Merge remote-tracking branch 'origin/develop' into HomeAssistantConnector

# Conflicts:
#	cookbook/forms.py
#	requirements.txt

cookbook/forms.py

@@ -10,7 +10,7 @@ from django_scopes import scopes_disabled
 from django_scopes.forms import SafeModelChoiceField, SafeModelMultipleChoiceField
 from hcaptcha.fields import hCaptchaField
 
-from .models import Comment, Food, InviteLink, Keyword, Recipe, RecipeBook, RecipeBookEntry, SearchPreference, Space, Storage, Sync, User, UserPreference, ConnectorConfig
+from .models import Comment, InviteLink, Keyword, Recipe, SearchPreference, Space, Storage, Sync, User, UserPreference, ConnectorConfig
 
 
 class SelectWidget(widgets.Select):

cookbook/helper/open_data_importer.py

@@ -1,5 +1,20 @@
+import traceback
+from collections import defaultdict
+from decimal import Decimal
+
 from cookbook.models import (Food, FoodProperty, Property, PropertyType, Supermarket,
                              SupermarketCategory, SupermarketCategoryRelation, Unit, UnitConversion)
+import re
+
+
+class OpenDataImportResponse:
+    total_created = 0
+    total_updated = 0
+    total_untouched = 0
+    total_errored = 0
+
+    def to_dict(self):
+        return {'total_created': self.total_created, 'total_updated': self.total_updated, 'total_untouched': self.total_untouched, 'total_errored': self.total_errored}
 
 
 class OpenDataImporter:
@@ -18,69 +33,269 @@ class OpenDataImporter:
     def _update_slug_cache(self, object_class, datatype):
         self.slug_id_cache[datatype] = dict(object_class.objects.filter(space=self.request.space, open_data_slug__isnull=False).values_list('open_data_slug', 'id', ))
 
-    def import_units(self):
-        datatype = 'unit'
+    @staticmethod
+    def _is_obj_identical(field_list, obj, existing_obj):
+        """
+        checks if the obj meant for import is identical to an already existing one
+        :param field_list: list of field names to check
+        :type field_list: list[str]
+        :param obj: object meant for import
+        :type obj: Object
+        :param existing_obj: object already in DB
+        :type existing_obj: Object
+        :return: if objects are identical
+        :rtype: bool
+        """
+        for field in field_list:
+            if isinstance(getattr(obj, field), float) or isinstance(getattr(obj, field), Decimal):
+                if abs(float(getattr(obj, field)) - float(existing_obj[field])) > 0.001:  # convert both to float and check if basically equal
+                    print(f'comparing FLOAT {obj} failed because field {field} is not equal ({getattr(obj, field)} != {existing_obj[field]})')
+                    return False
+            elif getattr(obj, field) != existing_obj[field]:
+                print(f'comparing {obj} failed because field {field} is not equal ({getattr(obj, field)} != {existing_obj[field]})')
+                return False
+        return True
+
+    @staticmethod
+    def _merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+        """
+        sometimes there might be two objects conflicting for open data import (one has the slug, the other the name)
+        this function checks if that is the case and merges the two objects if possible
+        :param model_type: type of model to check/merge
+        :type model_type: Model
+        :param obj: object that should be created/updated
+        :type obj: Model
+        :param existing_data_slugs: dict of open data slugs mapped to objects
+        :type existing_data_slugs: dict
+        :param existing_data_names: dict of names mapped to objects
+        :type existing_data_names: dict
+        :return: true if merge was successful or not necessary else false
+        :rtype: bool
+        """
+        if obj.open_data_slug in existing_data_slugs and obj.name in existing_data_names and existing_data_slugs[obj.open_data_slug]['pk'] != existing_data_names[obj.name]['pk']:
+            try:
+                source_obj = model_type.objects.get(pk=existing_data_slugs[obj.open_data_slug]['pk'])
+                del existing_data_slugs[obj.open_data_slug]
+                source_obj.merge_into(model_type.objects.get(pk=existing_data_names[obj.name]['pk']))
+                return True
+            except RuntimeError:
+                return False  # in the edge case (e.g. parent/child) that an object cannot be merged don't update it for now
+        else:
+            return True
+
+    @staticmethod
+    def _get_existing_obj(obj, existing_data_slugs, existing_data_names):
+        """
+        gets the existing object from slug or name cache
+        :param obj: object that should be found
+        :type obj: Model
+        :param existing_data_slugs: dict of open data slugs mapped to objects
+        :type existing_data_slugs: dict
+        :param existing_data_names: dict of names mapped to objects
+        :type existing_data_names: dict
+        :return: existing object
+        :rtype: dict
+        """
+        existing_obj = None
+        if obj.open_data_slug in existing_data_slugs:
+            existing_obj = existing_data_slugs[obj.open_data_slug]
+        elif obj.name in existing_data_names:
+            existing_obj = existing_data_names[obj.name]
+
+        return existing_obj
+
+    def import_units(self):
+        od_response = OpenDataImportResponse()
+        datatype = 'unit'
+        model_type = Unit
+        field_list = ['name', 'plural_name', 'base_unit', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
 
-        insert_list = []
         for u in list(self.data[datatype].keys()):
-            insert_list.append(Unit(
+            obj = model_type(
                 name=self.data[datatype][u]['name'],
                 plural_name=self.data[datatype][u]['plural_name'],
-                base_unit=self.data[datatype][u]['base_unit'] if self.data[datatype][u]['base_unit'] != '' else None,
+                base_unit=self.data[datatype][u]['base_unit'].lower() if self.data[datatype][u]['base_unit'] != '' else None,
                 open_data_slug=u,
                 space=self.request.space
-            ))
+            )
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
 
-        if self.update_existing:
-            return Unit.objects.bulk_create(insert_list, update_conflicts=True, update_fields=(
-                'name', 'plural_name', 'base_unit', 'open_data_slug'), unique_fields=('space', 'name',))
-        else:
-            return Unit.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        return od_response
 
     def import_category(self):
+        od_response = OpenDataImportResponse()
         datatype = 'category'
+        model_type = SupermarketCategory
+        field_list = ['name', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
 
-        insert_list = []
         for k in list(self.data[datatype].keys()):
-            insert_list.append(SupermarketCategory(
+            obj = model_type(
                 name=self.data[datatype][k]['name'],
                 open_data_slug=k,
                 space=self.request.space
-            ))
+            )
 
-        return SupermarketCategory.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        return od_response
 
     def import_property(self):
+        od_response = OpenDataImportResponse()
         datatype = 'property'
+        model_type = PropertyType
+        field_list = ['name', 'unit', 'fdc_id', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
 
-        insert_list = []
         for k in list(self.data[datatype].keys()):
-            insert_list.append(PropertyType(
+            obj = model_type(
                 name=self.data[datatype][k]['name'],
                 unit=self.data[datatype][k]['unit'],
+                fdc_id=self.data[datatype][k]['fdc_id'],
                 open_data_slug=k,
                 space=self.request.space
-            ))
+            )
 
-        return PropertyType.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        return od_response
 
     def import_supermarket(self):
+        od_response = OpenDataImportResponse()
         datatype = 'store'
+        model_type = Supermarket
+        field_list = ['name', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
 
         self._update_slug_cache(SupermarketCategory, 'category')
-        insert_list = []
         for k in list(self.data[datatype].keys()):
-            insert_list.append(Supermarket(
+            obj = model_type(
                 name=self.data[datatype][k]['name'],
                 open_data_slug=k,
                 space=self.request.space
-            ))
+            )
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
 
         # always add open data slug if matching supermarket is found, otherwise relation might fail
-        supermarkets = Supermarket.objects.bulk_create(insert_list, unique_fields=('space', 'name',), update_conflicts=True, update_fields=('open_data_slug',))
         self._update_slug_cache(Supermarket, 'store')
 
-        insert_list = []
         for k in list(self.data[datatype].keys()):
             relations = []
             order = 0
@@ -96,115 +311,186 @@ class OpenDataImporter:
 
             SupermarketCategoryRelation.objects.bulk_create(relations, ignore_conflicts=True, unique_fields=('supermarket', 'category',))
 
-        return supermarkets
+        return od_response
 
     def import_food(self):
-        identifier_list = []
+        od_response = OpenDataImportResponse()
         datatype = 'food'
-        for k in list(self.data[datatype].keys()):
-            identifier_list.append(self.data[datatype][k]['name'])
-            identifier_list.append(self.data[datatype][k]['plural_name'])
+        model_type = Food
+        field_list = ['name', 'open_data_slug']
 
-        existing_objects_flat = []
-        existing_objects = {}
-        for f in Food.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('id', 'name', 'plural_name'):
-            existing_objects_flat.append(f[1])
-            existing_objects_flat.append(f[2])
-            existing_objects[f[1]] = f
-            existing_objects[f[2]] = f
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
 
         self._update_slug_cache(Unit, 'unit')
         self._update_slug_cache(PropertyType, 'property')
+        self._update_slug_cache(SupermarketCategory, 'category')
+
+        unit_g = Unit.objects.filter(space=self.request.space, base_unit__iexact='g').first()
 
-        insert_list = []
-        insert_list_flat = []
-        update_list = []
-        update_field_list = []
         for k in list(self.data[datatype].keys()):
-            if not (self.data[datatype][k]['name'] in existing_objects_flat or self.data[datatype][k]['plural_name'] in existing_objects_flat):
-                if not (self.data[datatype][k]['name'] in insert_list_flat or self.data[datatype][k]['plural_name'] in insert_list_flat):
-                    insert_list.append({'data': {
-                        'name': self.data[datatype][k]['name'],
-                        'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
-                        'supermarket_category_id': self.slug_id_cache['category'][self.data[datatype][k]['store_category']],
-                        'fdc_id': self.data[datatype][k]['fdc_id'] if self.data[datatype][k]['fdc_id'] != '' else None,
-                        'open_data_slug': k,
-                        'space': self.request.space.id,
-                    }})
-                    # build a fake second flat array to prevent duplicate foods from being inserted.
-                    # trying to insert a duplicate would throw a db error :(
-                    insert_list_flat.append(self.data[datatype][k]['name'])
-                    insert_list_flat.append(self.data[datatype][k]['plural_name'])
+
+            obj_dict = {
+                'name': self.data[datatype][k]['name'],
+                'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
+                'supermarket_category_id': self.slug_id_cache['category'][self.data[datatype][k]['store_category']],
+                'fdc_id': re.sub(r'\D', '', self.data[datatype][k]['fdc_id']) if self.data[datatype][k]['fdc_id'] != '' else None,
+                'open_data_slug': k,
+                'properties_food_unit_id': None,
+                'space_id': self.request.space.id,
+            }
+
+            if unit_g:
+                obj_dict['properties_food_unit_id'] = unit_g.id
+
+            obj = model_type(**obj_dict)
+
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
             else:
-                if self.data[datatype][k]['name'] in existing_objects:
-                    existing_food_id = existing_objects[self.data[datatype][k]['name']][0]
-                else:
-                    existing_food_id = existing_objects[self.data[datatype][k]['plural_name']][0]
+                create_list.append({'data': obj_dict})
 
-                if self.update_existing:
-                    update_field_list = ['name', 'plural_name', 'preferred_unit_id', 'preferred_shopping_unit_id', 'supermarket_category_id', 'fdc_id', 'open_data_slug', ]
-                    update_list.append(Food(
-                        id=existing_food_id,
-                        name=self.data[datatype][k]['name'],
-                        plural_name=self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
-                        supermarket_category_id=self.slug_id_cache['category'][self.data[datatype][k]['store_category']],
-                        fdc_id=self.data[datatype][k]['fdc_id'] if self.data[datatype][k]['fdc_id'] != '' else None,
-                        open_data_slug=k,
-                    ))
-                else:
-                    update_field_list = ['open_data_slug', ]
-                    update_list.append(Food(id=existing_food_id, open_data_slug=k, ))
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
 
-        Food.load_bulk(insert_list, None)
-        if len(update_list) > 0:
-            Food.objects.bulk_update(update_list, update_field_list)
+        if len(create_list) > 0:
+            Food.load_bulk(create_list, None)
+            od_response.total_created += len(create_list)
+
+        # --------------- PROPERTY STUFF -----------------------
+        model_type = Property
+        field_list = ['property_type_id', 'property_amount', 'open_data_food_slug']
+
+        existing_data_slugs = {}
+        existing_data_property_types = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_food_slug']] = obj
+            existing_data_property_types[obj['property_type_id']] = obj
+
+        update_list = []
+        create_list = []
 
         self._update_slug_cache(Food, 'food')
 
-        food_property_list = []
-        # alias_list = []
+        for k in list(self.data['food'].keys()):
+            for fp in self.data['food'][k]['properties']['type_values']:
+                obj = model_type(
+                    property_type_id=self.slug_id_cache['property'][fp['property_type']],
+                    property_amount=fp['property_value'],
+                    open_data_food_slug=k,
+                    space=self.request.space,
+                )
 
-        for k in list(self.data[datatype].keys()):
-            for fp in self.data[datatype][k]['properties']['type_values']:
-                # try catch here because somettimes key "k" is not set for he food cache
-                try:
-                    food_property_list.append(Property(
-                        property_type_id=self.slug_id_cache['property'][fp['property_type']],
-                        property_amount=fp['property_value'],
-                        import_food_id=self.slug_id_cache['food'][k],
-                        space=self.request.space,
-                    ))
-                except KeyError:
-                    print(str(k) + ' is not in self.slug_id_cache["food"]')
+                if obj.open_data_food_slug in existing_data_slugs and obj.property_type_id in existing_data_property_types and existing_data_slugs[obj.open_data_food_slug] == existing_data_property_types[obj.property_type_id]:
+                    existing_obj = existing_data_slugs[obj.open_data_food_slug]
 
-        Property.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'import_food_id', 'property_type',))
+                    if not self._is_obj_identical(field_list, obj, existing_obj):
+                        obj.pk = existing_obj['pk']
+                        update_list.append(obj)
+                else:
+                    create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, ignore_conflicts=True, unique_fields=('space', 'open_data_food_slug', 'property_type',))
+
+        linked_properties = list(FoodProperty.objects.filter(food__space=self.request.space).values_list('property_id', flat=True).all())
 
         property_food_relation_list = []
-        for p in Property.objects.filter(space=self.request.space, import_food_id__isnull=False).values_list('import_food_id', 'id', ):
-            property_food_relation_list.append(Food.properties.through(food_id=p[0], property_id=p[1]))
+        for p in model_type.objects.filter(space=self.request.space, open_data_food_slug__isnull=False).values_list('open_data_food_slug', 'id', ):
+            if p[1] == 147:
+                pass
+            # slug_id_cache should always exist, don't create relations for already linked properties (ignore_conflicts would do that as well but this is more performant)
+            if p[0] in self.slug_id_cache['food'] and p[1] not in linked_properties:
+                property_food_relation_list.append(Food.properties.through(food_id=self.slug_id_cache['food'][p[0]], property_id=p[1]))
 
-        FoodProperty.objects.bulk_create(property_food_relation_list, ignore_conflicts=True, unique_fields=('food_id', 'property_id',))
+        FoodProperty.objects.bulk_create(property_food_relation_list, unique_fields=('food_id', 'property_id',))
 
-        return insert_list + update_list
+        return od_response
 
     def import_conversion(self):
+        od_response = OpenDataImportResponse()
         datatype = 'conversion'
+        model_type = UnitConversion
+        field_list = ['base_amount', 'base_unit_id', 'converted_amount', 'converted_unit_id', 'food_id', 'open_data_slug']
+
+        self._update_slug_cache(Food, 'food')
+        self._update_slug_cache(Unit, 'unit')
+
+        existing_data_slugs = {}
+        existing_data_foods = defaultdict(list)
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_foods[obj['food_id']].append(obj)
+
+        update_list = []
+        create_list = []
 
-        insert_list = []
         for k in list(self.data[datatype].keys()):
-            # try catch here because sometimes key "k" is not set for he food cache
+            # try catch here because sometimes key "k" is not set for the food cache
             try:
-                insert_list.append(UnitConversion(
-                    base_amount=self.data[datatype][k]['base_amount'],
+                obj = model_type(
+                    base_amount=Decimal(self.data[datatype][k]['base_amount']),
                     base_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['base_unit']],
-                    converted_amount=self.data[datatype][k]['converted_amount'],
+                    converted_amount=Decimal(self.data[datatype][k]['converted_amount']),
                     converted_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['converted_unit']],
                     food_id=self.slug_id_cache['food'][self.data[datatype][k]['food']],
                     open_data_slug=k,
                     space=self.request.space,
-                    created_by=self.request.user,
-                ))
-            except KeyError:
-                print(str(k) + ' is not in self.slug_id_cache["food"]')
+                    created_by_id=self.request.user.id,
+                )
 
-        return UnitConversion.objects.bulk_create(insert_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug'))
+                if obj.open_data_slug in existing_data_slugs:
+                    existing_obj = existing_data_slugs[obj.open_data_slug]
+                    if not self._is_obj_identical(field_list, obj, existing_obj):
+                        obj.pk = existing_obj['pk']
+                        update_list.append(obj)
+                    else:
+                        od_response.total_untouched += 1
+                else:
+                    matching_existing_found = False
+                    if obj.food_id in existing_data_foods:
+                        for edf in existing_data_foods[obj.food_id]:
+                            if obj.base_unit_id == edf['base_unit_id'] and obj.converted_unit_id == edf['converted_unit_id']:
+                                matching_existing_found = True
+                                if not self._is_obj_identical(field_list, obj, edf):
+                                    obj.pk = edf['pk']
+                                    update_list.append(obj)
+                                else:
+                                    od_response.total_untouched += 1
+                    if not matching_existing_found:
+                        create_list.append(obj)
+            except KeyError as e:
+                traceback.print_exc()
+                od_response.total_errored += 1
+                print(self.data[datatype][k]['food'] + ' is not in self.slug_id_cache["food"]')
+
+        if self.update_existing and len(update_list) > 0:
+            od_response.total_updated = model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_errored += len(update_list) - od_response.total_updated
+
+        if len(create_list) > 0:
+            objs_created = model_type.objects.bulk_create(create_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug'))
+            od_response.total_created = len(objs_created)
+            od_response.total_errored += len(create_list) - od_response.total_created
+
+        return od_response

cookbook/helper/property_helper.py

@@ -45,12 +45,12 @@ class FoodPropertyHelper:
                 conversions = uch.get_conversions(i)
                 for pt in property_types:
                     found_property = False
-                    if i.food.properties_food_amount == 0 or i.food.properties_food_unit is None:
-                        computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': 0}
-                        computed_properties[pt.id]['missing_value'] = i.food.properties_food_unit is None
+                    if i.food.properties_food_amount == 0 or i.food.properties_food_unit is None:  # if food is configured incorrectly
+                        computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': None}
+                        computed_properties[pt.id]['missing_value'] = True
                     else:
                         for p in i.food.properties.all():
-                            if p.property_type == pt:
+                            if p.property_type == pt and p.property_amount is not None:
                                 for c in conversions:
                                     if c.unit == i.food.properties_food_unit:
                                         found_property = True
@@ -58,13 +58,17 @@ class FoodPropertyHelper:
                                         computed_properties[pt.id]['food_values'] = self.add_or_create(
                                             computed_properties[p.property_type.id]['food_values'], c.food.id, (c.amount / i.food.properties_food_amount) * p.property_amount, c.food)
                     if not found_property:
-                        computed_properties[pt.id]['missing_value'] = True
-                        computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': 0}
+                        if i.amount == 0:  # don't count ingredients without an amount as missing
+                            computed_properties[pt.id]['missing_value'] = computed_properties[pt.id]['missing_value'] or False  # don't override if another food was already missing
+                            computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': 0}
+                        else:
+                            computed_properties[pt.id]['missing_value'] = True
+                            computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': None}
 
         return computed_properties
 
     # small dict helper to add to existing key or create new, probably a better way of doing this
-    # TODO move to central helper ?
+    # TODO move to central helper ? --> use defaultdict
     @staticmethod
     def add_or_create(d, key, value, food):
         if key in d:

cookbook/helper/recipe_url_import.py

@@ -231,7 +231,7 @@ def get_recipe_properties(space, property_data):
                             'id': pt.id,
                             'name': pt.name,
                         },
-                        'property_amount': parse_servings(property_data[properties[p]]) / float(property_data['servingSize']),
+                        'property_amount': parse_servings(property_data[properties[p]]) / parse_servings(property_data['servingSize']),
                     })
 
     return recipe_properties

cookbook/integration/rezeptsuitede.py

@@ -1,6 +1,6 @@
 import base64
 from io import BytesIO
-from xml import etree
+from lxml import etree
 
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text
@@ -53,7 +53,10 @@ class Rezeptsuitede(Integration):
                 u = ingredient_parser.get_unit(ingredient.attrib['unit'])
                 amount = 0
                 if ingredient.attrib['qty'].strip() != '':
-                    amount, unit, note = ingredient_parser.parse_amount(ingredient.attrib['qty'])
+                    try:
+                        amount, unit, note = ingredient_parser.parse_amount(ingredient.attrib['qty'])
+                    except ValueError:  # sometimes quantities contain words which cant be parsed
+                        pass
                 ingredient_step.ingredients.add(Ingredient.objects.create(food=f, unit=u, amount=amount, space=self.request.space, ))
 
         try:

cookbook/locale/it/LC_MESSAGES/django.po

@@ -12,8 +12,8 @@ msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2023-05-18 14:28+0200\n"
-"PO-Revision-Date: 2024-02-01 17:22+0000\n"
-"Last-Translator: Lorenzo <gerosa.lorenzo.gl@gmail.com>\n"
+"PO-Revision-Date: 2024-02-17 19:16+0000\n"
+"Last-Translator: Andrea <giovannibecco@mailo.com>\n"
 "Language-Team: Italian <http://translate.tandoor.dev/projects/tandoor/"
 "recipes-backend/it/>\n"
 "Language: it\n"
@@ -2093,7 +2093,7 @@ msgstr "Proprietario"
 #, fuzzy
 #| msgid "Create Space"
 msgid "Leave Space"
-msgstr "Crea Istanza"
+msgstr "Lascia Istanza"
 
 #: .\cookbook\templates\space_overview.html:78
 #: .\cookbook\templates\space_overview.html:88

cookbook/management/commands/fix_duplicate_properties.py

@@ -0,0 +1,34 @@
+from django.conf import settings
+from django.contrib.postgres.search import SearchVector
+from django.core.management.base import BaseCommand
+from django.db.models import Count
+from django.utils import translation
+from django.utils.translation import gettext_lazy as _
+from django_scopes import scopes_disabled
+
+from cookbook.managers import DICTIONARY
+from cookbook.models import Recipe, Step, FoodProperty, Food
+
+
+# can be executed at the command line with 'python manage.py rebuildindex'
+class Command(BaseCommand):
+    help = _('Fixes foods with ')
+
+    def add_arguments(self, parser):
+        parser.add_argument('-d', '--dry-run', help='does not delete properties but instead prints them', action='store_true')
+
+    def handle(self, *args, **options):
+        with scopes_disabled():
+            foods_with_duplicate_properties = Food.objects.annotate(property_type_count=Count('foodproperty__property__property_type') - Count('foodproperty__property__property_type', distinct=True)).filter(property_type_count__gt=0).all()
+            for f in foods_with_duplicate_properties:
+                found_property_types = []
+                for fp in f.properties.all():
+                    if fp.property_type.id in found_property_types:
+                        if options['dry_run']:
+                            print(f'Property id {fp.id} duplicate type {fp.property_type}({fp.property_type.id}) for food {f}({f.id})')
+                        else:
+                            print(f'DELETING property id {fp.id} duplicate type {fp.property_type}({fp.property_type.id}) for food {f}({f.id})')
+                            fp.delete()
+
+                    else:
+                        found_property_types.append(fp.property_type.id)

cookbook/migrations/0211_recipebook_order.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.7 on 2024-02-16 19:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cookbook', '0210_shoppinglistentry_updated_at'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='recipebook',
+            name='order',
+            field=models.IntegerField(default=0),
+        ),
+    ]

cookbook/migrations/0212_alter_property_property_amount.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.7 on 2024-02-18 07:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cookbook', '0211_recipebook_order'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='property',
+            name='property_amount',
+            field=models.DecimalField(decimal_places=4, default=None, max_digits=32, null=True),
+        ),
+    ]

cookbook/migrations/0213_remove_property_property_unique_import_food_per_space_and_more.py

@@ -0,0 +1,54 @@
+# Generated by Django 4.2.10 on 2024-02-19 13:48
+
+from django.db import migrations, models
+from django_scopes import scopes_disabled
+
+
+def migrate_property_import_slug(apps, schema_editor):
+    with scopes_disabled():
+        Property = apps.get_model('cookbook', 'Property')
+        Food = apps.get_model('cookbook', 'Food')
+
+        id_slug_mapping = {}
+        with scopes_disabled():
+            for f in Food.objects.filter(open_data_slug__isnull=False).values('id', 'open_data_slug').all():
+                id_slug_mapping[f['id']] = f['open_data_slug']
+
+            property_update_list = []
+
+            for p in Property.objects.filter().values('id', 'import_food_id').all():
+                if p['import_food_id'] in id_slug_mapping:
+                    property_update_list.append(Property(
+                        id=p['id'],
+                        open_data_food_slug=id_slug_mapping[p['import_food_id']]
+                    ))
+
+            Property.objects.bulk_update(property_update_list, ('open_data_food_slug',))
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('cookbook', '0212_alter_property_property_amount'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='property',
+            name='open_data_food_slug',
+            field=models.CharField(blank=True, default=None, max_length=128, null=True),
+        ),
+        migrations.RunPython(migrate_property_import_slug),
+        migrations.RemoveConstraint(
+            model_name='property',
+            name='property_unique_import_food_per_space',
+        ),
+        migrations.RemoveField(
+            model_name='property',
+            name='import_food_id',
+        ),
+
+        migrations.AddConstraint(
+            model_name='property',
+            constraint=models.UniqueConstraint(fields=('space', 'property_type', 'open_data_food_slug'), name='property_unique_import_food_per_space'),
+        ),
+    ]

cookbook/models.py

@@ -209,6 +209,27 @@ class TreeModel(MP_Node):
         abstract = True
 
 
+class MergeModelMixin:
+
+    def merge_into(self, target):
+        """
+        very simple merge function that replaces the current instance with the target instance
+        :param target: target object
+        :return: target with data merged
+        """
+
+        if self == target:
+            raise ValueError('Cannot merge an object with itself')
+
+        if getattr(self, 'space', 0) != getattr(target, 'space', 0):
+            raise RuntimeError('Cannot merge objects from different spaces')
+
+        if hasattr(self, 'get_descendants_and_self') and target in callable(getattr(self, 'get_descendants_and_self')):
+            raise RuntimeError('Cannot merge parent (source) with child (target) object')
+
+        # TODO copy field values
+
+
 class PermissionModelMixin:
     @staticmethod
     def get_space_key():
@@ -320,10 +341,18 @@ class Space(ExportModelOperationsMixin('space'), models.Model):
         BookmarkletImport.objects.filter(space=self).delete()
         CustomFilter.objects.filter(space=self).delete()
 
+        Property.objects.filter(space=self).delete()
+        PropertyType.objects.filter(space=self).delete()
+
         Comment.objects.filter(recipe__space=self).delete()
-        Keyword.objects.filter(space=self).delete()
         Ingredient.objects.filter(space=self).delete()
-        Food.objects.filter(space=self).delete()
+        Keyword.objects.filter(space=self).delete()
+
+        # delete food in batches because treabeard might fail to delete otherwise
+        while Food.objects.filter(space=self).count() > 0:
+            pks = Food.objects.filter(space=self).values_list('pk')[:200]
+            Food.objects.filter(pk__in=pks).delete()
+
         Unit.objects.filter(space=self).delete()
         Step.objects.filter(space=self).delete()
         NutritionInformation.objects.filter(space=self).delete()
@@ -348,9 +377,11 @@ class Space(ExportModelOperationsMixin('space'), models.Model):
         SupermarketCategory.objects.filter(space=self).delete()
         Supermarket.objects.filter(space=self).delete()
 
-        InviteLink.objects.filter(space=self).delete()
         UserFile.objects.filter(space=self).delete()
+        UserSpace.objects.filter(space=self).delete()
         Automation.objects.filter(space=self).delete()
+        InviteLink.objects.filter(space=self).delete()
+        TelegramBot.objects.filter(space=self).delete()
         self.delete()
 
     def get_owner(self):
@@ -468,6 +499,7 @@ class UserPreference(models.Model, PermissionModelMixin):
             self.use_fractions = FRACTION_PREF_DEFAULT
 
         return super().save(*args, **kwargs)
+
     def __str__(self):
         return str(self.user)
 
@@ -527,7 +559,7 @@ class Sync(models.Model, PermissionModelMixin):
         return self.path
 
 
-class SupermarketCategory(models.Model, PermissionModelMixin):
+class SupermarketCategory(models.Model, PermissionModelMixin, MergeModelMixin):
     name = models.CharField(max_length=128, validators=[MinLengthValidator(1)])
     description = models.TextField(blank=True, null=True)
     open_data_slug = models.CharField(max_length=128, null=True, blank=True, default=None)
@@ -538,6 +570,14 @@ class SupermarketCategory(models.Model, PermissionModelMixin):
     def __str__(self):
         return self.name
 
+    def merge_into(self, target):
+        super().merge_into(target)
+
+        Food.objects.filter(supermarket_category=self).update(supermarket_category=target)
+        SupermarketCategoryRelation.objects.filter(category=self).update(category=target)
+        self.delete()
+        return target
+
     class Meta:
         constraints = [
             models.UniqueConstraint(fields=['space', 'name'], name='smc_unique_name_per_space'),
@@ -612,7 +652,7 @@ class Keyword(ExportModelOperationsMixin('keyword'), TreeModel, PermissionModelM
         indexes = (Index(fields=['id', 'name']),)
 
 
-class Unit(ExportModelOperationsMixin('unit'), models.Model, PermissionModelMixin):
+class Unit(ExportModelOperationsMixin('unit'), models.Model, PermissionModelMixin, MergeModelMixin):
     name = models.CharField(max_length=128, validators=[MinLengthValidator(1)])
     plural_name = models.CharField(max_length=128, null=True, blank=True, default=None)
     description = models.TextField(blank=True, null=True)
@@ -622,6 +662,17 @@ class Unit(ExportModelOperationsMixin('unit'), models.Model, PermissionModelMixi
     space = models.ForeignKey(Space, on_delete=models.CASCADE)
     objects = ScopedManager(space='space')
 
+    def merge_into(self, target):
+        super().merge_into(target)
+
+        Ingredient.objects.filter(unit=self).update(unit=target)
+        ShoppingListEntry.objects.filter(unit=self).update(unit=target)
+        Food.objects.filter(properties_food_unit=self).update(properties_food_unit=target)
+        Food.objects.filter(preferred_unit=self).update(preferred_unit=target)
+        Food.objects.filter(preferred_shopping_unit=self).update(preferred_shopping_unit=target)
+        self.delete()
+        return target
+
     def __str__(self):
         return self.name
 
@@ -670,6 +721,32 @@ class Food(ExportModelOperationsMixin('food'), TreeModel, PermissionModelMixin):
     def __str__(self):
         return self.name
 
+    def merge_into(self, target):
+        """
+        very simple merge function that replaces the current food with the target food
+        also replaces a few attributes on the target field if they were empty before
+        :param target: target food object
+        :return: target with data merged
+        """
+        if self == target:
+            raise ValueError('Cannot merge an object with itself')
+
+        if self.space != target.space:
+            raise RuntimeError('Cannot merge objects from different spaces')
+
+        try:
+            if target in self.get_descendants_and_self():
+                raise RuntimeError('Cannot merge parent (source) with child (target) object')
+        except AttributeError:
+            pass  # AttributeError is raised when the object is not a tree and thus does not have the get_descendants_and_self() function
+
+        self.properties.all().delete()
+        self.properties.clear()
+        Ingredient.objects.filter(food=self).update(food=target)
+        ShoppingListEntry.objects.filter(food=self).update(food=target)
+        self.delete()
+        return target
+
     def delete(self):
         if self.ingredient_set.all().exclude(step=None).count() > 0:
             raise ProtectedError(self.name + _(" is part of a recipe step and cannot be deleted"), self.ingredient_set.all().exclude(step=None))
@@ -827,7 +904,7 @@ class Step(ExportModelOperationsMixin('step'), models.Model, PermissionModelMixi
         indexes = (GinIndex(fields=["search_vector"]),)
 
 
-class PropertyType(models.Model, PermissionModelMixin):
+class PropertyType(models.Model, PermissionModelMixin, MergeModelMixin):
     NUTRITION = 'NUTRITION'
     ALLERGEN = 'ALLERGEN'
     PRICE = 'PRICE'
@@ -852,6 +929,13 @@ class PropertyType(models.Model, PermissionModelMixin):
     def __str__(self):
         return f'{self.name}'
 
+    def merge_into(self, target):
+        super().merge_into(target)
+
+        Property.objects.filter(property_type=self).update(property_type=target)
+        self.delete()
+        return target
+
     class Meta:
         constraints = [
             models.UniqueConstraint(fields=['space', 'name'], name='property_type_unique_name_per_space'),
@@ -861,10 +945,10 @@ class PropertyType(models.Model, PermissionModelMixin):
 
 
 class Property(models.Model, PermissionModelMixin):
-    property_amount = models.DecimalField(default=0, decimal_places=4, max_digits=32)
+    property_amount = models.DecimalField(default=None, null=True, decimal_places=4, max_digits=32)
     property_type = models.ForeignKey(PropertyType, on_delete=models.PROTECT)
 
-    import_food_id = models.IntegerField(null=True, blank=True)  # field to hold food id when importing properties from the open data project
+    open_data_food_slug = models.CharField(max_length=128, null=True, blank=True, default=None)  # field to hold food id when importing properties from the open data project
 
     space = models.ForeignKey(Space, on_delete=models.CASCADE)
     objects = ScopedManager(space='space')
@@ -874,7 +958,7 @@ class Property(models.Model, PermissionModelMixin):
 
     class Meta:
         constraints = [
-            models.UniqueConstraint(fields=['space', 'property_type', 'import_food_id'], name='property_unique_import_food_per_space')
+            models.UniqueConstraint(fields=['space', 'property_type', 'open_data_food_slug'], name='property_unique_import_food_per_space')
         ]
 
 
@@ -1011,7 +1095,6 @@ class RecipeBook(ExportModelOperationsMixin('book'), models.Model, PermissionMod
     filter = models.ForeignKey('cookbook.CustomFilter', null=True, blank=True, on_delete=models.SET_NULL)
     order = models.IntegerField(default=0)
 
-
     space = models.ForeignKey(Space, on_delete=models.CASCADE)
     objects = ScopedManager(space='space')
 

cookbook/schemas.py

@@ -44,12 +44,12 @@ class TreeSchema(AutoSchema):
             "name": 'root', "in": "query", "required": False,
             "description": 'Return first level children of {obj} with ID [int].  Integer 0 will return root {obj}s.'.format(
                 obj=api_name),
-            'schema': {'type': 'int', },
+            'schema': {'type': 'integer', },
         })
         parameters.append({
             "name": 'tree', "in": "query", "required": False,
             "description": 'Return all self and children of {} with ID [int].'.format(api_name),
-            'schema': {'type': 'int', },
+            'schema': {'type': 'integer', },
         })
         return parameters
 

cookbook/serializer.py

@@ -584,7 +584,7 @@ class PropertyTypeSerializer(OpenDataModelMixin, WritableNestedModelSerializer,
 
 class PropertySerializer(UniqueFieldsMixin, WritableNestedModelSerializer):
     property_type = PropertyTypeSerializer()
-    property_amount = CustomDecimalField()
+    property_amount = CustomDecimalField(allow_null=True)
 
     def create(self, validated_data):
         validated_data['space'] = self.context['request'].space

cookbook/templates/space_manage.html

@@ -4,7 +4,7 @@
 {% load i18n %}
 {% load l10n %}
 
-{% block title %}{% trans 'Search' %}{% endblock %}
+{% block title %}{% trans 'Space Management' %}{% endblock %}
 
 {% block content %}
 

cookbook/templatetags/custom_tags.py

@@ -117,6 +117,7 @@ def page_help(page_name):
         'edit_connector_config': 'https://docs.tandoor.dev/features/connectors/',
         'view_shopping': 'https://docs.tandoor.dev/features/shopping/',
         'view_import': 'https://docs.tandoor.dev/features/import_export/',
+        'data_import_url': 'https://docs.tandoor.dev/features/import_export/',
         'view_export': 'https://docs.tandoor.dev/features/import_export/',
         'list_automation': 'https://docs.tandoor.dev/features/automation/',
     }

cookbook/tests/other/test_food_property.py

@@ -67,7 +67,7 @@ def test_food_property(space_1, space_2, u1_s1):
 
         assert property_values[property_fat.id]['name'] == property_fat.name
         assert abs(property_values[property_fat.id]['total_value']) < 0.0001
-        assert abs(property_values[property_fat.id]['food_values'][food_1.id]['value']) < 0.0001
+        assert property_values[property_fat.id]['food_values'][food_1.id]['value'] is None
 
         print('\n----------- TEST PROPERTY - PROPERTY CALCULATION UNIT CONVERSION ---------------')
         uc1 = UnitConversion.objects.create(

cookbook/views/api.py

@@ -262,7 +262,9 @@ class MergeMixin(ViewSetMixin):
 
             try:
                 if isinstance(source, Food):
-                    source.properties.remove()
+                    source.properties.all().delete()
+                    source.properties.clear()
+                    UnitConversion.objects.filter(food=source).delete()
 
                 for link in [field for field in source._meta.get_fields() if issubclass(type(field), ForeignObjectRel)]:
                     linkManager = getattr(source, link.get_accessor_name())
@@ -564,11 +566,8 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin):
             shared_users = c
         else:
             try:
-                shared_users = [x.id for x in list(self.request.user.get_shopping_share())] + [
-                    self.request.user.id]
-                caches['default'].set(
-                    f'shopping_shared_users_{self.request.space.id}_{self.request.user.id}',
-                    shared_users, timeout=5 * 60)
+                shared_users = [x.id for x in list(self.request.user.get_shopping_share())] + [self.request.user.id]
+                caches['default'].set(f'shopping_shared_users_{self.request.space.id}_{self.request.user.id}', shared_users, timeout=5 * 60)
                 # TODO ugly hack that improves API performance significantly, should be done properly
             except AttributeError:  # Anonymous users (using share links) don't have shared users
                 pass
@@ -616,11 +615,21 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin):
         if properties with a fdc_id already exist they will be overridden, if existing properties don't have a fdc_id they won't be changed
         """
         food = self.get_object()
+        if not food.fdc_id:
+            return JsonResponse({'msg': 'Food has no FDC ID associated.'}, status=400,
+                                json_dumps_params={'indent': 4})
 
         response = requests.get(f'https://api.nal.usda.gov/fdc/v1/food/{food.fdc_id}?api_key={FDC_API_KEY}')
         if response.status_code == 429:
-            return JsonResponse({'msg', 'API Key Rate Limit reached/exceeded, see https://api.data.gov/docs/rate-limits/ for more information. Configure your key in Tandoor using environment FDC_API_KEY variable.'}, status=429,
+            return JsonResponse({'msg': 'API Key Rate Limit reached/exceeded, see https://api.data.gov/docs/rate-limits/ for more information. Configure your key in Tandoor using environment FDC_API_KEY variable.'}, status=429,
                                 json_dumps_params={'indent': 4})
+        if response.status_code != 200:
+            return JsonResponse({'msg': f'Error while requesting FDC data using url https://api.nal.usda.gov/fdc/v1/food/{food.fdc_id}?api_key=****'}, status=response.status_code,
+                                json_dumps_params={'indent': 4})
+
+        food.properties_food_amount = 100
+        food.properties_food_unit = Unit.objects.get_or_create(base_unit__iexact='g', space=self.request.space, defaults={'name': 'g', 'base_unit': 'g', 'space': self.request.space})[0]
+        food.save()
 
         try:
             data = json.loads(response.content)
@@ -634,23 +643,29 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin):
 
             for pt in PropertyType.objects.filter(space=request.space, fdc_id__gte=0).all():
                 if pt.fdc_id:
+                    property_found = False
                     for fn in data['foodNutrients']:
                         if fn['nutrient']['id'] == pt.fdc_id:
+                            property_found = True
                             food_property_list.append(Property(
                                 property_type_id=pt.id,
-                                property_amount=round(fn['amount'], 2),
-                                import_food_id=food.id,
+                                property_amount=max(0, round(fn['amount'], 2)),  # sometimes FDC might return negative values which make no sense, set to 0
                                 space=self.request.space,
                             ))
+                    if not property_found:
+                        food_property_list.append(Property(
+                            property_type_id=pt.id,
+                            property_amount=0,  # if field not in FDC data the food does not have that property
+                            space=self.request.space,
+                        ))
 
-            Property.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'import_food_id', 'property_type',))
+            properties = Property.objects.bulk_create(food_property_list, unique_fields=('space', 'property_type',))
 
             property_food_relation_list = []
-            for p in Property.objects.filter(space=self.request.space, import_food_id=food.id).values_list('import_food_id', 'id', ):
-                property_food_relation_list.append(Food.properties.through(food_id=p[0], property_id=p[1]))
+            for p in properties:
+                property_food_relation_list.append(Food.properties.through(food_id=food.id, property_id=p.pk))
 
             FoodProperty.objects.bulk_create(property_food_relation_list, ignore_conflicts=True, unique_fields=('food_id', 'property_id',))
-            Property.objects.filter(space=self.request.space, import_food_id=food.id).update(import_food_id=None)
 
             return self.retrieve(request, pk)
         except Exception:
@@ -680,7 +695,7 @@ class RecipeBookViewSet(viewsets.ModelViewSet, StandardFilterMixin):
         ordering = f"{'' if order_direction == 'asc' else '-'}{order_field}"
 
         self.queryset = self.queryset.filter(Q(created_by=self.request.user) | Q(shared=self.request.user)).filter(
-        space=self.request.space).distinct().order_by(ordering)
+            space=self.request.space).distinct().order_by(ordering)
         return super().get_queryset()
 
 
@@ -728,7 +743,7 @@ class MealPlanViewSet(viewsets.ModelViewSet):
     query_params = [
         QueryParam(name='from_date', description=_('Filter meal plans from date (inclusive) in the format of YYYY-MM-DD.'), qtype='string'),
         QueryParam(name='to_date', description=_('Filter meal plans to date (inclusive) in the format of YYYY-MM-DD.'), qtype='string'),
-        QueryParam(name='meal_type', description=_('Filter meal plans with MealType ID. For multiple repeat parameter.'), qtype='int'),
+        QueryParam(name='meal_type', description=_('Filter meal plans with MealType ID. For multiple repeat parameter.'), qtype='integer'),
     ]
     schema = QueryParamAutoSchema()
 
@@ -858,7 +873,7 @@ class StepViewSet(viewsets.ModelViewSet):
     permission_classes = [CustomIsUser & CustomTokenHasReadWriteScope]
     pagination_class = DefaultPagination
     query_params = [
-        QueryParam(name='recipe', description=_('ID of recipe a step is part of. For multiple repeat parameter.'), qtype='int'),
+        QueryParam(name='recipe', description=_('ID of recipe a step is part of. For multiple repeat parameter.'), qtype='integer'),
         QueryParam(name='query', description=_('Query string matched (fuzzy) against object name.'), qtype='string'),
     ]
     schema = QueryParamAutoSchema()
@@ -901,27 +916,27 @@ class RecipeViewSet(viewsets.ModelViewSet):
 
     query_params = [
         QueryParam(name='query', description=_('Query string matched (fuzzy) against recipe name. In the future also fulltext search.')),
-        QueryParam(name='keywords', description=_('ID of keyword a recipe should have. For multiple repeat parameter. Equivalent to keywords_or'), qtype='int'),
-        QueryParam(name='keywords_or', description=_('Keyword IDs, repeat for multiple. Return recipes with any of the keywords'), qtype='int'),
-        QueryParam(name='keywords_and', description=_('Keyword IDs, repeat for multiple. Return recipes with all of the keywords.'), qtype='int'),
-        QueryParam(name='keywords_or_not', description=_('Keyword IDs, repeat for multiple. Exclude recipes with any of the keywords.'), qtype='int'),
-        QueryParam(name='keywords_and_not', description=_('Keyword IDs, repeat for multiple. Exclude recipes with all of the keywords.'), qtype='int'),
-        QueryParam(name='foods', description=_('ID of food a recipe should have. For multiple repeat parameter.'), qtype='int'),
-        QueryParam(name='foods_or', description=_('Food IDs, repeat for multiple. Return recipes with any of the foods'), qtype='int'),
-        QueryParam(name='foods_and', description=_('Food IDs, repeat for multiple. Return recipes with all of the foods.'), qtype='int'),
-        QueryParam(name='foods_or_not', description=_('Food IDs, repeat for multiple. Exclude recipes with any of the foods.'), qtype='int'),
-        QueryParam(name='foods_and_not', description=_('Food IDs, repeat for multiple. Exclude recipes with all of the foods.'), qtype='int'),
-        QueryParam(name='units', description=_('ID of unit a recipe should have.'), qtype='int'),
-        QueryParam(name='rating', description=_('Rating a recipe should have or greater. [0 - 5] Negative value filters rating less than.'), qtype='int'),
+        QueryParam(name='keywords', description=_('ID of keyword a recipe should have. For multiple repeat parameter. Equivalent to keywords_or'), qtype='integer'),
+        QueryParam(name='keywords_or', description=_('Keyword IDs, repeat for multiple. Return recipes with any of the keywords'), qtype='integer'),
+        QueryParam(name='keywords_and', description=_('Keyword IDs, repeat for multiple. Return recipes with all of the keywords.'), qtype='integer'),
+        QueryParam(name='keywords_or_not', description=_('Keyword IDs, repeat for multiple. Exclude recipes with any of the keywords.'), qtype='integer'),
+        QueryParam(name='keywords_and_not', description=_('Keyword IDs, repeat for multiple. Exclude recipes with all of the keywords.'), qtype='integer'),
+        QueryParam(name='foods', description=_('ID of food a recipe should have. For multiple repeat parameter.'), qtype='integer'),
+        QueryParam(name='foods_or', description=_('Food IDs, repeat for multiple. Return recipes with any of the foods'), qtype='integer'),
+        QueryParam(name='foods_and', description=_('Food IDs, repeat for multiple. Return recipes with all of the foods.'), qtype='integer'),
+        QueryParam(name='foods_or_not', description=_('Food IDs, repeat for multiple. Exclude recipes with any of the foods.'), qtype='integer'),
+        QueryParam(name='foods_and_not', description=_('Food IDs, repeat for multiple. Exclude recipes with all of the foods.'), qtype='integer'),
+        QueryParam(name='units', description=_('ID of unit a recipe should have.'), qtype='integer'),
+        QueryParam(name='rating', description=_('Rating a recipe should have or greater. [0 - 5] Negative value filters rating less than.'), qtype='integer'),
         QueryParam(name='books', description=_('ID of book a recipe should be in. For multiple repeat parameter.')),
-        QueryParam(name='books_or', description=_('Book IDs, repeat for multiple. Return recipes with any of the books'), qtype='int'),
-        QueryParam(name='books_and', description=_('Book IDs, repeat for multiple. Return recipes with all of the books.'), qtype='int'),
-        QueryParam(name='books_or_not', description=_('Book IDs, repeat for multiple. Exclude recipes with any of the books.'), qtype='int'),
-        QueryParam(name='books_and_not', description=_('Book IDs, repeat for multiple. Exclude recipes with all of the books.'), qtype='int'),
+        QueryParam(name='books_or', description=_('Book IDs, repeat for multiple. Return recipes with any of the books'), qtype='integer'),
+        QueryParam(name='books_and', description=_('Book IDs, repeat for multiple. Return recipes with all of the books.'), qtype='integer'),
+        QueryParam(name='books_or_not', description=_('Book IDs, repeat for multiple. Exclude recipes with any of the books.'), qtype='integer'),
+        QueryParam(name='books_and_not', description=_('Book IDs, repeat for multiple. Exclude recipes with all of the books.'), qtype='integer'),
         QueryParam(name='internal', description=_('If only internal recipes should be returned. [''true''/''<b>false</b>'']')),
         QueryParam(name='random', description=_('Returns the results in randomized order. [''true''/''<b>false</b>'']')),
         QueryParam(name='new', description=_('Returns new results first in search results. [''true''/''<b>false</b>'']')),
-        QueryParam(name='timescooked', description=_('Filter recipes cooked X times or more.  Negative values returns cooked less than X times'), qtype='int'),
+        QueryParam(name='timescooked', description=_('Filter recipes cooked X times or more.  Negative values returns cooked less than X times'), qtype='integer'),
         QueryParam(name='cookedon', description=_('Filter recipes last cooked on or after YYYY-MM-DD. Prepending ''-'' filters on or before date.')),
         QueryParam(name='createdon', description=_('Filter recipes created on or after YYYY-MM-DD. Prepending ''-'' filters on or before date.')),
         QueryParam(name='updatedon', description=_('Filter recipes updated on or after YYYY-MM-DD. Prepending ''-'' filters on or before date.')),
@@ -1095,7 +1110,7 @@ class UnitConversionViewSet(viewsets.ModelViewSet):
     serializer_class = UnitConversionSerializer
     permission_classes = [CustomIsUser & CustomTokenHasReadWriteScope]
     query_params = [
-        QueryParam(name='food_id', description='ID of food to filter for', qtype='int'),
+        QueryParam(name='food_id', description='ID of food to filter for', qtype='integer'),
     ]
     schema = QueryParamAutoSchema()
 
@@ -1146,10 +1161,10 @@ class ShoppingListEntryViewSet(viewsets.ModelViewSet):
     serializer_class = ShoppingListEntrySerializer
     permission_classes = [(CustomIsOwner | CustomIsShared) & CustomTokenHasReadWriteScope]
     query_params = [
-        QueryParam(name='id', description=_('Returns the shopping list entry with a primary key of id.  Multiple values allowed.'), qtype='int'),
+        QueryParam(name='id', description=_('Returns the shopping list entry with a primary key of id.  Multiple values allowed.'), qtype='integer'),
         QueryParam(name='checked', description=_('Filter shopping list entries on checked.  [''true'', ''false'', ''both'', ''<b>recent</b>'']<br>  - ''recent'' includes unchecked items and recently completed items.')
                    ),
-        QueryParam(name='supermarket', description=_('Returns the shopping list entries sorted by supermarket category order.'), qtype='int'),
+        QueryParam(name='supermarket', description=_('Returns the shopping list entries sorted by supermarket category order.'), qtype='integer'),
     ]
     schema = QueryParamAutoSchema()
 
@@ -1614,6 +1629,7 @@ class ImportOpenData(APIView):
         # TODO validate data
         print(request.data)
         selected_version = request.data['selected_version']
+        selected_datatypes = request.data['selected_datatypes']
         update_existing = str2bool(request.data['update_existing'])
         use_metric = str2bool(request.data['use_metric'])
 
@@ -1623,12 +1639,19 @@ class ImportOpenData(APIView):
         response_obj = {}
 
         data_importer = OpenDataImporter(request, data, update_existing=update_existing, use_metric=use_metric)
-        response_obj['unit'] = len(data_importer.import_units())
-        response_obj['category'] = len(data_importer.import_category())
-        response_obj['property'] = len(data_importer.import_property())
-        response_obj['store'] = len(data_importer.import_supermarket())
-        response_obj['food'] = len(data_importer.import_food())
-        response_obj['conversion'] = len(data_importer.import_conversion())
+
+        if selected_datatypes['unit']['selected']:
+            response_obj['unit'] = data_importer.import_units().to_dict()
+        if selected_datatypes['category']['selected']:
+            response_obj['category'] = data_importer.import_category().to_dict()
+        if selected_datatypes['property']['selected']:
+            response_obj['property'] = data_importer.import_property().to_dict()
+        if selected_datatypes['store']['selected']:
+            response_obj['store'] = data_importer.import_supermarket().to_dict()
+        if selected_datatypes['food']['selected']:
+            response_obj['food'] = data_importer.import_food().to_dict()
+        if selected_datatypes['conversion']['selected']:
+            response_obj['conversion'] = data_importer.import_conversion().to_dict()
 
         return Response(response_obj)