improved importer merging behavior

cookbook/helper/open_data_importer.py

@@ -7,10 +7,12 @@ class OpenDataImporter:
     request = None
     data = {}
     slug_id_cache = {}
+    update_existing = False
 
-    def __init__(self, request, data):
+    def __init__(self, request, data, update_existing=False):
         self.request = request
         self.data = data
+        self.update_existing = update_existing
 
     def _update_slug_cache(self, object_class, datatype):
         self.slug_id_cache[datatype] = dict(object_class.objects.filter(space=self.request.space, open_data_slug__isnull=False).values_list('open_data_slug', 'id', ))
@@ -80,37 +82,36 @@ class OpenDataImporter:
         return len(insert_list)
 
     def import_supermarket(self):
-        identifier_list = []
         datatype = 'supermarket'
-        for k in list(self.data[datatype].keys()):
-            identifier_list.append(self.data[datatype][k]['name'])
-
-        existing_objects = Supermarket.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('name', flat=True)
 
         self._update_slug_cache(SupermarketCategory, 'category')
+        insert_list = []
+        for k in list(self.data[datatype].keys()):
+            insert_list.append(Supermarket(
+                name=self.data[datatype][k]['name'],
+                open_data_slug=k,
+                space=self.request.space
+            ))
+
+        # always add open data slug if matching supermarket is found, otherwise relation might fail
+        Supermarket.objects.bulk_create(insert_list, unique_fields=('space', 'name',), update_conflicts=True, update_fields=('open_data_slug',))
+        self._update_slug_cache(Supermarket, 'supermarket')
 
         insert_list = []
         for k in list(self.data[datatype].keys()):
-            if not (self.data[datatype][k]['name'] in existing_objects):  # TODO on large datasets see if bulk creating supermarkets and then relations as well is better
-                supermarket = Supermarket.objects.create(
-                    name=self.data[datatype][k]['name'],
-                    open_data_slug=k,
-                    space=self.request.space
-                )
-
-                relations = []
-                order = 0
-                for c in self.data[datatype][k]['categories']:
-                    relations.append(
-                        SupermarketCategoryRelation(
-                            supermarket=supermarket,
-                            category_id=self.slug_id_cache['category'][c],
-                            order=order,
-                        )
+            relations = []
+            order = 0
+            for c in self.data[datatype][k]['categories']:
+                relations.append(
+                    SupermarketCategoryRelation(
+                        supermarket_id=self.slug_id_cache[datatype][k],
+                        category_id=self.slug_id_cache['category'][c],
+                        order=order,
                     )
-                    order += 1
+                )
+                order += 1
 
-                SupermarketCategoryRelation.objects.bulk_create(relations)
+            SupermarketCategoryRelation.objects.bulk_create(relations, ignore_conflicts=True, unique_fields=('supermarket', 'category',))
 
         return len(insert_list)
 
@@ -172,8 +173,8 @@ class OpenDataImporter:
                     created_by=self.request.user,
                 ))
 
-        FoodProperty.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'food', 'property_type'))
-        Automation.objects.bulk_create(alias_list)
+        FoodProperty.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'food', 'property_type',))
+        Automation.objects.bulk_create(alias_list, ignore_conflicts=True, unique_fields=('space', 'param_1', 'param_2',))
         return len(insert_list)
 
     def import_conversion(self):

cookbook/migrations/0194_supermarketcategoryrelation_unique_sm_category_relation.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.1.7 on 2023-05-04 13:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cookbook', '0193_unitconversion_f_unique_conversion_per_space'),
+    ]
+
+    operations = [
+        migrations.AddConstraint(
+            model_name='supermarketcategoryrelation',
+            constraint=models.UniqueConstraint(fields=('supermarket', 'category'), name='unique_sm_category_relation'),
+        ),
+    ]

cookbook/models.py

@@ -498,6 +498,9 @@ class SupermarketCategoryRelation(models.Model, PermissionModelMixin):
         return 'supermarket', 'space'
 
     class Meta:
+        constraints = [
+            models.UniqueConstraint(fields=['supermarket', 'category'], name='unique_sm_category_relation')
+        ]
         ordering = ('order',)
 
 

cookbook/serializer.py

@@ -497,7 +497,7 @@ class SupermarketSerializer(UniqueFieldsMixin, SpacedModelSerializer):
 
     class Meta:
         model = Supermarket
-        fields = ('id', 'name', 'description', 'category_to_supermarket')
+        fields = ('id', 'name', 'description', 'category_to_supermarket', 'open_data_slug')
 
 
 class RecipeSimpleSerializer(WritableNestedModelSerializer):
@@ -737,7 +737,7 @@ class UnitConversionSerializer(WritableNestedModelSerializer):
 
     class Meta:
         model = UnitConversion
-        fields = ('id', 'base_amount', 'base_unit', 'converted_amount', 'converted_unit', 'food')
+        fields = ('id', 'base_amount', 'base_unit', 'converted_amount', 'converted_unit', 'food', 'open_data_slug')
 
 
 class FoodPropertyTypeSerializer(serializers.ModelSerializer):

cookbook/views/api.py

@@ -1438,7 +1438,7 @@ class ImportOpenData(APIView):
         response = requests.get(f'https://raw.githubusercontent.com/TandoorRecipes/open-tandoor-data/main/build/{selected_version}.json')  # TODO catch 404, timeout, ...
         data = json.loads(response.content)
 
-        data_importer = OpenDataImporter(request, data)
+        data_importer = OpenDataImporter(request, data, update_existing=True)
         data_importer.import_units()
         data_importer.import_category()
         data_importer.import_property()