From 071926aadaafdfeba3a740fc21df5e275402745f Mon Sep 17 00:00:00 2001 From: vabene1111 Date: Thu, 4 May 2023 15:29:06 +0200 Subject: [PATCH] improved importer merging behavior --- cookbook/helper/open_data_importer.py | 53 ++++++++++--------- ...oryrelation_unique_sm_category_relation.py | 17 ++++++ cookbook/models.py | 3 ++ cookbook/serializer.py | 4 +- cookbook/views/api.py | 2 +- .../components/Modals/GenericModalForm.vue | 2 +- vue/src/components/Modals/TextInput.vue | 3 +- vue/src/locales/en.json | 2 + vue/src/utils/models.js | 20 ++++++- 9 files changed, 74 insertions(+), 32 deletions(-) create mode 100644 cookbook/migrations/0194_supermarketcategoryrelation_unique_sm_category_relation.py diff --git a/cookbook/helper/open_data_importer.py b/cookbook/helper/open_data_importer.py index 3a441fbbc..fbe4494a1 100644 --- a/cookbook/helper/open_data_importer.py +++ b/cookbook/helper/open_data_importer.py @@ -7,10 +7,12 @@ class OpenDataImporter: request = None data = {} slug_id_cache = {} + update_existing = False - def __init__(self, request, data): + def __init__(self, request, data, update_existing=False): self.request = request self.data = data + self.update_existing = update_existing def _update_slug_cache(self, object_class, datatype): self.slug_id_cache[datatype] = dict(object_class.objects.filter(space=self.request.space, open_data_slug__isnull=False).values_list('open_data_slug', 'id', )) @@ -80,37 +82,36 @@ class OpenDataImporter: return len(insert_list) def import_supermarket(self): - identifier_list = [] datatype = 'supermarket' - for k in list(self.data[datatype].keys()): - identifier_list.append(self.data[datatype][k]['name']) - - existing_objects = Supermarket.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('name', flat=True) self._update_slug_cache(SupermarketCategory, 'category') + insert_list = [] + for k in list(self.data[datatype].keys()): + insert_list.append(Supermarket( + name=self.data[datatype][k]['name'], + open_data_slug=k, + space=self.request.space + )) + + # always add open data slug if matching supermarket is found, otherwise relation might fail + Supermarket.objects.bulk_create(insert_list, unique_fields=('space', 'name',), update_conflicts=True, update_fields=('open_data_slug',)) + self._update_slug_cache(Supermarket, 'supermarket') insert_list = [] for k in list(self.data[datatype].keys()): - if not (self.data[datatype][k]['name'] in existing_objects): # TODO on large datasets see if bulk creating supermarkets and then relations as well is better - supermarket = Supermarket.objects.create( - name=self.data[datatype][k]['name'], - open_data_slug=k, - space=self.request.space - ) - - relations = [] - order = 0 - for c in self.data[datatype][k]['categories']: - relations.append( - SupermarketCategoryRelation( - supermarket=supermarket, - category_id=self.slug_id_cache['category'][c], - order=order, - ) + relations = [] + order = 0 + for c in self.data[datatype][k]['categories']: + relations.append( + SupermarketCategoryRelation( + supermarket_id=self.slug_id_cache[datatype][k], + category_id=self.slug_id_cache['category'][c], + order=order, ) - order += 1 + ) + order += 1 - SupermarketCategoryRelation.objects.bulk_create(relations) + SupermarketCategoryRelation.objects.bulk_create(relations, ignore_conflicts=True, unique_fields=('supermarket', 'category',)) return len(insert_list) @@ -172,8 +173,8 @@ class OpenDataImporter: created_by=self.request.user, )) - FoodProperty.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'food', 'property_type')) - Automation.objects.bulk_create(alias_list) + FoodProperty.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'food', 'property_type',)) + Automation.objects.bulk_create(alias_list, ignore_conflicts=True, unique_fields=('space', 'param_1', 'param_2',)) return len(insert_list) def import_conversion(self): diff --git a/cookbook/migrations/0194_supermarketcategoryrelation_unique_sm_category_relation.py b/cookbook/migrations/0194_supermarketcategoryrelation_unique_sm_category_relation.py new file mode 100644 index 000000000..a153d6d15 --- /dev/null +++ b/cookbook/migrations/0194_supermarketcategoryrelation_unique_sm_category_relation.py @@ -0,0 +1,17 @@ +# Generated by Django 4.1.7 on 2023-05-04 13:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('cookbook', '0193_unitconversion_f_unique_conversion_per_space'), + ] + + operations = [ + migrations.AddConstraint( + model_name='supermarketcategoryrelation', + constraint=models.UniqueConstraint(fields=('supermarket', 'category'), name='unique_sm_category_relation'), + ), + ] diff --git a/cookbook/models.py b/cookbook/models.py index e3109330a..382b6d0d4 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -498,6 +498,9 @@ class SupermarketCategoryRelation(models.Model, PermissionModelMixin): return 'supermarket', 'space' class Meta: + constraints = [ + models.UniqueConstraint(fields=['supermarket', 'category'], name='unique_sm_category_relation') + ] ordering = ('order',) diff --git a/cookbook/serializer.py b/cookbook/serializer.py index 4f0ff94f4..be1e8d8fd 100644 --- a/cookbook/serializer.py +++ b/cookbook/serializer.py @@ -497,7 +497,7 @@ class SupermarketSerializer(UniqueFieldsMixin, SpacedModelSerializer): class Meta: model = Supermarket - fields = ('id', 'name', 'description', 'category_to_supermarket') + fields = ('id', 'name', 'description', 'category_to_supermarket', 'open_data_slug') class RecipeSimpleSerializer(WritableNestedModelSerializer): @@ -737,7 +737,7 @@ class UnitConversionSerializer(WritableNestedModelSerializer): class Meta: model = UnitConversion - fields = ('id', 'base_amount', 'base_unit', 'converted_amount', 'converted_unit', 'food') + fields = ('id', 'base_amount', 'base_unit', 'converted_amount', 'converted_unit', 'food', 'open_data_slug') class FoodPropertyTypeSerializer(serializers.ModelSerializer): diff --git a/cookbook/views/api.py b/cookbook/views/api.py index 8e04d0ab0..2c81f4fa3 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -1438,7 +1438,7 @@ class ImportOpenData(APIView): response = requests.get(f'https://raw.githubusercontent.com/TandoorRecipes/open-tandoor-data/main/build/{selected_version}.json') # TODO catch 404, timeout, ... data = json.loads(response.content) - data_importer = OpenDataImporter(request, data) + data_importer = OpenDataImporter(request, data, update_existing=True) data_importer.import_units() data_importer.import_category() data_importer.import_property() diff --git a/vue/src/components/Modals/GenericModalForm.vue b/vue/src/components/Modals/GenericModalForm.vue index 34ff448de..1fe68b507 100644 --- a/vue/src/components/Modals/GenericModalForm.vue +++ b/vue/src/components/Modals/GenericModalForm.vue @@ -9,7 +9,7 @@

{{ f.label }}

- + diff --git a/vue/src/components/Modals/TextInput.vue b/vue/src/components/Modals/TextInput.vue index bd25ebc2d..4f85ca75f 100644 --- a/vue/src/components/Modals/TextInput.vue +++ b/vue/src/components/Modals/TextInput.vue @@ -1,7 +1,7 @@