From 12da77f03709393f8871504494afe50d99901a2a Mon Sep 17 00:00:00 2001 From: vabene1111 Date: Thu, 4 May 2023 17:12:49 +0200 Subject: [PATCH] beser response and stuff --- cookbook/helper/open_data_importer.py | 123 ++++++++++++++------------ cookbook/serializer.py | 4 +- cookbook/views/api.py | 22 ++--- vue/src/utils/models.js | 20 ++++- 4 files changed, 100 insertions(+), 69 deletions(-) diff --git a/cookbook/helper/open_data_importer.py b/cookbook/helper/open_data_importer.py index fbe4494a1..72f351081 100644 --- a/cookbook/helper/open_data_importer.py +++ b/cookbook/helper/open_data_importer.py @@ -8,78 +8,61 @@ class OpenDataImporter: data = {} slug_id_cache = {} update_existing = False + use_metric = True - def __init__(self, request, data, update_existing=False): + def __init__(self, request, data, update_existing=False, use_metric=True): self.request = request self.data = data self.update_existing = update_existing + self.use_metric = use_metric def _update_slug_cache(self, object_class, datatype): self.slug_id_cache[datatype] = dict(object_class.objects.filter(space=self.request.space, open_data_slug__isnull=False).values_list('open_data_slug', 'id', )) def import_units(self): - unit_name_list = [] - for u in list(self.data['unit'].keys()): - unit_name_list.append(self.data['unit'][u]['name']) - unit_name_list.append(self.data['unit'][u]['plural_name']) - - existing_units = Unit.objects.filter(space=self.request.space).filter(Q(name__in=unit_name_list) | Q(plural_name__in=unit_name_list)).values_list('name', 'plural_name') - existing_units = [item for sublist in existing_units for item in sublist] + datatype = 'category' insert_list = [] for u in list(self.data['unit'].keys()): - if not (self.data['unit'][u]['name'] in existing_units or self.data['unit'][u]['plural_name'] in existing_units): - insert_list.append(Unit( - name=self.data['unit'][u]['name'], - plural_name=self.data['unit'][u]['plural_name'], - base_unit=self.data['unit'][u]['base_unit'] if self.data['unit'][u]['base_unit'] != '' else None, - open_data_slug=u, - space=self.request.space - )) + insert_list.append(Unit( + name=self.data['unit'][u]['name'], + plural_name=self.data['unit'][u]['plural_name'], + base_unit=self.data['unit'][u]['base_unit'] if self.data['unit'][u]['base_unit'] != '' else None, + open_data_slug=u, + space=self.request.space + )) - Unit.objects.bulk_create(insert_list) - return len(insert_list) + if self.update_existing: + return Unit.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('name', 'plural_name', 'base_unit', 'open_data_slug'), unique_fields=('space', 'name',)) + else: + return Unit.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',)) def import_category(self): - identifier_list = [] datatype = 'category' - for k in list(self.data[datatype].keys()): - identifier_list.append(self.data[datatype][k]['name']) - - existing_objects = SupermarketCategory.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('name', flat=True) insert_list = [] for k in list(self.data[datatype].keys()): - if not (self.data[datatype][k]['name'] in existing_objects): - insert_list.append(SupermarketCategory( - name=self.data[datatype][k]['name'], - open_data_slug=k, - space=self.request.space - )) + insert_list.append(SupermarketCategory( + name=self.data[datatype][k]['name'], + open_data_slug=k, + space=self.request.space + )) - SupermarketCategory.objects.bulk_create(insert_list) - return len(insert_list) + return SupermarketCategory.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',)) def import_property(self): - identifier_list = [] datatype = 'property' - for k in list(self.data[datatype].keys()): - identifier_list.append(self.data[datatype][k]['name']) - - existing_objects = FoodPropertyType.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('name', flat=True) insert_list = [] for k in list(self.data[datatype].keys()): - if not (self.data[datatype][k]['name'] in existing_objects): - insert_list.append(FoodPropertyType( - name=self.data[datatype][k]['name'], - unit=self.data[datatype][k]['unit'], - open_data_slug=k, - space=self.request.space - )) + insert_list.append(FoodPropertyType( + name=self.data[datatype][k]['name'], + unit=self.data[datatype][k]['unit'], + open_data_slug=k, + space=self.request.space + )) - FoodPropertyType.objects.bulk_create(insert_list) - return len(insert_list) + return FoodPropertyType.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',)) def import_supermarket(self): datatype = 'supermarket' @@ -94,7 +77,7 @@ class OpenDataImporter: )) # always add open data slug if matching supermarket is found, otherwise relation might fail - Supermarket.objects.bulk_create(insert_list, unique_fields=('space', 'name',), update_conflicts=True, update_fields=('open_data_slug',)) + supermarkets = Supermarket.objects.bulk_create(insert_list, unique_fields=('space', 'name',), update_conflicts=True, update_fields=('open_data_slug',)) self._update_slug_cache(Supermarket, 'supermarket') insert_list = [] @@ -113,30 +96,37 @@ class OpenDataImporter: SupermarketCategoryRelation.objects.bulk_create(relations, ignore_conflicts=True, unique_fields=('supermarket', 'category',)) - return len(insert_list) + return supermarkets - def import_food(self, metric=True): + def import_food(self): identifier_list = [] datatype = 'food' for k in list(self.data[datatype].keys()): identifier_list.append(self.data[datatype][k]['name']) identifier_list.append(self.data[datatype][k]['plural_name']) - existing_objects = Food.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('name', flat=True) + existing_objects_flat = [] + existing_objects = {} + for f in Food.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('id', 'name', 'plural_name'): + existing_objects_flat.append(f[1]) + existing_objects_flat.append(f[2]) + existing_objects['name'] = f + existing_objects['plural_name'] = f self._update_slug_cache(Unit, 'unit') self._update_slug_cache(FoodPropertyType, 'property') pref_unit_key = 'preferred_unit_metric' pref_shopping_unit_key = 'preferred_packaging_unit_metric' - if not metric: + if not self.use_metric: pref_unit_key = 'preferred_unit_imperial' pref_shopping_unit_key = 'preferred_packaging_unit_imperial' insert_list = [] - + update_list = [] + update_field_list = [] for k in list(self.data[datatype].keys()): - if not (self.data[datatype][k]['name'] in existing_objects): + if not (self.data[datatype][k]['name'] in existing_objects_flat or self.data[datatype][k]['plural_name'] in existing_objects_flat): insert_list.append({'data': { 'name': self.data[datatype][k]['name'], 'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None, @@ -147,8 +137,30 @@ class OpenDataImporter: 'open_data_slug': k, 'space': self.request.space.id, }}) + else: + if self.data[datatype][k]['name'] in existing_objects: + existing_food_id = existing_objects[self.data[datatype][k]['name']][0] + else: + existing_food_id = existing_objects[self.data[datatype][k]['plural_name']][0] - Food.load_bulk(insert_list, None) + if self.update_existing: + update_field_list = ['name', 'plural_name', 'preferred_unit_id', 'preferred_shopping_unit_id', 'supermarket_category_id', 'fdc_id', 'open_data_slug', ] + update_list.append(Food( + id=existing_food_id, + name=self.data[datatype][k]['name'], + plural_name=self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None, + preferred_unit_id=self.slug_id_cache['unit'][self.data[datatype][k][pref_unit_key]], + preferred_shopping_unit_id=self.slug_id_cache['unit'][self.data[datatype][k][pref_shopping_unit_key]], + supermarket_category_id=self.slug_id_cache['category'][self.data[datatype][k]['supermarket_category']], + fdc_id=self.data[datatype][k]['fdc_id'] if self.data[datatype][k]['fdc_id'] != '' else None, + open_data_slug=k, + )) + else: + update_field_list = ['open_data_slug', ] + update_list.append(Food(id=existing_food_id, open_data_slug=k, )) + + foods = Food.load_bulk(insert_list, None) + Food.objects.bulk_update(update_list, update_field_list) self._update_slug_cache(Food, 'food') @@ -175,7 +187,7 @@ class OpenDataImporter: FoodProperty.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'food', 'property_type',)) Automation.objects.bulk_create(alias_list, ignore_conflicts=True, unique_fields=('space', 'param_1', 'param_2',)) - return len(insert_list) + return foods def import_conversion(self): datatype = 'conversion' @@ -193,5 +205,4 @@ class OpenDataImporter: created_by=self.request.user, )) - UnitConversion.objects.bulk_create(insert_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug')) - return len(insert_list) + return UnitConversion.objects.bulk_create(insert_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug')) diff --git a/cookbook/serializer.py b/cookbook/serializer.py index be1e8d8fd..bb581e988 100644 --- a/cookbook/serializer.py +++ b/cookbook/serializer.py @@ -464,7 +464,7 @@ class UnitSerializer(UniqueFieldsMixin, ExtendedRecipeMixin): class Meta: model = Unit - fields = ('id', 'name', 'plural_name', 'description', 'numrecipe', 'image') + fields = ('id', 'name', 'plural_name', 'description', 'numrecipe', 'image', 'open_data_slug') read_only_fields = ('id', 'numrecipe', 'image') @@ -627,7 +627,7 @@ class FoodSerializer(UniqueFieldsMixin, WritableNestedModelSerializer, ExtendedR fields = ( 'id', 'name', 'plural_name', 'description', 'shopping', 'recipe', 'food_onhand', 'supermarket_category', 'image', 'parent', 'numchild', 'numrecipe', 'inherit_fields', 'full_name', 'ignore_shopping', - 'substitute', 'substitute_siblings', 'substitute_children', 'substitute_onhand', 'child_inherit_fields' + 'substitute', 'substitute_siblings', 'substitute_children', 'substitute_onhand', 'child_inherit_fields', 'open_data_slug', ) read_only_fields = ('id', 'numchild', 'parent', 'image', 'numrecipe') diff --git a/cookbook/views/api.py b/cookbook/views/api.py index 2c81f4fa3..2db844405 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -1434,21 +1434,23 @@ class ImportOpenData(APIView): print(request.data) selected_version = request.data['selected_version'] selected_datatypes = request.data['selected_datatypes'] + update_existing = str2bool(request.data['update_existing']) + use_metric = str2bool(request.data['use_metric']) response = requests.get(f'https://raw.githubusercontent.com/TandoorRecipes/open-tandoor-data/main/build/{selected_version}.json') # TODO catch 404, timeout, ... data = json.loads(response.content) - data_importer = OpenDataImporter(request, data, update_existing=True) - data_importer.import_units() - data_importer.import_category() - data_importer.import_property() - data_importer.import_supermarket() - data_importer.import_food() # TODO pass metric parameter - data_importer.import_conversion() + response_obj = {} - return Response({ - 'test': '' - }) + data_importer = OpenDataImporter(request, data, update_existing=update_existing, use_metric=use_metric) + response_obj['unit'] = len(data_importer.import_units()) + response_obj['category'] = len(data_importer.import_category()) + response_obj['property'] = len(data_importer.import_property()) + response_obj['supermarket'] = len(data_importer.import_supermarket()) + response_obj['food'] = len(data_importer.import_food()) + response_obj['conversion'] = len(data_importer.import_conversion()) + + return Response(response_obj) def get_recipe_provider(recipe): diff --git a/vue/src/utils/models.js b/vue/src/utils/models.js index 7b7a6ae92..213a37874 100644 --- a/vue/src/utils/models.js +++ b/vue/src/utils/models.js @@ -91,6 +91,7 @@ export class Models { "substitute_children", "reset_inherit", "child_inherit_fields", + "open_data_slug", ], ], @@ -126,6 +127,14 @@ export class Models { label: "Recipe", // form.label always translated in utils.getForm() help_text: "food_recipe_help", // form.help_text always translated }, + open_data_slug: { + form_field: true, + type: "text", + field: "open_data_slug", + disabled: true, + label: "Open_Data_Slug", + help_text: "open_data_help_text", + }, onhand: { form_field: true, type: "checkbox", @@ -269,8 +278,9 @@ export class Models { apiName: "Unit", paginated: true, create: { - params: [["name", "plural_name", "description",]], + params: [["name", "plural_name", "description","open_data_slug",]], form: { + show_help: true, name: { form_field: true, type: "text", @@ -292,6 +302,14 @@ export class Models { label: "Description", placeholder: "", }, + open_data_slug: { + form_field: true, + type: "text", + field: "open_data_slug", + disabled: true, + label: "Open_Data_Slug", + help_text: "open_data_help_text", + }, }, }, merge: true,