diff --git a/.env.template b/.env.template index d7c44dfeb..bff198606 100644 --- a/.env.template +++ b/.env.template @@ -120,4 +120,13 @@ REVERSE_PROXY_AUTH=0 # Django session cookie settings. Can be changed to allow a single django application to authenticate several applications # when running under the same database # SESSION_COOKIE_DOMAIN=.example.com -# SESSION_COOKIE_NAME=sessionid # use this only to not interfere with non unified django applications under the same top level domain \ No newline at end of file +# SESSION_COOKIE_NAME=sessionid # use this only to not interfere with non unified django applications under the same top level domain + + +# by default SORT_TREE_BY_NAME is enabled this will store all Keywords and Food in case sensitive order +# this setting makes saving new keywords and foods very slow, which doesn't matter in most usecases. +# however, when doing large imports of recipes that will create new objects, can increase total run time by 5-10x +# Disabling SORT_TREE_BY_NAME (setting value to 0) will store objects unsorted, but will substantially increase speed of imports. +# Keywords and Food can be manually sorted by name in Admin +# This value can also be temporarily changed in Admin, it will revert the next time the application is started +# SORT_TREE_BY_NAME=0 \ No newline at end of file diff --git a/cookbook/admin.py b/cookbook/admin.py index 5b1025af6..4c4fc5220 100644 --- a/cookbook/admin.py +++ b/cookbook/admin.py @@ -102,9 +102,34 @@ class SyncLogAdmin(admin.ModelAdmin): admin.site.register(SyncLog, SyncLogAdmin) +@admin.action(description='Temporarily ENABLE sorting on Foods and Keywords.') +def enable_tree_sorting(modeladmin, request, queryset): + Food.node_order_by = ['name'] + Keyword.node_order_by = ['name'] + with scopes_disabled(): + Food.fix_tree(fix_paths=True) + Keyword.fix_tree(fix_paths=True) + + +@admin.action(description='Temporarily DISABLE sorting on Foods and Keywords.') +def disable_tree_sorting(modeladmin, request, queryset): + Food.node_order_by = [] + Keyword.node_order_by = [] + + +@admin.action(description='Fix problems and sort tree by name') +def sort_tree(modeladmin, request, queryset): + orginal_value = modeladmin.model.node_order_by[:] + modeladmin.model.node_order_by = ['name'] + with scopes_disabled(): + modeladmin.model.fix_tree(fix_paths=True) + modeladmin.model.node_order_by = orginal_value + + class KeywordAdmin(TreeAdmin): form = movenodeform_factory(Keyword) ordering = ('space', 'path',) + actions = [sort_tree, enable_tree_sorting, disable_tree_sorting] admin.site.register(Keyword, KeywordAdmin) @@ -151,6 +176,7 @@ admin.site.register(Unit) class FoodAdmin(TreeAdmin): form = movenodeform_factory(Keyword) ordering = ('space', 'path',) + actions = [sort_tree, enable_tree_sorting, disable_tree_sorting] admin.site.register(Food, FoodAdmin) diff --git a/cookbook/models.py b/cookbook/models.py index fe763c7ef..63db9b65e 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -11,7 +11,7 @@ from django.contrib.postgres.indexes import GinIndex from django.contrib.postgres.search import SearchVectorField from django.core.files.uploadedfile import UploadedFile, InMemoryUploadedFile from django.core.validators import MinLengthValidator -from django.db import models +from django.db import models, IntegrityError from django.db.models import Index, ProtectedError from django.utils import timezone from django.utils.translation import gettext as _ @@ -19,7 +19,7 @@ from treebeard.mp_tree import MP_Node, MP_NodeManager from django_scopes import ScopedManager, scopes_disabled from django_prometheus.models import ExportModelOperationsMixin from recipes.settings import (COMMENT_PREF_DEFAULT, FRACTION_PREF_DEFAULT, - STICKY_NAV_PREF_DEFAULT) + STICKY_NAV_PREF_DEFAULT, SORT_TREE_BY_NAME) def get_user_name(self): @@ -44,7 +44,12 @@ class TreeManager(MP_NodeManager): return self.get(name__exact=kwargs['name'], space=kwargs['space']), False except self.model.DoesNotExist: with scopes_disabled(): - return self.model.add_root(**kwargs), True + try: + return self.model.add_root(**kwargs), True + except IntegrityError as e: + if 'Key (path)' in e.args[0]: + self.model.fix_tree(fix_paths=True) + return self.model.add_root(**kwargs), True class TreeModel(MP_Node): @@ -335,8 +340,8 @@ class SyncLog(models.Model, PermissionModelMixin): class Keyword(ExportModelOperationsMixin('keyword'), TreeModel, PermissionModelMixin): - # TODO add find and fix problem functions - # node_order_by = ['name'] + if SORT_TREE_BY_NAME: + node_order_by = ['name'] name = models.CharField(max_length=64) icon = models.CharField(max_length=16, blank=True, null=True) description = models.TextField(default="", blank=True) @@ -353,6 +358,13 @@ class Keyword(ExportModelOperationsMixin('keyword'), TreeModel, PermissionModelM indexes = (Index(fields=['id', 'name']),) +# when starting up run fix_tree to: +# a) make sure that nodes are sorted when switching between sort modes +# b) fix problems, if any, with tree consistency +with scopes_disabled(): + Keyword.fix_tree(fix_paths=True) + + class Unit(ExportModelOperationsMixin('unit'), models.Model, PermissionModelMixin): name = models.CharField(max_length=128, validators=[MinLengthValidator(1)]) description = models.TextField(blank=True, null=True) @@ -370,8 +382,8 @@ class Unit(ExportModelOperationsMixin('unit'), models.Model, PermissionModelMixi class Food(ExportModelOperationsMixin('food'), TreeModel, PermissionModelMixin): - # TODO add find and fix problem functions - # node_order_by = ['name'] + if SORT_TREE_BY_NAME: + node_order_by = ['name'] name = models.CharField(max_length=128, validators=[MinLengthValidator(1)]) recipe = models.ForeignKey('Recipe', null=True, blank=True, on_delete=models.SET_NULL) supermarket_category = models.ForeignKey(SupermarketCategory, null=True, blank=True, on_delete=models.SET_NULL) @@ -400,6 +412,13 @@ class Food(ExportModelOperationsMixin('food'), TreeModel, PermissionModelMixin): ) +# when starting up run fix_tree to: +# a) make sure that nodes are sorted when switching between sort modes +# b) fix problems, if any, with tree consistency +with scopes_disabled(): + Food.fix_tree(fix_paths=True) + + class Ingredient(ExportModelOperationsMixin('ingredient'), models.Model, PermissionModelMixin): # a pre-delete signal on Food checks if the Ingredient is part of a Step, if it is raises a ProtectedError instead of cascading the delete food = models.ForeignKey(Food, on_delete=models.CASCADE, null=True, blank=True) @@ -793,7 +812,7 @@ class ViewLog(ExportModelOperationsMixin('view_log'), models.Model, PermissionMo class Meta(): indexes = ( Index(fields=['recipe']), - Index(fields=[ '-created_at']), + Index(fields=['-created_at']), Index(fields=['created_by']), Index(fields=['recipe', '-created_at', 'created_by']), ) diff --git a/cookbook/tests/api/test_api_food.py b/cookbook/tests/api/test_api_food.py index 8e53c753a..a1b30d3c1 100644 --- a/cookbook/tests/api/test_api_food.py +++ b/cookbook/tests/api/test_api_food.py @@ -21,6 +21,10 @@ LIST_URL = 'api:food-list' DETAIL_URL = 'api:food-detail' MOVE_URL = 'api:food-move' MERGE_URL = 'api:food-merge' +if (Food.node_order_by): + node_location = 'sorted-child' +else: + node_location = 'last-child' @pytest.fixture() @@ -264,7 +268,7 @@ def test_integrity(u1_s1, recipe_1_s1): i_1.step_set.first().ingredients.remove(i_1) assert Food.objects.count() == 10 assert Ingredient.objects.count() == 10 - + # deleting food will succeed because its not part of recipe and delete will cascade to Ingredient r = u1_s1.delete( reverse( @@ -429,7 +433,7 @@ def test_root_filter(obj_1, obj_1_1, obj_1_1_1, obj_2, obj_3, u1_s1): assert len(response['results']) == 2 with scopes_disabled(): - obj_2.move(obj_1, 'last-child') + obj_2.move(obj_1, node_location) # should return direct children of obj_1 (obj_1_1, obj_2), ignoring query filters response = json.loads(u1_s1.get(f'{reverse(LIST_URL)}?root={obj_1.id}').content) assert response['count'] == 2 @@ -439,7 +443,7 @@ def test_root_filter(obj_1, obj_1_1, obj_1_1_1, obj_2, obj_3, u1_s1): def test_tree_filter(obj_1, obj_1_1, obj_1_1_1, obj_2, obj_3, u1_s1): with scopes_disabled(): - obj_2.move(obj_1, 'last-child') + obj_2.move(obj_1, node_location) # should return full tree starting at obj_1 (obj_1_1_1, obj_2), ignoring query filters response = json.loads(u1_s1.get(f'{reverse(LIST_URL)}?tree={obj_1.id}').content) assert response['count'] == 4 diff --git a/cookbook/tests/api/test_api_keyword.py b/cookbook/tests/api/test_api_keyword.py index 82cad211e..7afcbf8a1 100644 --- a/cookbook/tests/api/test_api_keyword.py +++ b/cookbook/tests/api/test_api_keyword.py @@ -20,7 +20,10 @@ LIST_URL = 'api:keyword-list' DETAIL_URL = 'api:keyword-detail' MOVE_URL = 'api:keyword-move' MERGE_URL = 'api:keyword-merge' - +if (Keyword.node_order_by): + node_location = 'sorted-child' +else: + node_location = 'last-child' @pytest.fixture() def obj_1(space_1): @@ -350,7 +353,7 @@ def test_root_filter(obj_1, obj_1_1, obj_1_1_1, obj_2, obj_3, u1_s1): assert len(response['results']) == 2 with scopes_disabled(): - obj_2.move(obj_1, 'last-child') + obj_2.move(obj_1, node_location) # should return direct children of obj_1 (obj_1_1, obj_2), ignoring query filters response = json.loads(u1_s1.get(f'{reverse(LIST_URL)}?root={obj_1.id}').content) assert response['count'] == 2 @@ -360,7 +363,7 @@ def test_root_filter(obj_1, obj_1_1, obj_1_1_1, obj_2, obj_3, u1_s1): def test_tree_filter(obj_1, obj_1_1, obj_1_1_1, obj_2, obj_3, u1_s1): with scopes_disabled(): - obj_2.move(obj_1, 'last-child') + obj_2.move(obj_1, node_location) # should return full tree starting at obj_1 (obj_1_1_1, obj_2), ignoring query filters response = json.loads(u1_s1.get(f'{reverse(LIST_URL)}?tree={obj_1.id}').content) assert response['count'] == 4 diff --git a/cookbook/views/api.py b/cookbook/views/api.py index 135f8a94b..5c623c407 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -189,9 +189,14 @@ class MergeMixin(ViewSetMixin): # TODO update Units to use merge API # a new scenario exists and needs to be handled raise NotImplementedError if isTree: + if self.model.node_order_by: + node_location = 'sorted-child' + else: + node_location = 'last-child' + children = source.get_children().exclude(id=target.id) for c in children: - c.move(target, 'last-child') + c.move(target, node_location) content = {'msg': _(f'{source.name} was merged successfully with {target.name}')} source.delete() return Response(content, status=status.HTTP_200_OK) @@ -232,6 +237,10 @@ class TreeMixin(MergeMixin, FuzzyFilterMixin): @decorators.renderer_classes((TemplateHTMLRenderer, JSONRenderer)) def move(self, request, pk, parent): self.description = f"Move {self.basename} to be a child of {self.basename} with ID of [int]. Use ID: 0 to move {self.basename} to the root." + if self.model.node_order_by: + node_location = 'sorted' + else: + node_location = 'last' try: child = self.model.objects.get(pk=pk, space=self.request.space) @@ -244,7 +253,7 @@ class TreeMixin(MergeMixin, FuzzyFilterMixin): if parent == 0: try: with scopes_disabled(): - child.move(self.model.get_first_root_node(), 'last-sibling') + child.move(self.model.get_first_root_node(), f'{node_location}-sibling') content = {'msg': _(f'{child.name} was moved successfully to the root.')} return Response(content, status=status.HTTP_200_OK) except (PathOverflow, InvalidMoveToDescendant, InvalidPosition): @@ -262,7 +271,7 @@ class TreeMixin(MergeMixin, FuzzyFilterMixin): try: with scopes_disabled(): - child.move(parent, 'last-child') + child.move(parent, f'{node_location}-child') content = {'msg': _(f'{child.name} was moved successfully to parent {parent.name}')} return Response(content, status=status.HTTP_200_OK) except (PathOverflow, InvalidMoveToDescendant, InvalidPosition): diff --git a/recipes/settings.py b/recipes/settings.py index bfd58dc4b..95ef48c10 100644 --- a/recipes/settings.py +++ b/recipes/settings.py @@ -73,6 +73,8 @@ ACCOUNT_SIGNUP_FORM_CLASS = 'cookbook.forms.AllAuthSignupForm' TERMS_URL = os.getenv('TERMS_URL', '') PRIVACY_URL = os.getenv('PRIVACY_URL', '') IMPRINT_URL = os.getenv('IMPRINT_URL', '') +if SORT_TREE_BY_NAME:= bool(int(os.getenv('SQL_DEBUG', True))): + MIDDLEWARE += ('recipes.middleware.SqlPrintingMiddleware',) HOSTED = bool(int(os.getenv('HOSTED', False))) @@ -390,5 +392,3 @@ EMAIL_USE_SSL = bool(int(os.getenv('EMAIL_USE_SSL', False))) DEFAULT_FROM_EMAIL = os.getenv('DEFAULT_FROM_EMAIL', 'webmaster@localhost') ACCOUNT_EMAIL_SUBJECT_PREFIX = os.getenv('ACCOUNT_EMAIL_SUBJECT_PREFIX', '[Tandoor Recipes] ') # allauth sender prefix -if os.getenv('SQL_DEBUG', False): - MIDDLEWARE += ('recipes.middleware.SqlPrintingMiddleware',) diff --git a/recipes/wsgi.py b/recipes/wsgi.py index a33e55a2b..64e997379 100644 --- a/recipes/wsgi.py +++ b/recipes/wsgi.py @@ -8,7 +8,6 @@ https://docs.djangoproject.com/en/2.0/howto/deployment/wsgi/ """ import os - from django.core.wsgi import get_wsgi_application os.environ.setdefault("DJANGO_SETTINGS_MODULE", "recipes.settings")