From f7cb067b527ddab656e1b55ce101beda64dab5c0 Mon Sep 17 00:00:00 2001 From: smilerz Date: Tue, 11 Jan 2022 07:24:59 -0600 Subject: [PATCH 1/5] construct values in queryset instead of serializer methods --- cookbook/helper/HelperFunctions.py | 2 +- cookbook/serializer.py | 44 ++++++++++---------- cookbook/views/api.py | 65 +++++++++++++++++++++++++----- recipes/middleware.py | 7 ++-- recipes/settings.py | 4 +- 5 files changed, 82 insertions(+), 40 deletions(-) diff --git a/cookbook/helper/HelperFunctions.py b/cookbook/helper/HelperFunctions.py index cf04c3e2d..e2971c2ed 100644 --- a/cookbook/helper/HelperFunctions.py +++ b/cookbook/helper/HelperFunctions.py @@ -7,7 +7,7 @@ class Round(Func): def str2bool(v): - if type(v) == bool: + if type(v) == bool or v is None: return v else: return v.lower() in ("yes", "true", "1") diff --git a/cookbook/serializer.py b/cookbook/serializer.py index ac6238681..652e94f8e 100644 --- a/cookbook/serializer.py +++ b/cookbook/serializer.py @@ -12,6 +12,7 @@ from rest_framework import serializers from rest_framework.exceptions import NotFound, ValidationError from rest_framework.fields import empty +from cookbook.helper.HelperFunctions import str2bool from cookbook.helper.shopping_helper import list_from_recipe from cookbook.models import (Automation, BookmarkletImport, Comment, CookLog, Food, FoodInheritField, ImportLog, Ingredient, Keyword, MealPlan, MealType, @@ -21,13 +22,18 @@ from cookbook.models import (Automation, BookmarkletImport, Comment, CookLog, Fo SupermarketCategoryRelation, Sync, SyncLog, Unit, UserFile, UserPreference, ViewLog) from cookbook.templatetags.custom_tags import markdown +from recipes.settings import MEDIA_URL, SCRIPT_NAME class ExtendedRecipeMixin(serializers.ModelSerializer): # adds image and recipe count to serializer when query param extended=1 - image = serializers.SerializerMethodField('get_image') - numrecipe = serializers.SerializerMethodField('count_recipes') + # ORM path to this object from Recipe recipe_filter = None + # list of ORM paths to any image + images = None + + image = serializers.SerializerMethodField('get_image') + numrecipe = serializers.ReadOnlyField(source='count_recipes_test') def get_fields(self, *args, **kwargs): fields = super().get_fields(*args, **kwargs) @@ -37,8 +43,7 @@ class ExtendedRecipeMixin(serializers.ModelSerializer): api_serializer = None # extended values are computationally expensive and not needed in normal circumstances try: - if bool(int( - self.context['request'].query_params.get('extended', False))) and self.__class__ == api_serializer: + if str2bool(self.context['request'].query_params.get('extended', False)) and self.__class__ == api_serializer: return fields except (AttributeError, KeyError) as e: pass @@ -50,21 +55,8 @@ class ExtendedRecipeMixin(serializers.ModelSerializer): return fields def get_image(self, obj): - # TODO add caching - recipes = Recipe.objects.filter(**{self.recipe_filter: obj}, space=obj.space).exclude( - image__isnull=True).exclude(image__exact='') - try: - if recipes.count() == 0 and obj.has_children(): - obj__in = self.recipe_filter + '__in' - recipes = Recipe.objects.filter(**{obj__in: obj.get_descendants()}, space=obj.space).exclude( - image__isnull=True).exclude(image__exact='') # if no recipes found - check whole tree - except AttributeError: - # probably not a tree - pass - if recipes.count() != 0: - return random.choice(recipes).image.url - else: - return None + if obj.recipe_image: + return SCRIPT_NAME + MEDIA_URL + obj.recipe_image def count_recipes(self, obj): return Recipe.objects.filter(**{self.recipe_filter: obj}, space=obj.space).count() @@ -98,7 +90,11 @@ class CustomOnHandField(serializers.Field): return instance def to_representation(self, obj): - shared_users = [x.id for x in list(self.context['request'].user.get_shopping_share())] + [self.context['request'].user.id] + shared_users = [] + if request := self.context.get('request', None): + shared_users = request._shared_users + else: + shared_users = [x.id for x in list(self.context['request'].user.get_shopping_share())] + [self.context['request'].user.id] return obj.onhand_users.filter(id__in=shared_users).exists() def to_internal_value(self, data): @@ -379,14 +375,16 @@ class RecipeSimpleSerializer(serializers.ModelSerializer): class FoodSerializer(UniqueFieldsMixin, WritableNestedModelSerializer, ExtendedRecipeMixin): supermarket_category = SupermarketCategorySerializer(allow_null=True, required=False) recipe = RecipeSimpleSerializer(allow_null=True, required=False) - shopping = serializers.SerializerMethodField('get_shopping_status') + # shopping = serializers.SerializerMethodField('get_shopping_status') + shopping = serializers.ReadOnlyField(source='shopping_status') inherit_fields = FoodInheritFieldSerializer(many=True, allow_null=True, required=False) food_onhand = CustomOnHandField(required=False, allow_null=True) recipe_filter = 'steps__ingredients__food' + images = ['recipe__image'] - def get_shopping_status(self, obj): - return ShoppingListEntry.objects.filter(space=obj.space, food=obj, checked=False).count() > 0 + # def get_shopping_status(self, obj): + # return ShoppingListEntry.objects.filter(space=obj.space, food=obj, checked=False).count() > 0 def create(self, validated_data): validated_data['name'] = validated_data['name'].strip() diff --git a/cookbook/views/api.py b/cookbook/views/api.py index 86f976602..b0023d9ce 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -12,8 +12,9 @@ from django.contrib.auth.models import User from django.contrib.postgres.search import TrigramSimilarity from django.core.exceptions import FieldError, ValidationError from django.core.files import File -from django.db.models import Case, ProtectedError, Q, Value, When +from django.db.models import Case, Count, Exists, OuterRef, ProtectedError, Q, Subquery, Value, When from django.db.models.fields.related import ForeignObjectRel +from django.db.models.functions import Coalesce from django.http import FileResponse, HttpResponse, JsonResponse from django.shortcuts import get_object_or_404, redirect from django.urls import reverse @@ -30,6 +31,7 @@ from rest_framework.response import Response from rest_framework.viewsets import ViewSetMixin from treebeard.exceptions import InvalidMoveToDescendant, InvalidPosition, PathOverflow +from cookbook.helper.HelperFunctions import str2bool from cookbook.helper.image_processing import handle_image from cookbook.helper.ingredient_parser import IngredientParser from cookbook.helper.permission_helper import (CustomIsAdmin, CustomIsGuest, CustomIsOwner, @@ -100,7 +102,38 @@ class DefaultPagination(PageNumberPagination): max_page_size = 200 -class FuzzyFilterMixin(ViewSetMixin): +class ExtendedRecipeMixin(): + ''' + ExtendedRecipe annotates a queryset with recipe_image and recipe_count values + ''' + @classmethod + def annotate_recipe(self, queryset=None, request=None, serializer=None, tree=False): + extended = str2bool(request.query_params.get('extended', None)) + if extended: + recipe_filter = serializer.recipe_filter + images = serializer.images + space = request.space + + # add a recipe count annotation to the query + # explanation on construction https://stackoverflow.com/a/43771738/15762829 + recipe_count = Recipe.objects.filter(**{recipe_filter: OuterRef('id')}, space=space).values(recipe_filter).annotate(count=Count('pk')).values('count') + queryset = queryset.annotate(recipe_count_test=Coalesce(Subquery(recipe_count), 0)) + + # add a recipe image annotation to the query + image_subquery = Recipe.objects.filter(**{recipe_filter: OuterRef('id')}, space=space).exclude(image__isnull=True).exclude(image__exact='').order_by("?").values('image')[:1] + if tree: + image_children_subquery = Recipe.objects.filter(**{f"{recipe_filter}__path__startswith": OuterRef('path')}, + space=space).exclude(image__isnull=True).exclude(image__exact='').order_by("?").values('image')[:1] + else: + image_children_subquery = None + if images: + queryset = queryset.annotate(recipe_image=Coalesce(*images, image_subquery, image_children_subquery)) + else: + queryset = queryset.annotate(recipe_image=Coalesce(image_subquery, image_children_subquery)) + return queryset + + +class FuzzyFilterMixin(ViewSetMixin, ExtendedRecipeMixin): schema = FilterSchema() def get_queryset(self): @@ -141,12 +174,12 @@ class FuzzyFilterMixin(ViewSetMixin): if random: self.queryset = self.queryset.order_by("?") self.queryset = self.queryset[:int(limit)] - return self.queryset + return self.annotate_recipe(queryset=self.queryset, request=self.request, serializer=self.serializer_class) class MergeMixin(ViewSetMixin): - @decorators.action(detail=True, url_path='merge/(?P[^/.]+)', methods=['PUT'], ) - @decorators.renderer_classes((TemplateHTMLRenderer, JSONRenderer)) + @ decorators.action(detail=True, url_path='merge/(?P[^/.]+)', methods=['PUT'], ) + @ decorators.renderer_classes((TemplateHTMLRenderer, JSONRenderer)) def merge(self, request, pk, target): self.description = f"Merge {self.basename} onto target {self.basename} with ID of [int]." @@ -211,7 +244,7 @@ class MergeMixin(ViewSetMixin): return Response(content, status=status.HTTP_400_BAD_REQUEST) -class TreeMixin(MergeMixin, FuzzyFilterMixin): +class TreeMixin(MergeMixin, FuzzyFilterMixin, ExtendedRecipeMixin): schema = TreeSchema() model = None @@ -237,11 +270,13 @@ class TreeMixin(MergeMixin, FuzzyFilterMixin): except self.model.DoesNotExist: self.queryset = self.model.objects.none() else: - return super().get_queryset() - return self.queryset.filter(space=self.request.space).order_by('name') + self.queryset = super().get_queryset() + self.queryset = self.queryset.filter(space=self.request.space).order_by('name') - @decorators.action(detail=True, url_path='move/(?P[^/.]+)', methods=['PUT'], ) - @decorators.renderer_classes((TemplateHTMLRenderer, JSONRenderer)) + return self.annotate_recipe(queryset=self.queryset, request=self.request, serializer=self.serializer_class, tree=True) + + @ decorators.action(detail=True, url_path='move/(?P[^/.]+)', methods=['PUT'], ) + @ decorators.renderer_classes((TemplateHTMLRenderer, JSONRenderer)) def move(self, request, pk, parent): self.description = f"Move {self.basename} to be a child of {self.basename} with ID of [int]. Use ID: 0 to move {self.basename} to the root." if self.model.node_order_by: @@ -413,7 +448,15 @@ class FoodViewSet(viewsets.ModelViewSet, TreeMixin): permission_classes = [CustomIsUser] pagination_class = DefaultPagination - @decorators.action(detail=True, methods=['PUT'], serializer_class=FoodShoppingUpdateSerializer,) + def get_queryset(self): + self.request._shared_users = [x.id for x in list(self.request.user.get_shopping_share())] + [self.request.user.id] + + self.queryset = super().get_queryset() + shopping_status = ShoppingListEntry.objects.filter(space=self.request.space, food=OuterRef('id'), checked=False).values('id') + # onhand_status = self.queryset.annotate(onhand_status=Exists(onhand_users_set__in=[shared_users])) + return self.queryset.annotate(shopping_status=Exists(shopping_status)).prefetch_related('onhand_users', 'inherit_fields').select_related('recipe', 'supermarket_category') + + @ decorators.action(detail=True, methods=['PUT'], serializer_class=FoodShoppingUpdateSerializer,) # TODO DRF only allows one action in a decorator action without overriding get_operation_id_base() this should be PUT and DELETE probably def shopping(self, request, pk): if self.request.space.demo: diff --git a/recipes/middleware.py b/recipes/middleware.py index ebe9c51f0..09346e6a8 100644 --- a/recipes/middleware.py +++ b/recipes/middleware.py @@ -13,19 +13,20 @@ class CustomRemoteUser(RemoteUserMiddleware): Gist code by vstoykov, you can check his original gist at: https://gist.github.com/vstoykov/1390853/5d2e8fac3ca2b2ada8c7de2fb70c021e50927375 Changes: -Ignoring static file requests and a certain useless admin request from triggering the logger. +Ignoring static file requests and a certain useless admin request from triggering the logger. Updated statements to make it Python 3 friendly. """ - def terminal_width(): """ Function to compute the terminal width. """ width = 0 try: - import struct, fcntl, termios + import fcntl + import struct + import termios s = struct.pack('HHHH', 0, 0, 0, 0) x = fcntl.ioctl(1, termios.TIOCGWINSZ, s) width = struct.unpack('HHHH', x)[1] diff --git a/recipes/settings.py b/recipes/settings.py index 9b9dacbb1..dcbbfc07e 100644 --- a/recipes/settings.py +++ b/recipes/settings.py @@ -371,10 +371,10 @@ LANGUAGES = [ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.0/howto/static-files/ +SCRIPT_NAME = os.getenv('SCRIPT_NAME', '') # path for django_js_reverse to generate the javascript file containing all urls. Only done because the default command (collectstatic_js_reverse) fails to update the manifest JS_REVERSE_OUTPUT_PATH = os.path.join(BASE_DIR, "cookbook/static/django_js_reverse") - -JS_REVERSE_SCRIPT_PREFIX = os.getenv('JS_REVERSE_SCRIPT_PREFIX', os.getenv('SCRIPT_NAME', '')) +JS_REVERSE_SCRIPT_PREFIX = os.getenv('JS_REVERSE_SCRIPT_PREFIX', SCRIPT_NAME) STATIC_URL = os.getenv('STATIC_URL', '/static/') STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles") From f9b04a3f1ea41a4b32211ff1b9106b08ca010d59 Mon Sep 17 00:00:00 2001 From: smilerz Date: Tue, 11 Jan 2022 08:33:42 -0600 Subject: [PATCH 2/5] bug fix --- cookbook/serializer.py | 6 +++--- cookbook/views/api.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cookbook/serializer.py b/cookbook/serializer.py index 652e94f8e..09520545f 100644 --- a/cookbook/serializer.py +++ b/cookbook/serializer.py @@ -90,10 +90,10 @@ class CustomOnHandField(serializers.Field): return instance def to_representation(self, obj): - shared_users = [] + shared_users = None if request := self.context.get('request', None): - shared_users = request._shared_users - else: + shared_users = getattr(request, '_shared_users', None) + if shared_users is None: shared_users = [x.id for x in list(self.context['request'].user.get_shopping_share())] + [self.context['request'].user.id] return obj.onhand_users.filter(id__in=shared_users).exists() diff --git a/cookbook/views/api.py b/cookbook/views/api.py index b0023d9ce..1135cfec8 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -270,7 +270,7 @@ class TreeMixin(MergeMixin, FuzzyFilterMixin, ExtendedRecipeMixin): except self.model.DoesNotExist: self.queryset = self.model.objects.none() else: - self.queryset = super().get_queryset() + return self.annotate_recipe(queryset=super().get_queryset(), request=self.request, serializer=self.serializer_class, tree=True) self.queryset = self.queryset.filter(space=self.request.space).order_by('name') return self.annotate_recipe(queryset=self.queryset, request=self.request, serializer=self.serializer_class, tree=True) From 20d61160ba1edcd60bb6957f5fbcc0a57fecb8bc Mon Sep 17 00:00:00 2001 From: smilerz Date: Wed, 12 Jan 2022 12:21:28 -0600 Subject: [PATCH 3/5] refactor get_facets as RecipeFacets class --- cookbook/helper/recipe_search.py | 523 ++++++++++++------ cookbook/views/api.py | 14 +- recipes/middleware.py | 1 + .../RecipeSearchView/RecipeSearchView.vue | 51 +- 4 files changed, 412 insertions(+), 177 deletions(-) diff --git a/cookbook/helper/recipe_search.py b/cookbook/helper/recipe_search.py index 9ad2b81f5..608fd8a2e 100644 --- a/cookbook/helper/recipe_search.py +++ b/cookbook/helper/recipe_search.py @@ -3,8 +3,8 @@ from datetime import timedelta from django.contrib.postgres.search import SearchQuery, SearchRank, TrigramSimilarity from django.core.cache import caches -from django.db.models import Avg, Case, Count, Func, Max, Q, Subquery, Value, When -from django.db.models.functions import Coalesce +from django.db.models import Avg, Case, Count, Func, Max, OuterRef, Q, Subquery, Value, When +from django.db.models.functions import Coalesce, Substr from django.utils import timezone, translation from cookbook.filters import RecipeFilter @@ -145,6 +145,8 @@ def search_recipes(request, queryset, params): # TODO creating setting to include descendants of keywords a setting # for kw in Keyword.objects.filter(pk__in=search_keywords): # search_keywords += list(kw.get_descendants().values_list('pk', flat=True)) + for kw in Keyword.objects.filter(pk__in=search_keywords): + search_keywords = [*search_keywords, *list(kw.get_descendants_and_self().values_list('pk', flat=True))] queryset = queryset.filter(keywords__id__in=search_keywords) else: # when performing an 'and' search returned recipes should include a parent OR any of its descedants @@ -155,6 +157,8 @@ def search_recipes(request, queryset, params): if len(search_foods) > 0: if search_foods_or: # TODO creating setting to include descendants of food a setting + for fd in Food.objects.filter(pk__in=search_foods): + search_foods = [*search_foods, *list(fd.get_descendants_and_self().values_list('pk', flat=True))] queryset = queryset.filter(steps__ingredients__food__id__in=search_foods) else: # when performing an 'and' search returned recipes should include a parent OR any of its descedants @@ -170,6 +174,7 @@ def search_recipes(request, queryset, params): queryset = queryset.filter(recipebookentry__book__id=k) if search_rating: + # TODO make ratings a settings user-only vs all-users queryset = queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=request.user, then='cooklog__rating'), default=Value(0))))) if search_rating == -1: queryset = queryset.filter(rating=0) @@ -196,189 +201,375 @@ def search_recipes(request, queryset, params): return queryset -# TODO: This might be faster https://github.com/django-treebeard/django-treebeard/issues/115 -def get_facet(qs=None, request=None, use_cache=True, hash_key=None): - """ - Gets an annotated list from a queryset. - :param qs: - recipe queryset to build facets from - :param request: - the web request that contains the necessary query parameters - :param use_cache: - will find results in cache, if any, and return them or empty list. - will save the list of recipes IDs in the cache for future processing - :param hash_key: - the cache key of the recipe list to process - only evaluated if the use_cache parameter is false - """ - facets = {} - recipe_list = [] - cache_timeout = 600 +class CacheEmpty(Exception): + pass - if use_cache: - qs_hash = hash(frozenset(qs.values_list('pk'))) - facets['cache_key'] = str(qs_hash) - SEARCH_CACHE_KEY = f"recipes_filter_{qs_hash}" - if c := caches['default'].get(SEARCH_CACHE_KEY, None): - facets['Keywords'] = c['Keywords'] or [] - facets['Foods'] = c['Foods'] or [] - facets['Books'] = c['Books'] or [] - facets['Ratings'] = c['Ratings'] or [] - facets['Recent'] = c['Recent'] or [] - else: - facets['Keywords'] = [] - facets['Foods'] = [] - facets['Books'] = [] - rating_qs = qs.annotate(rating=Round(Avg(Case(When(cooklog__created_by=request.user, then='cooklog__rating'), default=Value(0))))) - facets['Ratings'] = dict(Counter(r.rating for r in rating_qs)) - facets['Recent'] = ViewLog.objects.filter( - created_by=request.user, space=request.space, - created_at__gte=timezone.now() - timedelta(days=14) # TODO make days of recent recipe a setting - ).values_list('recipe__pk', flat=True) - cached_search = { - 'recipe_list': list(qs.values_list('id', flat=True)), - 'keyword_list': request.query_params.getlist('keywords', []), - 'food_list': request.query_params.getlist('foods', []), - 'book_list': request.query_params.getlist('book', []), - 'search_keywords_or': str2bool(request.query_params.get('keywords_or', True)), - 'search_foods_or': str2bool(request.query_params.get('foods_or', True)), - 'search_books_or': str2bool(request.query_params.get('books_or', True)), - 'space': request.space, - 'Ratings': facets['Ratings'], - 'Recent': facets['Recent'], - 'Keywords': facets['Keywords'], - 'Foods': facets['Foods'], - 'Books': facets['Books'] +class RecipeFacet(): + def __init__(self, request, queryset=None, hash_key=None, cache_timeout=600): + if hash_key is None and queryset is None: + raise ValueError(_("One of queryset or hash_key must be provided")) + + self._request = request + self._queryset = queryset + self.hash_key = hash_key or str(hash(frozenset(self._queryset.values_list('pk')))) + self._SEARCH_CACHE_KEY = f"recipes_filter_{self.hash_key}" + self._cache_timeout = cache_timeout + self._cache = caches['default'].get(self._SEARCH_CACHE_KEY, None) + if self._cache is None and self._queryset is None: + raise CacheEmpty("No queryset provided and cache empty") + + self.Keywords = getattr(self._cache, 'Keywords', None) + self.Foods = getattr(self._cache, 'Foods', None) + self.Books = getattr(self._cache, 'Books', None) + self.Ratings = getattr(self._cache, 'Ratings', None) + self.Recent = getattr(self._cache, 'Recent', None) + + if self._queryset: + self._recipe_list = list(self._queryset.values_list('id', flat=True)) + self._search_params = { + 'keyword_list': self._request.query_params.getlist('keywords', []), + 'food_list': self._request.query_params.getlist('foods', []), + 'book_list': self._request.query_params.getlist('book', []), + 'search_keywords_or': str2bool(self._request.query_params.get('keywords_or', True)), + 'search_foods_or': str2bool(self._request.query_params.get('foods_or', True)), + 'search_books_or': str2bool(self._request.query_params.get('books_or', True)), + 'space': self._request.space, + } + elif self.hash_key: + self._recipe_list = self._cache.get('recipe_list', None) + self._search_params = { + 'keyword_list': self._cache.get('keyword_list', None), + 'food_list': self._cache.get('food_list', None), + 'book_list': self._cache.get('book_list', None), + 'search_keywords_or': self._cache.get('search_keywords_or', None), + 'search_foods_or': self._cache.get('search_foods_or', None), + 'search_books_or': self._cache.get('search_books_or', None), + 'space': self._cache.get('space', None), + } + + self._cache = { + **self._search_params, + 'recipe_list': self._recipe_list, + 'Ratings': self.Ratings, + 'Recent': self.Recent, + 'Keywords': self.Keywords, + 'Foods': self.Foods, + 'Books': self.Books + } - caches['default'].set(SEARCH_CACHE_KEY, cached_search, cache_timeout) - return facets + caches['default'].set(self._SEARCH_CACHE_KEY, self._cache, self._cache_timeout) - SEARCH_CACHE_KEY = f'recipes_filter_{hash_key}' - if c := caches['default'].get(SEARCH_CACHE_KEY, None): - recipe_list = c['recipe_list'] - keyword_list = c['keyword_list'] - food_list = c['food_list'] - book_list = c['book_list'] - search_keywords_or = c['search_keywords_or'] - search_foods_or = c['search_foods_or'] - search_books_or = c['search_books_or'] - else: - return {} + def get_facets(self): + if self._cache is None: + pass + return { + 'cache_key': self.hash_key, + 'Ratings': self.get_ratings(), + 'Recent': self.get_recent(), + 'Keywords': self.get_keywords(), + 'Foods': self.get_foods(), + 'Books': self.get_books() + } - # if using an OR search, will annotate all keywords, otherwise, just those that appear in results - if search_keywords_or: - keywords = Keyword.objects.filter(space=request.space).annotate(recipe_count=Count('recipe')) - else: - keywords = Keyword.objects.filter(recipe__in=recipe_list, space=request.space).annotate(recipe_count=Count('recipe')) - # custom django-tree function annotates a queryset to make building a tree easier. - # see https://django-treebeard.readthedocs.io/en/latest/api.html#treebeard.models.Node.get_annotated_list_qs for details - kw_a = annotated_qs(keywords, root=True, fill=True) + def set_cache(self, key, value): + self._cache = {**self._cache, key: value} + caches['default'].set( + self._SEARCH_CACHE_KEY, + self._cache, + self._cache_timeout + ) - # # if using an OR search, will annotate all keywords, otherwise, just those that appear in results - if search_foods_or: - foods = Food.objects.filter(space=request.space).annotate(recipe_count=Count('ingredient')) - else: - foods = Food.objects.filter(ingredient__step__recipe__in=recipe_list, space=request.space).annotate(recipe_count=Count('ingredient')) - food_a = annotated_qs(foods, root=True, fill=True) + def get_books(self): + if self.Books is None: + self.Books = [] + return self.Books - facets['Keywords'] = fill_annotated_parents(kw_a, keyword_list) - facets['Foods'] = fill_annotated_parents(food_a, food_list) - # TODO add book facet - facets['Books'] = [] - c['Keywords'] = facets['Keywords'] - c['Foods'] = facets['Foods'] - c['Books'] = facets['Books'] - caches['default'].set(SEARCH_CACHE_KEY, c, cache_timeout) - return facets + def get_keywords(self): + if self.Keywords is None: + if self._search_params['search_keywords_or']: + keywords = Keyword.objects.filter(space=self._request.space).distinct() + else: + keywords = Keyword.objects.filter(Q(recipe__in=self._recipe_list) | Q(depth=1)).filter(space=self._request.space).distinct() + + # Subquery that counts recipes for keyword including children + kw_recipe_count = Recipe.objects.filter(**{'keywords__path__startswith': OuterRef('path')}, id__in=self._recipe_list, space=self._request.space + ).values(kw=Substr('keywords__path', 1, Keyword.steplen) + ).annotate(count=Count('pk', distinct=True)).values('count') + + # set keywords to root objects only + keywords = keywords.annotate(count=Coalesce(Subquery(kw_recipe_count), 0) + ).filter(depth=1, count__gt=0 + ).values('id', 'name', 'count', 'numchild').order_by('name') + self.Keywords = list(keywords) + self.set_cache('Keywords', self.Keywords) + return self.Keywords + + def get_foods(self): + if self.Foods is None: + # # if using an OR search, will annotate all keywords, otherwise, just those that appear in results + if self._search_params['search_foods_or']: + foods = Food.objects.filter(space=self._request.space).distinct() + else: + foods = Food.objects.filter(Q(ingredient__step__recipe__in=self._recipe_list) | Q(depth=1)).filter(space=self._request.space).distinct() + + food_recipe_count = Recipe.objects.filter(**{'steps__ingredients__food__path__startswith': OuterRef('path')}, id__in=self._recipe_list, space=self._request.space + ).values(kw=Substr('steps__ingredients__food__path', 1, Food.steplen) + ).annotate(count=Count('pk', distinct=True)).values('count') + + # set keywords to root objects only + foods = foods.annotate(count=Coalesce(Subquery(food_recipe_count), 0) + ).filter(depth=1, count__gt=0 + ).values('id', 'name', 'count', 'numchild' + ).order_by('name') + self.Foods = list(foods) + self.set_cache('Foods', self.Foods) + return self.Foods + + def get_books(self): + if self.Books is None: + self.Books = [] + return self.Books + + def get_ratings(self): + if self.Ratings is None: + if self._queryset is None: + self._queryset = Recipe.objects.filter(id__in=self._recipe_list) + rating_qs = self._queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=self._request.user, then='cooklog__rating'), default=Value(0))))) + self.Ratings = dict(Counter(r.rating for r in rating_qs)) + self.set_cache('Ratings', self.Ratings) + return self.Ratings + + def get_recent(self): + if self.Recent is None: + # TODO make days of recent recipe a setting + recent_recipes = ViewLog.objects.filter(created_by=self._request.user, space=self._request.space, created_at__gte=timezone.now() - timedelta(days=14) + ).values_list('recipe__pk', flat=True) + self.Recent = list(recent_recipes) + self.set_cache('Recent', self.Recent) + return self.Recent -def fill_annotated_parents(annotation, filters): - tree_list = [] - parent = [] - i = 0 - level = -1 - for r in annotation: - expand = False +# # TODO: This might be faster https://github.com/django-treebeard/django-treebeard/issues/115 +# def get_facet(qs=None, request=None, use_cache=True, hash_key=None, food=None, keyword=None): +# """ +# Gets an annotated list from a queryset. +# :param qs: +# recipe queryset to build facets from +# :param request: +# the web request that contains the necessary query parameters +# :param use_cache: +# will find results in cache, if any, and return them or empty list. +# will save the list of recipes IDs in the cache for future processing +# :param hash_key: +# the cache key of the recipe list to process +# only evaluated if the use_cache parameter is false +# :param food: +# return children facets of food +# only evaluated if the use_cache parameter is false +# :param keyword: +# return children facets of keyword +# only evaluated if the use_cache parameter is false +# """ +# facets = {} +# recipe_list = [] +# cache_timeout = 600 - annotation[i][1]['id'] = r[0].id - annotation[i][1]['name'] = r[0].name - annotation[i][1]['count'] = getattr(r[0], 'recipe_count', 0) - annotation[i][1]['isDefaultExpanded'] = False +# # return cached values +# if use_cache: +# qs_hash = hash(frozenset(qs.values_list('pk'))) +# facets['cache_key'] = str(qs_hash) +# SEARCH_CACHE_KEY = f"recipes_filter_{qs_hash}" +# if c := caches['default'].get(SEARCH_CACHE_KEY, None): +# facets['Keywords'] = c['Keywords'] or [] +# facets['Foods'] = c['Foods'] or [] +# facets['Books'] = c['Books'] or [] +# facets['Ratings'] = c['Ratings'] or [] +# facets['Recent'] = c['Recent'] or [] +# else: +# facets['Keywords'] = [] +# facets['Foods'] = [] +# facets['Books'] = [] +# # TODO make ratings a settings user-only vs all-users +# rating_qs = qs.annotate(rating=Round(Avg(Case(When(cooklog__created_by=request.user, then='cooklog__rating'), default=Value(0))))) +# facets['Ratings'] = dict(Counter(r.rating for r in rating_qs)) +# facets['Recent'] = ViewLog.objects.filter( +# created_by=request.user, space=request.space, +# created_at__gte=timezone.now() - timedelta(days=14) # TODO make days of recent recipe a setting +# ).values_list('recipe__pk', flat=True) - if str(r[0].id) in filters: - expand = True - if r[1]['level'] < level: - parent = parent[:r[1]['level'] - level] - parent[-1] = i - level = r[1]['level'] - elif r[1]['level'] > level: - parent.extend([i]) - level = r[1]['level'] - else: - parent[-1] = i - j = 0 +# cached_search = { +# 'recipe_list': list(qs.values_list('id', flat=True)), +# 'keyword_list': request.query_params.getlist('keywords', []), +# 'food_list': request.query_params.getlist('foods', []), +# 'book_list': request.query_params.getlist('book', []), +# 'search_keywords_or': str2bool(request.query_params.get('keywords_or', True)), +# 'search_foods_or': str2bool(request.query_params.get('foods_or', True)), +# 'search_books_or': str2bool(request.query_params.get('books_or', True)), +# 'space': request.space, +# 'Ratings': facets['Ratings'], +# 'Recent': facets['Recent'], +# 'Keywords': facets['Keywords'], +# 'Foods': facets['Foods'], +# 'Books': facets['Books'] +# } +# caches['default'].set(SEARCH_CACHE_KEY, cached_search, cache_timeout) +# return facets - while j < level: - # this causes some double counting when a recipe has both a child and an ancestor - annotation[parent[j]][1]['count'] += getattr(r[0], 'recipe_count', 0) - if expand: - annotation[parent[j]][1]['isDefaultExpanded'] = True - j += 1 - if level == 0: - tree_list.append(annotation[i][1]) - elif level > 0: - annotation[parent[level - 1]][1].setdefault('children', []).append(annotation[i][1]) - i += 1 - return tree_list +# # construct and cache new values by retrieving search parameters from the cache +# SEARCH_CACHE_KEY = f'recipes_filter_{hash_key}' +# if c := caches['default'].get(SEARCH_CACHE_KEY, None): +# recipe_list = c['recipe_list'] +# keyword_list = c['keyword_list'] +# food_list = c['food_list'] +# book_list = c['book_list'] +# search_keywords_or = c['search_keywords_or'] +# search_foods_or = c['search_foods_or'] +# search_books_or = c['search_books_or'] +# else: +# return {} + +# # if using an OR search, will annotate all keywords, otherwise, just those that appear in results +# if search_keywords_or: +# keywords = Keyword.objects.filter(space=request.space).distinct() +# else: +# keywords = Keyword.objects.filter(Q(recipe__in=recipe_list) | Q(depth=1)).filter(space=request.space).distinct() + +# # Subquery that counts recipes for keyword including children +# kw_recipe_count = Recipe.objects.filter(**{'keywords__path__startswith': OuterRef('path')}, id__in=recipe_list, space=request.space +# ).values(kw=Substr('keywords__path', 1, Keyword.steplen) +# ).annotate(count=Count('pk', distinct=True)).values('count') + +# # set keywords to root objects only +# keywords = keywords.annotate(count=Coalesce(Subquery(kw_recipe_count), 0) +# ).filter(depth=1, count__gt=0 +# ).values('id', 'name', 'count', 'numchild' +# ).order_by('name') +# if keyword: +# facets['Keywords'] = list(keywords) +# return facets + +# # custom django-tree function annotates a queryset to make building a tree easier. +# # see https://django-treebeard.readthedocs.io/en/latest/api.html#treebeard.models.Node.get_annotated_list_qs for details +# # kw_a = annotated_qs(keywords, root=True, fill=True) + +# # # if using an OR search, will annotate all keywords, otherwise, just those that appear in results +# if search_foods_or: +# foods = Food.objects.filter(space=request.space).distinct() +# else: +# foods = Food.objects.filter(Q(ingredient__step__recipe__in=recipe_list) | Q(depth=1)).filter(space=request.space).distinct() + +# food_recipe_count = Recipe.objects.filter(**{'steps__ingredients__food__path__startswith': OuterRef('path')}, id__in=recipe_list, space=request.space +# ).values(kw=Substr('steps__ingredients__food__path', 1, Food.steplen * (1+getattr(food, 'depth', 0))) +# ).annotate(count=Count('pk', distinct=True)).values('count') + +# # set keywords to root objects only +# foods = foods.annotate(count=Coalesce(Subquery(food_recipe_count), 0) +# ).filter(depth=(1+getattr(food, 'depth', 0)), count__gt=0 +# ).values('id', 'name', 'count', 'numchild' +# ).order_by('name') +# if food: +# facets['Foods'] = list(foods) +# return facets + +# # food_a = annotated_qs(foods, root=True, fill=True) + +# # c['Keywords'] = facets['Keywords'] = fill_annotated_parents(kw_a, keyword_list) +# c['Keywords'] = facets['Keywords'] = list(keywords) +# # c['Foods'] = facets['Foods'] = fill_annotated_parents(food_a, food_list) +# c['Foods'] = facets['Foods'] = list(foods) +# # TODO add book facet +# c['Books'] = facets['Books'] = [] +# caches['default'].set(SEARCH_CACHE_KEY, c, cache_timeout) +# return facets -def annotated_qs(qs, root=False, fill=False): - """ - Gets an annotated list from a queryset. - :param root: +# def fill_annotated_parents(annotation, filters): +# tree_list = [] +# parent = [] +# i = 0 +# level = -1 +# for r in annotation: +# expand = False - Will backfill in annotation to include all parents to root node. +# annotation[i][1]['id'] = r[0].id +# annotation[i][1]['name'] = r[0].name +# annotation[i][1]['count'] = getattr(r[0], 'recipe_count', 0) +# annotation[i][1]['isDefaultExpanded'] = False - :param fill: - Will fill in gaps in annotation where nodes between children - and ancestors are not included in the queryset. - """ +# if str(r[0].id) in filters: +# expand = True +# if r[1]['level'] < level: +# parent = parent[:r[1]['level'] - level] +# parent[-1] = i +# level = r[1]['level'] +# elif r[1]['level'] > level: +# parent.extend([i]) +# level = r[1]['level'] +# else: +# parent[-1] = i +# j = 0 - result, info = [], {} - start_depth, prev_depth = (None, None) - nodes_list = list(qs.values_list('pk', flat=True)) - for node in qs.order_by('path'): - node_queue = [node] - while len(node_queue) > 0: - dirty = False - current_node = node_queue[-1] - depth = current_node.get_depth() - parent_id = current_node.parent - if root and depth > 1 and parent_id not in nodes_list: - parent_id = current_node.parent - nodes_list.append(parent_id) - node_queue.append(current_node.__class__.objects.get(pk=parent_id)) - dirty = True +# while j < level: +# # this causes some double counting when a recipe has both a child and an ancestor +# annotation[parent[j]][1]['count'] += getattr(r[0], 'recipe_count', 0) +# if expand: +# annotation[parent[j]][1]['isDefaultExpanded'] = True +# j += 1 +# if level == 0: +# tree_list.append(annotation[i][1]) +# elif level > 0: +# annotation[parent[level - 1]][1].setdefault('children', []).append(annotation[i][1]) +# i += 1 +# return tree_list - if fill and depth > 1 and prev_depth and depth > prev_depth and parent_id not in nodes_list: - nodes_list.append(parent_id) - node_queue.append(current_node.__class__.objects.get(pk=parent_id)) - dirty = True - if not dirty: - working_node = node_queue.pop() - if start_depth is None: - start_depth = depth - open = (depth and (prev_depth is None or depth > prev_depth)) - if prev_depth is not None and depth < prev_depth: - info['close'] = list(range(0, prev_depth - depth)) - info = {'open': open, 'close': [], 'level': depth - start_depth} - result.append((working_node, info,)) - prev_depth = depth - if start_depth and start_depth > 0: - info['close'] = list(range(0, prev_depth - start_depth + 1)) - return result +# def annotated_qs(qs, root=False, fill=False): +# """ +# Gets an annotated list from a queryset. +# :param root: + +# Will backfill in annotation to include all parents to root node. + +# :param fill: +# Will fill in gaps in annotation where nodes between children +# and ancestors are not included in the queryset. +# """ + +# result, info = [], {} +# start_depth, prev_depth = (None, None) +# nodes_list = list(qs.values_list('pk', flat=True)) +# for node in qs.order_by('path'): +# node_queue = [node] +# while len(node_queue) > 0: +# dirty = False +# current_node = node_queue[-1] +# depth = current_node.get_depth() +# parent_id = current_node.parent +# if root and depth > 1 and parent_id not in nodes_list: +# parent_id = current_node.parent +# nodes_list.append(parent_id) +# node_queue.append(current_node.__class__.objects.get(pk=parent_id)) +# dirty = True + +# if fill and depth > 1 and prev_depth and depth > prev_depth and parent_id not in nodes_list: +# nodes_list.append(parent_id) +# node_queue.append(current_node.__class__.objects.get(pk=parent_id)) +# dirty = True + +# if not dirty: +# working_node = node_queue.pop() +# if start_depth is None: +# start_depth = depth +# open = (depth and (prev_depth is None or depth > prev_depth)) +# if prev_depth is not None and depth < prev_depth: +# info['close'] = list(range(0, prev_depth - depth)) +# info = {'open': open, 'close': [], 'level': depth - start_depth} +# result.append((working_node, info,)) +# prev_depth = depth +# if start_depth and start_depth > 0: +# info['close'] = list(range(0, prev_depth - start_depth + 1)) +# return result def old_search(request): diff --git a/cookbook/views/api.py b/cookbook/views/api.py index 1135cfec8..e9ba46eed 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -38,7 +38,7 @@ from cookbook.helper.permission_helper import (CustomIsAdmin, CustomIsGuest, Cus CustomIsShare, CustomIsShared, CustomIsUser, group_required) from cookbook.helper.recipe_html_import import get_recipe_from_source -from cookbook.helper.recipe_search import get_facet, old_search, search_recipes +from cookbook.helper.recipe_search import RecipeFacet, old_search, search_recipes from cookbook.helper.recipe_url_import import get_from_scraper from cookbook.helper.shopping_helper import list_from_recipe, shopping_helper from cookbook.models import (Automation, BookmarkletImport, CookLog, Food, FoodInheritField, @@ -604,7 +604,7 @@ class RecipePagination(PageNumberPagination): max_page_size = 100 def paginate_queryset(self, queryset, request, view=None): - self.facets = get_facet(qs=queryset, request=request) + self.facets = RecipeFacet(request, queryset=queryset) return super().paginate_queryset(queryset, request, view) def get_paginated_response(self, data): @@ -613,7 +613,7 @@ class RecipePagination(PageNumberPagination): ('next', self.get_next_link()), ('previous', self.get_previous_link()), ('results', data), - ('facets', self.facets) + ('facets', self.facets.get_facets()) ])) @@ -651,8 +651,7 @@ class RecipeViewSet(viewsets.ModelViewSet): self.queryset = self.queryset.filter(space=self.request.space) self.queryset = search_recipes(self.request, self.queryset, self.request.GET) - - return super().get_queryset() + return super().get_queryset().prefetch_related('cooklog_set') def list(self, request, *args, **kwargs): if self.request.GET.get('debug', False): @@ -1132,10 +1131,13 @@ def ingredient_from_string(request): @group_required('user') def get_facets(request): key = request.GET.get('hash', None) + food = request.GET.get('food', None) + keyword = request.GET.get('keyword', None) + facets = RecipeFacet(request, hash_key=key) return JsonResponse( { - 'facets': get_facet(request=request, use_cache=False, hash_key=key), + 'facets': facets.get_facets(), }, status=200 ) diff --git a/recipes/middleware.py b/recipes/middleware.py index 09346e6a8..8f608794e 100644 --- a/recipes/middleware.py +++ b/recipes/middleware.py @@ -1,3 +1,4 @@ +import time from os import getenv from django.conf import settings diff --git a/vue/src/apps/RecipeSearchView/RecipeSearchView.vue b/vue/src/apps/RecipeSearchView/RecipeSearchView.vue index a76c7f051..b6c717897 100644 --- a/vue/src/apps/RecipeSearchView/RecipeSearchView.vue +++ b/vue/src/apps/RecipeSearchView/RecipeSearchView.vue @@ -99,7 +99,7 @@ :options="facets.Keywords" :flat="true" searchNested - multiple + :multiple="true" :placeholder="$t('Keywords')" :normalizer="normalizer" @input="refreshData(false)" @@ -123,10 +123,11 @@ { + if (x?.numchild > 0) { + return { ...x, children: null } + } else { + return x + } + }) + }, ratingOptions: function () { return [ { id: 5, label: "⭐⭐⭐⭐⭐" + " (" + (this.facets.Ratings?.["5.0"] ?? 0) + ")" }, @@ -403,6 +414,7 @@ export default { this.pagination_count = result.data.count this.facets = result.data.facets + console.log(this.facets) if (this.facets?.cache_key) { this.getFacets(this.facets.cache_key) } @@ -480,7 +492,7 @@ export default { } }, getFacets: function (hash) { - this.genericGetAPI("api_get_facets", { hash: hash }).then((response) => { + return this.genericGetAPI("api_get_facets", { hash: hash }).then((response) => { this.facets = { ...this.facets, ...response.data.facets } }) }, @@ -512,6 +524,35 @@ export default { console.log(result.data) }) }, + loadFoodChildren({ action, parentNode, callback }) { + // Typically, do the AJAX stuff here. + // Once the server has responded, + // assign children options to the parent node & call the callback. + + if (action === LOAD_CHILDREN_OPTIONS) { + switch (parentNode.id) { + case "success": { + console.log(parentNode) + break + } + // case "no-children": { + // simulateAsyncOperation(() => { + // parentNode.children = [] + // callback() + // }) + // break + // } + // case "failure": { + // simulateAsyncOperation(() => { + // callback(new Error("Failed to load options: network error.")) + // }) + // break + // } + default: /* empty */ + } + } + callback() + }, }, } From 22953b0591f8652938f993c7ee766738aadae809 Mon Sep 17 00:00:00 2001 From: smilerz Date: Wed, 12 Jan 2022 16:21:36 -0600 Subject: [PATCH 4/5] trees in recipe search loaded asynchronously --- cookbook/helper/recipe_search.py | 99 +++++++++++++------ cookbook/views/api.py | 9 +- .../RecipeSearchView/RecipeSearchView.vue | 71 ++++++------- vue/src/locales/en.json | 2 +- 4 files changed, 108 insertions(+), 73 deletions(-) diff --git a/cookbook/helper/recipe_search.py b/cookbook/helper/recipe_search.py index 608fd8a2e..13c06fe5c 100644 --- a/cookbook/helper/recipe_search.py +++ b/cookbook/helper/recipe_search.py @@ -201,12 +201,11 @@ def search_recipes(request, queryset, params): return queryset -class CacheEmpty(Exception): - pass - - class RecipeFacet(): - def __init__(self, request, queryset=None, hash_key=None, cache_timeout=600): + class CacheEmpty(Exception): + pass + + def __init__(self, request, queryset=None, hash_key=None, cache_timeout=3600): if hash_key is None and queryset is None: raise ValueError(_("One of queryset or hash_key must be provided")) @@ -215,15 +214,15 @@ class RecipeFacet(): self.hash_key = hash_key or str(hash(frozenset(self._queryset.values_list('pk')))) self._SEARCH_CACHE_KEY = f"recipes_filter_{self.hash_key}" self._cache_timeout = cache_timeout - self._cache = caches['default'].get(self._SEARCH_CACHE_KEY, None) + self._cache = caches['default'].get(self._SEARCH_CACHE_KEY, {}) if self._cache is None and self._queryset is None: - raise CacheEmpty("No queryset provided and cache empty") + raise self.CacheEmpty("No queryset provided and cache empty") - self.Keywords = getattr(self._cache, 'Keywords', None) - self.Foods = getattr(self._cache, 'Foods', None) - self.Books = getattr(self._cache, 'Books', None) - self.Ratings = getattr(self._cache, 'Ratings', None) - self.Recent = getattr(self._cache, 'Recent', None) + self.Keywords = self._cache.get('Keywords', None) + self.Foods = self._cache.get('Foods', None) + self.Books = self._cache.get('Books', None) + self.Ratings = self._cache.get('Ratings', None) + self.Recent = self._cache.get('Recent', None) if self._queryset: self._recipe_list = list(self._queryset.values_list('id', flat=True)) @@ -292,16 +291,9 @@ class RecipeFacet(): else: keywords = Keyword.objects.filter(Q(recipe__in=self._recipe_list) | Q(depth=1)).filter(space=self._request.space).distinct() - # Subquery that counts recipes for keyword including children - kw_recipe_count = Recipe.objects.filter(**{'keywords__path__startswith': OuterRef('path')}, id__in=self._recipe_list, space=self._request.space - ).values(kw=Substr('keywords__path', 1, Keyword.steplen) - ).annotate(count=Count('pk', distinct=True)).values('count') - # set keywords to root objects only - keywords = keywords.annotate(count=Coalesce(Subquery(kw_recipe_count), 0) - ).filter(depth=1, count__gt=0 - ).values('id', 'name', 'count', 'numchild').order_by('name') - self.Keywords = list(keywords) + keywords = self._keyword_queryset(keywords) + self.Keywords = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(keywords)] self.set_cache('Keywords', self.Keywords) return self.Keywords @@ -313,16 +305,10 @@ class RecipeFacet(): else: foods = Food.objects.filter(Q(ingredient__step__recipe__in=self._recipe_list) | Q(depth=1)).filter(space=self._request.space).distinct() - food_recipe_count = Recipe.objects.filter(**{'steps__ingredients__food__path__startswith': OuterRef('path')}, id__in=self._recipe_list, space=self._request.space - ).values(kw=Substr('steps__ingredients__food__path', 1, Food.steplen) - ).annotate(count=Count('pk', distinct=True)).values('count') - # set keywords to root objects only - foods = foods.annotate(count=Coalesce(Subquery(food_recipe_count), 0) - ).filter(depth=1, count__gt=0 - ).values('id', 'name', 'count', 'numchild' - ).order_by('name') - self.Foods = list(foods) + foods = self._food_queryset(foods) + + self.Foods = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(foods)] self.set_cache('Foods', self.Foods) return self.Foods @@ -349,6 +335,59 @@ class RecipeFacet(): self.set_cache('Recent', self.Recent) return self.Recent + def add_food_children(self, id): + try: + food = Food.objects.get(id=id) + nodes = food.get_ancestors() + except Food.DoesNotExist: + return self.get_facets() + foods = self._food_queryset(Food.objects.filter(path__startswith=food.path, depth=food.depth+1), food) + deep_search = self.Foods + for node in nodes: + index = next((i for i, x in enumerate(deep_search) if x["id"] == node.id), None) + deep_search = deep_search[index]['children'] + index = next((i for i, x in enumerate(deep_search) if x["id"] == food.id), None) + deep_search[index]['children'] = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(foods)] + self.set_cache('Foods', self.Foods) + return self.get_facets() + + def add_keyword_children(self, id): + try: + keyword = Keyword.objects.get(id=id) + nodes = keyword.get_ancestors() + except Keyword.DoesNotExist: + return self.get_facets() + keywords = self._keyword_queryset(Keyword.objects.filter(path__startswith=keyword.path, depth=keyword.depth+1), keyword) + deep_search = self.Keywords + for node in nodes: + index = next((i for i, x in enumerate(deep_search) if x["id"] == node.id), None) + deep_search = deep_search[index]['children'] + index = next((i for i, x in enumerate(deep_search) if x["id"] == keyword.id), None) + deep_search[index]['children'] = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(keywords)] + self.set_cache('Keywords', self.Keywords) + return self.get_facets() + + def _recipe_count_queryset(self, field, depth=1, steplen=4): + return Recipe.objects.filter(**{f'{field}__path__startswith': OuterRef('path')}, id__in=self._recipe_list, space=self._request.space + ).values(child=Substr(f'{field}__path', 1, steplen) + ).annotate(count=Count('pk', distinct=True)).values('count') + + def _keyword_queryset(self, queryset, keyword=None): + depth = getattr(keyword, 'depth', 0) + 1 + steplen = depth * Keyword.steplen + + return queryset.annotate(count=Coalesce(Subquery(self._recipe_count_queryset('keywords', depth, steplen)), 0) + ).filter(depth=depth, count__gt=0 + ).values('id', 'name', 'count', 'numchild').order_by('name') + + def _food_queryset(self, queryset, food=None): + depth = getattr(food, 'depth', 0) + 1 + steplen = depth * Food.steplen + + return queryset.annotate(count=Coalesce(Subquery(self._recipe_count_queryset('steps__ingredients__food', depth, steplen)), 0) + ).filter(depth__lte=depth, count__gt=0 + ).values('id', 'name', 'count', 'numchild').order_by('name') + # # TODO: This might be faster https://github.com/django-treebeard/django-treebeard/issues/115 # def get_facet(qs=None, request=None, use_cache=True, hash_key=None, food=None, keyword=None): diff --git a/cookbook/views/api.py b/cookbook/views/api.py index e9ba46eed..333e0c8bd 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -1135,9 +1135,16 @@ def get_facets(request): keyword = request.GET.get('keyword', None) facets = RecipeFacet(request, hash_key=key) + if food: + results = facets.add_food_children(food) + elif keyword: + results = facets.add_keyword_children(keyword) + else: + results = facets.get_facets() + return JsonResponse( { - 'facets': facets.get_facets(), + 'facets': results, }, status=200 ) diff --git a/vue/src/apps/RecipeSearchView/RecipeSearchView.vue b/vue/src/apps/RecipeSearchView/RecipeSearchView.vue index b6c717897..0e8d25a05 100644 --- a/vue/src/apps/RecipeSearchView/RecipeSearchView.vue +++ b/vue/src/apps/RecipeSearchView/RecipeSearchView.vue @@ -80,7 +80,7 @@
- {{ $t("Advanced Search Settings") }} + {{ $t("Search Settings") }}
@@ -97,6 +97,7 @@ { - if (x?.numchild > 0) { - return { ...x, children: null } - } else { - return x - } - }) - }, ratingOptions: function () { return [ { id: 5, label: "⭐⭐⭐⭐⭐" + " (" + (this.facets.Ratings?.["5.0"] ?? 0) + ")" }, @@ -414,10 +405,9 @@ export default { this.pagination_count = result.data.count this.facets = result.data.facets - console.log(this.facets) - if (this.facets?.cache_key) { - this.getFacets(this.facets.cache_key) - } + // if (this.facets?.cache_key) { + // this.getFacets(this.facets.cache_key) + // } this.recipes = this.removeDuplicates(result.data.results, (recipe) => recipe.id) if (!this.searchFiltered) { // if meal plans are being shown - filter out any meal plan recipes from the recipe list @@ -491,8 +481,12 @@ export default { return [undefined, undefined] } }, - getFacets: function (hash) { - return this.genericGetAPI("api_get_facets", { hash: hash }).then((response) => { + getFacets: function (hash, facet, id) { + let params = { hash: hash } + if (facet) { + params[facet] = id + } + return this.genericGetAPI("api_get_facets", params).then((response) => { this.facets = { ...this.facets, ...response.data.facets } }) }, @@ -520,9 +514,7 @@ export default { } else { params.options = { query: { debug: true } } } - this.genericAPI(this.Models.RECIPE, this.Actions.LIST, params).then((result) => { - console.log(result.data) - }) + this.genericAPI(this.Models.RECIPE, this.Actions.LIST, params).then((result) => {}) }, loadFoodChildren({ action, parentNode, callback }) { // Typically, do the AJAX stuff here. @@ -530,28 +522,25 @@ export default { // assign children options to the parent node & call the callback. if (action === LOAD_CHILDREN_OPTIONS) { - switch (parentNode.id) { - case "success": { - console.log(parentNode) - break - } - // case "no-children": { - // simulateAsyncOperation(() => { - // parentNode.children = [] - // callback() - // }) - // break - // } - // case "failure": { - // simulateAsyncOperation(() => { - // callback(new Error("Failed to load options: network error.")) - // }) - // break - // } - default: /* empty */ + if (this.facets?.cache_key) { + this.getFacets(this.facets.cache_key, "food", parentNode.id).then(callback()) } + } else { + callback() + } + }, + loadKeywordChildren({ action, parentNode, callback }) { + // Typically, do the AJAX stuff here. + // Once the server has responded, + // assign children options to the parent node & call the callback. + + if (action === LOAD_CHILDREN_OPTIONS) { + if (this.facets?.cache_key) { + this.getFacets(this.facets.cache_key, "keyword", parentNode.id).then(callback()) + } + } else { + callback() } - callback() }, }, } diff --git a/vue/src/locales/en.json b/vue/src/locales/en.json index fcf49e540..c404ed99a 100644 --- a/vue/src/locales/en.json +++ b/vue/src/locales/en.json @@ -116,7 +116,7 @@ "Information": "Information", "Download": "Download", "Create": "Create", - "Advanced Search Settings": "Advanced Search Settings", + "Search Settings": "Search Settings", "View": "View", "Recipes": "Recipes", "Move": "Move", From 798aa7f179370a4dbb6af37b22d41370ae477b0e Mon Sep 17 00:00:00 2001 From: smilerz Date: Wed, 12 Jan 2022 16:55:39 -0600 Subject: [PATCH 5/5] detect empty queryset --- cookbook/helper/recipe_search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cookbook/helper/recipe_search.py b/cookbook/helper/recipe_search.py index 13c06fe5c..e026a0b14 100644 --- a/cookbook/helper/recipe_search.py +++ b/cookbook/helper/recipe_search.py @@ -224,7 +224,7 @@ class RecipeFacet(): self.Ratings = self._cache.get('Ratings', None) self.Recent = self._cache.get('Recent', None) - if self._queryset: + if self._queryset is not None: self._recipe_list = list(self._queryset.values_list('id', flat=True)) self._search_params = { 'keyword_list': self._request.query_params.getlist('keywords', []), @@ -235,7 +235,7 @@ class RecipeFacet(): 'search_books_or': str2bool(self._request.query_params.get('books_or', True)), 'space': self._request.space, } - elif self.hash_key: + elif self.hash_key is not None: self._recipe_list = self._cache.get('recipe_list', None) self._search_params = { 'keyword_list': self._cache.get('keyword_list', None),