diff --git a/cookbook/helper/recipe_search.py b/cookbook/helper/recipe_search.py index 110ac2c43..0cfbcf6d1 100644 --- a/cookbook/helper/recipe_search.py +++ b/cookbook/helper/recipe_search.py @@ -1,8 +1,8 @@ import json from collections import Counter -from datetime import timedelta +from datetime import date, timedelta -from django.contrib.postgres.search import SearchQuery, SearchRank, TrigramSimilarity +from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector, TrigramSimilarity from django.core.cache import caches from django.db.models import Avg, Case, Count, F, Func, Max, OuterRef, Q, Subquery, Sum, Value, When from django.db.models.functions import Coalesce, Lower, Substr @@ -13,7 +13,8 @@ from cookbook.filters import RecipeFilter from cookbook.helper.HelperFunctions import Round, str2bool from cookbook.helper.permission_helper import has_group_permission from cookbook.managers import DICTIONARY -from cookbook.models import CookLog, CustomFilter, Food, Keyword, Recipe, SearchPreference, ViewLog +from cookbook.models import (CookLog, CustomFilter, Food, Keyword, Recipe, SearchFields, + SearchPreference, ViewLog) from recipes import settings @@ -66,10 +67,12 @@ class RecipeSearch(): self._internal = str2bool(self._params.get('internal', None)) self._random = str2bool(self._params.get('random', False)) self._new = str2bool(self._params.get('new', False)) - self._last_viewed = int(self._params.get('last_viewed', 0)) + self._num_recent = int(self._params.get('num_recent', 0)) self._include_children = str2bool(self._params.get('include_children', None)) self._timescooked = self._params.get('timescooked', None) - self._lastcooked = self._params.get('lastcooked', None) + self._cookedon = self._params.get('cookedon', None) + self._createdon = self._params.get('createdon', None) + self._viewedon = self._params.get('viewedon', None) # this supports hidden feature to find recipes missing X ingredients try: self._makenow = int(makenow := self._params.get('makenow', None)) @@ -81,16 +84,16 @@ class RecipeSearch(): self._search_type = self._search_prefs.search or 'plain' if self._string: - unaccent_include = self._search_prefs.unaccent.values_list('field', flat=True) - self._icontains_include = [x + '__unaccent' if x in unaccent_include else x for x in self._search_prefs.icontains.values_list('field', flat=True)] - self._istartswith_include = [x + '__unaccent' if x in unaccent_include else x for x in self._search_prefs.istartswith.values_list('field', flat=True)] + self._unaccent_include = self._search_prefs.unaccent.values_list('field', flat=True) + self._icontains_include = [x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.icontains.values_list('field', flat=True)] + self._istartswith_include = [x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.istartswith.values_list('field', flat=True)] self._trigram_include = None self._fulltext_include = None self._trigram = False if self._postgres and self._string: self._language = DICTIONARY.get(translation.get_language(), 'simple') - self._trigram_include = [x + '__unaccent' if x in unaccent_include else x for x in self._search_prefs.trigram.values_list('field', flat=True)] - self._fulltext_include = self._search_prefs.fulltext.values_list('field', flat=True) + self._trigram_include = [x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.trigram.values_list('field', flat=True)] + self._fulltext_include = self._search_prefs.fulltext.values_list('field', flat=True) or None if self._search_type not in ['websearch', 'raw'] and self._trigram_include: self._trigram = True @@ -99,11 +102,7 @@ class RecipeSearch(): search_type=self._search_type, config=self._language, ) - self.search_rank = ( - SearchRank('name_search_vector', self.search_query, cover_density=True) - + SearchRank('desc_search_vector', self.search_query, cover_density=True) - # + SearchRank('steps__search_vector', self.search_query, cover_density=True) # Is a large performance drag - ) + self.search_rank = None self.orderby = [] self._default_sort = ['-favorite'] # TODO add user setting self._filters = None @@ -112,8 +111,10 @@ class RecipeSearch(): def get_queryset(self, queryset): self._queryset = queryset self._build_sort_order() - self._recently_viewed(num_recent=self._last_viewed) - self._last_cooked(lastcooked=self._lastcooked) + self._recently_viewed(num_recent=self._num_recent) + self._cooked_on_filter(cooked_date=self._cookedon) + self._created_on_filter(created_date=self._createdon) + self._viewed_on_filter(viewed_date=self._viewedon) self._favorite_recipes(timescooked=self._timescooked) self._new_recipes() self.keyword_filters(**self._keywords) @@ -144,7 +145,7 @@ class RecipeSearch(): # TODO add userpreference for default sort order and replace '-favorite' default_order = ['-favorite'] # recent and new_recipe are always first; they float a few recipes to the top - if self._last_viewed: + if self._num_recent: order += ['-recent'] if self._new: order += ['-new_recipe'] @@ -162,7 +163,7 @@ class RecipeSearch(): if '-score' in order: order.remove('-score') # if no sort order provided prioritize text search, followed by the default search - elif self._postgres and self._string: + elif self._postgres and self._string and (self._trigram or self._fulltext_include): order += ['-score', *default_order] # otherwise sort by the remaining order_by attributes or favorite by default else: @@ -180,13 +181,11 @@ class RecipeSearch(): self.build_fulltext_filters(self._string) self.build_trigram(self._string) + query_filter = Q() if self._filters: - query_filter = None for f in self._filters: - if query_filter: - query_filter |= f - else: - query_filter = f + query_filter |= f + self._queryset = self._queryset.filter(query_filter).distinct() # this creates duplicate records which can screw up other aggregates, see makenow for workaround if self._fulltext_include: if self._fuzzy_match is None: @@ -197,27 +196,56 @@ class RecipeSearch(): if self._fuzzy_match is not None: simularity = self._fuzzy_match.filter(pk=OuterRef('pk')).values('simularity') if not self._fulltext_include: - self._queryset = self._queryset.annotate(score=Coalesce(Subquery(Max(simularity)), 0.0)) + self._queryset = self._queryset.annotate(score=Coalesce(Subquery(simularity), 0.0)) else: self._queryset = self._queryset.annotate(simularity=Coalesce(Subquery(simularity), 0.0)) if self._sort_includes('score') and self._fulltext_include and self._fuzzy_match is not None: self._queryset = self._queryset.annotate(score=Sum(F('rank')+F('simularity'))) else: - self._queryset = self._queryset.filter(name__icontains=self._string) + query_filter = Q() + for f in [x + '__unaccent__iexact' if x in self._unaccent_include else x + '__iexact' for x in SearchFields.objects.all().values_list('field', flat=True)]: + query_filter |= Q(**{"%s" % f: self._string}) + self._queryset = self._queryset.filter(query_filter).distinct() - def _last_cooked(self, lastcooked=None): - if self._sort_includes('lastcooked') or lastcooked: + def _cooked_on_filter(self, cooked_date=None): + if self._sort_includes('lastcooked') or cooked_date: longTimeAgo = timezone.now() - timedelta(days=100000) self._queryset = self._queryset.annotate(lastcooked=Coalesce( - Max(Case(When(created_by=self._request.user, space=self._request.space, then='cooklog__created_at'))), Value(longTimeAgo))) - if lastcooked is None: + Max(Case(When(cooklog__created_by=self._request.user, cooklog__space=self._request.space, then='cooklog__created_at'))), Value(longTimeAgo))) + if cooked_date is None: return - lessthan = '-' in lastcooked[:1] + lessthan = '-' in cooked_date[:1] + cooked_date = date(*[int(x) for x in cooked_date.split('-') if x != '']) if lessthan: - self._queryset = self._queryset.filter(lastcooked__lte=lastcooked[1:]).exclude(lastcooked=longTimeAgo) + self._queryset = self._queryset.filter(lastcooked__date__lte=cooked_date).exclude(lastcooked=longTimeAgo) else: - self._queryset = self._queryset.filter(lastcooked__gte=lastcooked).exclude(lastcooked=longTimeAgo) + self._queryset = self._queryset.filter(lastcooked__date__gte=cooked_date).exclude(lastcooked=longTimeAgo) + + def _created_on_filter(self, created_date=None): + if created_date is None: + return + lessthan = '-' in created_date[:1] + created_date = date(*[int(x) for x in created_date.split('-') if x != '']) + if lessthan: + self._queryset = self._queryset.filter(created_at__date__lte=created_date) + else: + self._queryset = self._queryset.filter(created_at__date__gte=created_date) + + def _viewed_on_filter(self, viewed_date=None): + if self._sort_includes('lastviewed') or viewed_date: + longTimeAgo = timezone.now() - timedelta(days=100000) + self._queryset = self._queryset.annotate(lastviewed=Coalesce( + Max(Case(When(viewlog__created_by=self._request.user, viewlog__space=self._request.space, then='viewlog__created_at'))), Value(longTimeAgo))) + if viewed_date is None: + return + lessthan = '-' in viewed_date[:1] + viewed_date = date(*[int(x) for x in viewed_date.split('-') if x != '']) + + if lessthan: + self._queryset = self._queryset.filter(lastviewed__date__lte=viewed_date).exclude(lastviewed=longTimeAgo) + else: + self._queryset = self._queryset.filter(lastviewed__date__gte=viewed_date).exclude(lastviewed=longTimeAgo) def _new_recipes(self, new_days=7): # TODO make new days a user-setting @@ -232,12 +260,12 @@ class RecipeSearch(): if not num_recent: if self._sort_includes('lastviewed'): self._queryset = self._queryset.annotate(lastviewed=Coalesce( - Max(Case(When(created_by=self._request.user, space=self._request.space, then='viewlog__pk'))), Value(0))) + Max(Case(When(viewlog__created_by=self._request.user, viewlog__space=self._request.space, then='viewlog__pk'))), Value(0))) return - last_viewed_recipes = ViewLog.objects.filter(created_by=self._request.user, space=self._request.space).values( + num_recent_recipes = ViewLog.objects.filter(created_by=self._request.user, space=self._request.space).values( 'recipe').annotate(recent=Max('created_at')).order_by('-recent')[:num_recent] - self._queryset = self._queryset.annotate(recent=Coalesce(Max(Case(When(pk__in=last_viewed_recipes.values('recipe'), then='viewlog__pk'))), Value(0))) + self._queryset = self._queryset.annotate(recent=Coalesce(Max(Case(When(pk__in=num_recent_recipes.values('recipe'), then='viewlog__pk'))), Value(0))) def _favorite_recipes(self, timescooked=None): if self._sort_includes('favorite') or timescooked: @@ -388,18 +416,32 @@ class RecipeSearch(): if not string: return if self._fulltext_include: - if not self._filters: - self._filters = [] + vectors = [] + rank = [] if 'name' in self._fulltext_include: - self._filters += [Q(name_search_vector=self.search_query)] + vectors.append('name_search_vector') + rank.append(SearchRank('name_search_vector', self.search_query, cover_density=True)) if 'description' in self._fulltext_include: - self._filters += [Q(desc_search_vector=self.search_query)] + vectors.append('desc_search_vector') + rank.append(SearchRank('desc_search_vector', self.search_query, cover_density=True)) if 'steps__instruction' in self._fulltext_include: - self._filters += [Q(steps__search_vector=self.search_query)] + vectors.append('steps__search_vector') + rank.append(SearchRank('steps__search_vector', self.search_query, cover_density=True)) if 'keywords__name' in self._fulltext_include: - self._filters += [Q(keywords__in=Keyword.objects.filter(name__search=self.search_query))] + # explicitly settings unaccent on keywords and foods so that they behave the same as search_vector fields + vectors.append('keywords__name__unaccent') + rank.append(SearchRank('keywords__name__unaccent', self.search_query, cover_density=True)) if 'steps__ingredients__food__name' in self._fulltext_include: - self._filters += [Q(steps__ingredients__food__in=Food.objects.filter(name__search=self.search_query))] + vectors.append('steps__ingredients__food__name__unaccent') + rank.append(SearchRank('steps__ingredients__food__name', self.search_query, cover_density=True)) + + for r in rank: + if self.search_rank is None: + self.search_rank = r + else: + self.search_rank += r + # modifying queryset will annotation creates duplicate results + self._filters.append(Q(id__in=Recipe.objects.annotate(vector=SearchVector(*vectors)).filter(Q(vector=self.search_query)))) def build_text_filters(self, string=None): if not string: @@ -653,7 +695,7 @@ class RecipeFacet(): def _recipe_count_queryset(self, field, depth=1, steplen=4): return Recipe.objects.filter(**{f'{field}__path__startswith': OuterRef('path')}, id__in=self._recipe_list, space=self._request.space - ).values(child=Substr(f'{field}__path', 1, steplen) + ).values(child=Substr(f'{field}__path', 1, steplen*depth) ).annotate(count=Count('pk', distinct=True)).values('count') def _keyword_queryset(self, queryset, keyword=None): diff --git a/cookbook/models.py b/cookbook/models.py index b704096bd..2cf87307c 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -1106,7 +1106,7 @@ class SearchPreference(models.Model, PermissionModelMixin): istartswith = models.ManyToManyField(SearchFields, related_name="istartswith_fields", blank=True) trigram = models.ManyToManyField(SearchFields, related_name="trigram_fields", blank=True, default=nameSearchField) fulltext = models.ManyToManyField(SearchFields, related_name="fulltext_fields", blank=True) - trigram_threshold = models.DecimalField(default=0.1, decimal_places=2, max_digits=3) + trigram_threshold = models.DecimalField(default=0.2, decimal_places=2, max_digits=3) class UserFile(ExportModelOperationsMixin('user_files'), models.Model, PermissionModelMixin): diff --git a/cookbook/signals.py b/cookbook/signals.py index f405a5218..32ff71400 100644 --- a/cookbook/signals.py +++ b/cookbook/signals.py @@ -37,6 +37,7 @@ def update_recipe_search_vector(sender, instance=None, created=False, **kwargs): if SQLITE: return language = DICTIONARY.get(translation.get_language(), 'simple') + # these indexed fields are space wide, reading user preferences would lead to inconsistent behavior instance.name_search_vector = SearchVector('name__unaccent', weight='A', config=language) instance.desc_search_vector = SearchVector('description__unaccent', weight='C', config=language) try: diff --git a/cookbook/tests/conftest.py b/cookbook/tests/conftest.py index 932a33aec..da724fc69 100644 --- a/cookbook/tests/conftest.py +++ b/cookbook/tests/conftest.py @@ -158,6 +158,31 @@ def dict_compare(d1, d2, details=False): return any([not added, not removed, not modified, not modified_dicts]) +def transpose(text, number=2): + + # select random token + tokens = text.split() + positions = list(i for i, e in enumerate(tokens) if len(e) > 1) + + if positions: + + token_pos = random.choice(positions) + + # select random positions in token + positions = random.sample(range(len(tokens[token_pos])), number) + + # swap the positions + l = list(tokens[token_pos]) + for first, second in zip(positions[::2], positions[1::2]): + l[first], l[second] = l[second], l[first] + + # replace original tokens with swapped + tokens[token_pos] = ''.join(l) + + # return text with the swapped token + return ' '.join(tokens) + + @pytest.fixture def recipe_1_s1(space_1, u1_s1): return get_random_recipe(space_1, u1_s1) diff --git a/cookbook/tests/factories/__init__.py b/cookbook/tests/factories/__init__.py index a35506c06..5cc215d3d 100644 --- a/cookbook/tests/factories/__init__.py +++ b/cookbook/tests/factories/__init__.py @@ -277,6 +277,15 @@ class ShoppingListEntryFactory(factory.django.DjangoModelFactory): delay_until = None space = factory.SubFactory(SpaceFactory) + @classmethod + def _create(cls, target_class, *args, **kwargs): # override create to prevent auto_add_now from changing the created_at date + created_at = kwargs.pop('created_at', None) + obj = super(ShoppingListEntryFactory, cls)._create(target_class, *args, **kwargs) + if created_at is not None: + obj.created_at = created_at + obj.save() + return obj + class Params: has_mealplan = False @@ -324,8 +333,6 @@ class StepFactory(factory.django.DjangoModelFactory): has_recipe = False self.ingredients.add(IngredientFactory(space=self.space, food__has_recipe=has_recipe)) num_header = kwargs.get('header', 0) - ####################################################### - ####################################################### if num_header > 0: for i in range(num_header): self.ingredients.add(IngredientFactory(food=None, unit=None, amount=0, is_header=True, space=self.space)) @@ -354,6 +361,15 @@ class RecipeFactory(factory.django.DjangoModelFactory): created_at = factory.LazyAttribute(lambda x: faker.date_this_decade()) space = factory.SubFactory(SpaceFactory) + @classmethod + def _create(cls, target_class, *args, **kwargs): # override create to prevent auto_add_now from changing the created_at date + created_at = kwargs.pop('created_at', None) + obj = super(RecipeFactory, cls)._create(target_class, *args, **kwargs) + if created_at is not None: + obj.created_at = created_at + obj.save() + return obj + @factory.post_generation def keywords(self, create, extracted, **kwargs): if not create: @@ -404,3 +420,47 @@ class RecipeFactory(factory.django.DjangoModelFactory): class Meta: model = 'cookbook.Recipe' + + +@register +class CookLogFactory(factory.django.DjangoModelFactory): + """CookLog factory.""" + recipe = factory.SubFactory(RecipeFactory, space=factory.SelfAttribute('..space')) + created_by = factory.SubFactory(UserFactory, space=factory.SelfAttribute('..space')) + created_at = factory.LazyAttribute(lambda x: faker.date_this_decade()) + rating = factory.LazyAttribute(lambda x: faker.random_int(min=1, max=5)) + servings = factory.LazyAttribute(lambda x: faker.random_int(min=1, max=32)) + space = factory.SubFactory(SpaceFactory) + + @classmethod + def _create(cls, target_class, *args, **kwargs): # override create to prevent auto_add_now from changing the created_at date + created_at = kwargs.pop('created_at', None) + obj = super(CookLogFactory, cls)._create(target_class, *args, **kwargs) + if created_at is not None: + obj.created_at = created_at + obj.save() + return obj + + class Meta: + model = 'cookbook.CookLog' + + +@register +class ViewLogFactory(factory.django.DjangoModelFactory): + """ViewLog factory.""" + recipe = factory.SubFactory(RecipeFactory, space=factory.SelfAttribute('..space')) + created_by = factory.SubFactory(UserFactory, space=factory.SelfAttribute('..space')) + created_at = factory.LazyAttribute(lambda x: faker.past_datetime(start_date='-365d')) + space = factory.SubFactory(SpaceFactory) + + @classmethod + def _create(cls, target_class, *args, **kwargs): # override create to prevent auto_add_now from changing the created_at date + created_at = kwargs.pop('created_at', None) + obj = super(ViewLogFactory, cls)._create(target_class, *args, **kwargs) + if created_at is not None: + obj.created_at = created_at + obj.save() + return obj + + class Meta: + model = 'cookbook.ViewLog' diff --git a/cookbook/tests/other/test_recipe_full_text_search.py b/cookbook/tests/other/test_recipe_full_text_search.py index d1893c7f2..b9da134b9 100644 --- a/cookbook/tests/other/test_recipe_full_text_search.py +++ b/cookbook/tests/other/test_recipe_full_text_search.py @@ -1,58 +1,78 @@ import itertools import json +from datetime import timedelta import pytest +from django.conf import settings from django.contrib import auth from django.urls import reverse +from django.utils import timezone from django_scopes import scope, scopes_disabled from cookbook.models import Food, Recipe, SearchFields -from cookbook.tests.factories import (FoodFactory, IngredientFactory, KeywordFactory, - RecipeBookEntryFactory, RecipeFactory, UnitFactory) - -# TODO recipe name/description/instructions/keyword/book/food test search with icontains, istarts with/ full text(?? probably when word changes based on conjugation??), trigram, unaccent - +from cookbook.tests.conftest import transpose +from cookbook.tests.factories import (CookLogFactory, FoodFactory, IngredientFactory, + KeywordFactory, RecipeBookEntryFactory, RecipeFactory, + UnitFactory, ViewLogFactory) # TODO test combining any/all of the above -# TODO search rating as user or when another user rated -# TODO search last cooked -# TODO changing lsat_viewed ## to return on search # TODO test sort_by # TODO test sort_by new X number of recipes are new within last Y days # TODO test loading custom filter # TODO test loading custom filter with overrided params # TODO makenow with above filters -# TODO test search for number of times cooked (self vs others) -# TODO test including children +# TODO test search food/keywords including/excluding children LIST_URL = 'api:recipe-list' +sqlite = settings.DATABASES['default']['ENGINE'] not in ['django.db.backends.postgresql_psycopg2', 'django.db.backends.postgresql'] @pytest.fixture def accent(): - return "àèìòù" + return "àbçđêf ğĦìĵķĽmñ öPqŕşŧ úvŵxyž" @pytest.fixture def unaccent(): - return "aeiou" + return "abcdef ghijklmn opqrst uvwxyz" @pytest.fixture -def user1(request, space_1, u1_s1): +def user1(request, space_1, u1_s1, unaccent): user = auth.get_user(u1_s1) - params = {x[0]: x[1] for x in request.param} + try: + params = {x[0]: x[1] for x in request.param} + except AttributeError: + params = {} + result = 1 + misspelled_result = 0 + search_term = unaccent + if params.get('fuzzy_lookups', False): user.searchpreference.lookup = True + misspelled_result = 1 if params.get('fuzzy_search', False): user.searchpreference.trigram.set(SearchFields.objects.all()) + misspelled_result = 1 + + if params.get('icontains', False): + user.searchpreference.icontains.set(SearchFields.objects.all()) + search_term = 'ghijklmn' + if params.get('istartswith', False): + user.searchpreference.istartswith.set(SearchFields.objects.all()) + search_term = 'abcdef' if params.get('unaccent', False): user.searchpreference.unaccent.set(SearchFields.objects.all()) + misspelled_result *= 2 + result *= 2 + # full text vectors are hard coded to use unaccent - put this after unaccent to override result + if params.get('fulltext', False): + user.searchpreference.fulltext.set(SearchFields.objects.all()) + # user.searchpreference.search = 'websearch' + search_term = 'ghijklmn uvwxyz' result = 2 - else: - result = 1 - - user.userpreference.save() - return (u1_s1, result, params) + user.searchpreference.save() + misspelled_term = transpose(search_term, number=3) + return (u1_s1, result, misspelled_result, search_term, misspelled_term, params) @pytest.fixture @@ -61,15 +81,21 @@ def recipes(space_1): @pytest.fixture -def found_recipe(request, space_1, accent, unaccent): +def found_recipe(request, space_1, accent, unaccent, u1_s1, u2_s1): + user1 = auth.get_user(u1_s1) + user2 = auth.get_user(u2_s1) + days_3 = timezone.now() - timedelta(days=3) + days_15 = timezone.now() - timedelta(days=15) + days_30 = timezone.now() - timedelta(days=30) recipe1 = RecipeFactory.create(space=space_1) recipe2 = RecipeFactory.create(space=space_1) recipe3 = RecipeFactory.create(space=space_1) + obj1 = None + obj2 = None if request.param.get('food', None): obj1 = FoodFactory.create(name=unaccent, space=space_1) obj2 = FoodFactory.create(name=accent, space=space_1) - recipe1.steps.first().ingredients.add(IngredientFactory.create(food=obj1)) recipe2.steps.first().ingredients.add(IngredientFactory.create(food=obj2)) recipe3.steps.first().ingredients.add(IngredientFactory.create(food=obj1), IngredientFactory.create(food=obj2)) @@ -79,6 +105,10 @@ def found_recipe(request, space_1, accent, unaccent): recipe1.keywords.add(obj1) recipe2.keywords.add(obj2) recipe3.keywords.add(obj1, obj2) + recipe1.name = unaccent + recipe2.name = accent + recipe1.save() + recipe2.save() if request.param.get('book', None): obj1 = RecipeBookEntryFactory.create(recipe=recipe1).book obj2 = RecipeBookEntryFactory.create(recipe=recipe2).book @@ -87,12 +117,51 @@ def found_recipe(request, space_1, accent, unaccent): if request.param.get('unit', None): obj1 = UnitFactory.create(name=unaccent, space=space_1) obj2 = UnitFactory.create(name=accent, space=space_1) - recipe1.steps.first().ingredients.add(IngredientFactory.create(unit=obj1)) recipe2.steps.first().ingredients.add(IngredientFactory.create(unit=obj2)) recipe3.steps.first().ingredients.add(IngredientFactory.create(unit=obj1), IngredientFactory.create(unit=obj2)) + if request.param.get('name', None): + recipe1.name = unaccent + recipe2.name = accent + recipe1.save() + recipe2.save() + if request.param.get('description', None): + recipe1.description = unaccent + recipe2.description = accent + recipe1.save() + recipe2.save() + if request.param.get('instruction', None): + i1 = recipe1.steps.first() + i2 = recipe2.steps.first() + i1.instruction = unaccent + i2.instruction = accent + i1.save() + i2.save() + if request.param.get('createdon', None): + recipe1.created_at = days_3 + recipe2.created_at = days_30 + recipe3.created_at = days_15 + recipe1.save() + recipe2.save() + recipe3.save() + if request.param.get('viewedon', None): + ViewLogFactory.create(recipe=recipe1, created_by=user1, created_at=days_3, space=space_1) + ViewLogFactory.create(recipe=recipe2, created_by=user1, created_at=days_30, space=space_1) + ViewLogFactory.create(recipe=recipe3, created_by=user2, created_at=days_15, space=space_1) + if request.param.get('cookedon', None): + CookLogFactory.create(recipe=recipe1, created_by=user1, created_at=days_3, space=space_1) + CookLogFactory.create(recipe=recipe2, created_by=user1, created_at=days_30, space=space_1) + CookLogFactory.create(recipe=recipe3, created_by=user2, created_at=days_15, space=space_1) + if request.param.get('timescooked', None): + CookLogFactory.create_batch(5, recipe=recipe1, created_by=user1, space=space_1) + CookLogFactory.create(recipe=recipe2, created_by=user1, space=space_1) + CookLogFactory.create_batch(3, recipe=recipe3, created_by=user2, space=space_1) + if request.param.get('rating', None): + CookLogFactory.create(recipe=recipe1, created_by=user1, rating=5.0, space=space_1) + CookLogFactory.create(recipe=recipe2, created_by=user1, rating=1.0, space=space_1) + CookLogFactory.create(recipe=recipe3, created_by=user2, rating=3.0, space=space_1) - return (recipe1, recipe2, recipe3, obj1, obj2) + return (recipe1, recipe2, recipe3, obj1, obj2, request.param) @pytest.mark.parametrize("found_recipe, param_type", [ @@ -101,7 +170,7 @@ def found_recipe(request, space_1, accent, unaccent): ({'book': True}, 'books'), ], indirect=['found_recipe']) @pytest.mark.parametrize('operator', [('_or', 3, 0), ('_and', 1, 2), ]) -def test_search_or_and_not(found_recipe, param_type, operator, recipes, user1, space_1): +def test_search_or_and_not(found_recipe, param_type, operator, recipes, u1_s1, space_1): with scope(space=space_1): param1 = f"{param_type}{operator[0]}={found_recipe[3].id}" param2 = f"{param_type}{operator[0]}={found_recipe[4].id}" @@ -163,9 +232,12 @@ def test_search_units(found_recipe, recipes, u1_s1, space_1): assert found_recipe[2].id in [x['id'] for x in r['results']] +@pytest.mark.skipif(sqlite, reason="requires PostgreSQL") @pytest.mark.parametrize("user1", itertools.product( - [('fuzzy_lookups', True), ('fuzzy_lookups', False)], - [('fuzzy_search', True), ('fuzzy_search', False)], + [ + ('fuzzy_search', True), ('fuzzy_search', False), + ('fuzzy_lookups', True), ('fuzzy_lookups', False) + ], [('unaccent', True), ('unaccent', False)] ), indirect=['user1']) @pytest.mark.parametrize("found_recipe, param_type", [ @@ -176,12 +248,99 @@ def test_search_units(found_recipe, recipes, u1_s1, space_1): def test_fuzzy_lookup(found_recipe, recipes, param_type, user1, space_1): with scope(space=space_1): list_url = f'api:{param_type}-list' - param1 = "query=aeiou" - param2 = "query=aoieu" + param1 = f"query={user1[3]}" + param2 = f"query={user1[4]}" - # test fuzzy off - also need search settings on/off r = json.loads(user1[0].get(reverse(list_url) + f'?{param1}&limit=2').content) assert len([x['id'] for x in r['results'] if x['id'] in [found_recipe[3].id, found_recipe[4].id]]) == user1[1] - r = json.loads(user1[0].get(reverse(list_url) + f'?{param2}').content) - assert len([x['id'] for x in r['results'] if x['id'] in [found_recipe[3].id, found_recipe[4].id]]) == user1[1] + r = json.loads(user1[0].get(reverse(list_url) + f'?{param2}&limit=10').content) + assert len([x['id'] for x in r['results'] if x['id'] in [found_recipe[3].id, found_recipe[4].id]]) == user1[2] + + +@pytest.mark.skipif(sqlite, reason="requires PostgreSQL") +@pytest.mark.parametrize("user1", itertools.product( + [ + ('fuzzy_search', True), ('fuzzy_search', False), + ('fulltext', True), ('fulltext', False), + ('icontains', True), ('icontains', False), + ('istartswith', True), ('istartswith', False), + ], + [('unaccent', True), ('unaccent', False)] +), indirect=['user1']) +@pytest.mark.parametrize("found_recipe", [ + ({'name': True}), + ({'description': True}), + ({'instruction': True}), + ({'keyword': True}), + ({'food': True}), +], indirect=['found_recipe']) +def test_search_string(found_recipe, recipes, user1, space_1): + with scope(space=space_1): + param1 = f"query={user1[3]}" + param2 = f"query={user1[4]}" + + r = json.loads(user1[0].get(reverse(LIST_URL) + f'?{param1}').content) + assert len([x['id'] for x in r['results'] if x['id'] in [found_recipe[0].id, found_recipe[1].id]]) == user1[1] + + r = json.loads(user1[0].get(reverse(LIST_URL) + f'?{param2}').content) + assert len([x['id'] for x in r['results'] if x['id'] in [found_recipe[0].id, found_recipe[1].id]]) == user1[2] + + +@pytest.mark.parametrize("found_recipe, param_type, result", [ + ({'viewedon': True}, 'viewedon', (1, 1)), + ({'cookedon': True}, 'cookedon', (1, 1)), + ({'createdon': True}, 'createdon', (2, 12)), # created dates are not filtered by user +], indirect=['found_recipe']) +def test_search_date(found_recipe, recipes, param_type, result, u1_s1, u2_s1, space_1): + date = (timezone.now() - timedelta(days=15)).strftime("%Y-%m-%d") + param1 = f"?{param_type}={date}" + param2 = f"?{param_type}=-{date}" + r = json.loads(u1_s1.get(reverse(LIST_URL) + f'{param1}').content) + assert r['count'] == result[0] + assert found_recipe[0].id in [x['id'] for x in r['results']] + + r = json.loads(u1_s1.get(reverse(LIST_URL) + f'{param2}').content) + assert r['count'] == result[1] + assert found_recipe[1].id in [x['id'] for x in r['results']] + + # test today's date returns for lte and gte searches + r = json.loads(u2_s1.get(reverse(LIST_URL) + f'{param1}').content) + assert r['count'] == result[0] + assert found_recipe[2].id in [x['id'] for x in r['results']] + + r = json.loads(u2_s1.get(reverse(LIST_URL) + f'{param2}').content) + assert r['count'] == result[1] + assert found_recipe[2].id in [x['id'] for x in r['results']] + + +@pytest.mark.parametrize("found_recipe, param_type", [ + ({'rating': True}, 'rating'), + ({'timescooked': True}, 'timescooked'), +], indirect=['found_recipe']) +def test_search_count(found_recipe, recipes, param_type, u1_s1, u2_s1, space_1): + param1 = f'?{param_type}=3' + param2 = f'?{param_type}=-3' + param3 = f'?{param_type}=0' + + r = json.loads(u1_s1.get(reverse(LIST_URL) + param1).content) + assert r['count'] == 1 + assert found_recipe[0].id in [x['id'] for x in r['results']] + + r = json.loads(u1_s1.get(reverse(LIST_URL) + param2).content) + assert r['count'] == 1 + assert found_recipe[1].id in [x['id'] for x in r['results']] + + # test search for not rated/cooked + r = json.loads(u1_s1.get(reverse(LIST_URL) + param3).content) + assert r['count'] == 11 + assert (found_recipe[0].id or found_recipe[1].id) not in [x['id'] for x in r['results']] + + # test matched returns for lte and gte searches + r = json.loads(u2_s1.get(reverse(LIST_URL) + param1).content) + assert r['count'] == 1 + assert found_recipe[2].id in [x['id'] for x in r['results']] + + r = json.loads(u2_s1.get(reverse(LIST_URL) + param2).content) + assert r['count'] == 1 + assert found_recipe[2].id in [x['id'] for x in r['results']] diff --git a/cookbook/views/api.py b/cookbook/views/api.py index f426994fa..a2c4b697f 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -652,7 +652,9 @@ class RecipeViewSet(viewsets.ModelViewSet): QueryParam(name='random', description=_('Returns the results in randomized order. [''true''/''false'']')), QueryParam(name='new', description=_('Returns new results first in search results. [''true''/''false'']')), QueryParam(name='timescooked', description=_('Filter recipes cooked X times or more. Negative values returns cooked less than X times'), qtype='int'), - QueryParam(name='lastcooked', description=_('Filter recipes last cooked on or after YYYY-MM-DD. Prepending ''-'' filters on or before date.')), + QueryParam(name='cookedon', description=_('Filter recipes last cooked on or after YYYY-MM-DD. Prepending ''-'' filters on or before date.')), + QueryParam(name='createdon', description=_('Filter recipes created on or after YYYY-MM-DD. Prepending ''-'' filters on or before date.')), + QueryParam(name='viewedon', description=_('Filter recipes lasts viewed on or after YYYY-MM-DD. Prepending ''-'' filters on or before date.')), QueryParam(name='makenow', description=_('Filter recipes that can be made with OnHand food. [''true''/''false'']')), ] schema = QueryParamAutoSchema() diff --git a/cookbook/views/views.py b/cookbook/views/views.py index 8a1462a25..23d2fe8a2 100644 --- a/cookbook/views/views.py +++ b/cookbook/views/views.py @@ -361,17 +361,18 @@ def user_settings(request): sp.istartswith.clear() sp.trigram.set([SearchFields.objects.get(name='Name')]) sp.fulltext.clear() - sp.trigram_threshold = 0.1 + sp.trigram_threshold = 0.2 if search_form.cleaned_data['preset'] == 'precise': sp.search = SearchPreference.WEB sp.lookup = True sp.unaccent.set(SearchFields.objects.all()) - sp.icontains.clear() + # full text on food is very slow, add search_vector field and index it (including Admin functions and postsave signal to rebuild index) + sp.icontains.set([SearchFields.objects.get(name__in=['Name', 'Ingredients'])]) sp.istartswith.set([SearchFields.objects.get(name='Name')]) sp.trigram.clear() - sp.fulltext.set(SearchFields.objects.all()) - sp.trigram_threshold = 0.1 + sp.fulltext.set(SearchFields.objects.filter(name__in=['Ingredients'])) + sp.trigram_threshold = 0.2 sp.save() elif 'shopping_form' in request.POST: diff --git a/vue/src/apps/RecipeSearchView/RecipeSearchView.vue b/vue/src/apps/RecipeSearchView/RecipeSearchView.vue index eada2392b..aaa825f76 100644 --- a/vue/src/apps/RecipeSearchView/RecipeSearchView.vue +++ b/vue/src/apps/RecipeSearchView/RecipeSearchView.vue @@ -122,7 +122,7 @@ - + @@ -393,7 +393,7 @@ -
+
@@ -410,9 +410,9 @@ - + - - >= + + >= <= @@ -563,8 +563,12 @@ export default { timescooked: undefined, timescooked_gte: true, makenow: false, - lastcooked: undefined, - lastcooked_gte: true, + cookedon: undefined, + cookedon_gte: true, + createdon: undefined, + createdon_gte: true, + viewedon: undefined, + viewedon_gte: true, sort_order: [], pagination_page: 1, expert_mode: false, @@ -594,7 +598,7 @@ export default { show_sortby: false, show_timescooked: false, show_makenow: false, - show_lastcooked: false, + show_cookedon: false, include_children: true, }, pagination_count: 0, @@ -681,7 +685,7 @@ export default { const field = [ [this.$t("search_rank"), "score"], [this.$t("Name"), "name"], - [this.$t("last_cooked"), "lastcooked"], + [this.$t("last_cooked"), "cookedon"], [this.$t("Rating"), "rating"], [this.$t("times_cooked"), "favorite"], [this.$t("date_created"), "created_at"], @@ -876,7 +880,7 @@ export default { this.search.pagination_page = 1 this.search.timescooked = undefined this.search.makenow = false - this.search.lastcooked = undefined + this.search.cookedon = undefined let fieldnum = { keywords: 1, @@ -976,9 +980,17 @@ export default { if (rating !== undefined && !this.search.search_rating_gte) { rating = rating * -1 } - let lastcooked = this.search.lastcooked || undefined - if (lastcooked !== undefined && !this.search.lastcooked_gte) { - lastcooked = "-" + lastcooked + let cookedon = this.search.cookedon || undefined + if (cookedon !== undefined && !this.search.cookedon_gte) { + cookedon = "-" + cookedon + } + let createdon = this.search.createdon || undefined + if (createdon !== undefined && !this.search.createdon_gte) { + createdon = "-" + createdon + } + let viewedon = this.search.viewedon || undefined + if (viewedon !== undefined && !this.search.viewedon_gte) { + viewedon = "-" + viewedon } let timescooked = parseInt(this.search.timescooked) if (isNaN(timescooked)) { @@ -999,7 +1011,9 @@ export default { random: this.random_search, timescooked: timescooked, makenow: this.search.makenow || undefined, - lastcooked: lastcooked, + cookedon: cookedon, + createdon: createdon, + viewedon: viewedon, page: this.search.pagination_page, pageSize: this.ui.page_size, } @@ -1009,7 +1023,7 @@ export default { include_children: this.ui.include_children, } if (!this.searchFiltered()) { - params.options.query.last_viewed = this.ui.recently_viewed + params.options.query.num_recent = this.ui.recently_viewed //TODO refactor as num_recent params._new = this.ui.sort_by_new } if (this.search.search_filter) { @@ -1030,12 +1044,12 @@ export default { this.search?.search_rating !== undefined || (this.search.timescooked !== undefined && this.search.timescooked !== "") || this.search.makenow !== false || - (this.search.lastcooked !== undefined && this.search.lastcooked !== "") + (this.search.cookedon !== undefined && this.search.cookedon !== "") if (ignore_string) { return filtered } else { - return filtered || this.search?.search_input != "" || this.search.sort_order.length <= 1 + return filtered || this.search?.search_input != "" || this.search.sort_order.length >= 1 } }, addFields(field) { diff --git a/vue/src/utils/models.js b/vue/src/utils/models.js index f283b9689..a977b0e98 100644 --- a/vue/src/utils/models.js +++ b/vue/src/utils/models.js @@ -530,7 +530,9 @@ export class Models { "random", "_new", "timescooked", - "lastcooked", + "cookedon", + "createdon", + "viewedon", "makenow", "page", "pageSize",