diff --git a/cookbook/apps.py b/cookbook/apps.py index b09926626..a506e060b 100644 --- a/cookbook/apps.py +++ b/cookbook/apps.py @@ -3,3 +3,6 @@ from django.apps import AppConfig class CookbookConfig(AppConfig): name = 'cookbook' + + def ready(self): + import cookbook.signals # noqa diff --git a/cookbook/filters.py b/cookbook/filters.py index b679a79fb..81eba379f 100644 --- a/cookbook/filters.py +++ b/cookbook/filters.py @@ -1,7 +1,5 @@ import django_filters from django.conf import settings -from django.contrib.postgres.search import TrigramSimilarity -from django.db.models import Q from django.utils.translation import gettext as _ from django_scopes import scopes_disabled @@ -53,6 +51,8 @@ with scopes_disabled(): 'django.db.backends.postgresql']: queryset = queryset.annotate(similarity=TrigramSimilarity('name', value), ).filter( Q(similarity__gt=0.1) | Q(name__unaccent__icontains=value)).order_by('-similarity') + if settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2', 'django.db.backends.postgresql']: + return queryset else: queryset = queryset.filter(name__icontains=value) return queryset @@ -61,7 +61,6 @@ with scopes_disabled(): model = Recipe fields = ['name', 'keywords', 'foods', 'internal'] - class FoodFilter(django_filters.FilterSet): name = django_filters.CharFilter(lookup_expr='icontains') @@ -69,7 +68,6 @@ with scopes_disabled(): model = Food fields = ['name'] - class ShoppingListFilter(django_filters.FilterSet): def __init__(self, data=None, *args, **kwargs): diff --git a/cookbook/managers.py b/cookbook/managers.py new file mode 100644 index 000000000..beb91dd11 --- /dev/null +++ b/cookbook/managers.py @@ -0,0 +1,30 @@ +from django.contrib.postgres.aggregates import StringAgg +from django.contrib.postgres.search import ( + SearchQuery, SearchRank, SearchVector, TrigramSimilarity, +) +from django.db import models + + +# TODO add search highlighting +# TODO add language support +# TODO add schedule index rebuild +# TODO add admin function to rebuild index +class RecipeSearchManager(models.Manager): + + def search(self, search_text, space): + search_query = SearchQuery( + search_text, config='english' + ) + search_vectors = ( + SearchVector('search_vector') + + SearchVector(StringAgg('steps__ingredients__food__name', delimiter=' '), weight='A', config='english') + + SearchVector(StringAgg('keywords__name', delimiter=' '), weight='A', config='english')) + search_rank = SearchRank(search_vectors, search_query) + # trigram_similarity = TrigramSimilarity( + # 'headline', search_text + # ) + return ( + self.get_queryset() + .annotate(search=search_vectors, rank=search_rank) + .filter(search=search_query) + .order_by('-rank')) diff --git a/cookbook/migrations/0119_auto_20210407_1828.py b/cookbook/migrations/0119_auto_20210407_1828.py new file mode 100644 index 000000000..ef8be186a --- /dev/null +++ b/cookbook/migrations/0119_auto_20210407_1828.py @@ -0,0 +1,37 @@ +# Generated by Django 3.1.7 on 2021-04-07 20:00 + +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVectorField, SearchVector +from django.db import migrations +from django_scopes import scopes_disabled +from cookbook.models import Recipe + + +def set_default_search_vector(apps, schema_editor): + with scopes_disabled(): + search_vector = ( + SearchVector('name', weight='A') + + SearchVector('description', weight='B')) + Recipe.objects.all().update(search_vector=search_vector) + + +class Migration(migrations.Migration): + + dependencies = [ + ('cookbook', '0118_auto_20210406_1805'), + ] + + operations = [ + migrations.AddField( + model_name='recipe', + name='search_vector', + field=SearchVectorField(null=True), + ), + migrations.AddIndex( + model_name='recipe', + index=GinIndex(fields=['search_vector'], name='cookbook_re_search__404e46_gin'), + ), + migrations.RunPython( + set_default_search_vector + ), + ] diff --git a/cookbook/models.py b/cookbook/models.py index d81811281..76ebf9c2b 100644 --- a/cookbook/models.py +++ b/cookbook/models.py @@ -7,7 +7,8 @@ from datetime import date, timedelta from annoying.fields import AutoOneToOneField from django.contrib import auth from django.contrib.auth.models import Group, User -from django.core.files.uploadedfile import UploadedFile, InMemoryUploadedFile +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVectorField from django.core.validators import MinLengthValidator from django.db import models from django.utils import timezone @@ -17,6 +18,7 @@ from django_scopes import ScopedManager from recipes.settings import (COMMENT_PREF_DEFAULT, FRACTION_PREF_DEFAULT, STICKY_NAV_PREF_DEFAULT) +from cookbook.managers import RecipeSearchManager def get_user_name(self): @@ -376,7 +378,11 @@ class NutritionInformation(models.Model, PermissionModelMixin): return f'Nutrition {self.pk}' -class Recipe(ExportModelOperationsMixin('recipe'), models.Model, PermissionModelMixin): +# TODO adjust model based on DB capabilities +# options to have multiple recipe models based on DB capability (to drive search) +# required_db_features, required-db-vendor +# https://docs.djangoproject.com/en/3.1/ref/models/options/#required-db-vendor +class Recipe(models.Model, PermissionModelMixin): name = models.CharField(max_length=128) description = models.CharField(max_length=512, blank=True, null=True) servings = models.IntegerField(default=1) @@ -400,13 +406,17 @@ class Recipe(ExportModelOperationsMixin('recipe'), models.Model, PermissionModel created_by = models.ForeignKey(User, on_delete=models.PROTECT) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) + search_vector = SearchVectorField(null=True) space = models.ForeignKey(Space, on_delete=models.CASCADE) - objects = ScopedManager(space='space') + objects = ScopedManager(_manager_class=RecipeSearchManager, space='space') def __str__(self): return self.name + class Meta(): + indexes = (GinIndex(fields=["search_vector"]),) + class Comment(ExportModelOperationsMixin('comment'), models.Model, PermissionModelMixin): recipe = models.ForeignKey(Recipe, on_delete=models.CASCADE) diff --git a/cookbook/signals.py b/cookbook/signals.py new file mode 100644 index 000000000..5c4cb69f4 --- /dev/null +++ b/cookbook/signals.py @@ -0,0 +1,25 @@ +from django.contrib.postgres.search import SearchVector +from django.db.models.signals import post_save +from django.dispatch import receiver + +from cookbook.models import Recipe + + +@receiver(post_save, sender=Recipe) +def update_recipe_search_vector(sender, instance=None, created=False, **kwargs): + if not instance: + return + + if hasattr(instance, '_dirty'): + return + + instance.search_vector = ( + SearchVector('name', weight='A', config='english') + + SearchVector('description', weight='B', config='english') + ) + + try: + instance._dirty = True + instance.save() + finally: + del instance._dirty diff --git a/cookbook/views/views.py b/cookbook/views/views.py index d56222bf8..0f9738e4a 100644 --- a/cookbook/views/views.py +++ b/cookbook/views/views.py @@ -12,19 +12,18 @@ from django.contrib.auth.forms import PasswordChangeForm from django.contrib.auth.models import Group from django.contrib.auth.password_validation import validate_password from django.core.exceptions import ValidationError -from django.db import IntegrityError -from django.db.models import Avg, Q, Sum -from django.http import HttpResponseRedirect, JsonResponse +from django.db.models import Avg, Q +from django.http import HttpResponseRedirect from django.shortcuts import get_object_or_404, render, redirect from django.urls import reverse, reverse_lazy from django.utils import timezone from django.utils.translation import gettext as _ -from django_scopes import scopes_disabled, scope +from django_scopes import scopes_disabled from django_tables2 import RequestConfig from rest_framework.authtoken.models import Token from cookbook.filters import RecipeFilter -from cookbook.forms import (CommentForm, Recipe, RecipeBookEntryForm, User, +from cookbook.forms import (CommentForm, Recipe, User, UserCreateForm, UserNameForm, UserPreference, UserPreferenceForm, SpaceJoinForm, SpaceCreateForm, AllAuthSignupForm) from cookbook.helper.ingredient_parser import parse @@ -57,6 +56,9 @@ def index(request): return HttpResponseRedirect(reverse('view_search')) +# faceting +# unaccent / likely will perform full table scan +# create tests def search(request): if has_group_permission(request.user, ('guest',)): if request.user.userpreference.search_style == UserPreference.NEW: @@ -65,11 +67,16 @@ def search(request): f = RecipeFilter(request.GET, queryset=Recipe.objects.filter(space=request.user.userpreference.space).all().order_by('name'), space=request.space) + if settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2', 'django.db.backends.postgresql']: + qs = Recipe.objects.search(request.GET.get('name', ''), space=request.space) + else: + qs = Recipe.objects.filter(space=request.user.userpreference.space).all().order_by('name') + f = RecipeFilter(request.GET, queryset=qs, space=request.space) if request.user.userpreference.search_style == UserPreference.LARGE: table = RecipeTable(f.qs) else: - table = RecipeTableSmall(f.qs) + table = RecipeTable(f.qs) RequestConfig(request, paginate={'per_page': 25}).configure(table) if request.GET == {} and request.user.userpreference.show_recent: @@ -372,8 +379,8 @@ def history(request): @group_required('admin') def system(request): postgres = False if ( - settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql_psycopg2' # noqa: E501 - or settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql' # noqa: E501 + settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql_psycopg2' # noqa: E501 + or settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql' # noqa: E501 ) else True secret_key = False if os.getenv('SECRET_KEY') else True