Merge pull request #1088 from smilerz/search_troubleshooting

add search debug

cookbook/helper/recipe_search.py

@@ -1,17 +1,17 @@
 from collections import Counter
 from datetime import timedelta
 
-from recipes import settings
-from django.contrib.postgres.search import (
-    SearchQuery, SearchRank, TrigramSimilarity
-)
+from django.contrib.postgres.search import SearchQuery, SearchRank, TrigramSimilarity
 from django.core.cache import caches
 from django.db.models import Avg, Case, Count, Func, Max, Q, Subquery, Value, When
 from django.db.models.functions import Coalesce
 from django.utils import timezone, translation
 
+from cookbook.filters import RecipeFilter
+from cookbook.helper.permission_helper import has_group_permission
 from cookbook.managers import DICTIONARY
-from cookbook.models import Food, Keyword, ViewLog, SearchPreference
+from cookbook.models import Food, Keyword, Recipe, SearchPreference, ViewLog
+from recipes import settings
 
 
 class Round(Func):
@@ -143,9 +143,9 @@ def search_recipes(request, queryset, params):
 
             # TODO add order by user settings - only do search rank and annotation if rank order is configured
             search_rank = (
-                    SearchRank('name_search_vector', search_query, cover_density=True)
-                    + SearchRank('desc_search_vector', search_query, cover_density=True)
-                    + SearchRank('steps__search_vector', search_query, cover_density=True)
+                SearchRank('name_search_vector', search_query, cover_density=True)
+                + SearchRank('desc_search_vector', search_query, cover_density=True)
+                + SearchRank('steps__search_vector', search_query, cover_density=True)
             )
             queryset = queryset.filter(query_filter).annotate(rank=search_rank)
             orderby += ['-rank']
@@ -400,3 +400,13 @@ def annotated_qs(qs, root=False, fill=False):
     if start_depth and start_depth > 0:
         info['close'] = list(range(0, prev_depth - start_depth + 1))
     return result
+
+
+def old_search(request):
+    if has_group_permission(request.user, ('guest',)):
+        params = dict(request.GET)
+        params['internal'] = None
+        f = RecipeFilter(params,
+                         queryset=Recipe.objects.filter(space=request.user.userpreference.space).all().order_by('name'),
+                         space=request.space)
+        return f.qs

cookbook/templates/base.html

@@ -345,6 +345,7 @@
     localStorage.setItem('SCRIPT_NAME', "{% base_path request 'script' %}")
     localStorage.setItem('BASE_PATH', "{% base_path request 'base' %}")
     localStorage.setItem('STATIC_URL', "{% base_path request 'static_base' %}")
+    localStorage.setItem('DEBUG', "{% is_debug %}")
     window.addEventListener("load", () => {
         if ("serviceWorker" in navigator) {
             navigator.serviceWorker.register("{% url 'service_worker' %}", {scope: "{% base_path request 'base' %}" + '/'}).then(function (reg) {

cookbook/urls.py

@@ -2,17 +2,17 @@ from pydoc import locate
 
 from django.urls import include, path
 from django.views.generic import TemplateView
-from recipes.version import VERSION_NUMBER
-from rest_framework import routers, permissions
+from rest_framework import permissions, routers
 from rest_framework.schemas import get_schema_view
 
 from cookbook.helper import dal
+from recipes.settings import DEBUG
+from recipes.version import VERSION_NUMBER
 
-from .models import (Comment, Food, InviteLink, Keyword, MealPlan, Recipe,
-                     RecipeBook, RecipeBookEntry, RecipeImport, ShoppingList,
-                     Storage, Supermarket, SupermarketCategory, Sync, SyncLog, Unit, get_model_name, Automation,
-                     UserFile, Step)
-from .views import api, data, delete, edit, import_export, lists, new, views, telegram
+from .models import (Automation, Comment, Food, InviteLink, Keyword, MealPlan, Recipe, RecipeBook,
+                     RecipeBookEntry, RecipeImport, ShoppingList, Step, Storage, Supermarket,
+                     SupermarketCategory, Sync, SyncLog, Unit, UserFile, get_model_name)
+from .views import api, data, delete, edit, import_export, lists, new, telegram, views
 
 router = routers.DefaultRouter()
 router.register(r'user-name', api.UserNameViewSet, basename='username')
@@ -68,8 +68,6 @@ urlpatterns = [
     path('history/', views.history, name='view_history'),
     path('supermarket/', views.supermarket, name='view_supermarket'),
     path('abuse/<slug:token>', views.report_share_abuse, name='view_report_share_abuse'),
-    path('test/', views.test, name='view_test'),
-    path('test2/', views.test2, name='view_test2'),
 
     path('import/', import_export.import_recipe, name='view_import'),
     path('import-response/<int:pk>/', import_export.import_response, name='view_import_response'),
@@ -189,3 +187,7 @@ for m in vue_models:
                 f'list/{url_name}/', c, name=f'list_{py_name}'
             )
         )
+
+if DEBUG:
+    urlpatterns.append(path('test/', views.test, name='view_test'))
+    urlpatterns.append(path('test2/', views.test2, name='view_test2'))

cookbook/views/api.py

@@ -2,11 +2,11 @@ import io
 import json
 import re
 import uuid
+from collections import OrderedDict
 
 import requests
 from annoying.decorators import ajax_request
 from annoying.functions import get_object_or_None
-from collections import OrderedDict
 from django.contrib import messages
 from django.contrib.auth.models import User
 from django.contrib.postgres.search import TrigramSimilarity
@@ -15,12 +15,12 @@ from django.core.files import File
 from django.db.models import Case, ProtectedError, Q, Value, When
 from django.db.models.fields.related import ForeignObjectRel
 from django.http import FileResponse, HttpResponse, JsonResponse
-from django_scopes import scopes_disabled
-from django.shortcuts import redirect, get_object_or_404
+from django.shortcuts import get_object_or_404, redirect
 from django.urls import reverse
 from django.utils.translation import gettext as _
+from django_scopes import scopes_disabled
 from icalendar import Calendar, Event
-from recipe_scrapers import scrape_me, WebsiteNotImplementedError, NoSchemaFoundInWildMode
+from recipe_scrapers import NoSchemaFoundInWildMode, WebsiteNotImplementedError, scrape_me
 from rest_framework import decorators, status, viewsets
 from rest_framework.exceptions import APIException, PermissionDenied
 from rest_framework.pagination import PageNumberPagination
@@ -28,41 +28,39 @@ from rest_framework.parsers import MultiPartParser
 from rest_framework.renderers import JSONRenderer, TemplateHTMLRenderer
 from rest_framework.response import Response
 from rest_framework.viewsets import ViewSetMixin
-from treebeard.exceptions import PathOverflow, InvalidMoveToDescendant, InvalidPosition
+from treebeard.exceptions import InvalidMoveToDescendant, InvalidPosition, PathOverflow
 
 from cookbook.helper.image_processing import handle_image
 from cookbook.helper.ingredient_parser import IngredientParser
-from cookbook.helper.permission_helper import (CustomIsAdmin, CustomIsGuest,
-                                               CustomIsOwner, CustomIsShare,
-                                               CustomIsShared, CustomIsUser,
+from cookbook.helper.permission_helper import (CustomIsAdmin, CustomIsGuest, CustomIsOwner,
+                                               CustomIsShare, CustomIsShared, CustomIsUser,
                                                group_required)
 from cookbook.helper.recipe_html_import import get_recipe_from_source
-
-from cookbook.helper.recipe_search import search_recipes, get_facet
+from cookbook.helper.recipe_search import get_facet, old_search, search_recipes
 from cookbook.helper.recipe_url_import import get_from_scraper
-from cookbook.models import (CookLog, Food, Ingredient, Keyword, MealPlan,
-                             MealType, Recipe, RecipeBook, ShoppingList,
-                             ShoppingListEntry, ShoppingListRecipe, Step,
-                             Storage, Sync, SyncLog, Unit, UserPreference,
-                             ViewLog, RecipeBookEntry, Supermarket, ImportLog, BookmarkletImport, SupermarketCategory, UserFile, ShareLink, SupermarketCategoryRelation, Automation)
+from cookbook.models import (Automation, BookmarkletImport, CookLog, Food, ImportLog, Ingredient,
+                             Keyword, MealPlan, MealType, Recipe, RecipeBook, RecipeBookEntry,
+                             ShareLink, ShoppingList, ShoppingListEntry, ShoppingListRecipe, Step,
+                             Storage, Supermarket, SupermarketCategory, SupermarketCategoryRelation,
+                             Sync, SyncLog, Unit, UserFile, UserPreference, ViewLog)
 from cookbook.provider.dropbox import Dropbox
 from cookbook.provider.local import Local
 from cookbook.provider.nextcloud import Nextcloud
-from cookbook.schemas import FilterSchema, RecipeSchema, TreeSchema, QueryOnlySchema
-from cookbook.serializer import (FoodSerializer, IngredientSerializer,
-                                 KeywordSerializer, MealPlanSerializer,
-                                 MealTypeSerializer, RecipeBookSerializer,
-                                 RecipeImageSerializer, RecipeSerializer,
-                                 ShoppingListAutoSyncSerializer,
-                                 ShoppingListEntrySerializer,
-                                 ShoppingListRecipeSerializer,
-                                 ShoppingListSerializer, StepSerializer,
-                                 StorageSerializer, SyncLogSerializer,
-                                 SyncSerializer, UnitSerializer,
-                                 UserNameSerializer, UserPreferenceSerializer,
-                                 ViewLogSerializer, CookLogSerializer, RecipeBookEntrySerializer,
-                                 RecipeOverviewSerializer, SupermarketSerializer, ImportLogSerializer,
-                                 BookmarkletImportSerializer, SupermarketCategorySerializer, UserFileSerializer, SupermarketCategoryRelationSerializer, AutomationSerializer)
+from cookbook.schemas import FilterSchema, QueryOnlySchema, RecipeSchema, TreeSchema
+from cookbook.serializer import (AutomationSerializer, BookmarkletImportSerializer,
+                                 CookLogSerializer, FoodSerializer, ImportLogSerializer,
+                                 IngredientSerializer, KeywordSerializer, MealPlanSerializer,
+                                 MealTypeSerializer, RecipeBookEntrySerializer,
+                                 RecipeBookSerializer, RecipeImageSerializer,
+                                 RecipeOverviewSerializer, RecipeSerializer,
+                                 ShoppingListAutoSyncSerializer, ShoppingListEntrySerializer,
+                                 ShoppingListRecipeSerializer, ShoppingListSerializer,
+                                 StepSerializer, StorageSerializer,
+                                 SupermarketCategoryRelationSerializer,
+                                 SupermarketCategorySerializer, SupermarketSerializer,
+                                 SyncLogSerializer, SyncSerializer, UnitSerializer,
+                                 UserFileSerializer, UserNameSerializer, UserPreferenceSerializer,
+                                 ViewLogSerializer)
 from recipes import settings
 
 
@@ -547,7 +545,16 @@ class RecipeViewSet(viewsets.ModelViewSet):
 
         return super().get_queryset()
 
+    def list(self, request, *args, **kwargs):
+        if self.request.GET.get('debug', False):
+            return JsonResponse({
+                'new': str(self.get_queryset().query),
+                'old': str(old_search(request).query)
+            })
+        return super().list(request, *args, **kwargs)
+
     # TODO write extensive tests for permissions
+
     def get_serializer_class(self):
         if self.action == 'list':
             return RecipeOverviewSerializer

cookbook/views/views.py

@@ -13,7 +13,7 @@ from django.contrib.auth.password_validation import validate_password
 from django.core.exceptions import ValidationError
 from django.db.models import Avg, Q, Sum
 from django.http import HttpResponseRedirect, JsonResponse
-from django.shortcuts import get_object_or_404, render, redirect
+from django.shortcuts import get_object_or_404, redirect, render
 from django.urls import reverse, reverse_lazy
 from django.utils import timezone
 from django.utils.translation import gettext as _
@@ -22,16 +22,15 @@ from django_tables2 import RequestConfig
 from rest_framework.authtoken.models import Token
 
 from cookbook.filters import RecipeFilter
-from cookbook.forms import (CommentForm, Recipe, User,
-                            UserCreateForm, UserNameForm, UserPreference,
-                            UserPreferenceForm, SpaceJoinForm, SpaceCreateForm,
-                            SearchPreferenceForm)
-from cookbook.helper.permission_helper import group_required, share_link_valid, has_group_permission
-from cookbook.models import (Comment, CookLog, InviteLink, MealPlan,
-                             ViewLog, ShoppingList, Space, Keyword, RecipeImport, Unit,
-                             Food, UserFile, ShareLink, SearchPreference, SearchFields)
-from cookbook.tables import (CookLogTable, RecipeTable, RecipeTableSmall,
-                             ViewLogTable, InviteLinkTable)
+from cookbook.forms import (CommentForm, Recipe, SearchPreferenceForm, SpaceCreateForm,
+                            SpaceJoinForm, User, UserCreateForm, UserNameForm, UserPreference,
+                            UserPreferenceForm)
+from cookbook.helper.permission_helper import group_required, has_group_permission, share_link_valid
+from cookbook.models import (Comment, CookLog, Food, InviteLink, Keyword, MealPlan, RecipeImport,
+                             SearchFields, SearchPreference, ShareLink, ShoppingList, Space, Unit,
+                             UserFile, ViewLog)
+from cookbook.tables import (CookLogTable, InviteLinkTable, RecipeTable, RecipeTableSmall,
+                             ViewLogTable)
 from cookbook.views.data import Object
 from recipes.version import BUILD_REF, VERSION_NUMBER
 
@@ -331,10 +330,10 @@ def user_settings(request):
                 if not sp:
                     sp = SearchPreferenceForm(user=request.user)
                 fields_searched = (
-                        len(search_form.cleaned_data['icontains'])
-                        + len(search_form.cleaned_data['istartswith'])
-                        + len(search_form.cleaned_data['trigram'])
-                        + len(search_form.cleaned_data['fulltext'])
+                    len(search_form.cleaned_data['icontains'])
+                    + len(search_form.cleaned_data['istartswith'])
+                    + len(search_form.cleaned_data['trigram'])
+                    + len(search_form.cleaned_data['fulltext'])
                 )
                 if fields_searched == 0:
                     search_form.add_error(None, _('You must select at least one field to search!'))
@@ -382,7 +381,7 @@ def user_settings(request):
     if up:
         preference_form = UserPreferenceForm(instance=up, space=request.space)
     else:
-        preference_form = UserPreferenceForm( space=request.space)
+        preference_form = UserPreferenceForm(space=request.space)
 
     fields_searched = len(sp.icontains.all()) + len(sp.istartswith.all()) + len(sp.trigram.all()) + len(
         sp.fulltext.all())