Merge branch 'develop' into feature/importer_to_vue

# Conflicts:
#	cookbook/helper/recipe_url_import.py

cookbook/helper/recipe_search.py

@@ -13,8 +13,8 @@ from cookbook.filters import RecipeFilter
 from cookbook.helper.HelperFunctions import Round, str2bool
 from cookbook.helper.permission_helper import has_group_permission
 from cookbook.managers import DICTIONARY
-from cookbook.models import (CookLog, CustomFilter, Food, Keyword, Recipe, SearchFields,
-                             SearchPreference, ViewLog, RecipeBook)
+from cookbook.models import (CookLog, CustomFilter, Food, Keyword, Recipe, RecipeBook, SearchFields,
+                             SearchPreference, ViewLog)
 from recipes import settings
 
 
@@ -28,7 +28,7 @@ class RecipeSearch():
         self._queryset = None
         if f := params.get('filter', None):
             custom_filter = CustomFilter.objects.filter(id=f, space=self._request.space).filter(Q(created_by=self._request.user) |
-                                                                                         Q(shared=self._request.user) | Q(recipebook__shared=self._request.user)).first()
+                                                                                                Q(shared=self._request.user) | Q(recipebook__shared=self._request.user)).first()
             if custom_filter:
                 self._params = {**json.loads(custom_filter.search)}
                 self._original_params = {**(params or {})}
@@ -40,7 +40,7 @@ class RecipeSearch():
             self._search_prefs = request.user.searchpreference
         else:
             self._search_prefs = SearchPreference()
-        self._string = params.get('query').strip() if params.get('query', None) else None
+        self._string = self._params.get('query').strip() if self._params.get('query', None) else None
         self._rating = self._params.get('rating', None)
         self._keywords = {
             'or': self._params.get('keywords_or', None) or self._params.get('keywords', None),
@@ -89,7 +89,10 @@ class RecipeSearch():
 
         self._search_type = self._search_prefs.search or 'plain'
         if self._string:
-            self._unaccent_include = self._search_prefs.unaccent.values_list('field', flat=True)
+            if self._postgres:
+                self._unaccent_include = self._search_prefs.unaccent.values_list('field', flat=True)
+            else:
+                self._unaccent_include = []
             self._icontains_include = [x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.icontains.values_list('field', flat=True)]
             self._istartswith_include = [x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.istartswith.values_list('field', flat=True)]
             self._trigram_include = None
@@ -205,7 +208,7 @@ class RecipeSearch():
                 else:
                     self._queryset = self._queryset.annotate(simularity=Coalesce(Subquery(simularity), 0.0))
             if self._sort_includes('score') and self._fulltext_include and self._fuzzy_match is not None:
-                self._queryset = self._queryset.annotate(score=Sum(F('rank')+F('simularity')))
+                self._queryset = self._queryset.annotate(score=F('rank')+F('simularity'))
         else:
             query_filter = Q()
             for f in [x + '__unaccent__iexact' if x in self._unaccent_include else x + '__iexact' for x in SearchFields.objects.all().values_list('field', flat=True)]:
@@ -726,9 +729,8 @@ class RecipeFacet():
         return self.get_facets()
 
     def _recipe_count_queryset(self, field, depth=1, steplen=4):
-        return Recipe.objects.filter(**{f'{field}__path__startswith': OuterRef('path')}, id__in=self._recipe_list, space=self._request.space
-                                     ).values(child=Substr(f'{field}__path',  1, steplen*depth)
-                                              ).annotate(count=Count('pk', distinct=True)).values('count')
+        return Recipe.objects.filter(**{f'{field}__path__startswith': OuterRef('path'), f'{field}__depth__gte': depth}, id__in=self._recipe_list, space=self._request.space
+                                     ).annotate(count=Coalesce(Func('pk', function='Count'), 0)).values('count')
 
     def _keyword_queryset(self, queryset, keyword=None):
         depth = getattr(keyword, 'depth', 0) + 1

cookbook/helper/recipe_url_import.py

@@ -4,8 +4,10 @@ from html import unescape
 from unicodedata import decomposition
 
 from django.utils.dateparse import parse_duration
+from django.utils.translation import gettext as _
 from isodate import parse_duration as iso_parse_duration
 from isodate.isoerror import ISO8601Error
+from recipe_scrapers._utils import get_minutes
 
 from cookbook.helper import recipe_url_import as helper
 from cookbook.helper.ingredient_parser import IngredientParser
@@ -29,9 +31,14 @@ def get_from_scraper(scrape, request):
             recipe_json['name'] = ''
 
     try:
-        description = scrape.schema.data.get("description") or ''
+        description = scrape.description()  or None
     except Exception:
-        description = ''
+        description = None
+    if not description:
+        try:
+            description = scrape.schema.data.get("description") or ''
+        except Exception:
+            description = ''
 
     recipe_json['description'] = parse_description(description)[:512]
     recipe_json['internal'] = True
@@ -53,13 +60,19 @@ def get_from_scraper(scrape, request):
     recipe_json['servings'] = max(servings, 1)
 
     try:
-        recipe_json['working_time'] = get_minutes(scrape.schema.data.get("prepTime")) or 0
+        recipe_json['working_time'] = get_minutes(scrape.prep_time()) or 0
     except Exception:
-        recipe_json['working_time'] = 0
+        try:
+            recipe_json['working_time'] = get_minutes(scrape.schema.data.get("prepTime")) or 0
+        except Exception:
+            recipe_json['working_time'] = 0
     try:
-        recipe_json['waiting_time'] = get_minutes(scrape.schema.data.get("cookTime")) or 0
+        recipe_json['waiting_time'] = get_minutes(scrape.cook_time()) or 0
     except Exception:
-        recipe_json['waiting_time'] = 0
+        try:
+            recipe_json['waiting_time'] = get_minutes(scrape.schema.data.get("cookTime")) or 0
+        except Exception:
+            recipe_json['waiting_time'] = 0
 
     if recipe_json['working_time'] + recipe_json['waiting_time'] == 0:
         try:
@@ -87,15 +100,23 @@ def get_from_scraper(scrape, request):
     except Exception:
         pass
     try:
-        if scrape.schema.data.get('recipeCategory'):
-            keywords += listify_keywords(scrape.schema.data.get("recipeCategory"))
+        if scrape.category():
+            keywords += listify_keywords(scrape.category())
     except Exception:
-        pass
+        try:
+            if scrape.schema.data.get('recipeCategory'):
+                keywords += listify_keywords(scrape.schema.data.get("recipeCategory"))
+        except Exception:
+            pass
     try:
-        if scrape.schema.data.get('recipeCuisine'):
-            keywords += listify_keywords(scrape.schema.data.get("recipeCuisine"))
+        if scrape.cuisine():
+            keywords += listify_keywords(scrape.cuisine())
     except Exception:
-        pass
+        try:
+            if scrape.schema.data.get('recipeCuisine'):
+                keywords += listify_keywords(scrape.schema.data.get("recipeCuisine"))
+        except Exception:
+            pass
     try:
         recipe_json['keywords'] = parse_keywords(list(set(map(str.casefold, keywords))), request.space)
     except AttributeError:
@@ -142,8 +163,8 @@ def get_from_scraper(scrape, request):
     except Exception:
         pass
 
-    if scrape.url:
-        recipe_json['source_url'] = scrape.url
+    if scrape.canonical_url():
+        recipe_json['source_url'] = scrape.canonical_url()
     return recipe_json
 
 
@@ -307,56 +328,6 @@ def normalize_string(string):
     return unescaped_string
 
 
-# TODO deprecate when merged into recipe_scapers
-
-
-def get_minutes(time_text):
-    if time_text is None:
-        return 0
-    TIME_REGEX = re.compile(
-        r"(\D*(?P<hours>\d*.?(\s\d)?\/?\d+)\s*(hours|hrs|hr|h|óra))?(\D*(?P<minutes>\d+)\s*(minutes|mins|min|m|perc))?",
-        re.IGNORECASE,
-    )
-    try:
-        return int(time_text)
-    except Exception:
-        pass
-
-    if time_text.startswith("P") and "T" in time_text:
-        time_text = time_text.split("T", 2)[1]
-    if "-" in time_text:
-        time_text = time_text.split("-", 2)[
-            1
-        ]  # sometimes formats are like this: '12-15 minutes'
-    if " to " in time_text:
-        time_text = time_text.split("to", 2)[
-            1
-        ]  # sometimes formats are like this: '12 to 15 minutes'
-
-    empty = ''
-    for x in time_text:
-        if 'fraction' in decomposition(x):
-            f = decomposition(x[-1:]).split()
-            empty += f" {f[1].replace('003', '')}/{f[3].replace('003', '')}"
-        else:
-            empty += x
-    time_text = empty
-    matched = TIME_REGEX.search(time_text)
-
-    minutes = int(matched.groupdict().get("minutes") or 0)
-
-    if "/" in (hours := matched.groupdict().get("hours") or ''):
-        number = hours.split(" ")
-        if len(number) == 2:
-            minutes += 60 * int(number[0])
-        fraction = number[-1:][0].split("/")
-        minutes += 60 * float(int(fraction[0]) / int(fraction[1]))
-    else:
-        minutes += 60 * float(hours)
-
-    return int(minutes)
-
-
 def iso_duration_to_minutes(string):
     match = re.match(
         r'P((?P<years>\d+)Y)?((?P<months>\d+)M)?((?P<weeks>\d+)W)?((?P<days>\d+)D)?T((?P<hours>\d+)H)?((?P<minutes>\d+)M)?((?P<seconds>\d+)S)?',

cookbook/helper/shopping_helper.py

@@ -35,7 +35,7 @@ def shopping_helper(qs, request):
         qs = qs.filter(Q(checked=False) | Q(completed_at__gte=week_ago))
         supermarket_order = ['checked'] + supermarket_order
 
-    return qs.order_by(*supermarket_order).select_related('unit', 'food', 'ingredient', 'created_by', 'list_recipe', 'list_recipe__mealplan', 'list_recipe__recipe')
+    return qs.distinct().order_by(*supermarket_order).select_related('unit', 'food', 'ingredient', 'created_by', 'list_recipe', 'list_recipe__mealplan', 'list_recipe__recipe')
 
 
 class RecipeShoppingEditor():