Merge branch 'develop' into feature/vue3

# Conflicts: # recipes/settings.py # requirements.txt
2026-01-01 04:10:06 -05:00 · 2024-08-01 15:03:40 +02:00
parent 65b451b465 fbadf14b58
commit 0e44243e55
17 changed files with 198 additions and 229 deletions
--- a/cookbook/connectors/connector_manager.py
+++ b/cookbook/connectors/connector_manager.py
@@ -5,6 +5,7 @@ import threading
 from asyncio import Task
 from dataclasses import dataclass
 from enum import Enum
+from logging import Logger
 from types import UnionType
 from typing import List, Any, Dict, Optional, Type

@@ -39,10 +40,12 @@ class Work:
 # 4. Work is marked as consumed, and next entry of the queue is consumed.
 # Each 'Work' is processed in sequential by the worker, so the throughput is about [workers * the slowest connector]
 class ConnectorManager:
+    _logger: Logger
    _queue: queue.Queue
    _listening_to_classes = REGISTERED_CLASSES | ConnectorConfig

    def __init__(self):
+        self._logger = logging.getLogger("recipes.connector")
        self._queue = queue.Queue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE)
        self._worker = threading.Thread(target=self.worker, args=(0, self._queue,), daemon=True)
        self._worker.start()
@@ -65,7 +68,7 @@ class ConnectorManager:
        try:
            self._queue.put_nowait(Work(instance, action_type))
        except queue.Full:
-            logging.info(f"queue was full, so skipping {action_type} of type {type(instance)}")
+            self._logger.info(f"queue was full, so skipping {action_type} of type {type(instance)}")
            return

    def stop(self):
@@ -74,10 +77,12 @@ class ConnectorManager:

    @staticmethod
    def worker(worker_id: int, worker_queue: queue.Queue):
+        logger = logging.getLogger("recipes.connector.worker")
+
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)

-        logging.info(f"started ConnectionManager worker {worker_id}")
+        logger.info(f"started ConnectionManager worker {worker_id}")

        # When multiple workers are used, please make sure the cache is shared across all threads, otherwise it might lead to un-expected behavior.
        _connectors_cache: Dict[int, List[Connector]] = dict()
@@ -91,6 +96,8 @@ class ConnectorManager:
            if item is None:
                break

+            logger.debug(f"received {item.instance=} with {item.actionType=}")
+
            # If a Connector was changed/updated, refresh connector from the database for said space
            refresh_connector_cache = isinstance(item.instance, ConnectorConfig)

@@ -111,7 +118,7 @@ class ConnectorManager:
                        try:
                            connector: Optional[Connector] = ConnectorManager.get_connected_for_config(config)
                        except BaseException:
-                            logging.exception(f"failed to initialize {config.name}")
+                            logger.exception(f"failed to initialize {config.name}")
                            continue

                        if connector is not None:
@@ -123,10 +130,12 @@ class ConnectorManager:
                worker_queue.task_done()
                continue

+            logger.debug(f"running {len(connectors)} connectors for {item.instance=} with {item.actionType=}")
+
            loop.run_until_complete(run_connectors(connectors, space, item.instance, item.actionType))
            worker_queue.task_done()

-        logging.info(f"terminating ConnectionManager worker {worker_id}")
+        logger.info(f"terminating ConnectionManager worker {worker_id}")

        asyncio.set_event_loop(None)
        loop.close()
--- a/cookbook/connectors/homeassistant.py
+++ b/cookbook/connectors/homeassistant.py
@@ -17,10 +17,11 @@ class HomeAssistant(Connector):
        if not config.token or not config.url or not config.todo_entity:
            raise ValueError("config for HomeAssistantConnector in incomplete")

+        self._logger = logging.getLogger(f"recipes.connector.homeassistant.{config.name}")
+
        if config.url[-1] != "/":
            config.url += "/"
        self._config = config
-        self._logger = logging.getLogger("connector.HomeAssistant")

    async def homeassistant_api_call(self, method: str, path: str, data: Dict) -> str:
        headers = {
@@ -37,7 +38,7 @@ class HomeAssistant(Connector):

        item, description = _format_shopping_list_entry(shopping_list_entry)

-        logging.debug(f"adding {item=} to {self._config.name}")
+        self._logger.debug(f"adding {item=}")

        data = {
            "entity_id": self._config.todo_entity,
@@ -48,7 +49,7 @@ class HomeAssistant(Connector):
        try:
            await self.homeassistant_api_call("POST", "services/todo/add_item", data)
        except ClientError as err:
-            self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}")
+            self._logger.warning(f"received an exception from the api: {err=}, {type(err)=}")

    async def on_shopping_list_entry_updated(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None:
        if not self._config.on_shopping_list_entry_updated_enabled:
@@ -66,7 +67,7 @@ class HomeAssistant(Connector):

        item, _ = _format_shopping_list_entry(shopping_list_entry)

-        logging.debug(f"removing {item=} from {self._config.name}")
+        self._logger.debug(f"removing {item=}")

        data = {
            "entity_id": self._config.todo_entity,
@@ -77,7 +78,7 @@ class HomeAssistant(Connector):
            await self.homeassistant_api_call("POST", "services/todo/remove_item", data)
        except ClientError as err:
            # This error will always trigger if the item is not present/found
-            self._logger.debug(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}")
+            self._logger.debug(f"received an exception from the api: {err=}, {type(err)=}")

    async def close(self) -> None:
        pass
--- a/cookbook/helper/scrapers/cooksillustrated.py
+++ b/cookbook/helper/scrapers/cooksillustrated.py
@@ -1,68 +0,0 @@
-import json
-from recipe_scrapers._abstract import AbstractScraper
-
-
-class CooksIllustrated(AbstractScraper):
-    @classmethod
-    def host(cls, site='cooksillustrated'):
-        return {
-            'cooksillustrated': f"{site}.com",
-            'americastestkitchen': f"{site}.com",
-            'cookscountry': f"{site}.com",
-        }.get(site)
-
-    def title(self):
-        return self.schema.title()
-
-    def image(self):
-        return self.schema.image()
-
-    def total_time(self):
-        if not self.recipe:
-            self.get_recipe()
-        return self.recipe['recipeTimeNote']
-
-    def yields(self):
-        if not self.recipe:
-            self.get_recipe()
-        return self.recipe['yields']
-
-    def ingredients(self):
-        if not self.recipe:
-            self.get_recipe()
-        ingredients = []
-        for group in self.recipe['ingredientGroups']:
-            ingredients += group['fields']['recipeIngredientItems']
-        return [
-            "{} {} {}{}".format(
-                i['fields']['qty'] or '',
-                i['fields']['measurement'] or '',
-                i['fields']['ingredient']['fields']['title'] or '',
-                i['fields']['postText'] or ''
-            )
-            for i in ingredients
-        ]
-
-    def instructions(self):
-        if not self.recipe:
-            self.get_recipe()
-        if self.recipe.get('headnote', False):
-            i = ['Note: ' + self.recipe.get('headnote', '')]
-        else:
-            i = []
-        return "\n".join(
-            i
-            + [self.recipe.get('whyThisWorks', '')]
-            + [
-                instruction['fields']['content']
-                for instruction in self.recipe['instructions']
-            ]
-        )
-
-    def nutrients(self):
-        raise NotImplementedError("This should be implemented.")
-
-    def get_recipe(self):
-        j = json.loads(self.soup.find(type='application/json').string)
-        name = list(j['props']['initialState']['content']['documents'])[0]
-        self.recipe = j['props']['initialState']['content']['documents'][name]
--- a/cookbook/helper/scrapers/scrapers.py
+++ b/cookbook/helper/scrapers/scrapers.py
@@ -1,43 +0,0 @@
-from json import JSONDecodeError
-
-from bs4 import BeautifulSoup
-from recipe_scrapers import SCRAPERS, get_host_name
-from recipe_scrapers._factory import SchemaScraperFactory
-from recipe_scrapers._schemaorg import SchemaOrg
-
-from .cooksillustrated import CooksIllustrated
-
-CUSTOM_SCRAPERS = {
-    CooksIllustrated.host(site="cooksillustrated"): CooksIllustrated,
-    CooksIllustrated.host(site="americastestkitchen"): CooksIllustrated,
-    CooksIllustrated.host(site="cookscountry"): CooksIllustrated,
-}
-SCRAPERS.update(CUSTOM_SCRAPERS)
-
-
-def text_scraper(text, url=None):
-    domain = None
-    if url:
-        domain = get_host_name(url)
-    if domain in SCRAPERS:
-        scraper_class = SCRAPERS[domain]
-    else:
-        scraper_class = SchemaScraperFactory.SchemaScraper
-
-    class TextScraper(scraper_class):
-        def __init__(
-            self,
-            html=None,
-            url=None,
-        ):
-            self.supported_only = False
-            self.meta_http_equiv = False
-            self.soup = BeautifulSoup(html, "html.parser")
-            self.url = url
-            self.recipe = None
-            try:
-                self.schema = SchemaOrg(html)
-            except (JSONDecodeError, AttributeError):
-                pass
-
-    return TextScraper(url=url, html=text)
--- a/cookbook/integration/cookbookapp.py
+++ b/cookbook/integration/cookbookapp.py
@@ -7,7 +7,7 @@ import validators
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import (get_from_scraper, get_images_from_soup,
                                               iso_duration_to_minutes)
-from cookbook.helper.scrapers.scrapers import text_scraper
+from recipe_scrapers import scrape_html
 from cookbook.integration.integration import Integration
 from cookbook.models import Ingredient, Recipe, Step

@@ -20,7 +20,7 @@ class CookBookApp(Integration):
    def get_recipe_from_file(self, file):
        recipe_html = file.getvalue().decode("utf-8")

-        scrape = text_scraper(text=recipe_html)
+        scrape = scrape_html(html=recipe_html, org_url="https://cookbookapp.import", supported_only=False)
        recipe_json = get_from_scraper(scrape, self.request)
        images = list(dict.fromkeys(get_images_from_soup(scrape.soup, None)))

--- a/cookbook/locale/de/LC_MESSAGES/django.po
+++ b/cookbook/locale/de/LC_MESSAGES/django.po
@@ -15,16 +15,16 @@ msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2024-07-08 17:43+0200\n"
-"PO-Revision-Date: 2024-05-11 00:33+0000\n"
-"Last-Translator: Jakob Priesner <jakob.priesner@outlook.de>\n"
-"Language-Team: German <http://translate.tandoor.dev/projects/tandoor/recipes-"
-"backend/de/>\n"
+"PO-Revision-Date: 2024-07-31 13:05+0000\n"
+"Last-Translator: vabene1111 <vabene1234@googlemail.com>\n"
+"Language-Team: German <http://translate.tandoor.dev/projects/tandoor/"
+"recipes-backend/de/>\n"
 "Language: de\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
-"X-Generator: Weblate 5.4.2\n"
+"X-Generator: Weblate 5.6.2\n"

 #: .\cookbook\forms.py:45
 msgid ""
@@ -395,7 +395,7 @@ msgstr "Sektion"

 #: .\cookbook\management\commands\fix_duplicate_properties.py:15
 msgid "Fixes foods with "
-msgstr ""
+msgstr "Behebt Lebensmittel mit "

 #: .\cookbook\management\commands\rebuildindex.py:14
 msgid "Rebuilds full text search index on Recipe"
--- a/cookbook/locale/he/LC_MESSAGES/django.po
+++ b/cookbook/locale/he/LC_MESSAGES/django.po
@@ -8,27 +8,27 @@ msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2024-07-08 17:43+0200\n"
-"PO-Revision-Date: 2023-11-15 08:20+0000\n"
-"Last-Translator: avi meyer <avmeyer@gmail.com>\n"
-"Language-Team: Hebrew <http://translate.tandoor.dev/projects/tandoor/recipes-"
-"backend/he/>\n"
+"PO-Revision-Date: 2024-07-28 08:38+0000\n"
+"Last-Translator: dudu dor <dudpon@gmail.com>\n"
+"Language-Team: Hebrew <http://translate.tandoor.dev/projects/tandoor/"
+"recipes-backend/he/>\n"
 "Language: he\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=4; plural=(n == 1) ? 0 : ((n == 2) ? 1 : ((n > 10 && "
 "n % 10 == 0) ? 2 : 3));\n"
-"X-Generator: Weblate 4.15\n"
+"X-Generator: Weblate 5.4.2\n"

 #: .\cookbook\forms.py:45
 msgid ""
 "Both fields are optional. If none are given the username will be displayed "
 "instead"
-msgstr ""
+msgstr "שני השדות אופציונלים. אם שני השדות ריקים, שם המשתמש יוצג במקום."

 #: .\cookbook\forms.py:62 .\cookbook\forms.py:246
 msgid "Name"
-msgstr ""
+msgstr "שם"

 #: .\cookbook\forms.py:62 .\cookbook\forms.py:246 .\cookbook\views\lists.py:103
 msgid "Keywords"
@@ -36,23 +36,23 @@ msgstr "מילות מפתח"

 #: .\cookbook\forms.py:62
 msgid "Preparation time in minutes"
-msgstr ""
+msgstr "זמן הכנה בדקות"

 #: .\cookbook\forms.py:62
 msgid "Waiting time (cooking/baking) in minutes"
-msgstr ""
+msgstr "זמן המתנה (בישול/אפייה) בדקות"

 #: .\cookbook\forms.py:63 .\cookbook\forms.py:222 .\cookbook\forms.py:246
 msgid "Path"
-msgstr ""
+msgstr "נתיב"

 #: .\cookbook\forms.py:63
 msgid "Storage UID"
-msgstr ""
+msgstr "אחסון UID"

 #: .\cookbook\forms.py:93
 msgid "Default"
-msgstr ""
+msgstr "ברירת מחדל"

 #: .\cookbook\forms.py:121
 msgid ""
@@ -62,21 +62,23 @@ msgstr ""

 #: .\cookbook\forms.py:143
 msgid "Add your comment: "
-msgstr ""
+msgstr "הוף את ההערות שלך:- "

 #: .\cookbook\forms.py:151
 msgid "Leave empty for dropbox and enter app password for nextcloud."
-msgstr ""
+msgstr "השאר ריק עבור dropbox והכנס סיסמאת יישום עבור nextcloud."

 #: .\cookbook\forms.py:154
 msgid "Leave empty for nextcloud and enter api token for dropbox."
-msgstr ""
+msgstr "השאר ריק עבור nextcloud והכנס טוקן API עבור dropbox."

 #: .\cookbook\forms.py:160
 msgid ""
 "Leave empty for dropbox and enter only base url for nextcloud (<code>/remote."
 "php/webdav/</code> is added automatically)"
 msgstr ""
+"השאר ריק עבור dropbox וכנס רק URL בסיסי עבור nextcloud (<code>/remote.php/"
+"webdav/</code> נוסף אוטומטי)"

 #: .\cookbook\forms.py:188
 msgid ""
@@ -86,49 +88,49 @@ msgstr ""

 #: .\cookbook\forms.py:193
 msgid "Something like http://homeassistant.local:8123/api"
-msgstr ""
+msgstr "משהו דומה לhttp://homeassistant.local:8123/api"

 #: .\cookbook\forms.py:205
 msgid "http://homeassistant.local:8123/api for example"
-msgstr ""
+msgstr "לדוגמא http://homeassistant.local:8123/api"

 #: .\cookbook\forms.py:222 .\cookbook\views\edit.py:117
 msgid "Storage"
-msgstr ""
+msgstr "אחסון"

 #: .\cookbook\forms.py:222
 msgid "Active"
-msgstr ""
+msgstr "פעיל"

 #: .\cookbook\forms.py:226
 msgid "Search String"
-msgstr ""
+msgstr "מחרוזת חיפוש"

 #: .\cookbook\forms.py:246
 msgid "File ID"
-msgstr ""
+msgstr "ID של הקובץ"

 #: .\cookbook\forms.py:262
 msgid "Maximum number of users for this space reached."
-msgstr ""
+msgstr "המספר המקסימלי של משתמשים עבור מרחב זה נוצל."

 #: .\cookbook\forms.py:268
 msgid "Email address already taken!"
-msgstr ""
+msgstr "כתובת האימייל כבר בשימוש!"

 #: .\cookbook\forms.py:275
 msgid ""
 "An email address is not required but if present the invite link will be sent "
 "to the user."
-msgstr ""
+msgstr "כתובת אימייל לא נדרשת אבל אם קיימת, קישור השיתוף ישלח למשתמש."

 #: .\cookbook\forms.py:287
 msgid "Name already taken."
-msgstr ""
+msgstr "שם כבר בשימוש."

 #: .\cookbook\forms.py:298
 msgid "Accept Terms and Privacy"
-msgstr ""
+msgstr "הסכם לתנאים ולפרטיות"

 #: .\cookbook\forms.py:332
 msgid ""
--- a/cookbook/tests/api/test_api_import_log.py
+++ b/cookbook/tests/api/test_api_import_log.py
@@ -51,7 +51,7 @@ def test_list_space(obj_1, obj_2, u1_s1, u1_s2, space_2):
    ['g1_s2', 403],
    ['u1_s2', 404],
    ['a1_s2', 404],
-])
+], ids=str)
 def test_update(arg, request, obj_1):
    c = request.getfixturevalue(arg[0])
    r = c.patch(
--- a/cookbook/tests/other/test_recipe_full_text_search.py
+++ b/cookbook/tests/other/test_recipe_full_text_search.py
@@ -272,12 +272,12 @@ def test_search_units(found_recipe, recipes, u1_s1, space_1):
        ('fuzzy_lookups', True), ('fuzzy_lookups', False)
    ],
    [('unaccent', True), ('unaccent', False)]
-), indirect=['user1'])
+), indirect=['user1'], ids=str)
@pytest.mark.parametrize("found_recipe, param_type", [
    ({'unit': True}, 'unit'),
    ({'keyword': True}, 'keyword'),
    ({'food': True}, 'food'),
-], indirect=['found_recipe'])
+], indirect=['found_recipe'], ids=str)
 def test_fuzzy_lookup(found_recipe, recipes, param_type, user1, space_1):
    with scope(space=space_1):
        list_url = f'api:{param_type}-list'
@@ -306,14 +306,14 @@ def test_fuzzy_lookup(found_recipe, recipes, param_type, user1, space_1):
        ('istartswith', True), ('istartswith', False),
    ],
    [('unaccent', True), ('unaccent', False)]
-), indirect=['user1'])
+), indirect=['user1'], ids=str)
@pytest.mark.parametrize("found_recipe", [
    ({'name': True}),
    ({'description': True}),
    ({'instruction': True}),
    ({'keyword': True}),
    ({'food': True}),
-], indirect=['found_recipe'])
+], indirect=['found_recipe'], ids=str)
 # user array contains: user client, expected count of search, expected count of mispelled search, search string, mispelled search string, user search preferences
 def test_search_string(found_recipe, recipes, user1, space_1):
    with scope(space=space_1):
--- a/cookbook/tests/other/test_url_import.py
+++ b/cookbook/tests/other/test_url_import.py
@@ -19,6 +19,23 @@ DATA_DIR = "cookbook/tests/other/test_data/"
 # plus the test that previously existed
 # plus the custom scraper that was created
 # plus any specific defects discovered along the way
+RECIPES = [
+    ALLRECIPES,
+    AMERICAS_TEST_KITCHEN,
+    CHEF_KOCH,
+    CHEF_KOCH2,  # test for empty ingredient in ingredient_parser
+    COOKPAD,
+    COOKS_COUNTRY,
+    DELISH,
+    FOOD_NETWORK,
+    GIALLOZAFFERANO,
+    JOURNAL_DES_FEMMES,
+    MADAME_DESSERT,  # example of json only source
+    MARMITON,
+    TASTE_OF_HOME,
+    THE_SPRUCE_EATS,  # example of non-json recipes_scraper
+    TUDOGOSTOSO,
+]


@pytest.mark.parametrize("arg", [
@@ -32,29 +49,7 @@ def test_import_permission(arg, request):
    assert c.get(reverse(IMPORT_SOURCE_URL)).status_code == arg[1]


-@pytest.mark.parametrize("arg", [
-    ALLRECIPES,
-    # test of custom scraper ATK
-    AMERICAS_TEST_KITCHEN,
-    CHEF_KOCH,
-    # test for empty ingredient in ingredient_parser
-    CHEF_KOCH2,
-    COOKPAD,
-    # test of custom scraper ATK
-    COOKS_COUNTRY,
-    DELISH,
-    FOOD_NETWORK,
-    GIALLOZAFFERANO,
-    JOURNAL_DES_FEMMES,
-    # example of recipes_scraper in with wildmode
-    # example of json only source
-    MADAME_DESSERT,
-    MARMITON,
-    TASTE_OF_HOME,
-    # example of non-json recipes_scraper
-    THE_SPRUCE_EATS,  # TODO seems to be broken in recipe scrapers
-    TUDOGOSTOSO,
-])
+@pytest.mark.parametrize("arg", RECIPES, ids=[x['file'][0] for x in RECIPES])
 def test_recipe_import(arg, u1_s1):
    url = arg['url']
    for f in list(arg['file']):  # url and files get popped later
--- a/cookbook/views/api.py
+++ b/cookbook/views/api.py
@@ -67,7 +67,6 @@ from cookbook.helper.permission_helper import (CustomIsAdmin, CustomIsOwner, Cus
                                               )
 from cookbook.helper.recipe_search import RecipeSearch
 from cookbook.helper.recipe_url_import import clean_dict, get_from_youtube_scraper, get_images_from_soup
-from cookbook.helper.scrapers.scrapers import text_scraper
 from cookbook.helper.shopping_helper import RecipeShoppingEditor, shopping_helper
 from cookbook.models import (Automation, BookmarkletImport, ConnectorConfig, CookLog, CustomFilter, ExportLog, Food, FoodInheritField, FoodProperty, ImportLog, Ingredient,
                             InviteLink, Keyword, MealPlan, MealType, Property, PropertyType, Recipe, RecipeBook, RecipeBookEntry, ShareLink, ShoppingListEntry,
@@ -1501,7 +1500,10 @@ class RecipeUrlImportView(APIView):
                else:
                    try:
                        if validators.url(url, public=True):
-                            html = requests.get(url).content
+                            html = requests.get(
+                                url,
+                                headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0"}
+                            ).content
                            scrape = scrape_html(org_url=url, html=html, supported_only=False)
                        else:
                            return Response({'error': True, 'msg': _('Invalid Url')}, status=status.HTTP_400_BAD_REQUEST)
@@ -1521,9 +1523,9 @@ class RecipeUrlImportView(APIView):
                    data = "<script type='application/ld+json'>" + json.dumps(data_json) + "</script>"
                except JSONDecodeError:
                    pass
-                scrape = text_scraper(text=data, url=url)
-                if not url and (found_url := scrape.schema.data.get('url', None)):
-                    scrape = text_scraper(text=data, url=found_url)
+                scrape = scrape_html(html=data, org_url=url, supported_only=False)
+                if not url and (found_url := scrape.schema.data.get('url', 'https://urlnotfound.none')):
+                    scrape = scrape_html(text=data, url=found_url, supported_only=False)

            if scrape:
                return Response({