Merge branch 'develop'

Currently translated at 53.8% (307 of 570 strings)

Translation: Tandoor/Recipes Frontend
Translate-URL: http://translate.tandoor.dev/projects/tandoor/recipes-frontend/pt/

.env.template

@@ -6,6 +6,9 @@
 # random secret key, use for example `base64 /dev/urandom | head -c50` to generate one
 SECRET_KEY=
 
+# allowed hosts (see documentation), should be set to your hostname(s) but might be * (default) for some proxies/providers
+# ALLOWED_HOSTS=recipes.mydomain.com
+
 # add only a database password if you want to run with the default postgres, otherwise change settings accordingly
 DB_ENGINE=django.db.backends.postgresql
 POSTGRES_HOST=db_recipes

.flake8

@@ -0,0 +1,22 @@
+[flake8]
+extend-ignore =
+    # Whitespace before ':' - Required for black compatibility
+    E203,
+    # Line break occurred before a binary operator - Required for black compatibility
+    W503,
+    # Comparison to False should be 'if cond is False:' or 'if not cond:'
+    E712
+exclude =
+    .git,
+    **/__pycache__,
+        **/.git,
+        **/.svn,
+        **/.hg,
+        **/CVS,
+        **/.DS_Store,
+        .vscode,
+        **/*.pyc
+per-file-ignores=
+    cookbook/apps.py:F401
+max-line-length = 179
+

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
 
         steps:
             - uses: actions/checkout@v4
-            - uses: awalsh128/cache-apt-pkgs-action@v1.4.1
+            - uses: awalsh128/cache-apt-pkgs-action@v1.4.2
               with:
                   packages: libsasl2-dev python3-dev libldap2-dev libssl-dev
                   version: 1.0

.gitignore

@@ -47,6 +47,11 @@ docs/reports/**
 
 # Django stuff:
 *.log
+mediafiles/
+*.sqlite3*
+staticfiles/
+postgresql/
+data/
 
 # Sphinx documentation
 docs/_build/
@@ -59,32 +64,25 @@ target/
 
 \.idea/dataSources\.local\.xml
 
-venv/
-
-mediafiles/
-
-*.sqlite3*
-
 \.idea/workspace\.xml
 
 \.idea/misc\.xml
 
 # Deployment
-
 \.env
-staticfiles/
-postgresql/
-data/
-
 
+cookbook/static/vue
+vue/webpack-stats.json
 /docker-compose.override.yml
 vue/node_modules
 plugins
-vetur.config.js
-cookbook/static/vue
-vue/webpack-stats.json
+vue3/node_modules
 cookbook/templates/sw.js
-.prettierignore
 vue/.yarn
 vue3/.vite
-vue3/node_modules
+
+# Configs
+vetur.config.js
+venv/
+.idea/easy-i18n.xml
+cookbook/static/vue3

.idea/watcherTasks.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectTasksOptions">
+    <TaskOptions isEnabled="false">
+      <option name="arguments" value="-m flake8 $FilePath$ --config $ContentRoot$\.flake8" />
+      <option name="checkSyntaxErrors" value="true" />
+      <option name="description" />
+      <option name="exitCodeBehavior" value="ALWAYS" />
+      <option name="fileExtension" value="py" />
+      <option name="immediateSync" value="false" />
+      <option name="name" value="Flake8 Watcher" />
+      <option name="output" value="$FilePath$" />
+      <option name="outputFilters">
+        <array>
+          <FilterInfo>
+            <option name="description" value="" />
+            <option name="name" value="" />
+            <option name="regExp" value="$FILE_PATH$:$LINE$:$COLUMN$: $MESSAGE$" />
+          </FilterInfo>
+        </array>
+      </option>
+      <option name="outputFromStdout" value="false" />
+      <option name="program" value="$PyInterpreterDirectory$/python" />
+      <option name="runOnExternalChanges" value="false" />
+      <option name="scopeName" value="Current File" />
+      <option name="trackOnlyRoot" value="false" />
+      <option name="workingDir" value="" />
+      <envs />
+    </TaskOptions>
+  </component>
+</project>

.prettierignore

@@ -0,0 +1,13 @@
+# generated files
+api.ts
+vue/src/apps/*.js
+vue/node_modules
+staticfiles/
+docs/reports/
+/vue3/src/openapi/
+
+# ignored files - prettier interferes with django templates and github actions
+*.html
+*.yml
+*.yaml
+

.prettierrc

@@ -0,0 +1,7 @@
+{
+    "printWidth": 179,
+    "trailingComma": "es5",
+    "tabWidth": 2,
+    "semi": false,
+    "experimentalTernaries": true
+}

.vscode/launch.json

@@ -4,7 +4,6 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
-      
       {
         "name": "Python Debugger: Django",
         "type": "debugpy",
@@ -13,6 +12,22 @@
         "args": ["runserver"],
         "django": true,
         "justMyCode": true
-      }
+      },
+    {
+      "name": "Python: Debug Tests",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "purpose": [
+        "debug-test"
+      ],
+      "console": "integratedTerminal",
+      "env": {
+        // coverage and pytest can't both be running at the same time
+        "PYTEST_ADDOPTS": "--no-cov"
+      },
+      "django": true,
+      "justMyCode": true
+    },
     ]
-  }
+  }

.vscode/settings.json

@@ -3,5 +3,12 @@
         "cookbook/tests"
     ],
     "python.testing.unittestEnabled": false,
-    "python.testing.pytestEnabled": true
-}
+    "python.testing.pytestEnabled": true,
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "[python]": {
+        "editor.defaultFormatter": "eeyore.yapf",
+    },
+    "yapf.args": [],
+    "isort.args": []
+}

Dockerfile

@@ -6,6 +6,8 @@ RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg li
 #Print all logs without buffering it.
 ENV PYTHONUNBUFFERED 1
 
+ENV DOCKER true
+
 #This port will be used by gunicorn.
 EXPOSE 8080
 
@@ -33,6 +35,13 @@ RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-de
 #Copy project and execute it.
 COPY . ./
 
+# commented for now https://github.com/TandoorRecipes/recipes/issues/3478
+#HEALTHCHECK --interval=30s \
+#            --timeout=5s \
+#            --start-period=10s \
+#            --retries=3 \
+#            CMD [ "/usr/bin/wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:8080/openapi" ]
+
 # collect information from git repositories
 RUN /opt/recipes/venv/bin/python version.py
 # delete git repositories to reduce image size

README.md

@@ -71,7 +71,7 @@ Because of that there are several ways you can support us
 - **Let us host for you** We are offering a [hosted version](https://app.tandoor.dev) where all profits support us and the development of tandoor (currently only available in germany).
 
 ## Contributing
-Contributions are welcome but please read [this](https://docs.tandoor.dev/contribute/#contributing-code) **BEFORE** contributing anything!
+Contributions are welcome but please read [this](https://docs.tandoor.dev/contribute/guidelines/) **BEFORE** contributing anything!
 
 ## Your Feedback
 

boot.sh

@@ -29,6 +29,18 @@ if [ -z "${SECRET_KEY}" ]; then
     display_warning "The environment variable 'SECRET_KEY' (or 'SECRET_KEY_FILE' that points to an existing file) is not set but REQUIRED for running Tandoor!"
 fi
 
+if [ -f "${AUTH_LDAP_BIND_PASSWORD_FILE}" ]; then
+    export AUTH_LDAP_BIND_PASSWORD=$(cat "$AUTH_LDAP_BIND_PASSWORD_FILE")
+fi
+
+if [ -f "${EMAIL_HOST_PASSWORD_FILE}" ]; then
+    export EMAIL_HOST_PASSWORD=$(cat "$EMAIL_HOST_PASSWORD_FILE")
+fi
+
+if [ -f "${SOCIALACCOUNT_PROVIDERS_FILE}" ]; then
+    export SOCIALACCOUNT_PROVIDERS=$(cat "$SOCIALACCOUNT_PROVIDERS_FILE")
+fi
+
 
 echo "Waiting for database to be ready..."
 
@@ -67,7 +79,7 @@ echo "Migrating database"
 
 python manage.py migrate
 
-echo "Generating static files"
+echo "Collecting static files, this may take a while..."
 
 python manage.py collectstatic_js_reverse
 python manage.py collectstatic --noinput
@@ -76,4 +88,11 @@ echo "Done"
 
 chmod -R 755 /opt/recipes/mediafiles
 
-exec gunicorn -b "[::]:$TANDOOR_PORT" --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --access-logfile - --error-logfile - --log-level $GUNICORN_LOG_LEVEL recipes.wsgi
+ipv6_disable=$(cat /sys/module/ipv6/parameters/disable)
+
+# Check if IPv6 is enabled, only then run gunicorn with ipv6 support
+if [ "$ipv6_disable" -eq 0 ]; then
+    exec gunicorn -b "[::]:$TANDOOR_PORT" --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --access-logfile - --error-logfile - --log-level $GUNICORN_LOG_LEVEL recipes.wsgi
+else
+    exec gunicorn -b ":$TANDOOR_PORT" --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --access-logfile - --error-logfile - --log-level $GUNICORN_LOG_LEVEL recipes.wsgi
+fi

cookbook/admin.py

@@ -185,7 +185,7 @@ class StepAdmin(admin.ModelAdmin):
     @admin.display(description="Name")
     def recipe_and_name(obj):
         if not obj.recipe_set.exists():
-            return f"Orphaned Step{'':s if not obj.name else f': {obj.name}'}"
+            return "Orphaned Step" + ('' if not obj.name else f': {obj.name}')
         return f"{obj.recipe_set.first().name}: {obj.name}" if obj.name else obj.recipe_set.first().name
 
 
@@ -376,10 +376,17 @@ class ShareLinkAdmin(admin.ModelAdmin):
 admin.site.register(ShareLink, ShareLinkAdmin)
 
 
+@admin.action(description='Delete all properties with type')
+def delete_properties_with_type(modeladmin, request, queryset):
+    for pt in queryset:
+        Property.objects.filter(property_type=pt).delete()
+
+
 class PropertyTypeAdmin(admin.ModelAdmin):
-    search_fields = ('space',)
+    search_fields = ('name',)
 
     list_display = ('id', 'space', 'name', 'fdc_id')
+    actions = [delete_properties_with_type]
 
 
 admin.site.register(PropertyType, PropertyTypeAdmin)

cookbook/apps.py

@@ -16,15 +16,11 @@ class CookbookConfig(AppConfig):
         import cookbook.signals  # noqa
 
         if not settings.DISABLE_EXTERNAL_CONNECTORS:
-            try:
-                from cookbook.connectors.connector_manager import ConnectorManager  # Needs to be here to prevent loading race condition of oauth2 modules in models.py
-                handler = ConnectorManager()
-                post_save.connect(handler, dispatch_uid="connector_manager")
-                post_delete.connect(handler, dispatch_uid="connector_manager")
-            except Exception as e:
-                traceback.print_exc()
-                print('Failed to initialize connectors')
-                pass
+            from cookbook.connectors.connector_manager import ConnectorManager  # Needs to be here to prevent loading race condition of oauth2 modules in models.py
+            handler = ConnectorManager()
+            post_save.connect(handler, dispatch_uid="post_save-connector_manager")
+            post_delete.connect(handler, dispatch_uid="post_delete-connector_manager")
+
         # if not settings.DISABLE_TREE_FIX_STARTUP:
         #     # when starting up run fix_tree to:
         #     #   a) make sure that nodes are sorted when switching between sort modes
@@ -45,4 +41,4 @@ class CookbookConfig(AppConfig):
         #         except Exception:
         #             if DEBUG:
         #                 traceback.print_exc()
-        #             pass  # dont break startup just because fix could not run, need to investigate cases when this happens
+        #             pass  # dont break startup just because fix could not run, need to investigate cases when this happens

cookbook/connectors/connector_manager.py

@@ -5,6 +5,7 @@ import threading
 from asyncio import Task
 from dataclasses import dataclass
 from enum import Enum
+from logging import Logger
 from types import UnionType
 from typing import List, Any, Dict, Optional, Type
 
@@ -30,6 +31,15 @@ class Work:
     actionType: ActionType
 
 
+class Singleton(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+
 # The way ConnectionManager works is as follows:
 # 1. On init, it starts a worker & creates a queue for 'Work'
 # 2. Then any time its called, it verifies the type of action (create/update/delete) and if the item is of interest, pushes the Work (non-blocking) to the queue.
@@ -38,11 +48,14 @@ class Work:
 # 3.2 If work is of type REGISTERED_CLASSES, it asynchronously fires of all connectors and wait for them to finish (runtime should depend on the 'slowest' connector)
 # 4. Work is marked as consumed, and next entry of the queue is consumed.
 # Each 'Work' is processed in sequential by the worker, so the throughput is about [workers * the slowest connector]
-class ConnectorManager:
+# The Singleton class is used for ConnectorManager to have a self-reference and so Python does not garbage collect it
+class ConnectorManager(metaclass=Singleton):
+    _logger: Logger
     _queue: queue.Queue
     _listening_to_classes = REGISTERED_CLASSES | ConnectorConfig
 
     def __init__(self):
+        self._logger = logging.getLogger("recipes.connector")
         self._queue = queue.Queue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE)
         self._worker = threading.Thread(target=self.worker, args=(0, self._queue,), daemon=True)
         self._worker.start()
@@ -65,7 +78,7 @@ class ConnectorManager:
         try:
             self._queue.put_nowait(Work(instance, action_type))
         except queue.Full:
-            logging.info(f"queue was full, so skipping {action_type} of type {type(instance)}")
+            self._logger.info(f"queue was full, so skipping {action_type} of type {type(instance)}")
             return
 
     def stop(self):
@@ -74,10 +87,12 @@ class ConnectorManager:
 
     @staticmethod
     def worker(worker_id: int, worker_queue: queue.Queue):
+        logger = logging.getLogger("recipes.connector.worker")
+
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
 
-        logging.info(f"started ConnectionManager worker {worker_id}")
+        logger.info(f"started ConnectionManager worker {worker_id}")
 
         # When multiple workers are used, please make sure the cache is shared across all threads, otherwise it might lead to un-expected behavior.
         _connectors_cache: Dict[int, List[Connector]] = dict()
@@ -91,6 +106,8 @@ class ConnectorManager:
             if item is None:
                 break
 
+            logger.debug(f"received {item.instance=} with {item.actionType=}")
+
             # If a Connector was changed/updated, refresh connector from the database for said space
             refresh_connector_cache = isinstance(item.instance, ConnectorConfig)
 
@@ -111,7 +128,7 @@ class ConnectorManager:
                         try:
                             connector: Optional[Connector] = ConnectorManager.get_connected_for_config(config)
                         except BaseException:
-                            logging.exception(f"failed to initialize {config.name}")
+                            logger.exception(f"failed to initialize {config.name}")
                             continue
 
                         if connector is not None:
@@ -123,10 +140,12 @@ class ConnectorManager:
                 worker_queue.task_done()
                 continue
 
+            logger.debug(f"running {len(connectors)} connectors for {item.instance=} with {item.actionType=}")
+
             loop.run_until_complete(run_connectors(connectors, space, item.instance, item.actionType))
             worker_queue.task_done()
 
-        logging.info(f"terminating ConnectionManager worker {worker_id}")
+        logger.info(f"terminating ConnectionManager worker {worker_id}")
 
         asyncio.set_event_loop(None)
         loop.close()

cookbook/connectors/homeassistant.py

@@ -1,26 +1,38 @@
 import logging
 from logging import Logger
+from typing import Dict, Tuple
+from urllib.parse import urljoin
 
-from homeassistant_api import Client, HomeassistantAPIError, Domain
+from aiohttp import request, ClientResponseError
 
 from cookbook.connectors.connector import Connector
 from cookbook.models import ShoppingListEntry, ConnectorConfig, Space
 
 
 class HomeAssistant(Connector):
-    _domains_cache: dict[str, Domain]
     _config: ConnectorConfig
     _logger: Logger
-    _client: Client
+
+    _required_foreign_keys = ("food", "unit", "created_by")
 
     def __init__(self, config: ConnectorConfig):
         if not config.token or not config.url or not config.todo_entity:
             raise ValueError("config for HomeAssistantConnector in incomplete")
 
-        self._domains_cache = dict()
+        self._logger = logging.getLogger(f"recipes.connector.homeassistant.{config.name}")
+
+        if config.url[-1] != "/":
+            config.url += "/"
         self._config = config
-        self._logger = logging.getLogger("connector.HomeAssistant")
-        self._client = Client(self._config.url, self._config.token, async_cache_session=False, use_async=True)
+
+    async def homeassistant_api_call(self, method: str, path: str, data: Dict) -> str:
+        headers = {
+            "Authorization": f"Bearer {self._config.token}",
+            "Content-Type": "application/json"
+        }
+        async with request(method, urljoin(self._config.url, path), headers=headers, json=data) as response:
+            response.raise_for_status()
+            return await response.json()
 
     async def on_shopping_list_entry_created(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None:
         if not self._config.on_shopping_list_entry_created_enabled:
@@ -28,16 +40,20 @@ class HomeAssistant(Connector):
 
         item, description = _format_shopping_list_entry(shopping_list_entry)
 
-        todo_domain = self._domains_cache.get('todo')
-        try:
-            if todo_domain is None:
-                todo_domain = await self._client.async_get_domain('todo')
-                self._domains_cache['todo'] = todo_domain
+        self._logger.debug(f"adding {item=} with {description=} to {self._config.todo_entity}")
 
-            logging.debug(f"pushing {item} to {self._config.name}")
-            await todo_domain.add_item(entity_id=self._config.todo_entity, item=item)
-        except HomeassistantAPIError as err:
-            self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}")
+        data = {
+            "entity_id": self._config.todo_entity,
+            "item": item,
+        }
+
+        if self._config.supports_description_field:
+            data["description"] = description
+
+        try:
+            await self.homeassistant_api_call("POST", "services/todo/add_item", data)
+        except ClientResponseError as err:
+            self._logger.warning(f"received an exception from the api: {err.request_info.url=}, {err.request_info.method=}, {err.status=}, {err.message=}, {type(err)=}")
 
     async def on_shopping_list_entry_updated(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None:
         if not self._config.on_shopping_list_entry_updated_enabled:
@@ -48,24 +64,31 @@ class HomeAssistant(Connector):
         if not self._config.on_shopping_list_entry_deleted_enabled:
             return
 
-        item, description = _format_shopping_list_entry(shopping_list_entry)
+        if not all(k in shopping_list_entry._state.fields_cache for k in self._required_foreign_keys):
+            # Sometimes the food foreign key is not loaded, and we cant load it from an async process
+            self._logger.debug("required property was not present in ShoppingListEntry")
+            return
+
+        item, _ = _format_shopping_list_entry(shopping_list_entry)
+
+        self._logger.debug(f"removing {item=} from {self._config.todo_entity}")
+
+        data = {
+            "entity_id": self._config.todo_entity,
+            "item": item,
+        }
 
-        todo_domain = self._domains_cache.get('todo')
         try:
-            if todo_domain is None:
-                todo_domain = await self._client.async_get_domain('todo')
-                self._domains_cache['todo'] = todo_domain
-
-            logging.debug(f"deleting {item} from {self._config.name}")
-            await todo_domain.remove_item(entity_id=self._config.todo_entity, item=item)
-        except HomeassistantAPIError as err:
-            self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}")
+            await self.homeassistant_api_call("POST", "services/todo/remove_item", data)
+        except ClientResponseError as err:
+            # This error will always trigger if the item is not present/found
+            self._logger.debug(f"received an exception from the api: {err.request_info.url=}, {err.request_info.method=}, {err.status=}, {err.message=}, {type(err)=}")
 
     async def close(self) -> None:
-        await self._client.async_cache_session.close()
+        pass
 
 
-def _format_shopping_list_entry(shopping_list_entry: ShoppingListEntry):
+def _format_shopping_list_entry(shopping_list_entry: ShoppingListEntry) -> Tuple[str, str]:
     item = shopping_list_entry.food.name
     if shopping_list_entry.amount > 0:
         item += f" ({shopping_list_entry.amount:.2f}".rstrip('0').rstrip('.')
@@ -76,10 +99,10 @@ def _format_shopping_list_entry(shopping_list_entry: ShoppingListEntry):
         else:
             item += ")"
 
-    description = "Imported by TandoorRecipes"
+    description = "From TandoorRecipes"
     if shopping_list_entry.created_by.first_name and len(shopping_list_entry.created_by.first_name) > 0:
-        description += f", created by {shopping_list_entry.created_by.first_name}"
+        description += f", by {shopping_list_entry.created_by.first_name}"
     else:
-        description += f", created by {shopping_list_entry.created_by.username}"
+        description += f", by {shopping_list_entry.created_by.username}"
 
     return item, description

cookbook/forms.py

@@ -1,6 +1,8 @@
 from datetime import datetime
 
+
 from allauth.account.forms import ResetPasswordForm, SignupForm
+from allauth.socialaccount.forms import SignupForm as SocialSignupForm
 from django import forms
 from django.conf import settings
 from django.core.exceptions import ValidationError
@@ -14,15 +16,13 @@ from .models import Comment, InviteLink, Keyword, Recipe, SearchPreference, Spac
 
 
 class SelectWidget(widgets.Select):
-
     class Media:
-        js = ('custom/js/form_select.js', )
+        js = ('custom/js/form_select.js',)
 
 
 class MultiSelectWidget(widgets.SelectMultiple):
-
     class Media:
-        js = ('custom/js/form_multiselect.js', )
+        js = ('custom/js/form_multiselect.js',)
 
 
 # Yes there are some stupid browsers that still dont support this but
@@ -89,12 +89,13 @@ class ImportExportBase(forms.Form):
     COOKMATE = 'COOKMATE'
     REZEPTSUITEDE = 'REZEPTSUITEDE'
     PDF = 'PDF'
+    GOURMET = 'GOURMET'
 
     type = forms.ChoiceField(choices=((DEFAULT, _('Default')), (PAPRIKA, 'Paprika'), (NEXTCLOUD, 'Nextcloud Cookbook'), (MEALIE, 'Mealie'), (CHOWDOWN, 'Chowdown'),
                                       (SAFFRON, 'Saffron'), (CHEFTAP, 'ChefTap'), (PEPPERPLATE, 'Pepperplate'), (RECETTETEK, 'RecetteTek'), (RECIPESAGE, 'Recipe Sage'),
                                       (DOMESTICA, 'Domestica'), (MEALMASTER, 'MealMaster'), (REZKONV, 'RezKonv'), (OPENEATS, 'Openeats'), (RECIPEKEEPER, 'Recipe Keeper'),
                                       (PLANTOEAT, 'Plantoeat'), (COOKBOOKAPP, 'CookBookApp'), (COPYMETHAT, 'CopyMeThat'), (PDF, 'PDF'), (MELARECIPES, 'Melarecipes'),
-                                      (COOKMATE, 'Cookmate'), (REZEPTSUITEDE, 'Recipesuite.de')))
+                                      (COOKMATE, 'Cookmate'), (REZEPTSUITEDE, 'Recipesuite.de'), (GOURMET, 'Gourmet')))
 
 
 class MultipleFileInput(forms.ClearableFileInput):
@@ -138,7 +139,7 @@ class CommentForm(forms.ModelForm):
 
     class Meta:
         model = Comment
-        fields = ('text', )
+        fields = ('text',)
 
         labels = {'text': _('Add your comment: '), }
         widgets = {'text': forms.Textarea(attrs={'rows': 2, 'cols': 15}), }
@@ -160,7 +161,6 @@ class StorageForm(forms.ModelForm):
         help_texts = {'url': _('Leave empty for dropbox and enter only base url for nextcloud (<code>/remote.php/webdav/</code> is added automatically)'), }
 
 
-
 class ConnectorConfigForm(forms.ModelForm):
     enabled = forms.BooleanField(
         help_text="Is the connector enabled",
@@ -182,6 +182,12 @@ class ConnectorConfigForm(forms.ModelForm):
         required=False,
     )
 
+    supports_description_field = forms.BooleanField(
+        help_text="Does the connector todo entity support the description field",
+        initial=True,
+        required=False,
+    )
+
     update_token = forms.CharField(
         widget=forms.TextInput(attrs={'autocomplete': 'update-token', 'type': 'password'}),
         required=False,
@@ -198,7 +204,7 @@ class ConnectorConfigForm(forms.ModelForm):
 
         fields = (
             'name', 'type', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled',
-            'on_shopping_list_entry_deleted_enabled', 'url', 'todo_entity',
+            'on_shopping_list_entry_deleted_enabled', 'supports_description_field', 'url', 'todo_entity',
         )
 
         help_texts = {
@@ -308,6 +314,18 @@ class AllAuthSignupForm(SignupForm):
         pass
 
 
+class AllAuthSocialSignupForm(SocialSignupForm):
+    terms = forms.BooleanField(label=_('Accept Terms and Privacy'))
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        if settings.PRIVACY_URL == '' and settings.TERMS_URL == '':
+            self.fields.pop('terms')
+
+    def signup(self, request, user):
+        pass
+
+
 class CustomPasswordResetForm(ResetPasswordForm):
     captcha = hCaptchaField()
 
@@ -338,12 +356,13 @@ class SearchPreferenceForm(forms.ModelForm):
 
         help_texts = {
             'search': _('Select type method of search.  Click <a href="/docs/search/">here</a> for full description of choices.'), 'lookup':
-            _('Use fuzzy matching on units, keywords and ingredients when editing and importing recipes.'), 'unaccent':
-            _('Fields to search ignoring accents.  Selecting this option can improve or degrade search quality depending on language'), 'icontains':
-            _("Fields to search for partial matches.  (e.g. searching for 'Pie' will return 'pie' and 'piece' and 'soapie')"), 'istartswith':
-            _("Fields to search for beginning of word matches. (e.g. searching for 'sa' will return 'salad' and 'sandwich')"), 'trigram':
-            _("Fields to 'fuzzy' search. (e.g. searching for 'recpie' will find 'recipe'.)  Note: this option will conflict with 'web' and 'raw' methods of search."), 'fulltext':
-            _("Fields to full text search.  Note: 'web', 'phrase', and 'raw' search methods only function with fulltext fields."),
+                _('Use fuzzy matching on units, keywords and ingredients when editing and importing recipes.'), 'unaccent':
+                _('Fields to search ignoring accents.  Selecting this option can improve or degrade search quality depending on language'), 'icontains':
+                _("Fields to search for partial matches.  (e.g. searching for 'Pie' will return 'pie' and 'piece' and 'soapie')"), 'istartswith':
+                _("Fields to search for beginning of word matches. (e.g. searching for 'sa' will return 'salad' and 'sandwich')"), 'trigram':
+                _("Fields to 'fuzzy' search. (e.g. searching for 'recpie' will find 'recipe'.)  Note: this option will conflict with 'web' and 'raw' methods of search."),
+            'fulltext':
+                _("Fields to full text search.  Note: 'web', 'phrase', and 'raw' search methods only function with fulltext fields."),
         }
 
         labels = {
@@ -353,5 +372,5 @@ class SearchPreferenceForm(forms.ModelForm):
 
         widgets = {
             'search': SelectWidget, 'unaccent': MultiSelectWidget, 'icontains': MultiSelectWidget, 'istartswith': MultiSelectWidget, 'trigram': MultiSelectWidget, 'fulltext':
-            MultiSelectWidget,
+                MultiSelectWidget,
         }

cookbook/helper/CustomTestRunner.py

@@ -1,8 +0,0 @@
-from django.test.runner import DiscoverRunner
-from django_scopes import scopes_disabled
-
-
-class CustomTestRunner(DiscoverRunner):
-    def run_tests(self, *args, **kwargs):
-        with scopes_disabled():
-            return super().run_tests(*args, **kwargs)

cookbook/helper/HelperFunctions.py

@@ -1,4 +1,12 @@
+import socket
+from urllib.parse import urlparse
+
+from django.core.exceptions import ValidationError
+from django.core.validators import URLValidator
 from django.db.models import Func
+from ipaddress import ip_address
+
+from recipes import settings
 
 
 class Round(Func):
@@ -11,3 +19,30 @@ def str2bool(v):
         return v
     else:
         return v.lower() in ("yes", "true", "1")
+
+
+"""
+validates an url that is supposed to be imported
+checks that the protocol used is http(s) and that no local address is accessed
+@:param url to test
+@:return true if url is valid, false otherwise
+"""
+
+
+def validate_import_url(url):
+    try:
+        validator = URLValidator(schemes=['http', 'https'])
+        validator(url)
+    except ValidationError:
+        # if schema is not http or https, consider url invalid
+        return False
+
+    # resolve IP address of url
+    try:
+        url_ip_address = ip_address(str(socket.gethostbyname(urlparse(url).hostname)))
+    except (ValueError, AttributeError, TypeError, Exception) as e:
+        # if ip cannot be parsed, consider url invalid
+        return False
+
+    # validate that IP is neither private nor any other special address
+    return not any([url_ip_address.is_private, url_ip_address.is_reserved, url_ip_address.is_loopback,  url_ip_address.is_multicast, url_ip_address.is_link_local, ])

cookbook/helper/image_processing.py

@@ -35,6 +35,20 @@ def get_filetype(name):
         return '.jpeg'
 
 
+def is_file_type_allowed(filename, image_only=False):
+    is_file_allowed = False
+    allowed_file_types = ['.pdf','.docx', '.xlsx']
+    allowed_image_types = ['.png', '.jpg', '.jpeg', '.gif']
+    check_list = allowed_image_types
+    if not image_only:
+        check_list += allowed_file_types
+
+    for file_type in check_list:
+        if filename.lower().endswith(file_type):
+            is_file_allowed = True
+
+    return is_file_allowed
+
 # TODO this whole file needs proper documentation, refactoring, and testing
 # TODO also add env variable to define which images sizes should be compressed
 # filetype argument can not be optional, otherwise this function will treat all images as if they were a jpeg

cookbook/helper/ingredient_parser.py

@@ -118,7 +118,7 @@ class IngredientParser:
         note = ''
         start = 0
         # search for first occurrence of an argument ending in a comma
-        while start < len(tokens) and not tokens[start].endswith(','):
+        while start < len(tokens) and not tokens[start].endswith((',', ';', ':')):
             start += 1
         if start == len(tokens):
             # no token ending in a comma found -> use everything as food

cookbook/helper/property_helper.py

@@ -71,7 +71,7 @@ class FoodPropertyHelper:
     # TODO move to central helper ? --> use defaultdict
     @staticmethod
     def add_or_create(d, key, value, food):
-        if key in d:
+        if key in d and d[key]['value']:
             d[key]['value'] += value
         else:
             d[key] = {'id': food.id, 'food': food.name, 'value': value}

cookbook/helper/recipe_url_import.py

@@ -6,7 +6,7 @@ from django.utils.dateparse import parse_duration
 from django.utils.translation import gettext as _
 from isodate import parse_duration as iso_parse_duration
 from isodate.isoerror import ISO8601Error
-from pytube import YouTube
+from pytubefix import YouTube
 from recipe_scrapers._utils import get_host_name, get_minutes
 
 from cookbook.helper.automation_helper import AutomationEngine
@@ -15,12 +15,9 @@ from cookbook.models import Automation, Keyword, PropertyType
 
 
 def get_from_scraper(scrape, request):
-    # converting the scrape_me object to the existing json format based on ld+json
+    # converting the scrape_html object to the existing json format based on ld+json
 
-    recipe_json = {
-        'steps': [],
-        'internal': True
-    }
+    recipe_json = {'steps': [], 'internal': True}
     keywords = []
 
     # assign source URL
@@ -31,7 +28,9 @@ def get_from_scraper(scrape, request):
             source_url = scrape.url
         except Exception:
             pass
-    if source_url:
+    if source_url == "https://urlnotfound.none" or not source_url:
+        recipe_json['source_url'] = ''
+    else:
         recipe_json['source_url'] = source_url
         try:
             keywords.append(source_url.replace('http://', '').replace('https://', '').split('/')[0])
@@ -157,11 +156,18 @@ def get_from_scraper(scrape, request):
     # assign steps
     try:
         for i in parse_instructions(scrape.instructions()):
-            recipe_json['steps'].append({'instruction': i, 'ingredients': [], 'show_ingredients_table': request.user.userpreference.show_step_ingredients, })
+            recipe_json['steps'].append({
+                'instruction': i,
+                'ingredients': [],
+                'show_ingredients_table': request.user.userpreference.show_step_ingredients,
+            })
     except Exception:
         pass
     if len(recipe_json['steps']) == 0:
-        recipe_json['steps'].append({'instruction': '', 'ingredients': [], })
+        recipe_json['steps'].append({
+            'instruction': '',
+            'ingredients': [],
+        })
 
     recipe_json['description'] = recipe_json['description'][:512]
     if len(recipe_json['description']) > 256:  # split at 256 as long descriptions don't look good on recipe cards
@@ -182,20 +188,20 @@ def get_from_scraper(scrape, request):
                         'original_text': x
                     }
                     if unit:
-                        ingredient['unit'] = {'name': unit, }
+                        ingredient['unit'] = {
+                            'name': unit,
+                        }
                     recipe_json['steps'][0]['ingredients'].append(ingredient)
                 except Exception:
-                    recipe_json['steps'][0]['ingredients'].append(
-                        {
-                            'amount': 0,
-                            'unit': None,
-                            'food': {
-                                'name': x,
-                            },
-                            'note': '',
-                            'original_text': x
-                        }
-                    )
+                    recipe_json['steps'][0]['ingredients'].append({
+                        'amount': 0,
+                        'unit': None,
+                        'food': {
+                            'name': x,
+                        },
+                        'note': '',
+                        'original_text': x
+                    })
     except Exception:
         pass
 
@@ -248,14 +254,16 @@ def get_from_youtube_scraper(url, request):
         'working_time': 0,
         'waiting_time': 0,
         'image': "",
-        'keywords': [{'name': kw.name, 'label': kw.name, 'id': kw.pk}],
+        'keywords': [{
+            'name': kw.name,
+            'label': kw.name,
+            'id': kw.pk
+        }],
         'source_url': url,
-        'steps': [
-            {
-                'ingredients': [],
-                'instruction': ''
-            }
-        ]
+        'steps': [{
+            'ingredients': [],
+            'instruction': ''
+        }]
     }
 
     try:
@@ -266,9 +274,8 @@ def get_from_youtube_scraper(url, request):
         default_recipe_json['image'] = video.thumbnail_url
         if video.description:
             default_recipe_json['steps'][0]['instruction'] = automation_engine.apply_regex_replace_automation(video.description, Automation.INSTRUCTION_REPLACE)
-
     except Exception:
-        pass
+        traceback.print_exc()
 
     return default_recipe_json
 
@@ -367,8 +374,8 @@ def parse_servings(servings):
             servings = 1
     elif isinstance(servings, list):
         try:
-            servings = int(re.findall(r'\b\d+\b', servings[0])[0])
-        except KeyError:
+            servings = int(re.findall(r'\b\d+\b', str(servings[0]))[0])
+        except (KeyError, IndexError):
             servings = 1
     return servings
 
@@ -452,10 +459,7 @@ def normalize_string(string):
 
 
 def iso_duration_to_minutes(string):
-    match = re.match(
-        r'P((?P<years>\d+)Y)?((?P<months>\d+)M)?((?P<weeks>\d+)W)?((?P<days>\d+)D)?T((?P<hours>\d+)H)?((?P<minutes>\d+)M)?((?P<seconds>\d+)S)?',
-        string
-    ).groupdict()
+    match = re.match(r'P((?P<years>\d+)Y)?((?P<months>\d+)M)?((?P<weeks>\d+)W)?((?P<days>\d+)D)?T((?P<hours>\d+)H)?((?P<minutes>\d+)M)?((?P<seconds>\d+)S)?', string).groupdict()
     return int(match['days'] or 0) * 24 * 60 + int(match['hours'] or 0) * 60 + int(match['minutes'] or 0)
 
 

cookbook/helper/scrapers/cooksillustrated.py

@@ -1,68 +0,0 @@
-import json
-from recipe_scrapers._abstract import AbstractScraper
-
-
-class CooksIllustrated(AbstractScraper):
-    @classmethod
-    def host(cls, site='cooksillustrated'):
-        return {
-            'cooksillustrated': f"{site}.com",
-            'americastestkitchen': f"{site}.com",
-            'cookscountry': f"{site}.com",
-        }.get(site)
-
-    def title(self):
-        return self.schema.title()
-
-    def image(self):
-        return self.schema.image()
-
-    def total_time(self):
-        if not self.recipe:
-            self.get_recipe()
-        return self.recipe['recipeTimeNote']
-
-    def yields(self):
-        if not self.recipe:
-            self.get_recipe()
-        return self.recipe['yields']
-
-    def ingredients(self):
-        if not self.recipe:
-            self.get_recipe()
-        ingredients = []
-        for group in self.recipe['ingredientGroups']:
-            ingredients += group['fields']['recipeIngredientItems']
-        return [
-            "{} {} {}{}".format(
-                i['fields']['qty'] or '',
-                i['fields']['measurement'] or '',
-                i['fields']['ingredient']['fields']['title'] or '',
-                i['fields']['postText'] or ''
-            )
-            for i in ingredients
-        ]
-
-    def instructions(self):
-        if not self.recipe:
-            self.get_recipe()
-        if self.recipe.get('headnote', False):
-            i = ['Note: ' + self.recipe.get('headnote', '')]
-        else:
-            i = []
-        return "\n".join(
-            i
-            + [self.recipe.get('whyThisWorks', '')]
-            + [
-                instruction['fields']['content']
-                for instruction in self.recipe['instructions']
-            ]
-        )
-
-    def nutrients(self):
-        raise NotImplementedError("This should be implemented.")
-
-    def get_recipe(self):
-        j = json.loads(self.soup.find(type='application/json').string)
-        name = list(j['props']['initialState']['content']['documents'])[0]
-        self.recipe = j['props']['initialState']['content']['documents'][name]

cookbook/helper/scrapers/scrapers.py

@@ -1,43 +0,0 @@
-from json import JSONDecodeError
-
-from bs4 import BeautifulSoup
-from recipe_scrapers import SCRAPERS, get_host_name
-from recipe_scrapers._factory import SchemaScraperFactory
-from recipe_scrapers._schemaorg import SchemaOrg
-
-from .cooksillustrated import CooksIllustrated
-
-CUSTOM_SCRAPERS = {
-    CooksIllustrated.host(site="cooksillustrated"): CooksIllustrated,
-    CooksIllustrated.host(site="americastestkitchen"): CooksIllustrated,
-    CooksIllustrated.host(site="cookscountry"): CooksIllustrated,
-}
-SCRAPERS.update(CUSTOM_SCRAPERS)
-
-
-def text_scraper(text, url=None):
-    domain = None
-    if url:
-        domain = get_host_name(url)
-    if domain in SCRAPERS:
-        scraper_class = SCRAPERS[domain]
-    else:
-        scraper_class = SchemaScraperFactory.SchemaScraper
-
-    class TextScraper(scraper_class):
-        def __init__(
-            self,
-            html=None,
-            url=None,
-        ):
-            self.wild_mode = False
-            self.meta_http_equiv = False
-            self.soup = BeautifulSoup(html, "html.parser")
-            self.url = url
-            self.recipe = None
-            try:
-                self.schema = SchemaOrg(html)
-            except (JSONDecodeError, AttributeError):
-                pass
-
-    return TextScraper(url=url, html=text)

cookbook/helper/template_helper.py

@@ -3,6 +3,8 @@ from gettext import gettext as _
 import bleach
 import markdown as md
 from jinja2 import Template, TemplateSyntaxError, UndefinedError
+from jinja2.exceptions import SecurityError
+from jinja2.sandbox import SandboxedEnvironment
 from markdown.extensions.tables import TableExtension
 
 from cookbook.helper.mdx_attributes import MarkdownFormatExtension
@@ -89,11 +91,13 @@ def render_instructions(step):  # TODO deduplicate markdown cleanup code
         return f"<scalable-number v-bind:number='{bleach.clean(str(number))}' v-bind:factor='ingredient_factor'></scalable-number>"
 
     try:
-        template = Template(instructions)
-        instructions = template.render(ingredients=ingredients, scale=scale)
+        env = SandboxedEnvironment()
+        instructions = env.from_string(instructions).render(ingredients=ingredients, scale=scale)
     except TemplateSyntaxError:
         return _('Could not parse template code.') + ' Error: Template Syntax broken'
     except UndefinedError:
         return _('Could not parse template code.') + ' Error: Undefined Error'
+    except SecurityError:
+        return _('Could not parse template code.') + ' Error: Security Error'
 
     return instructions

cookbook/helper/unit_conversion_helper.py

@@ -20,6 +20,7 @@ CONVERSION_TABLE = {
         'gallon': 0.264172,
         'tbsp': 67.628,
         'tsp': 202.884,
+        'us_cup': 4.22675,
         'imperial_fluid_ounce': 35.1951,
         'imperial_pint': 1.75975,
         'imperial_quart': 0.879877,

cookbook/integration/cookbookapp.py

@@ -2,12 +2,12 @@ import re
 from io import BytesIO
 
 import requests
-import validators
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import (get_from_scraper, get_images_from_soup,
                                                iso_duration_to_minutes)
-from cookbook.helper.scrapers.scrapers import text_scraper
+from recipe_scrapers import scrape_html
 from cookbook.integration.integration import Integration
 from cookbook.models import Ingredient, Recipe, Step
 
@@ -20,7 +20,7 @@ class CookBookApp(Integration):
     def get_recipe_from_file(self, file):
         recipe_html = file.getvalue().decode("utf-8")
 
-        scrape = text_scraper(text=recipe_html)
+        scrape = scrape_html(html=recipe_html, org_url="https://cookbookapp.import", supported_only=False)
         recipe_json = get_from_scraper(scrape, self.request)
         images = list(dict.fromkeys(get_images_from_soup(scrape.soup, None)))
 
@@ -63,7 +63,7 @@ class CookBookApp(Integration):
         if len(images) > 0:
             try:
                 url = images[0]
-                if validators.url(url, public=True):
+                if validate_import_url(url):
                     response = requests.get(url)
                     self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e:

cookbook/integration/cookmate.py

@@ -1,8 +1,8 @@
 from io import BytesIO
 
 import requests
-import validators
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
@@ -69,7 +69,7 @@ class Cookmate(Integration):
         if recipe_xml.find('imageurl') is not None:
             try:
                 url = recipe_xml.find('imageurl').text.strip()
-                if validators.url(url, public=True):
+                if validate_import_url(url):
                     response = requests.get(url)
                 self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e:

cookbook/integration/gourmet.py

@@ -0,0 +1,211 @@
+import base64
+from io import BytesIO
+from lxml import etree
+import requests
+from pathlib import Path
+
+from bs4 import BeautifulSoup, Tag
+
+from cookbook.helper.HelperFunctions import validate_import_url
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time, iso_duration_to_minutes
+from cookbook.integration.integration import Integration
+from cookbook.models import Ingredient, Recipe, Step, Keyword
+from recipe_scrapers import scrape_html
+
+
+class Gourmet(Integration):
+
+    def split_recipe_file(self, file):
+        encoding = 'utf-8'
+        byte_string = file.read()
+        text_obj = byte_string.decode(encoding, errors="ignore")
+        soup = BeautifulSoup(text_obj, "html.parser")
+        return soup.find_all("div", {"class": "recipe"})
+
+    def get_ingredients_recursive(self, step, ingredients, ingredient_parser):
+        if isinstance(ingredients, Tag):
+            for ingredient in ingredients.children:
+                if not isinstance(ingredient, Tag):
+                    continue
+
+                if ingredient.name in ["li"]:
+                    step_name = "".join(ingredient.findAll(text=True, recursive=False)).strip().rstrip(":")
+
+                    step.ingredients.add(Ingredient.objects.create(
+                        is_header=True,
+                        note=step_name[:256],
+                        original_text=step_name,
+                        space=self.request.space,
+                    ))
+                    next_ingrediets = ingredient.find("ul", {"class": "ing"})
+                    self.get_ingredients_recursive(step, next_ingrediets, ingredient_parser)
+
+                else:
+                    try:
+                        amount, unit, food, note = ingredient_parser.parse(ingredient.text.strip())
+                        f = ingredient_parser.get_food(food)
+                        u = ingredient_parser.get_unit(unit)
+                        step.ingredients.add(
+                            Ingredient.objects.create(
+                                food=f,
+                                unit=u,
+                                amount=amount,
+                                note=note,
+                                original_text=ingredient.text.strip(),
+                                space=self.request.space,
+                            )
+                        )
+                    except ValueError:
+                        pass
+
+    def get_recipe_from_file(self, file):
+        # 'file' comes is as a beautifulsoup object
+
+        source_url = None
+        for item in file.find_all('a'):
+            if item.has_attr('href'):
+                source_url = item.get("href")
+                break
+
+        name = file.find("p", {"class": "title"}).find("span", {"itemprop": "name"}).text.strip()
+
+        recipe = Recipe.objects.create(
+            name=name[:128],
+            source_url=source_url,
+            created_by=self.request.user,
+            internal=True,
+            space=self.request.space,
+        )
+
+        for category in file.find_all("span", {"itemprop": "recipeCategory"}):
+            keyword, created = Keyword.objects.get_or_create(name=category.text, space=self.request.space)
+            recipe.keywords.add(keyword)
+
+        try:
+            recipe.servings = parse_servings(file.find("span", {"itemprop": "recipeYield"}).text.strip())
+        except AttributeError:
+            pass
+
+        try:
+            prep_time = file.find("span", {"itemprop": "prepTime"}).text.strip().split()
+            prep_time[0] = prep_time[0].replace(',', '.')
+            if prep_time[1].lower() in ['stunde', 'stunden', 'hour', 'hours']:
+                prep_time_min = int(float(prep_time[0]) * 60)
+            elif prep_time[1].lower() in ['tag', 'tage', 'day', 'days']:
+                prep_time_min = int(float(prep_time[0]) * 60 * 24)
+            else:
+                prep_time_min = int(prep_time[0])
+            recipe.waiting_time = prep_time_min
+        except AttributeError:
+            pass
+
+        try:
+            cook_time = file.find("span", {"itemprop": "cookTime"}).text.strip().split()
+            cook_time[0] = cook_time[0].replace(',', '.')
+            if cook_time[1].lower() in ['stunde', 'stunden', 'hour', 'hours']:
+                cook_time_min = int(float(cook_time[0]) * 60)
+            elif cook_time[1].lower() in ['tag', 'tage', 'day', 'days']:
+                cook_time_min = int(float(cook_time[0]) * 60 * 24)
+            else:
+                cook_time_min = int(cook_time[0])
+
+            recipe.working_time = cook_time_min
+        except AttributeError:
+            pass
+
+        for cuisine in file.find_all('span', {'itemprop': 'recipeCuisine'}):
+            cuisine_name = cuisine.text
+            keyword = Keyword.objects.get_or_create(space=self.request.space, name=cuisine_name)
+            if len(keyword):
+                recipe.keywords.add(keyword[0])
+
+        for category in file.find_all('span', {'itemprop': 'recipeCategory'}):
+            category_name = category.text
+            keyword = Keyword.objects.get_or_create(space=self.request.space, name=category_name)
+            if len(keyword):
+                recipe.keywords.add(keyword[0])
+
+        step = Step.objects.create(
+            instruction='',
+            space=self.request.space,
+            show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+        )
+
+        ingredient_parser = IngredientParser(self.request, True)
+
+        ingredients = file.find("ul", {"class": "ing"})
+        self.get_ingredients_recursive(step, ingredients, ingredient_parser)
+
+        instructions = file.find("div", {"class": "instructions"})
+        if isinstance(instructions, Tag):
+            for instruction in instructions.children:
+                if not isinstance(instruction, Tag) or instruction.text == "":
+                    continue
+                if instruction.name == "h3":
+                    if step.instruction:
+                        step.save()
+                        recipe.steps.add(step)
+                        step = Step.objects.create(
+                            instruction='',
+                            space=self.request.space,
+                        )
+
+                    step.name = instruction.text.strip()[:128]
+                else:
+                    if instruction.name == "div":
+                        for instruction_step in instruction.children:
+                            for br in instruction_step.find_all("br"):
+                                br.replace_with("\n")
+                            step.instruction += instruction_step.text.strip() + ' \n\n'
+
+        notes = file.find("div", {"class": "modifications"})
+        if notes:
+            for n in notes.children:
+                if n.text == "":
+                    continue
+                if n.name == "h3":
+                    step.instruction += f'*{n.text.strip()}:* \n\n'
+                else:
+                    for br in n.find_all("br"):
+                        br.replace_with("\n")
+
+                    step.instruction += '*' + n.text.strip() + '* \n\n'
+
+        description = ''
+        try:
+            description = file.find("div", {"id": "description"}).text.strip()
+        except AttributeError:
+            pass
+        if len(description) <= 512:
+            recipe.description = description
+        else:
+            recipe.description = description[:480] + ' ... (full description below)'
+            step.instruction += '*Description:* \n\n*' + description + '* \n\n'
+
+        step.save()
+        recipe.steps.add(step)
+
+        # import the Primary recipe image that is stored in the Zip
+        try:
+            image_path = file.find("img").get("src")
+            image_filename = image_path.split("\\")[1]
+
+            for f in self.import_zip.filelist:
+                zip_file_name = Path(f.filename).name
+                if image_filename == zip_file_name:
+                    image_file = self.import_zip.read(f)
+                    image_bytes = BytesIO(image_file)
+                    self.import_recipe_image(recipe, image_bytes, filetype='.jpeg')
+                    break
+        except Exception as e:
+            print(recipe.name, ': failed to import image ', str(e))
+
+        recipe.save()
+        return recipe
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        raise NotImplementedError('Method not implemented in storage integration')
+
+    def get_file_from_recipe(self, recipe):
+        raise NotImplementedError('Method not implemented in storage integration')

cookbook/integration/integration.py

@@ -153,6 +153,19 @@ class Integration:
                             il.total_recipes = len(new_file_list)
                             file_list = new_file_list
 
+                        if isinstance(self, cookbook.integration.gourmet.Gourmet):
+                            self.import_zip = import_zip
+                            new_file_list = []
+                            for file in file_list:
+                                if file.file_size == 0:
+                                    next
+                                if file.filename.startswith("index.htm"):
+                                    next
+                                if file.filename.endswith(".htm"):
+                                    new_file_list += self.split_recipe_file(BytesIO(import_zip.read(file.filename)))
+                            il.total_recipes = len(new_file_list)
+                            file_list = new_file_list
+
                         for z in file_list:
                             try:
                                 if not hasattr(z, 'filename') or isinstance(z, Tag):

cookbook/integration/mealie.py

@@ -72,14 +72,14 @@ class Mealie(Integration):
             )
             recipe.steps.add(step)
 
-        if 'recipe_yield' in recipe_json:
+        if 'recipe_yield' in recipe_json and recipe_json['recipe_yield'] is not None:
             recipe.servings = parse_servings(recipe_json['recipe_yield'])
             recipe.servings_text = parse_servings_text(recipe_json['recipe_yield'])
 
         if 'total_time' in recipe_json and recipe_json['total_time'] is not None:
             recipe.working_time = parse_time(recipe_json['total_time'])
 
-        if 'org_url' in recipe_json:
+        if 'org_url' in recipe_json and recipe_json['org_url'] is not None:
             recipe.source_url = recipe_json['org_url']
 
         recipe.save()

cookbook/integration/nextcloud_cookbook.py

@@ -60,20 +60,20 @@ class NextcloudCookbook(Integration):
                 step = Step.objects.create(
                     instruction=s, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
                 )
-            if not ingredients_added:
-                if len(recipe_json['description'].strip()) > 500:
-                    step.instruction = recipe_json['description'].strip() + '\n\n' + step.instruction
 
-                ingredients_added = True
-
-                ingredient_parser = IngredientParser(self.request, True)
+            ingredient_parser = IngredientParser(self.request, True)
+            if ingredients_added == False:
                 for ingredient in recipe_json['recipeIngredient']:
-                    amount, unit, food, note = ingredient_parser.parse(ingredient)
-                    f = ingredient_parser.get_food(food)
-                    u = ingredient_parser.get_unit(unit)
-                    step.ingredients.add(Ingredient.objects.create(
-                        food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
-                    ))
+                    ingredients_added = True
+                    if ingredient.startswith('##'):
+                        subheader = ingredient.replace('##', '', 1)
+                        step.ingredients.add(Ingredient.objects.create(note=subheader, is_header=True, no_amount=True, space=self.request.space))
+                    else:
+                        amount, unit, food, note = ingredient_parser.parse(ingredient)
+                        f = ingredient_parser.get_food(food)
+                        u = ingredient_parser.get_unit(unit)
+                        step.ingredients.add(Ingredient.objects.create(
+                            food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,))
             recipe.steps.add(step)
 
         if 'nutrition' in recipe_json:

cookbook/integration/paprika.py

@@ -6,8 +6,8 @@ from gettext import gettext as _
 from io import BytesIO
 
 import requests
-import validators
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text
 from cookbook.integration.integration import Integration
@@ -84,13 +84,23 @@ class Paprika(Integration):
 
             recipe.steps.add(step)
 
+            # Paprika exports can have images in either of image_url, or photo_data.
+            # If a user takes an image himself, only photo_data will be set.
+            # If a user imports an image, both will be set. But the photo_data will be a center-cropped square resized version, so the image_url is preferred.
+            
+            # Try to download image if possible
             try:
                 if recipe_json.get("image_url", None):
                     url = recipe_json.get("image_url", None)
-                    if validators.url(url, public=True):
+                    if validate_import_url(url):
                         response = requests.get(url)
-                        self.import_recipe_image(recipe, BytesIO(response.content))
+                        if response.status_code == 200 and len(response.content) > 0:
+                            self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception:
+                pass
+
+            # If no image downloaded, try to extract from photo_data
+            if not recipe.image:
                 if recipe_json.get("photo_data", None):
                     self.import_recipe_image(recipe, BytesIO(base64.b64decode(recipe_json['photo_data'])), filetype='.jpeg')
 

cookbook/integration/plantoeat.py

@@ -1,9 +1,10 @@
 from io import BytesIO
 
 import requests
-import validators
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
 from cookbook.models import Ingredient, Keyword, Recipe, Step
 
@@ -18,32 +19,38 @@ class Plantoeat(Integration):
         tags = None
         ingredients = []
         directions = []
-        description = ''
+        fields = {}
         for line in file.replace('\r', '').split('\n'):
             if line.strip() != '':
                 if 'Title:' in line:
-                    title = line.replace('Title:', '').replace('"', '').strip()
+                    fields['name'] = line.replace('Title:', '').replace('"', '').strip()
                 if 'Description:' in line:
-                    description = line.replace('Description:', '').strip()
-                if 'Source:' in line or 'Serves:' in line or 'Prep Time:' in line or 'Cook Time:' in line:
-                    directions.append(line.strip() + '\n')
+                    fields['description'] = line.replace('Description:', '').strip()
+                if 'Serves:' in line:
+                    fields['servings'] = parse_servings(line.replace('Serves:', '').strip())
+                if 'Source:' in line:
+                    fields['source_url'] = line.replace('Source:', '').strip()
                 if 'Photo Url:' in line:
                     image_url = line.replace('Photo Url:', '').strip()
+                if 'Prep Time:' in line:
+                    fields['working_time'] = parse_time(line.replace('Prep Time:', '').strip())
+                if 'Cook Time:' in line:
+                    fields['waiting_time'] = parse_time(line.replace('Cook Time:', '').strip())
                 if 'Tags:' in line:
                     tags = line.replace('Tags:', '').strip()
-                if ingredient_mode:
-                    if len(line) > 2 and 'Instructions:' not in line:
-                        ingredients.append(line.strip())
-                if direction_mode:
-                    if len(line) > 2:
-                        directions.append(line.strip() + '\n')
                 if 'Ingredients:' in line:
                     ingredient_mode = True
                 if 'Directions:' in line:
                     ingredient_mode = False
                     direction_mode = True
+                if ingredient_mode:
+                    if len(line) > 2 and 'Ingredients:' not in line:
+                        ingredients.append(line.strip())
+                if direction_mode:
+                    if len(line) > 2:
+                        directions.append(line.strip() + '\n')                
 
-        recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space)
+        recipe = Recipe.objects.create(**fields, created_by=self.request.user, internal=True, space=self.request.space)
 
         step = Step.objects.create(
             instruction='\n'.join(directions) + '\n\n', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
@@ -68,7 +75,7 @@ class Plantoeat(Integration):
 
         if image_url:
             try:
-                if validators.url(image_url, public=True):
+                if validate_import_url(image_url):
                     response = requests.get(image_url)
                     self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e:

cookbook/integration/recettetek.py

@@ -5,9 +5,10 @@ from io import BytesIO
 from zipfile import ZipFile
 
 import requests
-import validators
 
 from django.utils.translation import gettext as _
+
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.image_processing import get_filetype
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
@@ -125,7 +126,7 @@ class RecetteTek(Integration):
             else:
                 if file['originalPicture'] != '':
                     url = file['originalPicture']
-                    if validators.url(url, public=True):
+                    if validate_import_url(url):
                         response = requests.get(url)
                         if imghdr.what(BytesIO(response.content)) is not None:
                             self.import_recipe_image(recipe, BytesIO(response.content), filetype=get_filetype(file['originalPicture']))

cookbook/integration/recipesage.py

@@ -2,8 +2,8 @@ import json
 from io import BytesIO
 
 import requests
-import validators
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
@@ -56,7 +56,7 @@ class RecipeSage(Integration):
         if len(file['image']) > 0:
             try:
                 url = file['image'][0]
-                if validators.url(url, public=True):
+                if validate_import_url(url):
                     response = requests.get(url)
                     self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e: