compiled and made messages

Shoppinglist fix

.coveragerc

@@ -0,0 +1,11 @@
+[run]
+omit =
+  */apps.py,
+  */migrations/*,
+  */settings*,
+  */test*,
+  */tests/*,
+  *urls.py,
+  */wsgi*,
+  manage.py,
+  *__init__*

.devcontainer/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.10-alpine3.18
+
+#Install all dependencies.
+RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg libwebp libxml2-dev libxslt-dev openldap git yarn
+
+#Print all logs without buffering it.
+ENV PYTHONUNBUFFERED 1
+
+#This port will be used by gunicorn.
+EXPOSE 8000
+
+#This port will be used by vue
+EXPOSE 8080
+
+#Install all python dependencies to the image
+COPY requirements.txt /tmp/pip-tmp/
+
+RUN \
+    if [ `apk --print-arch` = "armv7" ]; then \
+    printf "[global]\nextra-index-url=https://www.piwheels.org/simple\n" > /etc/pip.conf ; \
+    fi
+RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-dev jpeg-dev libwebp-dev openssl-dev libffi-dev cargo openldap-dev python3-dev && \
+    echo -n "INPUT ( libldap.so )" > /usr/lib/libldap_r.so && \
+    pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt && \
+    rm -rf /tmp/pip-tmp && \
+    apk --purge del .build-deps

.devcontainer/devcontainer.json

@@ -0,0 +1,27 @@
+// For format details, see https://aka.ms/devcontainer.json.
+{
+	"name": "Tandoor Dev Container",
+	"build": { "context": "..", "dockerfile": "Dockerfile" },
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [8000, 8080],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "pip3 install --user -r requirements.txt"
+
+	// Configure tool-specific properties.
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.debugpy",
+				"ms-python.python"
+			]
+		}
+	}
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.env.template

@@ -1,172 +1,15 @@
-# only set this to true when testing/debugging
-# when unset: 1 (true) - dont unset this, just for development
-DEBUG=0
-SQL_DEBUG=0
-DEBUG_TOOLBAR=0
-# Gunicorn log level for debugging (default value is "info" when unset)
-# (see https://docs.gunicorn.org/en/stable/settings.html#loglevel for available settings)
-# GUNICORN_LOG_LEVEL="debug"
-
-# HTTP port to bind to
-# TANDOOR_PORT=8080
-
-# hosts the application can run under e.g. recipes.mydomain.com,cooking.mydomain.com,...
-ALLOWED_HOSTS=*
+# ---------------------------------------------------------------------------
+# This template contains only required options.
+# Visit the docs to find more https://docs.tandoor.dev/system/configuration/
+# ---------------------------------------------------------------------------
 
 # random secret key, use for example `base64 /dev/urandom | head -c50` to generate one
-# ---------------------------- REQUIRED -------------------------
 SECRET_KEY=
-# ---------------------------------------------------------------
-
-# your default timezone See https://timezonedb.com/time-zones for a list of timezones
-TIMEZONE=Europe/Berlin
 
 # add only a database password if you want to run with the default postgres, otherwise change settings accordingly
 DB_ENGINE=django.db.backends.postgresql
-# DB_OPTIONS= {} # e.g. {"sslmode":"require"} to enable ssl
 POSTGRES_HOST=db_recipes
+POSTGRES_DB=djangodb
 POSTGRES_PORT=5432
 POSTGRES_USER=djangouser
-# ---------------------------- REQUIRED -------------------------
 POSTGRES_PASSWORD=
-# ---------------------------------------------------------------
-POSTGRES_DB=djangodb
-
-# database connection string, when used overrides other database settings.
-# format might vary depending on backend
-# DATABASE_URL = engine://username:password@host:port/dbname
-
-# the default value for the user preference 'fractions' (enable/disable fraction support)
-# default: disabled=0
-FRACTION_PREF_DEFAULT=0
-
-# the default value for the user preference 'comments' (enable/disable commenting system)
-# default comments enabled=1
-COMMENT_PREF_DEFAULT=1
-
-# Users can set a amount of time after which the shopping list is refreshed when they are in viewing mode
-# This is the minimum interval users can set. Setting this to low will allow users to refresh very frequently which
-# might cause high load on the server. (Technically they can obviously refresh as often as they want with their own scripts)
-SHOPPING_MIN_AUTOSYNC_INTERVAL=5
-
-# Default for user setting sticky navbar
-# STICKY_NAV_PREF_DEFAULT=1
-
-# If base URL is something other than just / (you are serving a subfolder in your proxy for instance http://recipe_app/recipes/)
-# Be sure to not have a trailing slash: e.g. '/recipes' instead of '/recipes/'
-# SCRIPT_NAME=/recipes
-
-# If staticfiles are stored at a different location uncomment and change accordingly, MUST END IN /
-# this is not required if you are just using a subfolder
-# This can either be a relative path from the applications base path or the url of an external host
-# STATIC_URL=/static/
-
-# If mediafiles are stored at a different location uncomment and change accordingly, MUST END IN /
-# this is not required if you are just using a subfolder
-# This can either be a relative path from the applications base path or the url of an external host
-# MEDIA_URL=/media/
-
-# Serve mediafiles directly using gunicorn. Basically everyone recommends not doing this. Please use any of the examples
-# provided that include an additional nxginx container to handle media file serving.
-# If you know what you are doing turn this back on (1) to serve media files using djangos serve() method.
-# when unset: 1 (true) - this is temporary until an appropriate amount of time has passed for everyone to migrate
-GUNICORN_MEDIA=0
-
-# GUNICORN SERVER RELATED SETTINGS (see https://docs.gunicorn.org/en/stable/design.html#how-many-workers for recommended settings)
-# GUNICORN_WORKERS=1
-# GUNICORN_THREADS=1
-
-# S3 Media settings: store mediafiles in s3 or any compatible storage backend (e.g. minio)
-# as long as S3_ACCESS_KEY is not set S3 features are disabled
-# S3_ACCESS_KEY=
-# S3_SECRET_ACCESS_KEY=
-# S3_BUCKET_NAME=
-# S3_REGION_NAME= # default none, set your region might be required
-# S3_QUERYSTRING_AUTH=1 # default true, set to 0 to serve media from a public bucket without signed urls
-# S3_QUERYSTRING_EXPIRE=3600 # number of seconds querystring are valid for
-# S3_ENDPOINT_URL= # when using a custom endpoint like minio
-# S3_CUSTOM_DOMAIN= # when using a CDN/proxy to S3 (see https://github.com/TandoorRecipes/recipes/issues/1943)
-
-# Email Settings, see https://docs.djangoproject.com/en/3.2/ref/settings/#email-host
-# Required for email confirmation and password reset (automatically activates if host is set)
-# EMAIL_HOST=
-# EMAIL_PORT=
-# EMAIL_HOST_USER=
-# EMAIL_HOST_PASSWORD=
-# EMAIL_USE_TLS=0
-# EMAIL_USE_SSL=0
-# email sender address (default 'webmaster@localhost')
-# DEFAULT_FROM_EMAIL=
-# prefix used for account related emails (default "[Tandoor Recipes] ")
-# ACCOUNT_EMAIL_SUBJECT_PREFIX=
-
-# allow authentication via reverse proxy (e.g. authelia), leave off if you dont know what you are doing
-# see docs for more information https://docs.tandoor.dev/features/authentication/
-# when unset: 0 (false)
-REVERSE_PROXY_AUTH=0
-
-# Default settings for spaces, apply per space and can be changed in the admin view
-# SPACE_DEFAULT_MAX_RECIPES=0 # 0=unlimited recipes
-# SPACE_DEFAULT_MAX_USERS=0 # 0=unlimited users per space
-# SPACE_DEFAULT_MAX_FILES=0 # Maximum file storage for space in MB. 0 for unlimited, -1 to disable file upload.
-# SPACE_DEFAULT_ALLOW_SHARING=1 # Allow users to share recipes with public links
-
-# allow people to create accounts on your application instance (without an invite link)
-# when unset: 0 (false)
-# ENABLE_SIGNUP=0
-
-# If signup is enabled you might want to add a captcha to it to prevent spam
-# HCAPTCHA_SITEKEY=
-# HCAPTCHA_SECRET=
-
-# if signup is enabled you might want to provide urls to data protection policies or terms and conditions
-# TERMS_URL=
-# PRIVACY_URL=
-# IMPRINT_URL=
-
-# enable serving of prometheus metrics under the /metrics path
-# ATTENTION: view is not secured (as per the prometheus default way) so make sure to secure it
-# trough your web server (or leave it open of you dont care if the stats are exposed)
-# ENABLE_METRICS=0
-
-# allows you to setup OAuth providers
-# see docs for more information https://docs.tandoor.dev/features/authentication/
-# SOCIAL_PROVIDERS = allauth.socialaccount.providers.github, allauth.socialaccount.providers.nextcloud,
-
-# Should a newly created user from a social provider get assigned to the default space and given permission by default ?
-# ATTENTION: This feature might be deprecated in favor of a space join and public viewing system in the future
-# default 0 (false), when 1 (true) users will be assigned space and group
-# SOCIAL_DEFAULT_ACCESS = 1
-
-# if SOCIAL_DEFAULT_ACCESS is used, which group should be added
-# SOCIAL_DEFAULT_GROUP=guest
-
-# Django session cookie settings. Can be changed to allow a single django application to authenticate several applications
-# when running under the same database
-# SESSION_COOKIE_DOMAIN=.example.com
-# SESSION_COOKIE_NAME=sessionid # use this only to not interfere with non unified django applications under the same top level domain
-
-# by default SORT_TREE_BY_NAME is disabled this will store all Keywords and Food in the order they are created
-# enabling this setting makes saving new keywords and foods very slow, which doesn't matter in most usecases.
-# however, when doing large imports of recipes that will create new objects, can increase total run time by 10-15x
-# Keywords and Food can be manually sorted by name in Admin
-# This value can also be temporarily changed in Admin, it will revert the next time the application is started
-# This will be fixed/changed in the future by changing the implementation or finding a better workaround for sorting
-# SORT_TREE_BY_NAME=0
-# LDAP authentication
-# default 0 (false), when 1 (true) list of allowed users will be fetched from LDAP server
-#LDAP_AUTH=
-#AUTH_LDAP_SERVER_URI=
-#AUTH_LDAP_BIND_DN=
-#AUTH_LDAP_BIND_PASSWORD=
-#AUTH_LDAP_USER_SEARCH_BASE_DN=
-#AUTH_LDAP_TLS_CACERTFILE=
-#AUTH_LDAP_START_TLS=
-
-# Enables exporting PDF (see export docs)
-# Disabled by default, uncomment to enable
-# ENABLE_PDF_EXPORT=1
-
-# Recipe exports are cached for a certain time by default, adjust time if needed
-# EXPORT_FILE_CACHE_DURATION=600
-

.github/dependabot.yml

@@ -1,10 +1,15 @@
 # To get started with Dependabot version updates, you'll need to specify which
 # package ecosystems to update and where the package manifests are located.
 # Please see the documentation for all configuration options:
-# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
 
 version: 2
 updates:
+  - package-ecosystem: "devcontainers"
+    directory: "/"
+    schedule:
+      interval: weekly
+
   - package-ecosystem: "pip"
     directory: "/"
     schedule:

.github/workflows/build-docker-open-data.yml

@@ -21,7 +21,7 @@ jobs:
             suffix: ""
             continue-on-error: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Get version number
         id: get_version
@@ -43,7 +43,7 @@ jobs:
           path: ./recipes/plugins/open_data_plugin
 
       # Build Vue frontend
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: '18'
           cache: yarn
@@ -64,17 +64,17 @@ jobs:
         run: yarn build
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: github.secret_source == 'Actions'
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: github.secret_source == 'Actions'
         with:
           registry: ghcr.io
@@ -82,7 +82,7 @@ jobs:
           password: ${{ github.token }}
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: |
             vabene1111/recipes
@@ -97,7 +97,7 @@ jobs:
             type=semver,suffix=-open-data-plugin,pattern={{major}}
             type=ref,suffix=-open-data-plugin,event=branch
       - name: Build and Push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: ${{ matrix.dockerfile }}

.github/workflows/build-docker.yml

@@ -21,7 +21,7 @@ jobs:
             suffix: ""
             continue-on-error: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Get version number
         id: get_version
@@ -35,7 +35,7 @@ jobs:
           fi
 
       # Build Vue frontend
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: '18'
           cache: yarn
@@ -48,17 +48,17 @@ jobs:
         run: yarn build
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: github.secret_source == 'Actions'
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: github.secret_source == 'Actions'
         with:
           registry: ghcr.io
@@ -66,7 +66,7 @@ jobs:
           password: ${{ github.token }}
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: |
             vabene1111/recipes
@@ -81,7 +81,7 @@ jobs:
             type=semver,pattern={{major}}
             type=ref,event=branch
       - name: Build and Push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: ${{ matrix.dockerfile }}

.github/workflows/ci.yml

@@ -3,38 +3,79 @@ name: Continuous Integration
 on: [push, pull_request]
 
 jobs:
-  build:
-    if: github.repository_owner == 'TandoorRecipes'
-    runs-on: ubuntu-latest
-    strategy:
-      max-parallel: 4
-      matrix:
-        python-version: ['3.10']
+    build:
+        if: github.repository_owner == 'TandoorRecipes'
+        runs-on: ubuntu-latest
+        strategy:
+            max-parallel: 4
+            matrix:
+                python-version: ["3.10"]
+                node-version: ["18"]
 
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.10'
-    # Build Vue frontend
-    - uses: actions/setup-node@v3
-      with:
-        node-version: '18'
-    - name: Install Vue dependencies
-      working-directory: ./vue
-      run: yarn install
-    - name: Build Vue dependencies
-      working-directory: ./vue
-      run: yarn build
-    - name: Install Django dependencies
-      run: |
-        sudo apt-get -y update
-        sudo apt-get install -y libsasl2-dev python3-dev libldap2-dev libssl-dev
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        python3 manage.py collectstatic --noinput
-        python3 manage.py collectstatic_js_reverse
-    - name: Django Testing project
-      run: |
-        pytest
+        steps:
+            - uses: actions/checkout@v4
+            - uses: awalsh128/cache-apt-pkgs-action@v1.4.1
+              with:
+                  packages: libsasl2-dev python3-dev libldap2-dev libssl-dev
+                  version: 1.0
+
+            # Setup python & dependencies
+            - name: Set up Python ${{ matrix.python-version }}
+              uses: actions/setup-python@v5
+              with:
+                  python-version: ${{ matrix.python-version }}
+                  cache: "pip"
+
+            - name: Install Python Dependencies
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install -r requirements.txt
+
+            - name: Cache StaticFiles
+              uses: actions/cache@v4
+              id: django_cache
+              with:
+                  path: |
+                      ./cookbook/static
+                      ./vue/webpack-stats.json
+                      ./staticfiles
+                  key: |
+                      ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }}
+
+            # Build Vue frontend & Dependencies
+            - name: Set up Node ${{ matrix.node-version }}
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              uses: actions/setup-node@v4
+              with:
+                  node-version: ${{ matrix.node-version }}
+                  cache: "yarn"
+                  cache-dependency-path: ./vue/yarn.lock
+
+            - name: Install Vue dependencies
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              working-directory: ./vue
+              run: yarn install
+
+            - name: Build Vue dependencies
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              working-directory: ./vue
+              run: yarn build
+
+            - name: Compile Django StaticFiles
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              run: |
+                  python3 manage.py collectstatic --noinput
+                  python3 manage.py collectstatic_js_reverse
+
+            - uses: actions/cache/save@v4
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              with:
+                  path: |
+                      ./cookbook/static
+                      ./vue/webpack-stats.json
+                      ./staticfiles
+                  key: |
+                      ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }}
+
+            - name: Django Testing project
+              run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml

.github/workflows/codeql-analysis.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           # We must fetch at least the immediate parents so that if this is
           # a pull request then we can checkout the head.
@@ -25,7 +25,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         # Override language selection by uncommenting this and choosing your languages
         with:
           languages: python, javascript
@@ -47,6 +47,6 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3
         with:
           languages: javascript, python

.github/workflows/docs.yml

@@ -1,17 +1,20 @@
 name: Make Docs
 on:
-  push:
-    branches:
-      - master
+  # the 1st condition
+  workflow_run:
+    workflows: ["Continuous Integration"]
+    branches: [master]
+    types:
+      - completed
 
 jobs:
   deploy:
-    if: github.repository_owner == 'TandoorRecipes'
+    if: github.repository_owner == 'TandoorRecipes' && ${{ github.event.workflow_run.conclusion == 'success' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.x
       - run: pip install mkdocs-material mkdocs-include-markdown-plugin
-      - run: mkdocs gh-deploy --force
+      - run: mkdocs gh-deploy --force

.gitignore

@@ -43,6 +43,7 @@ htmlcov/
 nosetests.xml
 coverage.xml
 *,cover
+docs/reports/**
 
 # Django stuff:
 *.log
@@ -54,7 +55,6 @@ docs/_build/
 target/
 
 \.idea/dataSources/
-
 \.idea/dataSources\.xml
 
 \.idea/dataSources\.local\.xml
@@ -74,15 +74,17 @@ mediafiles/
 \.env
 staticfiles/
 postgresql/
+data/
 
 
 /docker-compose.override.yml
 vue/node_modules
 plugins
-.vscode/
 vetur.config.js
 cookbook/static/vue
 vue/webpack-stats.json
 cookbook/templates/sw.js
 .prettierignore
 vue/.yarn
+vue3/.vite
+vue3/node_modules

.idea/dictionaries/vaben.xml

@@ -1,8 +1,10 @@
 <component name="ProjectDictionaryState">
   <dictionary name="vaben">
     <words>
+      <w>mealplan</w>
       <w>pinia</w>
       <w>selfhosted</w>
+      <w>unapplied</w>
     </words>
   </dictionary>
 </component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,5 +1,6 @@
 <component name="InspectionProjectProfileManager">
   <settings>
+    <option name="PROJECT_PROFILE" value="Default" />
     <option name="USE_PROJECT_PROFILE" value="false" />
     <version value="1.0" />
   </settings>

.idea/vcs.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+    <mapping directory="" vcs="Git" />
   </component>
 </project>

.vscode/launch.json

@@ -0,0 +1,34 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+      
+      {
+        "name": "Python Debugger: Django",
+        "type": "debugpy",
+        "request": "launch",
+        "program": "${workspaceFolder}/manage.py",
+        "args": ["runserver"],
+        "django": true,
+        "justMyCode": true
+      },
+    {
+      "name": "Python: Debug Tests",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "purpose": [
+        "debug-test"
+      ],
+      "console": "integratedTerminal",
+      "env": {
+        "//comment": "coverage and pytest can't both be running at the same time",
+        "PYTEST_ADDOPTS": "--no-cov -n 2"
+      },
+      "django": true,
+      "justMyCode": true
+    },
+    ]
+  }

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "python.testing.pytestArgs": [
+        "cookbook/tests"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

.vscode/tasks.json

@@ -0,0 +1,75 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+      {
+        "label": "Run Migrations",
+        "type": "shell",
+        "command": "python3 manage.py migrate",
+      },
+      {
+        "label": "Collect Static Files",
+        "type": "shell",
+        "command": "python3 manage.py collectstatic",
+        "dependsOn": ["Yarn Build"],
+      },
+      {
+        "label": "Setup Dev Server",
+        "dependsOn": ["Run Migrations", "Yarn Build"],
+      },
+      {
+        "label": "Run Dev Server",
+        "type": "shell",
+        "dependsOn": ["Setup Dev Server"],
+        "command": "python3 manage.py runserver",
+      },
+      {
+        "label": "Yarn Install",
+        "type": "shell",
+        "command": "yarn install",
+        "options": {
+          "cwd": "${workspaceFolder}/vue"
+        }
+      },
+      {
+        "label": "Yarn Serve",
+        "type": "shell",
+        "command": "yarn serve",
+        "dependsOn": ["Yarn Install"],
+        "options": {
+          "cwd": "${workspaceFolder}/vue"
+        }
+      },
+      {
+        "label": "Yarn Build",
+        "type": "shell",
+        "command": "yarn build",
+        "dependsOn": ["Yarn Install"],
+        "options": {
+          "cwd": "${workspaceFolder}/vue"
+        },
+        "group": "build",
+      },
+      {
+        "label": "Setup Tests",
+        "dependsOn": ["Run Migrations", "Collect Static Files"],
+      },
+      {
+        "label": "Run all pytests",
+        "type": "shell",
+        "command": "python3 -m pytest cookbook/tests",
+        "dependsOn": ["Setup Tests"],
+        "group": "test",
+      },
+      {
+        "label": "Setup Documentation Dependencies",
+        "type": "shell",
+        "command": "pip install mkdocs-material mkdocs-include-markdown-plugin",
+      },
+      {
+        "label": "Serve Documentation",
+        "type": "shell",
+        "command": "mkdocs serve",
+        "dependsOn": ["Setup Documentation Dependencies"],
+      }
+    ]
+}

CONTRIBUTERS.md

@@ -20,6 +20,7 @@ Below are some of the larger contributions made yet.
 - [murphy83] added support for IPv6 #1490
 - [TheHaf] added custom serving size component #1411
 - [lostlont] added LDAP support #960
+- [c0mputerguru] added devcontainers for ease of development
 
 ## Translations
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-alpine3.18
+FROM python:3.12-alpine3.19
 
 #Install all dependencies.
 RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg libwebp libxml2-dev libxslt-dev openldap git
@@ -19,12 +19,14 @@ RUN \
     if [ `apk --print-arch` = "armv7" ]; then \
     printf "[global]\nextra-index-url=https://www.piwheels.org/simple\n" > /etc/pip.conf ; \
     fi
+# remove Development dependencies from requirements.txt
+RUN sed -i '/# Development/,$d' requirements.txt
 RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-dev jpeg-dev libwebp-dev openssl-dev libffi-dev cargo openldap-dev python3-dev && \
     echo -n "INPUT ( libldap.so )" > /usr/lib/libldap_r.so && \
     python -m venv venv && \
     /opt/recipes/venv/bin/python -m pip install --upgrade pip && \
-    venv/bin/pip install wheel==0.37.1 && \
-    venv/bin/pip install setuptools_rust==1.1.2 && \
+    venv/bin/pip install wheel==0.42.0 && \
+    venv/bin/pip install setuptools_rust==1.9.0 && \
     venv/bin/pip install -r requirements.txt --no-cache-dir &&\
     apk --purge del .build-deps
 

README.md

@@ -39,13 +39,13 @@
 
 - 🔍 Powerful & customizable **search** with fulltext support and [TrigramSimilarity](https://docs.djangoproject.com/en/3.0/ref/contrib/postgres/search/#trigram-similarity)
 - 🏷️ Create and search for **tags**, assign them in batch to all files matching certain filters
-- ↔️ Quickly merge and rename ingredients, tags and units 
+- ↔️ Quickly merge and rename ingredients, tags and units
 - 📥️ **Import recipes** from thousands of websites supporting [ld+json or microdata](https://schema.org/Recipe)
 - ➗ Support for **fractions** or decimals
 - 🐳 Easy setup with **Docker** and included examples for **Kubernetes**, **Unraid** and **Synology**
 - 🎨 Customize your interface with **themes**
 - 📦 **Sync** files with Dropbox and Nextcloud
-  
+
 ## All the must haves
 
 - 📱Optimized for use on **mobile** devices
@@ -54,7 +54,7 @@
 - ➕ Many more like recipe scaling, image compression, printing views and supermarkets
 
 This application is meant for people with a collection of recipes they want to share with family and friends or simply
-store them in a nicely organized way. A basic permission system exists but this application is not meant to be run as 
+store them in a nicely organized way. A basic permission system exists but this application is not meant to be run as
 a public page.
 
 ## Docs
@@ -62,13 +62,13 @@ a public page.
 Documentation can be found [here](https://docs.tandoor.dev/).
 
 ## Support our work
-Tandoor is developed by volunteers in their free time just because its fun. That said earning 
+Tandoor is developed by volunteers in their free time just because its fun. That said earning
 some money with the project allows us to spend more time on it and thus make improvements we otherwise couldn't.
 Because of that there are several ways you can support us
 
 - **GitHub Sponsors** You can sponsor contributors of this project on GitHub: [vabene1111](https://github.com/sponsors/vabene1111)
 - **Host at Hetzner** We have been very happy customers of Hetzner for multiple years for all of our projects. If you want to get into self-hosting or are tired of the expensive big providers, their cloud servers are a great place to get started. When you sign up via our [referral link](https://hetzner.cloud/?ref=ISdlrLmr9kGj) you will get 20€ worth of cloud credits and we get a small kickback too.
-- **Let us host for you** We are offering a [hosted version](https://app.tandoor.dev) where all profits support us and the development of tandoor (currently only available in germany).  
+- **Let us host for you** We are offering a [hosted version](https://app.tandoor.dev) where all profits support us and the development of tandoor (currently only available in germany).
 
 ## Contributing
 Contributions are welcome but please read [this](https://docs.tandoor.dev/contribute/#contributing-code) **BEFORE** contributing anything!
@@ -96,11 +96,11 @@ Share some information on how you use Tandoor to help me improve the application
 Beginning with version 0.10.0 the code in this repository is licensed under the [GNU AGPL v3](https://www.gnu.org/licenses/agpl-3.0.de.html) license with a
 [common clause](https://commonsclause.com/) selling exception. See [LICENSE.md](https://github.com/vabene1111/recipes/blob/develop/LICENSE.md) for details.
 
-> NOTE: There appears to be a whole range of legal issues with licensing anything else then the standard completely open licenses.
+> NOTE: There appears to be a whole range of legal issues with licensing anything other than the standard completely open licenses.
 > I am in the process of getting some professional legal advice to sort out these issues. 
 > Please also see [Issue 238](https://github.com/vabene1111/recipes/issues/238) for some discussion and **reasoning** regarding the topic.
 
-**Reasoning**  
+**Reasoning**
 **This software and *all* its features are and will always be free for everyone to use and enjoy.**
 
 The reason for the selling exception is that a significant amount of time was spend over multiple years to develop this software.

boot.sh

@@ -19,9 +19,14 @@ if [ ! -f "$NGINX_CONF_FILE" ] && [ $GUNICORN_MEDIA -eq 0 ]; then
     display_warning "Nginx configuration file could not be found at the default location!\nPath: ${NGINX_CONF_FILE}"
 fi
 
-# SECRET_KEY must be set in .env file
+# SECRET_KEY (or a valid file at SECRET_KEY_FILE) must be set in .env file
+
+if [ -f "${SECRET_KEY_FILE}" ]; then
+    export SECRET_KEY=$(cat "$SECRET_KEY_FILE")
+fi
+
 if [ -z "${SECRET_KEY}" ]; then
-    display_warning "The environment variable 'SECRET_KEY' is not set but REQUIRED for running Tandoor!"
+    display_warning "The environment variable 'SECRET_KEY' (or 'SECRET_KEY_FILE' that points to an existing file) is not set but REQUIRED for running Tandoor!"
 fi
 
 
@@ -30,11 +35,16 @@ echo "Waiting for database to be ready..."
 attempt=0
 max_attempts=20
 
-if [ "${DB_ENGINE}" != 'django.db.backends.sqlite3' ]; then
+if [ "${DB_ENGINE}" == 'django.db.backends.postgresql' ] || [ "${DATABASE_URL}" == 'postgres'* ]; then
+
+  # POSTGRES_PASSWORD (or a valid file at POSTGRES_PASSWORD_FILE) must be set in .env file
+
+  if [ -f "${POSTGRES_PASSWORD_FILE}" ]; then
+    export POSTGRES_PASSWORD=$(cat "$POSTGRES_PASSWORD_FILE")
+  fi
 
-  # POSTGRES_PASSWORD must be set in .env file
   if [ -z "${POSTGRES_PASSWORD}" ]; then
-      display_warning "The environment variable 'POSTGRES_PASSWORD' is not set but REQUIRED for running Tandoor!"
+      display_warning "The environment variable 'POSTGRES_PASSWORD' (or 'POSTGRES_PASSWORD_FILE' that points to an existing file) is not set but REQUIRED for running Tandoor!"
   fi
 
   while pg_isready --host=${POSTGRES_HOST} --port=${POSTGRES_PORT} --user=${POSTGRES_USER} -q; status=$?; attempt=$((attempt+1)); [ $status -ne 0 ] && [ $attempt -le $max_attempts ]; do
@@ -66,4 +76,11 @@ echo "Done"
 
 chmod -R 755 /opt/recipes/mediafiles
 
-exec gunicorn -b :$TANDOOR_PORT --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --access-logfile - --error-logfile - --log-level $GUNICORN_LOG_LEVEL recipes.wsgi
+ipv6_disable=$(cat /sys/module/ipv6/parameters/disable)
+
+# Check if IPv6 is enabled, only then run gunicorn with ipv6 support
+if [ "$ipv6_disable" -eq 0 ]; then
+    exec gunicorn -b "[::]:$TANDOOR_PORT" --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --access-logfile - --error-logfile - --log-level $GUNICORN_LOG_LEVEL recipes.wsgi
+else
+    exec gunicorn -b ":$TANDOOR_PORT" --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --access-logfile - --error-logfile - --log-level $GUNICORN_LOG_LEVEL recipes.wsgi
+fi

cookbook/admin.py

@@ -10,13 +10,13 @@ from treebeard.forms import movenodeform_factory
 
 from cookbook.managers import DICTIONARY
 
-from .models import (Automation, BookmarkletImport, Comment, CookLog, Food, FoodInheritField,
-                     ImportLog, Ingredient, InviteLink, Keyword, MealPlan, MealType,
-                     NutritionInformation, Property, PropertyType, Recipe, RecipeBook,
-                     RecipeBookEntry, RecipeImport, SearchPreference, ShareLink, ShoppingList,
-                     ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage, Supermarket,
-                     SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog, TelegramBot,
-                     Unit, UnitConversion, UserFile, UserPreference, UserSpace, ViewLog)
+from .models import (BookmarkletImport, Comment, CookLog, Food, ImportLog, Ingredient, InviteLink,
+                     Keyword, MealPlan, MealType, NutritionInformation, Property, PropertyType,
+                     Recipe, RecipeBook, RecipeBookEntry, RecipeImport, SearchPreference, ShareLink,
+                     ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage,
+                     Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog,
+                     TelegramBot, Unit, UnitConversion, UserFile, UserPreference, UserSpace,
+                     ViewLog, ConnectorConfig)
 
 
 class CustomUserAdmin(UserAdmin):
@@ -60,9 +60,9 @@ admin.site.register(UserSpace, UserSpaceAdmin)
 
 
 class UserPreferenceAdmin(admin.ModelAdmin):
-    list_display = ('name', 'theme', 'nav_color', 'default_page',)
+    list_display = ('name', 'theme', 'default_page')
     search_fields = ('user__username',)
-    list_filter = ('theme', 'nav_color', 'default_page',)
+    list_filter = ('theme', 'default_page',)
     date_hierarchy = 'created_at'
     filter_horizontal = ('plan_share', 'shopping_share',)
 
@@ -95,6 +95,14 @@ class StorageAdmin(admin.ModelAdmin):
 admin.site.register(Storage, StorageAdmin)
 
 
+class ConnectorConfigAdmin(admin.ModelAdmin):
+    list_display = ('id', 'name', 'type', 'enabled', 'url')
+    search_fields = ('name', 'url')
+
+
+admin.site.register(ConnectorConfig, ConnectorConfigAdmin)
+
+
 class SyncAdmin(admin.ModelAdmin):
     list_display = ('storage', 'path', 'active', 'last_checked')
     search_fields = ('storage__name', 'path')
@@ -108,11 +116,16 @@ class SupermarketCategoryInline(admin.TabularInline):
 
 
 class SupermarketAdmin(admin.ModelAdmin):
+    list_display = ('name', 'space',)
     inlines = (SupermarketCategoryInline,)
 
 
+class SupermarketCategoryAdmin(admin.ModelAdmin):
+    list_display = ('name', 'space',)
+
+
 admin.site.register(Supermarket, SupermarketAdmin)
-admin.site.register(SupermarketCategory)
+admin.site.register(SupermarketCategory, SupermarketCategoryAdmin)
 
 
 class SyncLogAdmin(admin.ModelAdmin):
@@ -163,10 +176,18 @@ def delete_unattached_steps(modeladmin, request, queryset):
 
 
 class StepAdmin(admin.ModelAdmin):
-    list_display = ('name', 'order',)
-    search_fields = ('name',)
+    list_display = ('recipe_and_name', 'order', 'space')
+    ordering = ('recipe__name', 'name', 'space',)
+    search_fields = ('name', 'recipe__name')
     actions = [delete_unattached_steps]
 
+    @staticmethod
+    @admin.display(description="Name")
+    def recipe_and_name(obj):
+        if not obj.recipe_set.exists():
+            return f"Orphaned Step{'':s if not obj.name else f': {obj.name}'}"
+        return f"{obj.recipe_set.first().name}: {obj.name}" if obj.name else obj.recipe_set.first().name
+
 
 admin.site.register(Step, StepAdmin)
 
@@ -183,8 +204,9 @@ def rebuild_index(modeladmin, request, queryset):
 
 
 class RecipeAdmin(admin.ModelAdmin):
-    list_display = ('name', 'internal', 'created_by', 'storage')
+    list_display = ('name', 'internal', 'created_by', 'storage', 'space')
     search_fields = ('name', 'created_by__username')
+    ordering = ('name', 'created_by__username',)
     list_filter = ('internal',)
     date_hierarchy = 'created_at'
 
@@ -192,13 +214,20 @@ class RecipeAdmin(admin.ModelAdmin):
     def created_by(obj):
         return obj.created_by.get_user_display_name()
 
-    if settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2', 'django.db.backends.postgresql']:
+    if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql':
         actions = [rebuild_index]
 
 
 admin.site.register(Recipe, RecipeAdmin)
 
-admin.site.register(Unit)
+
+class UnitAdmin(admin.ModelAdmin):
+    list_display = ('name', 'space')
+    ordering = ('name', 'space',)
+    search_fields = ('name',)
+
+
+admin.site.register(Unit, UnitAdmin)
 
 
 # admin.site.register(FoodInheritField)
@@ -229,10 +258,16 @@ def delete_unattached_ingredients(modeladmin, request, queryset):
 
 
 class IngredientAdmin(admin.ModelAdmin):
-    list_display = ('food', 'amount', 'unit')
-    search_fields = ('food__name', 'unit__name')
+    list_display = ('recipe_name', 'amount', 'unit', 'food', 'space')
+    search_fields = ('food__name', 'unit__name', 'step__recipe__name')
     actions = [delete_unattached_ingredients]
 
+    @staticmethod
+    @admin.display(description="Recipe")
+    def recipe_name(obj):
+        recipes = obj.step_set.first().recipe_set.all() if obj.step_set.exists() else None
+        return recipes.first().name if recipes else 'Orphaned Ingredient'
+
 
 admin.site.register(Ingredient, IngredientAdmin)
 
@@ -258,7 +293,7 @@ admin.site.register(RecipeImport, RecipeImportAdmin)
 
 
 class RecipeBookAdmin(admin.ModelAdmin):
-    list_display = ('name', 'user_name')
+    list_display = ('name', 'user_name', 'space')
     search_fields = ('name', 'created_by__username')
 
     @staticmethod
@@ -277,7 +312,7 @@ admin.site.register(RecipeBookEntry, RecipeBookEntryAdmin)
 
 
 class MealPlanAdmin(admin.ModelAdmin):
-    list_display = ('user', 'recipe', 'meal_type', 'date')
+    list_display = ('user', 'recipe', 'meal_type', 'from_date', 'to_date')
 
     @staticmethod
     def user(obj):
@@ -288,8 +323,8 @@ admin.site.register(MealPlan, MealPlanAdmin)
 
 
 class MealTypeAdmin(admin.ModelAdmin):
-    list_display = ('name', 'created_by', 'order')
-    search_fields = ('name', 'created_by__username')
+    list_display = ('name', 'space', 'created_by', 'order')
+    search_fields = ('name', 'space', 'created_by__username')
 
 
 admin.site.register(MealType, MealTypeAdmin)
@@ -334,13 +369,6 @@ class ShoppingListEntryAdmin(admin.ModelAdmin):
 admin.site.register(ShoppingListEntry, ShoppingListEntryAdmin)
 
 
-class ShoppingListAdmin(admin.ModelAdmin):
-    list_display = ('id', 'created_by', 'created_at')
-
-
-admin.site.register(ShoppingList, ShoppingListAdmin)
-
-
 class ShareLinkAdmin(admin.ModelAdmin):
     list_display = ('recipe', 'created_by', 'uuid', 'created_at',)
 
@@ -349,7 +377,9 @@ admin.site.register(ShareLink, ShareLinkAdmin)
 
 
 class PropertyTypeAdmin(admin.ModelAdmin):
-    list_display = ('id', 'name')
+    search_fields = ('space',)
+
+    list_display = ('id', 'space', 'name', 'fdc_id')
 
 
 admin.site.register(PropertyType, PropertyTypeAdmin)

cookbook/apps.py

@@ -3,6 +3,7 @@ import traceback
 from django.apps import AppConfig
 from django.conf import settings
 from django.db import OperationalError, ProgrammingError
+from django.db.models.signals import post_save, post_delete
 from django_scopes import scopes_disabled
 
 from recipes.settings import DEBUG
@@ -14,6 +15,16 @@ class CookbookConfig(AppConfig):
     def ready(self):
         import cookbook.signals  # noqa
 
+        if not settings.DISABLE_EXTERNAL_CONNECTORS:
+            try:
+                from cookbook.connectors.connector_manager import ConnectorManager  # Needs to be here to prevent loading race condition of oauth2 modules in models.py
+                handler = ConnectorManager()
+                post_save.connect(handler, dispatch_uid="connector_manager")
+                post_delete.connect(handler, dispatch_uid="connector_manager")
+            except Exception as e:
+                traceback.print_exc()
+                print('Failed to initialize connectors')
+                pass
         # if not settings.DISABLE_TREE_FIX_STARTUP:
         #     # when starting up run fix_tree to:
         #     #   a) make sure that nodes are sorted when switching between sort modes

cookbook/connectors/connector.py

@@ -0,0 +1,29 @@
+from abc import ABC, abstractmethod
+
+from cookbook.models import ShoppingListEntry, Space, ConnectorConfig
+
+
+# A Connector is 'destroyed' & recreated each time 'any' ConnectorConfig in a space changes.
+class Connector(ABC):
+    @abstractmethod
+    def __init__(self, config: ConnectorConfig):
+        pass
+
+    @abstractmethod
+    async def on_shopping_list_entry_created(self, space: Space, instance: ShoppingListEntry) -> None:
+        pass
+
+    # This method might not trigger on 'direct' entry updates: https://stackoverflow.com/a/35238823
+    @abstractmethod
+    async def on_shopping_list_entry_updated(self, space: Space, instance: ShoppingListEntry) -> None:
+        pass
+
+    @abstractmethod
+    async def on_shopping_list_entry_deleted(self, space: Space, instance: ShoppingListEntry) -> None:
+        pass
+
+    @abstractmethod
+    async def close(self) -> None:
+        pass
+
+    # TODO: Add Recipes & possibly Meal Place listeners/hooks (And maybe more?)

cookbook/connectors/connector_manager.py

@@ -0,0 +1,179 @@
+import asyncio
+import logging
+import queue
+import threading
+from asyncio import Task
+from dataclasses import dataclass
+from enum import Enum
+from types import UnionType
+from typing import List, Any, Dict, Optional, Type
+
+from django.conf import settings
+from django_scopes import scope
+
+from cookbook.connectors.connector import Connector
+from cookbook.connectors.homeassistant import HomeAssistant
+from cookbook.models import ShoppingListEntry, Space, ConnectorConfig
+
+REGISTERED_CLASSES: UnionType | Type = ShoppingListEntry
+
+
+class ActionType(Enum):
+    CREATED = 1
+    UPDATED = 2
+    DELETED = 3
+
+
+@dataclass
+class Work:
+    instance: REGISTERED_CLASSES | ConnectorConfig
+    actionType: ActionType
+
+
+# The way ConnectionManager works is as follows:
+# 1. On init, it starts a worker & creates a queue for 'Work'
+# 2. Then any time its called, it verifies the type of action (create/update/delete) and if the item is of interest, pushes the Work (non-blocking) to the queue.
+# 3. The worker consumes said work from the queue.
+# 3.1 If the work is of type ConnectorConfig, it flushes its cache of known connectors (per space.id)
+# 3.2 If work is of type REGISTERED_CLASSES, it asynchronously fires of all connectors and wait for them to finish (runtime should depend on the 'slowest' connector)
+# 4. Work is marked as consumed, and next entry of the queue is consumed.
+# Each 'Work' is processed in sequential by the worker, so the throughput is about [workers * the slowest connector]
+class ConnectorManager:
+    _queue: queue.Queue
+    _listening_to_classes = REGISTERED_CLASSES | ConnectorConfig
+
+    def __init__(self):
+        self._queue = queue.Queue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE)
+        self._worker = threading.Thread(target=self.worker, args=(0, self._queue,), daemon=True)
+        self._worker.start()
+
+    # Called by post save & post delete signals
+    def __call__(self, instance: Any, **kwargs) -> None:
+        if not isinstance(instance, self._listening_to_classes) or not hasattr(instance, "space"):
+            return
+
+        action_type: ActionType
+        if "created" in kwargs and kwargs["created"]:
+            action_type = ActionType.CREATED
+        elif "created" in kwargs and not kwargs["created"]:
+            action_type = ActionType.UPDATED
+        elif "origin" in kwargs:
+            action_type = ActionType.DELETED
+        else:
+            return
+
+        try:
+            self._queue.put_nowait(Work(instance, action_type))
+        except queue.Full:
+            logging.info(f"queue was full, so skipping {action_type} of type {type(instance)}")
+            return
+
+    def stop(self):
+        self._queue.join()
+        self._worker.join()
+
+    @staticmethod
+    def worker(worker_id: int, worker_queue: queue.Queue):
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+        logging.info(f"started ConnectionManager worker {worker_id}")
+
+        # When multiple workers are used, please make sure the cache is shared across all threads, otherwise it might lead to un-expected behavior.
+        _connectors_cache: Dict[int, List[Connector]] = dict()
+
+        while True:
+            try:
+                item: Optional[Work] = worker_queue.get()
+            except KeyboardInterrupt:
+                break
+
+            if item is None:
+                break
+
+            # If a Connector was changed/updated, refresh connector from the database for said space
+            refresh_connector_cache = isinstance(item.instance, ConnectorConfig)
+
+            space: Space = item.instance.space
+            connectors: Optional[List[Connector]] = _connectors_cache.get(space.id)
+
+            if connectors is None or refresh_connector_cache:
+                if connectors is not None:
+                    loop.run_until_complete(close_connectors(connectors))
+
+                with scope(space=space):
+                    connectors: List[Connector] = list()
+                    for config in space.connectorconfig_set.all():
+                        config: ConnectorConfig = config
+                        if not config.enabled:
+                            continue
+
+                        try:
+                            connector: Optional[Connector] = ConnectorManager.get_connected_for_config(config)
+                        except BaseException:
+                            logging.exception(f"failed to initialize {config.name}")
+                            continue
+
+                        if connector is not None:
+                            connectors.append(connector)
+
+                    _connectors_cache[space.id] = connectors
+
+            if len(connectors) == 0 or refresh_connector_cache:
+                worker_queue.task_done()
+                continue
+
+            loop.run_until_complete(run_connectors(connectors, space, item.instance, item.actionType))
+            worker_queue.task_done()
+
+        logging.info(f"terminating ConnectionManager worker {worker_id}")
+
+        asyncio.set_event_loop(None)
+        loop.close()
+
+    @staticmethod
+    def get_connected_for_config(config: ConnectorConfig) -> Optional[Connector]:
+        match config.type:
+            case ConnectorConfig.HOMEASSISTANT:
+                return HomeAssistant(config)
+            case _:
+                return None
+
+
+async def close_connectors(connectors: List[Connector]):
+    tasks: List[Task] = [asyncio.create_task(connector.close()) for connector in connectors]
+
+    if len(tasks) == 0:
+        return
+
+    try:
+        await asyncio.gather(*tasks, return_exceptions=False)
+    except BaseException:
+        logging.exception("received an exception while closing one of the connectors")
+
+
+async def run_connectors(connectors: List[Connector], space: Space, instance: REGISTERED_CLASSES, action_type: ActionType):
+    tasks: List[Task] = list()
+
+    if isinstance(instance, ShoppingListEntry):
+        shopping_list_entry: ShoppingListEntry = instance
+
+        match action_type:
+            case ActionType.CREATED:
+                for connector in connectors:
+                    tasks.append(asyncio.create_task(connector.on_shopping_list_entry_created(space, shopping_list_entry)))
+            case ActionType.UPDATED:
+                for connector in connectors:
+                    tasks.append(asyncio.create_task(connector.on_shopping_list_entry_updated(space, shopping_list_entry)))
+            case ActionType.DELETED:
+                for connector in connectors:
+                    tasks.append(asyncio.create_task(connector.on_shopping_list_entry_deleted(space, shopping_list_entry)))
+
+    if len(tasks) == 0:
+        return
+
+    try:
+        # Wait for all async tasks to finish, if one fails, the others still continue.
+        await asyncio.gather(*tasks, return_exceptions=False)
+    except BaseException:
+        logging.exception("received an exception from one of the connectors")

cookbook/connectors/homeassistant.py

@@ -0,0 +1,85 @@
+import logging
+from logging import Logger
+
+from homeassistant_api import Client, HomeassistantAPIError, Domain
+
+from cookbook.connectors.connector import Connector
+from cookbook.models import ShoppingListEntry, ConnectorConfig, Space
+
+
+class HomeAssistant(Connector):
+    _domains_cache: dict[str, Domain]
+    _config: ConnectorConfig
+    _logger: Logger
+    _client: Client
+
+    def __init__(self, config: ConnectorConfig):
+        if not config.token or not config.url or not config.todo_entity:
+            raise ValueError("config for HomeAssistantConnector in incomplete")
+
+        self._domains_cache = dict()
+        self._config = config
+        self._logger = logging.getLogger("connector.HomeAssistant")
+        self._client = Client(self._config.url, self._config.token, async_cache_session=False, use_async=True)
+
+    async def on_shopping_list_entry_created(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None:
+        if not self._config.on_shopping_list_entry_created_enabled:
+            return
+
+        item, description = _format_shopping_list_entry(shopping_list_entry)
+
+        todo_domain = self._domains_cache.get('todo')
+        try:
+            if todo_domain is None:
+                todo_domain = await self._client.async_get_domain('todo')
+                self._domains_cache['todo'] = todo_domain
+
+            logging.debug(f"pushing {item} to {self._config.name}")
+            await todo_domain.add_item(entity_id=self._config.todo_entity, item=item)
+        except HomeassistantAPIError as err:
+            self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}")
+
+    async def on_shopping_list_entry_updated(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None:
+        if not self._config.on_shopping_list_entry_updated_enabled:
+            return
+        pass
+
+    async def on_shopping_list_entry_deleted(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None:
+        if not self._config.on_shopping_list_entry_deleted_enabled:
+            return
+
+        item, description = _format_shopping_list_entry(shopping_list_entry)
+
+        todo_domain = self._domains_cache.get('todo')
+        try:
+            if todo_domain is None:
+                todo_domain = await self._client.async_get_domain('todo')
+                self._domains_cache['todo'] = todo_domain
+
+            logging.debug(f"deleting {item} from {self._config.name}")
+            await todo_domain.remove_item(entity_id=self._config.todo_entity, item=item)
+        except HomeassistantAPIError as err:
+            self._logger.warning(f"[HomeAssistant {self._config.name}] Received an exception from the api: {err=}, {type(err)=}")
+
+    async def close(self) -> None:
+        await self._client.async_cache_session.close()
+
+
+def _format_shopping_list_entry(shopping_list_entry: ShoppingListEntry):
+    item = shopping_list_entry.food.name
+    if shopping_list_entry.amount > 0:
+        item += f" ({shopping_list_entry.amount:.2f}".rstrip('0').rstrip('.')
+        if shopping_list_entry.unit and shopping_list_entry.unit.base_unit and len(shopping_list_entry.unit.base_unit) > 0:
+            item += f" {shopping_list_entry.unit.base_unit})"
+        elif shopping_list_entry.unit and shopping_list_entry.unit.name and len(shopping_list_entry.unit.name) > 0:
+            item += f" {shopping_list_entry.unit.name})"
+        else:
+            item += ")"
+
+    description = "Imported by TandoorRecipes"
+    if shopping_list_entry.created_by.first_name and len(shopping_list_entry.created_by.first_name) > 0:
+        description += f", created by {shopping_list_entry.created_by.first_name}"
+    else:
+        description += f", created by {shopping_list_entry.created_by.username}"
+
+    return item, description

cookbook/forms.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 
+from allauth.account.forms import ResetPasswordForm, SignupForm
 from django import forms
 from django.conf import settings
 from django.core.exceptions import ValidationError
@@ -9,18 +10,19 @@ from django_scopes import scopes_disabled
 from django_scopes.forms import SafeModelChoiceField, SafeModelMultipleChoiceField
 from hcaptcha.fields import hCaptchaField
 
-from .models import (Comment, Food, InviteLink, Keyword, MealPlan, MealType, Recipe, RecipeBook,
-                     RecipeBookEntry, SearchPreference, Space, Storage, Sync, User, UserPreference)
+from .models import Comment, InviteLink, Keyword, Recipe, SearchPreference, Space, Storage, Sync, User, UserPreference, ConnectorConfig
 
 
 class SelectWidget(widgets.Select):
+
     class Media:
-        js = ('custom/js/form_select.js',)
+        js = ('custom/js/form_select.js', )
 
 
 class MultiSelectWidget(widgets.SelectMultiple):
+
     class Media:
-        js = ('custom/js/form_multiselect.js',)
+        js = ('custom/js/form_multiselect.js', )
 
 
 # Yes there are some stupid browsers that still dont support this but
@@ -33,64 +35,6 @@ class DateWidget(forms.DateInput):
         super().__init__(**kwargs)
 
 
-class UserPreferenceForm(forms.ModelForm):
-    prefix = 'preference'
-
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['plan_share'].queryset = User.objects.filter(userspace__space=space).all()
-
-    class Meta:
-        model = UserPreference
-        fields = (
-            'default_unit', 'use_fractions', 'use_kj', 'theme', 'nav_color',
-            'sticky_navbar', 'default_page', 'plan_share', 'ingredient_decimals', 'comments', 'left_handed',
-        )
-
-        labels = {
-            'default_unit': _('Default unit'),
-            'use_fractions': _('Use fractions'),
-            'use_kj': _('Use KJ'),
-            'theme': _('Theme'),
-            'nav_color': _('Navbar color'),
-            'sticky_navbar': _('Sticky navbar'),
-            'default_page': _('Default page'),
-            'plan_share': _('Plan sharing'),
-            'ingredient_decimals': _('Ingredient decimal places'),
-            'shopping_auto_sync': _('Shopping list auto sync period'),
-            'comments': _('Comments'),
-            'left_handed': _('Left-handed mode')
-        }
-
-        help_texts = {
-            'nav_color': _('Color of the top navigation bar. Not all colors work with all themes, just try them out!'),
-            
-            'default_unit': _('Default Unit to be used when inserting a new ingredient into a recipe.'),  
-            'use_fractions': _(
-                'Enables support for fractions in ingredient amounts (e.g. convert decimals to fractions automatically)'),
-            
-            'use_kj': _('Display nutritional energy amounts in joules instead of calories'),  
-            'plan_share': _('Users with whom newly created meal plans should be shared by default.'),
-            'shopping_share': _('Users with whom to share shopping lists.'),
-            'ingredient_decimals': _('Number of decimals to round ingredients.'),  
-            'comments': _('If you want to be able to create and see comments underneath recipes.'),  
-            'shopping_auto_sync': _(
-                'Setting to 0 will disable auto sync. When viewing a shopping list the list is updated every set seconds to sync changes someone else might have made. Useful when shopping with multiple people but might use a little bit '  
-                'of mobile data. If lower than instance limit it is reset when saving.'  
-            ),
-            'sticky_navbar': _('Makes the navbar stick to the top of the page.'),  
-            'mealplan_autoadd_shopping': _('Automatically add meal plan ingredients to shopping list.'),
-            'mealplan_autoexclude_onhand': _('Exclude ingredients that are on hand.'),
-            'left_handed': _('Will optimize the UI for use with your left hand.')
-        }
-
-        widgets = {
-            'plan_share': MultiSelectWidget,
-            'shopping_share': MultiSelectWidget,
-        }
-
-
 class UserNameForm(forms.ModelForm):
     prefix = 'name'
 
@@ -98,9 +42,7 @@ class UserNameForm(forms.ModelForm):
         model = User
         fields = ('first_name', 'last_name')
 
-        help_texts = {
-            'first_name': _('Both fields are optional. If none are given the username will be displayed instead')
-        }
+        help_texts = {'first_name': _('Both fields are optional. If none are given the username will be displayed instead')}
 
 
 class ExternalRecipeForm(forms.ModelForm):
@@ -114,23 +56,14 @@ class ExternalRecipeForm(forms.ModelForm):
 
     class Meta:
         model = Recipe
-        fields = (
-            'name', 'description', 'servings', 'working_time', 'waiting_time',
-            'file_path', 'file_uid', 'keywords'
-        )
+        fields = ('name', 'description', 'servings', 'working_time', 'waiting_time', 'file_path', 'file_uid', 'keywords')
 
         labels = {
-            'name': _('Name'),
-            'keywords': _('Keywords'),
-            'working_time': _('Preparation time in minutes'),
-            'waiting_time': _('Waiting time (cooking/baking) in minutes'),
-            'file_path': _('Path'),
-            'file_uid': _('Storage UID'),
+            'name': _('Name'), 'keywords': _('Keywords'), 'working_time': _('Preparation time in minutes'), 'waiting_time': _('Waiting time (cooking/baking) in minutes'),
+            'file_path': _('Path'), 'file_uid': _('Storage UID'),
         }
         widgets = {'keywords': MultiSelectWidget}
-        field_classes = {
-            'keywords': SafeModelMultipleChoiceField,
-        }
+        field_classes = {'keywords': SafeModelMultipleChoiceField, }
 
 
 class ImportExportBase(forms.Form):
@@ -157,14 +90,11 @@ class ImportExportBase(forms.Form):
     REZEPTSUITEDE = 'REZEPTSUITEDE'
     PDF = 'PDF'
 
-    type = forms.ChoiceField(choices=(
-        (DEFAULT, _('Default')), (PAPRIKA, 'Paprika'), (NEXTCLOUD, 'Nextcloud Cookbook'),
-        (MEALIE, 'Mealie'), (CHOWDOWN, 'Chowdown'), (SAFFRON, 'Saffron'), (CHEFTAP, 'ChefTap'),
-        (PEPPERPLATE, 'Pepperplate'), (RECETTETEK, 'RecetteTek'), (RECIPESAGE, 'Recipe Sage'), (DOMESTICA, 'Domestica'),
-        (MEALMASTER, 'MealMaster'), (REZKONV, 'RezKonv'), (OPENEATS, 'Openeats'), (RECIPEKEEPER, 'Recipe Keeper'),
-        (PLANTOEAT, 'Plantoeat'), (COOKBOOKAPP, 'CookBookApp'), (COPYMETHAT, 'CopyMeThat'), (PDF, 'PDF'), (MELARECIPES, 'Melarecipes'),
-        (COOKMATE, 'Cookmate'), (REZEPTSUITEDE, 'Recipesuite.de')
-    ))
+    type = forms.ChoiceField(choices=((DEFAULT, _('Default')), (PAPRIKA, 'Paprika'), (NEXTCLOUD, 'Nextcloud Cookbook'), (MEALIE, 'Mealie'), (CHOWDOWN, 'Chowdown'),
+                                      (SAFFRON, 'Saffron'), (CHEFTAP, 'ChefTap'), (PEPPERPLATE, 'Pepperplate'), (RECETTETEK, 'RecetteTek'), (RECIPESAGE, 'Recipe Sage'),
+                                      (DOMESTICA, 'Domestica'), (MEALMASTER, 'MealMaster'), (REZKONV, 'RezKonv'), (OPENEATS, 'Openeats'), (RECIPEKEEPER, 'Recipe Keeper'),
+                                      (PLANTOEAT, 'Plantoeat'), (COOKBOOKAPP, 'CookBookApp'), (COPYMETHAT, 'CopyMeThat'), (PDF, 'PDF'), (MELARECIPES, 'Melarecipes'),
+                                      (COOKMATE, 'Cookmate'), (REZEPTSUITEDE, 'Recipesuite.de')))
 
 
 class MultipleFileInput(forms.ClearableFileInput):
@@ -172,6 +102,7 @@ class MultipleFileInput(forms.ClearableFileInput):
 
 
 class MultipleFileField(forms.FileField):
+
     def __init__(self, *args, **kwargs):
         kwargs.setdefault("widget", MultipleFileInput())
         super().__init__(*args, **kwargs)
@@ -184,11 +115,11 @@ class MultipleFileField(forms.FileField):
             result = single_file_clean(data, initial)
         return result
 
+
 class ImportForm(ImportExportBase):
     files = MultipleFileField(required=True)
-    duplicates = forms.BooleanField(help_text=_(
-        'To prevent duplicates recipes with the same name as existing ones are ignored. Check this box to import everything.'),
-        required=False)
+    duplicates = forms.BooleanField(help_text=_('To prevent duplicates recipes with the same name as existing ones are ignored. Check this box to import everything.'),
+                                    required=False)
 
 
 class ExportForm(ImportExportBase):
@@ -207,59 +138,71 @@ class CommentForm(forms.ModelForm):
 
     class Meta:
         model = Comment
-        fields = ('text',)
+        fields = ('text', )
 
-        labels = {
-            'text': _('Add your comment: '),
-        }
-        widgets = {
-            'text': forms.Textarea(attrs={'rows': 2, 'cols': 15}),
-        }
+        labels = {'text': _('Add your comment: '), }
+        widgets = {'text': forms.Textarea(attrs={'rows': 2, 'cols': 15}), }
 
 
 class StorageForm(forms.ModelForm):
-    username = forms.CharField(
-        widget=forms.TextInput(attrs={'autocomplete': 'new-password'}),
-        required=False
-    )
-    password = forms.CharField(
-        widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}),
-        required=False,
-        help_text=_('Leave empty for dropbox and enter app password for nextcloud.')
-    )
-    token = forms.CharField(
-        widget=forms.TextInput(
-            attrs={'autocomplete': 'new-password', 'type': 'password'}
-        ),
-        required=False,
-        help_text=_('Leave empty for nextcloud and enter api token for dropbox.')
-    )
+    username = forms.CharField(widget=forms.TextInput(attrs={'autocomplete': 'new-password'}), required=False)
+    password = forms.CharField(widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}),
+                               required=False,
+                               help_text=_('Leave empty for dropbox and enter app password for nextcloud.'))
+    token = forms.CharField(widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}),
+                            required=False,
+                            help_text=_('Leave empty for nextcloud and enter api token for dropbox.'))
 
     class Meta:
         model = Storage
         fields = ('name', 'method', 'username', 'password', 'token', 'url', 'path')
 
-        help_texts = {
-            'url': _(
-                'Leave empty for dropbox and enter only base url for nextcloud (<code>/remote.php/webdav/</code> is added automatically)'),
-        }
+        help_texts = {'url': _('Leave empty for dropbox and enter only base url for nextcloud (<code>/remote.php/webdav/</code> is added automatically)'), }
 
 
-# TODO: Deprecate
-class RecipeBookEntryForm(forms.ModelForm):
-    prefix = 'bookmark'
 
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['book'].queryset = RecipeBook.objects.filter(space=space).all()
+class ConnectorConfigForm(forms.ModelForm):
+    enabled = forms.BooleanField(
+        help_text="Is the connector enabled",
+        required=False,
+    )
+
+    on_shopping_list_entry_created_enabled = forms.BooleanField(
+        help_text="Enable action for ShoppingListEntry created events",
+        required=False,
+    )
+
+    on_shopping_list_entry_updated_enabled = forms.BooleanField(
+        help_text="Enable action for ShoppingListEntry updated events",
+        required=False,
+    )
+
+    on_shopping_list_entry_deleted_enabled = forms.BooleanField(
+        help_text="Enable action for ShoppingListEntry deleted events",
+        required=False,
+    )
+
+    update_token = forms.CharField(
+        widget=forms.TextInput(attrs={'autocomplete': 'update-token', 'type': 'password'}),
+        required=False,
+        help_text=_('<a href="https://www.home-assistant.io/docs/authentication/#your-account-profile">Long Lived Access Token</a> for your HomeAssistant instance')
+    )
+
+    url = forms.URLField(
+        required=False,
+        help_text=_('Something like http://homeassistant.local:8123/api'),
+    )
 
     class Meta:
-        model = RecipeBookEntry
-        fields = ('book',)
+        model = ConnectorConfig
 
-        field_classes = {
-            'book': SafeModelChoiceField,
+        fields = (
+            'name', 'type', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled',
+            'on_shopping_list_entry_deleted_enabled', 'url', 'todo_entity',
+        )
+
+        help_texts = {
+            'url': _('http://homeassistant.local:8123/api for example'),
         }
 
 
@@ -274,25 +217,14 @@ class SyncForm(forms.ModelForm):
         model = Sync
         fields = ('storage', 'path', 'active')
 
-        field_classes = {
-            'storage': SafeModelChoiceField,
-        }
+        field_classes = {'storage': SafeModelChoiceField, }
 
-        labels = {
-            'storage': _('Storage'),
-            'path': _('Path'),
-            'active': _('Active')
-        }
+        labels = {'storage': _('Storage'), 'path': _('Path'), 'active': _('Active')}
 
 
-# TODO deprecate
 class BatchEditForm(forms.Form):
     search = forms.CharField(label=_('Search String'))
-    keywords = forms.ModelMultipleChoiceField(
-        queryset=Keyword.objects.none(),
-        required=False,
-        widget=MultiSelectWidget
-    )
+    keywords = forms.ModelMultipleChoiceField(queryset=Keyword.objects.none(), required=False, widget=MultiSelectWidget)
 
     def __init__(self, *args, **kwargs):
         space = kwargs.pop('space')
@@ -301,6 +233,7 @@ class BatchEditForm(forms.Form):
 
 
 class ImportRecipeForm(forms.ModelForm):
+
     def __init__(self, *args, **kwargs):
         space = kwargs.pop('space')
         super().__init__(*args, **kwargs)
@@ -310,63 +243,13 @@ class ImportRecipeForm(forms.ModelForm):
         model = Recipe
         fields = ('name', 'keywords', 'file_path', 'file_uid')
 
-        labels = {
-            'name': _('Name'),
-            'keywords': _('Keywords'),
-            'file_path': _('Path'),
-            'file_uid': _('File ID'),
-        }
+        labels = {'name': _('Name'), 'keywords': _('Keywords'), 'file_path': _('Path'), 'file_uid': _('File ID'), }
         widgets = {'keywords': MultiSelectWidget}
-        field_classes = {
-            'keywords': SafeModelChoiceField,
-        }
-
-
-# TODO deprecate
-class MealPlanForm(forms.ModelForm):
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['recipe'].queryset = Recipe.objects.filter(space=space).all()
-        self.fields['meal_type'].queryset = MealType.objects.filter(space=space).all()
-        self.fields['shared'].queryset = User.objects.filter(userpreference__space=space).all()
-
-    def clean(self):
-        cleaned_data = super(MealPlanForm, self).clean()
-
-        if cleaned_data['title'] == '' and cleaned_data['recipe'] is None:
-            raise forms.ValidationError(
-                _('You must provide at least a recipe or a title.')
-            )
-
-        return cleaned_data
-
-    class Meta:
-        model = MealPlan
-        fields = (
-            'recipe', 'title', 'meal_type', 'note',
-            'servings', 'date', 'shared'
-        )
-
-        help_texts = {
-            'shared': _('You can list default users to share recipes with in the settings.'),  
-            'note': _('You can use markdown to format this field. See the <a href="/docs/markdown/">docs here</a>')
-            
-        }
-
-        widgets = {
-            'recipe': SelectWidget,
-            'date': DateWidget,
-            'shared': MultiSelectWidget
-        }
-        field_classes = {
-            'recipe': SafeModelChoiceField,
-            'meal_type': SafeModelChoiceField,
-            'shared': SafeModelMultipleChoiceField,
-        }
+        field_classes = {'keywords': SafeModelChoiceField, }
 
 
 class InviteLinkForm(forms.ModelForm):
+
     def __init__(self, *args, **kwargs):
         user = kwargs.pop('user')
         super().__init__(*args, **kwargs)
@@ -374,8 +257,8 @@ class InviteLinkForm(forms.ModelForm):
 
     def clean(self):
         space = self.cleaned_data['space']
-        if space.max_users != 0 and (UserPreference.objects.filter(space=space).count() +
-                                     InviteLink.objects.filter(valid_until__gte=datetime.today(), used_by=None, space=space).count()) >= space.max_users:
+        if space.max_users != 0 and (UserPreference.objects.filter(space=space).count()
+                                     + InviteLink.objects.filter(valid_until__gte=datetime.today(), used_by=None, space=space).count()) >= space.max_users:
             raise ValidationError(_('Maximum number of users for this space reached.'))
 
     def clean_email(self):
@@ -389,12 +272,8 @@ class InviteLinkForm(forms.ModelForm):
     class Meta:
         model = InviteLink
         fields = ('email', 'group', 'valid_until', 'space')
-        help_texts = {
-            'email': _('An email address is not required but if present the invite link will be sent to the user.'),
-        }
-        field_classes = {
-            'space': SafeModelChoiceField,
-        }
+        help_texts = {'email': _('An email address is not required but if present the invite link will be sent to the user.'), }
+        field_classes = {'space': SafeModelChoiceField, }
 
 
 class SpaceCreateForm(forms.Form):
@@ -414,12 +293,12 @@ class SpaceJoinForm(forms.Form):
     token = forms.CharField()
 
 
-class AllAuthSignupForm(forms.Form):
+class AllAuthSignupForm(SignupForm):
     captcha = hCaptchaField()
     terms = forms.BooleanField(label=_('Accept Terms and Privacy'))
 
     def __init__(self, **kwargs):
-        super(AllAuthSignupForm, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         if settings.PRIVACY_URL == '' and settings.TERMS_URL == '':
             self.fields.pop('terms')
         if settings.HCAPTCHA_SECRET == '':
@@ -429,136 +308,50 @@ class AllAuthSignupForm(forms.Form):
         pass
 
 
+class CustomPasswordResetForm(ResetPasswordForm):
+    captcha = hCaptchaField()
+
+    def __init__(self, **kwargs):
+        super(CustomPasswordResetForm, self).__init__(**kwargs)
+        if settings.HCAPTCHA_SECRET == '':
+            self.fields.pop('captcha')
+
+
 class UserCreateForm(forms.Form):
     name = forms.CharField(label='Username')
-    password = forms.CharField(
-        widget=forms.TextInput(
-            attrs={'autocomplete': 'new-password', 'type': 'password'}
-        )
-    )
-    password_confirm = forms.CharField(
-        widget=forms.TextInput(
-            attrs={'autocomplete': 'new-password', 'type': 'password'}
-        )
-    )
+    password = forms.CharField(widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}))
+    password_confirm = forms.CharField(widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}))
 
 
 class SearchPreferenceForm(forms.ModelForm):
     prefix = 'search'
-    trigram_threshold = forms.DecimalField(min_value=0.01, max_value=1, decimal_places=2,
+    trigram_threshold = forms.DecimalField(min_value=0.01,
+                                           max_value=1,
+                                           decimal_places=2,
                                            widget=NumberInput(attrs={'class': "form-control-range", 'type': 'range'}),
-                                           help_text=_(
-                                               'Determines how fuzzy a search is if it uses trigram similarity matching (e.g. low values mean more typos are ignored).'))
+                                           help_text=_('Determines how fuzzy a search is if it uses trigram similarity matching (e.g. low values mean more typos are ignored).'))
     preset = forms.CharField(widget=forms.HiddenInput(), required=False)
 
     class Meta:
         model = SearchPreference
-        fields = (
-            'search', 'lookup', 'unaccent', 'icontains', 'istartswith', 'trigram', 'fulltext', 'trigram_threshold')
+        fields = ('search', 'lookup', 'unaccent', 'icontains', 'istartswith', 'trigram', 'fulltext', 'trigram_threshold')
 
         help_texts = {
-            'search': _(
-                'Select type method of search.  Click <a href="/docs/search/">here</a> for full description of choices.'),
-            'lookup': _('Use fuzzy matching on units, keywords and ingredients when editing and importing recipes.'),
-            'unaccent': _(
-                'Fields to search ignoring accents.  Selecting this option can improve or degrade search quality depending on language'),
-            'icontains': _(
-                "Fields to search for partial matches.  (e.g. searching for 'Pie' will return 'pie' and 'piece' and 'soapie')"),
-            'istartswith': _(
-                "Fields to search for beginning of word matches. (e.g. searching for 'sa' will return 'salad' and 'sandwich')"),
-            'trigram': _(
-                "Fields to 'fuzzy' search. (e.g. searching for 'recpie' will find 'recipe'.)  Note: this option will conflict with 'web' and 'raw' methods of search."),
-            'fulltext': _(
-                "Fields to full text search.  Note: 'web', 'phrase', and 'raw' search methods only function with fulltext fields."),
+            'search': _('Select type method of search.  Click <a href="/docs/search/">here</a> for full description of choices.'), 'lookup':
+            _('Use fuzzy matching on units, keywords and ingredients when editing and importing recipes.'), 'unaccent':
+            _('Fields to search ignoring accents.  Selecting this option can improve or degrade search quality depending on language'), 'icontains':
+            _("Fields to search for partial matches.  (e.g. searching for 'Pie' will return 'pie' and 'piece' and 'soapie')"), 'istartswith':
+            _("Fields to search for beginning of word matches. (e.g. searching for 'sa' will return 'salad' and 'sandwich')"), 'trigram':
+            _("Fields to 'fuzzy' search. (e.g. searching for 'recpie' will find 'recipe'.)  Note: this option will conflict with 'web' and 'raw' methods of search."), 'fulltext':
+            _("Fields to full text search.  Note: 'web', 'phrase', and 'raw' search methods only function with fulltext fields."),
         }
 
         labels = {
-            'search': _('Search Method'),
-            'lookup': _('Fuzzy Lookups'),
-            'unaccent': _('Ignore Accent'),
-            'icontains': _("Partial Match"),
-            'istartswith': _("Starts With"),
-            'trigram': _("Fuzzy Search"),
-            'fulltext': _("Full Text")
+            'search': _('Search Method'), 'lookup': _('Fuzzy Lookups'), 'unaccent': _('Ignore Accent'), 'icontains': _("Partial Match"), 'istartswith': _("Starts With"),
+            'trigram': _("Fuzzy Search"), 'fulltext': _("Full Text")
         }
 
         widgets = {
-            'search': SelectWidget,
-            'unaccent': MultiSelectWidget,
-            'icontains': MultiSelectWidget,
-            'istartswith': MultiSelectWidget,
-            'trigram': MultiSelectWidget,
-            'fulltext': MultiSelectWidget,
-        }
-
-
-class ShoppingPreferenceForm(forms.ModelForm):
-    prefix = 'shopping'
-
-    class Meta:
-        model = UserPreference
-
-        fields = (
-            'shopping_share', 'shopping_auto_sync', 'mealplan_autoadd_shopping', 'mealplan_autoexclude_onhand',
-            'mealplan_autoinclude_related', 'shopping_add_onhand', 'default_delay', 'filter_to_supermarket', 'shopping_recent_days', 'csv_delim', 'csv_prefix'
-        )
-
-        help_texts = {
-            'shopping_share': _('Users will see all items you add to your shopping list.  They must add you to see items on their list.'),
-            'shopping_auto_sync': _(
-                'Setting to 0 will disable auto sync. When viewing a shopping list the list is updated every set seconds to sync changes someone else might have made. Useful when shopping with multiple people but might use a little bit '  
-                'of mobile data. If lower than instance limit it is reset when saving.'  
-            ),
-            'mealplan_autoadd_shopping': _('Automatically add meal plan ingredients to shopping list.'),
-            'mealplan_autoinclude_related': _('When adding a meal plan to the shopping list (manually or automatically), include all related recipes.'),
-            'mealplan_autoexclude_onhand': _('When adding a meal plan to the shopping list (manually or automatically), exclude ingredients that are on hand.'),
-            'default_delay': _('Default number of hours to delay a shopping list entry.'),
-            'filter_to_supermarket': _('Filter shopping list to only include supermarket categories.'),
-            'shopping_recent_days': _('Days of recent shopping list entries to display.'),
-            'shopping_add_onhand': _("Mark food 'On Hand' when checked off shopping list."),
-            'csv_delim': _('Delimiter to use for CSV exports.'),
-            'csv_prefix': _('Prefix to add when copying list to the clipboard.'),
-
-        }
-        labels = {
-            'shopping_share': _('Share Shopping List'),
-            'shopping_auto_sync': _('Autosync'),
-            'mealplan_autoadd_shopping': _('Auto Add Meal Plan'),
-            'mealplan_autoexclude_onhand': _('Exclude On Hand'),
-            'mealplan_autoinclude_related': _('Include Related'),
-            'default_delay': _('Default Delay Hours'),
-            'filter_to_supermarket': _('Filter to Supermarket'),
-            'shopping_recent_days': _('Recent Days'),
-            'csv_delim': _('CSV Delimiter'),
-            "csv_prefix_label": _("List Prefix"),
-            'shopping_add_onhand': _("Auto On Hand"),
-        }
-
-        widgets = {
-            'shopping_share': MultiSelectWidget
-        }
-
-
-class SpacePreferenceForm(forms.ModelForm):
-    prefix = 'space'
-    reset_food_inherit = forms.BooleanField(label=_("Reset Food Inheritance"), initial=False, required=False,
-                                            help_text=_("Reset all food to inherit the fields configured."))
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)  # populates the post
-        self.fields['food_inherit'].queryset = Food.inheritable_fields
-
-    class Meta:
-        model = Space
-
-        fields = ('food_inherit', 'reset_food_inherit', 'show_facet_count', 'use_plural')
-
-        help_texts = {
-            'food_inherit': _('Fields on food that should be inherited by default.'),
-            'show_facet_count': _('Show recipe counts on search filters'), 
-            'use_plural': _('Use the plural form for units and food inside this space.'),
-        }
-
-        widgets = {
-            'food_inherit': MultiSelectWidget
+            'search': SelectWidget, 'unaccent': MultiSelectWidget, 'icontains': MultiSelectWidget, 'istartswith': MultiSelectWidget, 'trigram': MultiSelectWidget, 'fulltext':
+            MultiSelectWidget,
         }

cookbook/helper/AllAuthCustomAdapter.py

@@ -1,11 +1,10 @@
 import datetime
-
-from django.conf import settings
+from gettext import gettext as _
 
 from allauth.account.adapter import DefaultAccountAdapter
+from django.conf import settings
 from django.contrib import messages
 from django.core.cache import caches
-from gettext import gettext as _
 
 from cookbook.models import InviteLink
 
@@ -17,10 +16,13 @@ class AllAuthCustomAdapter(DefaultAccountAdapter):
         Whether to allow sign-ups.
         """
         signup_token = False
-        if 'signup_token' in request.session and InviteLink.objects.filter(valid_until__gte=datetime.datetime.today(), used_by=None, uuid=request.session['signup_token']).exists():
+        if 'signup_token' in request.session and InviteLink.objects.filter(
+                valid_until__gte=datetime.datetime.today(), used_by=None, uuid=request.session['signup_token']).exists():
             signup_token = True
 
-        if (request.resolver_match.view_name == 'account_signup' or request.resolver_match.view_name == 'socialaccount_signup') and not settings.ENABLE_SIGNUP and not signup_token:
+        if request.resolver_match.view_name == 'account_signup' and not settings.ENABLE_SIGNUP and not signup_token:
+            return False
+        elif request.resolver_match.view_name == 'socialaccount_signup' and len(settings.SOCIAL_PROVIDERS) < 1:
             return False
         else:
             return super(AllAuthCustomAdapter, self).is_open_for_signup(request)
@@ -33,7 +35,7 @@ class AllAuthCustomAdapter(DefaultAccountAdapter):
             if c == default:
                 try:
                     super(AllAuthCustomAdapter, self).send_mail(template_prefix, email, context)
-                except Exception: # dont fail signup just because confirmation mail could not be send
+                except Exception:  # dont fail signup just because confirmation mail could not be send
                     pass
             else:
                 messages.add_message(self.request, messages.ERROR, _('In order to prevent spam, the requested email was not send. Please wait a few minutes and try again.'))

cookbook/helper/HelperFunctions.py

@@ -7,7 +7,7 @@ class Round(Func):
 
 
 def str2bool(v):
-    if type(v) == bool or v is None:
+    if isinstance(v, bool) or v is None:
         return v
     else:
         return v.lower() in ("yes", "true", "1")

cookbook/helper/__init__.py

@@ -1,6 +1,4 @@
-import cookbook.helper.dal
 from cookbook.helper.AllAuthCustomAdapter import AllAuthCustomAdapter
 
 __all__ = [
-    'dal',
 ]

cookbook/helper/automation_helper.py

@@ -0,0 +1,227 @@
+import re
+
+from django.core.cache import caches
+from django.db.models.functions import Lower
+
+from cookbook.models import Automation
+
+
+class AutomationEngine:
+    request = None
+    source = None
+    use_cache = None
+    food_aliases = None
+    keyword_aliases = None
+    unit_aliases = None
+    never_unit = None
+    transpose_words = None
+    regex_replace = {
+        Automation.DESCRIPTION_REPLACE: None,
+        Automation.INSTRUCTION_REPLACE: None,
+        Automation.FOOD_REPLACE: None,
+        Automation.UNIT_REPLACE: None,
+        Automation.NAME_REPLACE: None,
+    }
+
+    def __init__(self, request, use_cache=True, source=None):
+        self.request = request
+        self.use_cache = use_cache
+        if not source:
+            self.source = "default_string_to_avoid_false_regex_match"
+        else:
+            self.source = source
+
+    def apply_keyword_automation(self, keyword):
+        keyword = keyword.strip()
+        if self.use_cache and self.keyword_aliases is None:
+            self.keyword_aliases = {}
+            KEYWORD_CACHE_KEY = f'automation_keyword_alias_{self.request.space.pk}'
+            if c := caches['default'].get(KEYWORD_CACHE_KEY, None):
+                self.keyword_aliases = c
+                caches['default'].touch(KEYWORD_CACHE_KEY, 30)
+            else:
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.KEYWORD_ALIAS).only('param_1', 'param_2').order_by('order').all():
+                    self.keyword_aliases[a.param_1.lower()] = a.param_2
+                caches['default'].set(KEYWORD_CACHE_KEY, self.keyword_aliases, 30)
+        else:
+            self.keyword_aliases = {}
+        if self.keyword_aliases:
+            try:
+                keyword = self.keyword_aliases[keyword.lower()]
+            except KeyError:
+                pass
+        else:
+            if automation := Automation.objects.filter(space=self.request.space, type=Automation.KEYWORD_ALIAS, param_1__iexact=keyword, disabled=False).order_by('order').first():
+                return automation.param_2
+        return keyword
+
+    def apply_unit_automation(self, unit):
+        unit = unit.strip()
+        if self.use_cache and self.unit_aliases is None:
+            self.unit_aliases = {}
+            UNIT_CACHE_KEY = f'automation_unit_alias_{self.request.space.pk}'
+            if c := caches['default'].get(UNIT_CACHE_KEY, None):
+                self.unit_aliases = c
+                caches['default'].touch(UNIT_CACHE_KEY, 30)
+            else:
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.UNIT_ALIAS).only('param_1', 'param_2').order_by('order').all():
+                    self.unit_aliases[a.param_1.lower()] = a.param_2
+                caches['default'].set(UNIT_CACHE_KEY, self.unit_aliases, 30)
+        else:
+            self.unit_aliases = {}
+        if self.unit_aliases:
+            try:
+                unit = self.unit_aliases[unit.lower()]
+            except KeyError:
+                pass
+        else:
+            if automation := Automation.objects.filter(space=self.request.space, type=Automation.UNIT_ALIAS, param_1__iexact=unit, disabled=False).order_by('order').first():
+                return automation.param_2
+        return self.apply_regex_replace_automation(unit, Automation.UNIT_REPLACE)
+
+    def apply_food_automation(self, food):
+        food = food.strip()
+        if self.use_cache and self.food_aliases is None:
+            self.food_aliases = {}
+            FOOD_CACHE_KEY = f'automation_food_alias_{self.request.space.pk}'
+            if c := caches['default'].get(FOOD_CACHE_KEY, None):
+                self.food_aliases = c
+                caches['default'].touch(FOOD_CACHE_KEY, 30)
+            else:
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.FOOD_ALIAS).only('param_1', 'param_2').order_by('order').all():
+                    self.food_aliases[a.param_1.lower()] = a.param_2
+                caches['default'].set(FOOD_CACHE_KEY, self.food_aliases, 30)
+        else:
+            self.food_aliases = {}
+
+        if self.food_aliases:
+            try:
+                return self.food_aliases[food.lower()]
+            except KeyError:
+                return self.apply_regex_replace_automation(food, Automation.FOOD_REPLACE)
+        else:
+            if automation := Automation.objects.filter(space=self.request.space, type=Automation.FOOD_ALIAS, param_1__iexact=food, disabled=False).order_by('order').first():
+                return automation.param_2
+        return self.apply_regex_replace_automation(food, Automation.FOOD_REPLACE)
+
+    def apply_never_unit_automation(self, tokens):
+        """
+        Moves a string that should never be treated as a unit to next token and optionally replaced with default unit
+        e.g. NEVER_UNIT: param1: egg, param2: None would modify ['1', 'egg', 'white'] to ['1', '', 'egg', 'white']
+        or NEVER_UNIT: param1: egg, param2: pcs would modify ['1', 'egg', 'yolk'] to ['1', 'pcs', 'egg', 'yolk']
+        :param1 string: string that should never be considered a unit, will be moved to token[2]
+        :param2 (optional) unit as string: will insert unit string into token[1]
+        :return: unit as string (possibly changed by automation)
+        """
+
+        if self.use_cache and self.never_unit is None:
+            self.never_unit = {}
+            NEVER_UNIT_CACHE_KEY = f'automation_never_unit_{self.request.space.pk}'
+            if c := caches['default'].get(NEVER_UNIT_CACHE_KEY, None):
+                self.never_unit = c
+                caches['default'].touch(NEVER_UNIT_CACHE_KEY, 30)
+            else:
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.NEVER_UNIT).only('param_1', 'param_2').order_by('order').all():
+                    self.never_unit[a.param_1.lower()] = a.param_2
+                caches['default'].set(NEVER_UNIT_CACHE_KEY, self.never_unit, 30)
+        else:
+            self.never_unit = {}
+
+        new_unit = None
+        alt_unit = self.apply_unit_automation(tokens[1])
+        never_unit = False
+        if self.never_unit:
+            try:
+                new_unit = self.never_unit[tokens[1].lower()]
+                never_unit = True
+            except KeyError:
+                return tokens
+        else:
+            if a := Automation.objects.annotate(param_1_lower=Lower('param_1')).filter(space=self.request.space, type=Automation.NEVER_UNIT, param_1_lower__in=[
+                    tokens[1].lower(), alt_unit.lower()], disabled=False).order_by('order').first():
+                new_unit = a.param_2
+                never_unit = True
+
+        if never_unit:
+            tokens.insert(1, new_unit)
+        return tokens
+
+    def apply_transpose_automation(self, string):
+        """
+        If two words (param_1 & param_2) are detected in sequence, swap their position in the ingredient string
+        :param 1: first word to detect
+        :param 2: second word to detect
+        return: new ingredient string
+        """
+        if self.use_cache and self.transpose_words is None:
+            self.transpose_words = {}
+            TRANSPOSE_WORDS_CACHE_KEY = f'automation_transpose_words_{self.request.space.pk}'
+            if c := caches['default'].get(TRANSPOSE_WORDS_CACHE_KEY, None):
+                self.transpose_words = c
+                caches['default'].touch(TRANSPOSE_WORDS_CACHE_KEY, 30)
+            else:
+                i = 0
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.TRANSPOSE_WORDS).only(
+                        'param_1', 'param_2').order_by('order').all()[:512]:
+                    self.transpose_words[i] = [a.param_1.lower(), a.param_2.lower()]
+                    i += 1
+                caches['default'].set(TRANSPOSE_WORDS_CACHE_KEY, self.transpose_words, 30)
+        else:
+            self.transpose_words = {}
+
+        tokens = [x.lower() for x in string.replace(',', ' ').split()]
+        if self.transpose_words:
+            for key, value in self.transpose_words.items():
+                if value[0] in tokens and value[1] in tokens:
+                    string = re.sub(rf"\b({value[0]})\W*({value[1]})\b", r"\2 \1", string, flags=re.IGNORECASE)
+        else:
+            for rule in Automation.objects.filter(space=self.request.space, type=Automation.TRANSPOSE_WORDS, disabled=False) \
+                    .annotate(param_1_lower=Lower('param_1'), param_2_lower=Lower('param_2')) \
+                    .filter(param_1_lower__in=tokens, param_2_lower__in=tokens).order_by('order')[:512]:
+                if rule.param_1 in tokens and rule.param_2 in tokens:
+                    string = re.sub(rf"\b({rule.param_1})\W*({rule.param_2})\b", r"\2 \1", string, flags=re.IGNORECASE)
+        return string
+
+    def apply_regex_replace_automation(self, string, automation_type):
+        # TODO add warning - maybe on SPACE page? when a max of 512 automations of a specific type is exceeded (ALIAS types excluded?)
+        """
+        Replaces strings in a recipe field that are from a matched source
+        field_type are Automation.type that apply regex replacements
+        Automation.DESCRIPTION_REPLACE
+        Automation.INSTRUCTION_REPLACE
+        Automation.FOOD_REPLACE
+        Automation.UNIT_REPLACE
+        Automation.NAME_REPLACE
+
+        regex replacment utilized the following fields from the Automation model
+        :param 1: source that should apply the automation in regex format ('.*' for all)
+        :param 2: regex pattern to match ()
+        :param 3: replacement string (leave blank to delete)
+        return: new string
+        """
+        if self.use_cache and self.regex_replace[automation_type] is None:
+            self.regex_replace[automation_type] = {}
+            REGEX_REPLACE_CACHE_KEY = f'automation_regex_replace_{self.request.space.pk}'
+            if c := caches['default'].get(REGEX_REPLACE_CACHE_KEY, None):
+                self.regex_replace[automation_type] = c[automation_type]
+                caches['default'].touch(REGEX_REPLACE_CACHE_KEY, 30)
+            else:
+                i = 0
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=automation_type).only(
+                        'param_1', 'param_2', 'param_3').order_by('order').all()[:512]:
+                    self.regex_replace[automation_type][i] = [a.param_1, a.param_2, a.param_3]
+                    i += 1
+                caches['default'].set(REGEX_REPLACE_CACHE_KEY, self.regex_replace, 30)
+        else:
+            self.regex_replace[automation_type] = {}
+
+        if self.regex_replace[automation_type]:
+            for rule in self.regex_replace[automation_type].values():
+                if re.match(rule[0], (self.source)[:512]):
+                    string = re.sub(rule[1], rule[2], string, flags=re.IGNORECASE)
+        else:
+            for rule in Automation.objects.filter(space=self.request.space, disabled=False, type=automation_type).only(
+                    'param_1', 'param_2', 'param_3').order_by('order').all()[:512]:
+                if re.match(rule.param_1, (self.source)[:512]):
+                    string = re.sub(rule.param_2, rule.param_3, string, flags=re.IGNORECASE)
+        return string

cookbook/helper/context_processors.py

@@ -11,4 +11,5 @@ def context_settings(request):
         'PRIVACY_URL': settings.PRIVACY_URL,
         'IMPRINT_URL': settings.IMPRINT_URL,
         'SHOPPING_MIN_AUTOSYNC_INTERVAL': settings.SHOPPING_MIN_AUTOSYNC_INTERVAL,
+        'DISABLE_EXTERNAL_CONNECTORS': settings.DISABLE_EXTERNAL_CONNECTORS,
     }

cookbook/helper/dal.py

@@ -1,34 +0,0 @@
-from cookbook.models import Food, Keyword, Recipe, Unit
-
-from dal import autocomplete
-
-
-class BaseAutocomplete(autocomplete.Select2QuerySetView):
-    model = None
-
-    def get_queryset(self):
-        if not self.request.user.is_authenticated:
-            return self.model.objects.none()
-
-        qs = self.model.objects.filter(space=self.request.space).all()
-
-        if self.q:
-            qs = qs.filter(name__icontains=self.q)
-
-        return qs
-
-
-class KeywordAutocomplete(BaseAutocomplete):
-    model = Keyword
-
-
-class IngredientsAutocomplete(BaseAutocomplete):
-    model = Food
-
-
-class RecipeAutocomplete(BaseAutocomplete):
-    model = Recipe
-
-
-class UnitAutocomplete(BaseAutocomplete):
-    model = Unit

cookbook/helper/fdc_helper.py

@@ -0,0 +1,19 @@
+import json
+
+
+def get_all_nutrient_types():
+    f = open('')  # <--- download the foundation food or any other dataset and retrieve all nutrition ID's from it https://fdc.nal.usda.gov/download-datasets.html
+    json_data = json.loads(f.read())
+
+    nutrients = {}
+    for food in json_data['FoundationFoods']:
+        for entry in food['foodNutrients']:
+            nutrients[entry['nutrient']['id']] = {'name': entry['nutrient']['name'], 'unit': entry['nutrient']['unitName']}
+
+    nutrient_ids = list(nutrients.keys())
+    nutrient_ids.sort()
+    for nid in nutrient_ids:
+        print('{', f'value: {nid}, text: "{nutrients[nid]["name"]} [{nutrients[nid]["unit"]}] ({nid})"', '},')
+
+
+get_all_nutrient_types()

cookbook/helper/image_processing.py

@@ -1,8 +1,7 @@
 import os
-import sys
+from io import BytesIO
 
 from PIL import Image
-from io import BytesIO
 
 
 def rescale_image_jpeg(image_object, base_width=1020):
@@ -11,7 +10,7 @@ def rescale_image_jpeg(image_object, base_width=1020):
     width_percent = (base_width / float(img.size[0]))
     height = int((float(img.size[1]) * float(width_percent)))
 
-    img = img.resize((base_width, height), Image.ANTIALIAS)
+    img = img.resize((base_width, height), Image.LANCZOS)
     img_bytes = BytesIO()
     img.save(img_bytes, 'JPEG', quality=90, optimize=True, icc_profile=icc_profile)
 
@@ -22,7 +21,7 @@ def rescale_image_png(image_object, base_width=1020):
     image_object = Image.open(image_object)
     wpercent = (base_width / float(image_object.size[0]))
     hsize = int((float(image_object.size[1]) * float(wpercent)))
-    img = image_object.resize((base_width, hsize), Image.ANTIALIAS)
+    img = image_object.resize((base_width, hsize), Image.LANCZOS)
 
     im_io = BytesIO()
     img.save(im_io, 'PNG', quality=90)

cookbook/helper/ingredient_parser.py

@@ -2,18 +2,16 @@ import re
 import string
 import unicodedata
 
-from django.core.cache import caches
-
-from cookbook.models import Unit, Food, Automation, Ingredient
+from cookbook.helper.automation_helper import AutomationEngine
+from cookbook.models import Food, Ingredient, Unit
 
 
 class IngredientParser:
     request = None
     ignore_rules = False
-    food_aliases = {}
-    unit_aliases = {}
+    automation = None
 
-    def __init__(self, request, cache_mode, ignore_automations=False):
+    def __init__(self, request, cache_mode=True, ignore_automations=False):
         """
         Initialize ingredient parser
         :param request: request context (to control caching, rule ownership, etc.)
@@ -22,65 +20,8 @@ class IngredientParser:
         """
         self.request = request
         self.ignore_rules = ignore_automations
-        if cache_mode:
-            FOOD_CACHE_KEY = f'automation_food_alias_{self.request.space.pk}'
-            if c := caches['default'].get(FOOD_CACHE_KEY, None):
-                self.food_aliases = c
-                caches['default'].touch(FOOD_CACHE_KEY, 30)
-            else:
-                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.FOOD_ALIAS).only('param_1', 'param_2').order_by('order').all():
-                    self.food_aliases[a.param_1] = a.param_2
-                caches['default'].set(FOOD_CACHE_KEY, self.food_aliases, 30)
-
-            UNIT_CACHE_KEY = f'automation_unit_alias_{self.request.space.pk}'
-            if c := caches['default'].get(UNIT_CACHE_KEY, None):
-                self.unit_aliases = c
-                caches['default'].touch(UNIT_CACHE_KEY, 30)
-            else:
-                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.UNIT_ALIAS).only('param_1', 'param_2').order_by('order').all():
-                    self.unit_aliases[a.param_1] = a.param_2
-                caches['default'].set(UNIT_CACHE_KEY, self.unit_aliases, 30)
-        else:
-            self.food_aliases = {}
-            self.unit_aliases = {}
-
-    def apply_food_automation(self, food):
-        """
-        Apply food alias automations to passed food
-        :param food: unit as string
-        :return: food as string (possibly changed by automation)
-        """
-        if self.ignore_rules:
-            return food
-        else:
-            if self.food_aliases:
-                try:
-                    return self.food_aliases[food]
-                except KeyError:
-                    return food
-            else:
-                if automation := Automation.objects.filter(space=self.request.space, type=Automation.FOOD_ALIAS, param_1=food, disabled=False).order_by('order').first():
-                    return automation.param_2
-        return food
-
-    def apply_unit_automation(self, unit):
-        """
-        Apply unit alias automations to passed unit
-        :param unit: unit as string
-        :return: unit as string (possibly changed by automation)
-        """
-        if self.ignore_rules:
-            return unit
-        else:
-            if self.unit_aliases:
-                try:
-                    return self.unit_aliases[unit]
-                except KeyError:
-                    return unit
-            else:
-                if automation := Automation.objects.filter(space=self.request.space, type=Automation.UNIT_ALIAS, param_1=unit, disabled=False).order_by('order').first():
-                    return automation.param_2
-        return unit
+        if not self.ignore_rules:
+            self.automation = AutomationEngine(self.request, use_cache=cache_mode)
 
     def get_unit(self, unit):
         """
@@ -91,7 +32,10 @@ class IngredientParser:
         if not unit:
             return None
         if len(unit) > 0:
-            u, created = Unit.objects.get_or_create(name=self.apply_unit_automation(unit), space=self.request.space)
+            if self.ignore_rules:
+                u, created = Unit.objects.get_or_create(name=unit.strip(), space=self.request.space)
+            else:
+                u, created = Unit.objects.get_or_create(name=self.automation.apply_unit_automation(unit), space=self.request.space)
             return u
         return None
 
@@ -104,7 +48,10 @@ class IngredientParser:
         if not food:
             return None
         if len(food) > 0:
-            f, created = Food.objects.get_or_create(name=self.apply_food_automation(food), space=self.request.space)
+            if self.ignore_rules:
+                f, created = Food.objects.get_or_create(name=food.strip(), space=self.request.space)
+            else:
+                f, created = Food.objects.get_or_create(name=self.automation.apply_food_automation(food), space=self.request.space)
             return f
         return None
 
@@ -133,10 +80,10 @@ class IngredientParser:
         end = 0
         while (end < len(x) and (x[end] in string.digits
                                  or (
-                                         (x[end] == '.' or x[end] == ',' or x[end] == '/')
-                                         and end + 1 < len(x)
-                                         and x[end + 1] in string.digits
-                                 ))):
+            (x[end] == '.' or x[end] == ',' or x[end] == '/')
+            and end + 1 < len(x)
+            and x[end + 1] in string.digits
+        ))):
             end += 1
         if end > 0:
             if "/" in x[:end]:
@@ -160,7 +107,8 @@ class IngredientParser:
         if unit is not None and unit.strip() == '':
             unit = None
 
-        if unit is not None and (unit.startswith('(') or unit.startswith('-')):  # i dont know any unit that starts with ( or - so its likely an alternative like 1L (500ml) Water or 2-3
+        if unit is not None and (unit.startswith('(') or unit.startswith(
+                '-')):  # i dont know any unit that starts with ( or - so its likely an alternative like 1L (500ml) Water or 2-3
             unit = None
             note = x
         return amount, unit, note
@@ -221,6 +169,9 @@ class IngredientParser:
         if len(ingredient) == 0:
             raise ValueError('string to parse cannot be empty')
 
+        if len(ingredient) > 512:
+            raise ValueError('cannot parse ingredients with more than 512 characters')
+
         # some people/languages put amount and unit at the end of the ingredient string
         # if something like this is detected move it to the beginning so the parser can handle it
         if len(ingredient) < 1000 and re.search(r'^([^\W\d_])+(.)*[1-9](\d)*\s*([^\W\d_])+', ingredient):
@@ -230,8 +181,8 @@ class IngredientParser:
 
         # if the string contains parenthesis early on remove it and place it at the end
         # because its likely some kind of note
-        if re.match('(.){1,6}\s\((.[^\(\)])+\)\s', ingredient):
-            match = re.search('\((.[^\(])+\)', ingredient)
+        if re.match('(.){1,6}\\s\\((.[^\\(\\)])+\\)\\s', ingredient):
+            match = re.search('\\((.[^\\(])+\\)', ingredient)
             ingredient = ingredient[:match.start()] + ingredient[match.end():] + ' ' + ingredient[match.start():match.end()]
 
         # leading spaces before commas result in extra tokens, clean them out
@@ -239,12 +190,15 @@ class IngredientParser:
 
         # handle "(from) - (to)" amounts by using the minimum amount and adding the range to the description
         # "10.5 - 200 g XYZ" => "100 g XYZ (10.5 - 200)"
-        ingredient = re.sub("^(\d+|\d+[\\.,]\d+) - (\d+|\d+[\\.,]\d+) (.*)", "\\1 \\3 (\\1 - \\2)", ingredient)
+        ingredient = re.sub("^(\\d+|\\d+[\\.,]\\d+) - (\\d+|\\d+[\\.,]\\d+) (.*)", "\\1 \\3 (\\1 - \\2)", ingredient)
 
         # if amount and unit are connected add space in between
-        if re.match('([0-9])+([A-z])+\s', ingredient):
+        if re.match('([0-9])+([A-z])+\\s', ingredient):
             ingredient = re.sub(r'(?<=([a-z])|\d)(?=(?(1)\d|[a-z]))', ' ', ingredient)
 
+        if not self.ignore_rules:
+            ingredient = self.automation.apply_transpose_automation(ingredient)
+
         tokens = ingredient.split()  # split at each space into tokens
         if len(tokens) == 1:
             # there only is one argument, that must be the food
@@ -257,6 +211,8 @@ class IngredientParser:
                 # three arguments if it already has a unit there can't be
                 # a fraction for the amount
                 if len(tokens) > 2:
+                    if not self.ignore_rules:
+                        tokens = self.automation.apply_never_unit_automation(tokens)
                     try:
                         if unit is not None:
                             # a unit is already found, no need to try the second argument for a fraction
@@ -303,10 +259,11 @@ class IngredientParser:
         if unit_note not in note:
             note += ' ' + unit_note
 
-        if unit:
-            unit = self.apply_unit_automation(unit.strip())
+        if unit and not self.ignore_rules:
+            unit = self.automation.apply_unit_automation(unit)
 
-        food = self.apply_food_automation(food.strip())
+        if food and not self.ignore_rules:
+            food = self.automation.apply_food_automation(food)
         if len(food) > Food._meta.get_field('name').max_length:  # test if food name is to long
             # try splitting it at a space and taking only the first arg
             if len(food.split()) > 1 and len(food.split()[0]) < Food._meta.get_field('name').max_length:

cookbook/helper/open_data_importer.py

@@ -1,6 +1,20 @@
-from django.db.models import Q
+import traceback
+from collections import defaultdict
+from decimal import Decimal
 
-from cookbook.models import Unit, SupermarketCategory, Property, PropertyType, Supermarket, SupermarketCategoryRelation, Food, Automation, UnitConversion, FoodProperty
+from cookbook.models import (Food, FoodProperty, Property, PropertyType, Supermarket,
+                             SupermarketCategory, SupermarketCategoryRelation, Unit, UnitConversion)
+import re
+
+
+class OpenDataImportResponse:
+    total_created = 0
+    total_updated = 0
+    total_untouched = 0
+    total_errored = 0
+
+    def to_dict(self):
+        return {'total_created': self.total_created, 'total_updated': self.total_updated, 'total_untouched': self.total_untouched, 'total_errored': self.total_errored}
 
 
 class OpenDataImporter:
@@ -19,68 +33,269 @@ class OpenDataImporter:
     def _update_slug_cache(self, object_class, datatype):
         self.slug_id_cache[datatype] = dict(object_class.objects.filter(space=self.request.space, open_data_slug__isnull=False).values_list('open_data_slug', 'id', ))
 
-    def import_units(self):
-        datatype = 'unit'
+    @staticmethod
+    def _is_obj_identical(field_list, obj, existing_obj):
+        """
+        checks if the obj meant for import is identical to an already existing one
+        :param field_list: list of field names to check
+        :type field_list: list[str]
+        :param obj: object meant for import
+        :type obj: Object
+        :param existing_obj: object already in DB
+        :type existing_obj: Object
+        :return: if objects are identical
+        :rtype: bool
+        """
+        for field in field_list:
+            if isinstance(getattr(obj, field), float) or isinstance(getattr(obj, field), Decimal):
+                if abs(float(getattr(obj, field)) - float(existing_obj[field])) > 0.001:  # convert both to float and check if basically equal
+                    print(f'comparing FLOAT {obj} failed because field {field} is not equal ({getattr(obj, field)} != {existing_obj[field]})')
+                    return False
+            elif getattr(obj, field) != existing_obj[field]:
+                print(f'comparing {obj} failed because field {field} is not equal ({getattr(obj, field)} != {existing_obj[field]})')
+                return False
+        return True
+
+    @staticmethod
+    def _merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+        """
+        sometimes there might be two objects conflicting for open data import (one has the slug, the other the name)
+        this function checks if that is the case and merges the two objects if possible
+        :param model_type: type of model to check/merge
+        :type model_type: Model
+        :param obj: object that should be created/updated
+        :type obj: Model
+        :param existing_data_slugs: dict of open data slugs mapped to objects
+        :type existing_data_slugs: dict
+        :param existing_data_names: dict of names mapped to objects
+        :type existing_data_names: dict
+        :return: true if merge was successful or not necessary else false
+        :rtype: bool
+        """
+        if obj.open_data_slug in existing_data_slugs and obj.name in existing_data_names and existing_data_slugs[obj.open_data_slug]['pk'] != existing_data_names[obj.name]['pk']:
+            try:
+                source_obj = model_type.objects.get(pk=existing_data_slugs[obj.open_data_slug]['pk'])
+                del existing_data_slugs[obj.open_data_slug]
+                source_obj.merge_into(model_type.objects.get(pk=existing_data_names[obj.name]['pk']))
+                return True
+            except RuntimeError:
+                return False  # in the edge case (e.g. parent/child) that an object cannot be merged don't update it for now
+        else:
+            return True
+
+    @staticmethod
+    def _get_existing_obj(obj, existing_data_slugs, existing_data_names):
+        """
+        gets the existing object from slug or name cache
+        :param obj: object that should be found
+        :type obj: Model
+        :param existing_data_slugs: dict of open data slugs mapped to objects
+        :type existing_data_slugs: dict
+        :param existing_data_names: dict of names mapped to objects
+        :type existing_data_names: dict
+        :return: existing object
+        :rtype: dict
+        """
+        existing_obj = None
+        if obj.open_data_slug in existing_data_slugs:
+            existing_obj = existing_data_slugs[obj.open_data_slug]
+        elif obj.name in existing_data_names:
+            existing_obj = existing_data_names[obj.name]
+
+        return existing_obj
+
+    def import_units(self):
+        od_response = OpenDataImportResponse()
+        datatype = 'unit'
+        model_type = Unit
+        field_list = ['name', 'plural_name', 'base_unit', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
 
-        insert_list = []
         for u in list(self.data[datatype].keys()):
-            insert_list.append(Unit(
+            obj = model_type(
                 name=self.data[datatype][u]['name'],
                 plural_name=self.data[datatype][u]['plural_name'],
-                base_unit=self.data[datatype][u]['base_unit'] if self.data[datatype][u]['base_unit'] != '' else None,
+                base_unit=self.data[datatype][u]['base_unit'].lower() if self.data[datatype][u]['base_unit'] != '' else None,
                 open_data_slug=u,
                 space=self.request.space
-            ))
+            )
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
 
-        if self.update_existing:
-            return Unit.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('name', 'plural_name', 'base_unit', 'open_data_slug'), unique_fields=('space', 'name',))
-        else:
-            return Unit.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        return od_response
 
     def import_category(self):
+        od_response = OpenDataImportResponse()
         datatype = 'category'
+        model_type = SupermarketCategory
+        field_list = ['name', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
 
-        insert_list = []
         for k in list(self.data[datatype].keys()):
-            insert_list.append(SupermarketCategory(
+            obj = model_type(
                 name=self.data[datatype][k]['name'],
                 open_data_slug=k,
                 space=self.request.space
-            ))
+            )
 
-        return SupermarketCategory.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        return od_response
 
     def import_property(self):
+        od_response = OpenDataImportResponse()
         datatype = 'property'
+        model_type = PropertyType
+        field_list = ['name', 'unit', 'fdc_id', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
 
-        insert_list = []
         for k in list(self.data[datatype].keys()):
-            insert_list.append(PropertyType(
+            obj = model_type(
                 name=self.data[datatype][k]['name'],
                 unit=self.data[datatype][k]['unit'],
+                fdc_id=self.data[datatype][k]['fdc_id'],
                 open_data_slug=k,
                 space=self.request.space
-            ))
+            )
 
-        return PropertyType.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        return od_response
 
     def import_supermarket(self):
+        od_response = OpenDataImportResponse()
         datatype = 'store'
+        model_type = Supermarket
+        field_list = ['name', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
 
         self._update_slug_cache(SupermarketCategory, 'category')
-        insert_list = []
         for k in list(self.data[datatype].keys()):
-            insert_list.append(Supermarket(
+            obj = model_type(
                 name=self.data[datatype][k]['name'],
                 open_data_slug=k,
                 space=self.request.space
-            ))
+            )
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
 
         # always add open data slug if matching supermarket is found, otherwise relation might fail
-        supermarkets = Supermarket.objects.bulk_create(insert_list, unique_fields=('space', 'name',), update_conflicts=True, update_fields=('open_data_slug',))
         self._update_slug_cache(Supermarket, 'store')
 
-        insert_list = []
         for k in list(self.data[datatype].keys()):
             relations = []
             order = 0
@@ -96,119 +311,186 @@ class OpenDataImporter:
 
             SupermarketCategoryRelation.objects.bulk_create(relations, ignore_conflicts=True, unique_fields=('supermarket', 'category',))
 
-        return supermarkets
+        return od_response
 
     def import_food(self):
-        identifier_list = []
+        od_response = OpenDataImportResponse()
         datatype = 'food'
-        for k in list(self.data[datatype].keys()):
-            identifier_list.append(self.data[datatype][k]['name'])
-            identifier_list.append(self.data[datatype][k]['plural_name'])
+        model_type = Food
+        field_list = ['name', 'open_data_slug']
 
-        existing_objects_flat = []
-        existing_objects = {}
-        for f in Food.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('id', 'name', 'plural_name'):
-            existing_objects_flat.append(f[1])
-            existing_objects_flat.append(f[2])
-            existing_objects[f[1]] = f
-            existing_objects[f[2]] = f
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
 
         self._update_slug_cache(Unit, 'unit')
         self._update_slug_cache(PropertyType, 'property')
+        self._update_slug_cache(SupermarketCategory, 'category')
 
-        # pref_unit_key = 'preferred_unit_metric'
-        # pref_shopping_unit_key = 'preferred_packaging_unit_metric'
-        # if not self.use_metric:
-        #     pref_unit_key = 'preferred_unit_imperial'
-        #     pref_shopping_unit_key = 'preferred_packaging_unit_imperial'
+        unit_g = Unit.objects.filter(space=self.request.space, base_unit__iexact='g').first()
 
-        insert_list = []
-        update_list = []
-        update_field_list = []
         for k in list(self.data[datatype].keys()):
-            if not (self.data[datatype][k]['name'] in existing_objects_flat or self.data[datatype][k]['plural_name'] in existing_objects_flat):
-                insert_list.append({'data': {
-                    'name': self.data[datatype][k]['name'],
-                    'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
-                    # 'preferred_unit_id': self.slug_id_cache['unit'][self.data[datatype][k][pref_unit_key]],
-                    # 'preferred_shopping_unit_id': self.slug_id_cache['unit'][self.data[datatype][k][pref_shopping_unit_key]],
-                    'supermarket_category_id': self.slug_id_cache['category'][self.data[datatype][k]['store_category']],
-                    'fdc_id': self.data[datatype][k]['fdc_id'] if self.data[datatype][k]['fdc_id'] != '' else None,
-                    'open_data_slug': k,
-                    'space': self.request.space.id,
-                }})
+
+            obj_dict = {
+                'name': self.data[datatype][k]['name'],
+                'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
+                'supermarket_category_id': self.slug_id_cache['category'][self.data[datatype][k]['store_category']],
+                'fdc_id': re.sub(r'\D', '', self.data[datatype][k]['fdc_id']) if self.data[datatype][k]['fdc_id'] != '' else None,
+                'open_data_slug': k,
+                'properties_food_unit_id': None,
+                'space_id': self.request.space.id,
+            }
+
+            if unit_g:
+                obj_dict['properties_food_unit_id'] = unit_g.id
+
+            obj = model_type(**obj_dict)
+
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
             else:
-                if self.data[datatype][k]['name'] in existing_objects:
-                    existing_food_id = existing_objects[self.data[datatype][k]['name']][0]
-                else:
-                    existing_food_id = existing_objects[self.data[datatype][k]['plural_name']][0]
+                create_list.append({'data': obj_dict})
 
-                if self.update_existing:
-                    update_field_list = ['name', 'plural_name', 'preferred_unit_id', 'preferred_shopping_unit_id', 'supermarket_category_id', 'fdc_id', 'open_data_slug', ]
-                    update_list.append(Food(
-                        id=existing_food_id,
-                        name=self.data[datatype][k]['name'],
-                        plural_name=self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
-                        # preferred_unit_id=self.slug_id_cache['unit'][self.data[datatype][k][pref_unit_key]],
-                        # preferred_shopping_unit_id=self.slug_id_cache['unit'][self.data[datatype][k][pref_shopping_unit_key]],
-                        supermarket_category_id=self.slug_id_cache['category'][self.data[datatype][k]['store_category']],
-                        fdc_id=self.data[datatype][k]['fdc_id'] if self.data[datatype][k]['fdc_id'] != '' else None,
-                        open_data_slug=k,
-                    ))
-                else:
-                    update_field_list = ['open_data_slug', ]
-                    update_list.append(Food(id=existing_food_id, open_data_slug=k, ))
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
 
-        Food.load_bulk(insert_list, None)
-        if len(update_list) > 0:
-            Food.objects.bulk_update(update_list, update_field_list)
+        if len(create_list) > 0:
+            Food.load_bulk(create_list, None)
+            od_response.total_created += len(create_list)
+
+        # --------------- PROPERTY STUFF -----------------------
+        model_type = Property
+        field_list = ['property_type_id', 'property_amount', 'open_data_food_slug']
+
+        existing_data_slugs = {}
+        existing_data_property_types = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_food_slug']] = obj
+            existing_data_property_types[obj['property_type_id']] = obj
+
+        update_list = []
+        create_list = []
 
         self._update_slug_cache(Food, 'food')
 
-        food_property_list = []
-        alias_list = []
-        for k in list(self.data[datatype].keys()):
-            for fp in self.data[datatype][k]['properties']['type_values']:
-                food_property_list.append(Property(
+        for k in list(self.data['food'].keys()):
+            for fp in self.data['food'][k]['properties']['type_values']:
+                obj = model_type(
                     property_type_id=self.slug_id_cache['property'][fp['property_type']],
                     property_amount=fp['property_value'],
-                    import_food_id=self.slug_id_cache['food'][k],
+                    open_data_food_slug=k,
                     space=self.request.space,
-                ))
+                )
 
-            # for a in self.data[datatype][k]['alias']:
-            #     alias_list.append(Automation(
-            #         param_1=a,
-            #         param_2=self.data[datatype][k]['name'],
-            #         space=self.request.space,
-            #         created_by=self.request.user,
-            #     ))
+                if obj.open_data_food_slug in existing_data_slugs and obj.property_type_id in existing_data_property_types and existing_data_slugs[obj.open_data_food_slug] == existing_data_property_types[obj.property_type_id]:
+                    existing_obj = existing_data_slugs[obj.open_data_food_slug]
 
-        Property.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'import_food_id', 'property_type',))
+                    if not self._is_obj_identical(field_list, obj, existing_obj):
+                        obj.pk = existing_obj['pk']
+                        update_list.append(obj)
+                else:
+                    create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, ignore_conflicts=True, unique_fields=('space', 'open_data_food_slug', 'property_type',))
+
+        linked_properties = list(FoodProperty.objects.filter(food__space=self.request.space).values_list('property_id', flat=True).all())
 
         property_food_relation_list = []
-        for p in Property.objects.filter(space=self.request.space, import_food_id__isnull=False).values_list('import_food_id', 'id', ):
-            property_food_relation_list.append(Food.properties.through(food_id=p[0], property_id=p[1]))
+        for p in model_type.objects.filter(space=self.request.space, open_data_food_slug__isnull=False).values_list('open_data_food_slug', 'id', ):
+            if p[1] == 147:
+                pass
+            # slug_id_cache should always exist, don't create relations for already linked properties (ignore_conflicts would do that as well but this is more performant)
+            if p[0] in self.slug_id_cache['food'] and p[1] not in linked_properties:
+                property_food_relation_list.append(Food.properties.through(food_id=self.slug_id_cache['food'][p[0]], property_id=p[1]))
 
-        FoodProperty.objects.bulk_create(property_food_relation_list, ignore_conflicts=True, unique_fields=('food_id', 'property_id',))
+        FoodProperty.objects.bulk_create(property_food_relation_list, unique_fields=('food_id', 'property_id',))
 
-        # Automation.objects.bulk_create(alias_list, ignore_conflicts=True, unique_fields=('space', 'param_1', 'param_2',))
-        return insert_list + update_list
+        return od_response
 
     def import_conversion(self):
+        od_response = OpenDataImportResponse()
         datatype = 'conversion'
+        model_type = UnitConversion
+        field_list = ['base_amount', 'base_unit_id', 'converted_amount', 'converted_unit_id', 'food_id', 'open_data_slug']
+
+        self._update_slug_cache(Food, 'food')
+        self._update_slug_cache(Unit, 'unit')
+
+        existing_data_slugs = {}
+        existing_data_foods = defaultdict(list)
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_foods[obj['food_id']].append(obj)
+
+        update_list = []
+        create_list = []
 
-        insert_list = []
         for k in list(self.data[datatype].keys()):
-            insert_list.append(UnitConversion(
-                base_amount=self.data[datatype][k]['base_amount'],
-                base_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['base_unit']],
-                converted_amount=self.data[datatype][k]['converted_amount'],
-                converted_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['converted_unit']],
-                food_id=self.slug_id_cache['food'][self.data[datatype][k]['food']],
-                open_data_slug=k,
-                space=self.request.space,
-                created_by=self.request.user,
-            ))
+            # try catch here because sometimes key "k" is not set for the food cache
+            try:
+                obj = model_type(
+                    base_amount=Decimal(self.data[datatype][k]['base_amount']),
+                    base_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['base_unit']],
+                    converted_amount=Decimal(self.data[datatype][k]['converted_amount']),
+                    converted_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['converted_unit']],
+                    food_id=self.slug_id_cache['food'][self.data[datatype][k]['food']],
+                    open_data_slug=k,
+                    space=self.request.space,
+                    created_by_id=self.request.user.id,
+                )
 
-        return UnitConversion.objects.bulk_create(insert_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug'))
+                if obj.open_data_slug in existing_data_slugs:
+                    existing_obj = existing_data_slugs[obj.open_data_slug]
+                    if not self._is_obj_identical(field_list, obj, existing_obj):
+                        obj.pk = existing_obj['pk']
+                        update_list.append(obj)
+                    else:
+                        od_response.total_untouched += 1
+                else:
+                    matching_existing_found = False
+                    if obj.food_id in existing_data_foods:
+                        for edf in existing_data_foods[obj.food_id]:
+                            if obj.base_unit_id == edf['base_unit_id'] and obj.converted_unit_id == edf['converted_unit_id']:
+                                matching_existing_found = True
+                                if not self._is_obj_identical(field_list, obj, edf):
+                                    obj.pk = edf['pk']
+                                    update_list.append(obj)
+                                else:
+                                    od_response.total_untouched += 1
+                    if not matching_existing_found:
+                        create_list.append(obj)
+            except KeyError as e:
+                traceback.print_exc()
+                od_response.total_errored += 1
+                print(self.data[datatype][k]['food'] + ' is not in self.slug_id_cache["food"]')
+
+        if self.update_existing and len(update_list) > 0:
+            od_response.total_updated = model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_errored += len(update_list) - od_response.total_updated
+
+        if len(create_list) > 0:
+            objs_created = model_type.objects.bulk_create(create_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug'))
+            od_response.total_created = len(objs_created)
+            od_response.total_errored += len(create_list) - od_response.total_created
+
+        return od_response

cookbook/helper/permission_helper.py

@@ -4,16 +4,16 @@ from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.decorators import user_passes_test
 from django.core.cache import cache
-from django.core.exceptions import ValidationError, ObjectDoesNotExist
+from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.http import HttpResponseRedirect
 from django.urls import reverse, reverse_lazy
 from django.utils.translation import gettext as _
-from oauth2_provider.contrib.rest_framework import TokenHasScope, TokenHasReadWriteScope
+from oauth2_provider.contrib.rest_framework import TokenHasReadWriteScope, TokenHasScope
 from oauth2_provider.models import AccessToken
 from rest_framework import permissions
 from rest_framework.permissions import SAFE_METHODS
 
-from cookbook.models import ShareLink, Recipe, UserSpace
+from cookbook.models import Recipe, ShareLink, UserSpace
 
 
 def get_allowed_groups(groups_required):
@@ -75,7 +75,7 @@ def is_object_owner(user, obj):
     if not user.is_authenticated:
         return False
     try:
-        return obj.get_owner() == user
+        return obj.get_owner() == 'orphan' or obj.get_owner() == user
     except Exception:
         return False
 
@@ -255,9 +255,6 @@ class CustomIsShared(permissions.BasePermission):
         return request.user.is_authenticated
 
     def has_object_permission(self, request, view, obj):
-        # # temporary hack to make old shopping list work with new shopping list
-        # if obj.__class__.__name__ in ['ShoppingList', 'ShoppingListEntry']:
-        #     return is_object_shared(request.user, obj) or obj.created_by in list(request.user.get_shopping_share())
         return is_object_shared(request.user, obj)
 
 
@@ -322,7 +319,8 @@ class CustomRecipePermission(permissions.BasePermission):
 
     def has_permission(self, request, view):  # user is either at least a guest or a share link is given and the request is safe
         share = request.query_params.get('share', None)
-        return ((has_group_permission(request.user, ['guest']) and request.method in SAFE_METHODS) or has_group_permission(request.user, ['user'])) or (share and request.method in SAFE_METHODS and 'pk' in view.kwargs)
+        return ((has_group_permission(request.user, ['guest']) and request.method in SAFE_METHODS) or has_group_permission(
+            request.user, ['user'])) or (share and request.method in SAFE_METHODS and 'pk' in view.kwargs)
 
     def has_object_permission(self, request, view, obj):
         share = request.query_params.get('share', None)
@@ -332,7 +330,8 @@ class CustomRecipePermission(permissions.BasePermission):
             if obj.private:
                 return ((obj.created_by == request.user) or (request.user in obj.shared.all())) and obj.space == request.space
             else:
-                return ((has_group_permission(request.user, ['guest']) and request.method in SAFE_METHODS) or has_group_permission(request.user, ['user'])) and obj.space == request.space
+                return ((has_group_permission(request.user, ['guest']) and request.method in SAFE_METHODS)
+                        or has_group_permission(request.user, ['user'])) and obj.space == request.space
 
 
 class CustomUserPermission(permissions.BasePermission):
@@ -361,7 +360,7 @@ class CustomTokenHasScope(TokenHasScope):
     """
 
     def has_permission(self, request, view):
-        if type(request.auth) == AccessToken:
+        if isinstance(request.auth, AccessToken):
             return super().has_permission(request, view)
         else:
             return request.user.is_authenticated
@@ -375,7 +374,7 @@ class CustomTokenHasReadWriteScope(TokenHasReadWriteScope):
     """
 
     def has_permission(self, request, view):
-        if type(request.auth) == AccessToken:
+        if isinstance(request.auth, AccessToken):
             return super().has_permission(request, view)
         else:
             return True

cookbook/helper/property_helper.py

@@ -2,7 +2,7 @@ from django.core.cache import caches
 
 from cookbook.helper.cache_helper import CacheHelper
 from cookbook.helper.unit_conversion_helper import UnitConversionHelper
-from cookbook.models import PropertyType, Unit, Food, Property, Recipe, Step
+from cookbook.models import PropertyType
 
 
 class FoodPropertyHelper:
@@ -31,10 +31,12 @@ class FoodPropertyHelper:
 
         if not property_types:
             property_types = PropertyType.objects.filter(space=self.space).all()
-            caches['default'].set(CacheHelper(self.space).PROPERTY_TYPE_CACHE_KEY, property_types, 60 * 60)  # cache is cleared on property type save signal so long duration is fine
+            # cache is cleared on property type save signal so long duration is fine
+            caches['default'].set(CacheHelper(self.space).PROPERTY_TYPE_CACHE_KEY, property_types, 60 * 60)
 
         for fpt in property_types:
-            computed_properties[fpt.id] = {'id': fpt.id, 'name': fpt.name, 'icon': fpt.icon, 'description': fpt.description, 'unit': fpt.unit, 'food_values': {}, 'total_value': 0, 'missing_value': False}
+            computed_properties[fpt.id] = {'id': fpt.id, 'name': fpt.name, 'description': fpt.description,
+                                           'unit': fpt.unit, 'order': fpt.order, 'food_values': {}, 'total_value': 0, 'missing_value': False}
 
         uch = UnitConversionHelper(self.space)
 
@@ -43,25 +45,30 @@ class FoodPropertyHelper:
                 conversions = uch.get_conversions(i)
                 for pt in property_types:
                     found_property = False
-                    if i.food.properties_food_amount == 0 or i.food.properties_food_unit is None:
-                        computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': 0}
-                        computed_properties[pt.id]['missing_value'] = i.food.properties_food_unit is None
+                    if i.food.properties_food_amount == 0 or i.food.properties_food_unit is None:  # if food is configured incorrectly
+                        computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': None}
+                        computed_properties[pt.id]['missing_value'] = True
                     else:
                         for p in i.food.properties.all():
-                            if p.property_type == pt:
+                            if p.property_type == pt and p.property_amount is not None:
                                 for c in conversions:
                                     if c.unit == i.food.properties_food_unit:
                                         found_property = True
                                         computed_properties[pt.id]['total_value'] += (c.amount / i.food.properties_food_amount) * p.property_amount
-                                        computed_properties[pt.id]['food_values'] = self.add_or_create(computed_properties[p.property_type.id]['food_values'], c.food.id, (c.amount / i.food.properties_food_amount) * p.property_amount, c.food)
+                                        computed_properties[pt.id]['food_values'] = self.add_or_create(
+                                            computed_properties[p.property_type.id]['food_values'], c.food.id, (c.amount / i.food.properties_food_amount) * p.property_amount, c.food)
                     if not found_property:
-                        computed_properties[pt.id]['missing_value'] = True
-                        computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': 0}
+                        if i.amount == 0:  # don't count ingredients without an amount as missing
+                            computed_properties[pt.id]['missing_value'] = computed_properties[pt.id]['missing_value'] or False  # don't override if another food was already missing
+                            computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': 0}
+                        else:
+                            computed_properties[pt.id]['missing_value'] = True
+                            computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': None}
 
         return computed_properties
 
     # small dict helper to add to existing key or create new, probably a better way of doing this
-    # TODO move to central helper ?
+    # TODO move to central helper ? --> use defaultdict
     @staticmethod
     def add_or_create(d, key, value, food):
         if key in d:

cookbook/helper/recipe_html_import.py

@@ -1,191 +0,0 @@
-# import json
-# import re
-# from json import JSONDecodeError
-# from urllib.parse import unquote
-
-# from bs4 import BeautifulSoup
-# from bs4.element import Tag
-# from recipe_scrapers import scrape_html, scrape_me
-# from recipe_scrapers._exceptions import NoSchemaFoundInWildMode
-# from recipe_scrapers._utils import get_host_name, normalize_string
-
-# from cookbook.helper import recipe_url_import as helper
-# from cookbook.helper.scrapers.scrapers import text_scraper
-
-
-# def get_recipe_from_source(text, url, request):
-#     def build_node(k, v):
-#         if isinstance(v, dict):
-#             node = {
-#                 'name': k,
-#                 'value': k,
-#                 'children': get_children_dict(v)
-#             }
-#         elif isinstance(v, list):
-#             node = {
-#                 'name': k,
-#                 'value': k,
-#                 'children': get_children_list(v)
-#             }
-#         else:
-#             node = {
-#                 'name': k + ": " + normalize_string(str(v)),
-#                 'value': normalize_string(str(v))
-#             }
-#         return node
-
-#     def get_children_dict(children):
-#         kid_list = []
-#         for k, v in children.items():
-#             kid_list.append(build_node(k, v))
-#         return kid_list
-
-#     def get_children_list(children):
-#         kid_list = []
-#         for kid in children:
-#             if type(kid) == list:
-#                 node = {
-#                     'name': "unknown list",
-#                     'value': "unknown list",
-#                     'children': get_children_list(kid)
-#                 }
-#                 kid_list.append(node)
-#             elif type(kid) == dict:
-#                 for k, v in kid.items():
-#                     kid_list.append(build_node(k, v))
-#             else:
-#                 kid_list.append({
-#                     'name': normalize_string(str(kid)),
-#                     'value': normalize_string(str(kid))
-#                 })
-#         return kid_list
-
-#     recipe_tree = []
-#     parse_list = []
-#     soup = BeautifulSoup(text, "html.parser")
-#     html_data = get_from_html(soup)
-#     images = get_images_from_source(soup, url)
-#     text = unquote(text)
-#     scrape = None
-
-#     if url and not text:
-#         try:
-#             scrape = scrape_me(url_path=url, wild_mode=True)
-#         except(NoSchemaFoundInWildMode):
-#             pass
-
-#     if not scrape:
-#         try:
-#             parse_list.append(remove_graph(json.loads(text)))
-#             if not url and 'url' in parse_list[0]:
-#                 url = parse_list[0]['url']
-#             scrape = text_scraper("<script type='application/ld+json'>" + text + "</script>", url=url)
-
-#         except JSONDecodeError:
-#             for el in soup.find_all('script', type='application/ld+json'):
-#                 el = remove_graph(el)
-#                 if not url and 'url' in el:
-#                     url = el['url']
-#                 if type(el) == list:
-#                     for le in el:
-#                         parse_list.append(le)
-#                 elif type(el) == dict:
-#                     parse_list.append(el)
-#             for el in soup.find_all(type='application/json'):
-#                 el = remove_graph(el)
-#                 if type(el) == list:
-#                     for le in el:
-#                         parse_list.append(le)
-#                 elif type(el) == dict:
-#                     parse_list.append(el)
-#             scrape = text_scraper(text, url=url)
-
-#     recipe_json = helper.get_from_scraper(scrape, request)
-
-#     # TODO: DEPRECATE recipe_tree & html_data.  first validate it isn't used anywhere
-#     for el in parse_list:
-#         temp_tree = []
-#         if isinstance(el, Tag):
-#             try:
-#                 el = json.loads(el.string)
-#             except TypeError:
-#                 continue
-
-#         for k, v in el.items():
-#             if isinstance(v, dict):
-#                 node = {
-#                     'name': k,
-#                     'value': k,
-#                     'children': get_children_dict(v)
-#                 }
-#             elif isinstance(v, list):
-#                 node = {
-#                     'name': k,
-#                     'value': k,
-#                     'children': get_children_list(v)
-#                 }
-#             else:
-#                 node = {
-#                     'name': k + ": " + normalize_string(str(v)),
-#                     'value': normalize_string(str(v))
-#                 }
-#             temp_tree.append(node)
-
-#         if '@type' in el and el['@type'] == 'Recipe':
-#             recipe_tree += [{'name': 'ld+json', 'children': temp_tree}]
-#         else:
-#             recipe_tree += [{'name': 'json', 'children': temp_tree}]
-
-#     return recipe_json, recipe_tree, html_data, images
-
-
-# def get_from_html(soup):
-#     INVISIBLE_ELEMS = ('style', 'script', 'head', 'title')
-#     html = []
-#     for s in soup.strings:
-#         if ((s.parent.name not in INVISIBLE_ELEMS) and (len(s.strip()) > 0)):
-#             html.append(s)
-#     return html
-
-
-# def get_images_from_source(soup, url):
-#     sources = ['src', 'srcset', 'data-src']
-#     images = []
-#     img_tags = soup.find_all('img')
-#     if url:
-#         site = get_host_name(url)
-#         prot = url.split(':')[0]
-
-#     urls = []
-#     for img in img_tags:
-#         for src in sources:
-#             try:
-#                 urls.append(img[src])
-#             except KeyError:
-#                 pass
-
-#     for u in urls:
-#         u = u.split('?')[0]
-#         filename = re.search(r'/([\w_-]+[.](jpg|jpeg|gif|png))$', u)
-#         if filename:
-#             if (('http' not in u) and (url)):
-#                 # sometimes an image source can be relative
-#                 # if it is provide the base url
-#                 u = '{}://{}{}'.format(prot, site, u)
-#             if 'http' in u:
-#                 images.append(u)
-#     return images
-
-
-# def remove_graph(el):
-#     # recipes type might be wrapped in @graph type
-#     if isinstance(el, Tag):
-#         try:
-#             el = json.loads(el.string)
-#             if '@graph' in el:
-#                 for x in el['@graph']:
-#                     if '@type' in x and x['@type'] == 'Recipe':
-#                         el = x
-#         except (TypeError, JSONDecodeError):
-#             pass
-#     return el

cookbook/helper/recipe_search.py

@@ -1,14 +1,11 @@
 import json
-from collections import Counter
 from datetime import date, timedelta
 
 from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector, TrigramSimilarity
-from django.core.cache import cache, caches
-from django.db.models import (Avg, Case, Count, Exists, F, Func, Max, OuterRef, Q, Subquery, Value,
-                              When)
+from django.core.cache import cache
+from django.db.models import Avg, Case, Count, Exists, F, Max, OuterRef, Q, Subquery, Value, When
 from django.db.models.functions import Coalesce, Lower, Substr
 from django.utils import timezone, translation
-from django.utils.translation import gettext as _
 
 from cookbook.helper.HelperFunctions import Round, str2bool
 from cookbook.managers import DICTIONARY
@@ -17,18 +14,19 @@ from cookbook.models import (CookLog, CustomFilter, Food, Keyword, Recipe, Searc
 from recipes import settings
 
 
-# TODO create extensive tests to make sure ORs ANDs and various filters, sorting, etc work as expected
 # TODO consider creating a simpleListRecipe API that only includes minimum of recipe info and minimal filtering
 class RecipeSearch():
-    _postgres = settings.DATABASES['default']['ENGINE'] in [
-        'django.db.backends.postgresql_psycopg2', 'django.db.backends.postgresql']
+    _postgres = settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql'
 
     def __init__(self, request, **params):
         self._request = request
         self._queryset = None
         if f := params.get('filter', None):
-            custom_filter = CustomFilter.objects.filter(id=f, space=self._request.space).filter(Q(created_by=self._request.user) |
-                                                                                                Q(shared=self._request.user) | Q(recipebook__shared=self._request.user)).first()
+            custom_filter = (
+                CustomFilter.objects.filter(id=f, space=self._request.space)
+                .filter(Q(created_by=self._request.user) | Q(shared=self._request.user) | Q(recipebook__shared=self._request.user))
+                .first()
+            )
             if custom_filter:
                 self._params = {**json.loads(custom_filter.search)}
                 self._original_params = {**(params or {})}
@@ -101,24 +99,18 @@ class RecipeSearch():
         self._search_type = self._search_prefs.search or 'plain'
         if self._string:
             if self._postgres:
-                self._unaccent_include = self._search_prefs.unaccent.values_list(
-                    'field', flat=True)
+                self._unaccent_include = self._search_prefs.unaccent.values_list('field', flat=True)
             else:
                 self._unaccent_include = []
-            self._icontains_include = [
-                x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.icontains.values_list('field', flat=True)]
-            self._istartswith_include = [
-                x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.istartswith.values_list('field', flat=True)]
+            self._icontains_include = [x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.icontains.values_list('field', flat=True)]
+            self._istartswith_include = [x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.istartswith.values_list('field', flat=True)]
             self._trigram_include = None
             self._fulltext_include = None
         self._trigram = False
         if self._postgres and self._string:
-            self._language = DICTIONARY.get(
-                translation.get_language(), 'simple')
-            self._trigram_include = [
-                x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.trigram.values_list('field', flat=True)]
-            self._fulltext_include = self._search_prefs.fulltext.values_list(
-                'field', flat=True) or None
+            self._language = DICTIONARY.get(translation.get_language(), 'simple')
+            self._trigram_include = [x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.trigram.values_list('field', flat=True)]
+            self._fulltext_include = self._search_prefs.fulltext.values_list('field', flat=True) or None
 
             if self._search_type not in ['websearch', 'raw'] and self._trigram_include:
                 self._trigram = True
@@ -169,7 +161,7 @@ class RecipeSearch():
         else:
             order = []
             # TODO add userpreference for default sort order and replace '-favorite'
-            default_order = ['-name']
+            default_order = ['name']
             # recent and new_recipe are always first; they float a few recipes to the top
             if self._num_recent:
                 order += ['-recent']
@@ -178,7 +170,6 @@ class RecipeSearch():
 
             # if a sort order is provided by user - use that order
             if self._sort_order:
-
                 if not isinstance(self._sort_order, list):
                     order += [self._sort_order]
                 else:
@@ -218,24 +209,18 @@ class RecipeSearch():
             self._queryset = self._queryset.filter(query_filter).distinct()
             if self._fulltext_include:
                 if self._fuzzy_match is None:
-                    self._queryset = self._queryset.annotate(
-                        score=Coalesce(Max(self.search_rank), 0.0))
+                    self._queryset = self._queryset.annotate(score=Coalesce(Max(self.search_rank), 0.0))
                 else:
-                    self._queryset = self._queryset.annotate(
-                        rank=Coalesce(Max(self.search_rank), 0.0))
+                    self._queryset = self._queryset.annotate(rank=Coalesce(Max(self.search_rank), 0.0))
 
             if self._fuzzy_match is not None:
-                simularity = self._fuzzy_match.filter(
-                    pk=OuterRef('pk')).values('simularity')
+                simularity = self._fuzzy_match.filter(pk=OuterRef('pk')).values('simularity')
                 if not self._fulltext_include:
-                    self._queryset = self._queryset.annotate(
-                        score=Coalesce(Subquery(simularity), 0.0))
+                    self._queryset = self._queryset.annotate(score=Coalesce(Subquery(simularity), 0.0))
                 else:
-                    self._queryset = self._queryset.annotate(
-                        simularity=Coalesce(Subquery(simularity), 0.0))
+                    self._queryset = self._queryset.annotate(simularity=Coalesce(Subquery(simularity), 0.0))
             if self._sort_includes('score') and self._fulltext_include and self._fuzzy_match is not None:
-                self._queryset = self._queryset.annotate(
-                    score=F('rank') + F('simularity'))
+                self._queryset = self._queryset.annotate(score=F('rank') + F('simularity'))
         else:
             query_filter = Q()
             for f in [x + '__unaccent__iexact' if x in self._unaccent_include else x + '__iexact' for x in SearchFields.objects.all().values_list('field', flat=True)]:
@@ -244,78 +229,69 @@ class RecipeSearch():
 
     def _cooked_on_filter(self, cooked_date=None):
         if self._sort_includes('lastcooked') or cooked_date:
-            lessthan = self._sort_includes(
-                '-lastcooked') or '-' in (cooked_date or [])[:1]
+            lessthan = self._sort_includes('-lastcooked') or '-' in (cooked_date or [])[:1]
             if lessthan:
                 default = timezone.now() - timedelta(days=100000)
             else:
                 default = timezone.now()
-            self._queryset = self._queryset.annotate(lastcooked=Coalesce(
-                Max(Case(When(cooklog__created_by=self._request.user, cooklog__space=self._request.space, then='cooklog__created_at'))), Value(default)))
+            self._queryset = self._queryset.annotate(
+                lastcooked=Coalesce(Max(Case(When(cooklog__created_by=self._request.user, cooklog__space=self._request.space, then='cooklog__created_at'))), Value(default))
+            )
         if cooked_date is None:
             return
 
-        cooked_date = date(*[int(x)
-                           for x in cooked_date.split('-') if x != ''])
+        cooked_date = date(*[int(x)for x in cooked_date.split('-') if x != ''])
 
         if lessthan:
-            self._queryset = self._queryset.filter(
-                lastcooked__date__lte=cooked_date).exclude(lastcooked=default)
+            self._queryset = self._queryset.filter(lastcooked__date__lte=cooked_date).exclude(lastcooked=default)
         else:
-            self._queryset = self._queryset.filter(
-                lastcooked__date__gte=cooked_date).exclude(lastcooked=default)
+            self._queryset = self._queryset.filter(lastcooked__date__gte=cooked_date).exclude(lastcooked=default)
 
     def _created_on_filter(self, created_date=None):
         if created_date is None:
             return
         lessthan = '-' in created_date[:1]
-        created_date = date(*[int(x)
-                            for x in created_date.split('-') if x != ''])
+        created_date = date(*[int(x) for x in created_date.split('-') if x != ''])
         if lessthan:
-            self._queryset = self._queryset.filter(
-                created_at__date__lte=created_date)
+            self._queryset = self._queryset.filter(created_at__date__lte=created_date)
         else:
-            self._queryset = self._queryset.filter(
-                created_at__date__gte=created_date)
+            self._queryset = self._queryset.filter(created_at__date__gte=created_date)
 
     def _updated_on_filter(self, updated_date=None):
         if updated_date is None:
             return
         lessthan = '-' in updated_date[:1]
-        updated_date = date(*[int(x)
-                            for x in updated_date.split('-') if x != ''])
+        updated_date = date(*[int(x)for x in updated_date.split('-') if x != ''])
         if lessthan:
-            self._queryset = self._queryset.filter(
-                updated_at__date__lte=updated_date)
+            self._queryset = self._queryset.filter(updated_at__date__lte=updated_date)
         else:
-            self._queryset = self._queryset.filter(
-                updated_at__date__gte=updated_date)
+            self._queryset = self._queryset.filter(updated_at__date__gte=updated_date)
 
     def _viewed_on_filter(self, viewed_date=None):
         if self._sort_includes('lastviewed') or viewed_date:
             longTimeAgo = timezone.now() - timedelta(days=100000)
-            self._queryset = self._queryset.annotate(lastviewed=Coalesce(
-                Max(Case(When(viewlog__created_by=self._request.user, viewlog__space=self._request.space, then='viewlog__created_at'))), Value(longTimeAgo)))
+            self._queryset = self._queryset.annotate(
+                lastviewed=Coalesce(Max(Case(When(viewlog__created_by=self._request.user, viewlog__space=self._request.space, then='viewlog__created_at'))), Value(longTimeAgo))
+            )
         if viewed_date is None:
             return
         lessthan = '-' in viewed_date[:1]
-        viewed_date = date(*[int(x)
-                           for x in viewed_date.split('-') if x != ''])
+        viewed_date = date(*[int(x)for x in viewed_date.split('-') if x != ''])
 
         if lessthan:
-            self._queryset = self._queryset.filter(
-                lastviewed__date__lte=viewed_date).exclude(lastviewed=longTimeAgo)
+            self._queryset = self._queryset.filter(lastviewed__date__lte=viewed_date).exclude(lastviewed=longTimeAgo)
         else:
-            self._queryset = self._queryset.filter(
-                lastviewed__date__gte=viewed_date).exclude(lastviewed=longTimeAgo)
+            self._queryset = self._queryset.filter(lastviewed__date__gte=viewed_date).exclude(lastviewed=longTimeAgo)
 
     def _new_recipes(self, new_days=7):
         # TODO make new days a user-setting
         if not self._new:
             return
-        self._queryset = (
-            self._queryset.annotate(new_recipe=Case(
-                When(created_at__gte=(timezone.now() - timedelta(days=new_days)), then=('pk')), default=Value(0), ))
+        self._queryset = self._queryset.annotate(
+            new_recipe=Case(
+                When(created_at__gte=(timezone.now() - timedelta(days=new_days)), then=('pk')),
+                default=Value(0),
+            )
         )
 
     def _recently_viewed(self, num_recent=None):
@@ -325,34 +301,35 @@ class RecipeSearch():
                     Max(Case(When(viewlog__created_by=self._request.user, viewlog__space=self._request.space, then='viewlog__pk'))), Value(0)))
             return
 
-        num_recent_recipes = ViewLog.objects.filter(created_by=self._request.user, space=self._request.space).values(
-            'recipe').annotate(recent=Max('created_at')).order_by('-recent')[:num_recent]
-        self._queryset = self._queryset.annotate(recent=Coalesce(Max(Case(When(
-            pk__in=num_recent_recipes.values('recipe'), then='viewlog__pk'))), Value(0)))
+        num_recent_recipes = (
+            ViewLog.objects.filter(created_by=self._request.user, space=self._request.space)
+            .values('recipe').annotate(recent=Max('created_at')).order_by('-recent')[:num_recent]
+        )
+        self._queryset = self._queryset.annotate(recent=Coalesce(Max(Case(When(pk__in=num_recent_recipes.values('recipe'), then='viewlog__pk'))), Value(0)))
 
     def _favorite_recipes(self, times_cooked=None):
         if self._sort_includes('favorite') or times_cooked:
-            less_than = '-' in (times_cooked or []
-                                ) and not self._sort_includes('-favorite')
+            less_than = '-' in (str(times_cooked) or []) and not self._sort_includes('-favorite')
             if less_than:
                 default = 1000
             else:
                 default = 0
-            favorite_recipes = CookLog.objects.filter(created_by=self._request.user, space=self._request.space, recipe=OuterRef('pk')
-                                                      ).values('recipe').annotate(count=Count('pk', distinct=True)).values('count')
-            self._queryset = self._queryset.annotate(
-                favorite=Coalesce(Subquery(favorite_recipes), default))
+            favorite_recipes = (
+                CookLog.objects.filter(created_by=self._request.user, space=self._request.space, recipe=OuterRef('pk'))
+                .values('recipe')
+                .annotate(count=Count('pk', distinct=True))
+                .values('count')
+            )
+            self._queryset = self._queryset.annotate(favorite=Coalesce(Subquery(favorite_recipes), default))
         if times_cooked is None:
             return
 
         if times_cooked == '0':
             self._queryset = self._queryset.filter(favorite=0)
         elif less_than:
-            self._queryset = self._queryset.filter(favorite__lte=int(
-                times_cooked.replace('-', ''))).exclude(favorite=0)
+            self._queryset = self._queryset.filter(favorite__lte=int(times_cooked.replace('-', ''))).exclude(favorite=0)
         else:
-            self._queryset = self._queryset.filter(
-                favorite__gte=int(times_cooked))
+            self._queryset = self._queryset.filter(favorite__gte=int(times_cooked))
 
     def keyword_filters(self, **kwargs):
         if all([kwargs[x] is None for x in kwargs]):
@@ -385,8 +362,7 @@ class RecipeSearch():
                     else:
                         self._queryset = self._queryset.filter(f_and)
                 if 'not' in kw_filter:
-                    self._queryset = self._queryset.exclude(
-                        id__in=recipes.values('id'))
+                    self._queryset = self._queryset.exclude(id__in=recipes.values('id'))
 
     def food_filters(self, **kwargs):
         if all([kwargs[x] is None for x in kwargs]):
@@ -400,8 +376,7 @@ class RecipeSearch():
             foods = Food.objects.filter(pk__in=kwargs[fd_filter])
             if 'or' in fd_filter:
                 if self._include_children:
-                    f_or = Q(
-                        steps__ingredients__food__in=Food.include_descendants(foods))
+                    f_or = Q(steps__ingredients__food__in=Food.include_descendants(foods))
                 else:
                     f_or = Q(steps__ingredients__food__in=foods)
 
@@ -413,8 +388,7 @@ class RecipeSearch():
                 recipes = Recipe.objects.all()
                 for food in foods:
                     if self._include_children:
-                        f_and = Q(
-                            steps__ingredients__food__in=food.get_descendants_and_self())
+                        f_and = Q(steps__ingredients__food__in=food.get_descendants_and_self())
                     else:
                         f_and = Q(steps__ingredients__food=food)
                     if 'not' in fd_filter:
@@ -422,8 +396,7 @@ class RecipeSearch():
                     else:
                         self._queryset = self._queryset.filter(f_and)
                 if 'not' in fd_filter:
-                    self._queryset = self._queryset.exclude(
-                        id__in=recipes.values('id'))
+                    self._queryset = self._queryset.exclude(id__in=recipes.values('id'))
 
     def unit_filters(self, units=None, operator=True):
         if operator != True:
@@ -432,8 +405,7 @@ class RecipeSearch():
             return
         if not isinstance(units, list):
             units = [units]
-        self._queryset = self._queryset.filter(
-            steps__ingredients__unit__in=units)
+        self._queryset = self._queryset.filter(steps__ingredients__unit__in=units)
 
     def rating_filter(self, rating=None):
         if rating or self._sort_includes('rating'):
@@ -479,14 +451,11 @@ class RecipeSearch():
                 recipes = Recipe.objects.all()
                 for book in kwargs[bk_filter]:
                     if 'not' in bk_filter:
-                        recipes = recipes.filter(
-                            recipebookentry__book__id=book)
+                        recipes = recipes.filter(recipebookentry__book__id=book)
                     else:
-                        self._queryset = self._queryset.filter(
-                            recipebookentry__book__id=book)
+                        self._queryset = self._queryset.filter(recipebookentry__book__id=book)
                 if 'not' in bk_filter:
-                    self._queryset = self._queryset.exclude(
-                        id__in=recipes.values('id'))
+                    self._queryset = self._queryset.exclude(id__in=recipes.values('id'))
 
     def step_filters(self, steps=None, operator=True):
         if operator != True:
@@ -505,25 +474,20 @@ class RecipeSearch():
             rank = []
             if 'name' in self._fulltext_include:
                 vectors.append('name_search_vector')
-                rank.append(SearchRank('name_search_vector',
-                            self.search_query, cover_density=True))
+                rank.append(SearchRank('name_search_vector', self.search_query, cover_density=True))
             if 'description' in self._fulltext_include:
                 vectors.append('desc_search_vector')
-                rank.append(SearchRank('desc_search_vector',
-                            self.search_query, cover_density=True))
+                rank.append(SearchRank('desc_search_vector', self.search_query, cover_density=True))
             if 'steps__instruction' in self._fulltext_include:
                 vectors.append('steps__search_vector')
-                rank.append(SearchRank('steps__search_vector',
-                            self.search_query, cover_density=True))
+                rank.append(SearchRank('steps__search_vector', self.search_query, cover_density=True))
             if 'keywords__name' in self._fulltext_include:
                 # explicitly settings unaccent on keywords and foods so that they behave the same as search_vector fields
                 vectors.append('keywords__name__unaccent')
-                rank.append(SearchRank('keywords__name__unaccent',
-                            self.search_query, cover_density=True))
+                rank.append(SearchRank('keywords__name__unaccent', self.search_query, cover_density=True))
             if 'steps__ingredients__food__name' in self._fulltext_include:
                 vectors.append('steps__ingredients__food__name__unaccent')
-                rank.append(SearchRank('steps__ingredients__food__name',
-                            self.search_query, cover_density=True))
+                rank.append(SearchRank('steps__ingredients__food__name', self.search_query, cover_density=True))
 
             for r in rank:
                 if self.search_rank is None:
@@ -531,8 +495,7 @@ class RecipeSearch():
                 else:
                     self.search_rank += r
             # modifying queryset will annotation creates duplicate results
-            self._filters.append(Q(id__in=Recipe.objects.annotate(
-                vector=SearchVector(*vectors)).filter(Q(vector=self.search_query))))
+            self._filters.append(Q(id__in=Recipe.objects.annotate(vector=SearchVector(*vectors)).filter(Q(vector=self.search_query))))
 
     def build_text_filters(self, string=None):
         if not string:
@@ -557,15 +520,19 @@ class RecipeSearch():
                     trigram += TrigramSimilarity(f, self._string)
                 else:
                     trigram = TrigramSimilarity(f, self._string)
-            self._fuzzy_match = Recipe.objects.annotate(trigram=trigram).distinct(
-            ).annotate(simularity=Max('trigram')).values('id', 'simularity').filter(simularity__gt=self._search_prefs.trigram_threshold)
+            self._fuzzy_match = (
+                Recipe.objects.annotate(trigram=trigram)
+                .distinct()
+                .annotate(simularity=Max('trigram'))
+                .values('id', 'simularity')
+                .filter(simularity__gt=self._search_prefs.trigram_threshold)
+            )
             self._filters += [Q(pk__in=self._fuzzy_match.values('pk'))]
 
     def _makenow_filter(self, missing=None):
         if missing is None or (isinstance(missing, bool) and missing == False):
             return
-        shopping_users = [
-            *self._request.user.get_shopping_share(), self._request.user]
+        shopping_users = [*self._request.user.get_shopping_share(), self._request.user]
 
         onhand_filter = (
             Q(steps__ingredients__food__onhand_users__in=shopping_users)  # food onhand
@@ -575,264 +542,40 @@ class RecipeSearch():
             | Q(steps__ingredients__food__in=self.__sibling_substitute_filter(shopping_users))
         )
         makenow_recipes = Recipe.objects.annotate(
-            count_food=Count('steps__ingredients__food__pk', filter=Q(
-                steps__ingredients__food__isnull=False), distinct=True),
-            count_onhand=Count('steps__ingredients__food__pk',
-                               filter=onhand_filter, distinct=True),
-            count_ignore_shopping=Count('steps__ingredients__food__pk', filter=Q(steps__ingredients__food__ignore_shopping=True,
-                                                                                 steps__ingredients__food__recipe__isnull=True), distinct=True),
-            has_child_sub=Case(When(steps__ingredients__food__in=self.__children_substitute_filter(
-                shopping_users), then=Value(1)), default=Value(0)),
-            has_sibling_sub=Case(When(steps__ingredients__food__in=self.__sibling_substitute_filter(
-                shopping_users), then=Value(1)), default=Value(0))
-        ).annotate(missingfood=F('count_food') - F('count_onhand') - F('count_ignore_shopping')).filter(missingfood=missing)
-        self._queryset = self._queryset.distinct().filter(
-            id__in=makenow_recipes.values('id'))
+            count_food=Count('steps__ingredients__food__pk', filter=Q(steps__ingredients__food__isnull=False), distinct=True),
+            count_onhand=Count('steps__ingredients__food__pk', filter=onhand_filter, distinct=True),
+            count_ignore_shopping=Count(
+                'steps__ingredients__food__pk', filter=Q(steps__ingredients__food__ignore_shopping=True, steps__ingredients__food__recipe__isnull=True), distinct=True
+            ),
+            has_child_sub=Case(When(steps__ingredients__food__in=self.__children_substitute_filter(shopping_users), then=Value(1)), default=Value(0)),
+            has_sibling_sub=Case(When(steps__ingredients__food__in=self.__sibling_substitute_filter(shopping_users), then=Value(1)), default=Value(0))
+        ).annotate(missingfood=F('count_food') - F('count_onhand') - F('count_ignore_shopping')).filter(missingfood__lte=missing)
+        self._queryset = self._queryset.distinct().filter(id__in=makenow_recipes.values('id'))
 
     @staticmethod
     def __children_substitute_filter(shopping_users=None):
-        children_onhand_subquery = Food.objects.filter(
-            path__startswith=OuterRef('path'),
-            depth__gt=OuterRef('depth'),
-            onhand_users__in=shopping_users
+        children_onhand_subquery = Food.objects.filter(path__startswith=OuterRef('path'), depth__gt=OuterRef('depth'), onhand_users__in=shopping_users)
+        return (
+            Food.objects.exclude(  # list of foods that are onhand and children of: foods that are not onhand and are set to use children as substitutes
+                Q(onhand_users__in=shopping_users) | Q(ignore_shopping=True, recipe__isnull=True) | Q(substitute__onhand_users__in=shopping_users)
+            )
+            .exclude(depth=1, numchild=0)
+            .filter(substitute_children=True)
+            .annotate(child_onhand_count=Exists(children_onhand_subquery))
+            .filter(child_onhand_count=True)
         )
-        return Food.objects.exclude(  # list of foods that are onhand and children of: foods that are not onhand and are set to use children as substitutes
-            Q(onhand_users__in=shopping_users)
-            | Q(ignore_shopping=True, recipe__isnull=True)
-            | Q(substitute__onhand_users__in=shopping_users)
-        ).exclude(depth=1, numchild=0
-                  ).filter(substitute_children=True
-                           ).annotate(child_onhand_count=Exists(children_onhand_subquery)
-                                      ).filter(child_onhand_count=True)
 
     @staticmethod
     def __sibling_substitute_filter(shopping_users=None):
         sibling_onhand_subquery = Food.objects.filter(
-            path__startswith=Substr(
-                OuterRef('path'), 1, Food.steplen * (OuterRef('depth') - 1)),
-            depth=OuterRef('depth'),
-            onhand_users__in=shopping_users
+            path__startswith=Substr(OuterRef('path'), 1, Food.steplen * (OuterRef('depth') - 1)), depth=OuterRef('depth'), onhand_users__in=shopping_users
         )
-        return Food.objects.exclude(  # list of foods that are onhand and siblings of: foods that are not onhand and are set to use siblings as substitutes
-            Q(onhand_users__in=shopping_users)
-            | Q(ignore_shopping=True, recipe__isnull=True)
-            | Q(substitute__onhand_users__in=shopping_users)
-        ).exclude(depth=1, numchild=0
-                  ).filter(substitute_siblings=True
-                           ).annotate(sibling_onhand=Exists(sibling_onhand_subquery)
-                                      ).filter(sibling_onhand=True)
-
-
-class RecipeFacet():
-    class CacheEmpty(Exception):
-        pass
-
-    def __init__(self, request, queryset=None, hash_key=None, cache_timeout=3600):
-        if hash_key is None and queryset is None:
-            raise ValueError(_("One of queryset or hash_key must be provided"))
-
-        self._request = request
-        self._queryset = queryset
-        self.hash_key = hash_key or str(hash(self._queryset.query))
-        self._SEARCH_CACHE_KEY = f"recipes_filter_{self.hash_key}"
-        self._cache_timeout = cache_timeout
-        self._cache = caches['default'].get(self._SEARCH_CACHE_KEY, {})
-        if self._cache is None and self._queryset is None:
-            raise self.CacheEmpty("No queryset provided and cache empty")
-
-        self.Keywords = self._cache.get('Keywords', None)
-        self.Foods = self._cache.get('Foods', None)
-        self.Books = self._cache.get('Books', None)
-        self.Ratings = self._cache.get('Ratings', None)
-        # TODO Move Recent to recipe annotation/serializer:  requrires change in RecipeSearch(), RecipeSearchView.vue and serializer
-        self.Recent = self._cache.get('Recent', None)
-
-        if self._queryset is not None:
-            self._recipe_list = list(
-                self._queryset.values_list('id', flat=True))
-            self._search_params = {
-                'keyword_list': self._request.query_params.getlist('keywords', []),
-                'food_list': self._request.query_params.getlist('foods', []),
-                'book_list': self._request.query_params.getlist('book', []),
-                'search_keywords_or': str2bool(self._request.query_params.get('keywords_or', True)),
-                'search_foods_or': str2bool(self._request.query_params.get('foods_or', True)),
-                'search_books_or': str2bool(self._request.query_params.get('books_or', True)),
-                'space': self._request.space,
-            }
-        elif self.hash_key is not None:
-            self._recipe_list = self._cache.get('recipe_list', [])
-            self._search_params = {
-                'keyword_list': self._cache.get('keyword_list', None),
-                'food_list': self._cache.get('food_list', None),
-                'book_list': self._cache.get('book_list', None),
-                'search_keywords_or': self._cache.get('search_keywords_or', None),
-                'search_foods_or': self._cache.get('search_foods_or', None),
-                'search_books_or': self._cache.get('search_books_or', None),
-                'space': self._cache.get('space', None),
-            }
-
-        self._cache = {
-            **self._search_params,
-            'recipe_list': self._recipe_list,
-            'Ratings': self.Ratings,
-            'Recent': self.Recent,
-            'Keywords': self.Keywords,
-            'Foods': self.Foods,
-            'Books': self.Books
-
-        }
-        caches['default'].set(self._SEARCH_CACHE_KEY,
-                              self._cache, self._cache_timeout)
-
-    def get_facets(self, from_cache=False):
-        if from_cache:
-            return {
-                'cache_key': self.hash_key or '',
-                'Ratings': self.Ratings or {},
-                'Recent': self.Recent or [],
-                'Keywords': self.Keywords or [],
-                'Foods': self.Foods or [],
-                'Books': self.Books or []
-            }
-        return {
-            'cache_key': self.hash_key,
-            'Ratings': self.get_ratings(),
-            'Recent': self.get_recent(),
-            'Keywords': self.get_keywords(),
-            'Foods': self.get_foods(),
-            'Books': self.get_books()
-        }
-
-    def set_cache(self, key, value):
-        self._cache = {**self._cache, key: value}
-        caches['default'].set(
-            self._SEARCH_CACHE_KEY,
-            self._cache,
-            self._cache_timeout
+        return (
+            Food.objects.exclude(  # list of foods that are onhand and siblings of: foods that are not onhand and are set to use siblings as substitutes
+                Q(onhand_users__in=shopping_users) | Q(ignore_shopping=True, recipe__isnull=True) | Q(substitute__onhand_users__in=shopping_users)
+            )
+            .exclude(depth=1, numchild=0)
+            .filter(substitute_siblings=True)
+            .annotate(sibling_onhand=Exists(sibling_onhand_subquery))
+            .filter(sibling_onhand=True)
         )
-
-    def get_books(self):
-        if self.Books is None:
-            self.Books = []
-        return self.Books
-
-    def get_keywords(self):
-        if self.Keywords is None:
-            if self._search_params['search_keywords_or']:
-                keywords = Keyword.objects.filter(
-                    space=self._request.space).distinct()
-            else:
-                keywords = Keyword.objects.filter(Q(recipe__in=self._recipe_list) | Q(
-                    depth=1)).filter(space=self._request.space).distinct()
-
-            # set keywords to root objects only
-            keywords = self._keyword_queryset(keywords)
-            self.Keywords = [{**x, 'children': None}
-                             if x['numchild'] > 0 else x for x in list(keywords)]
-            self.set_cache('Keywords', self.Keywords)
-        return self.Keywords
-
-    def get_foods(self):
-        if self.Foods is None:
-            # # if using an OR search, will annotate all keywords, otherwise, just those that appear in results
-            if self._search_params['search_foods_or']:
-                foods = Food.objects.filter(
-                    space=self._request.space).distinct()
-            else:
-                foods = Food.objects.filter(Q(ingredient__step__recipe__in=self._recipe_list) | Q(
-                    depth=1)).filter(space=self._request.space).distinct()
-
-            # set keywords to root objects only
-            foods = self._food_queryset(foods)
-
-            self.Foods = [{**x, 'children': None}
-                          if x['numchild'] > 0 else x for x in list(foods)]
-            self.set_cache('Foods', self.Foods)
-        return self.Foods
-
-    def get_ratings(self):
-        if self.Ratings is None:
-            if not self._request.space.demo and self._request.space.show_facet_count:
-                if self._queryset is None:
-                    self._queryset = Recipe.objects.filter(
-                        id__in=self._recipe_list)
-                rating_qs = self._queryset.annotate(rating=Round(Avg(Case(When(
-                    cooklog__created_by=self._request.user, then='cooklog__rating'), default=Value(0)))))
-                self.Ratings = dict(Counter(r.rating for r in rating_qs))
-            else:
-                self.Rating = {}
-            self.set_cache('Ratings', self.Ratings)
-        return self.Ratings
-
-    def get_recent(self):
-        if self.Recent is None:
-            # TODO make days of recent recipe a setting
-            recent_recipes = ViewLog.objects.filter(created_by=self._request.user, space=self._request.space, created_at__gte=timezone.now() - timedelta(days=14)
-                                                    ).values_list('recipe__pk', flat=True)
-            self.Recent = list(recent_recipes)
-            self.set_cache('Recent', self.Recent)
-        return self.Recent
-
-    def add_food_children(self, id):
-        try:
-            food = Food.objects.get(id=id)
-            nodes = food.get_ancestors()
-        except Food.DoesNotExist:
-            return self.get_facets()
-        foods = self._food_queryset(food.get_children(), food)
-        deep_search = self.Foods
-        for node in nodes:
-            index = next((i for i, x in enumerate(
-                deep_search) if x["id"] == node.id), None)
-            deep_search = deep_search[index]['children']
-        index = next((i for i, x in enumerate(
-            deep_search) if x["id"] == food.id), None)
-        deep_search[index]['children'] = [
-            {**x, 'children': None} if x['numchild'] > 0 else x for x in list(foods)]
-        self.set_cache('Foods', self.Foods)
-        return self.get_facets()
-
-    def add_keyword_children(self, id):
-        try:
-            keyword = Keyword.objects.get(id=id)
-            nodes = keyword.get_ancestors()
-        except Keyword.DoesNotExist:
-            return self.get_facets()
-        keywords = self._keyword_queryset(keyword.get_children(), keyword)
-        deep_search = self.Keywords
-        for node in nodes:
-            index = next((i for i, x in enumerate(
-                deep_search) if x["id"] == node.id), None)
-            deep_search = deep_search[index]['children']
-        index = next((i for i, x in enumerate(deep_search)
-                     if x["id"] == keyword.id), None)
-        deep_search[index]['children'] = [
-            {**x, 'children': None} if x['numchild'] > 0 else x for x in list(keywords)]
-        self.set_cache('Keywords', self.Keywords)
-        return self.get_facets()
-
-    def _recipe_count_queryset(self, field, depth=1, steplen=4):
-        return Recipe.objects.filter(**{f'{field}__path__startswith': OuterRef('path'), f'{field}__depth__gte': depth}, id__in=self._recipe_list, space=self._request.space
-                                     ).annotate(count=Coalesce(Func('pk', function='Count'), 0)).values('count')
-
-    def _keyword_queryset(self, queryset, keyword=None):
-        depth = getattr(keyword, 'depth', 0) + 1
-        steplen = depth * Keyword.steplen
-
-        if not self._request.space.demo and self._request.space.show_facet_count:
-            return queryset.annotate(count=Coalesce(Subquery(self._recipe_count_queryset('keywords', depth, steplen)), 0)
-                                     ).filter(depth=depth, count__gt=0
-                                              ).values('id', 'name', 'count', 'numchild').order_by(Lower('name').asc())[:200]
-        else:
-            return queryset.filter(depth=depth).values('id', 'name', 'numchild').order_by(Lower('name').asc())
-
-    def _food_queryset(self, queryset, food=None):
-        depth = getattr(food, 'depth', 0) + 1
-        steplen = depth * Food.steplen
-
-        if not self._request.space.demo and self._request.space.show_facet_count:
-            return queryset.annotate(count=Coalesce(Subquery(self._recipe_count_queryset('steps__ingredients__food', depth, steplen)), 0)
-                                     ).filter(depth__lte=depth, count__gt=0
-                                              ).values('id', 'name', 'count', 'numchild').order_by(Lower('name').asc())[:200]
-        else:
-            return queryset.filter(depth__lte=depth).values('id', 'name', 'numchild').order_by(Lower('name').asc())

cookbook/helper/recipe_url_import.py

@@ -1,9 +1,7 @@
-# import random
 import re
 import traceback
 from html import unescape
 
-from django.core.cache import caches
 from django.utils.dateparse import parse_duration
 from django.utils.translation import gettext as _
 from isodate import parse_duration as iso_parse_duration
@@ -11,20 +9,37 @@ from isodate.isoerror import ISO8601Error
 from pytube import YouTube
 from recipe_scrapers._utils import get_host_name, get_minutes
 
-# from cookbook.helper import recipe_url_import as helper
+from cookbook.helper.automation_helper import AutomationEngine
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.models import Automation, Keyword, PropertyType
 
 
-# from unicodedata import decomposition
-
-
-# from recipe_scrapers._utils import get_minutes  ## temporary until/unless upstream incorporates get_minutes() PR
-
-
 def get_from_scraper(scrape, request):
     # converting the scrape_me object to the existing json format based on ld+json
-    recipe_json = {}
+
+    recipe_json = {
+        'steps': [],
+        'internal': True
+    }
+    keywords = []
+
+    # assign source URL
+    try:
+        source_url = scrape.canonical_url()
+    except Exception:
+        try:
+            source_url = scrape.url
+        except Exception:
+            pass
+    if source_url:
+        recipe_json['source_url'] = source_url
+        try:
+            keywords.append(source_url.replace('http://', '').replace('https://', '').split('/')[0])
+        except Exception:
+            recipe_json['source_url'] = ''
+
+    automation_engine = AutomationEngine(request, source=recipe_json.get('source_url'))
+    # assign recipe name
     try:
         recipe_json['name'] = parse_name(scrape.title()[:128] or None)
     except Exception:
@@ -38,6 +53,10 @@ def get_from_scraper(scrape, request):
     if isinstance(recipe_json['name'], list) and len(recipe_json['name']) > 0:
         recipe_json['name'] = recipe_json['name'][0]
 
+    recipe_json['name'] = automation_engine.apply_regex_replace_automation(recipe_json['name'], Automation.NAME_REPLACE)
+
+    # assign recipe description
+    # TODO notify user about limit if reached - >256 description will be truncated
     try:
         description = scrape.description() or None
     except Exception:
@@ -48,16 +67,20 @@ def get_from_scraper(scrape, request):
         except Exception:
             description = ''
 
-    recipe_json['internal'] = True
+    recipe_json['description'] = parse_description(description)
+    recipe_json['description'] = automation_engine.apply_regex_replace_automation(recipe_json['description'], Automation.DESCRIPTION_REPLACE)
 
+    # assign servings attributes
     try:
-        servings = scrape.schema.data.get('recipeYield') or 1  # dont use scrape.yields() as this will always return "x servings" or "x items", should be improved in scrapers directly
+        # dont use scrape.yields() as this will always return "x servings" or "x items", should be improved in scrapers directly
+        servings = scrape.schema.data.get('recipeYield') or 1
     except Exception:
         servings = 1
 
     recipe_json['servings'] = parse_servings(servings)
     recipe_json['servings_text'] = parse_servings_text(servings)
 
+    # assign time attributes
     try:
         recipe_json['working_time'] = get_minutes(scrape.prep_time()) or 0
     except Exception:
@@ -82,6 +105,7 @@ def get_from_scraper(scrape, request):
             except Exception:
                 pass
 
+    # assign image
     try:
         recipe_json['image'] = parse_image(scrape.image()) or None
     except Exception:
@@ -92,7 +116,7 @@ def get_from_scraper(scrape, request):
         except Exception:
             recipe_json['image'] = ''
 
-    keywords = []
+    # assign keywords
     try:
         if scrape.schema.data.get("keywords"):
             keywords += listify_keywords(scrape.schema.data.get("keywords"))
@@ -117,20 +141,6 @@ def get_from_scraper(scrape, request):
         except Exception:
             pass
 
-    try:
-        source_url = scrape.canonical_url()
-    except Exception:
-        try:
-            source_url = scrape.url
-        except Exception:
-            pass
-    if source_url:
-        recipe_json['source_url'] = source_url
-        try:
-            keywords.append(source_url.replace('http://', '').replace('https://', '').split('/')[0])
-        except Exception:
-            recipe_json['source_url'] = ''
-
     try:
         if scrape.author():
             keywords.append(scrape.author())
@@ -138,33 +148,24 @@ def get_from_scraper(scrape, request):
         pass
 
     try:
-        recipe_json['keywords'] = parse_keywords(list(set(map(str.casefold, keywords))), request.space)
+        recipe_json['keywords'] = parse_keywords(list(set(map(str.casefold, keywords))), request)
     except AttributeError:
         recipe_json['keywords'] = keywords
 
     ingredient_parser = IngredientParser(request, True)
 
-    recipe_json['steps'] = []
+    # assign steps
     try:
         for i in parse_instructions(scrape.instructions()):
-            recipe_json['steps'].append({'instruction': i, 'ingredients': [], })
+            recipe_json['steps'].append({'instruction': i, 'ingredients': [], 'show_ingredients_table': request.user.userpreference.show_step_ingredients, })
     except Exception:
         pass
     if len(recipe_json['steps']) == 0:
         recipe_json['steps'].append({'instruction': '', 'ingredients': [], })
 
-    parsed_description = parse_description(description)
-    # TODO notify user about limit if reached
-    # limits exist to limit the attack surface for dos style attacks
-    automations = Automation.objects.filter(type=Automation.DESCRIPTION_REPLACE, space=request.space, disabled=False).only('param_1', 'param_2', 'param_3').all().order_by('order')[:512]
-    for a in automations:
-        if re.match(a.param_1, (recipe_json['source_url'])[:512]):
-            parsed_description = re.sub(a.param_2, a.param_3, parsed_description, count=1)
-
-    if len(parsed_description) > 256:  # split at 256 as long descriptions don't look good on recipe cards
-        recipe_json['steps'][0]['instruction'] = f'*{parsed_description}*  \n\n' + recipe_json['steps'][0]['instruction']
-    else:
-        recipe_json['description'] = parsed_description[:512]
+    recipe_json['description'] = recipe_json['description'][:512]
+    if len(recipe_json['description']) > 256:  # split at 256 as long descriptions don't look good on recipe cards
+        recipe_json['steps'][0]['instruction'] = f"*{recipe_json['description']}*  \n\n" + recipe_json['steps'][0]['instruction']
 
     try:
         for x in scrape.ingredients():
@@ -205,12 +206,9 @@ def get_from_scraper(scrape, request):
         traceback.print_exc()
         pass
 
-    if 'source_url' in recipe_json and recipe_json['source_url']:
-        automations = Automation.objects.filter(type=Automation.INSTRUCTION_REPLACE, space=request.space, disabled=False).only('param_1', 'param_2', 'param_3').order_by('order').all()[:512]
-        for a in automations:
-            if re.match(a.param_1, (recipe_json['source_url'])[:512]):
-                for s in recipe_json['steps']:
-                    s['instruction'] = re.sub(a.param_2, a.param_3, s['instruction'])
+    for s in recipe_json['steps']:
+        s['instruction'] = automation_engine.apply_regex_replace_automation(s['instruction'], Automation.INSTRUCTION_REPLACE)
+        # re.sub(a.param_2, a.param_3, s['instruction'])
 
     return recipe_json
 
@@ -233,7 +231,7 @@ def get_recipe_properties(space, property_data):
                             'id': pt.id,
                             'name': pt.name,
                         },
-                        'property_amount': parse_servings(property_data[properties[p]]) / float(property_data['servingSize']),
+                        'property_amount': parse_servings(property_data[properties[p]]) / parse_servings(property_data['servingSize']),
                     })
 
     return recipe_properties
@@ -261,10 +259,14 @@ def get_from_youtube_scraper(url, request):
     }
 
     try:
-        video = YouTube(url=url)
-        default_recipe_json['name'] = video.title
+        automation_engine = AutomationEngine(request, source=url)
+        video = YouTube(url)
+        video.streams.first()  # this is required to execute some kind of generator/web request that fetches the description
+        default_recipe_json['name'] = automation_engine.apply_regex_replace_automation(video.title, Automation.NAME_REPLACE)
         default_recipe_json['image'] = video.thumbnail_url
-        default_recipe_json['steps'][0]['instruction'] = video.description
+        if video.description:
+            default_recipe_json['steps'][0]['instruction'] = automation_engine.apply_regex_replace_automation(video.description, Automation.INSTRUCTION_REPLACE)
+
     except Exception:
         pass
 
@@ -272,7 +274,7 @@ def get_from_youtube_scraper(url, request):
 
 
 def parse_name(name):
-    if type(name) == list:
+    if isinstance(name, list):
         try:
             name = name[0]
         except Exception:
@@ -316,16 +318,16 @@ def parse_instructions(instructions):
     """
     instruction_list = []
 
-    if type(instructions) == list:
+    if isinstance(instructions, list):
         for i in instructions:
-            if type(i) == str:
+            if isinstance(i, str):
                 instruction_list.append(clean_instruction_string(i))
             else:
                 if 'text' in i:
                     instruction_list.append(clean_instruction_string(i['text']))
                 elif 'itemListElement' in i:
                     for ile in i['itemListElement']:
-                        if type(ile) == str:
+                        if isinstance(ile, str):
                             instruction_list.append(clean_instruction_string(ile))
                         elif 'text' in ile:
                             instruction_list.append(clean_instruction_string(ile['text']))
@@ -341,13 +343,13 @@ def parse_image(image):
     # check if list of images is returned, take first if so
     if not image:
         return None
-    if type(image) == list:
+    if isinstance(image, list):
         for pic in image:
-            if (type(pic) == str) and (pic[:4] == 'http'):
+            if (isinstance(pic, str)) and (pic[:4] == 'http'):
                 image = pic
             elif 'url' in pic:
                 image = pic['url']
-    elif type(image) == dict:
+    elif isinstance(image, dict):
         if 'url' in image:
             image = image['url']
 
@@ -358,12 +360,12 @@ def parse_image(image):
 
 
 def parse_servings(servings):
-    if type(servings) == str:
+    if isinstance(servings, str):
         try:
             servings = int(re.search(r'\d+', servings).group())
         except AttributeError:
             servings = 1
-    elif type(servings) == list:
+    elif isinstance(servings, list):
         try:
             servings = int(re.findall(r'\b\d+\b', servings[0])[0])
         except KeyError:
@@ -372,12 +374,12 @@ def parse_servings(servings):
 
 
 def parse_servings_text(servings):
-    if type(servings) == str:
+    if isinstance(servings, str):
         try:
-            servings = re.sub("\d+", '', servings).strip()
+            servings = re.sub("\\d+", '', servings).strip()
         except Exception:
             servings = ''
-    if type(servings) == list:
+    if isinstance(servings, list):
         try:
             servings = parse_servings_text(servings[1])
         except Exception:
@@ -394,7 +396,7 @@ def parse_time(recipe_time):
                 recipe_time = round(iso_parse_duration(recipe_time).seconds / 60)
             except ISO8601Error:
                 try:
-                    if (type(recipe_time) == list and len(recipe_time) > 0):
+                    if (isinstance(recipe_time, list) and len(recipe_time) > 0):
                         recipe_time = recipe_time[0]
                     recipe_time = round(parse_duration(recipe_time).seconds / 60)
                 except AttributeError:
@@ -403,18 +405,9 @@ def parse_time(recipe_time):
     return recipe_time
 
 
-def parse_keywords(keyword_json, space):
+def parse_keywords(keyword_json, request):
     keywords = []
-    keyword_aliases = {}
-    # retrieve keyword automation cache if it exists, otherwise build from database
-    KEYWORD_CACHE_KEY = f'automation_keyword_alias_{space.pk}'
-    if c := caches['default'].get(KEYWORD_CACHE_KEY, None):
-        keyword_aliases = c
-        caches['default'].touch(KEYWORD_CACHE_KEY, 30)
-    else:
-        for a in Automation.objects.filter(space=space, disabled=False, type=Automation.KEYWORD_ALIAS).only('param_1', 'param_2').order_by('order').all():
-            keyword_aliases[a.param_1] = a.param_2
-        caches['default'].set(KEYWORD_CACHE_KEY, keyword_aliases, 30)
+    automation_engine = AutomationEngine(request)
 
     # keywords as list
     for kw in keyword_json:
@@ -422,12 +415,8 @@ def parse_keywords(keyword_json, space):
         # if alias exists use that instead
 
         if len(kw) != 0:
-            if keyword_aliases:
-                try:
-                    kw = keyword_aliases[kw]
-                except KeyError:
-                    pass
-            if k := Keyword.objects.filter(name=kw, space=space).first():
+            kw = automation_engine.apply_keyword_automation(kw)
+            if k := Keyword.objects.filter(name__iexact=kw, space=request.space).first():
                 keywords.append({'label': str(k), 'name': k.name, 'id': k.id})
             else:
                 keywords.append({'label': kw, 'name': kw})
@@ -438,15 +427,15 @@ def parse_keywords(keyword_json, space):
 def listify_keywords(keyword_list):
     # keywords as string
     try:
-        if type(keyword_list[0]) == dict:
+        if isinstance(keyword_list[0], dict):
             return keyword_list
     except (KeyError, IndexError):
         pass
-    if type(keyword_list) == str:
+    if isinstance(keyword_list, str):
         keyword_list = keyword_list.split(',')
 
     # keywords as string in list
-    if (type(keyword_list) == list and len(keyword_list) == 1 and ',' in keyword_list[0]):
+    if (isinstance(keyword_list, list) and len(keyword_list) == 1 and ',' in keyword_list[0]):
         keyword_list = keyword_list[0].split(',')
     return [x.strip() for x in keyword_list]
 
@@ -500,13 +489,13 @@ def get_images_from_soup(soup, url):
 
 
 def clean_dict(input_dict, key):
-    if type(input_dict) == dict:
+    if isinstance(input_dict, dict):
         for x in list(input_dict):
             if x == key:
                 del input_dict[x]
-            elif type(input_dict[x]) == dict:
+            elif isinstance(input_dict[x], dict):
                 input_dict[x] = clean_dict(input_dict[x], key)
-            elif type(input_dict[x]) == list:
+            elif isinstance(input_dict[x], list):
                 temp_list = []
                 for e in input_dict[x]:
                     temp_list.append(clean_dict(e, key))

cookbook/helper/scope_middleware.py

@@ -1,8 +1,6 @@
 from django.urls import reverse
 from django_scopes import scope, scopes_disabled
 from oauth2_provider.contrib.rest_framework import OAuth2Authentication
-from rest_framework.authentication import TokenAuthentication
-from rest_framework.authtoken.models import Token
 from rest_framework.exceptions import AuthenticationFailed
 
 from cookbook.views import views
@@ -50,7 +48,6 @@ class ScopeMiddleware:
                 return views.no_groups(request)
 
             request.space = user_space.space
-            # with scopes_disabled():
             with scope(space=request.space):
                 return self.get_response(request)
         else:

cookbook/helper/shopping_helper.py

@@ -1,16 +1,12 @@
-from datetime import timedelta
+
 from decimal import Decimal
 
-from django.contrib.postgres.aggregates import ArrayAgg
 from django.db.models import F, OuterRef, Q, Subquery, Value
 from django.db.models.functions import Coalesce
-from django.utils import timezone
 from django.utils.translation import gettext as _
 
-from cookbook.helper.HelperFunctions import Round, str2bool
 from cookbook.models import (Ingredient, MealPlan, Recipe, ShoppingListEntry, ShoppingListRecipe,
                              SupermarketCategoryRelation)
-from recipes import settings
 
 
 def shopping_helper(qs, request):
@@ -30,9 +26,6 @@ def shopping_helper(qs, request):
     elif checked in ['true', 1, '1']:
         qs = qs.filter(checked=True)
     elif checked in ['recent']:
-        today_start = timezone.now().replace(hour=0, minute=0, second=0)
-        week_ago = today_start - timedelta(days=user.userpreference.shopping_recent_days)
-        qs = qs.filter(Q(checked=False) | Q(completed_at__gte=week_ago))
         supermarket_order = ['checked'] + supermarket_order
 
     return qs.distinct().order_by(*supermarket_order).select_related('unit', 'food', 'ingredient', 'created_by', 'list_recipe', 'list_recipe__mealplan', 'list_recipe__recipe')
@@ -47,7 +40,7 @@ class RecipeShoppingEditor():
         self.mealplan = self._kwargs.get('mealplan', None)
         if type(self.mealplan) in [int, float]:
             self.mealplan = MealPlan.objects.filter(id=self.mealplan, space=self.space)
-        if type(self.mealplan) == dict:
+        if isinstance(self.mealplan, dict):
             self.mealplan = MealPlan.objects.filter(id=self.mealplan['id'], space=self.space).first()
         self.id = self._kwargs.get('id', None)
 
@@ -69,11 +62,12 @@ class RecipeShoppingEditor():
 
     @property
     def _recipe_servings(self):
-        return getattr(self.recipe, 'servings', None) or getattr(getattr(self.mealplan, 'recipe', None), 'servings', None) or getattr(getattr(self._shopping_list_recipe, 'recipe', None), 'servings', None)
+        return getattr(self.recipe, 'servings', None) or getattr(getattr(self.mealplan, 'recipe', None), 'servings',
+                                                                 None) or getattr(getattr(self._shopping_list_recipe, 'recipe', None), 'servings', None)
 
     @property
     def _servings_factor(self):
-        return Decimal(self.servings)/Decimal(self._recipe_servings)
+        return Decimal(self.servings) / Decimal(self._recipe_servings)
 
     @property
     def _shared_users(self):
@@ -81,18 +75,17 @@ class RecipeShoppingEditor():
 
     @staticmethod
     def get_shopping_list_recipe(id, user, space):
-        return ShoppingListRecipe.objects.filter(id=id).filter(Q(shoppinglist__space=space) | Q(entries__space=space)).filter(
-            Q(shoppinglist__created_by=user)
-            | Q(shoppinglist__shared=user)
-            | Q(entries__created_by=user)
+        return ShoppingListRecipe.objects.filter(id=id).filter(entries__space=space).filter(
+            Q(entries__created_by=user)
             | Q(entries__created_by__in=list(user.get_shopping_share()))
         ).prefetch_related('entries').first()
 
     def get_recipe_ingredients(self, id, exclude_onhand=False):
         if exclude_onhand:
-            return Ingredient.objects.filter(step__recipe__id=id,  food__ignore_shopping=False,  space=self.space).exclude(food__onhand_users__id__in=[x.id for x in self._shared_users])
+            return Ingredient.objects.filter(step__recipe__id=id, food__ignore_shopping=False, space=self.space).exclude(
+                food__onhand_users__id__in=[x.id for x in self._shared_users])
         else:
-            return Ingredient.objects.filter(step__recipe__id=id,  food__ignore_shopping=False,  space=self.space)
+            return Ingredient.objects.filter(step__recipe__id=id, food__ignore_shopping=False, space=self.space)
 
     @property
     def _include_related(self):
@@ -109,7 +102,7 @@ class RecipeShoppingEditor():
             self.servings = float(servings)
 
         if mealplan := kwargs.get('mealplan', None):
-            if type(mealplan) == dict:
+            if isinstance(mealplan, dict):
                 self.mealplan = MealPlan.objects.filter(id=mealplan['id'], space=self.space).first()
             else:
                 self.mealplan = mealplan
@@ -170,14 +163,14 @@ class RecipeShoppingEditor():
         try:
             self._shopping_list_recipe.delete()
             return True
-        except:
+        except BaseException:
             return False
 
     def _add_ingredients(self, ingredients=None):
         if not ingredients:
             return
-        elif type(ingredients) == list:
-            ingredients = Ingredient.objects.filter(id__in=ingredients)
+        elif isinstance(ingredients, list):
+            ingredients = Ingredient.objects.filter(id__in=ingredients, food__ignore_shopping=False)
         existing = self._shopping_list_recipe.entries.filter(ingredient__in=ingredients).values_list('ingredient__pk', flat=True)
         add_ingredients = ingredients.exclude(id__in=existing)
 
@@ -199,120 +192,3 @@ class RecipeShoppingEditor():
         to_delete = self._shopping_list_recipe.entries.exclude(ingredient__in=ingredients)
         ShoppingListEntry.objects.filter(id__in=to_delete).delete()
         self._shopping_list_recipe = self.get_shopping_list_recipe(self.id, self.created_by, self.space)
-
-
-# # TODO refactor as class
-# def list_from_recipe(list_recipe=None, recipe=None, mealplan=None, servings=None, ingredients=None, created_by=None, space=None, append=False):
-#     """
-#     Creates ShoppingListRecipe and associated ShoppingListEntrys from a recipe or a meal plan with a recipe
-#     :param list_recipe: Modify an existing ShoppingListRecipe
-#     :param recipe: Recipe to use as list of ingredients.  One of [recipe, mealplan] are required
-#     :param mealplan: alternatively use a mealplan recipe as source of ingredients
-#     :param servings: Optional: Number of servings to use to scale shoppinglist.  If servings = 0 an existing recipe list will be deleted
-#     :param ingredients: Ingredients, list of ingredient IDs to include on the shopping list.  When not provided all ingredients will be used
-#     :param append: If False will remove any entries not included with ingredients, when True will append ingredients to the shopping list
-#     """
-#     r = recipe or getattr(mealplan, 'recipe', None) or getattr(list_recipe, 'recipe', None)
-#     if not r:
-#         raise ValueError(_("You must supply a recipe or mealplan"))
-
-#     created_by = created_by or getattr(ShoppingListEntry.objects.filter(list_recipe=list_recipe).first(), 'created_by', None)
-#     if not created_by:
-#         raise ValueError(_("You must supply a created_by"))
-
-#     try:
-#         servings = float(servings)
-#     except (ValueError, TypeError):
-#         servings = getattr(mealplan, 'servings', 1.0)
-
-#     servings_factor = servings / r.servings
-
-#     shared_users = list(created_by.get_shopping_share())
-#     shared_users.append(created_by)
-#     if list_recipe:
-#         created = False
-#     else:
-#         list_recipe = ShoppingListRecipe.objects.create(recipe=r, mealplan=mealplan, servings=servings)
-#         created = True
-
-#     related_step_ing = []
-#     if servings == 0 and not created:
-#         list_recipe.delete()
-#         return []
-#     elif ingredients:
-#         ingredients = Ingredient.objects.filter(pk__in=ingredients, space=space)
-#     else:
-#         ingredients = Ingredient.objects.filter(step__recipe=r,  food__ignore_shopping=False, space=space)
-
-#         if exclude_onhand := created_by.userpreference.mealplan_autoexclude_onhand:
-#             ingredients = ingredients.exclude(food__onhand_users__id__in=[x.id for x in shared_users])
-
-#         if related := created_by.userpreference.mealplan_autoinclude_related:
-#             # TODO: add levels of related recipes (related recipes of related recipes) to use when auto-adding mealplans
-#             related_recipes = r.get_related_recipes()
-
-#             for x in related_recipes:
-#                 # related recipe is a Step serving size is driven by recipe serving size
-#                 # TODO once/if Steps can have a serving size this needs to be refactored
-#                 if exclude_onhand:
-#                     # if steps are used more than once in a recipe or subrecipe - I don' think this results in the desired behavior
-#                     related_step_ing += Ingredient.objects.filter(step__recipe=x, space=space).exclude(food__onhand_users__id__in=[x.id for x in shared_users]).values_list('id', flat=True)
-#                 else:
-#                     related_step_ing += Ingredient.objects.filter(step__recipe=x, space=space).values_list('id', flat=True)
-
-#                 x_ing = []
-#                 if ingredients.filter(food__recipe=x).exists():
-#                     for ing in ingredients.filter(food__recipe=x):
-#                         if exclude_onhand:
-#                             x_ing = Ingredient.objects.filter(step__recipe=x,  food__ignore_shopping=False, space=space).exclude(food__onhand_users__id__in=[x.id for x in shared_users])
-#                         else:
-#                             x_ing = Ingredient.objects.filter(step__recipe=x,  food__ignore_shopping=False, space=space).exclude(food__ignore_shopping=True)
-#                         for i in [x for x in x_ing]:
-#                             ShoppingListEntry.objects.create(
-#                                 list_recipe=list_recipe,
-#                                 food=i.food,
-#                                 unit=i.unit,
-#                                 ingredient=i,
-#                                 amount=i.amount * Decimal(servings_factor),
-#                                 created_by=created_by,
-#                                 space=space,
-#                             )
-#                 # dont' add food to the shopping list that are actually recipes that will be added as ingredients
-#                 ingredients = ingredients.exclude(food__recipe=x)
-
-#     add_ingredients = list(ingredients.values_list('id', flat=True)) + related_step_ing
-#     if not append:
-#         existing_list = ShoppingListEntry.objects.filter(list_recipe=list_recipe)
-#         # delete shopping list entries not included in ingredients
-#         existing_list.exclude(ingredient__in=ingredients).delete()
-#         # add shopping list entries that did not previously exist
-#         add_ingredients = set(add_ingredients) - set(existing_list.values_list('ingredient__id', flat=True))
-#     add_ingredients = Ingredient.objects.filter(id__in=add_ingredients, space=space)
-
-#     # if servings have changed, update the ShoppingListRecipe and existing Entries
-#     if servings <= 0:
-#         servings = 1
-
-#     if not created and list_recipe.servings != servings:
-#         update_ingredients = set(ingredients.values_list('id', flat=True)) - set(add_ingredients.values_list('id', flat=True))
-#         list_recipe.servings = servings
-#         list_recipe.save()
-#         for sle in ShoppingListEntry.objects.filter(list_recipe=list_recipe, ingredient__id__in=update_ingredients):
-#             sle.amount = sle.ingredient.amount * Decimal(servings_factor)
-#             sle.save()
-
-#     # add any missing Entries
-#     for i in [x for x in add_ingredients if x.food]:
-
-#         ShoppingListEntry.objects.create(
-#             list_recipe=list_recipe,
-#             food=i.food,
-#             unit=i.unit,
-#             ingredient=i,
-#             amount=i.amount * Decimal(servings_factor),
-#             created_by=created_by,
-#             space=space,
-#         )
-
-#     # return all shopping list items
-#     return list_recipe

cookbook/helper/template_helper.py

@@ -2,7 +2,6 @@ from gettext import gettext as _
 
 import bleach
 import markdown as md
-from bleach_allowlist import markdown_attrs, markdown_tags
 from jinja2 import Template, TemplateSyntaxError, UndefinedError
 from markdown.extensions.tables import TableExtension
 
@@ -15,12 +14,14 @@ class IngredientObject(object):
     unit = ""
     food = ""
     note = ""
+    numeric_amount = 0
 
     def __init__(self, ingredient):
         if ingredient.no_amount:
             self.amount = ""
         else:
             self.amount = f"<scalable-number v-bind:number='{bleach.clean(str(ingredient.amount))}' v-bind:factor='ingredient_factor'></scalable-number>"
+            self.numeric_amount = float(ingredient.amount)
         if ingredient.unit:
             if ingredient.unit.plural_name in (None, ""):
                 self.unit = bleach.clean(str(ingredient.unit))
@@ -53,9 +54,17 @@ class IngredientObject(object):
 def render_instructions(step):  # TODO deduplicate markdown cleanup code
     instructions = step.instruction
 
-    tags = markdown_tags + [
-        'pre', 'table', 'td', 'tr', 'th', 'tbody', 'style', 'thead', 'img'
-    ]
+    tags = {
+        "h1", "h2", "h3", "h4", "h5", "h6",
+        "b", "i", "strong", "em", "tt",
+        "p", "br",
+        "span", "div", "blockquote", "code", "pre", "hr",
+        "ul", "ol", "li", "dd", "dt",
+        "img",
+        "a",
+        "sub", "sup",
+        'pre', 'table', 'td', 'tr', 'th', 'tbody', 'style', 'thead'
+    }
     parsed_md = md.markdown(
         instructions,
         extensions=[
@@ -63,7 +72,11 @@ def render_instructions(step):  # TODO deduplicate markdown cleanup code
             UrlizeExtension(), MarkdownFormatExtension()
         ]
     )
-    markdown_attrs['*'] = markdown_attrs['*'] + ['class', 'width', 'height']
+    markdown_attrs = {
+        "*": ["id", "class", 'width', 'height'],
+        "img": ["src", "alt", "title"],
+        "a": ["href", "alt", "title"],
+    }
 
     instructions = bleach.clean(parsed_md, tags, markdown_attrs)
 
@@ -72,9 +85,12 @@ def render_instructions(step):  # TODO deduplicate markdown cleanup code
     for i in step.ingredients.all():
         ingredients.append(IngredientObject(i))
 
+    def scale(number):
+        return f"<scalable-number v-bind:number='{bleach.clean(str(number))}' v-bind:factor='ingredient_factor'></scalable-number>"
+
     try:
         template = Template(instructions)
-        instructions = template.render(ingredients=ingredients)
+        instructions = template.render(ingredients=ingredients, scale=scale)
     except TemplateSyntaxError:
         return _('Could not parse template code.') + ' Error: Template Syntax broken'
     except UndefinedError:

cookbook/integration/cheftap.py

@@ -36,7 +36,7 @@ class ChefTap(Integration):
 
         recipe = Recipe.objects.create(name=title, created_by=self.request.user, internal=True, space=self.request.space, )
 
-        step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space,)
+        step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,)
 
         if source_url != '':
             step.instruction += '\n' + source_url

cookbook/integration/chowdown.py

@@ -4,6 +4,7 @@ from zipfile import ZipFile
 
 from cookbook.helper.image_processing import get_filetype
 from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
 from cookbook.models import Ingredient, Keyword, Recipe, Step
 
@@ -19,6 +20,10 @@ class Chowdown(Integration):
         direction_mode = False
         description_mode = False
 
+        description = None
+        prep_time = None
+        serving = None
+
         ingredients = []
         directions = []
         descriptions = []
@@ -26,6 +31,12 @@ class Chowdown(Integration):
             line = fl.decode("utf-8")
             if 'title:' in line:
                 title = line.replace('title:', '').replace('"', '').strip()
+            if 'description:' in line:
+                description = line.replace('description:', '').replace('"', '').strip()
+            if 'prep_time:' in line:
+                prep_time = line.replace('prep_time:', '').replace('"', '').strip()
+            if 'yield:' in line:
+                serving = line.replace('yield:', '').replace('"', '').strip()
             if 'image:' in line:
                 image = line.replace('image:', '').strip()
             if 'tags:' in line:
@@ -48,15 +59,43 @@ class Chowdown(Integration):
                 descriptions.append(line)
 
         recipe = Recipe.objects.create(name=title, created_by=self.request.user, internal=True, space=self.request.space)
+        if description:
+            recipe.description = description
 
         for k in tags.split(','):
-            print(f'adding keyword {k.strip()}')
             keyword, created = Keyword.objects.get_or_create(name=k.strip(), space=self.request.space)
             recipe.keywords.add(keyword)
 
-        step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n' + '\n'.join(descriptions), space=self.request.space,
-        )
+        ingredients_added = False
+        for direction in directions:
+            if len(direction.strip()) > 0:
+                step = Step.objects.create(
+                    instruction=direction, name='', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+                )
+            else:
+                step = Step.objects.create(
+                    instruction=direction, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+                )
+            if not ingredients_added:
+                ingredients_added = True
+
+                ingredient_parser = IngredientParser(self.request, True)
+                for ingredient in ingredients:
+                    if len(ingredient.strip()) > 0:
+                        amount, unit, food, note = ingredient_parser.parse(ingredient)
+                        f = ingredient_parser.get_food(food)
+                        u = ingredient_parser.get_unit(unit)
+                        step.ingredients.add(Ingredient.objects.create(
+                            food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
+                        ))
+            recipe.steps.add(step)
+
+        if serving:
+            recipe.servings = parse_servings(serving)
+            recipe.servings_text = 'servings'
+
+        if prep_time:
+            recipe.working_time = parse_time(prep_time)
 
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
@@ -76,6 +115,7 @@ class Chowdown(Integration):
                     if re.match(f'^images/{image}$', z.filename):
                         self.import_recipe_image(recipe, BytesIO(import_zip.read(z.filename)), filetype=get_filetype(z.filename))
 
+        recipe.save()
         return recipe
 
     def get_file_from_recipe(self, recipe):

cookbook/integration/cookbookapp.py

@@ -1,20 +1,15 @@
-import base64
-import gzip
-import json
 import re
-from gettext import gettext as _
 from io import BytesIO
 
 import requests
 import validators
-import yaml
 
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import (get_from_scraper, get_images_from_soup,
                                                iso_duration_to_minutes)
 from cookbook.helper.scrapers.scrapers import text_scraper
 from cookbook.integration.integration import Integration
-from cookbook.models import Ingredient, Keyword, Recipe, Step
+from cookbook.models import Ingredient, Recipe, Step
 
 
 class CookBookApp(Integration):
@@ -25,7 +20,6 @@ class CookBookApp(Integration):
     def get_recipe_from_file(self, file):
         recipe_html = file.getvalue().decode("utf-8")
 
-        # recipe_json, recipe_tree, html_data, images = get_recipe_from_source(recipe_html, 'CookBookApp', self.request)
         scrape = text_scraper(text=recipe_html)
         recipe_json = get_from_scraper(scrape, self.request)
         images = list(dict.fromkeys(get_images_from_soup(scrape.soup, None)))
@@ -37,7 +31,7 @@ class CookBookApp(Integration):
 
         try:
             recipe.servings = re.findall('([0-9])+', recipe_json['recipeYield'])[0]
-        except Exception as e:
+        except Exception:
             pass
 
         try:
@@ -47,7 +41,8 @@ class CookBookApp(Integration):
             pass
 
         # assuming import files only contain single step
-        step = Step.objects.create(instruction=recipe_json['steps'][0]['instruction'], space=self.request.space, )
+        step = Step.objects.create(instruction=recipe_json['steps'][0]['instruction'], space=self.request.space,
+                                   show_ingredients_table=self.request.user.userpreference.show_step_ingredients, )
 
         if 'nutrition' in recipe_json:
             step.instruction = step.instruction + '\n\n' + recipe_json['nutrition']
@@ -62,7 +57,7 @@ class CookBookApp(Integration):
             if unit := ingredient.get('unit', None):
                 u = ingredient_parser.get_unit(unit.get('name', None))
             step.ingredients.add(Ingredient.objects.create(
-                food=f, unit=u, amount=ingredient.get('amount', None), note=ingredient.get('note', None),  original_text=ingredient.get('original_text', None), space=self.request.space,
+                food=f, unit=u, amount=ingredient.get('amount', None), note=ingredient.get('note', None), original_text=ingredient.get('original_text', None), space=self.request.space,
             ))
 
         if len(images) > 0:

cookbook/integration/cookmate.py

@@ -1,17 +1,12 @@
-import base64
-import json
 from io import BytesIO
 
-from gettext import gettext as _
-
 import requests
 import validators
-from lxml import etree
 
 from cookbook.helper.ingredient_parser import IngredientParser
-from cookbook.helper.recipe_url_import import parse_servings, parse_time, parse_servings_text
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
-from cookbook.models import Ingredient, Keyword, Recipe, Step
+from cookbook.models import Ingredient, Recipe, Step
 
 
 class Cookmate(Integration):
@@ -50,7 +45,7 @@ class Cookmate(Integration):
             for step in recipe_text.getchildren():
                 if step.text:
                     step = Step.objects.create(
-                        instruction=step.text.strip(), space=self.request.space,
+                        instruction=step.text.strip(), space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
                     )
                     recipe.steps.add(step)
 

cookbook/integration/copymethat.py

@@ -1,4 +1,3 @@
-import re
 from io import BytesIO
 from zipfile import ZipFile
 
@@ -26,12 +25,13 @@ class CopyMeThat(Integration):
         except AttributeError:
             source = None
 
-        recipe = Recipe.objects.create(name=file.find("div", {"id": "name"}).text.strip()[:128], source_url=source, created_by=self.request.user, internal=True, space=self.request.space, )
+        recipe = Recipe.objects.create(name=file.find("div", {"id": "name"}).text.strip(
+        )[:128], source_url=source, created_by=self.request.user, internal=True, space=self.request.space, )
 
         for category in file.find_all("span", {"class": "recipeCategory"}):
             keyword, created = Keyword.objects.get_or_create(name=category.text, space=self.request.space)
             recipe.keywords.add(keyword)
-        
+
         try:
             recipe.servings = parse_servings(file.find("a", {"id": "recipeYield"}).text.strip())
             recipe.working_time = iso_duration_to_minutes(file.find("span", {"meta": "prepTime"}).text.strip())
@@ -51,7 +51,7 @@ class CopyMeThat(Integration):
         except AttributeError:
             pass
 
-        step = Step.objects.create(instruction='', space=self.request.space, )
+        step = Step.objects.create(instruction='', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients, )
 
         ingredient_parser = IngredientParser(self.request, True)
 
@@ -61,7 +61,14 @@ class CopyMeThat(Integration):
                 if not isinstance(ingredient, Tag) or not ingredient.text.strip() or "recipeIngredient_spacer" in ingredient['class']:
                     continue
                 if any(x in ingredient['class'] for x in ["recipeIngredient_subheader", "recipeIngredient_note"]):
-                    step.ingredients.add(Ingredient.objects.create(is_header=True, note=ingredient.text.strip()[:256], original_text=ingredient.text.strip(), space=self.request.space, ))
+                    step.ingredients.add(
+                        Ingredient.objects.create(
+                            is_header=True,
+                            note=ingredient.text.strip()[
+                                :256],
+                            original_text=ingredient.text.strip(),
+                            space=self.request.space,
+                        ))
                 else:
                     amount, unit, food, note = ingredient_parser.parse(ingredient.text.strip())
                     f = ingredient_parser.get_food(food)
@@ -78,7 +85,7 @@ class CopyMeThat(Integration):
                         step.save()
                         recipe.steps.add(step)
                         step = Step.objects.create(instruction='', space=self.request.space, )
-                    
+
                     step.name = instruction.text.strip()[:128]
                 else:
                     step.instruction += instruction.text.strip() + ' \n\n'

cookbook/integration/default.py

@@ -22,7 +22,7 @@ class Default(Integration):
         if images:
             try:
                 self.import_recipe_image(recipe, BytesIO(recipe_zip.read(images[0])), filetype=get_filetype(images[0]))
-            except AttributeError as e:
+            except AttributeError:
                 traceback.print_exc()
         return recipe
 
@@ -58,7 +58,7 @@ class Default(Integration):
 
                 try:
                     recipe_zip_obj.writestr(f'image{get_filetype(r.image.file.name)}', r.image.file.read())
-                except ValueError:
+                except (ValueError, FileNotFoundError):
                     pass
 
                 recipe_zip_obj.close()
@@ -71,4 +71,4 @@ class Default(Integration):
 
         export_zip_obj.close()
 
-        return [[ self.get_export_file_name(), export_zip_stream.getvalue() ]]
+        return [[self.get_export_file_name(), export_zip_stream.getvalue()]]

cookbook/integration/domestica.py

@@ -28,7 +28,7 @@ class Domestica(Integration):
         recipe.save()
 
         step = Step.objects.create(
-            instruction=file['directions'], space=self.request.space,
+            instruction=file['directions'], space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
         )
 
         if file['source'] != '':

cookbook/integration/integration.py

@@ -1,4 +1,3 @@
-import traceback
 import datetime
 import traceback
 import uuid
@@ -18,8 +17,7 @@ from lxml import etree
 
 from cookbook.helper.image_processing import handle_image
 from cookbook.models import Keyword, Recipe
-from recipes.settings import DEBUG
-from recipes.settings import EXPORT_FILE_CACHE_DURATION
+from recipes.settings import DEBUG, EXPORT_FILE_CACHE_DURATION
 
 
 class Integration:
@@ -39,7 +37,6 @@ class Integration:
         self.ignored_recipes = []
 
         description = f'Imported by {request.user.get_user_display_name()} at {date_format(datetime.datetime.now(), "DATETIME_FORMAT")}. Type: {export_type}'
-        icon = '📥'
 
         try:
             last_kw = Keyword.objects.filter(name__regex=r'^(Import [0-9]+)', space=request.space).latest('created_at')
@@ -52,23 +49,19 @@ class Integration:
             self.keyword = parent.add_child(
                 name=name,
                 description=description,
-                icon=icon,
                 space=request.space
             )
         except (IntegrityError, ValueError):  # in case, for whatever reason, the name does exist append UUID to it. Not nice but works for now.
             self.keyword = parent.add_child(
                 name=f'{name} {str(uuid.uuid4())[0:8]}',
                 description=description,
-                icon=icon,
                 space=request.space
             )
 
-
-
     def do_export(self, recipes, el):
 
         with scope(space=self.request.space):
-            el.total_recipes = len(recipes) 
+            el.total_recipes = len(recipes)
             el.cache_duration = EXPORT_FILE_CACHE_DURATION
             el.save()
 
@@ -80,7 +73,7 @@ class Integration:
                 export_file = file
 
             else:
-                #zip the files if there is more then one file
+                # zip the files if there is more then one file
                 export_filename = self.get_export_file_name()
                 export_stream = BytesIO()
                 export_obj = ZipFile(export_stream, 'w')
@@ -91,8 +84,7 @@ class Integration:
                 export_obj.close()
                 export_file = export_stream.getvalue()
 
-
-            cache.set('export_file_'+str(el.pk), {'filename': export_filename, 'file': export_file}, EXPORT_FILE_CACHE_DURATION)
+            cache.set('export_file_' + str(el.pk), {'filename': export_filename, 'file': export_file}, EXPORT_FILE_CACHE_DURATION)
             el.running = False
             el.save()
 
@@ -100,7 +92,6 @@ class Integration:
         response['Content-Disposition'] = 'attachment; filename="' + export_filename + '"'
         return response
 
-
     def import_file_name_filter(self, zip_info_object):
         """
         Since zipfile.namelist() returns all files in all subdirectories this function allows filtering of files
@@ -164,7 +155,7 @@ class Integration:
 
                         for z in file_list:
                             try:
-                                if not hasattr(z, 'filename') or type(z) == Tag:
+                                if not hasattr(z, 'filename') or isinstance(z, Tag):
                                     recipe = self.get_recipe_from_file(z)
                                 else:
                                     recipe = self.get_recipe_from_file(BytesIO(import_zip.read(z.filename)))
@@ -298,7 +289,6 @@ class Integration:
         if DEBUG:
             traceback.print_exc()
 
-
     def get_export_file_name(self, format='zip'):
         return "export_{}.{}".format(datetime.datetime.now().strftime("%Y-%m-%d"), format)
 

cookbook/integration/mealie.py

@@ -7,7 +7,7 @@ from cookbook.helper.image_processing import get_filetype
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
-from cookbook.models import Ingredient, Recipe, Step
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class Mealie(Integration):
@@ -25,7 +25,7 @@ class Mealie(Integration):
             created_by=self.request.user, internal=True, space=self.request.space)
 
         for s in recipe_json['recipe_instructions']:
-            step = Step.objects.create(instruction=s['text'], space=self.request.space, )
+            step = Step.objects.create(instruction=s['text'], space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients, )
             recipe.steps.add(step)
 
         step = recipe.steps.first()
@@ -56,6 +56,12 @@ class Mealie(Integration):
             except Exception:
                 pass
 
+        if 'tags' in recipe_json and len(recipe_json['tags']) > 0:
+            for k in recipe_json['tags']:
+                if 'name' in k:
+                    keyword, created = Keyword.objects.get_or_create(name=k['name'].strip(), space=self.request.space)
+                    recipe.keywords.add(keyword)
+
         if 'notes' in recipe_json and len(recipe_json['notes']) > 0:
             notes_text = "#### Notes  \n\n"
             for n in recipe_json['notes']:

cookbook/integration/mealmaster.py

@@ -22,7 +22,7 @@ class MealMaster(Integration):
                         if 'Yield:' in line:
                             servings_text = line.replace('Yield:', '').strip()
                         else:
-                            if re.match('\s{2,}([0-9])+', line):
+                            if re.match(r'\s{2,}([0-9])+', line):
                                 ingredients.append(line.strip())
                             else:
                                 directions.append(line.strip())
@@ -39,7 +39,7 @@ class MealMaster(Integration):
             recipe.keywords.add(keyword)
 
         step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n', space=self.request.space,
+            instruction='\n'.join(directions) + '\n\n', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
         )
 
         ingredient_parser = IngredientParser(self.request, True)

cookbook/integration/melarecipes.py

@@ -57,7 +57,7 @@ class MelaRecipes(Integration):
             recipe.source_url = recipe_json['link']
 
         step = Step.objects.create(
-            instruction=instruction, space=self.request.space,
+            instruction=instruction, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients
         )
 
         ingredient_parser = IngredientParser(self.request, True)

cookbook/integration/nextcloud_cookbook.py

@@ -2,13 +2,14 @@ import json
 import re
 from io import BytesIO, StringIO
 from zipfile import ZipFile
+
 from PIL import Image
 
 from cookbook.helper.image_processing import get_filetype
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import iso_duration_to_minutes
 from cookbook.integration.integration import Integration
-from cookbook.models import Ingredient, Keyword, Recipe, Step, NutritionInformation
+from cookbook.models import Ingredient, Keyword, NutritionInformation, Recipe, Step
 
 
 class NextcloudCookbook(Integration):
@@ -51,14 +52,13 @@ class NextcloudCookbook(Integration):
 
         ingredients_added = False
         for s in recipe_json['recipeInstructions']:
-            instruction_text = ''
             if 'text' in s:
                 step = Step.objects.create(
-                    instruction=s['text'], name=s['name'], space=self.request.space,
+                    instruction=s['text'], name=s['name'], space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
                 )
             else:
                 step = Step.objects.create(
-                    instruction=s, space=self.request.space,
+                    instruction=s, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
                 )
             if not ingredients_added:
                 if len(recipe_json['description'].strip()) > 500:
@@ -91,7 +91,7 @@ class NextcloudCookbook(Integration):
                 if nutrition != {}:
                     recipe.nutrition = NutritionInformation.objects.create(**nutrition, space=self.request.space)
                     recipe.save()
-            except Exception as e:
+            except Exception:
                 pass
 
         for f in self.files:

cookbook/integration/openeats.py

@@ -1,9 +1,11 @@
 import json
 
+from django.utils.translation import gettext as _
+
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Ingredient, Recipe, Step, Keyword, Comment, CookLog
-from django.utils.translation import gettext as _
+from cookbook.models import Comment, CookLog, Ingredient, Keyword, Recipe, Step
+
 
 class OpenEats(Integration):
 
@@ -25,16 +27,16 @@ class OpenEats(Integration):
         if file["source"] != '':
             instructions += '\n' + _('Recipe source:') + f'[{file["source"]}]({file["source"]})'
 
-        cuisine_keyword, created = Keyword.objects.get_or_create(name="Cuisine", space=self.request.space)      
+        cuisine_keyword, created = Keyword.objects.get_or_create(name="Cuisine", space=self.request.space)
         if file["cuisine"] != '':
-            keyword, created = Keyword.objects.get_or_create(name=file["cuisine"].strip(), space=self.request.space)      
+            keyword, created = Keyword.objects.get_or_create(name=file["cuisine"].strip(), space=self.request.space)
             if created:
                 keyword.move(cuisine_keyword, pos="last-child")
             recipe.keywords.add(keyword)
 
-        course_keyword, created = Keyword.objects.get_or_create(name="Course", space=self.request.space)      
+        course_keyword, created = Keyword.objects.get_or_create(name="Course", space=self.request.space)
         if file["course"] != '':
-            keyword, created = Keyword.objects.get_or_create(name=file["course"].strip(), space=self.request.space)      
+            keyword, created = Keyword.objects.get_or_create(name=file["course"].strip(), space=self.request.space)
             if created:
                 keyword.move(course_keyword, pos="last-child")
             recipe.keywords.add(keyword)
@@ -51,7 +53,7 @@ class OpenEats(Integration):
             recipe.image = f'recipes/openeats-import/{file["photo"]}'
             recipe.save()
 
-        step = Step.objects.create(instruction=instructions, space=self.request.space,)
+        step = Step.objects.create(instruction=instructions, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,)
 
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in file['ingredients']:

cookbook/integration/paprika.py

@@ -58,7 +58,7 @@ class Paprika(Integration):
                 pass
 
             step = Step.objects.create(
-                instruction=instructions, space=self.request.space,
+                instruction=instructions, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
             )
 
             if 'description' in recipe_json and len(recipe_json['description'].strip()) > 500:
@@ -90,7 +90,7 @@ class Paprika(Integration):
                     if validators.url(url, public=True):
                         response = requests.get(url)
                         self.import_recipe_image(recipe, BytesIO(response.content))
-            except:
+            except Exception:
                 if recipe_json.get("photo_data", None):
                     self.import_recipe_image(recipe, BytesIO(base64.b64decode(recipe_json['photo_data'])), filetype='.jpeg')
 

cookbook/integration/pdfexport.py

@@ -1,21 +1,11 @@
-import json
-from io import BytesIO
-from re import match
-from zipfile import ZipFile
 import asyncio
-from pyppeteer import launch
-
-from rest_framework.renderers import JSONRenderer
-
-from cookbook.helper.image_processing import get_filetype
-from cookbook.integration.integration import Integration
-from cookbook.serializer import RecipeExportSerializer
-
-from cookbook.models import ExportLog
-from asgiref.sync import sync_to_async
 
 import django.core.management.commands.runserver as runserver
-import logging
+from asgiref.sync import sync_to_async
+from pyppeteer import launch
+
+from cookbook.integration.integration import Integration
+
 
 class PDFexport(Integration):
 
@@ -42,7 +32,6 @@ class PDFexport(Integration):
                    }
                    }
 
-
         files = []
         for recipe in recipes:
 
@@ -50,20 +39,18 @@ class PDFexport(Integration):
             await page.emulateMedia('print')
             await page.setCookie(cookies)
 
-            await page.goto('http://'+cmd.default_addr+':'+cmd.default_port+'/view/recipe/'+str(recipe.id), {'waitUntil': 'domcontentloaded'})
-            await page.waitForSelector('#printReady');
+            await page.goto('http://' + cmd.default_addr + ':' + cmd.default_port + '/view/recipe/' + str(recipe.id), {'waitUntil': 'domcontentloaded'})
+            await page.waitForSelector('#printReady')
 
             files.append([recipe.name + '.pdf', await page.pdf(options)])
-            await page.close();
+            await page.close()
 
             el.exported_recipes += 1
             el.msg += self.get_recipe_processed_msg(recipe)
             await sync_to_async(el.save, thread_sensitive=True)()
 
-
         await browser.close()
         return files
 
-
     def get_files_from_recipes(self, recipes, el, cookie):
         return asyncio.run(self.get_files_from_recipes_async(recipes, el, cookie))

cookbook/integration/pepperplate.py

@@ -35,7 +35,7 @@ class Pepperplate(Integration):
         recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space)
 
         step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n', space=self.request.space,
+            instruction='\n'.join(directions) + '\n\n', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
         )
 
         ingredient_parser = IngredientParser(self.request, True)

cookbook/integration/plantoeat.py

@@ -46,7 +46,7 @@ class Plantoeat(Integration):
         recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space)
 
         step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n', space=self.request.space,
+            instruction='\n'.join(directions) + '\n\n', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
         )
 
         if tags:

cookbook/integration/recettetek.py

@@ -46,7 +46,7 @@ class RecetteTek(Integration):
         if not instructions:
             instructions = ''
 
-        step = Step.objects.create(instruction=instructions, space=self.request.space,)
+        step = Step.objects.create(instruction=instructions, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,)
 
         # Append the original import url to the step (if it exists)
         try:

cookbook/integration/recipekeeper.py

@@ -41,7 +41,7 @@ class RecipeKeeper(Integration):
         except AttributeError:
             pass
 
-        step = Step.objects.create(instruction='', space=self.request.space, )
+        step = Step.objects.create(instruction='', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients, )
 
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in file.find("div", {"itemprop": "recipeIngredients"}).findChildren("p"):

cookbook/integration/recipesage.py

@@ -39,7 +39,7 @@ class RecipeSage(Integration):
         ingredients_added = False
         for s in file['recipeInstructions']:
             step = Step.objects.create(
-                instruction=s['text'], space=self.request.space,
+                instruction=s['text'], space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
             )
             if not ingredients_added:
                 ingredients_added = True

cookbook/integration/rezeptsuitede.py

@@ -1,13 +1,11 @@
 import base64
 from io import BytesIO
-from xml import etree
-
 from lxml import etree
 
 from cookbook.helper.ingredient_parser import IngredientParser
-from cookbook.helper.recipe_url_import import parse_time, parse_servings, parse_servings_text
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text
 from cookbook.integration.integration import Integration
-from cookbook.models import Ingredient, Recipe, Step, Keyword
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class Rezeptsuitede(Integration):
@@ -37,7 +35,7 @@ class Rezeptsuitede(Integration):
             try:
                 if prep.find('step').text:
                     step = Step.objects.create(
-                        instruction=prep.find('step').text.strip(), space=self.request.space,
+                        instruction=prep.find('step').text.strip(), space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
                     )
                     recipe.steps.add(step)
             except Exception:
@@ -55,20 +53,23 @@ class Rezeptsuitede(Integration):
                 u = ingredient_parser.get_unit(ingredient.attrib['unit'])
                 amount = 0
                 if ingredient.attrib['qty'].strip() != '':
-                    amount, unit, note = ingredient_parser.parse_amount(ingredient.attrib['qty'])
+                    try:
+                        amount, unit, note = ingredient_parser.parse_amount(ingredient.attrib['qty'])
+                    except ValueError:  # sometimes quantities contain words which cant be parsed
+                        pass
                 ingredient_step.ingredients.add(Ingredient.objects.create(food=f, unit=u, amount=amount, space=self.request.space, ))
 
         try:
             k, created = Keyword.objects.get_or_create(name=recipe_xml.find('head').find('cat').text.strip(), space=self.request.space)
             recipe.keywords.add(k)
-        except Exception as e:
+        except Exception:
             pass
 
         recipe.save()
 
         try:
             self.import_recipe_image(recipe, BytesIO(base64.b64decode(recipe_xml.find('head').find('picbin').text)), filetype='.jpeg')
-        except:
+        except BaseException:
             pass
 
         return recipe

cookbook/integration/rezkonv.py

@@ -38,7 +38,7 @@ class RezKonv(Integration):
             recipe.keywords.add(keyword)
 
         step = Step.objects.create(
-            instruction='  \n'.join(directions) + '\n\n', space=self.request.space,
+            instruction='  \n'.join(directions) + '\n\n', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
         )
 
         ingredient_parser = IngredientParser(self.request, True)
@@ -60,8 +60,8 @@ class RezKonv(Integration):
     def split_recipe_file(self, file):
         recipe_list = []
         current_recipe = ''
-        encoding_list = ['windows-1250',
-                         'latin-1']  # TODO build algorithm to try trough encodings and fail if none work, use for all importers
+        # TODO build algorithm to try trough encodings and fail if none work, use for all importers
+        # encoding_list = ['windows-1250', 'latin-1']
         encoding = 'windows-1250'
         for fl in file.readlines():
             try:

cookbook/integration/saffron.py

@@ -43,7 +43,7 @@ class Saffron(Integration):
 
         recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space, )
 
-        step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space, )
+        step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients, )
 
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
@@ -59,11 +59,11 @@ class Saffron(Integration):
 
     def get_file_from_recipe(self, recipe):
 
-        data = "Title: "+recipe.name if recipe.name else ""+"\n"
-        data += "Description: "+recipe.description if recipe.description else ""+"\n"
+        data = "Title: " + recipe.name if recipe.name else "" + "\n"
+        data += "Description: " + recipe.description if recipe.description else "" + "\n"
         data += "Source: \n"
         data += "Original URL: \n"
-        data += "Yield: "+str(recipe.servings)+"\n"
+        data += "Yield: " + str(recipe.servings) + "\n"
         data += "Cookbook: \n"
         data += "Section: \n"
         data += "Image: \n"
@@ -78,13 +78,13 @@ class Saffron(Integration):
 
         data += "Ingredients: \n"
         for ingredient in recipeIngredient:
-            data += ingredient+"\n"
+            data += ingredient + "\n"
 
         data += "Instructions: \n"
         for instruction in recipeInstructions:
-            data += instruction+"\n"
+            data += instruction + "\n"
 
-        return recipe.name+'.txt', data
+        return recipe.name + '.txt', data
 
     def get_files_from_recipes(self, recipes, el, cookie):
         files = []