fixed space overview

# Conflicts:
#	vue3/src/locales/de.json

.coveragerc

@@ -0,0 +1,11 @@
+[run]
+omit =
+  */apps.py,
+  */migrations/*,
+  */settings*,
+  */test*,
+  */tests/*,
+  *urls.py,
+  */wsgi*,
+  manage.py,
+  *__init__*

.devcontainer/Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.13-alpine3.22
+
+#Install all dependencies.
+RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg libwebp libxml2-dev libxslt-dev openldap git yarn libgcc libstdc++ nginx tini envsubst nodejs npm
+
+#Print all logs without buffering it.
+ENV PYTHONUNBUFFERED 1
+
+#This port will be used by gunicorn.
+EXPOSE 8000
+
+#This port will be used by vue
+EXPOSE 8080
+
+#Install all python dependencies to the image
+COPY requirements.txt /tmp/pip-tmp/
+
+RUN \
+    if [ `apk --print-arch` = "armv7" ]; then \
+    printf "[global]\nextra-index-url=https://www.piwheels.org/simple\n" > /etc/pip.conf ; \
+    fi
+RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-dev jpeg-dev libwebp-dev openssl-dev libffi-dev cargo openldap-dev python3-dev xmlsec-dev xmlsec build-base g++ curl rust && \
+    python -m pip install --upgrade pip && \
+    pip debug -v && \
+    pip install wheel==0.45.1 && \
+    pip install setuptools_rust==1.10.2 && \
+    pip install -r /tmp/pip-tmp/requirements.txt --no-cache-dir &&\
+    apk --purge del .build-deps

.devcontainer/devcontainer.json

@@ -0,0 +1,31 @@
+// For format details, see https://aka.ms/devcontainer.json.
+{
+	"name": "Tandoor Dev Container",
+	"build": { "context": "..", "dockerfile": "Dockerfile" },
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [8000, 8080],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "pip3 install --user -r requirements.txt"
+
+	// Configure tool-specific properties.
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.debugpy",
+				"ms-python.python"
+			]
+		}
+	},
+
+	"containerEnv": {
+		"CSRF_TRUSTED_ORIGINS": "http://localhost:8000,http://localhost:8080"
+	}
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.dockerignore

@@ -3,7 +3,6 @@ npm-debug.log
 Dockerfile*
 docker-compose*
 .dockerignore
-.git
 .gitignore
 README.md
 LICENSE
@@ -30,3 +29,4 @@ vue/babel.config*
 vue/package.json
 vue/tsconfig.json
 vue/src/utils/openapi
+venv

.env.template

@@ -1,162 +1,21 @@
-# only set this to true when testing/debugging
-# when unset: 1 (true) - dont unset this, just for development
-DEBUG=0
-SQL_DEBUG=0
-
-# HTTP port to bind to
-# TANDOOR_PORT=8080
-
-# hosts the application can run under e.g. recipes.mydomain.com,cooking.mydomain.com,...
-ALLOWED_HOSTS=*
+# ---------------------------------------------------------------------------
+# This template contains only required options.
+# Visit the docs to find more https://docs.tandoor.dev/system/configuration/
+# ---------------------------------------------------------------------------
 
 # random secret key, use for example `base64 /dev/urandom | head -c50` to generate one
-# ---------------------------- REQUIRED -------------------------
 SECRET_KEY=
-# ---------------------------------------------------------------
 
 # your default timezone See https://timezonedb.com/time-zones for a list of timezones
-TIMEZONE=Europe/Berlin
+TZ=Europe/Berlin
+
+# allowed hosts (see documentation), should be set to your hostname(s) but might be * (default) for some proxies/providers
+# ALLOWED_HOSTS=recipes.mydomain.com
 
 # add only a database password if you want to run with the default postgres, otherwise change settings accordingly
 DB_ENGINE=django.db.backends.postgresql
-# DB_OPTIONS= {} # e.g. {"sslmode":"require"} to enable ssl
 POSTGRES_HOST=db_recipes
+POSTGRES_DB=djangodb
 POSTGRES_PORT=5432
 POSTGRES_USER=djangouser
-# ---------------------------- REQUIRED -------------------------
 POSTGRES_PASSWORD=
-# ---------------------------------------------------------------
-POSTGRES_DB=djangodb
-
-# database connection string, when used overrides other database settings.
-# format might vary depending on backend
-# DATABASE_URL = engine://username:password@host:port/dbname
-
-# the default value for the user preference 'fractions' (enable/disable fraction support)
-# default: disabled=0
-FRACTION_PREF_DEFAULT=0
-
-# the default value for the user preference 'comments' (enable/disable commenting system)
-# default comments enabled=1
-COMMENT_PREF_DEFAULT=1
-
-# Users can set a amount of time after which the shopping list is refreshed when they are in viewing mode
-# This is the minimum interval users can set. Setting this to low will allow users to refresh very frequently which
-# might cause high load on the server. (Technically they can obviously refresh as often as they want with their own scripts)
-SHOPPING_MIN_AUTOSYNC_INTERVAL=5
-
-# Default for user setting sticky navbar
-# STICKY_NAV_PREF_DEFAULT=1
-
-# If base URL is something other than just / (you are serving a subfolder in your proxy for instance http://recipe_app/recipes/)
-# Be sure to not have a trailing slash: e.g. '/recipes' instead of '/recipes/'
-# SCRIPT_NAME=/recipes
-
-# If staticfiles are stored at a different location uncomment and change accordingly, MUST END IN /
-# this is not required if you are just using a subfolder
-# This can either be a relative path from the applications base path or the url of an external host
-# STATIC_URL=/static/
-
-# If mediafiles are stored at a different location uncomment and change accordingly, MUST END IN /
-# this is not required if you are just using a subfolder
-# This can either be a relative path from the applications base path or the url of an external host
-# MEDIA_URL=/media/
-
-# Serve mediafiles directly using gunicorn. Basically everyone recommends not doing this. Please use any of the examples
-# provided that include an additional nxginx container to handle media file serving.
-# If you know what you are doing turn this back on (1) to serve media files using djangos serve() method.
-# when unset: 1 (true) - this is temporary until an appropriate amount of time has passed for everyone to migrate
-GUNICORN_MEDIA=0
-
-# S3 Media settings: store mediafiles in s3 or any compatible storage backend (e.g. minio)
-# as long as S3_ACCESS_KEY is not set S3 features are disabled
-# S3_ACCESS_KEY=
-# S3_SECRET_ACCESS_KEY=
-# S3_BUCKET_NAME=
-# S3_REGION_NAME= # default none, set your region might be required
-# S3_QUERYSTRING_AUTH=1 # default true, set to 0 to serve media from a public bucket without signed urls
-# S3_QUERYSTRING_EXPIRE=3600 # number of seconds querystring are valid for
-# S3_ENDPOINT_URL= # when using a custom endpoint like minio
-
-# Email Settings, see https://docs.djangoproject.com/en/3.2/ref/settings/#email-host
-# Required for email confirmation and password reset (automatically activates if host is set)
-# EMAIL_HOST=
-# EMAIL_PORT=
-# EMAIL_HOST_USER=
-# EMAIL_HOST_PASSWORD=
-# EMAIL_USE_TLS=0
-# EMAIL_USE_SSL=0
-# email sender address (default 'webmaster@localhost')
-# DEFAULT_FROM_EMAIL=
-# prefix used for account related emails (default "[Tandoor Recipes] ")
-# ACCOUNT_EMAIL_SUBJECT_PREFIX=
-
-# allow authentication via reverse proxy (e.g. authelia), leave off if you dont know what you are doing
-# see docs for more information https://vabene1111.github.io/recipes/features/authentication/
-# when unset: 0 (false)
-REVERSE_PROXY_AUTH=0
-
-# Default settings for spaces, apply per space and can be changed in the admin view
-# SPACE_DEFAULT_MAX_RECIPES=0 # 0=unlimited recipes
-# SPACE_DEFAULT_MAX_USERS=0 # 0=unlimited users per space
-# SPACE_DEFAULT_MAX_FILES=0 # Maximum file storage for space in MB. 0 for unlimited, -1 to disable file upload.
-# SPACE_DEFAULT_ALLOW_SHARING=1 # Allow users to share recipes with public links
-
-# allow people to create accounts on your application instance (without an invite link)
-# when unset: 0 (false)
-# ENABLE_SIGNUP=0
-
-# If signup is enabled you might want to add a captcha to it to prevent spam
-# HCAPTCHA_SITEKEY=
-# HCAPTCHA_SECRET=
-
-# if signup is enabled you might want to provide urls to data protection policies or terms and conditions
-# TERMS_URL=
-# PRIVACY_URL=
-# IMPRINT_URL=
-
-# enable serving of prometheus metrics under the /metrics path
-# ATTENTION: view is not secured (as per the prometheus default way) so make sure to secure it
-# trough your web server (or leave it open of you dont care if the stats are exposed)
-# ENABLE_METRICS=0
-
-# allows you to setup OAuth providers
-# see docs for more information https://vabene1111.github.io/recipes/features/authentication/
-# SOCIAL_PROVIDERS = allauth.socialaccount.providers.github, allauth.socialaccount.providers.nextcloud,
-
-# Should a newly created user from a social provider get assigned to the default space and given permission by default ?
-# ATTENTION: This feature might be deprecated in favor of a space join and public viewing system in the future
-# default 0 (false), when 1 (true) users will be assigned space and group
-# SOCIAL_DEFAULT_ACCESS = 1
-
-# if SOCIAL_DEFAULT_ACCESS is used, which group should be added
-# SOCIAL_DEFAULT_GROUP=guest
-
-# Django session cookie settings. Can be changed to allow a single django application to authenticate several applications
-# when running under the same database
-# SESSION_COOKIE_DOMAIN=.example.com
-# SESSION_COOKIE_NAME=sessionid # use this only to not interfere with non unified django applications under the same top level domain
-
-# by default SORT_TREE_BY_NAME is disabled this will store all Keywords and Food in the order they are created
-# enabling this setting makes saving new keywords and foods very slow, which doesn't matter in most usecases.
-# however, when doing large imports of recipes that will create new objects, can increase total run time by 10-15x
-# Keywords and Food can be manually sorted by name in Admin
-# This value can also be temporarily changed in Admin, it will revert the next time the application is started
-# This will be fixed/changed in the future by changing the implementation or finding a better workaround for sorting
-# SORT_TREE_BY_NAME=0
-# LDAP authentication
-# default 0 (false), when 1 (true) list of allowed users will be fetched from LDAP server
-#LDAP_AUTH=
-#AUTH_LDAP_SERVER_URI=
-#AUTH_LDAP_BIND_DN=
-#AUTH_LDAP_BIND_PASSWORD=
-#AUTH_LDAP_USER_SEARCH_BASE_DN=
-#AUTH_LDAP_TLS_CACERTFILE=
-
-# Enables exporting PDF (see export docs)
-# Disabled by default, uncomment to enable
-# ENABLE_PDF_EXPORT=1
-
-# Recipe exports are cached for a certain time by default, adjust time if needed
-# EXPORT_FILE_CACHE_DURATION=600
-

.flake8

@@ -0,0 +1,22 @@
+[flake8]
+extend-ignore =
+    # Whitespace before ':' - Required for black compatibility
+    E203,
+    # Line break occurred before a binary operator - Required for black compatibility
+    W503,
+    # Comparison to False should be 'if cond is False:' or 'if not cond:'
+    E712
+exclude =
+    .git,
+    **/__pycache__,
+        **/.git,
+        **/.svn,
+        **/.hg,
+        **/CVS,
+        **/.DS_Store,
+        .vscode,
+        **/*.pyc
+per-file-ignores=
+    cookbook/apps.py:F401
+max-line-length = 179
+

.github/dependabot.yml

@@ -1,16 +1,26 @@
 # To get started with Dependabot version updates, you'll need to specify which
 # package ecosystems to update and where the package manifests are located.
 # Please see the documentation for all configuration options:
-# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
 
 version: 2
 updates:
+  - package-ecosystem: "devcontainers"
+    directory: "/"
+    schedule:
+      interval: weekly
+
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
       interval: "monthly"
 
   - package-ecosystem: "npm"
-    directory: "/vue/"
+    directory: "/vue3/"
+    schedule:
+      interval: "monthly"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
     schedule:
       interval: "monthly"

.github/workflows/build-docker.yml

@@ -0,0 +1,126 @@
+name: Build Docker Container
+
+on: push
+
+jobs:
+  build-container:
+    name: Build ${{ matrix.name }} Container
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'TandoorRecipes'
+    continue-on-error: ${{ matrix.continue-on-error }}
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      matrix:
+        include:
+          # Standard build config
+          - name: Standard
+            dockerfile: Dockerfile
+            platforms: linux/amd64,linux/arm64
+            suffix: ""
+            continue-on-error: false
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get version number
+        id: get_version
+        run: |
+          if [[ "$GITHUB_REF" = refs/tags/* ]]; then
+            echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
+          elif [[ "$GITHUB_REF" = refs/heads/beta ]]; then
+            echo VERSION=beta >> $GITHUB_OUTPUT
+          else
+            echo VERSION=develop >> $GITHUB_OUTPUT
+          fi
+
+      # Build Vue 3 frontend
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: yarn
+          cache-dependency-path: vue3/yarn.lock
+      - name: Install dependencies
+        working-directory: ./vue3
+        run: yarn install --frozen-lockfile
+      - name: Build dependencies
+        working-directory: ./vue3
+        run: yarn build
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        if: github.secret_source == 'Actions'
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        if: github.secret_source == 'Actions'
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            vabene1111/recipes
+            ghcr.io/TandoorRecipes/recipes
+          flavor: |
+            latest=false
+            suffix=${{ matrix.suffix }}
+          tags: |
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') }}
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=ref,event=branch
+      - name: Build and Push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.dockerfile }}
+          pull: true
+          push: ${{ github.secret_source == 'Actions' }}
+          platforms: ${{ matrix.platforms }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  notify-stable:
+    name: Notify Stable
+    runs-on: ubuntu-latest
+    needs: build-container
+    if: startsWith(github.ref, 'refs/tags/')
+    steps:
+      - name: Set tag name
+        run: |
+          # Strip "refs/tags/" prefix
+          echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+      # Send stable discord notification
+      - name: Discord notification
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.DISCORD_RELEASE_WEBHOOK }}
+        uses: Ilshidur/action-discord@0.4.0
+        with:
+          args: '🚀 Version {{ VERSION }} of tandoor has been released 🥳 Check it out https://github.com/vabene1111/recipes/releases/tag/{{ VERSION }}'
+
+  notify-beta:
+    name: Notify Beta
+    runs-on: ubuntu-latest
+    needs: build-container
+    if: github.ref == 'refs/heads/beta'
+    steps:
+      # Send beta discord notification
+      - name: Discord notification
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.DISCORD_BETA_WEBHOOK }}
+        uses: Ilshidur/action-discord@0.4.0
+        with:
+          args: '🚀 The Tandoor 2 Image has been updated! 🥳'

.github/workflows/ci.yml

@@ -1,39 +1,77 @@
 name: Continuous Integration
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
-  build:
-    if: github.repository_owner == 'TandoorRecipes'
-    runs-on: ubuntu-latest
-    strategy:
-      max-parallel: 4
-      matrix:
-        python-version: ['3.10']
+    build:
+        if: github.repository_owner == 'TandoorRecipes'
+        runs-on: ubuntu-latest
+        strategy:
+            max-parallel: 4
+            matrix:
+                python-version: ["3.12"]
+                node-version: ["22"]
+        steps:
+            - uses: actions/checkout@v4
+            - uses: awalsh128/cache-apt-pkgs-action@v1.5.1
+              with:
+                  packages: libsasl2-dev python3-dev libxml2-dev libxmlsec1-dev libxslt-dev libxmlsec1-openssl libxslt-dev libldap2-dev libssl-dev gcc musl-dev postgresql-dev zlib-dev jpeg-dev libwebp-dev openssl-dev libffi-dev cargo openldap-dev python3-dev xmlsec-dev xmlsec build-base g++ curl
+                  version: 1.0
 
-    steps:
-    - uses: actions/checkout@v1
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v1
-      with:
-        python-version: '3.10'
-    # Build Vue frontend
-    - uses: actions/setup-node@v2
-      with:
-        node-version: '14'
-    - name: Install Vue dependencies
-      working-directory: ./vue
-      run: yarn install
-    - name: Build Vue dependencies
-      working-directory: ./vue
-      run: yarn build
-    - name: Install Django dependencies
-      run: |
-        sudo apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        python3 manage.py collectstatic --noinput
-        python3 manage.py collectstatic_js_reverse
-    - name: Django Testing project
-      run: |
-        pytest
+            # Setup python & dependencies
+            - name: Set up Python ${{ matrix.python-version }}
+              uses: actions/setup-python@v5
+              with:
+                  python-version: ${{ matrix.python-version }}
+                  cache: "pip"
+
+            - name: Install Python Dependencies
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install -r requirements.txt
+
+            - name: Cache StaticFiles
+              uses: actions/cache@v4
+              id: django_cache
+              with:
+                  path: |
+                      ./cookbook/static
+                      ./staticfiles
+                  key: |
+                      ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue3/src/*') }}
+
+            # Build Vue frontend & Dependencies
+            - name: Set up Node ${{ matrix.node-version }}
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              uses: actions/setup-node@v4
+              with:
+                  node-version: ${{ matrix.node-version }}
+                  cache: "yarn"
+                  cache-dependency-path: ./vue3/yarn.lock
+
+            - name: Install Vue dependencies
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              working-directory: ./vue3
+              run: yarn install
+
+            - name: Build Vue dependencies
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              working-directory: ./vue3
+              run: yarn build
+
+            - name: Compile Django StaticFiles
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              run: |
+                  python3 manage.py collectstatic --noinput
+
+            - uses: actions/cache/save@v4
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              with:
+                  path: |
+                      ./cookbook/static
+                      ./staticfiles
+                  key: |
+                      ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }}
+
+            - name: Django Testing project
+              run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml

.github/workflows/codeql-analysis.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           # We must fetch at least the immediate parents so that if this is
           # a pull request then we can checkout the head.
@@ -25,7 +25,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v1
+        uses: github/codeql-action/init@v3
         # Override language selection by uncommenting this and choosing your languages
         with:
           languages: python, javascript
@@ -47,6 +47,6 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v1
+        uses: github/codeql-action/analyze@v3
         with:
           languages: javascript, python

.github/workflows/docker-publish-beta-raspi.yml

@@ -1,48 +0,0 @@
-name: publish beta raspi image docker
-on:
-  push:
-    branches:
-      - 'beta'
-jobs:
-  build:
-    if: github.repository_owner == 'TandoorRecipes'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-      # Update Version number
-      - name: Update version file
-        uses: DamianReeves/write-file-action@v1.0
-        with:
-          path: recipes/version.py
-          contents: |
-            VERSION_NUMBER = 'beta'
-            BUILD_REF = '${{ github.sha }}'
-          write-mode: overwrite
-      # Build Vue frontend
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-      - name: Install dependencies
-        working-directory: ./vue
-        run: yarn install
-      - name: Build dependencies
-        working-directory: ./vue
-        run: yarn build
-      # Build container
-      - name: Build and publish image
-        uses: ilteoood/docker_buildx@master
-        with:
-          publish: true
-          imageName: vabene1111/recipes
-          tag: beta-raspi
-          dockerFile: Dockerfile-raspi
-          platform: linux/arm/v7
-          dockerUser: ${{ secrets.DOCKER_USERNAME }}
-          dockerPassword: ${{ secrets.DOCKER_PASSWORD }}
-      # Send discord notification
-      - name: Discord notification
-        env:
-          DISCORD_WEBHOOK: ${{ secrets.DISCORD_BETA_WEBHOOK }}
-        uses: Ilshidur/action-discord@0.3.2
-        with:
-          args: '🚀 The BETA Image has been updated! 🥳'

.github/workflows/docker-publish-beta.yml

@@ -1,47 +0,0 @@
-name: publish beta image docker
-on:
-  push:
-    branches:
-      - 'beta'
-jobs:
-  build:
-    if: github.repository_owner == 'TandoorRecipes'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-      # Update Version number
-      - name: Update version file
-        uses: DamianReeves/write-file-action@v1.0
-        with:
-          path: recipes/version.py
-          contents: |
-            VERSION_NUMBER = 'beta'
-            BUILD_REF = '${{ github.sha }}'
-          write-mode: overwrite
-      # Build Vue frontend
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-      - name: Install dependencies
-        working-directory: ./vue
-        run: yarn install
-      - name: Build dependencies
-        working-directory: ./vue
-        run: yarn build
-      # Build container
-      - name: Build and publish image
-        uses: ilteoood/docker_buildx@master
-        with:
-          publish: true
-          imageName: vabene1111/recipes
-          tag: beta
-          platform: linux/amd64,linux/arm64
-          dockerUser: ${{ secrets.DOCKER_USERNAME }}
-          dockerPassword: ${{ secrets.DOCKER_PASSWORD }}
-      # Send discord notification
-      - name: Discord notification
-        env:
-          DISCORD_WEBHOOK: ${{ secrets.DISCORD_BETA_WEBHOOK }}
-        uses: Ilshidur/action-discord@0.3.2
-        with:
-          args: '🚀 The BETA Image has been updated! 🥳'

.github/workflows/docker-publish-dev.yml

@@ -1,42 +0,0 @@
-name: publish dev image docker
-on:
-  push:
-    branches:
-      - '*'
-      - '*/*'
-      - '!master'
-jobs:
-  build:
-    if: github.repository_owner == 'TandoorRecipes'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-      # Update Version number
-      - name: Update version file
-        uses: DamianReeves/write-file-action@v1.0
-        with:
-          path: recipes/version.py
-          contents: |
-            VERSION_NUMBER = 'develop'
-            BUILD_REF = '${{ github.sha }}'
-          write-mode: overwrite
-      # Build Vue frontend
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-      - name: Clear Cache
-        working-directory: ./vue
-        run: yarn cache clean --all
-      - name: Install dependencies
-        working-directory: ./vue
-        run: yarn install
-      - name: Build dependencies
-        working-directory: ./vue
-        run: yarn build
-      # Build container
-      - name: Publish to Registry
-        uses: elgohr/Publish-Docker-Github-Action@2.13
-        with:
-          name: vabene1111/recipes
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/docker-publish-latest-raspi.yml

@@ -1,45 +0,0 @@
-name: publish latest raspi image docker
-on:
-  push:
-    tags:
-    - '*'
-
-jobs:
-  build:
-    if: github.repository_owner == 'TandoorRecipes'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-      - name: Get version number
-        id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}-raspi
-      # Update Version number
-      - name: Update version file
-        uses: DamianReeves/write-file-action@v1.0
-        with:
-          path: recipes/version.py
-          contents: |
-            VERSION_NUMBER = '${{ steps.get_version.outputs.VERSION }}-raspi'
-            BUILD_REF = '${{ github.sha }}'
-          write-mode: overwrite
-      # Build Vue frontend
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-      - name: Install dependencies
-        working-directory: ./vue
-        run: yarn install
-      - name: Build dependencies
-        working-directory: ./vue
-        run: yarn build
-      # Build container
-      - name: Build and publish image
-        uses: ilteoood/docker_buildx@master
-        with:
-          publish: true
-          imageName: vabene1111/recipes
-          dockerFile: Dockerfile-raspi
-          platform: linux/arm/v7
-          tag: latest-raspi
-          dockerUser: ${{ secrets.DOCKER_USERNAME }}
-          dockerPassword: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/docker-publish-latest.yml

@@ -1,44 +0,0 @@
-name: publish latest image docker
-on:
-  push:
-    tags:
-    - '*'
-
-jobs:
-  build:
-    if: github.repository_owner == 'TandoorRecipes'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-      - name: Get version number
-        id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
-      # Update Version number
-      - name: Update version file
-        uses: DamianReeves/write-file-action@v1.0
-        with:
-          path: recipes/version.py
-          contents: |
-            VERSION_NUMBER = '${{ steps.get_version.outputs.VERSION }}'
-            BUILD_REF = '${{ github.sha }}'
-          write-mode: overwrite
-      # Build Vue frontend
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-      - name: Install dependencies
-        working-directory: ./vue
-        run: yarn install
-      - name: Build dependencies
-        working-directory: ./vue
-        run: yarn build
-      # Build container
-      - name: Build and publish image
-        uses: ilteoood/docker_buildx@master
-        with:
-          publish: true
-          imageName: vabene1111/recipes
-          platform: linux/amd64,linux/arm64
-          tag: latest
-          dockerUser: ${{ secrets.DOCKER_USERNAME }}
-          dockerPassword: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/docker-publish-release-raspi.yml

@@ -1,47 +0,0 @@
-name: publish tagged raspi release docker
-
-on:
-  release:
-    types: [published]
-
-jobs:
-  build:
-    if: github.repository_owner == 'TandoorRecipes'
-    runs-on: ubuntu-latest
-    name: Build image job
-    steps:
-      - name: Checkout master
-        uses: actions/checkout@master
-      - name: Get version number
-        id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
-      # Update Version number
-      - name: Update version file
-        uses: DamianReeves/write-file-action@v1.0
-        with:
-          path: recipes/version.py
-          contents: |
-            VERSION_NUMBER = '${{ steps.get_version.outputs.VERSION }}'
-            BUILD_REF = '${{ github.sha }}'
-          write-mode: overwrite
-      # Build Vue frontend
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-      - name: Install dependencies
-        working-directory: ./vue
-        run: yarn install
-      - name: Build dependencies
-        working-directory: ./vue
-        run: yarn build
-      # Build container
-      - name: Build and publish image
-        uses: ilteoood/docker_buildx@master
-        with:
-          publish: true
-          imageName: vabene1111/recipes
-          dockerFile: Dockerfile-raspi
-          platform: linux/arm/v7
-          tag: ${{ steps.get_version.outputs.VERSION }}-raspi
-          dockerUser: ${{ secrets.DOCKER_USERNAME }}
-          dockerPassword: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/docker-publish-release.yml

@@ -1,53 +0,0 @@
-name: publish tagged release docker
-
-on:
-  release:
-    types: [published]
-
-jobs:
-  build:
-    if: github.repository_owner == 'TandoorRecipes'
-    runs-on: ubuntu-latest
-    name: Build image job
-    steps:
-      - name: Checkout master
-        uses: actions/checkout@master
-      - name: Get version number
-        id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
-      # Update Version number
-      - name: Update version file
-        uses: DamianReeves/write-file-action@v1.0
-        with:
-          path: recipes/version.py
-          contents: |
-            VERSION_NUMBER = '${{ steps.get_version.outputs.VERSION }}'
-            BUILD_REF = '${{ github.sha }}'
-          write-mode: overwrite
-      # Build Vue frontend
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-      - name: Install dependencies
-        working-directory: ./vue
-        run: yarn install
-      - name: Build dependencies
-        working-directory: ./vue
-        run: yarn build
-      # Build container
-      - name: Build and publish image
-        uses: ilteoood/docker_buildx@master
-        with:
-          publish: true
-          imageName: vabene1111/recipes
-          platform: linux/amd64,linux/arm64
-          tag: ${{ steps.get_version.outputs.VERSION }}
-          dockerUser: ${{ secrets.DOCKER_USERNAME }}
-          dockerPassword: ${{ secrets.DOCKER_PASSWORD }}
-      # Send discord notification
-      - name: Discord notification
-        env:
-          DISCORD_WEBHOOK: ${{ secrets.DISCORD_RELEASE_WEBHOOK }}
-        uses: Ilshidur/action-discord@0.3.2
-        with:
-          args: '🚀 Version {{ EVENT_PAYLOAD.release.tag_name }} of tandoor has been released 🥳 Check it out https://github.com/vabene1111/recipes/releases/tag/{{ EVENT_PAYLOAD.release.tag_name }}'

.github/workflows/docs.yml

@@ -1,17 +1,20 @@
 name: Make Docs
 on:
-  push:
-    branches:
-      - master
+  # the 1st condition
+  workflow_run:
+    workflows: ["Continuous Integration"]
+    branches: [master]
+    types:
+      - completed
 
 jobs:
   deploy:
-    if: github.repository_owner == 'TandoorRecipes'
+    if: github.repository_owner == 'TandoorRecipes' && ${{ github.event.workflow_run.conclusion == 'success' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.x
       - run: pip install mkdocs-material mkdocs-include-markdown-plugin
-      - run: mkdocs gh-deploy --force
+      - run: mkdocs gh-deploy --force

.gitignore

@@ -43,9 +43,15 @@ htmlcov/
 nosetests.xml
 coverage.xml
 *,cover
+docs/reports/**
 
 # Django stuff:
 *.log
+mediafiles/
+*.sqlite3*
+staticfiles/
+postgresql/
+data/
 
 # Sphinx documentation
 docs/_build/
@@ -54,33 +60,35 @@ docs/_build/
 target/
 
 \.idea/dataSources/
-
 \.idea/dataSources\.xml
 
 \.idea/dataSources\.local\.xml
 
-venv/
-
-mediafiles/
-
-*.sqlite3*
-
 \.idea/workspace\.xml
 
 \.idea/misc\.xml
 
 # Deployment
-
 \.env
-staticfiles/
-postgresql/
-
 
+cookbook/static/vue
+vue/webpack-stats.json
 /docker-compose.override.yml
 vue/node_modules
-.vscode/
+/recipes/plugins
 vetur.config.js
 cookbook/static/vue
 vue/webpack-stats.json
 cookbook/templates/sw.js
-.prettierignore
+vue/.yarn
+vue3/.vite
+
+# Configs
+vetur.config.js
+venv/
+.idea/easy-i18n.xml
+cookbook/static/vue3
+vue3/node_modules
+cookbook/tests/other/docs/reports/tests/tests.html
+cookbook/tests/other/docs/reports/tests/pytest.xml
+vue3/src/plugins

.idea/dictionaries/vaben.xml

@@ -0,0 +1,10 @@
+<component name="ProjectDictionaryState">
+  <dictionary name="vaben">
+    <words>
+      <w>mealplan</w>
+      <w>pinia</w>
+      <w>selfhosted</w>
+      <w>unapplied</w>
+    </words>
+  </dictionary>
+</component>

.idea/dictionaries/vabene1111_PC.xml

@@ -6,6 +6,7 @@
       <w>csrftoken</w>
       <w>gunicorn</w>
       <w>ical</w>
+      <w>invitelink</w>
       <w>mealie</w>
       <w>pepperplate</w>
       <w>safron</w>

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,5 +1,6 @@
 <component name="InspectionProjectProfileManager">
   <settings>
+    <option name="PROJECT_PROFILE" value="Default" />
     <option name="USE_PROJECT_PROFILE" value="false" />
     <version value="1.0" />
   </settings>

.idea/prettier.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PrettierConfiguration">
+    <option name="myConfigurationMode" value="AUTOMATIC" />
+  </component>
+</project>

.idea/recipes.iml

@@ -18,7 +18,7 @@
       <excludeFolder url="file://$MODULE_DIR$/staticfiles" />
       <excludeFolder url="file://$MODULE_DIR$/venv" />
     </content>
-    <orderEntry type="jdk" jdkName="Python 3.9 (recipes)" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.12 (recipes)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="TemplatesService">

.idea/vcs.xml

@@ -1,6 +1,8 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+    <mapping directory="" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/recipes/plugins/enterprise_plugin" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/recipes/plugins/open_data_plugin" vcs="Git" />
   </component>
 </project>

.idea/watcherTasks.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectTasksOptions">
+    <TaskOptions isEnabled="false">
+      <option name="arguments" value="-m flake8 $FilePath$ --config $ContentRoot$\.flake8" />
+      <option name="checkSyntaxErrors" value="true" />
+      <option name="description" />
+      <option name="exitCodeBehavior" value="NEVER" />
+      <option name="fileExtension" value="py" />
+      <option name="immediateSync" value="false" />
+      <option name="name" value="Flake8 Watcher" />
+      <option name="output" value="$FilePath$" />
+      <option name="outputFilters">
+        <array>
+          <FilterInfo>
+            <option name="description" value="" />
+            <option name="name" value="" />
+            <option name="regExp" value="$FILE_PATH$:$LINE$:$COLUMN$: $MESSAGE$" />
+          </FilterInfo>
+        </array>
+      </option>
+      <option name="outputFromStdout" value="false" />
+      <option name="program" value="$PyInterpreterDirectory$/python" />
+      <option name="runOnExternalChanges" value="false" />
+      <option name="scopeName" value="Current File" />
+      <option name="trackOnlyRoot" value="false" />
+      <option name="workingDir" value="" />
+      <envs />
+    </TaskOptions>
+    <TaskOptions isEnabled="false">
+      <option name="arguments" value="-m isort $FilePath$" />
+      <option name="checkSyntaxErrors" value="true" />
+      <option name="description" />
+      <option name="exitCodeBehavior" value="ERROR" />
+      <option name="fileExtension" value="py" />
+      <option name="immediateSync" value="false" />
+      <option name="name" value="isort Watcher" />
+      <option name="output" value="$FilePath$" />
+      <option name="outputFilters">
+        <array />
+      </option>
+      <option name="outputFromStdout" value="false" />
+      <option name="program" value="$PyInterpreterDirectory$/python" />
+      <option name="runOnExternalChanges" value="false" />
+      <option name="scopeName" value="Project Files" />
+      <option name="trackOnlyRoot" value="false" />
+      <option name="workingDir" value="" />
+      <envs />
+    </TaskOptions>
+    <TaskOptions isEnabled="false">
+      <option name="arguments" value="-m yapf -i $FilePath$" />
+      <option name="checkSyntaxErrors" value="true" />
+      <option name="description" />
+      <option name="exitCodeBehavior" value="NEVER" />
+      <option name="fileExtension" value="py" />
+      <option name="immediateSync" value="false" />
+      <option name="name" value="YAPF" />
+      <option name="output" value="$FilePath$" />
+      <option name="outputFilters">
+        <array />
+      </option>
+      <option name="outputFromStdout" value="false" />
+      <option name="program" value="$PyInterpreterDirectory$/python" />
+      <option name="runOnExternalChanges" value="false" />
+      <option name="scopeName" value="Project Files" />
+      <option name="trackOnlyRoot" value="false" />
+      <option name="workingDir" value="" />
+      <envs />
+    </TaskOptions>
+    <TaskOptions isEnabled="false">
+      <option name="arguments" value="--cwd $ProjectFileDir$\vue prettier -w --config $ProjectFileDir$\.prettierrc $FilePath$" />
+      <option name="checkSyntaxErrors" value="true" />
+      <option name="description" />
+      <option name="exitCodeBehavior" value="ERROR" />
+      <option name="fileExtension" value="*" />
+      <option name="immediateSync" value="true" />
+      <option name="name" value="Prettier" />
+      <option name="output" value="" />
+      <option name="outputFilters">
+        <array />
+      </option>
+      <option name="outputFromStdout" value="false" />
+      <option name="program" value="yarn" />
+      <option name="runOnExternalChanges" value="true" />
+      <option name="scopeName" value="Prettier" />
+      <option name="trackOnlyRoot" value="false" />
+      <option name="workingDir" value="" />
+      <envs />
+    </TaskOptions>
+  </component>
+</project>

.prettierignore

@@ -0,0 +1,13 @@
+# generated files
+api.ts
+vue/src/apps/*.js
+vue/node_modules
+staticfiles/
+docs/reports/
+/vue3/src/openapi/
+
+# ignored files - prettier interferes with django templates and github actions
+*.html
+*.yml
+*.yaml
+

.prettierrc

@@ -0,0 +1,7 @@
+{
+    "printWidth": 179,
+    "trailingComma": "es5",
+    "tabWidth": 2,
+    "semi": false,
+    "experimentalTernaries": true
+}

.vscode/launch.json

@@ -0,0 +1,33 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+      {
+        "name": "Python Debugger: Django",
+        "type": "debugpy",
+        "request": "launch",
+        "program": "${workspaceFolder}/manage.py",
+        "args": ["runserver"],
+        "django": true,
+        "justMyCode": true
+      },
+    {
+      "name": "Python: Debug Tests",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "purpose": [
+        "debug-test"
+      ],
+      "console": "integratedTerminal",
+      "env": {
+        // coverage and pytest can't both be running at the same time
+        "PYTEST_ADDOPTS": "--no-cov -n 0"
+      },
+      "django": true,
+      "justMyCode": true
+    },
+    ]
+  }

.vscode/settings.json

@@ -0,0 +1,14 @@
+{
+    "python.testing.pytestArgs": [
+        "cookbook/tests"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true,
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "[python]": {
+        "editor.defaultFormatter": "eeyore.yapf",
+    },
+    "yapf.args": [],
+    "isort.args": []
+}

.vscode/tasks.json

@@ -0,0 +1,83 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "Run Migrations",
+      "type": "shell",
+      "command": "python3 manage.py migrate"
+    },
+    {
+      "label": "Collect Static Files",
+      "type": "shell",
+      "command": "python3 manage.py collectstatic",
+      "dependsOn": ["Yarn Build"]
+    },
+    {
+      "label": "Setup Dev Server",
+      "dependsOn": ["Run Migrations"]
+    },
+    {
+      "label": "Run Dev Server",
+      "type": "shell", 
+      "dependsOn": ["Setup Dev Server"],
+      "command": "DEBUG=1 python3 manage.py runserver"
+    },
+    {
+      "label": "Yarn Install",
+      "type": "shell",
+      "command": "yarn install --force",
+      "options": {
+        "cwd": "${workspaceFolder}/vue3"
+      }
+    },
+    {
+      "label": "Generate API",
+      "type": "shell",
+      "command": "openapi-generator-cli generate -g typescript-fetch -i http://127.0.0.1:8000/openapi/",
+      "options": {
+        "cwd": "${workspaceFolder}/vue3/src/openapi"
+      }
+    },
+    {
+      "label": "Yarn Dev",
+      "type": "shell",
+      "command": "yarn dev",
+      "dependsOn": ["Yarn Install"],
+      "options": {
+        "cwd": "${workspaceFolder}/vue3"
+      }
+    },
+    {
+      "label": "Yarn Build",
+      "type": "shell",
+      "command": "yarn build",
+      "dependsOn": ["Yarn Install"],
+      "options": {
+        "cwd": "${workspaceFolder}/vue3"
+      },
+      "group": "build"
+    },
+    {
+      "label": "Setup Tests",
+      "dependsOn": ["Run Migrations", "Collect Static Files"]
+    },
+    {
+      "label": "Run all pytests",
+      "type": "shell",
+      "command": "python3 -m pytest cookbook/tests",
+      "dependsOn": ["Setup Tests"],
+      "group": "test"
+    },
+    {
+      "label": "Setup Documentation Dependencies",
+      "type": "shell",
+      "command": "pip install mkdocs-material mkdocs-include-markdown-plugin"
+    },
+    {
+      "label": "Serve Documentation",
+      "type": "shell",
+      "command": "mkdocs serve",
+      "dependsOn": ["Setup Documentation Dependencies"]
+    }
+  ]
+}

CONTRIBUTERS.md

@@ -20,6 +20,7 @@ Below are some of the larger contributions made yet.
 - [murphy83] added support for IPv6 #1490
 - [TheHaf] added custom serving size component #1411
 - [lostlont] added LDAP support #960
+- [c0mputerguru] added devcontainers for ease of development
 
 ## Translations
 

Dockerfile

@@ -1,13 +1,14 @@
-FROM python:3.10-alpine3.15
+FROM python:3.13-alpine3.22
 
 #Install all dependencies.
-RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg libwebp libxml2-dev libxslt-dev py-cryptography openldap
+RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg libwebp libxml2-dev libxslt-dev openldap git libgcc libstdc++ nginx tini envsubst nodejs npm
 
 #Print all logs without buffering it.
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONUNBUFFERED=1 \
+    DOCKER=true
 
 #This port will be used by gunicorn.
-EXPOSE 8080
+EXPOSE 80 8080
 
 #Create app dir and install requirements.
 RUN mkdir /opt/recipes
@@ -15,16 +16,38 @@ WORKDIR /opt/recipes
 
 COPY requirements.txt ./
 
-RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-dev jpeg-dev libwebp-dev openssl-dev libffi-dev cargo openldap-dev python3-dev && \
-    echo -n "INPUT ( libldap.so )" > /usr/lib/libldap_r.so && \
+# remove Development dependencies from requirements.txt
+RUN sed -i '/# Development/,$d' requirements.txt
+RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-dev jpeg-dev libwebp-dev openssl-dev libffi-dev cargo openldap-dev python3-dev xmlsec-dev xmlsec build-base g++ curl rust && \
     python -m venv venv && \
     /opt/recipes/venv/bin/python -m pip install --upgrade pip && \
-    venv/bin/pip install wheel==0.37.1 && \
-    venv/bin/pip install setuptools_rust==1.1.2 && \
+    venv/bin/pip debug -v && \
+    venv/bin/pip install wheel==0.45.1 && \
+    venv/bin/pip install setuptools_rust==1.10.2 && \
     venv/bin/pip install -r requirements.txt --no-cache-dir &&\
     apk --purge del .build-deps
 
 #Copy project and execute it.
 COPY . ./
+
+# delete default nginx config and link it to tandoors config
+# create symlinks to access and error log to show them on stdout
+RUN rm -rf /etc/nginx/http.d && \
+    ln -s /opt/recipes/http.d /etc/nginx/http.d && \
+    ln -sf /dev/stdout /var/log/nginx/access.log && \
+    ln -sf /dev/stderr /var/log/nginx/error.log
+
+# commented for now https://github.com/TandoorRecipes/recipes/issues/3478
+#HEALTHCHECK --interval=30s \
+#            --timeout=5s \
+#            --start-period=10s \
+#            --retries=3 \
+#            CMD [ "/usr/bin/wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:8080/openapi" ]
+
+# collect information from git repositories
+RUN /opt/recipes/venv/bin/python version.py
+# delete git repositories to reduce image size
+RUN find . -type d -name ".git" | xargs rm -rf
+
 RUN chmod +x boot.sh
-ENTRYPOINT ["/opt/recipes/boot.sh"]
+ENTRYPOINT ["/sbin/tini", "--", "/opt/recipes/boot.sh"]

Dockerfile-raspi

@@ -1,33 +0,0 @@
-# builds of cryptography for raspberry pi (or better arm v7) fail for some
-FROM python:3.9-alpine3.15
-
-#Install all dependencies.
-RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg libwebp libxml2-dev libxslt-dev py-cryptography openldap gcompat
-
-#Print all logs without buffering it.
-ENV PYTHONUNBUFFERED 1
-
-#This port will be used by gunicorn.
-EXPOSE 8080
-
-#Create app dir and install requirements.
-RUN mkdir /opt/recipes
-WORKDIR /opt/recipes
-
-COPY requirements.txt ./
-RUN \
-    if [ `apk --print-arch` = "armv7" ]; then \
-    printf "[global]\nextra-index-url=https://www.piwheels.org/simple\n" > /etc/pip.conf ; \
-    fi
-RUN apk add --no-cache --virtual .build-deps gcc musl-dev zlib-dev jpeg-dev libwebp-dev python3-dev && \
-    echo -n "INPUT ( libldap.so )" > /usr/lib/libldap_r.so && \
-    python -m venv venv && \
-    /opt/recipes/venv/bin/python -m pip install --upgrade pip && \
-    venv/bin/pip install wheel==0.37.1 && \
-    venv/bin/pip install -r requirements.txt --no-cache-dir --no-binary=Pillow && \
-    apk --purge del .build-deps
-
-#Copy project and execute it.
-COPY . ./
-RUN chmod +x boot.sh
-ENTRYPOINT ["/opt/recipes/boot.sh"]

README.md

@@ -23,6 +23,7 @@
 <a href="https://docs.tandoor.dev/install/docker/" target="_blank" rel="noopener noreferrer">Installation</a> •
 <a href="https://docs.tandoor.dev/" target="_blank" rel="noopener noreferrer">Docs</a> •
 <a href="https://app.tandoor.dev/accounts/login/?demo" target="_blank" rel="noopener noreferrer">Demo</a> •
+<a href="https://community.tandoor.dev" target="_blank" rel="noopener noreferrer">Community</a> •
 <a href="https://discord.gg/RhzBrfWgtp" target="_blank" rel="noopener noreferrer">Discord</a>
 </p>
 
@@ -39,13 +40,13 @@
 
 - 🔍 Powerful & customizable **search** with fulltext support and [TrigramSimilarity](https://docs.djangoproject.com/en/3.0/ref/contrib/postgres/search/#trigram-similarity)
 - 🏷️ Create and search for **tags**, assign them in batch to all files matching certain filters
-- ↔️ Quickly merge and rename ingredients, tags and units 
+- ↔️ Quickly merge and rename ingredients, tags and units
 - 📥️ **Import recipes** from thousands of websites supporting [ld+json or microdata](https://schema.org/Recipe)
 - ➗ Support for **fractions** or decimals
 - 🐳 Easy setup with **Docker** and included examples for **Kubernetes**, **Unraid** and **Synology**
 - 🎨 Customize your interface with **themes**
 - 📦 **Sync** files with Dropbox and Nextcloud
-  
+
 ## All the must haves
 
 - 📱Optimized for use on **mobile** devices
@@ -54,16 +55,24 @@
 - ➕ Many more like recipe scaling, image compression, printing views and supermarkets
 
 This application is meant for people with a collection of recipes they want to share with family and friends or simply
-store them in a nicely organized way. A basic permission system exists but this application is not meant to be run as 
+store them in a nicely organized way. A basic permission system exists but this application is not meant to be run as
 a public page.
 
 ## Docs
 
 Documentation can be found [here](https://docs.tandoor.dev/).
 
-## Contributing
+## Support our work
+Tandoor is developed by volunteers in their free time just because its fun. That said earning
+some money with the project allows us to spend more time on it and thus make improvements we otherwise couldn't.
+Because of that there are several ways you can support us
 
-You can help out with the ongoing development by looking for potential bugs in our code base, or by contributing new features. We are always welcoming new pull requests containing bug fixes, refactors and new features. We have a list of tasks and bugs on our issue tracker on Github. Please comment on issues if you want to contribute with, to avoid duplicating effort.
+- **GitHub Sponsors** You can sponsor contributors of this project on GitHub: [vabene1111](https://github.com/sponsors/vabene1111)
+- **Host at Hetzner** We have been very happy customers of Hetzner for multiple years for all of our projects. If you want to get into self-hosting or are tired of the expensive big providers, their cloud servers are a great place to get started. When you sign up via our [referral link](https://hetzner.cloud/?ref=ISdlrLmr9kGj) you will get 20€ worth of cloud credits and we get a small kickback too.
+- **Let us host for you** We are offering a [hosted version](https://app.tandoor.dev) where all profits support us and the development of tandoor (currently only available in germany).
+
+## Contributing
+Contributions are welcome but please read [this](https://docs.tandoor.dev/contribute/guidelines/) **BEFORE** contributing anything!
 
 ## Your Feedback
 
@@ -73,13 +82,13 @@ Share some information on how you use Tandoor to help me improve the application
 
 <table>
   <tr>
-    <td><a href="https://discord.gg/RhzBrfWgtp">Discord</a></td>
-    <td>We have a public Discord server that anyone can join. This is where all our developers and contributors hang out and where we make announcements</td>
+    <td><a href="https://community.tandoor.dev">Community</a></td>
+    <td>Get support, share best practices, discuss feature ideas, and meet other Tandoor users.</td>
   </tr>
 
   <tr>
-    <td><a href="https://twitter.com/TandoorRecipes">Twitter</a></td>
-    <td>You can follow our Twitter account to get updates on new features or releases</td>
+    <td><a href="https://discord.gg/RhzBrfWgtp">Discord</a></td>
+    <td>We have a public Discord server that anyone can join. This is where all our developers and contributors hang out and where we make announcements</td>
   </tr>
 </table>
 
@@ -88,11 +97,11 @@ Share some information on how you use Tandoor to help me improve the application
 Beginning with version 0.10.0 the code in this repository is licensed under the [GNU AGPL v3](https://www.gnu.org/licenses/agpl-3.0.de.html) license with a
 [common clause](https://commonsclause.com/) selling exception. See [LICENSE.md](https://github.com/vabene1111/recipes/blob/develop/LICENSE.md) for details.
 
-> NOTE: There appears to be a whole range of legal issues with licensing anything else then the standard completely open licenses.
+> NOTE: There appears to be a whole range of legal issues with licensing anything other than the standard completely open licenses.
 > I am in the process of getting some professional legal advice to sort out these issues. 
 > Please also see [Issue 238](https://github.com/vabene1111/recipes/issues/238) for some discussion and **reasoning** regarding the topic.
 
-**Reasoning**  
+**Reasoning**
 **This software and *all* its features are and will always be free for everyone to use and enjoy.**
 
 The reason for the selling exception is that a significant amount of time was spend over multiple years to develop this software.

SECURITY.md

@@ -6,5 +6,4 @@ Since this software is still considered beta/WIP support is always only given fo
 
 ## Reporting a Vulnerability
 
-Please open a normal public issue if you have any security related concerns. If you feel like the issue should not be discussed in 
-public just open a generic issue and we will discuss further communication there (since GitHub does not allow everyone to create a security advisory :/).
+Please use GitHub Security Advisories to report any kind of security vulnerabilities.

boot.sh

@@ -1,8 +1,21 @@
 #!/bin/sh
 source venv/bin/activate
 
-TANDOOR_PORT="${TANDOOR_PORT:-8080}"
-NGINX_CONF_FILE=/opt/recipes/nginx/conf.d/Recipes.conf
+# these are envsubst in the nginx config, make sure they default to something sensible when unset
+export TANDOOR_PORT="${TANDOOR_PORT:-8080}"
+export MEDIA_ROOT=${MEDIA_ROOT:-/opt/recipes/mediafiles};
+export STATIC_ROOT=${STATIC_ROOT:-/opt/recipes/staticfiles};
+
+GUNICORN_WORKERS="${GUNICORN_WORKERS:-3}"
+GUNICORN_THREADS="${GUNICORN_THREADS:-2}"
+GUNICORN_LOG_LEVEL="${GUNICORN_LOG_LEVEL:-'info'}"
+
+PLUGINS_BUILD="${PLUGINS_BUILD:-0}"
+
+if [ "${TANDOOR_PORT}" -eq 80 ]; then
+    echo "TANDOOR_PORT set to 8080 because 80 is now taken by the integrated nginx"
+    TANDOOR_PORT=8080
+fi
 
 display_warning() {
     echo "[WARNING]"
@@ -11,27 +24,47 @@ display_warning() {
 
 echo "Checking configuration..."
 
-# Nginx config file must exist if gunicorn is not active
-if [ ! -f "$NGINX_CONF_FILE" ] && [ $GUNICORN_MEDIA -eq 0 ]; then
-    display_warning "Nginx configuration file could not be found at the default location!\nPath: ${NGINX_CONF_FILE}"
+# SECRET_KEY (or a valid file at SECRET_KEY_FILE) must be set in .env file
+
+if [ -f "${SECRET_KEY_FILE}" ]; then
+    export SECRET_KEY=$(cat "$SECRET_KEY_FILE")
 fi
 
-# SECRET_KEY must be set in .env file
 if [ -z "${SECRET_KEY}" ]; then
-    display_warning "The environment variable 'SECRET_KEY' is not set but REQUIRED for running Tandoor!"
+    display_warning "The environment variable 'SECRET_KEY' (or 'SECRET_KEY_FILE' that points to an existing file) is not set but REQUIRED for running Tandoor!"
 fi
 
+if [ -f "${AUTH_LDAP_BIND_PASSWORD_FILE}" ]; then
+    export AUTH_LDAP_BIND_PASSWORD=$(cat "$AUTH_LDAP_BIND_PASSWORD_FILE")
+fi
+
+if [ -f "${EMAIL_HOST_PASSWORD_FILE}" ]; then
+    export EMAIL_HOST_PASSWORD=$(cat "$EMAIL_HOST_PASSWORD_FILE")
+fi
+
+if [ -f "${SOCIALACCOUNT_PROVIDERS_FILE}" ]; then
+    export SOCIALACCOUNT_PROVIDERS=$(cat "$SOCIALACCOUNT_PROVIDERS_FILE")
+fi
+
+if [ -f "${S3_SECRET_ACCESS_KEY_FILE}" ]; then
+    export S3_SECRET_ACCESS_KEY=$(cat "$S3_SECRET_ACCESS_KEY_FILE")
+fi
 
 echo "Waiting for database to be ready..."
 
 attempt=0
 max_attempts=20
 
-if [ "${DB_ENGINE}" != 'django.db.backends.sqlite3' ]; then
+if [ "${DB_ENGINE}" == 'django.db.backends.postgresql' ] || [ "${DATABASE_URL}" == 'postgres'* ]; then
+
+  # POSTGRES_PASSWORD (or a valid file at POSTGRES_PASSWORD_FILE) must be set in .env file
+
+  if [ -f "${POSTGRES_PASSWORD_FILE}" ]; then
+    export POSTGRES_PASSWORD=$(cat "$POSTGRES_PASSWORD_FILE")
+  fi
 
-  # POSTGRES_PASSWORD must be set in .env file
   if [ -z "${POSTGRES_PASSWORD}" ]; then
-      display_warning "The environment variable 'POSTGRES_PASSWORD' is not set but REQUIRED for running Tandoor!"
+      display_warning "The environment variable 'POSTGRES_PASSWORD' (or 'POSTGRES_PASSWORD_FILE' that points to an existing file) is not set but REQUIRED for running Tandoor!"
   fi
 
   while pg_isready --host=${POSTGRES_HOST} --port=${POSTGRES_PORT} --user=${POSTGRES_USER} -q; status=$?; attempt=$((attempt+1)); [ $status -ne 0 ] && [ $attempt -le $max_attempts ]; do
@@ -51,16 +84,34 @@ echo "Database is ready"
 
 echo "Migrating database"
 
-
 python manage.py migrate
 
-echo "Generating static files"
+if [ "${PLUGINS_BUILD}" -eq 1 ]; then
+    echo "Running yarn build at startup because PLUGINS_BUILD is enabled"
+    python plugin.py
+fi
+
+echo "Collecting static files, this may take a while..."
 
-python manage.py collectstatic_js_reverse
 python manage.py collectstatic --noinput
 
 echo "Done"
 
-chmod -R 755 /opt/recipes/mediafiles
+chmod -R 755 ${MEDIA_ROOT:-/opt/recipes/mediafiles}
 
-exec gunicorn -b :$TANDOOR_PORT --access-logfile - --error-logfile - --log-level INFO recipes.wsgi
+ipv6_disable=$(cat /sys/module/ipv6/parameters/disable)
+
+# prepare nginx config
+envsubst '$MEDIA_ROOT $STATIC_ROOT $TANDOOR_PORT' < /opt/recipes/http.d/Recipes.conf.template > /opt/recipes/http.d/Recipes.conf
+
+# start nginx
+echo "Starting nginx"
+nginx
+
+echo "Starting gunicorn"
+# Check if IPv6 is enabled, only then run gunicorn with ipv6 support
+if [ "$ipv6_disable" -eq 0 ]; then
+    exec gunicorn -b "[::]:$TANDOOR_PORT" --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --access-logfile - --error-logfile - --log-level $GUNICORN_LOG_LEVEL recipes.wsgi
+else
+    exec gunicorn -b ":$TANDOOR_PORT" --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --access-logfile - --error-logfile - --log-level $GUNICORN_LOG_LEVEL recipes.wsgi
+fi

cookbook/admin.py

@@ -7,15 +7,19 @@ from django.utils import translation
 from django_scopes import scopes_disabled
 from treebeard.admin import TreeAdmin
 from treebeard.forms import movenodeform_factory
+from allauth.account.decorators import secure_admin_login
 
 from cookbook.managers import DICTIONARY
 
-from .models import (BookmarkletImport, Comment, CookLog, Food, FoodInheritField, ImportLog,
-                     Ingredient, InviteLink, Keyword, MealPlan, MealType, NutritionInformation,
+from .models import (BookmarkletImport, Comment, CookLog, CustomFilter, Food, ImportLog, Ingredient, InviteLink,
+                     Keyword, MealPlan, MealType, NutritionInformation, Property, PropertyType,
                      Recipe, RecipeBook, RecipeBookEntry, RecipeImport, SearchPreference, ShareLink,
-                     ShoppingList, ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage,
+                     ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage,
                      Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog,
-                     TelegramBot, Unit, UserFile, UserPreference, ViewLog, Automation)
+                     TelegramBot, Unit, UnitConversion, UserFile, UserPreference, UserSpace,
+                     ViewLog, ConnectorConfig)
+
+admin.site.login = secure_admin_login(admin.site.login)
 
 
 class CustomUserAdmin(UserAdmin):
@@ -32,46 +36,14 @@ admin.site.unregister(Group)
 @admin.action(description='Delete all data from a space')
 def delete_space_action(modeladmin, request, queryset):
     for space in queryset:
-        CookLog.objects.filter(space=space).delete()
-        ViewLog.objects.filter(space=space).delete()
-        ImportLog.objects.filter(space=space).delete()
-        BookmarkletImport.objects.filter(space=space).delete()
-
-        Comment.objects.filter(recipe__space=space).delete()
-        Keyword.objects.filter(space=space).delete()
-        Ingredient.objects.filter(space=space).delete()
-        Food.objects.filter(space=space).delete()
-        Unit.objects.filter(space=space).delete()
-        Step.objects.filter(space=space).delete()
-        NutritionInformation.objects.filter(space=space).delete()
-        RecipeBookEntry.objects.filter(book__space=space).delete()
-        RecipeBook.objects.filter(space=space).delete()
-        MealType.objects.filter(space=space).delete()
-        MealPlan.objects.filter(space=space).delete()
-        ShareLink.objects.filter(space=space).delete()
-        Recipe.objects.filter(space=space).delete()
-
-        RecipeImport.objects.filter(space=space).delete()
-        SyncLog.objects.filter(sync__space=space).delete()
-        Sync.objects.filter(space=space).delete()
-        Storage.objects.filter(space=space).delete()
-
-        ShoppingListEntry.objects.filter(shoppinglist__space=space).delete()
-        ShoppingListRecipe.objects.filter(shoppinglist__space=space).delete()
-        ShoppingList.objects.filter(space=space).delete()
-
-        SupermarketCategoryRelation.objects.filter(supermarket__space=space).delete()
-        SupermarketCategory.objects.filter(space=space).delete()
-        Supermarket.objects.filter(space=space).delete()
-
-        InviteLink.objects.filter(space=space).delete()
-        UserFile.objects.filter(space=space).delete()
-        Automation.objects.filter(space=space).delete()
+        space.safe_delete()
 
 
 class SpaceAdmin(admin.ModelAdmin):
     list_display = ('name', 'created_by', 'max_recipes', 'max_users', 'max_file_storage_mb', 'allow_sharing')
     search_fields = ('name', 'created_by__username')
+    autocomplete_fields = ('created_by',)
+    filter_horizontal = ('food_inherit',)
     list_filter = ('max_recipes', 'max_users', 'max_file_storage_mb', 'allow_sharing')
     date_hierarchy = 'created_at'
     actions = [delete_space_action]
@@ -80,15 +52,26 @@ class SpaceAdmin(admin.ModelAdmin):
 admin.site.register(Space, SpaceAdmin)
 
 
+class UserSpaceAdmin(admin.ModelAdmin):
+    list_display = ('user', 'space',)
+    search_fields = ('user__username', 'space__name',)
+    filter_horizontal = ('groups',)
+    autocomplete_fields = ('user', 'space',)
+
+
+admin.site.register(UserSpace, UserSpaceAdmin)
+
+
 class UserPreferenceAdmin(admin.ModelAdmin):
-    list_display = ('name', 'space', 'theme', 'nav_color', 'default_page', 'search_style',)  # TODO add new fields
-    search_fields = ('user__username', 'space__name')
-    list_filter = ('theme', 'nav_color', 'default_page', 'search_style')
+    list_display = ('name', 'theme', 'default_page')
+    search_fields = ('user__username',)
+    list_filter = ('theme', 'default_page',)
     date_hierarchy = 'created_at'
+    filter_horizontal = ('plan_share', 'shopping_share',)
 
     @staticmethod
     def name(obj):
-        return obj.user.get_user_name()
+        return obj.user.get_user_display_name()
 
 
 admin.site.register(UserPreference, UserPreferenceAdmin)
@@ -101,7 +84,7 @@ class SearchPreferenceAdmin(admin.ModelAdmin):
 
     @staticmethod
     def name(obj):
-        return obj.user.get_user_name()
+        return obj.user.get_user_display_name()
 
 
 admin.site.register(SearchPreference, SearchPreferenceAdmin)
@@ -115,6 +98,21 @@ class StorageAdmin(admin.ModelAdmin):
 admin.site.register(Storage, StorageAdmin)
 
 
+class ConnectorConfigAdmin(admin.ModelAdmin):
+    list_display = ('id', 'name', 'type', 'enabled', 'url')
+    search_fields = ('name', 'url')
+
+
+admin.site.register(ConnectorConfig, ConnectorConfigAdmin)
+
+
+class CustomFilterAdmin(admin.ModelAdmin):
+    list_display = ('id', 'type', 'name')
+
+
+admin.site.register(CustomFilter, CustomFilterAdmin)
+
+
 class SyncAdmin(admin.ModelAdmin):
     list_display = ('storage', 'path', 'active', 'last_checked')
     search_fields = ('storage__name', 'path')
@@ -128,11 +126,16 @@ class SupermarketCategoryInline(admin.TabularInline):
 
 
 class SupermarketAdmin(admin.ModelAdmin):
+    list_display = ('name', 'space',)
     inlines = (SupermarketCategoryInline,)
 
 
+class SupermarketCategoryAdmin(admin.ModelAdmin):
+    list_display = ('name', 'space',)
+
+
 admin.site.register(Supermarket, SupermarketAdmin)
-admin.site.register(SupermarketCategory)
+admin.site.register(SupermarketCategory, SupermarketCategoryAdmin)
 
 
 class SyncLogAdmin(admin.ModelAdmin):
@@ -176,9 +179,24 @@ class KeywordAdmin(TreeAdmin):
 admin.site.register(Keyword, KeywordAdmin)
 
 
+@admin.action(description='Delete Steps not part of a Recipe.')
+def delete_unattached_steps(modeladmin, request, queryset):
+    with scopes_disabled():
+        Step.objects.filter(recipe=None).delete()
+
+
 class StepAdmin(admin.ModelAdmin):
-    list_display = ('name', 'order',)
-    search_fields = ('name',)
+    list_display = ('recipe_and_name', 'order', 'space')
+    ordering = ('recipe__name', 'name', 'space',)
+    search_fields = ('name', 'recipe__name')
+    actions = [delete_unattached_steps]
+
+    @staticmethod
+    @admin.display(description="Name")
+    def recipe_and_name(obj):
+        if not obj.recipe_set.exists():
+            return "Orphaned Step" + ('' if not obj.name else f': {obj.name}')
+        return f"{obj.recipe_set.first().name}: {obj.name}" if obj.name else obj.recipe_set.first().name
 
 
 admin.site.register(Step, StepAdmin)
@@ -196,22 +214,30 @@ def rebuild_index(modeladmin, request, queryset):
 
 
 class RecipeAdmin(admin.ModelAdmin):
-    list_display = ('name', 'internal', 'created_by', 'storage')
+    list_display = ('name', 'internal', 'created_by', 'storage', 'space')
     search_fields = ('name', 'created_by__username')
+    ordering = ('name', 'created_by__username',)
     list_filter = ('internal',)
     date_hierarchy = 'created_at'
 
     @staticmethod
     def created_by(obj):
-        return obj.created_by.get_user_name()
+        return obj.created_by.get_user_display_name()
 
-    if settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2', 'django.db.backends.postgresql']:
+    if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql':
         actions = [rebuild_index]
 
 
 admin.site.register(Recipe, RecipeAdmin)
 
-admin.site.register(Unit)
+
+class UnitAdmin(admin.ModelAdmin):
+    list_display = ('name', 'space')
+    ordering = ('name', 'space',)
+    search_fields = ('name',)
+
+
+admin.site.register(Unit, UnitAdmin)
 
 
 # admin.site.register(FoodInheritField)
@@ -227,11 +253,32 @@ class FoodAdmin(TreeAdmin):
 admin.site.register(Food, FoodAdmin)
 
 
-class IngredientAdmin(admin.ModelAdmin):
-    list_display = ('food', 'amount', 'unit')
+class UnitConversionAdmin(admin.ModelAdmin):
+    list_display = ('base_amount', 'base_unit', 'food', 'converted_amount', 'converted_unit')
     search_fields = ('food__name', 'unit__name')
 
 
+admin.site.register(UnitConversion, UnitConversionAdmin)
+
+
+@admin.action(description='Delete Ingredients not part of a Recipe.')
+def delete_unattached_ingredients(modeladmin, request, queryset):
+    with scopes_disabled():
+        Ingredient.objects.filter(step__recipe=None).delete()
+
+
+class IngredientAdmin(admin.ModelAdmin):
+    list_display = ('recipe_name', 'amount', 'unit', 'food', 'space')
+    search_fields = ('food__name', 'unit__name', 'step__recipe__name')
+    actions = [delete_unattached_ingredients]
+
+    @staticmethod
+    @admin.display(description="Recipe")
+    def recipe_name(obj):
+        recipes = obj.step_set.first().recipe_set.all() if obj.step_set.exists() else None
+        return recipes.first().name if recipes else 'Orphaned Ingredient'
+
+
 admin.site.register(Ingredient, IngredientAdmin)
 
 
@@ -242,7 +289,7 @@ class CommentAdmin(admin.ModelAdmin):
 
     @staticmethod
     def name(obj):
-        return obj.created_by.get_user_name()
+        return obj.created_by.get_user_display_name()
 
 
 admin.site.register(Comment, CommentAdmin)
@@ -256,12 +303,12 @@ admin.site.register(RecipeImport, RecipeImportAdmin)
 
 
 class RecipeBookAdmin(admin.ModelAdmin):
-    list_display = ('name', 'user_name')
+    list_display = ('name', 'user_name', 'space')
     search_fields = ('name', 'created_by__username')
 
     @staticmethod
     def user_name(obj):
-        return obj.created_by.get_user_name()
+        return obj.created_by.get_user_display_name()
 
 
 admin.site.register(RecipeBook, RecipeBookAdmin)
@@ -275,19 +322,19 @@ admin.site.register(RecipeBookEntry, RecipeBookEntryAdmin)
 
 
 class MealPlanAdmin(admin.ModelAdmin):
-    list_display = ('user', 'recipe', 'meal_type', 'date')
+    list_display = ('user', 'recipe', 'meal_type', 'from_date', 'to_date')
 
     @staticmethod
     def user(obj):
-        return obj.created_by.get_user_name()
+        return obj.created_by.get_user_display_name()
 
 
 admin.site.register(MealPlan, MealPlanAdmin)
 
 
 class MealTypeAdmin(admin.ModelAdmin):
-    list_display = ('name', 'created_by', 'order')
-    search_fields = ('name', 'created_by__username')
+    list_display = ('name', 'space', 'created_by', 'order')
+    search_fields = ('name', 'space', 'created_by__username')
 
 
 admin.site.register(MealType, MealTypeAdmin)
@@ -312,6 +359,7 @@ admin.site.register(InviteLink, InviteLinkAdmin)
 
 class CookLogAdmin(admin.ModelAdmin):
     list_display = ('recipe', 'created_by', 'created_at', 'rating', 'servings')
+    search_fields = ('recipe__name', 'space__name',)
 
 
 admin.site.register(CookLog, CookLogAdmin)
@@ -331,13 +379,6 @@ class ShoppingListEntryAdmin(admin.ModelAdmin):
 admin.site.register(ShoppingListEntry, ShoppingListEntryAdmin)
 
 
-class ShoppingListAdmin(admin.ModelAdmin):
-    list_display = ('id', 'created_by', 'created_at')
-
-
-admin.site.register(ShoppingList, ShoppingListAdmin)
-
-
 class ShareLinkAdmin(admin.ModelAdmin):
     list_display = ('recipe', 'created_by', 'uuid', 'created_at',)
 
@@ -345,6 +386,29 @@ class ShareLinkAdmin(admin.ModelAdmin):
 admin.site.register(ShareLink, ShareLinkAdmin)
 
 
+@admin.action(description='Delete all properties with type')
+def delete_properties_with_type(modeladmin, request, queryset):
+    for pt in queryset:
+        Property.objects.filter(property_type=pt).delete()
+
+
+class PropertyTypeAdmin(admin.ModelAdmin):
+    search_fields = ('name',)
+
+    list_display = ('id', 'space', 'name', 'fdc_id')
+    actions = [delete_properties_with_type]
+
+
+admin.site.register(PropertyType, PropertyTypeAdmin)
+
+
+class PropertyAdmin(admin.ModelAdmin):
+    list_display = ('property_amount', 'property_type')
+
+
+admin.site.register(Property, PropertyAdmin)
+
+
 class NutritionInformationAdmin(admin.ModelAdmin):
     list_display = ('id',)
 

cookbook/apps.py

@@ -3,6 +3,7 @@ import traceback
 from django.apps import AppConfig
 from django.conf import settings
 from django.db import OperationalError, ProgrammingError
+from django.db.models.signals import post_save, post_delete
 from django_scopes import scopes_disabled
 
 from recipes.settings import DEBUG
@@ -14,6 +15,12 @@ class CookbookConfig(AppConfig):
     def ready(self):
         import cookbook.signals  # noqa
 
+        if not settings.DISABLE_EXTERNAL_CONNECTORS:
+            from cookbook.connectors.connector_manager import ConnectorManager  # Needs to be here to prevent loading race condition of oauth2 modules in models.py
+            handler = ConnectorManager()
+            post_save.connect(handler, dispatch_uid="post_save-connector_manager")
+            post_delete.connect(handler, dispatch_uid="post_delete-connector_manager")
+
         # if not settings.DISABLE_TREE_FIX_STARTUP:
         #     # when starting up run fix_tree to:
         #     #   a) make sure that nodes are sorted when switching between sort modes
@@ -34,4 +41,4 @@ class CookbookConfig(AppConfig):
         #         except Exception:
         #             if DEBUG:
         #                 traceback.print_exc()
-        #             pass  # dont break startup just because fix could not run, need to investigate cases when this happens
+        #             pass  # dont break startup just because fix could not run, need to investigate cases when this happens

cookbook/connectors/connector.py

@@ -0,0 +1,64 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Optional
+
+from cookbook.models import ShoppingListEntry, User, ConnectorConfig
+
+
+@dataclass
+class UserDTO:
+    username: str
+    first_name: Optional[str]
+
+    @staticmethod
+    def create_from_user(instance: User) -> 'UserDTO':
+        return UserDTO(
+            username=instance.username,
+            first_name=instance.first_name if instance.first_name else None
+        )
+
+
+@dataclass
+class ShoppingListEntryDTO:
+    food_name: str
+    amount: Optional[float]
+    base_unit: Optional[str]
+    unit_name: Optional[str]
+    created_by: UserDTO
+
+    @staticmethod
+    def try_create_from_entry(instance: ShoppingListEntry) -> Optional['ShoppingListEntryDTO']:
+        if instance.food is None or instance.created_by is None:
+            return None
+
+        return ShoppingListEntryDTO(
+            food_name=instance.food.name,
+            amount=instance.amount if instance.amount else None,
+            unit_name=instance.unit.name if instance.unit else None,
+            base_unit=instance.unit.base_unit if instance.unit and instance.unit.base_unit else None,
+            created_by=UserDTO.create_from_user(instance.created_by),
+        )
+
+
+# A Connector is 'destroyed' & recreated each time 'any' ConnectorConfig in a space changes.
+class Connector(ABC):
+    @abstractmethod
+    def __init__(self, config: ConnectorConfig):
+        pass
+
+    @abstractmethod
+    async def on_shopping_list_entry_created(self, instance: ShoppingListEntryDTO) -> None:
+        pass
+
+    # This method might not trigger on 'direct' entry updates: https://stackoverflow.com/a/35238823
+    @abstractmethod
+    async def on_shopping_list_entry_updated(self, instance: ShoppingListEntryDTO) -> None:
+        pass
+
+    @abstractmethod
+    async def on_shopping_list_entry_deleted(self, instance: ShoppingListEntryDTO) -> None:
+        pass
+
+    @abstractmethod
+    async def close(self) -> None:
+        pass

cookbook/connectors/connector_manager.py

@@ -0,0 +1,226 @@
+import asyncio
+import logging
+import queue
+import threading
+from asyncio import Task
+from dataclasses import dataclass
+from enum import Enum
+from logging import Logger
+from types import UnionType
+from typing import List, Any, Dict, Optional, Type
+
+from django.conf import settings
+from django_scopes import scope
+
+from cookbook.connectors.connector import Connector, ShoppingListEntryDTO
+from cookbook.connectors.homeassistant import HomeAssistant
+from cookbook.models import ShoppingListEntry, Space, ConnectorConfig
+
+REGISTERED_CLASSES: UnionType | Type = ShoppingListEntry
+
+
+class ActionType(Enum):
+    CREATED = 1
+    UPDATED = 2
+    DELETED = 3
+
+
+@dataclass
+class Work:
+    instance: REGISTERED_CLASSES | ConnectorConfig
+    actionType: ActionType
+
+
+class Singleton(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+
+# The way ConnectionManager works is as follows:
+# 1. On init, it starts a worker & creates a queue for 'Work'
+# 2. Then any time its called, it verifies the type of action (create/update/delete) and if the item is of interest, pushes the Work (non-blocking) to the queue.
+# 3. The worker consumes said work from the queue.
+# 3.1 If the work is of type ConnectorConfig, it flushes its cache of known connectors (per space.id)
+# 3.2 If work is of type REGISTERED_CLASSES, it asynchronously fires of all connectors and wait for them to finish (runtime should depend on the 'slowest' connector)
+# 4. Work is marked as consumed, and next entry of the queue is consumed.
+# Each 'Work' is processed in sequential by the worker, so the throughput is about [workers * the slowest connector]
+# The Singleton class is used for ConnectorManager to have a self-reference and so Python does not garbage collect it
+class ConnectorManager(metaclass=Singleton):
+    _logger: Logger
+    _queue: queue.Queue
+    _listening_to_classes = REGISTERED_CLASSES | ConnectorConfig
+
+    def __init__(self):
+        self._logger = logging.getLogger("recipes.connector")
+        self._logger.debug("ConnectorManager initializing")
+        self._queue = queue.Queue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE)
+        self._worker = threading.Thread(target=self.worker, args=(0, self._queue,), daemon=True)
+        self._worker.start()
+
+    # Called by post save & post delete signals
+    def __call__(self, instance: Any, **kwargs) -> None:
+        action_type: ActionType
+        if "created" in kwargs and kwargs["created"]:
+            action_type = ActionType.CREATED
+        elif "created" in kwargs and not kwargs["created"]:
+            action_type = ActionType.UPDATED
+        elif "origin" in kwargs:
+            action_type = ActionType.DELETED
+        else:
+            return
+
+        self._add_work(action_type, instance)
+
+    def _add_work(self, action_type: ActionType, *instances: REGISTERED_CLASSES):
+        for instance in instances:
+            if not isinstance(instance, self._listening_to_classes) or not hasattr(instance, "space"):
+                continue
+            try:
+                _force_load_instance(instance)
+                self._queue.put_nowait(Work(instance, action_type))
+            except queue.Full:
+                self._logger.info(f"queue was full, so skipping {action_type} of type {type(instance)}")
+
+    def stop(self):
+        self._queue.join()
+        self._worker.join()
+
+    @classmethod
+    def is_initialized(cls):
+        return cls in cls._instances
+
+    @staticmethod
+    def add_work(action_type: ActionType, *instances: REGISTERED_CLASSES):
+        """
+        Manually inject work that failed to come in through the __call__ (aka Django signal)
+        Before the work is processed, we check if the connectionManager is initialized, because if it's not, we don't want to accidentally initialize it.
+        Be careful calling it, because it might result in a instance being processed twice.
+        """
+        if not ConnectorManager.is_initialized():
+            return
+        ConnectorManager()._add_work(action_type, *instances)
+
+    @staticmethod
+    def worker(worker_id: int, worker_queue: queue.Queue):
+        logger = logging.getLogger("recipes.connector.worker")
+
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+        logger.info(f"started ConnectionManager worker {worker_id}")
+
+        # When multiple workers are used, please make sure the cache is shared across all threads, otherwise it might lead to un-expected behavior.
+        _connectors_cache: Dict[int, List[Connector]] = dict()
+
+        while True:
+            try:
+                item: Optional[Work] = worker_queue.get()
+            except KeyboardInterrupt:
+                break
+
+            if item is None:
+                break
+
+            logger.debug(f"received {item.instance=} with {item.actionType=}")
+
+            # If a Connector was changed/updated, refresh connector from the database for said space
+            refresh_connector_cache = isinstance(item.instance, ConnectorConfig)
+
+            space: Space = item.instance.space
+            connectors: Optional[List[Connector]] = _connectors_cache.get(space.id)
+
+            if connectors is None or refresh_connector_cache:
+                if connectors is not None:
+                    loop.run_until_complete(_close_connectors(connectors))
+
+                with scope(space=space):
+                    connectors: List[Connector] = list()
+                    for config in space.connectorconfig_set.all():
+                        config: ConnectorConfig = config
+                        if not config.enabled:
+                            continue
+
+                        try:
+                            connector: Optional[Connector] = ConnectorManager.get_connected_for_config(config)
+                        except BaseException:
+                            logger.exception(f"failed to initialize {config.name}")
+                            continue
+
+                        if connector is not None:
+                            connectors.append(connector)
+
+                    _connectors_cache[space.id] = connectors
+
+            if len(connectors) == 0 or refresh_connector_cache:
+                worker_queue.task_done()
+                continue
+
+            logger.debug(f"running {len(connectors)} connectors for {item.instance=} with {item.actionType=}")
+
+            loop.run_until_complete(run_connectors(connectors, item.instance, item.actionType))
+            worker_queue.task_done()
+
+        logger.info(f"terminating ConnectionManager worker {worker_id}")
+
+        asyncio.set_event_loop(None)
+        loop.close()
+
+    @staticmethod
+    def get_connected_for_config(config: ConnectorConfig) -> Optional[Connector]:
+        match config.type:
+            case ConnectorConfig.HOMEASSISTANT:
+                return HomeAssistant(config)
+            case _:
+                return None
+
+
+def _force_load_instance(instance: REGISTERED_CLASSES):
+    if isinstance(instance, ShoppingListEntry):
+        _ = instance.food  # Force load food
+        _ = instance.unit  # Force load unit
+        _ = instance.created_by  # Force load created_by
+
+
+async def _close_connectors(connectors: List[Connector]):
+    tasks: List[Task] = [asyncio.create_task(connector.close()) for connector in connectors]
+
+    if len(tasks) == 0:
+        return
+
+    try:
+        await asyncio.gather(*tasks, return_exceptions=False)
+    except BaseException:
+        logging.exception("received an exception while closing one of the connectors")
+
+
+async def run_connectors(connectors: List[Connector], instance: REGISTERED_CLASSES, action_type: ActionType):
+    tasks: List[Task] = list()
+
+    if isinstance(instance, ShoppingListEntry):
+        shopping_list_entry = ShoppingListEntryDTO.try_create_from_entry(instance)
+        if shopping_list_entry is None:
+            return
+
+        match action_type:
+            case ActionType.CREATED:
+                for connector in connectors:
+                    tasks.append(asyncio.create_task(connector.on_shopping_list_entry_created(shopping_list_entry)))
+            case ActionType.UPDATED:
+                for connector in connectors:
+                    tasks.append(asyncio.create_task(connector.on_shopping_list_entry_updated(shopping_list_entry)))
+            case ActionType.DELETED:
+                for connector in connectors:
+                    tasks.append(asyncio.create_task(connector.on_shopping_list_entry_deleted(shopping_list_entry)))
+
+    if len(tasks) == 0:
+        return
+
+    try:
+        # Wait for all async tasks to finish, if one fails, the others still continue.
+        await asyncio.gather(*tasks, return_exceptions=False)
+    except BaseException:
+        logging.exception("received an exception from one of the connectors")

cookbook/connectors/homeassistant.py

@@ -0,0 +1,101 @@
+import logging
+from logging import Logger
+from typing import Dict, Tuple
+from urllib.parse import urljoin
+
+from aiohttp import request, ClientResponseError
+
+from cookbook.connectors.connector import Connector, ShoppingListEntryDTO
+from cookbook.models import ConnectorConfig
+
+
+class HomeAssistant(Connector):
+    _config: ConnectorConfig
+    _logger: Logger
+
+    def __init__(self, config: ConnectorConfig):
+        if not config.token or not config.url or not config.todo_entity:
+            raise ValueError("config for HomeAssistantConnector in incomplete")
+
+        self._logger = logging.getLogger(f"recipes.connector.homeassistant.{config.name}")
+
+        if config.url[-1] != "/":
+            config.url += "/"
+        self._config = config
+
+    async def homeassistant_api_call(self, method: str, path: str, data: Dict) -> str:
+        headers = {
+            "Authorization": f"Bearer {self._config.token}",
+            "Content-Type": "application/json"
+        }
+        async with request(method, urljoin(self._config.url, path), headers=headers, json=data) as response:
+            response.raise_for_status()
+            return await response.json()
+
+    async def on_shopping_list_entry_created(self, shopping_list_entry: ShoppingListEntryDTO) -> None:
+        if not self._config.on_shopping_list_entry_created_enabled:
+            return
+
+        item, description = _format_shopping_list_entry(shopping_list_entry)
+
+        self._logger.debug(f"adding {item=} with {description=} to {self._config.todo_entity}")
+
+        data = {
+            "entity_id": self._config.todo_entity,
+            "item": item,
+        }
+
+        if self._config.supports_description_field:
+            data["description"] = description
+
+        try:
+            await self.homeassistant_api_call("POST", "services/todo/add_item", data)
+        except ClientResponseError as err:
+            self._logger.warning(f"received an exception from the api: {err.request_info.url=}, {err.request_info.method=}, {err.status=}, {err.message=}, {type(err)=}")
+
+    async def on_shopping_list_entry_updated(self, shopping_list_entry: ShoppingListEntryDTO) -> None:
+        if not self._config.on_shopping_list_entry_updated_enabled:
+            return
+        pass
+
+    async def on_shopping_list_entry_deleted(self, shopping_list_entry: ShoppingListEntryDTO) -> None:
+        if not self._config.on_shopping_list_entry_deleted_enabled:
+            return
+
+        item, _ = _format_shopping_list_entry(shopping_list_entry)
+
+        self._logger.debug(f"removing {item=} from {self._config.todo_entity}")
+
+        data = {
+            "entity_id": self._config.todo_entity,
+            "item": item,
+        }
+
+        try:
+            await self.homeassistant_api_call("POST", "services/todo/remove_item", data)
+        except ClientResponseError as err:
+            # This error will always trigger if the item is not present/found
+            self._logger.debug(f"received an exception from the api: {err.request_info.url=}, {err.request_info.method=}, {err.status=}, {err.message=}, {type(err)=}")
+
+    async def close(self) -> None:
+        pass
+
+
+def _format_shopping_list_entry(shopping_list_entry: ShoppingListEntryDTO) -> Tuple[str, str]:
+    item = shopping_list_entry.food_name
+    if shopping_list_entry.amount:
+        item += f" ({shopping_list_entry.amount:.2f}".rstrip('0').rstrip('.')
+        if shopping_list_entry.base_unit:
+            item += f" {shopping_list_entry.base_unit})"
+        elif shopping_list_entry.unit_name:
+            item += f" {shopping_list_entry.unit_name})"
+        else:
+            item += ")"
+
+    description = "From TandoorRecipes"
+    if shopping_list_entry.created_by.first_name:
+        description += f", by {shopping_list_entry.created_by.first_name}"
+    else:
+        description += f", by {shopping_list_entry.created_by.username}"
+
+    return item, description

cookbook/filters.py

@@ -1,62 +0,0 @@
-import django_filters
-from django.conf import settings
-from django.contrib.postgres.search import TrigramSimilarity
-from django.db.models import Q
-from django.utils.translation import gettext as _
-from django_scopes import scopes_disabled
-
-from cookbook.forms import MultiSelectWidget
-from cookbook.models import Food, Keyword, Recipe
-
-with scopes_disabled():
-    class RecipeFilter(django_filters.FilterSet):
-        name = django_filters.CharFilter(method='filter_name')
-        keywords = django_filters.ModelMultipleChoiceFilter(
-            queryset=Keyword.objects.none(),
-            widget=MultiSelectWidget,
-            method='filter_keywords'
-        )
-        foods = django_filters.ModelMultipleChoiceFilter(
-            queryset=Food.objects.none(),
-            widget=MultiSelectWidget,
-            method='filter_foods',
-            label=_('Ingredients')
-        )
-
-        def __init__(self, data=None, *args, **kwargs):
-            space = kwargs.pop('space')
-            super().__init__(data, *args, **kwargs)
-            self.filters['foods'].queryset = Food.objects.filter(space=space).all()
-            self.filters['keywords'].queryset = Keyword.objects.filter(space=space).all()
-
-        @staticmethod
-        def filter_keywords(queryset, name, value):
-            if not name == 'keywords':
-                return queryset
-            for x in value:
-                queryset = queryset.filter(keywords=x)
-            return queryset
-
-        @staticmethod
-        def filter_foods(queryset, name, value):
-            if not name == 'foods':
-                return queryset
-            for x in value:
-                queryset = queryset.filter(steps__ingredients__food__name=x).distinct()
-            return queryset
-
-        @staticmethod
-        def filter_name(queryset, name, value):
-            if not name == 'name':
-                return queryset
-            if settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2',
-                                                           'django.db.backends.postgresql']:
-                queryset = queryset.annotate(similarity=TrigramSimilarity('name', value), ).filter(
-                    Q(similarity__gt=0.1) | Q(name__unaccent__icontains=value)).order_by('-similarity')
-            else:
-                queryset = queryset.filter(name__icontains=value)
-            return queryset
-
-        class Meta:
-            model = Recipe
-            fields = ['name', 'keywords', 'foods', 'internal']

cookbook/forms.py

@@ -1,5 +1,7 @@
 from datetime import datetime
 
+from allauth.account.forms import ResetPasswordForm, SignupForm
+from allauth.socialaccount.forms import SignupForm as SocialSignupForm
 from django import forms
 from django.conf import settings
 from django.core.exceptions import ValidationError
@@ -9,9 +11,7 @@ from django_scopes import scopes_disabled
 from django_scopes.forms import SafeModelChoiceField, SafeModelMultipleChoiceField
 from hcaptcha.fields import hCaptchaField
 
-from .models import (Comment, Food, InviteLink, Keyword, MealPlan, MealType, Recipe, RecipeBook,
-                     RecipeBookEntry, SearchPreference, Space, Storage, Sync, User, UserPreference)
-
+from .models import Comment, InviteLink, Keyword, Recipe, SearchPreference, Space, Storage, Sync, User, UserPreference, ConnectorConfig
 
 class SelectWidget(widgets.Select):
     class Media:
@@ -21,126 +21,6 @@ class SelectWidget(widgets.Select):
 class MultiSelectWidget(widgets.SelectMultiple):
     class Media:
         js = ('custom/js/form_multiselect.js',)
-
-
-# Yes there are some stupid browsers that still dont support this but
-# I dont support people using these browsers.
-class DateWidget(forms.DateInput):
-    input_type = 'date'
-
-    def __init__(self, **kwargs):
-        kwargs["format"] = "%Y-%m-%d"
-        super().__init__(**kwargs)
-
-
-class UserPreferenceForm(forms.ModelForm):
-    prefix = 'preference'
-
-    def __init__(self, *args, **kwargs):
-        if x := kwargs.get('instance', None):
-            space = x.space
-        else:
-            space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['plan_share'].queryset = User.objects.filter(userpreference__space=space).all()
-
-    class Meta:
-        model = UserPreference
-        fields = (
-            'default_unit', 'use_fractions', 'use_kj', 'theme', 'nav_color',
-            'sticky_navbar', 'default_page', 'show_recent', 'search_style',
-            'plan_share', 'ingredient_decimals', 'comments', 'left_handed',
-        )
-
-        labels = {
-            'default_unit': _('Default unit'),
-            'use_fractions': _('Use fractions'),
-            'use_kj': _('Use KJ'),
-            'theme': _('Theme'),
-            'nav_color': _('Navbar color'),
-            'sticky_navbar': _('Sticky navbar'),
-            'default_page': _('Default page'),
-            'show_recent': _('Show recent recipes'),
-            'search_style': _('Search style'),
-            'plan_share': _('Plan sharing'),
-            'ingredient_decimals': _('Ingredient decimal places'),
-            'shopping_auto_sync': _('Shopping list auto sync period'),
-            'comments': _('Comments'),
-            'left_handed': _('Left-handed mode')
-        }
-
-        help_texts = {
-            'nav_color': _('Color of the top navigation bar. Not all colors work with all themes, just try them out!'),
-            # noqa: E501
-            'default_unit': _('Default Unit to be used when inserting a new ingredient into a recipe.'),  # noqa: E501
-            'use_fractions': _(
-                'Enables support for fractions in ingredient amounts (e.g. convert decimals to fractions automatically)'),
-            # noqa: E501
-            'use_kj': _('Display nutritional energy amounts in joules instead of calories'),  # noqa: E501
-            'plan_share': _('Users with whom newly created meal plans should be shared by default.'),
-            'shopping_share': _('Users with whom to share shopping lists.'),
-            # noqa: E501
-            'show_recent': _('Show recently viewed recipes on search page.'),  # noqa: E501
-            'ingredient_decimals': _('Number of decimals to round ingredients.'),  # noqa: E501
-            'comments': _('If you want to be able to create and see comments underneath recipes.'),  # noqa: E501
-            'shopping_auto_sync': _(
-                'Setting to 0 will disable auto sync. When viewing a shopping list the list is updated every set seconds to sync changes someone else might have made. Useful when shopping with multiple people but might use a little bit '  # noqa: E501
-                'of mobile data. If lower than instance limit it is reset when saving.'  # noqa: E501
-            ),
-            'sticky_navbar': _('Makes the navbar stick to the top of the page.'),  # noqa: E501
-            'mealplan_autoadd_shopping': _('Automatically add meal plan ingredients to shopping list.'),
-            'mealplan_autoexclude_onhand': _('Exclude ingredients that are on hand.'),
-            'left_handed': _('Will optimize the UI for use with your left hand.')
-        }
-
-        widgets = {
-            'plan_share': MultiSelectWidget,
-            'shopping_share': MultiSelectWidget,
-        }
-
-
-class UserNameForm(forms.ModelForm):
-    prefix = 'name'
-
-    class Meta:
-        model = User
-        fields = ('first_name', 'last_name')
-
-        help_texts = {
-            'first_name': _('Both fields are optional. If none are given the username will be displayed instead')
-        }
-
-
-class ExternalRecipeForm(forms.ModelForm):
-    file_path = forms.CharField(disabled=True, required=False)
-    file_uid = forms.CharField(disabled=True, required=False)
-
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['keywords'].queryset = Keyword.objects.filter(space=space).all()
-
-    class Meta:
-        model = Recipe
-        fields = (
-            'name', 'description', 'servings', 'working_time', 'waiting_time',
-            'file_path', 'file_uid', 'keywords'
-        )
-
-        labels = {
-            'name': _('Name'),
-            'keywords': _('Keywords'),
-            'working_time': _('Preparation time in minutes'),
-            'waiting_time': _('Waiting time (cooking/baking) in minutes'),
-            'file_path': _('Path'),
-            'file_uid': _('Storage UID'),
-        }
-        widgets = {'keywords': MultiSelectWidget}
-        field_classes = {
-            'keywords': SafeModelMultipleChoiceField,
-        }
-
-
 class ImportExportBase(forms.Form):
     DEFAULT = 'DEFAULT'
     PAPRIKA = 'PAPRIKA'
@@ -162,25 +42,39 @@ class ImportExportBase(forms.Form):
     COOKBOOKAPP = 'COOKBOOKAPP'
     COPYMETHAT = 'COPYMETHAT'
     COOKMATE = 'COOKMATE'
+    REZEPTSUITEDE = 'REZEPTSUITEDE'
     PDF = 'PDF'
+    GOURMET = 'GOURMET'
 
-    type = forms.ChoiceField(choices=(
-        (DEFAULT, _('Default')), (PAPRIKA, 'Paprika'), (NEXTCLOUD, 'Nextcloud Cookbook'),
-        (MEALIE, 'Mealie'), (CHOWDOWN, 'Chowdown'), (SAFFRON, 'Saffron'), (CHEFTAP, 'ChefTap'),
-        (PEPPERPLATE, 'Pepperplate'), (RECETTETEK, 'RecetteTek'), (RECIPESAGE, 'Recipe Sage'), (DOMESTICA, 'Domestica'),
-        (MEALMASTER, 'MealMaster'), (REZKONV, 'RezKonv'), (OPENEATS, 'Openeats'), (RECIPEKEEPER, 'Recipe Keeper'),
-        (PLANTOEAT, 'Plantoeat'), (COOKBOOKAPP, 'CookBookApp'), (COPYMETHAT, 'CopyMeThat'), (PDF, 'PDF'), (MELARECIPES, 'Melarecipes'),
-        (COOKMATE, 'Cookmate')
-    ))
+    type = forms.ChoiceField(choices=((DEFAULT, _('Default')), (PAPRIKA, 'Paprika'), (NEXTCLOUD, 'Nextcloud Cookbook'), (MEALIE, 'Mealie'), (CHOWDOWN, 'Chowdown'),
+                                      (SAFFRON, 'Saffron'), (CHEFTAP, 'ChefTap'), (PEPPERPLATE, 'Pepperplate'), (RECETTETEK, 'RecetteTek'), (RECIPESAGE, 'Recipe Sage'),
+                                      (DOMESTICA, 'Domestica'), (MEALMASTER, 'MealMaster'), (REZKONV, 'RezKonv'), (OPENEATS, 'Openeats'), (RECIPEKEEPER, 'Recipe Keeper'),
+                                      (PLANTOEAT, 'Plantoeat'), (COOKBOOKAPP, 'CookBookApp'), (COPYMETHAT, 'CopyMeThat'), (PDF, 'PDF'), (MELARECIPES, 'Melarecipes'),
+                                      (COOKMATE, 'Cookmate'), (REZEPTSUITEDE, 'Recipesuite.de'), (GOURMET, 'Gourmet')))
+
+class MultipleFileInput(forms.ClearableFileInput):
+    allow_multiple_selected = True
+
+
+class MultipleFileField(forms.FileField):
+
+    def __init__(self, *args, **kwargs):
+        kwargs.setdefault("widget", MultipleFileInput())
+        super().__init__(*args, **kwargs)
+
+    def clean(self, data, initial=None):
+        single_file_clean = super().clean
+        if isinstance(data, (list, tuple)):
+            result = [single_file_clean(d, initial) for d in data]
+        else:
+            result = single_file_clean(data, initial)
+        return result
 
 
 class ImportForm(ImportExportBase):
-    files = forms.FileField(required=True, widget=forms.ClearableFileInput(attrs={'multiple': True}))
-    duplicates = forms.BooleanField(help_text=_(
-        'To prevent duplicates recipes with the same name as existing ones are ignored. Check this box to import everything.'),
-        required=False)
-
-
+    files = MultipleFileField(required=True)
+    duplicates = forms.BooleanField(help_text=_('To prevent duplicates recipes with the same name as existing ones are ignored. Check this box to import everything.'),
+                                    required=False)
 class ExportForm(ImportExportBase):
     recipes = forms.ModelMultipleChoiceField(widget=MultiSelectWidget, queryset=Recipe.objects.none(), required=False)
     all = forms.BooleanField(required=False)
@@ -191,172 +85,11 @@ class ExportForm(ImportExportBase):
         super().__init__(*args, **kwargs)
         self.fields['recipes'].queryset = Recipe.objects.filter(space=space).all()
 
-
-class CommentForm(forms.ModelForm):
-    prefix = 'comment'
-
-    class Meta:
-        model = Comment
-        fields = ('text',)
-
-        labels = {
-            'text': _('Add your comment: '),
-        }
-        widgets = {
-            'text': forms.Textarea(attrs={'rows': 2, 'cols': 15}),
-        }
-
-
-class StorageForm(forms.ModelForm):
-    username = forms.CharField(
-        widget=forms.TextInput(attrs={'autocomplete': 'new-password'}),
-        required=False
-    )
-    password = forms.CharField(
-        widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}),
-        required=False,
-        help_text=_('Leave empty for dropbox and enter app password for nextcloud.')
-    )
-    token = forms.CharField(
-        widget=forms.TextInput(
-            attrs={'autocomplete': 'new-password', 'type': 'password'}
-        ),
-        required=False,
-        help_text=_('Leave empty for nextcloud and enter api token for dropbox.')
-    )
-
-    class Meta:
-        model = Storage
-        fields = ('name', 'method', 'username', 'password', 'token', 'url', 'path')
-
-        help_texts = {
-            'url': _(
-                'Leave empty for dropbox and enter only base url for nextcloud (<code>/remote.php/webdav/</code> is added automatically)'),
-        }
-
-
-# TODO: Deprecate
-class RecipeBookEntryForm(forms.ModelForm):
-    prefix = 'bookmark'
-
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['book'].queryset = RecipeBook.objects.filter(space=space).all()
-
-    class Meta:
-        model = RecipeBookEntry
-        fields = ('book',)
-
-        field_classes = {
-            'book': SafeModelChoiceField,
-        }
-
-
-class SyncForm(forms.ModelForm):
-
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['storage'].queryset = Storage.objects.filter(space=space).all()
-
-    class Meta:
-        model = Sync
-        fields = ('storage', 'path', 'active')
-
-        field_classes = {
-            'storage': SafeModelChoiceField,
-        }
-
-        labels = {
-            'storage': _('Storage'),
-            'path': _('Path'),
-            'active': _('Active')
-        }
-
-
-# TODO deprecate
-class BatchEditForm(forms.Form):
-    search = forms.CharField(label=_('Search String'))
-    keywords = forms.ModelMultipleChoiceField(
-        queryset=Keyword.objects.none(),
-        required=False,
-        widget=MultiSelectWidget
-    )
-
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['keywords'].queryset = Keyword.objects.filter(space=space).all().order_by('id')
-
-
-class ImportRecipeForm(forms.ModelForm):
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['keywords'].queryset = Keyword.objects.filter(space=space).all()
-
-    class Meta:
-        model = Recipe
-        fields = ('name', 'keywords', 'file_path', 'file_uid')
-
-        labels = {
-            'name': _('Name'),
-            'keywords': _('Keywords'),
-            'file_path': _('Path'),
-            'file_uid': _('File ID'),
-        }
-        widgets = {'keywords': MultiSelectWidget}
-        field_classes = {
-            'keywords': SafeModelChoiceField,
-        }
-
-
-# TODO deprecate
-class MealPlanForm(forms.ModelForm):
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['recipe'].queryset = Recipe.objects.filter(space=space).all()
-        self.fields['meal_type'].queryset = MealType.objects.filter(space=space).all()
-        self.fields['shared'].queryset = User.objects.filter(userpreference__space=space).all()
-
-    def clean(self):
-        cleaned_data = super(MealPlanForm, self).clean()
-
-        if cleaned_data['title'] == '' and cleaned_data['recipe'] is None:
-            raise forms.ValidationError(
-                _('You must provide at least a recipe or a title.')
-            )
-
-        return cleaned_data
-
-    class Meta:
-        model = MealPlan
-        fields = (
-            'recipe', 'title', 'meal_type', 'note',
-            'servings', 'date', 'shared'
-        )
-
-        help_texts = {
-            'shared': _('You can list default users to share recipes with in the settings.'),  # noqa: E501
-            'note': _('You can use markdown to format this field. See the <a href="/docs/markdown/">docs here</a>')
-            # noqa: E501
-        }
-
-        widgets = {
-            'recipe': SelectWidget,
-            'date': DateWidget,
-            'shared': MultiSelectWidget
-        }
-        field_classes = {
-            'recipe': SafeModelChoiceField,
-            'meal_type': SafeModelChoiceField,
-            'shared': SafeModelMultipleChoiceField,
-        }
+from .models import InviteLink, SearchPreference, Space, User, UserPreference
 
 
 class InviteLinkForm(forms.ModelForm):
+
     def __init__(self, *args, **kwargs):
         user = kwargs.pop('user')
         super().__init__(*args, **kwargs)
@@ -364,8 +97,8 @@ class InviteLinkForm(forms.ModelForm):
 
     def clean(self):
         space = self.cleaned_data['space']
-        if space.max_users != 0 and (UserPreference.objects.filter(space=space).count() +
-                                     InviteLink.objects.filter(valid_until__gte=datetime.today(), used_by=None, space=space).count()) >= space.max_users:
+        if space.max_users != 0 and (UserPreference.objects.filter(space=space).count()
+                                     + InviteLink.objects.filter(valid_until__gte=datetime.today(), used_by=None, space=space).count()) >= space.max_users:
             raise ValidationError(_('Maximum number of users for this space reached.'))
 
     def clean_email(self):
@@ -379,12 +112,8 @@ class InviteLinkForm(forms.ModelForm):
     class Meta:
         model = InviteLink
         fields = ('email', 'group', 'valid_until', 'space')
-        help_texts = {
-            'email': _('An email address is not required but if present the invite link will be sent to the user.'),
-        }
-        field_classes = {
-            'space': SafeModelChoiceField,
-        }
+        help_texts = {'email': _('An email address is not required but if present the invite link will be sent to the user.'), }
+        field_classes = {'space': SafeModelChoiceField, }
 
 
 class SpaceCreateForm(forms.Form):
@@ -404,12 +133,12 @@ class SpaceJoinForm(forms.Form):
     token = forms.CharField()
 
 
-class AllAuthSignupForm(forms.Form):
+class AllAuthSignupForm(SignupForm):
     captcha = hCaptchaField()
     terms = forms.BooleanField(label=_('Accept Terms and Privacy'))
 
     def __init__(self, **kwargs):
-        super(AllAuthSignupForm, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         if settings.PRIVACY_URL == '' and settings.TERMS_URL == '':
             self.fields.pop('terms')
         if settings.HCAPTCHA_SECRET == '':
@@ -419,134 +148,28 @@ class AllAuthSignupForm(forms.Form):
         pass
 
 
+class AllAuthSocialSignupForm(SocialSignupForm):
+    terms = forms.BooleanField(label=_('Accept Terms and Privacy'))
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        if settings.PRIVACY_URL == '' and settings.TERMS_URL == '':
+            self.fields.pop('terms')
+
+    def signup(self, request, user):
+        pass
+
+
+class CustomPasswordResetForm(ResetPasswordForm):
+    captcha = hCaptchaField()
+
+    def __init__(self, **kwargs):
+        super(CustomPasswordResetForm, self).__init__(**kwargs)
+        if settings.HCAPTCHA_SECRET == '':
+            self.fields.pop('captcha')
+
+
 class UserCreateForm(forms.Form):
     name = forms.CharField(label='Username')
-    password = forms.CharField(
-        widget=forms.TextInput(
-            attrs={'autocomplete': 'new-password', 'type': 'password'}
-        )
-    )
-    password_confirm = forms.CharField(
-        widget=forms.TextInput(
-            attrs={'autocomplete': 'new-password', 'type': 'password'}
-        )
-    )
-
-
-class SearchPreferenceForm(forms.ModelForm):
-    prefix = 'search'
-    trigram_threshold = forms.DecimalField(min_value=0.01, max_value=1, decimal_places=2,
-                                           widget=NumberInput(attrs={'class': "form-control-range", 'type': 'range'}),
-                                           help_text=_(
-                                               'Determines how fuzzy a search is if it uses trigram similarity matching (e.g. low values mean more typos are ignored).'))
-    preset = forms.CharField(widget=forms.HiddenInput(), required=False)
-
-    class Meta:
-        model = SearchPreference
-        fields = (
-            'search', 'lookup', 'unaccent', 'icontains', 'istartswith', 'trigram', 'fulltext', 'trigram_threshold')
-
-        help_texts = {
-            'search': _(
-                'Select type method of search.  Click <a href="/docs/search/">here</a> for full description of choices.'),
-            'lookup': _('Use fuzzy matching on units, keywords and ingredients when editing and importing recipes.'),
-            'unaccent': _(
-                'Fields to search ignoring accents.  Selecting this option can improve or degrade search quality depending on language'),
-            'icontains': _(
-                "Fields to search for partial matches.  (e.g. searching for 'Pie' will return 'pie' and 'piece' and 'soapie')"),
-            'istartswith': _(
-                "Fields to search for beginning of word matches. (e.g. searching for 'sa' will return 'salad' and 'sandwich')"),
-            'trigram': _(
-                "Fields to 'fuzzy' search. (e.g. searching for 'recpie' will find 'recipe'.)  Note: this option will conflict with 'web' and 'raw' methods of search."),
-            'fulltext': _(
-                "Fields to full text search.  Note: 'web', 'phrase', and 'raw' search methods only function with fulltext fields."),
-        }
-
-        labels = {
-            'search': _('Search Method'),
-            'lookup': _('Fuzzy Lookups'),
-            'unaccent': _('Ignore Accent'),
-            'icontains': _("Partial Match"),
-            'istartswith': _("Starts With"),
-            'trigram': _("Fuzzy Search"),
-            'fulltext': _("Full Text")
-        }
-
-        widgets = {
-            'search': SelectWidget,
-            'unaccent': MultiSelectWidget,
-            'icontains': MultiSelectWidget,
-            'istartswith': MultiSelectWidget,
-            'trigram': MultiSelectWidget,
-            'fulltext': MultiSelectWidget,
-        }
-
-
-class ShoppingPreferenceForm(forms.ModelForm):
-    prefix = 'shopping'
-
-    class Meta:
-        model = UserPreference
-
-        fields = (
-            'shopping_share', 'shopping_auto_sync', 'mealplan_autoadd_shopping', 'mealplan_autoexclude_onhand',
-            'mealplan_autoinclude_related', 'shopping_add_onhand', 'default_delay', 'filter_to_supermarket', 'shopping_recent_days', 'csv_delim', 'csv_prefix'
-        )
-
-        help_texts = {
-            'shopping_share': _('Users will see all items you add to your shopping list.  They must add you to see items on their list.'),
-            'shopping_auto_sync': _(
-                'Setting to 0 will disable auto sync. When viewing a shopping list the list is updated every set seconds to sync changes someone else might have made. Useful when shopping with multiple people but might use a little bit '  # noqa: E501
-                'of mobile data. If lower than instance limit it is reset when saving.'  # noqa: E501
-            ),
-            'mealplan_autoadd_shopping': _('Automatically add meal plan ingredients to shopping list.'),
-            'mealplan_autoinclude_related': _('When adding a meal plan to the shopping list (manually or automatically), include all related recipes.'),
-            'mealplan_autoexclude_onhand': _('When adding a meal plan to the shopping list (manually or automatically), exclude ingredients that are on hand.'),
-            'default_delay': _('Default number of hours to delay a shopping list entry.'),
-            'filter_to_supermarket': _('Filter shopping list to only include supermarket categories.'),
-            'shopping_recent_days': _('Days of recent shopping list entries to display.'),
-            'shopping_add_onhand': _("Mark food 'On Hand' when checked off shopping list."),
-            'csv_delim': _('Delimiter to use for CSV exports.'),
-            'csv_prefix': _('Prefix to add when copying list to the clipboard.'),
-
-        }
-        labels = {
-            'shopping_share': _('Share Shopping List'),
-            'shopping_auto_sync': _('Autosync'),
-            'mealplan_autoadd_shopping': _('Auto Add Meal Plan'),
-            'mealplan_autoexclude_onhand': _('Exclude On Hand'),
-            'mealplan_autoinclude_related': _('Include Related'),
-            'default_delay': _('Default Delay Hours'),
-            'filter_to_supermarket': _('Filter to Supermarket'),
-            'shopping_recent_days': _('Recent Days'),
-            'csv_delim': _('CSV Delimiter'),
-            "csv_prefix_label": _("List Prefix"),
-            'shopping_add_onhand': _("Auto On Hand"),
-        }
-
-        widgets = {
-            'shopping_share': MultiSelectWidget
-        }
-
-
-class SpacePreferenceForm(forms.ModelForm):
-    prefix = 'space'
-    reset_food_inherit = forms.BooleanField(label=_("Reset Food Inheritance"), initial=False, required=False,
-                                            help_text=_("Reset all food to inherit the fields configured."))
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)  # populates the post
-        self.fields['food_inherit'].queryset = Food.inheritable_fields
-
-    class Meta:
-        model = Space
-
-        fields = ('food_inherit', 'reset_food_inherit', 'show_facet_count')
-
-        help_texts = {
-            'food_inherit': _('Fields on food that should be inherited by default.'),
-            'show_facet_count': _('Show recipe counts on search filters'), }
-
-        widgets = {
-            'food_inherit': MultiSelectWidget
-        }
+    password = forms.CharField(widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}))
+    password_confirm = forms.CharField(widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}))

cookbook/helper/AllAuthCustomAdapter.py

@@ -1,11 +1,10 @@
 import datetime
-
-from django.conf import settings
+from gettext import gettext as _
 
 from allauth.account.adapter import DefaultAccountAdapter
+from django.conf import settings
 from django.contrib import messages
 from django.core.cache import caches
-from gettext import gettext as _
 
 from cookbook.models import InviteLink
 
@@ -14,13 +13,16 @@ class AllAuthCustomAdapter(DefaultAccountAdapter):
 
     def is_open_for_signup(self, request):
         """
-        Whether to allow sign ups.
+        Whether to allow sign-ups.
         """
         signup_token = False
-        if 'signup_token' in request.session and InviteLink.objects.filter(valid_until__gte=datetime.datetime.today(), used_by=None, uuid=request.session['signup_token']).exists():
+        if 'signup_token' in request.session and InviteLink.objects.filter(
+                valid_until__gte=datetime.datetime.today(), used_by=None, uuid=request.session['signup_token']).exists():
             signup_token = True
 
-        if (request.resolver_match.view_name == 'account_signup' or request.resolver_match.view_name == 'socialaccount_signup') and not settings.ENABLE_SIGNUP and not signup_token:
+        if request.resolver_match.view_name == 'account_signup' and not settings.ENABLE_SIGNUP and not signup_token:
+            return False
+        elif request.resolver_match.view_name == 'socialaccount_signup' and len(settings.SOCIAL_PROVIDERS) < 1:
             return False
         else:
             return super(AllAuthCustomAdapter, self).is_open_for_signup(request)
@@ -31,7 +33,10 @@ class AllAuthCustomAdapter(DefaultAccountAdapter):
             default = datetime.datetime.now()
             c = caches['default'].get_or_set(email, default, timeout=360)
             if c == default:
-                super(AllAuthCustomAdapter, self).send_mail(template_prefix, email, context)
+                try:
+                    super(AllAuthCustomAdapter, self).send_mail(template_prefix, email, context)
+                except Exception:  # dont fail signup just because confirmation mail could not be send
+                    pass
             else:
                 messages.add_message(self.request, messages.ERROR, _('In order to prevent spam, the requested email was not send. Please wait a few minutes and try again.'))
         else:

cookbook/helper/CustomTestRunner.py

@@ -1,8 +0,0 @@
-from django.test.runner import DiscoverRunner
-from django_scopes import scopes_disabled
-
-
-class CustomTestRunner(DiscoverRunner):
-    def run_tests(self, *args, **kwargs):
-        with scopes_disabled():
-            return super().run_tests(*args, **kwargs)

cookbook/helper/HelperFunctions.py

@@ -1,4 +1,12 @@
+import socket
+from urllib.parse import urlparse
+
+from django.core.exceptions import ValidationError
+from django.core.validators import URLValidator
 from django.db.models import Func
+from ipaddress import ip_address
+
+from recipes import settings
 
 
 class Round(Func):
@@ -7,7 +15,34 @@ class Round(Func):
 
 
 def str2bool(v):
-    if type(v) == bool or v is None:
+    if isinstance(v, bool) or v is None:
         return v
     else:
         return v.lower() in ("yes", "true", "1")
+
+
+"""
+validates an url that is supposed to be imported
+checks that the protocol used is http(s) and that no local address is accessed
+@:param url to test
+@:return true if url is valid, false otherwise
+"""
+
+
+def validate_import_url(url):
+    try:
+        validator = URLValidator(schemes=['http', 'https'])
+        validator(url)
+    except ValidationError:
+        # if schema is not http or https, consider url invalid
+        return False
+
+    # resolve IP address of url
+    try:
+        url_ip_address = ip_address(str(socket.gethostbyname(urlparse(url).hostname)))
+    except (ValueError, AttributeError, TypeError, Exception) as e:
+        # if ip cannot be parsed, consider url invalid
+        return False
+
+    # validate that IP is neither private nor any other special address
+    return not any([url_ip_address.is_private, url_ip_address.is_reserved, url_ip_address.is_loopback,  url_ip_address.is_multicast, url_ip_address.is_link_local, ])

cookbook/helper/__init__.py

@@ -1,6 +1,4 @@
-import cookbook.helper.dal
 from cookbook.helper.AllAuthCustomAdapter import AllAuthCustomAdapter
 
 __all__ = [
-    'dal',
 ]

cookbook/helper/automation_helper.py

@@ -0,0 +1,227 @@
+import re
+
+from django.core.cache import caches
+from django.db.models.functions import Lower
+
+from cookbook.models import Automation
+
+
+class AutomationEngine:
+    request = None
+    source = None
+    use_cache = None
+    food_aliases = None
+    keyword_aliases = None
+    unit_aliases = None
+    never_unit = None
+    transpose_words = None
+    regex_replace = {
+        Automation.DESCRIPTION_REPLACE: None,
+        Automation.INSTRUCTION_REPLACE: None,
+        Automation.FOOD_REPLACE: None,
+        Automation.UNIT_REPLACE: None,
+        Automation.NAME_REPLACE: None,
+    }
+
+    def __init__(self, request, use_cache=True, source=None):
+        self.request = request
+        self.use_cache = use_cache
+        if not source:
+            self.source = "default_string_to_avoid_false_regex_match"
+        else:
+            self.source = source
+
+    def apply_keyword_automation(self, keyword):
+        keyword = keyword.strip()
+        if self.use_cache and self.keyword_aliases is None:
+            self.keyword_aliases = {}
+            KEYWORD_CACHE_KEY = f'automation_keyword_alias_{self.request.space.pk}'
+            if c := caches['default'].get(KEYWORD_CACHE_KEY, None):
+                self.keyword_aliases = c
+                caches['default'].touch(KEYWORD_CACHE_KEY, 30)
+            else:
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.KEYWORD_ALIAS).only('param_1', 'param_2').order_by('order').all():
+                    self.keyword_aliases[a.param_1.lower()] = a.param_2
+                caches['default'].set(KEYWORD_CACHE_KEY, self.keyword_aliases, 30)
+        else:
+            self.keyword_aliases = {}
+        if self.keyword_aliases:
+            try:
+                keyword = self.keyword_aliases[keyword.lower()]
+            except KeyError:
+                pass
+        else:
+            if automation := Automation.objects.filter(space=self.request.space, type=Automation.KEYWORD_ALIAS, param_1__iexact=keyword, disabled=False).order_by('order').first():
+                return automation.param_2
+        return keyword
+
+    def apply_unit_automation(self, unit):
+        unit = unit.strip()
+        if self.use_cache and self.unit_aliases is None:
+            self.unit_aliases = {}
+            UNIT_CACHE_KEY = f'automation_unit_alias_{self.request.space.pk}'
+            if c := caches['default'].get(UNIT_CACHE_KEY, None):
+                self.unit_aliases = c
+                caches['default'].touch(UNIT_CACHE_KEY, 30)
+            else:
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.UNIT_ALIAS).only('param_1', 'param_2').order_by('order').all():
+                    self.unit_aliases[a.param_1.lower()] = a.param_2
+                caches['default'].set(UNIT_CACHE_KEY, self.unit_aliases, 30)
+        else:
+            self.unit_aliases = {}
+        if self.unit_aliases:
+            try:
+                unit = self.unit_aliases[unit.lower()]
+            except KeyError:
+                pass
+        else:
+            if automation := Automation.objects.filter(space=self.request.space, type=Automation.UNIT_ALIAS, param_1__iexact=unit, disabled=False).order_by('order').first():
+                return automation.param_2
+        return self.apply_regex_replace_automation(unit, Automation.UNIT_REPLACE)
+
+    def apply_food_automation(self, food):
+        food = food.strip()
+        if self.use_cache and self.food_aliases is None:
+            self.food_aliases = {}
+            FOOD_CACHE_KEY = f'automation_food_alias_{self.request.space.pk}'
+            if c := caches['default'].get(FOOD_CACHE_KEY, None):
+                self.food_aliases = c
+                caches['default'].touch(FOOD_CACHE_KEY, 30)
+            else:
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.FOOD_ALIAS).only('param_1', 'param_2').order_by('order').all():
+                    self.food_aliases[a.param_1.lower()] = a.param_2
+                caches['default'].set(FOOD_CACHE_KEY, self.food_aliases, 30)
+        else:
+            self.food_aliases = {}
+
+        if self.food_aliases:
+            try:
+                return self.food_aliases[food.lower()]
+            except KeyError:
+                return self.apply_regex_replace_automation(food, Automation.FOOD_REPLACE)
+        else:
+            if automation := Automation.objects.filter(space=self.request.space, type=Automation.FOOD_ALIAS, param_1__iexact=food, disabled=False).order_by('order').first():
+                return automation.param_2
+        return self.apply_regex_replace_automation(food, Automation.FOOD_REPLACE)
+
+    def apply_never_unit_automation(self, tokens):
+        """
+        Moves a string that should never be treated as a unit to next token and optionally replaced with default unit
+        e.g. NEVER_UNIT: param1: egg, param2: None would modify ['1', 'egg', 'white'] to ['1', '', 'egg', 'white']
+        or NEVER_UNIT: param1: egg, param2: pcs would modify ['1', 'egg', 'yolk'] to ['1', 'pcs', 'egg', 'yolk']
+        :param1 tokens: string that should never be considered a unit, will be moved to token[2]
+        :param2 (optional) unit as string: will insert unit string into token[1]
+        :return: unit as string (possibly changed by automation)
+        """
+
+        if self.use_cache and self.never_unit is None:
+            self.never_unit = {}
+            NEVER_UNIT_CACHE_KEY = f'automation_never_unit_{self.request.space.pk}'
+            if c := caches['default'].get(NEVER_UNIT_CACHE_KEY, None):
+                self.never_unit = c
+                caches['default'].touch(NEVER_UNIT_CACHE_KEY, 30)
+            else:
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.NEVER_UNIT).only('param_1', 'param_2').order_by('order').all():
+                    self.never_unit[a.param_1.lower()] = a.param_2
+                caches['default'].set(NEVER_UNIT_CACHE_KEY, self.never_unit, 30)
+        else:
+            self.never_unit = {}
+
+        new_unit = None
+        alt_unit = self.apply_unit_automation(tokens[1])
+        never_unit = False
+        if self.never_unit:
+            try:
+                new_unit = self.never_unit[tokens[1].lower()]
+                never_unit = True
+            except KeyError:
+                return tokens, never_unit
+        else:
+            if a := Automation.objects.annotate(param_1_lower=Lower('param_1')).filter(space=self.request.space, type=Automation.NEVER_UNIT, param_1_lower__in=[
+                    tokens[1].lower(), alt_unit.lower()], disabled=False).order_by('order').first():
+                new_unit = a.param_2
+                never_unit = True
+
+        if never_unit:
+            tokens.insert(1, new_unit)
+        return tokens, never_unit
+
+    def apply_transpose_automation(self, string):
+        """
+        If two words (param_1 & param_2) are detected in sequence, swap their position in the ingredient string
+        :param 1: first word to detect
+        :param 2: second word to detect
+        return: new ingredient string
+        """
+        if self.use_cache and self.transpose_words is None:
+            self.transpose_words = {}
+            TRANSPOSE_WORDS_CACHE_KEY = f'automation_transpose_words_{self.request.space.pk}'
+            if c := caches['default'].get(TRANSPOSE_WORDS_CACHE_KEY, None):
+                self.transpose_words = c
+                caches['default'].touch(TRANSPOSE_WORDS_CACHE_KEY, 30)
+            else:
+                i = 0
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.TRANSPOSE_WORDS).only(
+                        'param_1', 'param_2').order_by('order').all()[:512]:
+                    self.transpose_words[i] = [a.param_1.lower(), a.param_2.lower()]
+                    i += 1
+                caches['default'].set(TRANSPOSE_WORDS_CACHE_KEY, self.transpose_words, 30)
+        else:
+            self.transpose_words = {}
+
+        tokens = [x.lower() for x in string.replace(',', ' ').split()]
+        if self.transpose_words:
+            for key, value in self.transpose_words.items():
+                if value[0] in tokens and value[1] in tokens:
+                    string = re.sub(rf"\b({value[0]})\W*({value[1]})\b", r"\2 \1", string, flags=re.IGNORECASE)
+        else:
+            for rule in Automation.objects.filter(space=self.request.space, type=Automation.TRANSPOSE_WORDS, disabled=False) \
+                    .annotate(param_1_lower=Lower('param_1'), param_2_lower=Lower('param_2')) \
+                    .filter(param_1_lower__in=tokens, param_2_lower__in=tokens).order_by('order')[:512]:
+                if rule.param_1 in tokens and rule.param_2 in tokens:
+                    string = re.sub(rf"\b({rule.param_1})\W*({rule.param_2})\b", r"\2 \1", string, flags=re.IGNORECASE)
+        return string
+
+    def apply_regex_replace_automation(self, string, automation_type):
+        # TODO add warning - maybe on SPACE page? when a max of 512 automations of a specific type is exceeded (ALIAS types excluded?)
+        """
+        Replaces strings in a recipe field that are from a matched source
+        field_type are Automation.type that apply regex replacements
+        Automation.DESCRIPTION_REPLACE
+        Automation.INSTRUCTION_REPLACE
+        Automation.FOOD_REPLACE
+        Automation.UNIT_REPLACE
+        Automation.NAME_REPLACE
+
+        regex replacment utilized the following fields from the Automation model
+        :param 1: source that should apply the automation in regex format ('.*' for all)
+        :param 2: regex pattern to match ()
+        :param 3: replacement string (leave blank to delete)
+        return: new string
+        """
+        if self.use_cache and self.regex_replace[automation_type] is None:
+            self.regex_replace[automation_type] = {}
+            REGEX_REPLACE_CACHE_KEY = f'automation_regex_replace_{self.request.space.pk}'
+            if c := caches['default'].get(REGEX_REPLACE_CACHE_KEY, None):
+                self.regex_replace[automation_type] = c[automation_type]
+                caches['default'].touch(REGEX_REPLACE_CACHE_KEY, 30)
+            else:
+                i = 0
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=automation_type).only(
+                        'param_1', 'param_2', 'param_3').order_by('order').all()[:512]:
+                    self.regex_replace[automation_type][i] = [a.param_1, a.param_2, a.param_3]
+                    i += 1
+                caches['default'].set(REGEX_REPLACE_CACHE_KEY, self.regex_replace, 30)
+        else:
+            self.regex_replace[automation_type] = {}
+
+        if self.regex_replace[automation_type]:
+            for rule in self.regex_replace[automation_type].values():
+                if re.match(rule[0], (self.source)[:512]):
+                    string = re.sub(rule[1], rule[2], string, flags=re.IGNORECASE)
+        else:
+            for rule in Automation.objects.filter(space=self.request.space, disabled=False, type=automation_type).only(
+                    'param_1', 'param_2', 'param_3').order_by('order').all()[:512]:
+                if re.match(rule.param_1, (self.source)[:512]):
+                    string = re.sub(rule.param_2, rule.param_3, string, flags=re.IGNORECASE)
+        return string

cookbook/helper/cache_helper.py

@@ -0,0 +1,11 @@
+class CacheHelper:
+    space = None
+
+    BASE_UNITS_CACHE_KEY = None
+    PROPERTY_TYPE_CACHE_KEY = None
+
+    def __init__(self, space):
+        self.space = space
+
+        self.BASE_UNITS_CACHE_KEY = f'SPACE_{space.id}_BASE_UNITS'
+        self.PROPERTY_TYPE_CACHE_KEY = f'SPACE_{space.id}_PROPERTY_TYPES'

cookbook/helper/context_processors.py

@@ -10,4 +10,6 @@ def context_settings(request):
         'TERMS_URL': settings.TERMS_URL,
         'PRIVACY_URL': settings.PRIVACY_URL,
         'IMPRINT_URL': settings.IMPRINT_URL,
+        'SHOPPING_MIN_AUTOSYNC_INTERVAL': settings.SHOPPING_MIN_AUTOSYNC_INTERVAL,
+        'DISABLE_EXTERNAL_CONNECTORS': settings.DISABLE_EXTERNAL_CONNECTORS,
     }

cookbook/helper/dal.py

@@ -1,34 +0,0 @@
-from cookbook.models import Food, Keyword, Recipe, Unit
-
-from dal import autocomplete
-
-
-class BaseAutocomplete(autocomplete.Select2QuerySetView):
-    model = None
-
-    def get_queryset(self):
-        if not self.request.user.is_authenticated:
-            return self.model.objects.none()
-
-        qs = self.model.objects.filter(space=self.request.space).all()
-
-        if self.q:
-            qs = qs.filter(name__icontains=self.q)
-
-        return qs
-
-
-class KeywordAutocomplete(BaseAutocomplete):
-    model = Keyword
-
-
-class IngredientsAutocomplete(BaseAutocomplete):
-    model = Food
-
-
-class RecipeAutocomplete(BaseAutocomplete):
-    model = Recipe
-
-
-class UnitAutocomplete(BaseAutocomplete):
-    model = Unit

cookbook/helper/drf_spectacular_hooks.py

@@ -0,0 +1,102 @@
+# custom processing for schema
+# reason: DRF writable nested needs ID's to decide if a nested object should be created or updated
+# the API schema/client make ID's read only by default and strips them entirely in request objects (with COMPONENT_SPLIT_REQUEST enabled)
+# change the schema to make IDs optional but writable so they are included in the request
+
+def custom_postprocessing_hook(result, generator, request, public):
+    for c in result['components']['schemas'].keys():
+        # handle schemas used by the client to do requests on the server
+        if 'properties' in result['components']['schemas'][c] and 'id' in result['components']['schemas'][c]['properties']:
+            # make ID field not read only so it's not stripped from the request on the client
+            result['components']['schemas'][c]['properties']['id']['readOnly'] = False
+            # make ID field not required
+            if 'required' in result['components']['schemas'][c] and 'id' in result['components']['schemas'][c]['required']:
+                result['components']['schemas'][c]['required'].remove('id')
+
+    return result
+
+
+# TODO remove below once legacy API has been fully deprecated
+from drf_spectacular.openapi import AutoSchema  # noqa: E402 isort: skip
+import functools  # noqa: E402 isort: skip
+import re  # noqa: E402 isort: skip
+
+
+class LegacySchema(AutoSchema):
+    operation_id_base = None
+
+    @functools.cached_property
+    def path(self):
+        path = re.sub(pattern=self.path_prefix, repl='', string=self.path, flags=re.IGNORECASE)
+        # remove path variables
+        return re.sub(pattern=r'\{[\w\-]+\}', repl='', string=path)
+
+    def get_operation_id(self):
+        """
+        Compute an operation ID from the view type and get_operation_id_base method.
+        """
+        method_name = getattr(self.view, 'action', self.method.lower())
+        if self._is_list_view():
+            action = 'list'
+        elif method_name not in self.method_mapping:
+            action = self._to_camel_case(method_name)
+        else:
+            action = self.method_mapping[self.method.lower()]
+
+        name = self.get_operation_id_base(action)
+
+        return action + name
+
+    def get_operation_id_base(self, action):
+        """
+        Compute the base part for operation ID from the model, serializer or view name.
+        """
+        model = getattr(getattr(self.view, 'queryset', None), 'model', None)
+
+        if self.operation_id_base is not None:
+            name = self.operation_id_base
+
+        # Try to deduce the ID from the view's model
+        elif model is not None:
+            name = model.__name__
+
+        # Try with the serializer class name
+        elif self.get_serializer() is not None:
+            name = self.get_serializer().__class__.__name__
+            if name.endswith('Serializer'):
+                name = name[:-10]
+
+        # Fallback to the view name
+        else:
+            name = self.view.__class__.__name__
+            if name.endswith('APIView'):
+                name = name[:-7]
+            elif name.endswith('View'):
+                name = name[:-4]
+
+            # Due to camel-casing of classes and `action` being lowercase, apply title in order to find if action truly
+            # comes at the end of the name
+            if name.endswith(action.title()):  # ListView, UpdateAPIView, ThingDelete ...
+                name = name[:-len(action)]
+
+        if action == 'list' and not name.endswith('s'):  # listThings instead of listThing
+            name += 's'
+
+        return name
+
+    def get_serializer(self):
+        view = self.view
+
+        if not hasattr(view, 'get_serializer'):
+            return None
+
+        try:
+            return view.get_serializer()
+        except Exception:
+            return None
+
+    def _to_camel_case(self, snake_str):
+        components = snake_str.split('_')
+        # We capitalize the first letter of each component except the first one
+        # with the 'title' method and join them together.
+        return components[0] + ''.join(x.title() for x in components[1:])

cookbook/helper/fdc_helper.py

@@ -0,0 +1,19 @@
+import json
+
+
+def get_all_nutrient_types():
+    f = open('')  # <--- download the foundation food or any other dataset and retrieve all nutrition ID's from it https://fdc.nal.usda.gov/download-datasets.html
+    json_data = json.loads(f.read())
+
+    nutrients = {}
+    for food in json_data['FoundationFoods']:
+        for entry in food['foodNutrients']:
+            nutrients[entry['nutrient']['id']] = {'name': entry['nutrient']['name'], 'unit': entry['nutrient']['unitName']}
+
+    nutrient_ids = list(nutrients.keys())
+    nutrient_ids.sort()
+    for nid in nutrient_ids:
+        print('{', f'value: {nid}, text: "{nutrients[nid]["name"]} [{nutrients[nid]["unit"]}] ({nid})"', '},')
+
+
+get_all_nutrient_types()

cookbook/helper/image_processing.py

@@ -1,8 +1,7 @@
 import os
-import sys
+from io import BytesIO
 
 from PIL import Image
-from io import BytesIO
 
 
 def rescale_image_jpeg(image_object, base_width=1020):
@@ -11,7 +10,7 @@ def rescale_image_jpeg(image_object, base_width=1020):
     width_percent = (base_width / float(img.size[0]))
     height = int((float(img.size[1]) * float(width_percent)))
 
-    img = img.resize((base_width, height), Image.ANTIALIAS)
+    img = img.resize((base_width, height), Image.LANCZOS)
     img_bytes = BytesIO()
     img.save(img_bytes, 'JPEG', quality=90, optimize=True, icc_profile=icc_profile)
 
@@ -22,7 +21,7 @@ def rescale_image_png(image_object, base_width=1020):
     image_object = Image.open(image_object)
     wpercent = (base_width / float(image_object.size[0]))
     hsize = int((float(image_object.size[1]) * float(wpercent)))
-    img = image_object.resize((base_width, hsize), Image.ANTIALIAS)
+    img = image_object.resize((base_width, hsize), Image.LANCZOS)
 
     im_io = BytesIO()
     img.save(im_io, 'PNG', quality=90)
@@ -36,14 +35,58 @@ def get_filetype(name):
         return '.jpeg'
 
 
+def is_file_type_allowed(filename, image_only=False):
+    is_file_allowed = False
+    allowed_file_types = ['.pdf', '.docx', '.xlsx', '.css', '.mp4', '.mov']
+    allowed_image_types = ['.png', '.jpg', '.jpeg', '.gif', '.webp']
+    check_list = allowed_image_types
+    if not image_only:
+        check_list += allowed_file_types
+
+    for file_type in check_list:
+        if filename.lower().endswith(file_type):
+            is_file_allowed = True
+
+    return is_file_allowed
+
+
+def strip_image_meta(image_object, file_format):
+    image_object = Image.open(image_object)
+
+    data = list(image_object.getdata())
+    image_without_exif = Image.new(image_object.mode, image_object.size)
+    image_without_exif.putdata(data)
+
+    im_io = BytesIO()
+    image_without_exif.save(im_io, file_format)
+    return im_io
+
+
 # TODO this whole file needs proper documentation, refactoring, and testing
 # TODO also add env variable to define which images sizes should be compressed
 # filetype argument can not be optional, otherwise this function will treat all images as if they were a jpeg
 # Because it's no longer optional, no reason to return it
-def handle_image(request, image_object, filetype): 
+def handle_image(request, image_object, filetype):
+    try:
+        Image.open(image_object).verify()
+    except Exception:
+        return None
+
+    file_format = None
+    if filetype == '.jpeg' or filetype == '.jpg':
+        file_format = 'JPEG'
+    if filetype == '.png':
+        file_format = 'PNG'
+    if filetype == '.webp':
+        file_format = 'WEBP'
+
     if (image_object.size / 1000) > 500:  # if larger than 500 kb compress
         if filetype == '.jpeg' or filetype == '.jpg':
             return rescale_image_jpeg(image_object)
         if filetype == '.png':
             return rescale_image_png(image_object)
+    else:
+        return strip_image_meta(image_object, file_format)
+
+    # TODO webp and gifs bypass the scaling and metadata checks, fix
     return image_object

cookbook/helper/ingredient_parser.py

@@ -2,18 +2,16 @@ import re
 import string
 import unicodedata
 
-from django.core.cache import caches
-
-from cookbook.models import Unit, Food, Automation, Ingredient
+from cookbook.helper.automation_helper import AutomationEngine
+from cookbook.models import Food, Ingredient, Unit
 
 
 class IngredientParser:
     request = None
     ignore_rules = False
-    food_aliases = {}
-    unit_aliases = {}
+    automation = None
 
-    def __init__(self, request, cache_mode, ignore_automations=False):
+    def __init__(self, request, cache_mode=True, ignore_automations=False):
         """
         Initialize ingredient parser
         :param request: request context (to control caching, rule ownership, etc.)
@@ -22,65 +20,8 @@ class IngredientParser:
         """
         self.request = request
         self.ignore_rules = ignore_automations
-        if cache_mode:
-            FOOD_CACHE_KEY = f'automation_food_alias_{self.request.space.pk}'
-            if c := caches['default'].get(FOOD_CACHE_KEY, None):
-                self.food_aliases = c
-                caches['default'].touch(FOOD_CACHE_KEY, 30)
-            else:
-                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.FOOD_ALIAS).only('param_1', 'param_2').all():
-                    self.food_aliases[a.param_1] = a.param_2
-                caches['default'].set(FOOD_CACHE_KEY, self.food_aliases, 30)
-
-            UNIT_CACHE_KEY = f'automation_unit_alias_{self.request.space.pk}'
-            if c := caches['default'].get(UNIT_CACHE_KEY, None):
-                self.unit_aliases = c
-                caches['default'].touch(UNIT_CACHE_KEY, 30)
-            else:
-                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.UNIT_ALIAS).only('param_1', 'param_2').all():
-                    self.unit_aliases[a.param_1] = a.param_2
-                caches['default'].set(UNIT_CACHE_KEY, self.unit_aliases, 30)
-        else:
-            self.food_aliases = {}
-            self.unit_aliases = {}
-
-    def apply_food_automation(self, food):
-        """
-        Apply food alias automations to passed food
-        :param food: unit as string
-        :return: food as string (possibly changed by automation)
-        """
-        if self.ignore_rules:
-            return food
-        else:
-            if self.food_aliases:
-                try:
-                    return self.food_aliases[food]
-                except KeyError:
-                    return food
-            else:
-                if automation := Automation.objects.filter(space=self.request.space, type=Automation.FOOD_ALIAS, param_1=food, disabled=False).first():
-                    return automation.param_2
-        return food
-
-    def apply_unit_automation(self, unit):
-        """
-        Apply unit alias automations to passed unit
-        :param unit: unit as string
-        :return: unit as string (possibly changed by automation)
-        """
-        if self.ignore_rules:
-            return unit
-        else:
-            if self.unit_aliases:
-                try:
-                    return self.unit_aliases[unit]
-                except KeyError:
-                    return unit
-            else:
-                if automation := Automation.objects.filter(space=self.request.space, type=Automation.UNIT_ALIAS, param_1=unit, disabled=False).first():
-                    return automation.param_2
-        return unit
+        if not self.ignore_rules:
+            self.automation = AutomationEngine(self.request, use_cache=cache_mode)
 
     def get_unit(self, unit):
         """
@@ -91,7 +32,10 @@ class IngredientParser:
         if not unit:
             return None
         if len(unit) > 0:
-            u, created = Unit.objects.get_or_create(name=self.apply_unit_automation(unit), space=self.request.space)
+            if self.ignore_rules:
+                u, created = Unit.objects.get_or_create(name=unit.strip(), space=self.request.space)
+            else:
+                u, created = Unit.objects.get_or_create(name=self.automation.apply_unit_automation(unit), space=self.request.space)
             return u
         return None
 
@@ -104,7 +48,10 @@ class IngredientParser:
         if not food:
             return None
         if len(food) > 0:
-            f, created = Food.objects.get_or_create(name=self.apply_food_automation(food), space=self.request.space)
+            if self.ignore_rules:
+                f, created = Food.objects.get_or_create(name=food.strip(), space=self.request.space)
+            else:
+                f, created = Food.objects.get_or_create(name=self.automation.apply_food_automation(food), space=self.request.space)
             return f
         return None
 
@@ -126,15 +73,17 @@ class IngredientParser:
         amount = 0
         unit = None
         note = ''
+        if x.strip() == '':
+            return amount, unit, note
 
         did_check_frac = False
         end = 0
         while (end < len(x) and (x[end] in string.digits
                                  or (
-                                         (x[end] == '.' or x[end] == ',' or x[end] == '/')
-                                         and end + 1 < len(x)
-                                         and x[end + 1] in string.digits
-                                 ))):
+            (x[end] == '.' or x[end] == ',' or x[end] == '/')
+            and end + 1 < len(x)
+            and x[end + 1] in string.digits
+        ))):
             end += 1
         if end > 0:
             if "/" in x[:end]:
@@ -158,7 +107,8 @@ class IngredientParser:
         if unit is not None and unit.strip() == '':
             unit = None
 
-        if unit is not None and (unit.startswith('(') or unit.startswith('-')):  # i dont know any unit that starts with ( or - so its likely an alternative like 1L (500ml) Water or 2-3
+        if unit is not None and (unit.startswith('(') or unit.startswith(
+                '-')):  # i dont know any unit that starts with ( or - so its likely an alternative like 1L (500ml) Water or 2-3
             unit = None
             note = x
         return amount, unit, note
@@ -168,7 +118,7 @@ class IngredientParser:
         note = ''
         start = 0
         # search for first occurrence of an argument ending in a comma
-        while start < len(tokens) and not tokens[start].endswith(','):
+        while start < len(tokens) and not tokens[start].endswith((',', ';', ':')):
             start += 1
         if start == len(tokens):
             # no token ending in a comma found -> use everything as food
@@ -219,19 +169,35 @@ class IngredientParser:
         if len(ingredient) == 0:
             raise ValueError('string to parse cannot be empty')
 
+        if len(ingredient) > 512:
+            raise ValueError('cannot parse ingredients with more than 512 characters')
+
         # some people/languages put amount and unit at the end of the ingredient string
         # if something like this is detected move it to the beginning so the parser can handle it
-        if len(ingredient) < 1000 and re.search(r'^([A-z])+(.)*[1-9](\d)*\s([A-z])+', ingredient):
-            match = re.search(r'[1-9](\d)*\s([A-z])+', ingredient)
-            print(f'reording from {ingredient} to {ingredient[match.start():match.end()] + " " + ingredient.replace(ingredient[match.start():match.end()], "")}')
+        if len(ingredient) < 1000 and re.search(r'^([^\W\d_])+(.)*[1-9](\d)*\s*([^\W\d_])+', ingredient):
+            match = re.search(r'[1-9](\d)*\s*([^\W\d_])+', ingredient)
             ingredient = ingredient[match.start():match.end()] + ' ' + ingredient.replace(ingredient[match.start():match.end()], '')
 
         # if the string contains parenthesis early on remove it and place it at the end
         # because its likely some kind of note
-        if re.match('(.){1,6}\s\((.[^\(\)])+\)\s', ingredient):
-            match = re.search('\((.[^\(])+\)', ingredient)
+        if re.match('(.){1,6}\\s\\((.[^\\(\\)])+\\)\\s', ingredient):
+            match = re.search('\\((.[^\\(])+\\)', ingredient)
             ingredient = ingredient[:match.start()] + ingredient[match.end():] + ' ' + ingredient[match.start():match.end()]
 
+        # leading spaces before commas result in extra tokens, clean them out
+        ingredient = ingredient.replace(' ,', ',')
+
+        # handle "(from) - (to)" amounts by using the minimum amount and adding the range to the description
+        # "10.5 - 200 g XYZ" => "100 g XYZ (10.5 - 200)"
+        ingredient = re.sub("^(\\d+|\\d+[\\.,]\\d+) - (\\d+|\\d+[\\.,]\\d+) (.*)", "\\1 \\3 (\\1 - \\2)", ingredient)
+
+        # if amount and unit are connected add space in between
+        if re.match('([0-9])+([A-z])+\\s', ingredient):
+            ingredient = re.sub(r'(?<=([a-z])|\d)(?=(?(1)\d|[a-z]))', ' ', ingredient)
+
+        if not self.ignore_rules:
+            ingredient = self.automation.apply_transpose_automation(ingredient)
+
         tokens = ingredient.split()  # split at each space into tokens
         if len(tokens) == 1:
             # there only is one argument, that must be the food
@@ -244,37 +210,46 @@ class IngredientParser:
                 # three arguments if it already has a unit there can't be
                 # a fraction for the amount
                 if len(tokens) > 2:
-                    try:
-                        if unit is not None:
-                            # a unit is already found, no need to try the second argument for a fraction
-                            # probably not the best method to do it, but I didn't want to make an if check and paste the exact same thing in the else as already is in the except
-                            raise ValueError
-                        # try to parse second argument as amount and add that, in case of '2 1/2' or '2 ½'
-                        amount += self.parse_fraction(tokens[1])
-                        # assume that units can't end with a comma
-                        if len(tokens) > 3 and not tokens[2].endswith(','):
-                            # try to use third argument as unit and everything else as food, use everything as food if it fails
-                            try:
-                                food, note = self.parse_food(tokens[3:])
-                                unit = tokens[2]
-                            except ValueError:
+                    never_unit_applied = False
+                    if not self.ignore_rules:
+                        tokens, never_unit_applied = self.automation.apply_never_unit_automation(tokens)
+
+                    if never_unit_applied:
+                        unit = tokens[1]
+                        food, note = self.parse_food(tokens[2:])
+                    else:
+                        try:
+                            if unit is not None:
+                                # a unit is already found, no need to try the second argument for a fraction
+                                # probably not the best method to do it, but I didn't want to make an if check and paste the exact same thing in the else as already is in the except
+                                raise ValueError
+                            # try to parse second argument as amount and add that, in case of '2 1/2' or '2 ½'
+                            if tokens[1]:
+                                amount += self.parse_fraction(tokens[1])
+                            # assume that units can't end with a comma
+                            if len(tokens) > 3 and not tokens[2].endswith(','):
+                                # try to use third argument as unit and everything else as food, use everything as food if it fails
+                                try:
+                                    food, note = self.parse_food(tokens[3:])
+                                    unit = tokens[2]
+                                except ValueError:
+                                    food, note = self.parse_food(tokens[2:])
+                            else:
                                 food, note = self.parse_food(tokens[2:])
-                        else:
-                            food, note = self.parse_food(tokens[2:])
-                    except ValueError:
-                        # assume that units can't end with a comma
-                        if not tokens[1].endswith(','):
-                            # try to use second argument as unit and everything else as food, use everything as food if it fails
-                            try:
-                                food, note = self.parse_food(tokens[2:])
-                                if unit is None:
-                                    unit = tokens[1]
-                                else:
-                                    note = tokens[1]
-                            except ValueError:
+                        except ValueError:
+                            # assume that units can't end with a comma
+                            if not tokens[1].endswith(','):
+                                # try to use second argument as unit and everything else as food, use everything as food if it fails
+                                try:
+                                    food, note = self.parse_food(tokens[2:])
+                                    if unit is None:
+                                        unit = tokens[1]
+                                    else:
+                                        note = tokens[1]
+                                except ValueError:
+                                    food, note = self.parse_food(tokens[1:])
+                            else:
                                 food, note = self.parse_food(tokens[1:])
-                        else:
-                            food, note = self.parse_food(tokens[1:])
                 else:
                     # only two arguments, first one is the amount
                     # which means this is the food
@@ -290,10 +265,12 @@ class IngredientParser:
         if unit_note not in note:
             note += ' ' + unit_note
 
-        if unit:
-            unit = self.apply_unit_automation(unit.strip())
+        if unit and not self.ignore_rules:
+            unit = self.automation.apply_unit_automation(unit)
+
+        if food and not self.ignore_rules:
+            food = self.automation.apply_food_automation(food)
 
-        food = self.apply_food_automation(food.strip())
         if len(food) > Food._meta.get_field('name').max_length:  # test if food name is to long
             # try splitting it at a space and taking only the first arg
             if len(food.split()) > 1 and len(food.split()[0]) < Food._meta.get_field('name').max_length:
@@ -303,4 +280,7 @@ class IngredientParser:
                 note = food + ' ' + note
                 food = food[:Food._meta.get_field('name').max_length]
 
+        if len(food.strip()) == 0:
+            raise ValueError(f'Error parsing string {ingredient}, food cannot be empty')
+
         return amount, unit, food, note[:Ingredient._meta.get_field('note').max_length].strip()

cookbook/helper/mdx_attributes.py

@@ -7,7 +7,9 @@ class StyleTreeprocessor(Treeprocessor):
     def run_processor(self, node):
         for child in node:
             if child.tag == "table":
-                child.set("class", "table table-bordered")
+                child.set("class", "markdown-table")
+            if child.tag == "th" or child.tag == "td":
+                child.set("class", "markdown-table-cell")
             if child.tag == "img":
                 child.set("class", "img-fluid")
             self.run_processor(child)

cookbook/helper/mdx_urlize.py

@@ -35,6 +35,7 @@ Negative examples:
 u'<p>del.icio.us</p>'
 
 """
+from xml.etree.ElementTree import Element
 
 import markdown
 
@@ -64,7 +65,7 @@ class UrlizePattern(markdown.inlinepatterns.Pattern):
             else:
                 url = 'http://' + url
 
-        el = markdown.util.etree.Element("a")
+        el = Element("a")
         el.set('href', url)
         el.text = markdown.util.AtomicString(text)
         return el
@@ -73,9 +74,9 @@ class UrlizePattern(markdown.inlinepatterns.Pattern):
 class UrlizeExtension(markdown.Extension):
     """ Urlize Extension for Python-Markdown. """
 
-    def extendMarkdown(self, md, md_globals):
+    def extendMarkdown(self, md):
         """ Replace autolink with UrlizePattern """
-        md.inlinePatterns['autolink'] = UrlizePattern(URLIZE_RE, md)
+        md.inlinePatterns.register(UrlizePattern(URLIZE_RE, md), 'autolink', 120)
 
 
 def makeExtension(*args, **kwargs):

cookbook/helper/open_data_importer.py

@@ -0,0 +1,514 @@
+import traceback
+from collections import defaultdict
+from decimal import Decimal
+
+from cookbook.models import (Food, FoodProperty, Property, PropertyType, Supermarket,
+                             SupermarketCategory, SupermarketCategoryRelation, Unit, UnitConversion)
+import re
+
+from recipes.settings import DEBUG
+
+
+class OpenDataImportResponse:
+    total_created = 0
+    total_updated = 0
+    total_untouched = 0
+    total_errored = 0
+
+    def to_dict(self):
+        return {'total_created': self.total_created, 'total_updated': self.total_updated, 'total_untouched': self.total_untouched, 'total_errored': self.total_errored}
+
+
+class OpenDataImporter:
+    request = None
+    data = {}
+    slug_id_cache = {}
+    update_existing = False
+    use_metric = True
+
+    def __init__(self, request, data, update_existing=False, use_metric=True):
+        self.request = request
+        self.data = data
+        self.update_existing = update_existing
+        self.use_metric = use_metric
+
+    def _update_slug_cache(self, object_class, datatype):
+        self.slug_id_cache[datatype] = dict(object_class.objects.filter(space=self.request.space, open_data_slug__isnull=False).values_list('open_data_slug', 'id', ))
+
+    @staticmethod
+    def _is_obj_identical(field_list, obj, existing_obj):
+        """
+        checks if the obj meant for import is identical to an already existing one
+        :param field_list: list of field names to check
+        :type field_list: list[str]
+        :param obj: object meant for import
+        :type obj: Object
+        :param existing_obj: object already in DB
+        :type existing_obj: Object
+        :return: if objects are identical
+        :rtype: bool
+        """
+        for field in field_list:
+            if isinstance(getattr(obj, field), float) or isinstance(getattr(obj, field), Decimal):
+                if abs(float(getattr(obj, field)) - float(existing_obj[field])) > 0.001:  # convert both to float and check if basically equal
+                    print(f'comparing FLOAT {obj} failed because field {field} is not equal ({getattr(obj, field)} != {existing_obj[field]})')
+                    return False
+            elif getattr(obj, field) != existing_obj[field]:
+                print(f'comparing {obj} failed because field {field} is not equal ({getattr(obj, field)} != {existing_obj[field]})')
+                return False
+        return True
+
+    @staticmethod
+    def _merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+        """
+        sometimes there might be two objects conflicting for open data import (one has the slug, the other the name)
+        this function checks if that is the case and merges the two objects if possible
+        :param model_type: type of model to check/merge
+        :type model_type: Model
+        :param obj: object that should be created/updated
+        :type obj: Model
+        :param existing_data_slugs: dict of open data slugs mapped to objects
+        :type existing_data_slugs: dict
+        :param existing_data_names: dict of names mapped to objects
+        :type existing_data_names: dict
+        :return: true if merge was successful or not necessary else false
+        :rtype: bool
+        """
+        if obj.open_data_slug in existing_data_slugs and obj.name in existing_data_names and existing_data_slugs[obj.open_data_slug]['pk'] != existing_data_names[obj.name]['pk']:
+            try:
+                source_obj = model_type.objects.get(pk=existing_data_slugs[obj.open_data_slug]['pk'])
+                del existing_data_slugs[obj.open_data_slug]
+                source_obj.merge_into(model_type.objects.get(pk=existing_data_names[obj.name]['pk']))
+                return True
+            except RuntimeError:
+                return False  # in the edge case (e.g. parent/child) that an object cannot be merged don't update it for now
+        else:
+            return True
+
+    @staticmethod
+    def _get_existing_obj(obj, existing_data_slugs, existing_data_names):
+        """
+        gets the existing object from slug or name cache
+        :param obj: object that should be found
+        :type obj: Model
+        :param existing_data_slugs: dict of open data slugs mapped to objects
+        :type existing_data_slugs: dict
+        :param existing_data_names: dict of names mapped to objects
+        :type existing_data_names: dict
+        :return: existing object
+        :rtype: dict
+        """
+        existing_obj = None
+        if obj.open_data_slug in existing_data_slugs:
+            existing_obj = existing_data_slugs[obj.open_data_slug]
+        elif obj.name in existing_data_names:
+            existing_obj = existing_data_names[obj.name]
+
+        return existing_obj
+
+    def import_units(self):
+        od_response = OpenDataImportResponse()
+        datatype = 'unit'
+        model_type = Unit
+        field_list = ['name', 'plural_name', 'base_unit', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
+
+        for u in list(self.data[datatype].keys()):
+            obj = model_type(
+                name=self.data[datatype][u]['name'],
+                plural_name=self.data[datatype][u]['plural_name'],
+                base_unit=self.data[datatype][u]['base_unit'].lower() if self.data[datatype][u]['base_unit'] != '' else None,
+                open_data_slug=u,
+                space=self.request.space
+            )
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        return od_response
+
+    def import_category(self):
+        od_response = OpenDataImportResponse()
+        datatype = 'category'
+        model_type = SupermarketCategory
+        field_list = ['name', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
+
+        for k in list(self.data[datatype].keys()):
+            obj = model_type(
+                name=self.data[datatype][k]['name'],
+                open_data_slug=k,
+                space=self.request.space
+            )
+
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        return od_response
+
+    def import_property(self):
+        od_response = OpenDataImportResponse()
+        datatype = 'property'
+        model_type = PropertyType
+        field_list = ['name', 'unit', 'fdc_id', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
+
+        for k in list(self.data[datatype].keys()):
+            obj = model_type(
+                name=self.data[datatype][k]['name'],
+                unit=self.data[datatype][k]['unit'],
+                fdc_id=self.data[datatype][k]['fdc_id'],
+                open_data_slug=k,
+                space=self.request.space
+            )
+
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        return od_response
+
+    def import_supermarket(self):
+        od_response = OpenDataImportResponse()
+        datatype = 'store'
+        model_type = Supermarket
+        field_list = ['name', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
+
+        self._update_slug_cache(SupermarketCategory, 'category')
+        for k in list(self.data[datatype].keys()):
+            obj = model_type(
+                name=self.data[datatype][k]['name'],
+                open_data_slug=k,
+                space=self.request.space
+            )
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        # always add open data slug if matching supermarket is found, otherwise relation might fail
+        self._update_slug_cache(Supermarket, 'store')
+
+        for k in list(self.data[datatype].keys()):
+            relations = []
+            order = 0
+            for c in self.data[datatype][k]['categories']:
+                relations.append(
+                    SupermarketCategoryRelation(
+                        supermarket_id=self.slug_id_cache[datatype][k],
+                        category_id=self.slug_id_cache['category'][c],
+                        order=order,
+                    )
+                )
+                order += 1
+
+            SupermarketCategoryRelation.objects.bulk_create(relations, ignore_conflicts=True, unique_fields=('supermarket', 'category',))
+
+        return od_response
+
+    def import_food(self):
+        od_response = OpenDataImportResponse()
+        datatype = 'food'
+        model_type = Food
+        field_list = ['name', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
+
+        self._update_slug_cache(Unit, 'unit')
+        self._update_slug_cache(PropertyType, 'property')
+        self._update_slug_cache(SupermarketCategory, 'category')
+
+        unit_g = Unit.objects.filter(space=self.request.space, base_unit__iexact='g').first()
+
+        for k in list(self.data[datatype].keys()):
+
+            obj_dict = {
+                'name': self.data[datatype][k]['name'],
+                'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
+                'supermarket_category_id': self.slug_id_cache['category'][self.data[datatype][k]['store_category']] if self.data[datatype][k]['store_category'] in self.slug_id_cache['category'] else None,
+                'fdc_id': re.sub(r'\D', '', self.data[datatype][k]['fdc_id']) if self.data[datatype][k]['fdc_id'] != '' else None,
+                'open_data_slug': k,
+                'properties_food_unit_id': None,
+                'space_id': self.request.space.id,
+            }
+
+            if unit_g:
+                obj_dict['properties_food_unit_id'] = unit_g.id
+
+            obj = model_type(**obj_dict)
+
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append({'data': obj_dict})
+
+        if self.update_existing and len(update_list) > 0:
+            try:
+                model_type.objects.bulk_update(update_list, field_list)
+                od_response.total_updated += len(update_list)
+            except Exception:
+                if DEBUG:
+                    print('========= LOAD FOOD FAILED ============')
+                    print(update_list)
+                    print(existing_data_names)
+                    print(existing_data_slugs)
+                    traceback.print_exc()
+
+        if len(create_list) > 0:
+            try:
+                Food.load_bulk(create_list, None)
+                od_response.total_created += len(create_list)
+            except Exception:
+                if DEBUG:
+                    print('========= LOAD FOOD FAILED ============')
+                    print(create_list)
+                    print(existing_data_names)
+                    print(existing_data_slugs)
+                    traceback.print_exc()
+
+        # --------------- PROPERTY STUFF -----------------------
+        model_type = Property
+        field_list = ['property_type_id', 'property_amount', 'open_data_food_slug']
+
+        existing_data_slugs = {}
+        existing_data_property_types = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_food_slug']] = obj
+            existing_data_property_types[obj['property_type_id']] = obj
+
+        update_list = []
+        create_list = []
+
+        self._update_slug_cache(Food, 'food')
+
+        for k in list(self.data['food'].keys()):
+            for fp in self.data['food'][k]['properties']['type_values']:
+                obj = model_type(
+                    property_type_id=self.slug_id_cache['property'][fp['property_type']],
+                    property_amount=fp['property_value'],
+                    open_data_food_slug=k,
+                    space=self.request.space,
+                )
+
+                if obj.open_data_food_slug in existing_data_slugs and obj.property_type_id in existing_data_property_types and existing_data_slugs[obj.open_data_food_slug] == existing_data_property_types[obj.property_type_id]:
+                    existing_obj = existing_data_slugs[obj.open_data_food_slug]
+
+                    if not self._is_obj_identical(field_list, obj, existing_obj):
+                        obj.pk = existing_obj['pk']
+                        update_list.append(obj)
+                else:
+                    create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, ignore_conflicts=True, unique_fields=('space', 'open_data_food_slug', 'property_type',))
+
+        linked_properties = list(FoodProperty.objects.filter(food__space=self.request.space).values_list('property_id', flat=True).all())
+
+        property_food_relation_list = []
+        for p in model_type.objects.filter(space=self.request.space, open_data_food_slug__isnull=False).values_list('open_data_food_slug', 'id', ):
+            if p[1] == 147:
+                pass
+            # slug_id_cache should always exist, don't create relations for already linked properties (ignore_conflicts would do that as well but this is more performant)
+            if p[0] in self.slug_id_cache['food'] and p[1] not in linked_properties:
+                property_food_relation_list.append(Food.properties.through(food_id=self.slug_id_cache['food'][p[0]], property_id=p[1]))
+
+        FoodProperty.objects.bulk_create(property_food_relation_list, unique_fields=('food_id', 'property_id',))
+
+        return od_response
+
+    def import_conversion(self):
+        od_response = OpenDataImportResponse()
+        datatype = 'conversion'
+        model_type = UnitConversion
+        field_list = ['base_amount', 'base_unit_id', 'converted_amount', 'converted_unit_id', 'food_id', 'open_data_slug']
+
+        self._update_slug_cache(Food, 'food')
+        self._update_slug_cache(Unit, 'unit')
+
+        existing_data_slugs = {}
+        existing_data_foods = defaultdict(list)
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_foods[obj['food_id']].append(obj)
+
+        update_list = []
+        create_list = []
+
+        for k in list(self.data[datatype].keys()):
+            # try catch here because sometimes key "k" is not set for the food cache
+            try:
+                obj = model_type(
+                    base_amount=Decimal(self.data[datatype][k]['base_amount']),
+                    base_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['base_unit']],
+                    converted_amount=Decimal(self.data[datatype][k]['converted_amount']),
+                    converted_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['converted_unit']],
+                    food_id=self.slug_id_cache['food'][self.data[datatype][k]['food']],
+                    open_data_slug=k,
+                    space=self.request.space,
+                    created_by_id=self.request.user.id,
+                )
+
+                if obj.open_data_slug in existing_data_slugs:
+                    existing_obj = existing_data_slugs[obj.open_data_slug]
+                    if not self._is_obj_identical(field_list, obj, existing_obj):
+                        obj.pk = existing_obj['pk']
+                        update_list.append(obj)
+                    else:
+                        od_response.total_untouched += 1
+                else:
+                    matching_existing_found = False
+                    if obj.food_id in existing_data_foods:
+                        for edf in existing_data_foods[obj.food_id]:
+                            if obj.base_unit_id == edf['base_unit_id'] and obj.converted_unit_id == edf['converted_unit_id']:
+                                matching_existing_found = True
+                                if not self._is_obj_identical(field_list, obj, edf):
+                                    obj.pk = edf['pk']
+                                    update_list.append(obj)
+                                else:
+                                    od_response.total_untouched += 1
+                    if not matching_existing_found:
+                        create_list.append(obj)
+            except KeyError as e:
+                traceback.print_exc()
+                od_response.total_errored += 1
+                print(self.data[datatype][k]['food'] + ' is not in self.slug_id_cache["food"]')
+
+        if self.update_existing and len(update_list) > 0:
+            od_response.total_updated = model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_errored += len(update_list) - od_response.total_updated
+
+        if len(create_list) > 0:
+            objs_created = model_type.objects.bulk_create(create_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug'))
+            od_response.total_created = len(objs_created)
+            od_response.total_errored += len(create_list) - od_response.total_created
+
+        return od_response

cookbook/helper/permission_helper.py

@@ -1,15 +1,19 @@
+import inspect
+
 from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.decorators import user_passes_test
-from django.core.cache import caches
-from django.core.exceptions import ValidationError
+from django.core.cache import cache
+from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.http import HttpResponseRedirect
 from django.urls import reverse, reverse_lazy
 from django.utils.translation import gettext as _
+from oauth2_provider.contrib.rest_framework import TokenHasReadWriteScope, TokenHasScope
+from oauth2_provider.models import AccessToken
 from rest_framework import permissions
 from rest_framework.permissions import SAFE_METHODS
 
-from cookbook.models import ShareLink, Recipe, UserPreference
+from cookbook.models import Recipe, ShareLink, UserSpace
 
 
 def get_allowed_groups(groups_required):
@@ -27,11 +31,12 @@ def get_allowed_groups(groups_required):
     return groups_allowed
 
 
-def has_group_permission(user, groups):
+def has_group_permission(user, groups, no_cache=False):
     """
     Tests if a given user is member of a certain group (or any higher group)
     Superusers always bypass permission checks.
     Unauthenticated users can't be member of any group thus always return false.
+    :param no_cache: (optional) do not return cached results, always check agains DB
     :param user: django auth user object
     :param groups: list or tuple of groups the user should be checked for
     :return: True if user is in allowed groups, false otherwise
@@ -39,10 +44,23 @@ def has_group_permission(user, groups):
     if not user.is_authenticated:
         return False
     groups_allowed = get_allowed_groups(groups)
+
+    CACHE_KEY = hash((inspect.stack()[0][3], (user.pk, user.username, user.email), groups_allowed))
+    if not no_cache:
+        cached_result = cache.get(CACHE_KEY, default=None)
+        if cached_result is not None:
+            return cached_result
+
+    result = False
     if user.is_authenticated:
-        if bool(user.groups.filter(name__in=groups_allowed)):
-            return True
-    return False
+        if user_space := user.userspace_set.filter(active=True):
+            if len(user_space) != 1:
+                result = False  # do not allow any group permission if more than one space is active, needs to be changed when simultaneous multi-space-tenancy is added
+            elif bool(user_space.first().groups.filter(name__in=groups_allowed)):
+                result = True
+
+    cache.set(CACHE_KEY, result, timeout=10)
+    return result
 
 
 def is_object_owner(user, obj):
@@ -50,7 +68,6 @@ def is_object_owner(user, obj):
     Tests if a given user is the owner of a given object
     test performed by checking user against the objects user
     and create_by field (if exists)
-    superusers bypass all checks, unauthenticated users cannot own anything
     :param user django auth user object
     :param obj any object that should be tested
     :return: true if user is owner of object, false otherwise
@@ -58,7 +75,22 @@ def is_object_owner(user, obj):
     if not user.is_authenticated:
         return False
     try:
-        return obj.get_owner() == user
+        return obj.get_owner() == 'orphan' or obj.get_owner() == user
+    except Exception:
+        return False
+
+
+def is_space_owner(user, obj):
+    """
+    Tests if a given user is the owner the space of a given object
+    :param user django auth user object
+    :param obj any object that should be tested
+    :return: true if user is owner of the objects space, false otherwise
+    """
+    if not user.is_authenticated:
+        return False
+    try:
+        return obj.get_space().get_owner() == user
     except Exception:
         return False
 
@@ -67,7 +99,6 @@ def is_object_shared(user, obj):
     """
     Tests if a given user is shared for a given object
     test performed by checking user against the objects shared table
-    superusers bypass all checks, unauthenticated users cannot own anything
     :param user django auth user object
     :param obj any object that should be tested
     :return: true if user is shared for object, false otherwise
@@ -88,15 +119,15 @@ def share_link_valid(recipe, share):
     """
     try:
         CACHE_KEY = f'recipe_share_{recipe.pk}_{share}'
-        if c := caches['default'].get(CACHE_KEY, False):
+        if c := cache.get(CACHE_KEY, False):
             return c
 
         if link := ShareLink.objects.filter(recipe=recipe, uuid=share, abuse_blocked=False).first():
-            if 0 < settings.SHARING_LIMIT < link.request_count:
+            if 0 < settings.SHARING_LIMIT < link.request_count and not link.space.no_sharing_limit:
                 return False
             link.request_count += 1
             link.save()
-            caches['default'].set(CACHE_KEY, True, timeout=3)
+            cache.set(CACHE_KEY, True, timeout=3)
             return True
         return False
     except ValidationError:
@@ -129,18 +160,15 @@ class GroupRequiredMixin(object):
     def dispatch(self, request, *args, **kwargs):
         if not has_group_permission(request.user, self.groups_required):
             if not request.user.is_authenticated:
-                messages.add_message(request, messages.ERROR,
-                                     _('You are not logged in and therefore cannot view this page!'))
+                messages.add_message(request, messages.ERROR, _('You are not logged in and therefore cannot view this page!'))
                 return HttpResponseRedirect(reverse_lazy('account_login') + '?next=' + request.path)
             else:
-                messages.add_message(request, messages.ERROR,
-                                     _('You do not have the required permissions to view this page!'))
+                messages.add_message(request, messages.ERROR, _('You do not have the required permissions to view this page!'))
                 return HttpResponseRedirect(reverse_lazy('index'))
         try:
             obj = self.get_object()
             if obj.get_space() != request.space:
-                messages.add_message(request, messages.ERROR,
-                                     _('You do not have the required permissions to view this page!'))
+                messages.add_message(request, messages.ERROR, _('You do not have the required permissions to view this page!'))
                 return HttpResponseRedirect(reverse_lazy('index'))
         except AttributeError:
             pass
@@ -163,7 +191,7 @@ class OwnerRequiredMixin(object):
 
         try:
             obj = self.get_object()
-            if obj.get_space() != request.space:
+            if not request.user.userspace.filter(space=obj.get_space()).exists():
                 messages.add_message(request, messages.ERROR,
                                      _('You do not have the required permissions to view this page!'))
                 return HttpResponseRedirect(reverse_lazy('index'))
@@ -181,7 +209,7 @@ class CustomIsOwner(permissions.BasePermission):
     verifies user has ownership over object
     (either user or created_by or user is request user)
     """
-    message = _('You cannot interact with this object as it is not owned by you!')  # noqa: E501
+    message = _('You cannot interact with this object as it is not owned by you!')
 
     def has_permission(self, request, view):
         return request.user.is_authenticated
@@ -190,6 +218,28 @@ class CustomIsOwner(permissions.BasePermission):
         return is_object_owner(request.user, obj)
 
 
+class CustomIsOwnerReadOnly(CustomIsOwner):
+    def has_permission(self, request, view):
+        return super().has_permission(request, view) and request.method in SAFE_METHODS
+
+    def has_object_permission(self, request, view, obj):
+        return super().has_object_permission(request, view) and request.method in SAFE_METHODS
+
+
+class CustomIsSpaceOwner(permissions.BasePermission):
+    """
+    Custom permission class for django rest framework views
+    verifies if the user is the owner of the space the object belongs to
+    """
+    message = _('You cannot interact with this object as it is not owned by you!')
+
+    def has_permission(self, request, view):
+        return request.user.is_authenticated and request.space.created_by == request.user
+
+    def has_object_permission(self, request, view, obj):
+        return is_space_owner(request.user, obj)
+
+
 # TODO function duplicate/too similar name
 class CustomIsShared(permissions.BasePermission):
     """
@@ -202,9 +252,6 @@ class CustomIsShared(permissions.BasePermission):
         return request.user.is_authenticated
 
     def has_object_permission(self, request, view, obj):
-        # # temporary hack to make old shopping list work with new shopping list
-        # if obj.__class__.__name__ in ['ShoppingList', 'ShoppingListEntry']:
-        #     return is_object_shared(request.user, obj) or obj.created_by in list(request.user.get_shopping_share())
         return is_object_shared(request.user, obj)
 
 
@@ -261,6 +308,75 @@ class CustomIsShare(permissions.BasePermission):
         return False
 
 
+class CustomRecipePermission(permissions.BasePermission):
+    """
+    Custom permission class for recipe api endpoint
+    """
+    message = _('You do not have the required permissions to view this page!')
+
+    def has_permission(self, request, view):  # user is either at least a guest or a share link is given and the request is safe
+        share = request.query_params.get('share', None)
+        return ((has_group_permission(request.user, ['guest']) and request.method in SAFE_METHODS) or has_group_permission(
+            request.user, ['user'])) or (share and request.method in SAFE_METHODS and 'pk' in view.kwargs)
+
+    def has_object_permission(self, request, view, obj):
+        share = request.query_params.get('share', None)
+        if share:
+            return share_link_valid(obj, share)
+        else:
+            if obj.private:
+                return ((obj.created_by == request.user) or (request.user in obj.shared.all())) and obj.space == request.space
+            else:
+                return ((has_group_permission(request.user, ['guest']) and request.method in SAFE_METHODS)
+                        or has_group_permission(request.user, ['user'])) and obj.space == request.space
+
+
+class CustomUserPermission(permissions.BasePermission):
+    """
+    Custom permission class for user api endpoint
+    """
+    message = _('You do not have the required permissions to view this page!')
+
+    def has_permission(self, request, view):  # a space filtered user list is visible for everyone
+        return has_group_permission(request.user, ['guest'])
+
+    def has_object_permission(self, request, view, obj):  # object write permissions are only available for user
+        if request.method in SAFE_METHODS and 'pk' in view.kwargs and has_group_permission(request.user, ['guest']) and request.space in obj.userspace_set.all():
+            return True
+        elif request.user == obj:
+            return True
+        else:
+            return False
+
+
+class CustomTokenHasScope(TokenHasScope):
+    """
+    Custom implementation of Django OAuth Toolkit TokenHasScope class
+    Only difference: if any other authentication method except OAuth2Authentication is used the scope check is ignored
+    IMPORTANT: do not use this class without any other permission class as it will not check anything besides token scopes
+    """
+
+    def has_permission(self, request, view):
+        if isinstance(request.auth, AccessToken):
+            return super().has_permission(request, view)
+        else:
+            return request.user.is_authenticated
+
+
+class CustomTokenHasReadWriteScope(TokenHasReadWriteScope):
+    """
+    Custom implementation of Django OAuth Toolkit TokenHasReadWriteScope class
+    Only difference: if any other authentication method except OAuth2Authentication is used the scope check is ignored
+    IMPORTANT: do not use this class without any other permission class as it will not check anything besides token scopes
+    """
+
+    def has_permission(self, request, view):
+        if isinstance(request.auth, AccessToken):
+            return super().has_permission(request, view)
+        else:
+            return True
+
+
 def above_space_limit(space):  # TODO add file storage limit
     """
     Test if the space has reached any limit (e.g. max recipes, users, ..)
@@ -290,7 +406,34 @@ def above_space_user_limit(space):
     :param space: Space to test for limits
     :return: Tuple (True if above or equal limit else false, message)
     """
-    limit = space.max_users != 0 and UserPreference.objects.filter(space=space).count() > space.max_users
+    limit = space.max_users != 0 and UserSpace.objects.filter(space=space).count() > space.max_users
     if limit:
         return True, _('You have more users than allowed in your space.')
     return False, ''
+
+
+def switch_user_active_space(user, space):
+    """
+    Switch the currently active space of a user by setting all spaces to inactive and activating the one passed
+    :param user: user to change active space for
+    :param space: space to activate user for
+    :return user space object or none if not found/no permission
+    """
+    try:
+        us = UserSpace.objects.get(space=space, user=user)
+        if not us.active:
+            UserSpace.objects.filter(user=user).update(active=False)
+            us.active = True
+            us.save()
+            return us
+        else:
+            return us
+    except ObjectDoesNotExist:
+        return None
+
+
+class IsReadOnlyDRF(permissions.BasePermission):
+    message = 'You cannot interact with this object as it is not owned by you!'
+
+    def has_permission(self, request, view):
+        return request.method in SAFE_METHODS

cookbook/helper/property_helper.py

@@ -0,0 +1,89 @@
+from django.core.cache import caches
+
+from cookbook.helper.cache_helper import CacheHelper
+from cookbook.helper.unit_conversion_helper import UnitConversionHelper
+from cookbook.models import PropertyType
+
+
+class FoodPropertyHelper:
+    space = None
+
+    def __init__(self, space):
+        """
+        Helper to perform food property calculations
+        :param space: space to limit scope to
+        """
+        self.space = space
+
+    def calculate_recipe_properties(self, recipe):
+        """
+        Calculate all food properties for a given recipe.
+        :param recipe: recipe to calculate properties for
+        :return: dict of with property keys and total/food values for each property available
+        """
+        ingredients = []
+        computed_properties = {}
+
+        for s in recipe.steps.all():
+            ingredients += s.ingredients.all()
+
+        property_types = caches['default'].get(CacheHelper(self.space).PROPERTY_TYPE_CACHE_KEY, None)
+
+        if not property_types:
+            property_types = PropertyType.objects.filter(space=self.space).all()
+            # cache is cleared on property type save signal so long duration is fine
+            caches['default'].set(CacheHelper(self.space).PROPERTY_TYPE_CACHE_KEY, property_types, 60 * 60)
+
+        for fpt in property_types:
+            computed_properties[fpt.id] = {'id': fpt.id, 'name': fpt.name, 'description': fpt.description,
+                                           'unit': fpt.unit, 'order': fpt.order, 'food_values': {}, 'total_value': 0, 'missing_value': False}
+
+        uch = UnitConversionHelper(self.space)
+
+        for i in ingredients:
+            if i.food is not None:
+                conversions = uch.get_conversions(i)
+                for pt in property_types:
+                    # if a property could be calculated with an actual value
+                    found_property = False
+                    # if food has a value for the given property type (no matter if conversion is possible)
+                    has_property_value = False
+                    if i.food.properties_food_amount == 0 or i.food.properties_food_unit is None and not (i.amount == 0 or i.no_amount):  # if food is configured incorrectly
+                        computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': {'id': i.food.id, 'name': i.food.name}, 'value': None}
+                        computed_properties[pt.id]['missing_value'] = True
+                    else:
+                        for p in i.food.properties.all():
+                            if p.property_type == pt and p.property_amount is not None:
+                                has_property_value = True
+                                for c in conversions:
+                                    if c.unit == i.food.properties_food_unit:
+                                        found_property = True
+                                        computed_properties[pt.id]['total_value'] += (c.amount / i.food.properties_food_amount) * p.property_amount
+                                        computed_properties[pt.id]['food_values'] = self.add_or_create(
+                                            computed_properties[p.property_type.id]['food_values'], c.food.id, (c.amount / i.food.properties_food_amount) * p.property_amount, c.food)
+                    if not found_property:
+                        # if no amount and food does not exist yet add it but don't count as missing
+                        if i.amount == 0 or i.no_amount and i.food.id not in computed_properties[pt.id]['food_values']:
+                            computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': {'id': i.food.id, 'name': i.food.name}, 'value': 0}
+                        # if amount is present but unit is missing indicate it in the result
+                        elif i.unit is None:
+                            if i.food.id not in computed_properties[pt.id]['food_values']:
+                                computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': {'id': i.food.id, 'name': i.food.name}, 'value': 0}
+                            computed_properties[pt.id]['food_values'][i.food.id]['missing_unit'] = True
+                        else:
+                            computed_properties[pt.id]['missing_value'] = True
+                            computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': {'id': i.food.id, 'name': i.food.name}, 'value': None}
+                            if has_property_value and i.unit is not None:
+                                computed_properties[pt.id]['food_values'][i.food.id]['missing_conversion'] = {'base_unit': {'id': i.unit.id, 'name': i.unit.name}, 'converted_unit': {'id': i.food.properties_food_unit.id, 'name': i.food.properties_food_unit.name}}
+
+        return computed_properties
+
+    # small dict helper to add to existing key or create new, probably a better way of doing this
+    # TODO move to central helper ? --> use defaultdict
+    @staticmethod
+    def add_or_create(d, key, value, food):
+        if key in d and d[key]['value']:
+            d[key]['value'] += value
+        else:
+            d[key] = {'id': food.id, 'food': {'id': food.id, 'name': food.name}, 'value': value}
+        return d

cookbook/helper/recipe_html_import.py

@@ -1,191 +0,0 @@
-import json
-import re
-from json import JSONDecodeError
-from urllib.parse import unquote
-
-from bs4 import BeautifulSoup
-from bs4.element import Tag
-from recipe_scrapers._utils import get_host_name, normalize_string
-
-from cookbook.helper import recipe_url_import as helper
-from cookbook.helper.scrapers.scrapers import text_scraper
-from recipe_scrapers import scrape_me
-from recipe_scrapers._exceptions import NoSchemaFoundInWildMode
-
-
-def get_recipe_from_source(text, url, request):
-    def build_node(k, v):
-        if isinstance(v, dict):
-            node = {
-                'name': k,
-                'value': k,
-                'children': get_children_dict(v)
-            }
-        elif isinstance(v, list):
-            node = {
-                'name': k,
-                'value': k,
-                'children': get_children_list(v)
-            }
-        else:
-            node = {
-                'name': k + ": " + normalize_string(str(v)),
-                'value': normalize_string(str(v))
-            }
-        return node
-
-    def get_children_dict(children):
-        kid_list = []
-        for k, v in children.items():
-            kid_list.append(build_node(k, v))
-        return kid_list
-
-    def get_children_list(children):
-        kid_list = []
-        for kid in children:
-            if type(kid) == list:
-                node = {
-                    'name': "unknown list",
-                    'value': "unknown list",
-                    'children': get_children_list(kid)
-                }
-                kid_list.append(node)
-            elif type(kid) == dict:
-                for k, v in kid.items():
-                    kid_list.append(build_node(k, v))
-            else:
-                kid_list.append({
-                    'name': normalize_string(str(kid)),
-                    'value': normalize_string(str(kid))
-                })
-        return kid_list
-
-    recipe_tree = []
-    parse_list = []
-    html_data = []
-    images = []
-    text = unquote(text)
-    scrape = None
-
-    if url:
-        try:
-            scrape = scrape_me(url_path=url, wild_mode=True)
-        except(NoSchemaFoundInWildMode):
-            pass
-    if not scrape:
-        try:
-            parse_list.append(remove_graph(json.loads(text)))
-            if not url and 'url' in parse_list[0]:
-                url = parse_list[0]['url']
-            scrape = text_scraper("<script type='application/ld+json'>" + text + "</script>", url=url)
-
-        except JSONDecodeError:
-            soup = BeautifulSoup(text, "html.parser")
-            html_data = get_from_html(soup)
-            images += get_images_from_source(soup, url)
-            for el in soup.find_all('script', type='application/ld+json'):
-                el = remove_graph(el)
-                if not url and 'url' in el:
-                    url = el['url']
-                if type(el) == list:
-                    for le in el:
-                        parse_list.append(le)
-                elif type(el) == dict:
-                    parse_list.append(el)
-            for el in soup.find_all(type='application/json'):
-                el = remove_graph(el)
-                if type(el) == list:
-                    for le in el:
-                        parse_list.append(le)
-                elif type(el) == dict:
-                    parse_list.append(el)
-            scrape = text_scraper(text, url=url)
-
-    recipe_json = helper.get_from_scraper(scrape, request)
-
-    for el in parse_list:
-        temp_tree = []
-        if isinstance(el, Tag):
-            try:
-                el = json.loads(el.string)
-            except TypeError:
-                continue
-
-        for k, v in el.items():
-            if isinstance(v, dict):
-                node = {
-                    'name': k,
-                    'value': k,
-                    'children': get_children_dict(v)
-                }
-            elif isinstance(v, list):
-                node = {
-                    'name': k,
-                    'value': k,
-                    'children': get_children_list(v)
-                }
-            else:
-                node = {
-                    'name': k + ": " + normalize_string(str(v)),
-                    'value': normalize_string(str(v))
-                }
-            temp_tree.append(node)
-
-        if '@type' in el and el['@type'] == 'Recipe':
-            recipe_tree += [{'name': 'ld+json', 'children': temp_tree}]
-        else:
-            recipe_tree += [{'name': 'json', 'children': temp_tree}]
-
-    return recipe_json, recipe_tree, html_data, images
-
-
-def get_from_html(soup):
-    INVISIBLE_ELEMS = ('style', 'script', 'head', 'title')
-    html = []
-    for s in soup.strings:
-        if ((s.parent.name not in INVISIBLE_ELEMS) and (len(s.strip()) > 0)):
-            html.append(s)
-    return html
-
-
-def get_images_from_source(soup, url):
-    sources = ['src', 'srcset', 'data-src']
-    images = []
-    img_tags = soup.find_all('img')
-    if url:
-        site = get_host_name(url)
-        prot = url.split(':')[0]
-
-    urls = []
-    for img in img_tags:
-        for src in sources:
-            try:
-                urls.append(img[src])
-            except KeyError:
-                pass
-
-    for u in urls:
-        u = u.split('?')[0]
-        filename = re.search(r'/([\w_-]+[.](jpg|jpeg|gif|png))$', u)
-        if filename:
-            if (('http' not in u) and (url)):
-                # sometimes an image source can be relative
-                # if it is provide the base url
-                u = '{}://{}{}'.format(prot, site, u)
-            if 'http' in u:
-                images.append(u)
-    return images
-
-
-def remove_graph(el):
-    # recipes type might be wrapped in @graph type
-    if isinstance(el, Tag):
-        try:
-            el = json.loads(el.string)
-            if '@graph' in el:
-                for x in el['@graph']:
-                    if '@type' in x and x['@type'] == 'Recipe':
-                        el = x
-        except (TypeError, JSONDecodeError):
-            pass
-    return el

cookbook/helper/recipe_search.py

@@ -1,47 +1,59 @@
 import json
-from collections import Counter
 from datetime import date, timedelta
 
 from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector, TrigramSimilarity
-from django.core.cache import caches
-from django.db.models import Avg, Case, Count, F, Func, Max, OuterRef, Q, Subquery, Sum, Value, When
+from django.core.cache import cache
+from django.db.models import Avg, Case, Count, Exists, F, Max, OuterRef, Q, Subquery, Value, When
 from django.db.models.functions import Coalesce, Lower, Substr
 from django.utils import timezone, translation
-from django.utils.translation import gettext as _
 
-from cookbook.filters import RecipeFilter
 from cookbook.helper.HelperFunctions import Round, str2bool
-from cookbook.helper.permission_helper import has_group_permission
 from cookbook.managers import DICTIONARY
-from cookbook.models import (CookLog, CustomFilter, Food, Keyword, Recipe, RecipeBook, SearchFields,
+from cookbook.models import (CookLog, CustomFilter, Food, Keyword, Recipe, SearchFields,
                              SearchPreference, ViewLog)
 from recipes import settings
 
 
-# TODO create extensive tests to make sure ORs ANDs and various filters, sorting, etc work as expected
 # TODO consider creating a simpleListRecipe API that only includes minimum of recipe info and minimal filtering
 class RecipeSearch():
-    _postgres = settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2', 'django.db.backends.postgresql']
+    _postgres = settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql'
 
-    def __init__(self, request,  **params):
+    def __init__(self, request, **params):
         self._request = request
         self._queryset = None
         if f := params.get('filter', None):
-            custom_filter = CustomFilter.objects.filter(id=f, space=self._request.space).filter(Q(created_by=self._request.user) |
-                                                                                                Q(shared=self._request.user) | Q(recipebook__shared=self._request.user)).first()
+            custom_filter = (
+                CustomFilter.objects.filter(id=f, space=self._request.space)
+                .filter(Q(created_by=self._request.user) | Q(shared=self._request.user) | Q(recipebook__shared=self._request.user))
+                .first()
+            )
             if custom_filter:
                 self._params = {**json.loads(custom_filter.search)}
                 self._original_params = {**(params or {})}
+                # json.loads casts rating as an integer, expecting string
+                if isinstance(self._params.get('rating', None), int):
+                    self._params['rating'] = str(self._params['rating'])
             else:
                 self._params = {**(params or {})}
         else:
             self._params = {**(params or {})}
         if self._request.user.is_authenticated:
-            self._search_prefs = request.user.searchpreference
+            CACHE_KEY = f'search_pref_{request.user.id}'
+            cached_result = cache.get(CACHE_KEY, default=None)
+            if cached_result is not None:
+                self._search_prefs = cached_result
+            else:
+                self._search_prefs = request.user.searchpreference
+            cache.set(CACHE_KEY, self._search_prefs, timeout=10)
         else:
             self._search_prefs = SearchPreference()
-        self._string = self._params.get('query').strip() if self._params.get('query', None) else None
+        self._string = self._params.get('query').strip(
+        ) if self._params.get('query', None) else None
+
         self._rating = self._params.get('rating', None)
+        self._rating_gte = self._params.get('rating_gte', None)
+        self._rating_lte = self._params.get('rating_lte', None)
+
         self._keywords = {
             'or': self._params.get('keywords_or', None) or self._params.get('keywords', None),
             'and': self._params.get('keywords_and', None),
@@ -62,24 +74,41 @@ class RecipeSearch():
         }
         self._steps = self._params.get('steps', None)
         self._units = self._params.get('units', None)
-        # TODO add created by
-        # TODO image exists
-        self._sort_order = self._params.get('sort_order', None)
         self._internal = str2bool(self._params.get('internal', None))
-        self._random = str2bool(self._params.get('random', False))
+        self._sort_order = self._params.get('sort_order', None)
+        if self._sort_order == 'random':
+            self._random = True
+            self.sort_order = None
+        else:
+            self._random = str2bool(self._params.get('random', False))
         self._new = str2bool(self._params.get('new', False))
         self._num_recent = int(self._params.get('num_recent', 0))
-        self._include_children = str2bool(self._params.get('include_children', None))
+        self._include_children = str2bool(
+            self._params.get('include_children', None))
         self._timescooked = self._params.get('timescooked', None)
-        self._cookedon = self._params.get('cookedon', None)
+        self._timescooked_gte = self._params.get('timescooked_gte', None)
+        self._timescooked_lte = self._params.get('timescooked_lte', None)
+
         self._createdon = self._params.get('createdon', None)
+        self._createdon_gte = self._params.get('createdon_gte', None)
+        self._createdon_lte = self._params.get('createdon_lte', None)
+
         self._updatedon = self._params.get('updatedon', None)
-        self._viewedon = self._params.get('viewedon', None)
+        self._updatedon_gte = self._params.get('updatedon_gte', None)
+        self._updatedon_lte = self._params.get('updatedon_lte', None)
+
+        self._viewedon_gte = self._params.get('viewedon_gte', None)
+        self._viewedon_lte = self._params.get('viewedon_lte', None)
+
+        self._cookedon_gte = self._params.get('cookedon_gte', None)
+        self._cookedon_lte = self._params.get('cookedon_lte', None)
+
+        self._createdby = self._params.get('createdby', None)
         self._makenow = self._params.get('makenow', None)
         # this supports hidden feature to find recipes missing X ingredients
-        if type(self._makenow) == bool and self._makenow == True:
+        if isinstance(self._makenow, bool) and self._makenow == True:
             self._makenow = 0
-        elif type(self._makenow) == str and self._makenow in ["yes", "true"]:
+        elif isinstance(self._makenow, str) and self._makenow in ["yes", "true"]:
             self._makenow = 0
         else:
             try:
@@ -112,30 +141,34 @@ class RecipeSearch():
             )
             self.search_rank = None
         self.orderby = []
-        self._default_sort = ['-favorite']  # TODO add user setting
         self._filters = None
         self._fuzzy_match = None
 
     def get_queryset(self, queryset):
         self._queryset = queryset
+        self._queryset = self._queryset.prefetch_related('keywords')
+
         self._build_sort_order()
         self._recently_viewed(num_recent=self._num_recent)
-        self._cooked_on_filter(cooked_date=self._cookedon)
-        self._created_on_filter(created_date=self._createdon)
-        self._updated_on_filter(updated_date=self._updatedon)
-        self._viewed_on_filter(viewed_date=self._viewedon)
-        self._favorite_recipes(timescooked=self._timescooked)
+
+        self._cooked_on_filter()
+        self._created_on_filter()
+        self._updated_on_filter()
+        self._viewed_on_filter()
+
+        self._created_by_filter(created_by_user_id=self._createdby)
+        self._favorite_recipes()
         self._new_recipes()
         self.keyword_filters(**self._keywords)
         self.food_filters(**self._foods)
         self.book_filters(**self._books)
-        self.rating_filter(rating=self._rating)
+        self.rating_filter()
         self.internal_filter(internal=self._internal)
         self.step_filters(steps=self._steps)
         self.unit_filters(units=self._units)
         self._makenow_filter(missing=self._makenow)
         self.string_filters(string=self._string)
-        return self._queryset.filter(space=self._request.space).distinct().order_by(*self.orderby)
+        return self._queryset.filter(space=self._request.space).order_by(*self.orderby)
 
     def _sort_includes(self, *args):
         for x in args:
@@ -147,11 +180,11 @@ class RecipeSearch():
 
     def _build_sort_order(self):
         if self._random:
-            self._queryset = self._queryset.order_by("?")
+            self.orderby = ['?']
         else:
             order = []
             # TODO add userpreference for default sort order and replace '-favorite'
-            default_order = ['-favorite']
+            default_order = ['name']
             # recent and new_recipe are always first; they float a few recipes to the top
             if self._num_recent:
                 order += ['-recent']
@@ -160,7 +193,6 @@ class RecipeSearch():
 
             # if a sort order is provided by user - use that order
             if self._sort_order:
-
                 if not isinstance(self._sort_order, list):
                     order += [self._sort_order]
                 else:
@@ -176,8 +208,10 @@ class RecipeSearch():
             # otherwise sort by the remaining order_by attributes or favorite by default
             else:
                 order += default_order
-            order[:] = [Lower('name').asc() if x == 'name' else x for x in order]
-            order[:] = [Lower('name').desc() if x == '-name' else x for x in order]
+            order[:] = [Lower('name').asc() if x ==
+                                               'name' else x for x in order]
+            order[:] = [Lower('name').desc() if x ==
+                                                '-name' else x for x in order]
             self.orderby = order
 
     def string_filters(self, string=None):
@@ -194,7 +228,8 @@ class RecipeSearch():
             for f in self._filters:
                 query_filter |= f
 
-            self._queryset = self._queryset.filter(query_filter).distinct()  # this creates duplicate records which can screw up other aggregates, see makenow for workaround
+            # this creates duplicate records which can screw up other aggregates, see makenow for workaround
+            self._queryset = self._queryset.filter(query_filter).distinct()
             if self._fulltext_include:
                 if self._fuzzy_match is None:
                     self._queryset = self._queryset.annotate(score=Coalesce(Max(self.search_rank), 0.0))
@@ -208,74 +243,71 @@ class RecipeSearch():
                 else:
                     self._queryset = self._queryset.annotate(simularity=Coalesce(Subquery(simularity), 0.0))
             if self._sort_includes('score') and self._fulltext_include and self._fuzzy_match is not None:
-                self._queryset = self._queryset.annotate(score=F('rank')+F('simularity'))
+                self._queryset = self._queryset.annotate(score=F('rank') + F('simularity'))
         else:
             query_filter = Q()
             for f in [x + '__unaccent__iexact' if x in self._unaccent_include else x + '__iexact' for x in SearchFields.objects.all().values_list('field', flat=True)]:
                 query_filter |= Q(**{"%s" % f: self._string})
             self._queryset = self._queryset.filter(query_filter).distinct()
 
-    def _cooked_on_filter(self, cooked_date=None):
-        if self._sort_includes('lastcooked') or cooked_date:
-            lessthan = self._sort_includes('-lastcooked') or '-' in (cooked_date or [])[:1]
+    def _cooked_on_filter(self):
+        if self._sort_includes('lastcooked') or self._cookedon_gte or self._cookedon_lte:
+            lessthan = self._sort_includes('-lastcooked') or self._cookedon_lte
             if lessthan:
                 default = timezone.now() - timedelta(days=100000)
             else:
                 default = timezone.now()
-            self._queryset = self._queryset.annotate(lastcooked=Coalesce(
-                Max(Case(When(cooklog__created_by=self._request.user, cooklog__space=self._request.space, then='cooklog__created_at'))), Value(default)))
-        if cooked_date is None:
-            return
+            self._queryset = self._queryset.annotate(
+                lastcooked=Coalesce(Max(Case(When(cooklog__created_by=self._request.user, cooklog__space=self._request.space, then='cooklog__created_at'))), Value(default))
+            )
 
-        cooked_date = date(*[int(x) for x in cooked_date.split('-') if x != ''])
-
-        if lessthan:
-            self._queryset = self._queryset.filter(lastcooked__date__lte=cooked_date).exclude(lastcooked=default)
-        else:
-            self._queryset = self._queryset.filter(lastcooked__date__gte=cooked_date).exclude(lastcooked=default)
-
-    def _created_on_filter(self, created_date=None):
-        if created_date is None:
-            return
-        lessthan = '-' in created_date[:1]
-        created_date = date(*[int(x) for x in created_date.split('-') if x != ''])
-        if lessthan:
-            self._queryset = self._queryset.filter(created_at__date__lte=created_date)
-        else:
-            self._queryset = self._queryset.filter(created_at__date__gte=created_date)
-
-    def _updated_on_filter(self, updated_date=None):
-        if updated_date is None:
-            return
-        lessthan = '-' in updated_date[:1]
-        updated_date = date(*[int(x) for x in updated_date.split('-') if x != ''])
-        if lessthan:
-            self._queryset = self._queryset.filter(updated_at__date__lte=updated_date)
-        else:
-            self._queryset = self._queryset.filter(updated_at__date__gte=updated_date)
+        if self._cookedon_lte:
+            self._queryset = self._queryset.filter(lastcooked__date__lte=self._cookedon_lte).exclude(lastcooked=default)
+        elif self._cookedon_gte:
+            self._queryset = self._queryset.filter(lastcooked__date__gte=self._cookedon_gte).exclude(lastcooked=default)
 
     def _viewed_on_filter(self, viewed_date=None):
-        if self._sort_includes('lastviewed') or viewed_date:
+        if self._sort_includes('lastviewed') or self._viewedon_gte or self._viewedon_lte:
             longTimeAgo = timezone.now() - timedelta(days=100000)
-            self._queryset = self._queryset.annotate(lastviewed=Coalesce(
-                Max(Case(When(viewlog__created_by=self._request.user, viewlog__space=self._request.space, then='viewlog__created_at'))), Value(longTimeAgo)))
-        if viewed_date is None:
-            return
-        lessthan = '-' in viewed_date[:1]
-        viewed_date = date(*[int(x) for x in viewed_date.split('-') if x != ''])
+            self._queryset = self._queryset.annotate(
+                lastviewed=Coalesce(Max(Case(When(viewlog__created_by=self._request.user, viewlog__space=self._request.space, then='viewlog__created_at'))), Value(longTimeAgo))
+            )
 
-        if lessthan:
-            self._queryset = self._queryset.filter(lastviewed__date__lte=viewed_date).exclude(lastviewed=longTimeAgo)
-        else:
-            self._queryset = self._queryset.filter(lastviewed__date__gte=viewed_date).exclude(lastviewed=longTimeAgo)
+        if self._viewedon_lte:
+            self._queryset = self._queryset.filter(lastviewed__date__lte=self._viewedon_lte).exclude(lastviewed=longTimeAgo)
+        elif self._viewedon_gte:
+            self._queryset = self._queryset.filter(lastviewed__date__gte=self._viewedon_gte).exclude(lastviewed=longTimeAgo)
+
+    def _created_on_filter(self):
+        if self._createdon:
+            self._queryset = self._queryset.filter(created_at__date=self._createdon)
+        elif self._createdon_lte:
+            self._queryset = self._queryset.filter(created_at__date__lte=self._createdon_lte)
+        elif self._createdon_gte:
+            self._queryset = self._queryset.filter(created_at__date__gte=self._createdon_gte)
+
+    def _updated_on_filter(self):
+        if self._updatedon:
+            self._queryset = self._queryset.filter(updated_at__date__date=self._updatedon)
+        elif self._updatedon_lte:
+            self._queryset = self._queryset.filter(updated_at__date__lte=self._updatedon_lte)
+        elif self._updatedon_gte:
+            self._queryset = self._queryset.filter(updated_at__date__gte=self._updatedon_gte)
+
+    def _created_by_filter(self, created_by_user_id=None):
+        if created_by_user_id is None:
+            return
+        self._queryset = self._queryset.filter(created_by__id=created_by_user_id)
 
     def _new_recipes(self, new_days=7):
         # TODO make new days a user-setting
         if not self._new:
             return
-        self._queryset = (
-            self._queryset.annotate(new_recipe=Case(
-                When(created_at__gte=(timezone.now() - timedelta(days=new_days)), then=('pk')), default=Value(0), ))
+        self._queryset = self._queryset.annotate(
+            new_recipe=Case(
+                When(created_at__gte=(timezone.now() - timedelta(days=new_days)), then=('pk')),
+                default=Value(0),
+            )
         )
 
     def _recently_viewed(self, num_recent=None):
@@ -285,29 +317,33 @@ class RecipeSearch():
                     Max(Case(When(viewlog__created_by=self._request.user, viewlog__space=self._request.space, then='viewlog__pk'))), Value(0)))
             return
 
-        num_recent_recipes = ViewLog.objects.filter(created_by=self._request.user, space=self._request.space).values(
-            'recipe').annotate(recent=Max('created_at')).order_by('-recent')[:num_recent]
+        num_recent_recipes = (
+            ViewLog.objects.filter(created_by=self._request.user, space=self._request.space)
+            .values('recipe').annotate(recent=Max('created_at')).order_by('-recent')[:num_recent]
+        )
         self._queryset = self._queryset.annotate(recent=Coalesce(Max(Case(When(pk__in=num_recent_recipes.values('recipe'), then='viewlog__pk'))), Value(0)))
 
-    def _favorite_recipes(self, timescooked=None):
-        if self._sort_includes('favorite') or timescooked:
-            lessthan = '-' in (timescooked or []) or not self._sort_includes('-favorite')
-            if lessthan:
+    def _favorite_recipes(self):
+        if self._sort_includes('favorite') or self._timescooked or self._timescooked_gte or self._timescooked_lte:
+            less_than = self._timescooked_lte and not self._sort_includes('-favorite')
+            if less_than:
                 default = 1000
             else:
                 default = 0
-            favorite_recipes = CookLog.objects.filter(created_by=self._request.user, space=self._request.space, recipe=OuterRef('pk')
-                                                      ).values('recipe').annotate(count=Count('pk', distinct=True)).values('count')
+            favorite_recipes = (
+                CookLog.objects.filter(created_by=self._request.user, space=self._request.space, recipe=OuterRef('pk'))
+                .values('recipe')
+                .annotate(count=Count('pk', distinct=True))
+                .values('count')
+            )
             self._queryset = self._queryset.annotate(favorite=Coalesce(Subquery(favorite_recipes), default))
-        if timescooked is None:
-            return
 
-        if timescooked == '0':
+        if self._timescooked:
             self._queryset = self._queryset.filter(favorite=0)
-        elif lessthan:
-            self._queryset = self._queryset.filter(favorite__lte=int(timescooked[1:])).exclude(favorite=0)
-        else:
-            self._queryset = self._queryset.filter(favorite__gte=int(timescooked))
+        elif self._timescooked_lte:
+            self._queryset = self._queryset.filter(favorite__lte=int(self._timescooked_lte)).exclude(favorite=0)
+        elif self._timescooked_gte:
+            self._queryset = self._queryset.filter(favorite__gte=int(self._timescooked_gte))
 
     def keyword_filters(self, **kwargs):
         if all([kwargs[x] is None for x in kwargs]):
@@ -385,24 +421,16 @@ class RecipeSearch():
             units = [units]
         self._queryset = self._queryset.filter(steps__ingredients__unit__in=units)
 
-    def rating_filter(self, rating=None):
-        if rating or self._sort_includes('rating'):
-            lessthan = self._sort_includes('-rating') or '-' in (rating or [])
-            if lessthan:
-                default = 100
-            else:
-                default = 0
-            # TODO make ratings a settings user-only vs all-users
-            self._queryset = self._queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=self._request.user, then='cooklog__rating'), default=default))))
-        if rating is None:
-            return
+    def rating_filter(self):
+        if self._rating or self._rating_lte or self._rating_gte or self._sort_includes('rating'):
+            self._queryset = self._queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=self._request.user, then='cooklog__rating'), default=0))))
 
-        if rating == '0':
-            self._queryset = self._queryset.filter(rating=0)
-        elif lessthan:
-            self._queryset = self._queryset.filter(rating__lte=int(rating[1:])).exclude(rating=0)
-        else:
-            self._queryset = self._queryset.filter(rating__gte=int(rating))
+        if self._rating:
+            self._queryset = self._queryset.filter(rating=round(int(self._rating)))
+        elif self._rating_gte:
+            self._queryset = self._queryset.filter(rating__gte=int(self._rating_gte))
+        elif self._rating_lte:
+            self._queryset = self._queryset.filter(rating__gte=int(self._rating_lte)).exclude(rating=0)
 
     def internal_filter(self, internal=None):
         if not internal:
@@ -440,7 +468,7 @@ class RecipeSearch():
         if not steps:
             return
         if not isinstance(steps, list):
-            steps = [unistepsts]
+            steps = [steps]
         self._queryset = self._queryset.filter(steps__id__in=steps)
 
     def build_fulltext_filters(self, string=None):
@@ -497,269 +525,62 @@ class RecipeSearch():
                     trigram += TrigramSimilarity(f, self._string)
                 else:
                     trigram = TrigramSimilarity(f, self._string)
-            self._fuzzy_match = Recipe.objects.annotate(trigram=trigram).distinct(
-            ).annotate(simularity=Max('trigram')).values('id', 'simularity').filter(simularity__gt=self._search_prefs.trigram_threshold)
+            self._fuzzy_match = (
+                Recipe.objects.annotate(trigram=trigram)
+                .distinct()
+                .annotate(simularity=Max('trigram'))
+                .values('id', 'simularity')
+                .filter(simularity__gt=self._search_prefs.trigram_threshold)
+            )
             self._filters += [Q(pk__in=self._fuzzy_match.values('pk'))]
 
     def _makenow_filter(self, missing=None):
-        if missing is None or (type(missing) == bool and missing == False):
+        if missing is None or (isinstance(missing, bool) and missing == False):
             return
         shopping_users = [*self._request.user.get_shopping_share(), self._request.user]
 
         onhand_filter = (
-            Q(steps__ingredients__food__onhand_users__in=shopping_users)  # food onhand
-            | Q(steps__ingredients__food__substitute__onhand_users__in=shopping_users)  # or substitute food onhand
-            | Q(steps__ingredients__food__in=self.__children_substitute_filter(shopping_users))
-            | Q(steps__ingredients__food__in=self.__sibling_substitute_filter(shopping_users))
+                Q(steps__ingredients__food__onhand_users__in=shopping_users)  # food onhand
+                # or substitute food onhand
+                | Q(steps__ingredients__food__substitute__onhand_users__in=shopping_users)
+                | Q(steps__ingredients__food__in=self.__children_substitute_filter(shopping_users))
+                | Q(steps__ingredients__food__in=self.__sibling_substitute_filter(shopping_users))
         )
         makenow_recipes = Recipe.objects.annotate(
             count_food=Count('steps__ingredients__food__pk', filter=Q(steps__ingredients__food__isnull=False), distinct=True),
             count_onhand=Count('steps__ingredients__food__pk', filter=onhand_filter, distinct=True),
-            count_ignore_shopping=Count('steps__ingredients__food__pk', filter=Q(steps__ingredients__food__ignore_shopping=True,
-                                                                                 steps__ingredients__food__recipe__isnull=True), distinct=True),
+            count_ignore_shopping=Count(
+                'steps__ingredients__food__pk', filter=Q(steps__ingredients__food__ignore_shopping=True, steps__ingredients__food__recipe__isnull=True), distinct=True
+            ),
             has_child_sub=Case(When(steps__ingredients__food__in=self.__children_substitute_filter(shopping_users), then=Value(1)), default=Value(0)),
             has_sibling_sub=Case(When(steps__ingredients__food__in=self.__sibling_substitute_filter(shopping_users), then=Value(1)), default=Value(0))
-        ).annotate(missingfood=F('count_food')-F('count_onhand')-F('count_ignore_shopping')).filter(missingfood=missing)
+        ).annotate(missingfood=F('count_food') - F('count_onhand') - F('count_ignore_shopping')).filter(missingfood__lte=missing)
         self._queryset = self._queryset.distinct().filter(id__in=makenow_recipes.values('id'))
 
-    @ staticmethod
+    @staticmethod
     def __children_substitute_filter(shopping_users=None):
-        children_onhand_subquery = Food.objects.filter(
-            path__startswith=Substr(OuterRef('path'), 1, Food.steplen*OuterRef('depth')),
-            depth__gt=OuterRef('depth'),
-            onhand_users__in=shopping_users
-        ).annotate(child_onhand=Coalesce(Func('pk', function='Count'), 0)).values('child_onhand')
-        return Food.objects.exclude(  # list of foods that are onhand and children of: foods that are not onhand and are set to use children as substitutes
-            Q(onhand_users__in=shopping_users)
-            | Q(ignore_shopping=True, recipe__isnull=True)
-            | Q(substitute__onhand_users__in=shopping_users)
-        ).exclude(depth=1, numchild=0).filter(substitute_children=True
-                                              ).annotate(child_onhand=Coalesce(Subquery(children_onhand_subquery), 0)).exclude(child_onhand=0)
-
-    @ staticmethod
-    def __sibling_substitute_filter(shopping_users=None):
-        sibling_onhand_subquery = Food.objects.filter(
-            path__startswith=Substr(OuterRef('path'), 1, Food.steplen*(OuterRef('depth')-1)),
-            depth=OuterRef('depth'),
-            onhand_users__in=shopping_users
-        ).annotate(sibling_onhand=Coalesce(Func('pk', function='Count'), 0)).values('sibling_onhand')
-        return Food.objects.exclude(  # list of foods that are onhand and siblings of: foods that are not onhand and are set to use siblings as substitutes
-            Q(onhand_users__in=shopping_users)
-            | Q(ignore_shopping=True, recipe__isnull=True)
-            | Q(substitute__onhand_users__in=shopping_users)
-        ).exclude(depth=1, numchild=0).filter(substitute_siblings=True
-                                              ).annotate(sibling_onhand=Coalesce(Subquery(sibling_onhand_subquery), 0)).exclude(sibling_onhand=0)
-
-
-class RecipeFacet():
-    class CacheEmpty(Exception):
-        pass
-
-    def __init__(self, request, queryset=None, hash_key=None, cache_timeout=3600):
-        if hash_key is None and queryset is None:
-            raise ValueError(_("One of queryset or hash_key must be provided"))
-
-        self._request = request
-        self._queryset = queryset
-        self.hash_key = hash_key or str(hash(frozenset(self._queryset.values_list('pk'))))
-        self._SEARCH_CACHE_KEY = f"recipes_filter_{self.hash_key}"
-        self._cache_timeout = cache_timeout
-        self._cache = caches['default'].get(self._SEARCH_CACHE_KEY, {})
-        if self._cache is None and self._queryset is None:
-            raise self.CacheEmpty("No queryset provided and cache empty")
-
-        self.Keywords = self._cache.get('Keywords', None)
-        self.Foods = self._cache.get('Foods', None)
-        self.Books = self._cache.get('Books', None)
-        self.Ratings = self._cache.get('Ratings', None)
-        # TODO Move Recent to recipe annotation/serializer:  requrires change in RecipeSearch(), RecipeSearchView.vue and serializer
-        self.Recent = self._cache.get('Recent', None)
-
-        if self._queryset is not None:
-            self._recipe_list = list(self._queryset.values_list('id', flat=True))
-            self._search_params = {
-                'keyword_list': self._request.query_params.getlist('keywords', []),
-                'food_list': self._request.query_params.getlist('foods', []),
-                'book_list': self._request.query_params.getlist('book', []),
-                'search_keywords_or': str2bool(self._request.query_params.get('keywords_or', True)),
-                'search_foods_or': str2bool(self._request.query_params.get('foods_or', True)),
-                'search_books_or': str2bool(self._request.query_params.get('books_or', True)),
-                'space': self._request.space,
-            }
-        elif self.hash_key is not None:
-            self._recipe_list = self._cache.get('recipe_list', [])
-            self._search_params = {
-                'keyword_list': self._cache.get('keyword_list', None),
-                'food_list': self._cache.get('food_list', None),
-                'book_list': self._cache.get('book_list', None),
-                'search_keywords_or': self._cache.get('search_keywords_or', None),
-                'search_foods_or': self._cache.get('search_foods_or', None),
-                'search_books_or': self._cache.get('search_books_or', None),
-                'space': self._cache.get('space', None),
-            }
-
-        self._cache = {
-            **self._search_params,
-            'recipe_list': self._recipe_list,
-            'Ratings': self.Ratings,
-            'Recent': self.Recent,
-            'Keywords': self.Keywords,
-            'Foods': self.Foods,
-            'Books': self.Books
-
-        }
-        caches['default'].set(self._SEARCH_CACHE_KEY, self._cache, self._cache_timeout)
-
-    def get_facets(self, from_cache=False):
-        if from_cache:
-            return {
-                'cache_key': self.hash_key or '',
-                'Ratings': self.Ratings or {},
-                'Recent': self.Recent or [],
-                'Keywords': self.Keywords or [],
-                'Foods': self.Foods or [],
-                'Books': self.Books or []
-            }
-        return {
-            'cache_key': self.hash_key,
-            'Ratings': self.get_ratings(),
-            'Recent': self.get_recent(),
-            'Keywords': self.get_keywords(),
-            'Foods': self.get_foods(),
-            'Books': self.get_books()
-        }
-
-    def set_cache(self, key, value):
-        self._cache = {**self._cache, key: value}
-        caches['default'].set(
-            self._SEARCH_CACHE_KEY,
-            self._cache,
-            self._cache_timeout
+        children_onhand_subquery = Food.objects.filter(path__startswith=OuterRef('path'), depth__gt=OuterRef('depth'), onhand_users__in=shopping_users)
+        return (
+            Food.objects.exclude(  # list of foods that are onhand and children of: foods that are not onhand and are set to use children as substitutes
+                Q(onhand_users__in=shopping_users) | Q(ignore_shopping=True, recipe__isnull=True) | Q(substitute__onhand_users__in=shopping_users)
+            )
+            .exclude(depth=1, numchild=0)
+            .filter(substitute_children=True)
+            .annotate(child_onhand_count=Exists(children_onhand_subquery))
+            .filter(child_onhand_count=True)
         )
 
-    def get_books(self):
-        if self.Books is None:
-            self.Books = []
-        return self.Books
-
-    def get_keywords(self):
-        if self.Keywords is None:
-            if self._search_params['search_keywords_or']:
-                keywords = Keyword.objects.filter(space=self._request.space).distinct()
-            else:
-                keywords = Keyword.objects.filter(Q(recipe__in=self._recipe_list) | Q(depth=1)).filter(space=self._request.space).distinct()
-
-            # set keywords to root objects only
-            keywords = self._keyword_queryset(keywords)
-            self.Keywords = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(keywords)]
-            self.set_cache('Keywords', self.Keywords)
-        return self.Keywords
-
-    def get_foods(self):
-        if self.Foods is None:
-            # # if using an OR search, will annotate all keywords, otherwise, just those that appear in results
-            if self._search_params['search_foods_or']:
-                foods = Food.objects.filter(space=self._request.space).distinct()
-            else:
-                foods = Food.objects.filter(Q(ingredient__step__recipe__in=self._recipe_list) | Q(depth=1)).filter(space=self._request.space).distinct()
-
-            # set keywords to root objects only
-            foods = self._food_queryset(foods)
-
-            self.Foods = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(foods)]
-            self.set_cache('Foods', self.Foods)
-        return self.Foods
-
-    def get_books(self):
-        if self.Books is None:
-            self.Books = []
-        return self.Books
-
-    def get_ratings(self):
-        if self.Ratings is None:
-            if not self._request.space.demo and self._request.space.show_facet_count:
-                if self._queryset is None:
-                    self._queryset = Recipe.objects.filter(id__in=self._recipe_list)
-                rating_qs = self._queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=self._request.user, then='cooklog__rating'), default=Value(0)))))
-                self.Ratings = dict(Counter(r.rating for r in rating_qs))
-            else:
-                self.Rating = {}
-            self.set_cache('Ratings', self.Ratings)
-        return self.Ratings
-
-    def get_recent(self):
-        if self.Recent is None:
-            # TODO make days of recent recipe a setting
-            recent_recipes = ViewLog.objects.filter(created_by=self._request.user, space=self._request.space, created_at__gte=timezone.now() - timedelta(days=14)
-                                                    ).values_list('recipe__pk', flat=True)
-            self.Recent = list(recent_recipes)
-            self.set_cache('Recent', self.Recent)
-        return self.Recent
-
-    def add_food_children(self, id):
-        try:
-            food = Food.objects.get(id=id)
-            nodes = food.get_ancestors()
-        except Food.DoesNotExist:
-            return self.get_facets()
-        foods = self._food_queryset(food.get_children(), food)
-        deep_search = self.Foods
-        for node in nodes:
-            index = next((i for i, x in enumerate(deep_search) if x["id"] == node.id), None)
-            deep_search = deep_search[index]['children']
-        index = next((i for i, x in enumerate(deep_search) if x["id"] == food.id), None)
-        deep_search[index]['children'] = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(foods)]
-        self.set_cache('Foods', self.Foods)
-        return self.get_facets()
-
-    def add_keyword_children(self, id):
-        try:
-            keyword = Keyword.objects.get(id=id)
-            nodes = keyword.get_ancestors()
-        except Keyword.DoesNotExist:
-            return self.get_facets()
-        keywords = self._keyword_queryset(keyword.get_children(), keyword)
-        deep_search = self.Keywords
-        for node in nodes:
-            index = next((i for i, x in enumerate(deep_search) if x["id"] == node.id), None)
-            deep_search = deep_search[index]['children']
-        index = next((i for i, x in enumerate(deep_search) if x["id"] == keyword.id), None)
-        deep_search[index]['children'] = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(keywords)]
-        self.set_cache('Keywords', self.Keywords)
-        return self.get_facets()
-
-    def _recipe_count_queryset(self, field, depth=1, steplen=4):
-        return Recipe.objects.filter(**{f'{field}__path__startswith': OuterRef('path'), f'{field}__depth__gte': depth}, id__in=self._recipe_list, space=self._request.space
-                                     ).annotate(count=Coalesce(Func('pk', function='Count'), 0)).values('count')
-
-    def _keyword_queryset(self, queryset, keyword=None):
-        depth = getattr(keyword, 'depth', 0) + 1
-        steplen = depth * Keyword.steplen
-
-        if not self._request.space.demo and self._request.space.show_facet_count:
-            return queryset.annotate(count=Coalesce(Subquery(self._recipe_count_queryset('keywords', depth, steplen)), 0)
-                                     ).filter(depth=depth, count__gt=0
-                                              ).values('id', 'name', 'count', 'numchild').order_by(Lower('name').asc())[:200]
-        else:
-            return queryset.filter(depth=depth).values('id', 'name',  'numchild').order_by(Lower('name').asc())
-
-    def _food_queryset(self, queryset, food=None):
-        depth = getattr(food, 'depth', 0) + 1
-        steplen = depth * Food.steplen
-
-        if not self._request.space.demo and self._request.space.show_facet_count:
-            return queryset.annotate(count=Coalesce(Subquery(self._recipe_count_queryset('steps__ingredients__food', depth, steplen)), 0)
-                                     ).filter(depth__lte=depth, count__gt=0
-                                              ).values('id', 'name', 'count', 'numchild').order_by(Lower('name').asc())[:200]
-        else:
-            return queryset.filter(depth__lte=depth).values('id', 'name', 'numchild').order_by(Lower('name').asc())
-
-
-def old_search(request):
-    if has_group_permission(request.user, ('guest',)):
-        params = dict(request.GET)
-        params['internal'] = None
-        f = RecipeFilter(params,
-                         queryset=Recipe.objects.filter(space=request.user.userpreference.space).all().order_by(Lower('name').asc()),
-                         space=request.space)
-        return f.qs
+    @staticmethod
+    def __sibling_substitute_filter(shopping_users=None):
+        sibling_onhand_subquery = Food.objects.filter(
+            path__startswith=Substr(OuterRef('path'), 1, Food.steplen * (OuterRef('depth') - 1)), depth=OuterRef('depth'), onhand_users__in=shopping_users
+        )
+        return (
+            Food.objects.exclude(  # list of foods that are onhand and siblings of: foods that are not onhand and are set to use siblings as substitutes
+                Q(onhand_users__in=shopping_users) | Q(ignore_shopping=True, recipe__isnull=True) | Q(substitute__onhand_users__in=shopping_users)
+            )
+            .exclude(depth=1, numchild=0)
+            .filter(substitute_siblings=True)
+            .annotate(sibling_onhand=Exists(sibling_onhand_subquery))
+            .filter(sibling_onhand=True)
+        )

cookbook/helper/recipe_url_import.py

@@ -1,27 +1,46 @@
-import random
 import re
+import traceback
 from html import unescape
-from unicodedata import decomposition
 
 from django.utils.dateparse import parse_duration
 from django.utils.translation import gettext as _
 from isodate import parse_duration as iso_parse_duration
 from isodate.isoerror import ISO8601Error
-from recipe_scrapers._utils import get_minutes
+from pytubefix import YouTube
+from recipe_scrapers._utils import get_host_name, get_minutes
 
-from cookbook.helper import recipe_url_import as helper
+from cookbook.helper.automation_helper import AutomationEngine
 from cookbook.helper.ingredient_parser import IngredientParser
-from cookbook.models import Keyword
-
-
-# from recipe_scrapers._utils import get_minutes  ## temporary until/unless upstream incorporates get_minutes() PR
+from cookbook.models import Automation, Keyword, PropertyType
 
 
 def get_from_scraper(scrape, request):
-    # converting the scrape_me object to the existing json format based on ld+json
-    recipe_json = {}
+    # converting the scrape_html object to the existing json format based on ld+json
+
+    recipe_json = {'steps': [], 'internal': True}
+    keywords = []
+
+    # assign source URL
     try:
-        recipe_json['name'] = parse_name(scrape.title() or None)
+        source_url = scrape.canonical_url()
+    except Exception:
+        try:
+            source_url = scrape.url
+        except Exception:
+            pass
+    if source_url == "https://urlnotfound.none" or not source_url:
+        recipe_json['source_url'] = ''
+    else:
+        recipe_json['source_url'] = source_url
+        try:
+            keywords.append(source_url.replace('http://', '').replace('https://', '').split('/')[0])
+        except Exception:
+            recipe_json['source_url'] = ''
+
+    automation_engine = AutomationEngine(request, source=recipe_json.get('source_url'))
+    # assign recipe name
+    try:
+        recipe_json['name'] = parse_name(scrape.title()[:128] or None)
     except Exception:
         recipe_json['name'] = None
     if not recipe_json['name']:
@@ -30,6 +49,13 @@ def get_from_scraper(scrape, request):
         except Exception:
             recipe_json['name'] = ''
 
+    if isinstance(recipe_json['name'], list) and len(recipe_json['name']) > 0:
+        recipe_json['name'] = recipe_json['name'][0]
+
+    recipe_json['name'] = automation_engine.apply_regex_replace_automation(recipe_json['name'], Automation.NAME_REPLACE)
+
+    # assign recipe description
+    # TODO notify user about limit if reached - >256 description will be truncated
     try:
         description = scrape.description() or None
     except Exception:
@@ -40,21 +66,20 @@ def get_from_scraper(scrape, request):
         except Exception:
             description = ''
 
-    recipe_json['internal'] = True
+    recipe_json['description'] = parse_description(description)
+    recipe_json['description'] = automation_engine.apply_regex_replace_automation(recipe_json['description'], Automation.DESCRIPTION_REPLACE)
 
+    # assign servings attributes
     try:
-        servings = scrape.yields() or None
+        # dont use scrape.yields() as this will always return "x servings" or "x items", should be improved in scrapers directly
+        servings = scrape.schema.data.get('recipeYield') or 1
     except Exception:
-        servings = None
-    if not servings:
-        try:
-            servings = scrape.schema.data.get('recipeYield') or 1
-        except Exception:
-            servings = 1
+        servings = 1
 
     recipe_json['servings'] = parse_servings(servings)
     recipe_json['servings_text'] = parse_servings_text(servings)
 
+    # assign time attributes
     try:
         recipe_json['working_time'] = get_minutes(scrape.prep_time()) or 0
     except Exception:
@@ -79,17 +104,18 @@ def get_from_scraper(scrape, request):
             except Exception:
                 pass
 
+    # assign image
     try:
-        recipe_json['image'] = parse_image(scrape.image()) or None
+        recipe_json['image_url'] = parse_image(scrape.image()) or None
     except Exception:
-        recipe_json['image'] = None
-    if not recipe_json['image']:
+        recipe_json['image_url'] = None
+    if not recipe_json['image_url']:
         try:
-            recipe_json['image'] = parse_image(scrape.schema.data.get('image')) or ''
+            recipe_json['image_url'] = parse_image(scrape.schema.data.get('image')) or ''
         except Exception:
-            recipe_json['image'] = ''
+            recipe_json['image_url'] = ''
 
-    keywords = []
+    # assign keywords
     try:
         if scrape.schema.data.get("keywords"):
             keywords += listify_keywords(scrape.schema.data.get("keywords"))
@@ -115,59 +141,59 @@ def get_from_scraper(scrape, request):
             pass
 
     try:
-        source_url = scrape.canonical_url()
+        if scrape.author():
+            keywords.append(scrape.author())
     except Exception:
-        try: 
-            source_url = scrape.url
-        except Exception:
-            pass
-    if source_url:
-        recipe_json['source_url'] = source_url
-        try:
-            keywords.append(source_url.replace('http://', '').replace('https://', '').split('/')[0])
-        except Exception:
-            pass
+        pass
 
     try:
-        recipe_json['keywords'] = parse_keywords(list(set(map(str.casefold, keywords))), request.space)
+        recipe_json['keywords'] = parse_keywords(list(set(map(str.casefold, keywords))), request)
     except AttributeError:
         recipe_json['keywords'] = keywords
 
     ingredient_parser = IngredientParser(request, True)
 
-    recipe_json['steps'] = []
+    # assign steps
     try:
         for i in parse_instructions(scrape.instructions()):
-            recipe_json['steps'].append({'instruction': i, 'ingredients': [], })
+            recipe_json['steps'].append({
+                'instruction': i,
+                'ingredients': [],
+                'show_ingredients_table': request.user.userpreference.show_step_ingredients,
+            })
     except Exception:
         pass
     if len(recipe_json['steps']) == 0:
-        recipe_json['steps'].append({'instruction': '', 'ingredients': [], })
+        recipe_json['steps'].append({
+            'instruction': '',
+            'ingredients': [],
+        })
 
-    if len(parse_description(description)) > 256:  # split at 256 as long descriptions dont look good on recipe cards
-        recipe_json['steps'][0]['instruction'] = f'*{parse_description(description)}*  \n\n' + recipe_json['steps'][0]['instruction']
-    else:
-        recipe_json['description'] = parse_description(description)[:512]
+    recipe_json['description'] = recipe_json['description'][:512]
+    if len(recipe_json['description']) > 256:  # split at 256 as long descriptions don't look good on recipe cards
+        recipe_json['steps'][0]['instruction'] = f"*{recipe_json['description']}*  \n\n" + recipe_json['steps'][0]['instruction']
 
     try:
         for x in scrape.ingredients():
-            try:
-                amount, unit, ingredient, note = ingredient_parser.parse(x)
-                ingredient = {
-                    'amount': amount,
-                    'food': {
-                        'name': ingredient,
-                    },
-                    'unit': None,
-                    'note': note,
-                    'original_text': x
-                }
-                if unit:
-                    ingredient['unit'] = {'name': unit, }
-                recipe_json['steps'][0]['ingredients'].append(ingredient)
-            except Exception:
-                recipe_json['steps'][0]['ingredients'].append(
-                    {
+            if x.strip() != '':
+                try:
+                    amount, unit, ingredient, note = ingredient_parser.parse(x)
+                    ingredient = {
+                        'amount': amount,
+                        'food': {
+                            'name': ingredient,
+                        },
+                        'unit': None,
+                        'note': note,
+                        'original_text': x
+                    }
+                    if unit:
+                        ingredient['unit'] = {
+                            'name': unit,
+                        }
+                    recipe_json['steps'][0]['ingredients'].append(ingredient)
+                except Exception:
+                    recipe_json['steps'][0]['ingredients'].append({
                         'amount': 0,
                         'unit': None,
                         'food': {
@@ -175,16 +201,95 @@ def get_from_scraper(scrape, request):
                         },
                         'note': '',
                         'original_text': x
-                    }
-                )
+                    })
     except Exception:
         pass
 
+    recipe_json['properties'] = []
+    try:
+        recipe_json['properties'] = get_recipe_properties(request.space, scrape.schema.nutrients())
+        print(recipe_json['properties'])
+    except Exception:
+        traceback.print_exc()
+        pass
+
+    for s in recipe_json['steps']:
+        s['instruction'] = automation_engine.apply_regex_replace_automation(s['instruction'], Automation.INSTRUCTION_REPLACE)
+        # re.sub(a.param_2, a.param_3, s['instruction'])
+
     return recipe_json
 
 
+def get_recipe_properties(space, property_data):
+    # {'servingSize': '1', 'calories': '302 kcal', 'proteinContent': '7,66g', 'fatContent': '11,56g', 'carbohydrateContent': '41,33g'}
+    properties = {
+        "property-calories": "calories",
+        "property-carbohydrates": "carbohydrateContent",
+        "property-proteins": "proteinContent",
+        "property-fats": "fatContent",
+    }
+
+    serving_size = 1
+    try:
+        serving_size = parse_servings(property_data['servingSize'])
+    except KeyError:
+        pass
+
+    recipe_properties = []
+    for pt in PropertyType.objects.filter(space=space, open_data_slug__in=list(properties.keys())).all():
+        for p in list(properties.keys()):
+            if pt.open_data_slug == p:
+                if properties[p] in property_data:
+                    recipe_properties.append({
+                        'property_type': {
+                            'id': pt.id,
+                            'name': pt.name,
+                        },
+                        'property_amount': parse_servings(property_data[properties[p]]) / serving_size,
+                    })
+
+    return recipe_properties
+
+
+def get_from_youtube_scraper(url, request):
+    """A YouTube Information Scraper."""
+    kw, created = Keyword.objects.get_or_create(name='YouTube', space=request.space)
+    default_recipe_json = {
+        'name': '',
+        'internal': True,
+        'description': '',
+        'servings': 1,
+        'working_time': 0,
+        'waiting_time': 0,
+        'image': "",
+        'keywords': [{
+            'name': kw.name,
+            'label': kw.name,
+            'id': kw.pk
+        }],
+        'source_url': url,
+        'steps': [{
+            'ingredients': [],
+            'instruction': ''
+        }]
+    }
+
+    try:
+        automation_engine = AutomationEngine(request, source=url)
+        video = YouTube(url)
+        video.streams.first()  # this is required to execute some kind of generator/web request that fetches the description
+        default_recipe_json['name'] = automation_engine.apply_regex_replace_automation(video.title, Automation.NAME_REPLACE)
+        default_recipe_json['image'] = video.thumbnail_url
+        if video.description:
+            default_recipe_json['steps'][0]['instruction'] = automation_engine.apply_regex_replace_automation(video.description, Automation.INSTRUCTION_REPLACE)
+    except Exception:
+        traceback.print_exc()
+
+    return default_recipe_json
+
+
 def parse_name(name):
-    if type(name) == list:
+    if isinstance(name, list):
         try:
             name = name[0]
         except Exception:
@@ -197,10 +302,27 @@ def parse_description(description):
 
 
 def clean_instruction_string(instruction):
-    normalized_string = normalize_string(instruction)
+    # handle HTML tags that can be converted to markup
+    normalized_string = instruction \
+        .replace("<nobr>", "**") \
+        .replace("</nobr>", "**") \
+        .replace("<strong>", "**") \
+        .replace("</strong>", "**")
+    normalized_string = normalize_string(normalized_string)
     normalized_string = normalized_string.replace('\n', '  \n')
     normalized_string = normalized_string.replace('  \n  \n', '\n\n')
-    return normalized_string
+
+    # handle unsupported, special UTF8 character in Thermomix-specific instructions,
+    # that happen in nearly every recipe on Cookidoo, Zaubertopf Club, Rezeptwelt
+    # and in Thermomix-specific recipes on many other sites
+    return normalized_string \
+        .replace("", _('reverse rotation')) \
+        .replace("", _('careful rotation')) \
+        .replace("", _('knead')) \
+        .replace("Andicken ", _('thicken')) \
+        .replace("Erwärmen ", _('warm up')) \
+        .replace("Fermentieren ", _('ferment')) \
+        .replace("Sous-vide ", _("sous-vide"))
 
 
 def parse_instructions(instructions):
@@ -211,16 +333,16 @@ def parse_instructions(instructions):
     """
     instruction_list = []
 
-    if type(instructions) == list:
+    if isinstance(instructions, list):
         for i in instructions:
-            if type(i) == str:
+            if isinstance(i, str):
                 instruction_list.append(clean_instruction_string(i))
             else:
                 if 'text' in i:
                     instruction_list.append(clean_instruction_string(i['text']))
                 elif 'itemListElement' in i:
                     for ile in i['itemListElement']:
-                        if type(ile) == str:
+                        if isinstance(ile, str):
                             instruction_list.append(clean_instruction_string(ile))
                         elif 'text' in ile:
                             instruction_list.append(clean_instruction_string(ile['text']))
@@ -236,13 +358,13 @@ def parse_image(image):
     # check if list of images is returned, take first if so
     if not image:
         return None
-    if type(image) == list:
+    if isinstance(image, list):
         for pic in image:
-            if (type(pic) == str) and (pic[:4] == 'http'):
+            if (isinstance(pic, str)) and (pic[:4] == 'http'):
                 image = pic
             elif 'url' in pic:
                 image = pic['url']
-    elif type(image) == dict:
+    elif isinstance(image, dict):
         if 'url' in image:
             image = image['url']
 
@@ -253,26 +375,31 @@ def parse_image(image):
 
 
 def parse_servings(servings):
-    if type(servings) == str:
+    if isinstance(servings, str):
         try:
             servings = int(re.search(r'\d+', servings).group())
         except AttributeError:
             servings = 1
-    elif type(servings) == list:
+    elif isinstance(servings, list):
         try:
-            servings = int(re.findall(r'\b\d+\b', servings[0])[0])
-        except KeyError:
+            servings = int(re.findall(r'\b\d+\b', str(servings[0]))[0])
+        except (KeyError, IndexError):
             servings = 1
     return servings
 
 
 def parse_servings_text(servings):
-    if type(servings) == str:
+    if isinstance(servings, str):
         try:
-            servings = re.sub("\d+", '', servings).strip()
+            servings = re.sub("\\d+", '', servings).strip()
         except Exception:
             servings = ''
-    return servings
+    if isinstance(servings, list):
+        try:
+            servings = parse_servings_text(servings[1])
+        except Exception:
+            pass
+    return str(servings)[:32]
 
 
 def parse_time(recipe_time):
@@ -284,7 +411,7 @@ def parse_time(recipe_time):
                 recipe_time = round(iso_parse_duration(recipe_time).seconds / 60)
             except ISO8601Error:
                 try:
-                    if (type(recipe_time) == list and len(recipe_time) > 0):
+                    if (isinstance(recipe_time, list) and len(recipe_time) > 0):
                         recipe_time = recipe_time[0]
                     recipe_time = round(parse_duration(recipe_time).seconds / 60)
                 except AttributeError:
@@ -293,16 +420,21 @@ def parse_time(recipe_time):
     return recipe_time
 
 
-def parse_keywords(keyword_json, space):
+def parse_keywords(keyword_json, request):
     keywords = []
+    automation_engine = AutomationEngine(request)
+
     # keywords as list
     for kw in keyword_json:
         kw = normalize_string(kw)
+        # if alias exists use that instead
+
         if len(kw) != 0:
-            if k := Keyword.objects.filter(name=kw, space=space).first():
-                keywords.append({'label': str(k), 'name': k.name, 'id': k.id})
+            kw = automation_engine.apply_keyword_automation(kw)
+            if k := Keyword.objects.filter(name__iexact=kw, space=request.space).first():
+                keywords.append({'label': str(k), 'name': k.name, 'id': k.id, 'import_keyword': True})
             else:
-                keywords.append({'label': kw, 'name': kw})
+                keywords.append({'label': kw, 'name': kw, 'import_keyword': False})
 
     return keywords
 
@@ -310,15 +442,15 @@ def parse_keywords(keyword_json, space):
 def listify_keywords(keyword_list):
     # keywords as string
     try:
-        if type(keyword_list[0]) == dict:
+        if isinstance(keyword_list[0], dict):
             return keyword_list
     except (KeyError, IndexError):
         pass
-    if type(keyword_list) == str:
+    if isinstance(keyword_list, str):
         keyword_list = keyword_list.split(',')
 
     # keywords as string in list
-    if (type(keyword_list) == list and len(keyword_list) == 1 and ',' in keyword_list[0]):
+    if (isinstance(keyword_list, list) and len(keyword_list) == 1 and ',' in keyword_list[0]):
         keyword_list = keyword_list[0].split(',')
     return [x.strip() for x in keyword_list]
 
@@ -335,8 +467,53 @@ def normalize_string(string):
 
 
 def iso_duration_to_minutes(string):
-    match = re.match(
-        r'P((?P<years>\d+)Y)?((?P<months>\d+)M)?((?P<weeks>\d+)W)?((?P<days>\d+)D)?T((?P<hours>\d+)H)?((?P<minutes>\d+)M)?((?P<seconds>\d+)S)?',
-        string
-    ).groupdict()
+    match = re.match(r'P((?P<years>\d+)Y)?((?P<months>\d+)M)?((?P<weeks>\d+)W)?((?P<days>\d+)D)?T((?P<hours>\d+)H)?((?P<minutes>\d+)M)?((?P<seconds>\d+)S)?', string).groupdict()
     return int(match['days'] or 0) * 24 * 60 + int(match['hours'] or 0) * 60 + int(match['minutes'] or 0)
+
+
+def get_images_from_soup(soup, url):
+    sources = ['src', 'srcset', 'data-src']
+    images = []
+    img_tags = soup.find_all('img')
+    if url:
+        site = get_host_name(url)
+        prot = url.split(':')[0]
+
+    urls = []
+    for img in img_tags:
+        for src in sources:
+            try:
+                urls.append(img[src])
+            except KeyError:
+                pass
+
+    for u in urls:
+        u = u.split('?')[0]
+        filename = re.search(r'/([\w_-]+[.](jpg|jpeg|gif|png))$', u)
+        if filename:
+            if u.startswith('//'):
+                # urls from e.g. ottolenghi.co.uk start with //
+                u = 'https:' + u
+            if ('http' not in u) and url:
+                print(f'rewriting URL {u}')
+                # sometimes an image source can be relative
+                # if it is provide the base url
+                u = '{}://{}{}'.format(prot, site, u)
+            if 'http' in u:
+                images.append(u)
+    return images
+
+
+def clean_dict(input_dict, key):
+    if isinstance(input_dict, dict):
+        for x in list(input_dict):
+            if x == key:
+                del input_dict[x]
+            elif isinstance(input_dict[x], dict):
+                input_dict[x] = clean_dict(input_dict[x], key)
+            elif isinstance(input_dict[x], list):
+                temp_list = []
+                for e in input_dict[x]:
+                    temp_list.append(clean_dict(e, key))
+
+    return input_dict

cookbook/helper/scope_middleware.py

@@ -1,7 +1,6 @@
 from django.urls import reverse
 from django_scopes import scope, scopes_disabled
-from rest_framework.authentication import TokenAuthentication
-from rest_framework.authtoken.models import Token
+from oauth2_provider.contrib.rest_framework import OAuth2Authentication
 from rest_framework.exceptions import AuthenticationFailed
 
 from cookbook.views import views
@@ -13,7 +12,13 @@ class ScopeMiddleware:
         self.get_response = get_response
 
     def __call__(self, request):
-        prefix = settings.JS_REVERSE_SCRIPT_PREFIX or ''
+        prefix = settings.SCRIPT_NAME or ''
+
+        # need to disable scopes for writing requests into userpref and enable for loading ?
+        if request.path.startswith(prefix + '/api/user-preference/'):
+            with scopes_disabled():
+                return self.get_response(request)
+
         if request.user.is_authenticated:
 
             if request.path.startswith(prefix + '/admin/'):
@@ -26,24 +31,34 @@ class ScopeMiddleware:
             if request.path.startswith(prefix + '/accounts/'):
                 return self.get_response(request)
 
-            with scopes_disabled():
-                if request.user.userpreference.space is None and not reverse('account_logout') in request.path:
-                    return views.no_space(request)
+            if request.path.startswith(prefix + '/switch-space/'):
+                return self.get_response(request)
 
-            if request.user.groups.count() == 0 and not reverse('account_logout') in request.path:
+            with scopes_disabled():
+                if request.user.userspace_set.count() == 0 and not reverse('account_logout') in request.path:
+                    return views.space_overview(request)
+
+            # get active user space, if for some reason more than one space is active select first (group permission checks will fail, this is not intended at this point)
+            user_space = request.user.userspace_set.filter(active=True).first()
+
+            if not user_space:
+                return views.space_overview(request)
+
+            if user_space.groups.count() == 0 and not reverse('account_logout') in request.path:
                 return views.no_groups(request)
 
-            request.space = request.user.userpreference.space
-            # with scopes_disabled():
+            request.space = user_space.space
             with scope(space=request.space):
                 return self.get_response(request)
         else:
             if request.path.startswith(prefix + '/api/'):
                 try:
-                    if auth := TokenAuthentication().authenticate(request):
-                        request.space = auth[0].userpreference.space
-                        with scope(space=request.space):
-                            return self.get_response(request)
+                    if auth := OAuth2Authentication().authenticate(request):
+                        user_space = auth[0].userspace_set.filter(active=True).first()
+                        if user_space:
+                            request.space = user_space.space
+                            with scope(space=request.space):
+                                return self.get_response(request)
                 except AuthenticationFailed:
                     pass
 

cookbook/helper/scrapers/cooksillustrated.py

@@ -1,68 +0,0 @@
-import json
-from recipe_scrapers._abstract import AbstractScraper
-
-
-class CooksIllustrated(AbstractScraper):
-    @classmethod
-    def host(cls, site='cooksillustrated'):
-        return {
-            'cooksillustrated': f"{site}.com",
-            'americastestkitchen': f"{site}.com",
-            'cookscountry': f"{site}.com",
-        }.get(site)
-
-    def title(self):
-        return self.schema.title()
-
-    def image(self):
-        return self.schema.image()
-
-    def total_time(self):
-        if not self.recipe:
-            self.get_recipe()
-        return self.recipe['recipeTimeNote']
-
-    def yields(self):
-        if not self.recipe:
-            self.get_recipe()
-        return self.recipe['yields']
-
-    def ingredients(self):
-        if not self.recipe:
-            self.get_recipe()
-        ingredients = []
-        for group in self.recipe['ingredientGroups']:
-            ingredients += group['fields']['recipeIngredientItems']
-        return [
-            "{} {} {}{}".format(
-                i['fields']['qty'] or '',
-                i['fields']['measurement'] or '',
-                i['fields']['ingredient']['fields']['title'] or '',
-                i['fields']['postText'] or ''
-            )
-            for i in ingredients
-        ]
-
-    def instructions(self):
-        if not self.recipe:
-            self.get_recipe()
-        if self.recipe.get('headnote', False):
-            i = ['Note: ' + self.recipe.get('headnote', '')]
-        else:
-            i = []
-        return "\n".join(
-            i
-            + [self.recipe.get('whyThisWorks', '')]
-            + [
-                instruction['fields']['content']
-                for instruction in self.recipe['instructions']
-            ]
-        )
-
-    def nutrients(self):
-        raise NotImplementedError("This should be implemented.")
-
-    def get_recipe(self):
-        j = json.loads(self.soup.find(type='application/json').string)
-        name = list(j['props']['initialState']['content']['documents'])[0]
-        self.recipe = j['props']['initialState']['content']['documents'][name]

cookbook/helper/scrapers/scrapers.py

@@ -1,36 +0,0 @@
-from bs4 import BeautifulSoup
-from json import JSONDecodeError
-from recipe_scrapers import SCRAPERS 
-from recipe_scrapers._factory import SchemaScraperFactory
-from recipe_scrapers._schemaorg import SchemaOrg
-
-from .cooksillustrated import CooksIllustrated
-
-CUSTOM_SCRAPERS = {
-    CooksIllustrated.host(site="cooksillustrated"): CooksIllustrated,
-    CooksIllustrated.host(site="americastestkitchen"): CooksIllustrated,
-    CooksIllustrated.host(site="cookscountry"): CooksIllustrated,
-}
-SCRAPERS.update(CUSTOM_SCRAPERS)
-
-
-def text_scraper(text, url=None):
-    scraper_class = SchemaScraperFactory.SchemaScraper
-
-    class TextScraper(scraper_class):
-        def __init__(
-            self,
-            page_data,
-            url=None
-        ):
-            self.wild_mode = False
-            self.meta_http_equiv = False
-            self.soup = BeautifulSoup(page_data, "html.parser")
-            self.url = url
-            self.recipe = None
-            try:
-                self.schema = SchemaOrg(page_data)
-            except (JSONDecodeError, AttributeError):
-                pass
-
-    return TextScraper(text, url)

cookbook/helper/shopping_helper.py

@@ -1,16 +1,12 @@
-from datetime import timedelta
+
 from decimal import Decimal
 
-from django.contrib.postgres.aggregates import ArrayAgg
 from django.db.models import F, OuterRef, Q, Subquery, Value
 from django.db.models.functions import Coalesce
-from django.utils import timezone
 from django.utils.translation import gettext as _
 
-from cookbook.helper.HelperFunctions import Round, str2bool
 from cookbook.models import (Ingredient, MealPlan, Recipe, ShoppingListEntry, ShoppingListRecipe,
                              SupermarketCategoryRelation)
-from recipes import settings
 
 
 def shopping_helper(qs, request):
@@ -30,9 +26,6 @@ def shopping_helper(qs, request):
     elif checked in ['true', 1, '1']:
         qs = qs.filter(checked=True)
     elif checked in ['recent']:
-        today_start = timezone.now().replace(hour=0, minute=0, second=0)
-        week_ago = today_start - timedelta(days=user.userpreference.shopping_recent_days)
-        qs = qs.filter(Q(checked=False) | Q(completed_at__gte=week_ago))
         supermarket_order = ['checked'] + supermarket_order
 
     return qs.distinct().order_by(*supermarket_order).select_related('unit', 'food', 'ingredient', 'created_by', 'list_recipe', 'list_recipe__mealplan', 'list_recipe__recipe')
@@ -47,6 +40,8 @@ class RecipeShoppingEditor():
         self.mealplan = self._kwargs.get('mealplan', None)
         if type(self.mealplan) in [int, float]:
             self.mealplan = MealPlan.objects.filter(id=self.mealplan, space=self.space)
+        if isinstance(self.mealplan, dict):
+            self.mealplan = MealPlan.objects.filter(id=self.mealplan['id'], space=self.space).first()
         self.id = self._kwargs.get('id', None)
 
         self._shopping_list_recipe = self.get_shopping_list_recipe(self.id, self.created_by, self.space)
@@ -67,11 +62,12 @@ class RecipeShoppingEditor():
 
     @property
     def _recipe_servings(self):
-        return getattr(self.recipe, 'servings', None) or getattr(getattr(self.mealplan, 'recipe', None), 'servings', None) or getattr(getattr(self._shopping_list_recipe, 'recipe', None), 'servings', None)
+        return getattr(self.recipe, 'servings', None) or getattr(getattr(self.mealplan, 'recipe', None), 'servings',
+                                                                 None) or getattr(getattr(self._shopping_list_recipe, 'recipe', None), 'servings', None)
 
     @property
     def _servings_factor(self):
-        return Decimal(self.servings)/Decimal(self._recipe_servings)
+        return Decimal(self.servings) / Decimal(self._recipe_servings)
 
     @property
     def _shared_users(self):
@@ -79,18 +75,17 @@ class RecipeShoppingEditor():
 
     @staticmethod
     def get_shopping_list_recipe(id, user, space):
-        return ShoppingListRecipe.objects.filter(id=id).filter(Q(shoppinglist__space=space) | Q(entries__space=space)).filter(
-            Q(shoppinglist__created_by=user)
-            | Q(shoppinglist__shared=user)
-            | Q(entries__created_by=user)
+        return ShoppingListRecipe.objects.filter(id=id).filter(entries__space=space).filter(
+            Q(entries__created_by=user)
             | Q(entries__created_by__in=list(user.get_shopping_share()))
         ).prefetch_related('entries').first()
 
     def get_recipe_ingredients(self, id, exclude_onhand=False):
         if exclude_onhand:
-            return Ingredient.objects.filter(step__recipe__id=id,  food__ignore_shopping=False,  space=self.space).exclude(food__onhand_users__id__in=[x.id for x in self._shared_users])
+            return Ingredient.objects.filter(step__recipe__id=id, food__ignore_shopping=False, space=self.space).exclude(
+                food__onhand_users__id__in=[x.id for x in self._shared_users])
         else:
-            return Ingredient.objects.filter(step__recipe__id=id,  food__ignore_shopping=False,  space=self.space)
+            return Ingredient.objects.filter(step__recipe__id=id, food__ignore_shopping=False, space=self.space)
 
     @property
     def _include_related(self):
@@ -107,7 +102,10 @@ class RecipeShoppingEditor():
             self.servings = float(servings)
 
         if mealplan := kwargs.get('mealplan', None):
-            self.mealplan = mealplan
+            if isinstance(mealplan, dict):
+                self.mealplan = MealPlan.objects.filter(id=mealplan['id'], space=self.space).first()
+            else:
+                self.mealplan = mealplan
             self.recipe = mealplan.recipe
         elif recipe := kwargs.get('recipe', None):
             self.recipe = recipe
@@ -115,7 +113,7 @@ class RecipeShoppingEditor():
         if not self.servings:
             self.servings = getattr(self.mealplan, 'servings', None) or getattr(self.recipe, 'servings', 1.0)
 
-        self._shopping_list_recipe = ShoppingListRecipe.objects.create(recipe=self.recipe, mealplan=self.mealplan, servings=self.servings)
+        self._shopping_list_recipe = ShoppingListRecipe.objects.create(recipe=self.recipe, mealplan=self.mealplan, servings=self.servings, space=self.space, created_by=self.created_by)
 
         if ingredients:
             self._add_ingredients(ingredients=ingredients)
@@ -155,8 +153,9 @@ class RecipeShoppingEditor():
             return True
 
         for sle in ShoppingListEntry.objects.filter(list_recipe=self._shopping_list_recipe):
-            sle.amount = sle.ingredient.amount * Decimal(self._servings_factor)
-            sle.save()
+            if sle.ingredient: # TODO temporarily dont scale manual entries until ingredient_amount or some other base amount has been migrated to SLE
+                sle.amount = sle.ingredient.amount * Decimal(self._servings_factor)
+                sle.save()
         self._shopping_list_recipe.servings = self.servings
         self._shopping_list_recipe.save()
         return True
@@ -165,14 +164,14 @@ class RecipeShoppingEditor():
         try:
             self._shopping_list_recipe.delete()
             return True
-        except:
+        except BaseException:
             return False
 
     def _add_ingredients(self, ingredients=None):
         if not ingredients:
             return
-        elif type(ingredients) == list:
-            ingredients = Ingredient.objects.filter(id__in=ingredients)
+        elif isinstance(ingredients, list):
+            ingredients = Ingredient.objects.filter(id__in=ingredients, food__ignore_shopping=False)
         existing = self._shopping_list_recipe.entries.filter(ingredient__in=ingredients).values_list('ingredient__pk', flat=True)
         add_ingredients = ingredients.exclude(id__in=existing)
 
@@ -194,120 +193,3 @@ class RecipeShoppingEditor():
         to_delete = self._shopping_list_recipe.entries.exclude(ingredient__in=ingredients)
         ShoppingListEntry.objects.filter(id__in=to_delete).delete()
         self._shopping_list_recipe = self.get_shopping_list_recipe(self.id, self.created_by, self.space)
-
-
-# # TODO refactor as class
-# def list_from_recipe(list_recipe=None, recipe=None, mealplan=None, servings=None, ingredients=None, created_by=None, space=None, append=False):
-#     """
-#     Creates ShoppingListRecipe and associated ShoppingListEntrys from a recipe or a meal plan with a recipe
-#     :param list_recipe: Modify an existing ShoppingListRecipe
-#     :param recipe: Recipe to use as list of ingredients.  One of [recipe, mealplan] are required
-#     :param mealplan: alternatively use a mealplan recipe as source of ingredients
-#     :param servings: Optional: Number of servings to use to scale shoppinglist.  If servings = 0 an existing recipe list will be deleted
-#     :param ingredients: Ingredients, list of ingredient IDs to include on the shopping list.  When not provided all ingredients will be used
-#     :param append: If False will remove any entries not included with ingredients, when True will append ingredients to the shopping list
-#     """
-#     r = recipe or getattr(mealplan, 'recipe', None) or getattr(list_recipe, 'recipe', None)
-#     if not r:
-#         raise ValueError(_("You must supply a recipe or mealplan"))
-
-#     created_by = created_by or getattr(ShoppingListEntry.objects.filter(list_recipe=list_recipe).first(), 'created_by', None)
-#     if not created_by:
-#         raise ValueError(_("You must supply a created_by"))
-
-#     try:
-#         servings = float(servings)
-#     except (ValueError, TypeError):
-#         servings = getattr(mealplan, 'servings', 1.0)
-
-#     servings_factor = servings / r.servings
-
-#     shared_users = list(created_by.get_shopping_share())
-#     shared_users.append(created_by)
-#     if list_recipe:
-#         created = False
-#     else:
-#         list_recipe = ShoppingListRecipe.objects.create(recipe=r, mealplan=mealplan, servings=servings)
-#         created = True
-
-#     related_step_ing = []
-#     if servings == 0 and not created:
-#         list_recipe.delete()
-#         return []
-#     elif ingredients:
-#         ingredients = Ingredient.objects.filter(pk__in=ingredients, space=space)
-#     else:
-#         ingredients = Ingredient.objects.filter(step__recipe=r,  food__ignore_shopping=False, space=space)
-
-#         if exclude_onhand := created_by.userpreference.mealplan_autoexclude_onhand:
-#             ingredients = ingredients.exclude(food__onhand_users__id__in=[x.id for x in shared_users])
-
-#         if related := created_by.userpreference.mealplan_autoinclude_related:
-#             # TODO: add levels of related recipes (related recipes of related recipes) to use when auto-adding mealplans
-#             related_recipes = r.get_related_recipes()
-
-#             for x in related_recipes:
-#                 # related recipe is a Step serving size is driven by recipe serving size
-#                 # TODO once/if Steps can have a serving size this needs to be refactored
-#                 if exclude_onhand:
-#                     # if steps are used more than once in a recipe or subrecipe - I don' think this results in the desired behavior
-#                     related_step_ing += Ingredient.objects.filter(step__recipe=x, space=space).exclude(food__onhand_users__id__in=[x.id for x in shared_users]).values_list('id', flat=True)
-#                 else:
-#                     related_step_ing += Ingredient.objects.filter(step__recipe=x, space=space).values_list('id', flat=True)
-
-#                 x_ing = []
-#                 if ingredients.filter(food__recipe=x).exists():
-#                     for ing in ingredients.filter(food__recipe=x):
-#                         if exclude_onhand:
-#                             x_ing = Ingredient.objects.filter(step__recipe=x,  food__ignore_shopping=False, space=space).exclude(food__onhand_users__id__in=[x.id for x in shared_users])
-#                         else:
-#                             x_ing = Ingredient.objects.filter(step__recipe=x,  food__ignore_shopping=False, space=space).exclude(food__ignore_shopping=True)
-#                         for i in [x for x in x_ing]:
-#                             ShoppingListEntry.objects.create(
-#                                 list_recipe=list_recipe,
-#                                 food=i.food,
-#                                 unit=i.unit,
-#                                 ingredient=i,
-#                                 amount=i.amount * Decimal(servings_factor),
-#                                 created_by=created_by,
-#                                 space=space,
-#                             )
-#                 # dont' add food to the shopping list that are actually recipes that will be added as ingredients
-#                 ingredients = ingredients.exclude(food__recipe=x)
-
-#     add_ingredients = list(ingredients.values_list('id', flat=True)) + related_step_ing
-#     if not append:
-#         existing_list = ShoppingListEntry.objects.filter(list_recipe=list_recipe)
-#         # delete shopping list entries not included in ingredients
-#         existing_list.exclude(ingredient__in=ingredients).delete()
-#         # add shopping list entries that did not previously exist
-#         add_ingredients = set(add_ingredients) - set(existing_list.values_list('ingredient__id', flat=True))
-#     add_ingredients = Ingredient.objects.filter(id__in=add_ingredients, space=space)
-
-#     # if servings have changed, update the ShoppingListRecipe and existing Entries
-#     if servings <= 0:
-#         servings = 1
-
-#     if not created and list_recipe.servings != servings:
-#         update_ingredients = set(ingredients.values_list('id', flat=True)) - set(add_ingredients.values_list('id', flat=True))
-#         list_recipe.servings = servings
-#         list_recipe.save()
-#         for sle in ShoppingListEntry.objects.filter(list_recipe=list_recipe, ingredient__id__in=update_ingredients):
-#             sle.amount = sle.ingredient.amount * Decimal(servings_factor)
-#             sle.save()
-
-#     # add any missing Entries
-#     for i in [x for x in add_ingredients if x.food]:
-
-#         ShoppingListEntry.objects.create(
-#             list_recipe=list_recipe,
-#             food=i.food,
-#             unit=i.unit,
-#             ingredient=i,
-#             amount=i.amount * Decimal(servings_factor),
-#             created_by=created_by,
-#             space=space,
-#         )
-
-#     # return all shopping list items
-#     return list_recipe

cookbook/helper/template_helper.py

@@ -2,8 +2,9 @@ from gettext import gettext as _
 
 import bleach
 import markdown as md
-from bleach_allowlist import markdown_attrs, markdown_tags
 from jinja2 import Template, TemplateSyntaxError, UndefinedError
+from jinja2.exceptions import SecurityError
+from jinja2.sandbox import SandboxedEnvironment
 from markdown.extensions.tables import TableExtension
 
 from cookbook.helper.mdx_attributes import MarkdownFormatExtension
@@ -15,17 +16,34 @@ class IngredientObject(object):
     unit = ""
     food = ""
     note = ""
+    numeric_amount = 0
 
     def __init__(self, ingredient):
         if ingredient.no_amount:
             self.amount = ""
         else:
             self.amount = f"<scalable-number v-bind:number='{bleach.clean(str(ingredient.amount))}' v-bind:factor='ingredient_factor'></scalable-number>"
+            self.numeric_amount = float(ingredient.amount)
         if ingredient.unit:
-            self.unit = bleach.clean(str(ingredient.unit))
+            if ingredient.unit.plural_name in (None, ""):
+                self.unit = bleach.clean(str(ingredient.unit))
+            else:
+                if ingredient.always_use_plural_unit or ingredient.amount > 1 and not ingredient.no_amount:
+                    self.unit = bleach.clean(ingredient.unit.plural_name)
+                else:
+                    self.unit = bleach.clean(str(ingredient.unit))
         else:
             self.unit = ""
-        self.food = bleach.clean(str(ingredient.food))
+        if ingredient.food:
+            if ingredient.food.plural_name in (None, ""):
+                self.food = bleach.clean(str(ingredient.food))
+            else:
+                if ingredient.always_use_plural_food or ingredient.amount > 1 and not ingredient.no_amount:
+                    self.food = bleach.clean(str(ingredient.food.plural_name))
+                else:
+                    self.food = bleach.clean(str(ingredient.food))
+        else:
+            self.food = ""
         self.note = bleach.clean(str(ingredient.note))
 
     def __str__(self):
@@ -38,17 +56,29 @@ class IngredientObject(object):
 def render_instructions(step):  # TODO deduplicate markdown cleanup code
     instructions = step.instruction
 
-    tags = markdown_tags + [
-        'pre', 'table', 'td', 'tr', 'th', 'tbody', 'style', 'thead', 'img'
-    ]
+    tags = {
+        "h1", "h2", "h3", "h4", "h5", "h6",
+        "b", "i", "strong", "em", "tt",
+        "p", "br",
+        "span", "div", "blockquote", "code", "pre", "hr",
+        "ul", "ol", "li", "dd", "dt",
+        "img",
+        "a",
+        "sub", "sup",
+        'pre', 'table', 'td', 'tr', 'th', 'tbody', 'style', 'thead'
+    }
     parsed_md = md.markdown(
         instructions,
         extensions=[
-            'markdown.extensions.fenced_code', TableExtension(),
+            'markdown.extensions.fenced_code', 'markdown.extensions.sane_lists', 'markdown.extensions.nl2br', TableExtension(),
             UrlizeExtension(), MarkdownFormatExtension()
         ]
     )
-    markdown_attrs['*'] = markdown_attrs['*'] + ['class', 'width', 'height']
+    markdown_attrs = {
+        "*": ["id", "class", 'width', 'height'],
+        "img": ["src", "alt", "title"],
+        "a": ["href", "alt", "title"],
+    }
 
     instructions = bleach.clean(parsed_md, tags, markdown_attrs)
 
@@ -57,12 +87,17 @@ def render_instructions(step):  # TODO deduplicate markdown cleanup code
     for i in step.ingredients.all():
         ingredients.append(IngredientObject(i))
 
+    def scale(number):
+        return f"<scalable-number v-bind:number='{bleach.clean(str(number))}' v-bind:factor='ingredient_factor'></scalable-number>"
+
     try:
-        template = Template(instructions)
-        instructions = template.render(ingredients=ingredients)
+        env = SandboxedEnvironment()
+        instructions = env.from_string(instructions).render(ingredients=ingredients, scale=scale)
     except TemplateSyntaxError:
         return _('Could not parse template code.') + ' Error: Template Syntax broken'
     except UndefinedError:
         return _('Could not parse template code.') + ' Error: Undefined Error'
+    except SecurityError:
+        return _('Could not parse template code.') + ' Error: Security Error'
 
     return instructions

cookbook/helper/unit_conversion_helper.py

@@ -0,0 +1,142 @@
+from django.core.cache import caches
+from decimal import Decimal
+
+from cookbook.helper.cache_helper import CacheHelper
+from cookbook.models import Ingredient, Unit
+
+CONVERSION_TABLE = {
+    'weight': {
+        'g': 1000,
+        'kg': 1,
+        'ounce': 35.274,
+        'pound': 2.20462
+    },
+    'volume': {
+        'ml': 1000,
+        'l': 1,
+        'fluid_ounce': 33.814,
+        'pint': 2.11338,
+        'quart': 1.05669,
+        'gallon': 0.264172,
+        'tbsp': 67.628,
+        'tsp': 202.884,
+        'us_cup': 4.22675,
+        'imperial_fluid_ounce': 35.1951,
+        'imperial_pint': 1.75975,
+        'imperial_quart': 0.879877,
+        'imperial_gallon': 0.219969,
+        'imperial_tbsp': 56.3121,
+        'imperial_tsp': 168.936,
+    },
+}
+
+BASE_UNITS_WEIGHT = list(CONVERSION_TABLE['weight'].keys())
+BASE_UNITS_VOLUME = list(CONVERSION_TABLE['volume'].keys())
+
+
+class ConversionException(Exception):
+    pass
+
+
+class UnitConversionHelper:
+    space = None
+
+    def __init__(self, space):
+        """
+        Initializes unit conversion helper
+        :param space: space to perform conversions on
+        """
+        self.space = space
+
+    @staticmethod
+    def convert_from_to(from_unit, to_unit, amount):
+        """
+        Convert from one base unit to another. Throws ConversionException if trying to convert between different systems (weight/volume) or if units are not supported.
+        :param from_unit: str unit to convert from
+        :param to_unit: str unit to convert to
+        :param amount: amount to convert
+        :return: Decimal converted amount
+        """
+        system = None
+        if from_unit in BASE_UNITS_WEIGHT and to_unit in BASE_UNITS_WEIGHT:
+            system = 'weight'
+        if from_unit in BASE_UNITS_VOLUME and to_unit in BASE_UNITS_VOLUME:
+            system = 'volume'
+
+        if not system:
+            raise ConversionException('Trying to convert units not existing or not in one unit system (weight/volume)')
+
+        return Decimal(amount / Decimal(CONVERSION_TABLE[system][from_unit] / CONVERSION_TABLE[system][to_unit]))
+
+    def base_conversions(self, ingredient_list):
+        """
+        Calculates all possible base unit conversions for each ingredient give.
+        Converts to all common base units IF they exist in the unit database of the space.
+        For useful results all ingredients passed should be of the same food, otherwise filtering afterwards might be required.
+        :param ingredient_list: list of ingredients to convert
+        :return: ingredient list with appended conversions
+        """
+        base_conversion_ingredient_list = ingredient_list.copy()
+        for i in ingredient_list:
+            try:
+                conversion_unit = i.unit.name
+                if i.unit.base_unit:
+                    conversion_unit = i.unit.base_unit
+
+                # TODO allow setting which units to convert to? possibly only once conversions become visible
+                units = caches['default'].get(CacheHelper(self.space).BASE_UNITS_CACHE_KEY, None)
+                if not units:
+                    units = Unit.objects.filter(space=self.space, base_unit__in=(BASE_UNITS_VOLUME + BASE_UNITS_WEIGHT)).all()
+                    caches['default'].set(CacheHelper(self.space).BASE_UNITS_CACHE_KEY, units, 60 * 60)  # cache is cleared on unit save signal so long duration is fine
+
+                for u in units:
+                    try:
+                        ingredient = Ingredient(amount=self.convert_from_to(conversion_unit, u.base_unit, i.amount), unit=u, food=ingredient_list[0].food, )
+                        if not any((x.unit.name == ingredient.unit.name or x.unit.base_unit == ingredient.unit.name) for x in base_conversion_ingredient_list):
+                            base_conversion_ingredient_list.append(ingredient)
+                    except ConversionException:
+                        pass
+            except Exception:
+                pass
+
+        return base_conversion_ingredient_list
+
+    def get_conversions(self, ingredient):
+        """
+        Converts an ingredient to all possible conversions based on the custom unit conversion database.
+        After that passes conversion to UnitConversionHelper.base_conversions() to get all base conversions possible.
+        :param ingredient: Ingredient object
+        :return: list of ingredients with all possible custom and base conversions
+        """
+        conversions = [ingredient]
+        if ingredient.unit:
+            for c in ingredient.unit.unit_conversion_base_relation.all():
+                if c.space == self.space:
+                    r = self._uc_convert(c, ingredient.amount, ingredient.unit, ingredient.food)
+                    if r and r not in conversions:
+                        conversions.append(r)
+            for c in ingredient.unit.unit_conversion_converted_relation.all():
+                if c.space == self.space:
+                    r = self._uc_convert(c, ingredient.amount, ingredient.unit, ingredient.food)
+                    if r and r not in conversions:
+                        conversions.append(r)
+
+        conversions = self.base_conversions(conversions)
+
+        return conversions
+
+    def _uc_convert(self, uc, amount, unit, food):
+        """
+        Helper to calculate values for custom unit conversions.
+        Converts given base values using the passed UnitConversion object into a converted Ingredient
+        :param uc: UnitConversion object
+        :param amount: base amount
+        :param unit: base unit
+        :param food: base food
+        :return: converted ingredient object from base amount/unit/food
+        """
+        if uc.food is None or uc.food == food:
+            if unit == uc.base_unit:
+                return Ingredient(amount=amount * (uc.converted_amount / uc.base_amount), unit=uc.converted_unit, food=food, space=self.space)
+            else:
+                return Ingredient(amount=amount * (uc.base_amount / uc.converted_amount), unit=uc.base_unit, food=food, space=self.space)

cookbook/integration/cheftap.py

@@ -36,7 +36,7 @@ class ChefTap(Integration):
 
         recipe = Recipe.objects.create(name=title, created_by=self.request.user, internal=True, space=self.request.space, )
 
-        step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space,)
+        step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,)
 
         if source_url != '':
             step.instruction += '\n' + source_url

cookbook/integration/chowdown.py

@@ -4,6 +4,7 @@ from zipfile import ZipFile
 
 from cookbook.helper.image_processing import get_filetype
 from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
 from cookbook.models import Ingredient, Keyword, Recipe, Step
 
@@ -19,6 +20,10 @@ class Chowdown(Integration):
         direction_mode = False
         description_mode = False
 
+        description = None
+        prep_time = None
+        serving = None
+
         ingredients = []
         directions = []
         descriptions = []
@@ -26,6 +31,12 @@ class Chowdown(Integration):
             line = fl.decode("utf-8")
             if 'title:' in line:
                 title = line.replace('title:', '').replace('"', '').strip()
+            if 'description:' in line:
+                description = line.replace('description:', '').replace('"', '').strip()
+            if 'prep_time:' in line:
+                prep_time = line.replace('prep_time:', '').replace('"', '').strip()
+            if 'yield:' in line:
+                serving = line.replace('yield:', '').replace('"', '').strip()
             if 'image:' in line:
                 image = line.replace('image:', '').strip()
             if 'tags:' in line:
@@ -48,15 +59,43 @@ class Chowdown(Integration):
                 descriptions.append(line)
 
         recipe = Recipe.objects.create(name=title, created_by=self.request.user, internal=True, space=self.request.space)
+        if description:
+            recipe.description = description
 
         for k in tags.split(','):
-            print(f'adding keyword {k.strip()}')
             keyword, created = Keyword.objects.get_or_create(name=k.strip(), space=self.request.space)
             recipe.keywords.add(keyword)
 
-        step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n' + '\n'.join(descriptions), space=self.request.space,
-        )
+        ingredients_added = False
+        for direction in directions:
+            if len(direction.strip()) > 0:
+                step = Step.objects.create(
+                    instruction=direction, name='', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+                )
+            else:
+                step = Step.objects.create(
+                    instruction=direction, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+                )
+            if not ingredients_added:
+                ingredients_added = True
+
+                ingredient_parser = IngredientParser(self.request, True)
+                for ingredient in ingredients:
+                    if len(ingredient.strip()) > 0:
+                        amount, unit, food, note = ingredient_parser.parse(ingredient)
+                        f = ingredient_parser.get_food(food)
+                        u = ingredient_parser.get_unit(unit)
+                        step.ingredients.add(Ingredient.objects.create(
+                            food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
+                        ))
+            recipe.steps.add(step)
+
+        if serving:
+            recipe.servings = parse_servings(serving)
+            recipe.servings_text = 'servings'
+
+        if prep_time:
+            recipe.working_time = parse_time(prep_time)
 
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
@@ -76,6 +115,7 @@ class Chowdown(Integration):
                     if re.match(f'^images/{image}$', z.filename):
                         self.import_recipe_image(recipe, BytesIO(import_zip.read(z.filename)), filetype=get_filetype(z.filename))
 
+        recipe.save()
         return recipe
 
     def get_file_from_recipe(self, recipe):

cookbook/integration/cookbookapp.py

@@ -1,19 +1,15 @@
-import base64
-import gzip
-import json
 import re
-from gettext import gettext as _
 from io import BytesIO
 
 import requests
-import validators
-import yaml
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
-from cookbook.helper.recipe_html_import import get_recipe_from_source
-from cookbook.helper.recipe_url_import import iso_duration_to_minutes
+from cookbook.helper.recipe_url_import import (get_from_scraper, get_images_from_soup,
+                                               iso_duration_to_minutes)
+from recipe_scrapers import scrape_html
 from cookbook.integration.integration import Integration
-from cookbook.models import Ingredient, Keyword, Recipe, Step
+from cookbook.models import Ingredient, Recipe, Step
 
 
 class CookBookApp(Integration):
@@ -24,7 +20,9 @@ class CookBookApp(Integration):
     def get_recipe_from_file(self, file):
         recipe_html = file.getvalue().decode("utf-8")
 
-        recipe_json, recipe_tree, html_data, images = get_recipe_from_source(recipe_html, 'CookBookApp', self.request)
+        scrape = scrape_html(html=recipe_html, org_url="https://cookbookapp.import", supported_only=False)
+        recipe_json = get_from_scraper(scrape, self.request)
+        images = list(dict.fromkeys(get_images_from_soup(scrape.soup, None)))
 
         recipe = Recipe.objects.create(
             name=recipe_json['name'].strip(),
@@ -33,7 +31,7 @@ class CookBookApp(Integration):
 
         try:
             recipe.servings = re.findall('([0-9])+', recipe_json['recipeYield'])[0]
-        except Exception as e:
+        except Exception:
             pass
 
         try:
@@ -42,7 +40,9 @@ class CookBookApp(Integration):
         except Exception:
             pass
 
-        step = Step.objects.create(instruction=recipe_json['recipeInstructions'], space=self.request.space, )
+        # assuming import files only contain single step
+        step = Step.objects.create(instruction=recipe_json['steps'][0]['instruction'], space=self.request.space,
+                                   show_ingredients_table=self.request.user.userpreference.show_step_ingredients, )
 
         if 'nutrition' in recipe_json:
             step.instruction = step.instruction + '\n\n' + recipe_json['nutrition']
@@ -51,21 +51,24 @@ class CookBookApp(Integration):
         recipe.steps.add(step)
 
         ingredient_parser = IngredientParser(self.request, True)
-        for ingredient in recipe_json['recipeIngredient']:
-            f = ingredient_parser.get_food(ingredient['ingredient']['text'])
-            u = ingredient_parser.get_unit(ingredient['unit']['text'])
+        for ingredient in recipe_json['steps'][0]['ingredients']:
+            f = ingredient_parser.get_food(ingredient['food']['name'])
+            u = None
+            if unit := ingredient.get('unit', None):
+                u = ingredient_parser.get_unit(unit.get('name', None))
             step.ingredients.add(Ingredient.objects.create(
-                food=f, unit=u, amount=ingredient['amount'], note=ingredient['note'],  space=self.request.space,
+                food=f, unit=u, amount=ingredient.get('amount', None), note=ingredient.get('note', None), original_text=ingredient.get('original_text', None), space=self.request.space,
             ))
 
-        if len(images) > 0:
-            try:
-                url = images[0]
-                if validators.url(url, public=True):
+        try:
+            for url in images:
+                # import the first valid image which is not cookbookapp branding
+                if validate_import_url(url) and not url.startswith("https://media.cookbookmanager.com/brand/"):
                     response = requests.get(url)
                     self.import_recipe_image(recipe, BytesIO(response.content))
-            except Exception as e:
-                print('failed to import image ', str(e))
+                    break
+        except Exception as e:
+            print('failed to import image ', str(e))
 
         recipe.save()
         return recipe

cookbook/integration/cookmate.py

@@ -1,17 +1,12 @@
-import base64
-import json
 from io import BytesIO
 
-from gettext import gettext as _
-
 import requests
-import validators
-from lxml import etree
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
-from cookbook.helper.recipe_url_import import parse_servings, parse_time, parse_servings_text
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
-from cookbook.models import Ingredient, Keyword, Recipe, Step
+from cookbook.models import Ingredient, Recipe, Step
 
 
 class Cookmate(Integration):
@@ -46,27 +41,35 @@ class Cookmate(Integration):
             if len(recipe_xml.find('description')) > 0:
                 recipe.description = recipe_xml.find('description')[0].text[:512]
 
-        for step in recipe_xml.find('recipetext').getchildren():
-            step = Step.objects.create(
-                instruction=step.text.strip(), space=self.request.space,
-            )
-            recipe.steps.add(step)
+        if recipe_text := recipe_xml.find('recipetext'):
+            for step in recipe_text.getchildren():
+                if step.text:
+                    step = Step.objects.create(
+                        instruction=step.text.strip(), space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+                    )
+                    recipe.steps.add(step)
 
         ingredient_parser = IngredientParser(self.request, True)
 
-        for ingredient in recipe_xml.find('ingredient').getchildren():
-            if ingredient.text.strip() != '':
-                amount, unit, food, note = ingredient_parser.parse(ingredient.text.strip())
-                f = ingredient_parser.get_food(food)
-                u = ingredient_parser.get_unit(unit)
-                recipe.steps.first().ingredients.add(Ingredient.objects.create(
-                    food=f, unit=u, amount=amount, note=note, original_text=ingredient.text.strip(), space=self.request.space,
-                ))
+        if recipe_ingredients := recipe_xml.find('ingredient'):
+            ingredient_step = recipe.steps.first()
+            if ingredient_step is None:
+                ingredient_step = Step.objects.create(space=self.request.space, instruction='')
+
+            for ingredient in recipe_ingredients.getchildren():
+                if ingredient.text:
+                    if ingredient.text.strip() != '':
+                        amount, unit, food, note = ingredient_parser.parse(ingredient.text.strip())
+                        f = ingredient_parser.get_food(food)
+                        u = ingredient_parser.get_unit(unit)
+                        ingredient_step.ingredients.add(Ingredient.objects.create(
+                            food=f, unit=u, amount=amount, note=note, original_text=ingredient.text.strip(), space=self.request.space,
+                        ))
 
         if recipe_xml.find('imageurl') is not None:
             try:
                 url = recipe_xml.find('imageurl').text.strip()
-                if validators.url(url, public=True):
+                if validate_import_url(url):
                     response = requests.get(url)
                 self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e:

cookbook/integration/copymethat.py

@@ -1,12 +1,10 @@
-import re
 from io import BytesIO
 from zipfile import ZipFile
 
-from bs4 import BeautifulSoup
-
+from bs4 import BeautifulSoup, Tag
 from django.utils.translation import gettext as _
+
 from cookbook.helper.ingredient_parser import IngredientParser
-from cookbook.helper.recipe_html_import import get_recipe_from_source
 from cookbook.helper.recipe_url_import import iso_duration_to_minutes, parse_servings
 from cookbook.integration.integration import Integration
 from cookbook.models import Ingredient, Keyword, Recipe, Step
@@ -22,7 +20,13 @@ class CopyMeThat(Integration):
 
     def get_recipe_from_file(self, file):
         # 'file' comes is as a beautifulsoup object
-        recipe = Recipe.objects.create(name=file.find("div", {"id": "name"}).text.strip(), created_by=self.request.user, internal=True, space=self.request.space, )
+        try:
+            source = file.find("a", {"id": "original_link"}).text
+        except AttributeError:
+            source = None
+
+        recipe = Recipe.objects.create(name=file.find("div", {"id": "name"}).text.strip(
+        )[:128], source_url=source, created_by=self.request.user, internal=True, space=self.request.space, )
 
         for category in file.find_all("span", {"class": "recipeCategory"}):
             keyword, created = Keyword.objects.get_or_create(name=category.text, space=self.request.space)
@@ -32,8 +36,6 @@ class CopyMeThat(Integration):
             recipe.servings = parse_servings(file.find("a", {"id": "recipeYield"}).text.strip())
             recipe.working_time = iso_duration_to_minutes(file.find("span", {"meta": "prepTime"}).text.strip())
             recipe.waiting_time = iso_duration_to_minutes(file.find("span", {"meta": "cookTime"}).text.strip())
-            recipe.description = (file.find("div ", {"id": "description"}).text.strip())[:512]
-
         except AttributeError:
             pass
 
@@ -43,36 +45,72 @@ class CopyMeThat(Integration):
         except AttributeError:
             pass
 
-        step = Step.objects.create(instruction='', space=self.request.space, )
-
-        ingredient_parser = IngredientParser(self.request, True)
-        for ingredient in file.find_all("li", {"class": "recipeIngredient"}):
-            if ingredient.text == "":
-                continue
-            amount, unit, food, note = ingredient_parser.parse(ingredient.text.strip())
-            f = ingredient_parser.get_food(food)
-            u = ingredient_parser.get_unit(unit)
-            step.ingredients.add(Ingredient.objects.create(
-                food=f, unit=u, amount=amount, note=note, original_text=ingredient.text.strip(), space=self.request.space,
-            ))
-
-        for s in file.find_all("li", {"class": "instruction"}):
-            if s.text == "":
-                continue
-            step.instruction += s.text.strip() + ' \n\n'
-
-        for s in file.find_all("li", {"class": "recipeNote"}):
-            if s.text == "":
-                continue
-            step.instruction += s.text.strip() + ' \n\n'
-
         try:
-            if file.find("a", {"id": "original_link"}).text != '':
-                step.instruction += "\n\n" + _("Imported from") + ": " + file.find("a", {"id": "original_link"}).text
-                step.save()
+            if len(file.find("span", {"id": "made_this"}).text.strip()) > 0:
+                recipe.keywords.add(Keyword.objects.get_or_create(space=self.request.space, name=_('I made this'))[0])
         except AttributeError:
             pass
 
+        step = Step.objects.create(instruction='', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients, )
+
+        ingredient_parser = IngredientParser(self.request, True)
+
+        ingredients = file.find("ul", {"id": "recipeIngredients"})
+        if isinstance(ingredients, Tag):
+            for ingredient in ingredients.children:
+                if not isinstance(ingredient, Tag) or not ingredient.text.strip() or "recipeIngredient_spacer" in ingredient['class']:
+                    continue
+                if any(x in ingredient['class'] for x in ["recipeIngredient_subheader", "recipeIngredient_note"]):
+                    step.ingredients.add(
+                        Ingredient.objects.create(
+                            is_header=True,
+                            note=ingredient.text.strip()[
+                                :256],
+                            original_text=ingredient.text.strip(),
+                            space=self.request.space,
+                        ))
+                else:
+                    amount, unit, food, note = ingredient_parser.parse(ingredient.text.strip())
+                    f = ingredient_parser.get_food(food)
+                    u = ingredient_parser.get_unit(unit)
+                    step.ingredients.add(Ingredient.objects.create(food=f, unit=u, amount=amount, note=note, original_text=ingredient.text.strip(), space=self.request.space, ))
+
+        instructions = file.find("ol", {"id": "recipeInstructions"})
+        if isinstance(instructions, Tag):
+            for instruction in instructions.children:
+                if not isinstance(instruction, Tag) or instruction.text == "":
+                    continue
+                if "instruction_subheader" in instruction['class']:
+                    if step.instruction:
+                        step.save()
+                        recipe.steps.add(step)
+                        step = Step.objects.create(instruction='', space=self.request.space, )
+
+                    step.name = instruction.text.strip()[:128]
+                else:
+                    step.instruction += instruction.text.strip() + ' \n\n'
+
+        notes = file.find_all("li", {"class": "recipeNote"})
+        if notes:
+            step.instruction += '*Notes:* \n\n'
+
+            for n in notes:
+                if n.text == "":
+                    continue
+                step.instruction += '*' + n.text.strip() + '* \n\n'
+
+        description = ''
+        try:
+            description = file.find("div", {"id": "description"}).text.strip()
+        except AttributeError:
+            pass
+        if len(description) <= 512:
+            recipe.description = description
+        else:
+            recipe.description = description[:480] + ' ... (full description below)'
+            step.instruction += '*Description:* \n\n*' + description + '* \n\n'
+
+        step.save()
         recipe.steps.add(step)
 
         # import the Primary recipe image that is stored in the Zip

cookbook/integration/default.py

@@ -1,4 +1,5 @@
 import json
+import traceback
 from io import BytesIO, StringIO
 from re import match
 from zipfile import ZipFile
@@ -19,7 +20,10 @@ class Default(Integration):
         recipe = self.decode_recipe(recipe_string)
         images = list(filter(lambda v: match('image.*', v), recipe_zip.namelist()))
         if images:
-            self.import_recipe_image(recipe, BytesIO(recipe_zip.read(images[0])), filetype=get_filetype(images[0]))
+            try:
+                self.import_recipe_image(recipe, BytesIO(recipe_zip.read(images[0])), filetype=get_filetype(images[0]))
+            except AttributeError:
+                traceback.print_exc()
         return recipe
 
     def decode_recipe(self, string):
@@ -54,7 +58,7 @@ class Default(Integration):
 
                 try:
                     recipe_zip_obj.writestr(f'image{get_filetype(r.image.file.name)}', r.image.file.read())
-                except ValueError:
+                except (ValueError, FileNotFoundError):
                     pass
 
                 recipe_zip_obj.close()
@@ -67,4 +71,4 @@ class Default(Integration):
 
         export_zip_obj.close()
 
-        return [[ self.get_export_file_name(), export_zip_stream.getvalue() ]]
+        return [[self.get_export_file_name(), export_zip_stream.getvalue()]]

cookbook/integration/domestica.py

@@ -28,7 +28,7 @@ class Domestica(Integration):
         recipe.save()
 
         step = Step.objects.create(
-            instruction=file['directions'], space=self.request.space,
+            instruction=file['directions'], space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
         )
 
         if file['source'] != '':

cookbook/integration/gourmet.py

@@ -0,0 +1,211 @@
+import base64
+from io import BytesIO
+from lxml import etree
+import requests
+from pathlib import Path
+
+from bs4 import BeautifulSoup, Tag
+
+from cookbook.helper.HelperFunctions import validate_import_url
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time, iso_duration_to_minutes
+from cookbook.integration.integration import Integration
+from cookbook.models import Ingredient, Recipe, Step, Keyword
+from recipe_scrapers import scrape_html
+
+
+class Gourmet(Integration):
+
+    def split_recipe_file(self, file):
+        encoding = 'utf-8'
+        byte_string = file.read()
+        text_obj = byte_string.decode(encoding, errors="ignore")
+        soup = BeautifulSoup(text_obj, "html.parser")
+        return soup.find_all("div", {"class": "recipe"})
+
+    def get_ingredients_recursive(self, step, ingredients, ingredient_parser):
+        if isinstance(ingredients, Tag):
+            for ingredient in ingredients.children:
+                if not isinstance(ingredient, Tag):
+                    continue
+
+                if ingredient.name in ["li"]:
+                    step_name = "".join(ingredient.findAll(text=True, recursive=False)).strip().rstrip(":")
+
+                    step.ingredients.add(Ingredient.objects.create(
+                        is_header=True,
+                        note=step_name[:256],
+                        original_text=step_name,
+                        space=self.request.space,
+                    ))
+                    next_ingrediets = ingredient.find("ul", {"class": "ing"})
+                    self.get_ingredients_recursive(step, next_ingrediets, ingredient_parser)
+
+                else:
+                    try:
+                        amount, unit, food, note = ingredient_parser.parse(ingredient.text.strip())
+                        f = ingredient_parser.get_food(food)
+                        u = ingredient_parser.get_unit(unit)
+                        step.ingredients.add(
+                            Ingredient.objects.create(
+                                food=f,
+                                unit=u,
+                                amount=amount,
+                                note=note,
+                                original_text=ingredient.text.strip(),
+                                space=self.request.space,
+                            )
+                        )
+                    except ValueError:
+                        pass
+
+    def get_recipe_from_file(self, file):
+        # 'file' comes is as a beautifulsoup object
+
+        source_url = None
+        for item in file.find_all('a'):
+            if item.has_attr('href'):
+                source_url = item.get("href")
+                break
+
+        name = file.find("p", {"class": "title"}).find("span", {"itemprop": "name"}).text.strip()
+
+        recipe = Recipe.objects.create(
+            name=name[:128],
+            source_url=source_url,
+            created_by=self.request.user,
+            internal=True,
+            space=self.request.space,
+        )
+
+        for category in file.find_all("span", {"itemprop": "recipeCategory"}):
+            keyword, created = Keyword.objects.get_or_create(name=category.text, space=self.request.space)
+            recipe.keywords.add(keyword)
+
+        try:
+            recipe.servings = parse_servings(file.find("span", {"itemprop": "recipeYield"}).text.strip())
+        except AttributeError:
+            pass
+
+        try:
+            prep_time = file.find("span", {"itemprop": "prepTime"}).text.strip().split()
+            prep_time[0] = prep_time[0].replace(',', '.')
+            if prep_time[1].lower() in ['stunde', 'stunden', 'hour', 'hours']:
+                prep_time_min = int(float(prep_time[0]) * 60)
+            elif prep_time[1].lower() in ['tag', 'tage', 'day', 'days']:
+                prep_time_min = int(float(prep_time[0]) * 60 * 24)
+            else:
+                prep_time_min = int(prep_time[0])
+            recipe.waiting_time = prep_time_min
+        except AttributeError:
+            pass
+
+        try:
+            cook_time = file.find("span", {"itemprop": "cookTime"}).text.strip().split()
+            cook_time[0] = cook_time[0].replace(',', '.')
+            if cook_time[1].lower() in ['stunde', 'stunden', 'hour', 'hours']:
+                cook_time_min = int(float(cook_time[0]) * 60)
+            elif cook_time[1].lower() in ['tag', 'tage', 'day', 'days']:
+                cook_time_min = int(float(cook_time[0]) * 60 * 24)
+            else:
+                cook_time_min = int(cook_time[0])
+
+            recipe.working_time = cook_time_min
+        except AttributeError:
+            pass
+
+        for cuisine in file.find_all('span', {'itemprop': 'recipeCuisine'}):
+            cuisine_name = cuisine.text
+            keyword = Keyword.objects.get_or_create(space=self.request.space, name=cuisine_name)
+            if len(keyword):
+                recipe.keywords.add(keyword[0])
+
+        for category in file.find_all('span', {'itemprop': 'recipeCategory'}):
+            category_name = category.text
+            keyword = Keyword.objects.get_or_create(space=self.request.space, name=category_name)
+            if len(keyword):
+                recipe.keywords.add(keyword[0])
+
+        step = Step.objects.create(
+            instruction='',
+            space=self.request.space,
+            show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+        )
+
+        ingredient_parser = IngredientParser(self.request, True)
+
+        ingredients = file.find("ul", {"class": "ing"})
+        self.get_ingredients_recursive(step, ingredients, ingredient_parser)
+
+        instructions = file.find("div", {"class": "instructions"})
+        if isinstance(instructions, Tag):
+            for instruction in instructions.children:
+                if not isinstance(instruction, Tag) or instruction.text == "":
+                    continue
+                if instruction.name == "h3":
+                    if step.instruction:
+                        step.save()
+                        recipe.steps.add(step)
+                        step = Step.objects.create(
+                            instruction='',
+                            space=self.request.space,
+                        )
+
+                    step.name = instruction.text.strip()[:128]
+                else:
+                    if instruction.name == "div":
+                        for instruction_step in instruction.children:
+                            for br in instruction_step.find_all("br"):
+                                br.replace_with("\n")
+                            step.instruction += instruction_step.text.strip() + ' \n\n'
+
+        notes = file.find("div", {"class": "modifications"})
+        if notes:
+            for n in notes.children:
+                if n.text == "":
+                    continue
+                if n.name == "h3":
+                    step.instruction += f'*{n.text.strip()}:* \n\n'
+                else:
+                    for br in n.find_all("br"):
+                        br.replace_with("\n")
+
+                    step.instruction += '*' + n.text.strip() + '* \n\n'
+
+        description = ''
+        try:
+            description = file.find("div", {"id": "description"}).text.strip()
+        except AttributeError:
+            pass
+        if len(description) <= 512:
+            recipe.description = description
+        else:
+            recipe.description = description[:480] + ' ... (full description below)'
+            step.instruction += '*Description:* \n\n*' + description + '* \n\n'
+
+        step.save()
+        recipe.steps.add(step)
+
+        # import the Primary recipe image that is stored in the Zip
+        try:
+            image_path = file.find("img").get("src")
+            image_filename = image_path.split("\\")[1]
+
+            for f in self.import_zip.filelist:
+                zip_file_name = Path(f.filename).name
+                if image_filename == zip_file_name:
+                    image_file = self.import_zip.read(f)
+                    image_bytes = BytesIO(image_file)
+                    self.import_recipe_image(recipe, image_bytes, filetype='.jpeg')
+                    break
+        except Exception as e:
+            print(recipe.name, ': failed to import image ', str(e))
+
+        recipe.save()
+        return recipe
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        raise NotImplementedError('Method not implemented in storage integration')
+
+    def get_file_from_recipe(self, recipe):
+        raise NotImplementedError('Method not implemented in storage integration')

cookbook/integration/integration.py

@@ -1,16 +1,11 @@
-import time
 import datetime
-import json
 import traceback
 import uuid
-from io import BytesIO, StringIO
+from io import BytesIO
 from zipfile import BadZipFile, ZipFile
 
-import lxml
-from django.core.cache import cache
-import datetime
-
 from bs4 import Tag
+from django.core.cache import cache
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.files import File
 from django.db import IntegrityError
@@ -20,11 +15,9 @@ from django.utils.translation import gettext as _
 from django_scopes import scope
 from lxml import etree
 
-from cookbook.forms import ImportExportBase
-from cookbook.helper.image_processing import get_filetype, handle_image
+from cookbook.helper.image_processing import handle_image
 from cookbook.models import Keyword, Recipe
-from recipes.settings import DEBUG
-from recipes.settings import EXPORT_FILE_CACHE_DURATION
+from recipes.settings import DEBUG, EXPORT_FILE_CACHE_DURATION
 
 
 class Integration:
@@ -43,8 +36,7 @@ class Integration:
         self.export_type = export_type
         self.ignored_recipes = []
 
-        description = f'Imported by {request.user.get_user_name()} at {date_format(datetime.datetime.now(), "DATETIME_FORMAT")}. Type: {export_type}'
-        icon = '📥'
+        description = f'Imported by {request.user.get_user_display_name()} at {date_format(datetime.datetime.now(), "DATETIME_FORMAT")}. Type: {export_type}'
 
         try:
             last_kw = Keyword.objects.filter(name__regex=r'^(Import [0-9]+)', space=request.space).latest('created_at')
@@ -57,23 +49,19 @@ class Integration:
             self.keyword = parent.add_child(
                 name=name,
                 description=description,
-                icon=icon,
                 space=request.space
             )
         except (IntegrityError, ValueError):  # in case, for whatever reason, the name does exist append UUID to it. Not nice but works for now.
             self.keyword = parent.add_child(
                 name=f'{name} {str(uuid.uuid4())[0:8]}',
                 description=description,
-                icon=icon,
                 space=request.space
             )
 
-
-
     def do_export(self, recipes, el):
 
         with scope(space=self.request.space):
-            el.total_recipes = len(recipes) 
+            el.total_recipes = len(recipes)
             el.cache_duration = EXPORT_FILE_CACHE_DURATION
             el.save()
 
@@ -85,7 +73,7 @@ class Integration:
                 export_file = file
 
             else:
-                #zip the files if there is more then one file
+                # zip the files if there is more then one file
                 export_filename = self.get_export_file_name()
                 export_stream = BytesIO()
                 export_obj = ZipFile(export_stream, 'w')
@@ -96,8 +84,7 @@ class Integration:
                 export_obj.close()
                 export_file = export_stream.getvalue()
 
-
-            cache.set('export_file_'+str(el.pk), {'filename': export_filename, 'file': export_file}, EXPORT_FILE_CACHE_DURATION)
+            cache.set('export_file_' + str(el.pk), {'filename': export_filename, 'file': export_file}, EXPORT_FILE_CACHE_DURATION)
             el.running = False
             el.save()
 
@@ -105,7 +92,6 @@ class Integration:
         response['Content-Disposition'] = 'attachment; filename="' + export_filename + '"'
         return response
 
-
     def import_file_name_filter(self, zip_info_object):
         """
         Since zipfile.namelist() returns all files in all subdirectories this function allows filtering of files
@@ -167,9 +153,22 @@ class Integration:
                             il.total_recipes = len(new_file_list)
                             file_list = new_file_list
 
+                        if isinstance(self, cookbook.integration.gourmet.Gourmet):
+                            self.import_zip = import_zip
+                            new_file_list = []
+                            for file in file_list:
+                                if file.file_size == 0:
+                                    next
+                                if file.filename.startswith("index.htm"):
+                                    next
+                                if file.filename.endswith(".htm"):
+                                    new_file_list += self.split_recipe_file(BytesIO(import_zip.read(file.filename)))
+                            il.total_recipes = len(new_file_list)
+                            file_list = new_file_list
+
                         for z in file_list:
                             try:
-                                if not hasattr(z, 'filename'):
+                                if not hasattr(z, 'filename') or isinstance(z, Tag):
                                     recipe = self.get_recipe_from_file(z)
                                 else:
                                     recipe = self.get_recipe_from_file(BytesIO(import_zip.read(z.filename)))
@@ -182,7 +181,7 @@ class Integration:
                                 traceback.print_exc()
                                 self.handle_exception(e, log=il, message=f'-------------------- \nERROR \n{e}\n--------------------\n')
                         import_zip.close()
-                    elif '.json' in f['name'] or '.txt' in f['name'] or '.mmf' in f['name'] or '.rk' in f['name'] or '.melarecipe' in f['name']:
+                    elif '.json' in f['name'] or '.xml' in f['name'] or '.txt' in f['name'] or '.mmf' in f['name'] or '.rk' in f['name'] or '.melarecipe' in f['name']:
                         data_list = self.split_recipe_file(f['file'])
                         il.total_recipes += len(data_list)
                         for d in data_list:
@@ -303,7 +302,6 @@ class Integration:
         if DEBUG:
             traceback.print_exc()
 
-
     def get_export_file_name(self, format='zip'):
         return "export_{}.{}".format(datetime.datetime.now().strftime("%Y-%m-%d"), format)
 

cookbook/integration/mealie.py

@@ -5,8 +5,9 @@ from zipfile import ZipFile
 
 from cookbook.helper.image_processing import get_filetype
 from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
-from cookbook.models import Ingredient, Recipe, Step
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class Mealie(Integration):
@@ -23,41 +24,66 @@ class Mealie(Integration):
             name=recipe_json['name'].strip(), description=description,
             created_by=self.request.user, internal=True, space=self.request.space)
 
-        # TODO parse times (given in PT2H3M )
-        # @vabene check recipe_url_import.iso_duration_to_minutes  I think it does what you are looking for
-
-        ingredients_added = False
         for s in recipe_json['recipe_instructions']:
-            step = Step.objects.create(
-                instruction=s['text'], space=self.request.space,
-            )
-            if not ingredients_added:
-                ingredients_added = True
-
-                if len(recipe_json['description'].strip()) > 500:
-                    step.instruction = recipe_json['description'].strip() + '\n\n' + step.instruction
-
-                ingredient_parser = IngredientParser(self.request, True)
-                for ingredient in recipe_json['recipe_ingredient']:
-                    try:
-                        if ingredient['food']:
-                            f = ingredient_parser.get_food(ingredient['food'])
-                            u = ingredient_parser.get_unit(ingredient['unit'])
-                            amount = ingredient['quantity']
-                            note = ingredient['note']
-                            original_text = None
-                        else:
-                            amount, unit, food, note = ingredient_parser.parse(ingredient['note'])
-                            f = ingredient_parser.get_food(food)
-                            u = ingredient_parser.get_unit(unit)
-                            original_text = ingredient['note']
-                        step.ingredients.add(Ingredient.objects.create(
-                            food=f, unit=u, amount=amount, note=note, original_text=original_text, space=self.request.space,
-                        ))
-                    except Exception:
-                        pass
+            step = Step.objects.create(instruction=s['text'], space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients, )
             recipe.steps.add(step)
 
+        step = recipe.steps.first()
+        if not step:  # if there is no step in the exported data
+            step = Step.objects.create(instruction='', space=self.request.space, )
+            recipe.steps.add(step)
+
+        if len(recipe_json['description'].strip()) > 500:
+            step.instruction = recipe_json['description'].strip() + '\n\n' + step.instruction
+
+        ingredient_parser = IngredientParser(self.request, True)
+        for ingredient in recipe_json['recipe_ingredient']:
+            try:
+                if ingredient['food']:
+                    f = ingredient_parser.get_food(ingredient['food'])
+                    u = ingredient_parser.get_unit(ingredient['unit'])
+                    amount = ingredient['quantity']
+                    note = ingredient['note']
+                    original_text = None
+                else:
+                    amount, unit, food, note = ingredient_parser.parse(ingredient['note'])
+                    f = ingredient_parser.get_food(food)
+                    u = ingredient_parser.get_unit(unit)
+                    original_text = ingredient['note']
+                step.ingredients.add(Ingredient.objects.create(
+                    food=f, unit=u, amount=amount, note=note, original_text=original_text, space=self.request.space,
+                ))
+            except Exception:
+                pass
+
+        if 'tags' in recipe_json and len(recipe_json['tags']) > 0:
+            for k in recipe_json['tags']:
+                if 'name' in k:
+                    keyword, created = Keyword.objects.get_or_create(name=k['name'].strip(), space=self.request.space)
+                    recipe.keywords.add(keyword)
+
+        if 'notes' in recipe_json and len(recipe_json['notes']) > 0:
+            notes_text = "#### Notes  \n\n"
+            for n in recipe_json['notes']:
+                notes_text += f'{n["text"]}  \n'
+
+            step = Step.objects.create(
+                instruction=notes_text, space=self.request.space,
+            )
+            recipe.steps.add(step)
+
+        if 'recipe_yield' in recipe_json and recipe_json['recipe_yield'] is not None:
+            recipe.servings = parse_servings(recipe_json['recipe_yield'])
+            recipe.servings_text = parse_servings_text(recipe_json['recipe_yield'])
+
+        if 'total_time' in recipe_json and recipe_json['total_time'] is not None:
+            recipe.working_time = parse_time(recipe_json['total_time'])
+
+        if 'org_url' in recipe_json and recipe_json['org_url'] is not None:
+            recipe.source_url = recipe_json['org_url']
+
+        recipe.save()
+
         for f in self.files:
             if '.zip' in f['name']:
                 import_zip = ZipFile(f['file'])

cookbook/integration/mealmaster.py

@@ -22,7 +22,7 @@ class MealMaster(Integration):
                         if 'Yield:' in line:
                             servings_text = line.replace('Yield:', '').strip()
                         else:
-                            if re.match('\s{2,}([0-9])+', line):
+                            if re.match(r'\s{2,}([0-9])+', line):
                                 ingredients.append(line.strip())
                             else:
                                 directions.append(line.strip())
@@ -39,7 +39,7 @@ class MealMaster(Integration):
             recipe.keywords.add(keyword)
 
         step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n', space=self.request.space,
+            instruction='\n'.join(directions) + '\n\n', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
         )
 
         ingredient_parser = IngredientParser(self.request, True)

cookbook/integration/melarecipes.py

@@ -57,7 +57,7 @@ class MelaRecipes(Integration):
             recipe.source_url = recipe_json['link']
 
         step = Step.objects.create(
-            instruction=instruction, space=self.request.space,
+            instruction=instruction, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients
         )
 
         ingredient_parser = IngredientParser(self.request, True)

cookbook/integration/nextcloud_cookbook.py

@@ -1,13 +1,15 @@
 import json
 import re
-from io import BytesIO
+from io import BytesIO, StringIO
 from zipfile import ZipFile
 
+from PIL import Image
+
 from cookbook.helper.image_processing import get_filetype
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import iso_duration_to_minutes
 from cookbook.integration.integration import Integration
-from cookbook.models import Ingredient, Keyword, Recipe, Step
+from cookbook.models import Ingredient, Keyword, NutritionInformation, Recipe, Step
 
 
 class NextcloudCookbook(Integration):
@@ -50,31 +52,45 @@ class NextcloudCookbook(Integration):
 
         ingredients_added = False
         for s in recipe_json['recipeInstructions']:
-            step = Step.objects.create(
-                instruction=s, space=self.request.space,
-            )
-            if not ingredients_added:
-                if len(recipe_json['description'].strip()) > 500:
-                    step.instruction = recipe_json['description'].strip() + '\n\n' + step.instruction
+            if 'text' in s:
+                step = Step.objects.create(
+                    instruction=s['text'], name=s['name'], space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+                )
+            else:
+                step = Step.objects.create(
+                    instruction=s, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+                )
 
-                ingredients_added = True
-
-                ingredient_parser = IngredientParser(self.request, True)
+            ingredient_parser = IngredientParser(self.request, True)
+            if ingredients_added == False:
                 for ingredient in recipe_json['recipeIngredient']:
-                    amount, unit, food, note = ingredient_parser.parse(ingredient)
-                    f = ingredient_parser.get_food(food)
-                    u = ingredient_parser.get_unit(unit)
-                    step.ingredients.add(Ingredient.objects.create(
-                        food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
-                    ))
+                    ingredients_added = True
+                    if ingredient.startswith('##'):
+                        subheader = ingredient.replace('##', '', 1)
+                        step.ingredients.add(Ingredient.objects.create(note=subheader, is_header=True, no_amount=True, space=self.request.space))
+                    else:
+                        amount, unit, food, note = ingredient_parser.parse(ingredient)
+                        f = ingredient_parser.get_food(food)
+                        u = ingredient_parser.get_unit(unit)
+                        step.ingredients.add(Ingredient.objects.create(
+                            food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,))
             recipe.steps.add(step)
 
         if 'nutrition' in recipe_json:
+            nutrition = {}
             try:
-                recipe.nutrition.calories = recipe_json['nutrition']['calories'].replace(' kcal', '').replace(' ', '')
-                recipe.nutrition.proteins = recipe_json['nutrition']['calories'].replace(' g', '').replace(',', '.').replace(' ', '')
-                recipe.nutrition.fats = recipe_json['nutrition']['calories'].replace(' g', '').replace(',', '.').replace(' ', '')
-                recipe.nutrition.carbohydrates = recipe_json['nutrition']['calories'].replace(' g', '').replace(',', '.').replace(' ', '')
+                if 'calories' in recipe_json['nutrition']:
+                    nutrition['calories'] = int(re.search(r'\d+', recipe_json['nutrition']['calories']).group())
+                if 'proteinContent' in recipe_json['nutrition']:
+                    nutrition['proteins'] = int(re.search(r'\d+', recipe_json['nutrition']['proteinContent']).group())
+                if 'fatContent' in recipe_json['nutrition']:
+                    nutrition['fats'] = int(re.search(r'\d+', recipe_json['nutrition']['fatContent']).group())
+                if 'carbohydrateContent' in recipe_json['nutrition']:
+                    nutrition['carbohydrates'] = int(re.search(r'\d+', recipe_json['nutrition']['carbohydrateContent']).group())
+
+                if nutrition != {}:
+                    recipe.nutrition = NutritionInformation.objects.create(**nutrition, space=self.request.space)
+                    recipe.save()
             except Exception:
                 pass
 
@@ -87,5 +103,90 @@ class NextcloudCookbook(Integration):
 
         return recipe
 
+    def formatTime(self, min):
+        h = min // 60
+        m = min % 60
+        return f'PT{h}H{m}M0S'
+
     def get_file_from_recipe(self, recipe):
-        raise NotImplementedError('Method not implemented in storage integration')
+
+        export = {}
+        export['name'] = recipe.name
+        export['description'] = recipe.description
+        export['url'] = recipe.source_url
+        export['prepTime'] = self.formatTime(recipe.working_time)
+        export['cookTime'] = self.formatTime(recipe.waiting_time)
+        export['totalTime'] = self.formatTime(recipe.working_time + recipe.waiting_time)
+        export['recipeYield'] = recipe.servings
+        export['image'] = f'/Recipes/{recipe.name}/full.jpg'
+        export['imageUrl'] = f'/Recipes/{recipe.name}/full.jpg'
+
+        recipeKeyword = []
+        for k in recipe.keywords.all():
+            recipeKeyword.append(k.name)
+
+        export['keywords'] = recipeKeyword
+
+        recipeInstructions = []
+        recipeIngredient = []
+        for s in recipe.steps.all():
+            recipeInstructions.append(s.instruction)
+
+            for i in s.ingredients.all():
+                recipeIngredient.append(f'{float(i.amount)} {i.unit} {i.food}')
+
+        export['recipeIngredient'] = recipeIngredient
+        export['recipeInstructions'] = recipeInstructions
+
+        return "recipe.json", json.dumps(export)
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        export_zip_stream = BytesIO()
+        export_zip_obj = ZipFile(export_zip_stream, 'w')
+
+        for recipe in recipes:
+            if recipe.internal and recipe.space == self.request.space:
+
+                recipe_stream = StringIO()
+                filename, data = self.get_file_from_recipe(recipe)
+                recipe_stream.write(data)
+                export_zip_obj.writestr(f'{recipe.name}/{filename}', recipe_stream.getvalue())
+                recipe_stream.close()
+
+                try:
+                    imageByte = recipe.image.file.read()
+                    export_zip_obj.writestr(f'{recipe.name}/full.jpg', self.getJPEG(imageByte))
+                    export_zip_obj.writestr(f'{recipe.name}/thumb.jpg', self.getThumb(171, imageByte))
+                    export_zip_obj.writestr(f'{recipe.name}/thumb16.jpg', self.getThumb(16, imageByte))
+                except ValueError:
+                    pass
+
+            el.exported_recipes += 1
+            el.msg += self.get_recipe_processed_msg(recipe)
+            el.save()
+
+        export_zip_obj.close()
+
+        return [[self.get_export_file_name(), export_zip_stream.getvalue()]]
+
+    def getJPEG(self, imageByte):
+        image = Image.open(BytesIO(imageByte))
+        image = image.convert('RGB')
+
+        bytes = BytesIO()
+        image.save(bytes, "JPEG")
+        return bytes.getvalue()
+
+    def getThumb(self, size, imageByte):
+        image = Image.open(BytesIO(imageByte))
+
+        w, h = image.size
+        m = min(w, h)
+
+        image = image.crop(((w - m) // 2, (h - m) // 2, (w + m) // 2, (h + m) // 2))
+        image = image.resize([size, size], Image.Resampling.LANCZOS)
+        image = image.convert('RGB')
+
+        bytes = BytesIO()
+        image.save(bytes, "JPEG")
+        return bytes.getvalue()

cookbook/integration/openeats.py

@@ -1,27 +1,59 @@
 import json
 
+from django.utils.translation import gettext as _
+
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Ingredient, Recipe, Step
+from cookbook.models import Comment, CookLog, Ingredient, Keyword, Recipe, Step
 
 
 class OpenEats(Integration):
 
     def get_recipe_from_file(self, file):
-        recipe = Recipe.objects.create(name=file['name'].strip(), created_by=self.request.user, internal=True,
+
+        description = file['info']
+        description_max_length = Recipe._meta.get_field('description').max_length
+        if len(description) > description_max_length:
+            description = description[0:description_max_length]
+
+        recipe = Recipe.objects.create(name=file['name'].strip(), description=description, created_by=self.request.user, internal=True,
                                        servings=file['servings'], space=self.request.space, waiting_time=file['cook_time'], working_time=file['prep_time'])
 
         instructions = ''
-        if file["info"] != '':
-            instructions += file["info"]
 
         if file["directions"] != '':
             instructions += file["directions"]
 
         if file["source"] != '':
-            instructions += file["source"]
+            instructions += '\n' + _('Recipe source:') + f'[{file["source"]}]({file["source"]})'
 
-        step = Step.objects.create(instruction=instructions, space=self.request.space,)
+        cuisine_keyword, created = Keyword.objects.get_or_create(name="Cuisine", space=self.request.space)
+        if file["cuisine"] != '':
+            keyword, created = Keyword.objects.get_or_create(name=file["cuisine"].strip(), space=self.request.space)
+            if created:
+                keyword.move(cuisine_keyword, pos="last-child")
+            recipe.keywords.add(keyword)
+
+        course_keyword, created = Keyword.objects.get_or_create(name="Course", space=self.request.space)
+        if file["course"] != '':
+            keyword, created = Keyword.objects.get_or_create(name=file["course"].strip(), space=self.request.space)
+            if created:
+                keyword.move(course_keyword, pos="last-child")
+            recipe.keywords.add(keyword)
+
+        for tag in file["tags"]:
+            keyword, created = Keyword.objects.get_or_create(name=tag.strip(), space=self.request.space)
+            recipe.keywords.add(keyword)
+
+        for comment in file['comments']:
+            Comment.objects.create(recipe=recipe, text=comment['text'], created_by=self.request.user)
+            CookLog.objects.create(recipe=recipe, rating=comment['rating'], created_by=self.request.user, space=self.request.space)
+
+        if file["photo"] != '':
+            recipe.image = f'recipes/openeats-import/{file["photo"]}'
+            recipe.save()
+
+        step = Step.objects.create(instruction=instructions, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,)
 
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in file['ingredients']:
@@ -38,6 +70,9 @@ class OpenEats(Integration):
         recipe_json = json.loads(file.read())
         recipe_dict = {}
         ingredient_group_dict = {}
+        cuisine_group_dict = {}
+        course_group_dict = {}
+        tag_group_dict = {}
 
         for o in recipe_json:
             if o['model'] == 'recipe.recipe':
@@ -50,11 +85,27 @@ class OpenEats(Integration):
                     'cook_time': o['fields']['cook_time'],
                     'servings': o['fields']['servings'],
                     'ingredients': [],
+                    'photo': o['fields']['photo'],
+                    'cuisine': o['fields']['cuisine'],
+                    'course': o['fields']['course'],
+                    'tags': o['fields']['tags'],
+                    'comments': [],
                 }
             if o['model'] == 'ingredient.ingredientgroup':
                 ingredient_group_dict[o['pk']] = o['fields']['recipe']
+            if o['model'] == 'recipe_groups.cuisine':
+                cuisine_group_dict[o['pk']] = o['fields']['title']
+            if o['model'] == 'recipe_groups.course':
+                course_group_dict[o['pk']] = o['fields']['title']
+            if o['model'] == 'recipe_groups.tag':
+                tag_group_dict[o['pk']] = o['fields']['title']
 
         for o in recipe_json:
+            if o['model'] == 'rating.rating':
+                recipe_dict[o['fields']['recipe']]["comments"].append({
+                    "text": o['fields']['comment'],
+                    "rating": o['fields']['rating']
+                })
             if o['model'] == 'ingredient.ingredient':
                 ingredient = {
                     'food': o['fields']['title'],
@@ -63,6 +114,15 @@ class OpenEats(Integration):
                 }
                 recipe_dict[ingredient_group_dict[o['fields']['ingredient_group']]]['ingredients'].append(ingredient)
 
+        for k, r in recipe_dict.items():
+            if r["cuisine"] in cuisine_group_dict:
+                r["cuisine"] = cuisine_group_dict[r["cuisine"]]
+            if r["course"] in course_group_dict:
+                r["course"] = course_group_dict[r["course"]]
+            for index in range(len(r["tags"])):
+                if r["tags"][index] in tag_group_dict:
+                    r["tags"][index] = tag_group_dict[r["tags"][index]]
+
         return list(recipe_dict.values())
 
     def get_file_from_recipe(self, recipe):

cookbook/integration/paprika.py

@@ -5,6 +5,9 @@ import re
 from gettext import gettext as _
 from io import BytesIO
 
+import requests
+
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text
 from cookbook.integration.integration import Integration
@@ -55,7 +58,7 @@ class Paprika(Integration):
                 pass
 
             step = Step.objects.create(
-                instruction=instructions, space=self.request.space,
+                instruction=instructions, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
             )
 
             if 'description' in recipe_json and len(recipe_json['description'].strip()) > 500:
@@ -81,7 +84,24 @@ class Paprika(Integration):
 
             recipe.steps.add(step)
 
-            if recipe_json.get("photo_data", None):
-                self.import_recipe_image(recipe, BytesIO(base64.b64decode(recipe_json['photo_data'])), filetype='.jpeg')
+            # Paprika exports can have images in either of image_url, or photo_data.
+            # If a user takes an image himself, only photo_data will be set.
+            # If a user imports an image, both will be set. But the photo_data will be a center-cropped square resized version, so the image_url is preferred.
+            
+            # Try to download image if possible
+            try:
+                if recipe_json.get("image_url", None):
+                    url = recipe_json.get("image_url", None)
+                    if validate_import_url(url):
+                        response = requests.get(url)
+                        if response.status_code == 200 and len(response.content) > 0:
+                            self.import_recipe_image(recipe, BytesIO(response.content))
+            except Exception:
+                pass
+
+            # If no image downloaded, try to extract from photo_data
+            if not recipe.image:
+                if recipe_json.get("photo_data", None):
+                    self.import_recipe_image(recipe, BytesIO(base64.b64decode(recipe_json['photo_data'])), filetype='.jpeg')
 
             return recipe

cookbook/integration/pdfexport.py

@@ -1,21 +1,11 @@
-import json
-from io import BytesIO
-from re import match
-from zipfile import ZipFile
 import asyncio
-from pyppeteer import launch
-
-from rest_framework.renderers import JSONRenderer
-
-from cookbook.helper.image_processing import get_filetype
-from cookbook.integration.integration import Integration
-from cookbook.serializer import RecipeExportSerializer
-
-from cookbook.models import ExportLog
-from asgiref.sync import sync_to_async
 
 import django.core.management.commands.runserver as runserver
-import logging
+from asgiref.sync import sync_to_async
+from pyppeteer import launch
+
+from cookbook.integration.integration import Integration
+
 
 class PDFexport(Integration):
 
@@ -42,7 +32,6 @@ class PDFexport(Integration):
                    }
                    }
 
-
         files = []
         for recipe in recipes:
 
@@ -50,20 +39,18 @@ class PDFexport(Integration):
             await page.emulateMedia('print')
             await page.setCookie(cookies)
 
-            await page.goto('http://'+cmd.default_addr+':'+cmd.default_port+'/view/recipe/'+str(recipe.id), {'waitUntil': 'domcontentloaded'})
-            await page.waitForSelector('#printReady');
+            await page.goto('http://' + cmd.default_addr + ':' + cmd.default_port + '/view/recipe/' + str(recipe.id), {'waitUntil': 'domcontentloaded'})
+            await page.waitForSelector('#printReady')
 
             files.append([recipe.name + '.pdf', await page.pdf(options)])
-            await page.close();
+            await page.close()
 
             el.exported_recipes += 1
             el.msg += self.get_recipe_processed_msg(recipe)
             await sync_to_async(el.save, thread_sensitive=True)()
 
-
         await browser.close()
         return files
 
-
     def get_files_from_recipes(self, recipes, el, cookie):
         return asyncio.run(self.get_files_from_recipes_async(recipes, el, cookie))

cookbook/integration/pepperplate.py

@@ -35,7 +35,7 @@ class Pepperplate(Integration):
         recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space)
 
         step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n', space=self.request.space,
+            instruction='\n'.join(directions) + '\n\n', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
         )
 
         ingredient_parser = IngredientParser(self.request, True)

cookbook/integration/plantoeat.py

@@ -2,7 +2,9 @@ from io import BytesIO
 
 import requests
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
 from cookbook.models import Ingredient, Keyword, Recipe, Step
 
@@ -17,38 +19,45 @@ class Plantoeat(Integration):
         tags = None
         ingredients = []
         directions = []
-        description = ''
+        fields = {}
         for line in file.replace('\r', '').split('\n'):
             if line.strip() != '':
                 if 'Title:' in line:
-                    title = line.replace('Title:', '').replace('"', '').strip()
+                    fields['name'] = line.replace('Title:', '').replace('"', '').strip()
                 if 'Description:' in line:
-                    description = line.replace('Description:', '').strip()
-                if 'Source:' in line or 'Serves:' in line or 'Prep Time:' in line or 'Cook Time:' in line:
-                    directions.append(line.strip() + '\n')
+                    fields['description'] = line.replace('Description:', '').strip()
+                if 'Serves:' in line:
+                    fields['servings'] = parse_servings(line.replace('Serves:', '').strip())
+                if 'Source:' in line:
+                    fields['source_url'] = line.replace('Source:', '').strip()
                 if 'Photo Url:' in line:
                     image_url = line.replace('Photo Url:', '').strip()
+                if 'Prep Time:' in line:
+                    fields['working_time'] = parse_time(line.replace('Prep Time:', '').strip())
+                if 'Cook Time:' in line:
+                    fields['waiting_time'] = parse_time(line.replace('Cook Time:', '').strip())
                 if 'Tags:' in line:
                     tags = line.replace('Tags:', '').strip()
-                if ingredient_mode:
-                    if len(line) > 2 and 'Instructions:' not in line:
-                        ingredients.append(line.strip())
-                if direction_mode:
-                    if len(line) > 2:
-                        directions.append(line.strip() + '\n')
                 if 'Ingredients:' in line:
                     ingredient_mode = True
                 if 'Directions:' in line:
                     ingredient_mode = False
                     direction_mode = True
+                if ingredient_mode:
+                    if len(line) > 2 and 'Ingredients:' not in line:
+                        ingredients.append(line.strip())
+                if direction_mode:
+                    if len(line) > 2:
+                        directions.append(line.strip() + '\n')                
 
-        recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space)
+        recipe = Recipe.objects.create(**fields, created_by=self.request.user, internal=True, space=self.request.space)
 
         step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n', space=self.request.space,
+            instruction='\n'.join(directions) + '\n\n', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
         )
 
         if tags:
+            tags = tags.replace('^',',')
             for k in tags.split(','):
                 keyword, created = Keyword.objects.get_or_create(name=k.strip(), space=self.request.space)
                 recipe.keywords.add(keyword)
@@ -66,8 +75,9 @@ class Plantoeat(Integration):
 
         if image_url:
             try:
-                response = requests.get(image_url)
-                self.import_recipe_image(recipe, BytesIO(response.content))
+                if validate_import_url(image_url):
+                    response = requests.get(image_url)
+                    self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e:
                 print('failed to import image ', str(e))
 

cookbook/integration/recettetek.py

@@ -1,13 +1,14 @@
-import imghdr
 import json
 import re
 from io import BytesIO
 from zipfile import ZipFile
 
 import requests
-import validators
+from PIL import Image
 
 from django.utils.translation import gettext as _
+
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.image_processing import get_filetype
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
@@ -46,7 +47,7 @@ class RecetteTek(Integration):
         if not instructions:
             instructions = ''
 
-        step = Step.objects.create(instruction=instructions, space=self.request.space,)
+        step = Step.objects.create(instruction=instructions, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,)
 
         # Append the original import url to the step (if it exists)
         try:
@@ -61,7 +62,7 @@ class RecetteTek(Integration):
             ingredient_parser = IngredientParser(self.request, True)
             for ingredient in file['ingredients'].split('\n'):
                 if len(ingredient.strip()) > 0:
-                    amount, unit, food, note = ingredient_parser.parse(food)
+                    amount, unit, food, note = ingredient_parser.parse(ingredient.strip())
                     f = ingredient_parser.get_food(ingredient)
                     u = ingredient_parser.get_unit(unit)
                     step.ingredients.add(Ingredient.objects.create(
@@ -125,9 +126,9 @@ class RecetteTek(Integration):
             else:
                 if file['originalPicture'] != '':
                     url = file['originalPicture']
-                    if validators.url(url, public=True):
+                    if validate_import_url(url):
                         response = requests.get(url)
-                        if imghdr.what(BytesIO(response.content)) is not None:
+                        if Image.open(BytesIO(response.content)).verify():
                             self.import_recipe_image(recipe, BytesIO(response.content), filetype=get_filetype(file['originalPicture']))
                         else:
                             raise Exception("Original image failed to download.")

cookbook/integration/recipekeeper.py

@@ -41,7 +41,7 @@ class RecipeKeeper(Integration):
         except AttributeError:
             pass
 
-        step = Step.objects.create(instruction='', space=self.request.space,)
+        step = Step.objects.create(instruction='', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients, )
 
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in file.find("div", {"itemprop": "recipeIngredients"}).findChildren("p"):
@@ -51,13 +51,20 @@ class RecipeKeeper(Integration):
             f = ingredient_parser.get_food(food)
             u = ingredient_parser.get_unit(unit)
             step.ingredients.add(Ingredient.objects.create(
-                food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
+                food=f, unit=u, amount=amount, note=note, original_text=str(ingredient).replace('<p>', '').replace('</p>', ''), space=self.request.space,
             ))
 
         for s in file.find("div", {"itemprop": "recipeDirections"}).find_all("p"):
             if s.text == "":
                 continue
             step.instruction += s.text + ' \n'
+            step.save()
+
+        for s in file.find("div", {"itemprop": "recipeNotes"}).find_all("p"):
+            if s.text == "":
+                continue
+            step.instruction += s.text + ' \n'
+            step.save()
 
         if file.find("span", {"itemprop": "recipeSource"}).text != '':
             step.instruction += "\n\n" + _("Imported from") + ": " + file.find("span", {"itemprop": "recipeSource"}).text

cookbook/integration/recipesage.py

@@ -2,9 +2,10 @@ import json
 from io import BytesIO
 
 import requests
-import validators
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
 from cookbook.models import Ingredient, Recipe, Step
 
@@ -18,25 +19,27 @@ class RecipeSage(Integration):
             created_by=self.request.user, internal=True,
             space=self.request.space)
 
+        if file['recipeYield'] != '':
+            recipe.servings = parse_servings(file['recipeYield'])
+            recipe.servings_text = parse_servings_text(file['recipeYield'])
+
         try:
-            if file['recipeYield'] != '':
-                recipe.servings = int(file['recipeYield'])
+            if 'totalTime' in file and file['totalTime'] != '':
+                recipe.working_time = parse_time(file['totalTime'])
 
-            if file['totalTime'] != '':
-                recipe.waiting_time = int(file['totalTime']) - int(file['timePrep'])
-
-            if file['prepTime'] != '':
-                recipe.working_time = int(file['timePrep'])
-
-            recipe.save()
+            if 'timePrep' in file and file['prepTime'] != '':
+                recipe.working_time = parse_time(file['timePrep'])
+                recipe.waiting_time = parse_time(file['totalTime']) - parse_time(file['timePrep'])
         except Exception as e:
-            print('failed to parse yield or time ', str(e))
+            print('failed to parse time ', str(e))
+
+        recipe.save()
 
         ingredient_parser = IngredientParser(self.request, True)
         ingredients_added = False
         for s in file['recipeInstructions']:
             step = Step.objects.create(
-                instruction=s['text'], space=self.request.space,
+                instruction=s['text'], space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
             )
             if not ingredients_added:
                 ingredients_added = True
@@ -53,7 +56,7 @@ class RecipeSage(Integration):
         if len(file['image']) > 0:
             try:
                 url = file['image'][0]
-                if validators.url(url, public=True):
+                if validate_import_url(url):
                     response = requests.get(url)
                     self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e:

cookbook/integration/rezeptsuitede.py

@@ -0,0 +1,78 @@
+import base64
+from io import BytesIO
+from lxml import etree
+
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text
+from cookbook.integration.integration import Integration
+from cookbook.models import Ingredient, Keyword, Recipe, Step
+
+
+class Rezeptsuitede(Integration):
+
+    def split_recipe_file(self, file):
+        return etree.parse(file).getroot().getchildren()
+
+    def get_recipe_from_file(self, file):
+        recipe_xml = file
+
+        recipe = Recipe.objects.create(
+            name=recipe_xml.find('head').attrib['title'].strip(),
+            created_by=self.request.user, internal=True, space=self.request.space)
+
+        try:
+            if recipe_xml.find('head').attrib['servingtype']:
+                recipe.servings = parse_servings(recipe_xml.find('head').attrib['servingtype'].strip())
+                recipe.servings_text = parse_servings_text(recipe_xml.find('head').attrib['servingtype'].strip())
+        except KeyError:
+            pass
+
+        if recipe_xml.find('remark') is not None:  # description is a list of <li>'s with text
+            if recipe_xml.find('remark').find('line') is not None:
+                recipe.description = recipe_xml.find('remark').find('line').text[:512]
+
+        for prep in recipe_xml.findall('preparation'):
+            try:
+                if prep.find('step').text:
+                    step = Step.objects.create(
+                        instruction=prep.find('step').text.strip(), space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+                    )
+                    recipe.steps.add(step)
+            except Exception:
+                pass
+
+        ingredient_parser = IngredientParser(self.request, True)
+
+        if recipe_xml.find('part').find('ingredient') is not None:
+            ingredient_step = recipe.steps.first()
+            if ingredient_step is None:
+                ingredient_step = Step.objects.create(space=self.request.space, instruction='')
+
+            for ingredient in recipe_xml.find('part').findall('ingredient'):
+                f = ingredient_parser.get_food(ingredient.attrib['item'])
+                u = ingredient_parser.get_unit(ingredient.attrib['unit'])
+                amount = 0
+                if ingredient.attrib['qty'].strip() != '':
+                    try:
+                        amount, unit, note = ingredient_parser.parse_amount(ingredient.attrib['qty'])
+                    except ValueError:  # sometimes quantities contain words which cant be parsed
+                        pass
+                ingredient_step.ingredients.add(Ingredient.objects.create(food=f, unit=u, amount=amount, space=self.request.space, ))
+
+        try:
+            k, created = Keyword.objects.get_or_create(name=recipe_xml.find('head').find('cat').text.strip(), space=self.request.space)
+            recipe.keywords.add(k)
+        except Exception:
+            pass
+
+        recipe.save()
+
+        try:
+            self.import_recipe_image(recipe, BytesIO(base64.b64decode(recipe_xml.find('head').find('picbin').text)), filetype='.jpeg')
+        except BaseException:
+            pass
+
+        return recipe
+
+    def get_file_from_recipe(self, recipe):
+        raise NotImplementedError('Method not implemented in storage integration')

cookbook/integration/rezkonv.py

@@ -38,7 +38,7 @@ class RezKonv(Integration):
             recipe.keywords.add(keyword)
 
         step = Step.objects.create(
-            instruction='  \n'.join(directions) + '\n\n', space=self.request.space,
+            instruction='  \n'.join(directions) + '\n\n', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
         )
 
         ingredient_parser = IngredientParser(self.request, True)
@@ -60,8 +60,8 @@ class RezKonv(Integration):
     def split_recipe_file(self, file):
         recipe_list = []
         current_recipe = ''
-        encoding_list = ['windows-1250',
-                         'latin-1']  # TODO build algorithm to try trough encodings and fail if none work, use for all importers
+        # TODO build algorithm to try trough encodings and fail if none work, use for all importers
+        # encoding_list = ['windows-1250', 'latin-1']
         encoding = 'windows-1250'
         for fl in file.readlines():
             try:

cookbook/integration/saffron.py

@@ -43,7 +43,7 @@ class Saffron(Integration):
 
         recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space, )
 
-        step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space, )
+        step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients, )
 
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
@@ -59,11 +59,11 @@ class Saffron(Integration):
 
     def get_file_from_recipe(self, recipe):
 
-        data = "Title: "+recipe.name if recipe.name else ""+"\n"
-        data += "Description: "+recipe.description if recipe.description else ""+"\n"
+        data = "Title: " + recipe.name if recipe.name else "" + "\n"
+        data += "Description: " + recipe.description if recipe.description else "" + "\n"
         data += "Source: \n"
         data += "Original URL: \n"
-        data += "Yield: "+str(recipe.servings)+"\n"
+        data += "Yield: " + str(recipe.servings) + "\n"
         data += "Cookbook: \n"
         data += "Section: \n"
         data += "Image: \n"
@@ -78,13 +78,13 @@ class Saffron(Integration):
 
         data += "Ingredients: \n"
         for ingredient in recipeIngredient:
-            data += ingredient+"\n"
+            data += ingredient + "\n"
 
         data += "Instructions: \n"
         for instruction in recipeInstructions:
-            data += instruction+"\n"
+            data += instruction + "\n"
 
-        return recipe.name+'.txt', data
+        return recipe.name + '.txt', data
 
     def get_files_from_recipes(self, recipes, el, cookie):
         files = []