Merge branch 'develop'

add community links

.coveragerc

@@ -0,0 +1,11 @@
+[run]
+omit =
+  */apps.py,
+  */migrations/*,
+  */settings*,
+  */test*,
+  */tests/*,
+  *urls.py,
+  */wsgi*,
+  manage.py,
+  *__init__*

.devcontainer/Dockerfile

@@ -0,0 +1,31 @@
+FROM python:3.10-alpine3.18
+
+#Install all dependencies.
+RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg libwebp libxml2-dev libxslt-dev openldap git yarn
+
+# Fix libxml error from xmlsec https://github.com/xmlsec/python-xmlsec/issues/257#issuecomment-1738620862
+RUN echo "https://dl-cdn.alpinelinux.org/alpine/v3.15/community/" | tee -a /etc/apk/repositories
+RUN echo "https://dl-cdn.alpinelinux.org/alpine/v3.15/main" | tee -a /etc/apk/repositories
+RUN apk add --no-cache libxml2-dev=2.9.14-r2  xmlsec-dev=1.2.33-r0
+
+#Print all logs without buffering it.
+ENV PYTHONUNBUFFERED 1
+
+#This port will be used by gunicorn.
+EXPOSE 8000
+
+#This port will be used by vue
+EXPOSE 8080
+
+#Install all python dependencies to the image
+COPY requirements.txt /tmp/pip-tmp/
+
+RUN \
+    if [ `apk --print-arch` = "armv7" ]; then \
+    printf "[global]\nextra-index-url=https://www.piwheels.org/simple\n" > /etc/pip.conf ; \
+    fi
+RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-dev jpeg-dev libwebp-dev openssl-dev libffi-dev cargo openldap-dev python3-dev && \
+    echo -n "INPUT ( libldap.so )" > /usr/lib/libldap_r.so && \
+    pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt && \
+    rm -rf /tmp/pip-tmp && \
+    apk --purge del .build-deps

.devcontainer/devcontainer.json

@@ -0,0 +1,31 @@
+// For format details, see https://aka.ms/devcontainer.json.
+{
+	"name": "Tandoor Dev Container",
+	"build": { "context": "..", "dockerfile": "Dockerfile" },
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [8000, 8080],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "pip3 install --user -r requirements.txt"
+
+	// Configure tool-specific properties.
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.debugpy",
+				"ms-python.python"
+			]
+		}
+	},
+
+	"containerEnv": {
+		"CSRF_TRUSTED_ORIGINS": "http://localhost:8000,http://localhost:8080"
+	}
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.dockerignore

@@ -29,3 +29,4 @@ vue/babel.config*
 vue/package.json
 vue/tsconfig.json
 vue/src/utils/openapi
+venv

.env.template

@@ -6,6 +6,12 @@
 # random secret key, use for example `base64 /dev/urandom | head -c50` to generate one
 SECRET_KEY=
 
+# your default timezone See https://timezonedb.com/time-zones for a list of timezones
+TZ=Europe/Berlin
+
+# allowed hosts (see documentation), should be set to your hostname(s) but might be * (default) for some proxies/providers
+# ALLOWED_HOSTS=recipes.mydomain.com
+
 # add only a database password if you want to run with the default postgres, otherwise change settings accordingly
 DB_ENGINE=django.db.backends.postgresql
 POSTGRES_HOST=db_recipes

.flake8

@@ -0,0 +1,22 @@
+[flake8]
+extend-ignore =
+    # Whitespace before ':' - Required for black compatibility
+    E203,
+    # Line break occurred before a binary operator - Required for black compatibility
+    W503,
+    # Comparison to False should be 'if cond is False:' or 'if not cond:'
+    E712
+exclude =
+    .git,
+    **/__pycache__,
+        **/.git,
+        **/.svn,
+        **/.hg,
+        **/CVS,
+        **/.DS_Store,
+        .vscode,
+        **/*.pyc
+per-file-ignores=
+    cookbook/apps.py:F401
+max-line-length = 179
+

.github/dependabot.yml

@@ -1,17 +1,22 @@
 # To get started with Dependabot version updates, you'll need to specify which
 # package ecosystems to update and where the package manifests are located.
 # Please see the documentation for all configuration options:
-# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
 
 version: 2
 updates:
+  - package-ecosystem: "devcontainers"
+    directory: "/"
+    schedule:
+      interval: weekly
+
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
       interval: "monthly"
 
   - package-ecosystem: "npm"
-    directory: "/vue/"
+    directory: "/vue3/"
     schedule:
       interval: "monthly"
 

.github/workflows/build-docker-open-data.yml

@@ -1,110 +0,0 @@
-name: Build Docker Container with open data plugin installed
-
-on: push
-
-jobs:
-  build-container:
-    name: Build ${{ matrix.name }} Container
-    runs-on: ubuntu-latest
-    if: github.repository_owner == 'TandoorRecipes'
-    continue-on-error: ${{ matrix.continue-on-error }}
-    permissions:
-      contents: read
-      packages: write
-    strategy:
-      matrix:
-        include:
-          # Standard build config
-          - name: Standard
-            dockerfile: Dockerfile
-            platforms: linux/amd64,linux/arm64
-            suffix: ""
-            continue-on-error: false
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Get version number
-        id: get_version
-        run: |
-          if [[ "$GITHUB_REF" = refs/tags/* ]]; then
-            echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
-          elif [[ "$GITHUB_REF" = refs/heads/beta ]]; then
-            echo VERSION=beta >> $GITHUB_OUTPUT
-          else
-            echo VERSION=develop >> $GITHUB_OUTPUT
-          fi
-
-      # clone open data plugin
-      - name: clone open data plugin repo
-        uses: actions/checkout@master
-        with:
-          repository: TandoorRecipes/open_data_plugin
-          ref: master
-          path: ./recipes/plugins/open_data_plugin
-
-      # Build Vue frontend
-      - uses: actions/setup-node@v4
-        with:
-          node-version: '18'
-          cache: yarn
-          cache-dependency-path: vue/yarn.lock
-      - name: Install dependencies
-        working-directory: ./vue
-        run: yarn install --frozen-lockfile
-      - name: Build dependencies
-        working-directory: ./vue
-        run: yarn build
-
-      - name: Setup Open Data Plugin Links
-        working-directory: ./recipes/plugins/open_data_plugin
-        run: python setup_repo.py
-        
-      - name: Build Open Data Frontend
-        working-directory: ./recipes/plugins/open_data_plugin/vue
-        run: yarn build
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Set up Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        if: github.secret_source == 'Actions'
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        if: github.secret_source == 'Actions'
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ github.token }}
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            vabene1111/recipes
-            ghcr.io/TandoorRecipes/recipes
-          flavor: |
-            latest=false
-            suffix=${{ matrix.suffix }}
-          tags: |
-            type=raw,value=latest,suffix=-open-data-plugin,enable=${{ startsWith(github.ref, 'refs/tags/') }}
-            type=semver,suffix=-open-data-plugin,pattern={{version}}
-            type=semver,suffix=-open-data-plugin,pattern={{major}}.{{minor}}
-            type=semver,suffix=-open-data-plugin,pattern={{major}}
-            type=ref,suffix=-open-data-plugin,event=branch
-      - name: Build and Push
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ${{ matrix.dockerfile }}
-          pull: true
-          push: ${{ github.secret_source == 'Actions' }}
-          platforms: ${{ matrix.platforms }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max

.github/workflows/build-docker.yml

@@ -17,7 +17,7 @@ jobs:
           # Standard build config
           - name: Standard
             dockerfile: Dockerfile
-            platforms: linux/amd64,linux/arm64,linux/arm/v7
+            platforms: linux/amd64,linux/arm64
             suffix: ""
             continue-on-error: false
     steps:
@@ -34,17 +34,17 @@ jobs:
             echo VERSION=develop >> $GITHUB_OUTPUT
           fi
 
-      # Build Vue frontend
+      # Build Vue 3 frontend
       - uses: actions/setup-node@v4
         with:
-          node-version: '18'
+          node-version: '22'
           cache: yarn
-          cache-dependency-path: vue/yarn.lock
+          cache-dependency-path: vue3/yarn.lock
       - name: Install dependencies
-        working-directory: ./vue
+        working-directory: ./vue3
         run: yarn install --frozen-lockfile
       - name: Build dependencies
-        working-directory: ./vue
+        working-directory: ./vue3
         run: yarn build
 
       - name: Set up QEMU
@@ -107,7 +107,7 @@ jobs:
       - name: Discord notification
         env:
           DISCORD_WEBHOOK: ${{ secrets.DISCORD_RELEASE_WEBHOOK }}
-        uses: Ilshidur/action-discord@0.3.2
+        uses: Ilshidur/action-discord@0.4.0
         with:
           args: '🚀 Version {{ VERSION }} of tandoor has been released 🥳 Check it out https://github.com/vabene1111/recipes/releases/tag/{{ VERSION }}'
 
@@ -121,6 +121,6 @@ jobs:
       - name: Discord notification
         env:
           DISCORD_WEBHOOK: ${{ secrets.DISCORD_BETA_WEBHOOK }}
-        uses: Ilshidur/action-discord@0.3.2
+        uses: Ilshidur/action-discord@0.4.0
         with:
-          args: '🚀 The BETA Image has been updated! 🥳'
+          args: '🚀 The Tandoor 2 Image has been updated! 🥳'

.github/workflows/ci.yml

@@ -3,38 +3,75 @@ name: Continuous Integration
 on: [push, pull_request]
 
 jobs:
-  build:
-    if: github.repository_owner == 'TandoorRecipes'
-    runs-on: ubuntu-latest
-    strategy:
-      max-parallel: 4
-      matrix:
-        python-version: ['3.10']
+    build:
+        if: github.repository_owner == 'TandoorRecipes'
+        runs-on: ubuntu-latest
+        strategy:
+            max-parallel: 4
+            matrix:
+                python-version: ["3.12"]
+                node-version: ["22"]
+        steps:
+            - uses: actions/checkout@v4
+            - uses: awalsh128/cache-apt-pkgs-action@v1.5.1
+              with:
+                  packages: libsasl2-dev python3-dev libxml2-dev libxmlsec1-dev libxslt-dev libxmlsec1-openssl libxslt-dev libldap2-dev libssl-dev gcc musl-dev postgresql-dev zlib-dev jpeg-dev libwebp-dev openssl-dev libffi-dev cargo openldap-dev python3-dev xmlsec-dev xmlsec build-base g++ curl
+                  version: 1.0
 
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.10'
-    # Build Vue frontend
-    - uses: actions/setup-node@v4
-      with:
-        node-version: '18'
-    - name: Install Vue dependencies
-      working-directory: ./vue
-      run: yarn install
-    - name: Build Vue dependencies
-      working-directory: ./vue
-      run: yarn build
-    - name: Install Django dependencies
-      run: |
-        sudo apt-get -y update
-        sudo apt-get install -y libsasl2-dev python3-dev libldap2-dev libssl-dev
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        python3 manage.py collectstatic --noinput
-        python3 manage.py collectstatic_js_reverse
-    - name: Django Testing project
-      run: |
-        pytest
+            # Setup python & dependencies
+            - name: Set up Python ${{ matrix.python-version }}
+              uses: actions/setup-python@v5
+              with:
+                  python-version: ${{ matrix.python-version }}
+                  cache: "pip"
+
+            - name: Install Python Dependencies
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install -r requirements.txt
+
+            - name: Cache StaticFiles
+              uses: actions/cache@v4
+              id: django_cache
+              with:
+                  path: |
+                      ./cookbook/static
+                      ./staticfiles
+                  key: |
+                      ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue3/src/*') }}
+
+            # Build Vue frontend & Dependencies
+            - name: Set up Node ${{ matrix.node-version }}
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              uses: actions/setup-node@v4
+              with:
+                  node-version: ${{ matrix.node-version }}
+                  cache: "yarn"
+                  cache-dependency-path: ./vue3/yarn.lock
+
+            - name: Install Vue dependencies
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              working-directory: ./vue3
+              run: yarn install
+
+            - name: Build Vue dependencies
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              working-directory: ./vue3
+              run: yarn build
+
+            - name: Compile Django StaticFiles
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              run: |
+                  python3 manage.py collectstatic --noinput
+
+            - uses: actions/cache/save@v4
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              with:
+                  path: |
+                      ./cookbook/static
+                      ./staticfiles
+                  key: |
+                      ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }}
+
+            - name: Django Testing project
+              run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml

.github/workflows/docs.yml

@@ -1,12 +1,15 @@
 name: Make Docs
 on:
-  push:
-    branches:
-      - master
+  # the 1st condition
+  workflow_run:
+    workflows: ["Continuous Integration"]
+    branches: [master]
+    types:
+      - completed
 
 jobs:
   deploy:
-    if: github.repository_owner == 'TandoorRecipes'
+    if: github.repository_owner == 'TandoorRecipes' && ${{ github.event.workflow_run.conclusion == 'success' }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -14,4 +17,4 @@ jobs:
         with:
           python-version: 3.x
       - run: pip install mkdocs-material mkdocs-include-markdown-plugin
-      - run: mkdocs gh-deploy --force
+      - run: mkdocs gh-deploy --force

.gitignore

@@ -43,9 +43,15 @@ htmlcov/
 nosetests.xml
 coverage.xml
 *,cover
+docs/reports/**
 
 # Django stuff:
 *.log
+mediafiles/
+*.sqlite3*
+staticfiles/
+postgresql/
+data/
 
 # Sphinx documentation
 docs/_build/
@@ -54,36 +60,34 @@ docs/_build/
 target/
 
 \.idea/dataSources/
-
 \.idea/dataSources\.xml
 
 \.idea/dataSources\.local\.xml
 
-venv/
-
-mediafiles/
-
-*.sqlite3*
-
 \.idea/workspace\.xml
 
 \.idea/misc\.xml
 
 # Deployment
-
 \.env
-staticfiles/
-postgresql/
-data/
-
 
+cookbook/static/vue
+vue/webpack-stats.json
 /docker-compose.override.yml
 vue/node_modules
-plugins
-.vscode/
+/recipes/plugins
 vetur.config.js
 cookbook/static/vue
 vue/webpack-stats.json
 cookbook/templates/sw.js
-.prettierignore
 vue/.yarn
+vue3/.vite
+
+# Configs
+vetur.config.js
+venv/
+.idea/easy-i18n.xml
+cookbook/static/vue3
+vue3/node_modules
+cookbook/tests/other/docs/reports/tests/tests.html
+cookbook/tests/other/docs/reports/tests/pytest.xml

.idea/dictionaries/vaben.xml

@@ -1,6 +1,7 @@
 <component name="ProjectDictionaryState">
   <dictionary name="vaben">
     <words>
+      <w>mealplan</w>
       <w>pinia</w>
       <w>selfhosted</w>
       <w>unapplied</w>

.idea/prettier.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PrettierConfiguration">
+    <option name="myConfigurationMode" value="AUTOMATIC" />
+  </component>
+</project>

.idea/recipes.iml

@@ -18,7 +18,7 @@
       <excludeFolder url="file://$MODULE_DIR$/staticfiles" />
       <excludeFolder url="file://$MODULE_DIR$/venv" />
     </content>
-    <orderEntry type="inheritedJdk" />
+    <orderEntry type="jdk" jdkName="Python 3.12 (recipes)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="TemplatesService">

.idea/vcs.xml

@@ -2,5 +2,7 @@
 <project version="4">
   <component name="VcsDirectoryMappings">
     <mapping directory="" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/recipes/plugins/enterprise_plugin" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/recipes/plugins/open_data_plugin" vcs="Git" />
   </component>
 </project>

.idea/watcherTasks.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectTasksOptions">
+    <TaskOptions isEnabled="false">
+      <option name="arguments" value="-m flake8 $FilePath$ --config $ContentRoot$\.flake8" />
+      <option name="checkSyntaxErrors" value="true" />
+      <option name="description" />
+      <option name="exitCodeBehavior" value="NEVER" />
+      <option name="fileExtension" value="py" />
+      <option name="immediateSync" value="false" />
+      <option name="name" value="Flake8 Watcher" />
+      <option name="output" value="$FilePath$" />
+      <option name="outputFilters">
+        <array>
+          <FilterInfo>
+            <option name="description" value="" />
+            <option name="name" value="" />
+            <option name="regExp" value="$FILE_PATH$:$LINE$:$COLUMN$: $MESSAGE$" />
+          </FilterInfo>
+        </array>
+      </option>
+      <option name="outputFromStdout" value="false" />
+      <option name="program" value="$PyInterpreterDirectory$/python" />
+      <option name="runOnExternalChanges" value="false" />
+      <option name="scopeName" value="Current File" />
+      <option name="trackOnlyRoot" value="false" />
+      <option name="workingDir" value="" />
+      <envs />
+    </TaskOptions>
+    <TaskOptions isEnabled="false">
+      <option name="arguments" value="-m isort $FilePath$" />
+      <option name="checkSyntaxErrors" value="true" />
+      <option name="description" />
+      <option name="exitCodeBehavior" value="ERROR" />
+      <option name="fileExtension" value="py" />
+      <option name="immediateSync" value="false" />
+      <option name="name" value="isort Watcher" />
+      <option name="output" value="$FilePath$" />
+      <option name="outputFilters">
+        <array />
+      </option>
+      <option name="outputFromStdout" value="false" />
+      <option name="program" value="$PyInterpreterDirectory$/python" />
+      <option name="runOnExternalChanges" value="false" />
+      <option name="scopeName" value="Project Files" />
+      <option name="trackOnlyRoot" value="false" />
+      <option name="workingDir" value="" />
+      <envs />
+    </TaskOptions>
+    <TaskOptions isEnabled="false">
+      <option name="arguments" value="-m yapf -i $FilePath$" />
+      <option name="checkSyntaxErrors" value="true" />
+      <option name="description" />
+      <option name="exitCodeBehavior" value="NEVER" />
+      <option name="fileExtension" value="py" />
+      <option name="immediateSync" value="false" />
+      <option name="name" value="YAPF" />
+      <option name="output" value="$FilePath$" />
+      <option name="outputFilters">
+        <array />
+      </option>
+      <option name="outputFromStdout" value="false" />
+      <option name="program" value="$PyInterpreterDirectory$/python" />
+      <option name="runOnExternalChanges" value="false" />
+      <option name="scopeName" value="Project Files" />
+      <option name="trackOnlyRoot" value="false" />
+      <option name="workingDir" value="" />
+      <envs />
+    </TaskOptions>
+    <TaskOptions isEnabled="false">
+      <option name="arguments" value="--cwd $ProjectFileDir$\vue prettier -w --config $ProjectFileDir$\.prettierrc $FilePath$" />
+      <option name="checkSyntaxErrors" value="true" />
+      <option name="description" />
+      <option name="exitCodeBehavior" value="ERROR" />
+      <option name="fileExtension" value="*" />
+      <option name="immediateSync" value="true" />
+      <option name="name" value="Prettier" />
+      <option name="output" value="" />
+      <option name="outputFilters">
+        <array />
+      </option>
+      <option name="outputFromStdout" value="false" />
+      <option name="program" value="yarn" />
+      <option name="runOnExternalChanges" value="true" />
+      <option name="scopeName" value="Prettier" />
+      <option name="trackOnlyRoot" value="false" />
+      <option name="workingDir" value="" />
+      <envs />
+    </TaskOptions>
+  </component>
+</project>

.prettierignore

@@ -0,0 +1,13 @@
+# generated files
+api.ts
+vue/src/apps/*.js
+vue/node_modules
+staticfiles/
+docs/reports/
+/vue3/src/openapi/
+
+# ignored files - prettier interferes with django templates and github actions
+*.html
+*.yml
+*.yaml
+

.prettierrc

@@ -0,0 +1,7 @@
+{
+    "printWidth": 179,
+    "trailingComma": "es5",
+    "tabWidth": 2,
+    "semi": false,
+    "experimentalTernaries": true
+}

.vscode/launch.json

@@ -0,0 +1,33 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+      {
+        "name": "Python Debugger: Django",
+        "type": "debugpy",
+        "request": "launch",
+        "program": "${workspaceFolder}/manage.py",
+        "args": ["runserver"],
+        "django": true,
+        "justMyCode": true
+      },
+    {
+      "name": "Python: Debug Tests",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "purpose": [
+        "debug-test"
+      ],
+      "console": "integratedTerminal",
+      "env": {
+        // coverage and pytest can't both be running at the same time
+        "PYTEST_ADDOPTS": "--no-cov -n 0"
+      },
+      "django": true,
+      "justMyCode": true
+    },
+    ]
+  }

.vscode/settings.json

@@ -0,0 +1,14 @@
+{
+    "python.testing.pytestArgs": [
+        "cookbook/tests"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true,
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "[python]": {
+        "editor.defaultFormatter": "eeyore.yapf",
+    },
+    "yapf.args": [],
+    "isort.args": []
+}

.vscode/tasks.json

@@ -0,0 +1,131 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "Run Migrations",
+      "type": "shell",
+      "command": "python3 manage.py migrate"
+    },
+    {
+      "label": "Collect Static Files",
+      "type": "shell",
+      "command": "python3 manage.py collectstatic",
+      "dependsOn": ["Yarn Build"]
+    },
+    {
+      "label": "Setup Dev Server",
+      "dependsOn": ["Run Migrations", "Yarn Build"]
+    },
+    {
+      "label": "Run Dev Server",
+      "type": "shell",
+      "dependsOn": ["Setup Dev Server"],
+      "command": "python3 manage.py runserver"
+    },
+    {
+      "label": "Yarn Install",
+      "dependsOn": ["Yarn Install - Vue", "Yarn Install - Vue3"]
+    },
+    {
+      "label": "Yarn Install - Vue",
+      "type": "shell",
+      "command": "yarn install --force",
+      "options": {
+        "cwd": "${workspaceFolder}/vue"
+      }
+    },
+    {
+      "label": "Yarn Install - Vue3",
+      "type": "shell",
+      "command": "yarn install --force",
+      "options": {
+        "cwd": "${workspaceFolder}/vue3"
+      }
+    },
+    {
+      "label": "Generate API",
+      "dependsOn": ["Generate API - Vue", "Generate API - Vue3"]
+    },
+    {
+      "label": "Generate API - Vue",
+      "type": "shell",
+      "command": "openapi-generator-cli generate -g typescript-axios -i http://127.0.0.1:8000/openapi/",
+      "options": {
+        "cwd": "${workspaceFolder}/vue/src/utils/openapi"
+      }
+    },
+    {
+      "label": "Generate API - Vue3",
+      "type": "shell",
+      "command": "openapi-generator-cli generate -g typescript-fetch -i http://127.0.0.1:8000/openapi/",
+      "options": {
+        "cwd": "${workspaceFolder}/vue3/src/openapi"
+      }
+    },
+    {
+      "label": "Yarn Serve",
+      "type": "shell",
+      "command": "yarn serve",
+      "dependsOn": ["Yarn Install - Vue"],
+      "options": {
+        "cwd": "${workspaceFolder}/vue"
+      }
+    },
+    {
+      "label": "Vite Serve",
+      "type": "shell",
+      "command": "vite",
+      "dependsOn": ["Yarn Install - Vue3"],
+      "options": {
+        "cwd": "${workspaceFolder}/vue3"
+      }
+    },
+    {
+      "label": "Yarn Build",
+      "dependsOn": ["Yarn Build - Vue", "Vite Build - Vue3"],
+      "group": "build"
+    },
+    {
+      "label": "Yarn Build - Vue",
+      "type": "shell",
+      "command": "yarn build",
+      "dependsOn": ["Yarn Install - Vue"],
+      "options": {
+        "cwd": "${workspaceFolder}/vue"
+      },
+      "group": "build"
+    },
+    {
+      "label": "Vite Build - Vue3",
+      "type": "shell",
+      "command": "vite build",
+      "dependsOn": ["Yarn Install - Vue3"],
+      "options": {
+        "cwd": "${workspaceFolder}/vue3"
+      },
+      "group": "build"
+    },
+    {
+      "label": "Setup Tests",
+      "dependsOn": ["Run Migrations", "Collect Static Files"]
+    },
+    {
+      "label": "Run all pytests",
+      "type": "shell",
+      "command": "python3 -m pytest cookbook/tests",
+      "dependsOn": ["Setup Tests"],
+      "group": "test"
+    },
+    {
+      "label": "Setup Documentation Dependencies",
+      "type": "shell",
+      "command": "pip install mkdocs-material mkdocs-include-markdown-plugin"
+    },
+    {
+      "label": "Serve Documentation",
+      "type": "shell",
+      "command": "mkdocs serve",
+      "dependsOn": ["Setup Documentation Dependencies"]
+    }
+  ]
+}

CONTRIBUTERS.md

@@ -20,6 +20,7 @@ Below are some of the larger contributions made yet.
 - [murphy83] added support for IPv6 #1490
 - [TheHaf] added custom serving size component #1411
 - [lostlont] added LDAP support #960
+- [c0mputerguru] added devcontainers for ease of development
 
 ## Translations
 

Dockerfile

@@ -1,13 +1,14 @@
-FROM python:3.10-alpine3.18
+FROM python:3.13-alpine3.22
 
 #Install all dependencies.
-RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg libwebp libxml2-dev libxslt-dev openldap git
+RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg libwebp libxml2-dev libxslt-dev openldap git libgcc libstdc++ nginx tini envsubst
 
 #Print all logs without buffering it.
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONUNBUFFERED=1 \
+    DOCKER=true
 
 #This port will be used by gunicorn.
-EXPOSE 8080
+EXPOSE 80 8080
 
 #Create app dir and install requirements.
 RUN mkdir /opt/recipes
@@ -15,26 +16,38 @@ WORKDIR /opt/recipes
 
 COPY requirements.txt ./
 
-RUN \
-    if [ `apk --print-arch` = "armv7" ]; then \
-    printf "[global]\nextra-index-url=https://www.piwheels.org/simple\n" > /etc/pip.conf ; \
-    fi
-RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-dev jpeg-dev libwebp-dev openssl-dev libffi-dev cargo openldap-dev python3-dev && \
-    echo -n "INPUT ( libldap.so )" > /usr/lib/libldap_r.so && \
+# remove Development dependencies from requirements.txt
+RUN sed -i '/# Development/,$d' requirements.txt
+RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-dev jpeg-dev libwebp-dev openssl-dev libffi-dev cargo openldap-dev python3-dev xmlsec-dev xmlsec build-base g++ curl rust && \
     python -m venv venv && \
     /opt/recipes/venv/bin/python -m pip install --upgrade pip && \
-    venv/bin/pip install wheel==0.37.1 && \
-    venv/bin/pip install setuptools_rust==1.1.2 && \
+    venv/bin/pip debug -v && \
+    venv/bin/pip install wheel==0.45.1 && \
+    venv/bin/pip install setuptools_rust==1.10.2 && \
     venv/bin/pip install -r requirements.txt --no-cache-dir &&\
     apk --purge del .build-deps
 
 #Copy project and execute it.
 COPY . ./
 
+# delete default nginx config and link it to tandoors config
+# create symlinks to access and error log to show them on stdout
+RUN rm -rf /etc/nginx/http.d && \
+    ln -s /opt/recipes/http.d /etc/nginx/http.d && \
+    ln -sf /dev/stdout /var/log/nginx/access.log && \
+    ln -sf /dev/stderr /var/log/nginx/error.log
+
+# commented for now https://github.com/TandoorRecipes/recipes/issues/3478
+#HEALTHCHECK --interval=30s \
+#            --timeout=5s \
+#            --start-period=10s \
+#            --retries=3 \
+#            CMD [ "/usr/bin/wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:8080/openapi" ]
+
 # collect information from git repositories
 RUN /opt/recipes/venv/bin/python version.py
 # delete git repositories to reduce image size
 RUN find . -type d -name ".git" | xargs rm -rf
 
 RUN chmod +x boot.sh
-ENTRYPOINT ["/opt/recipes/boot.sh"]
+ENTRYPOINT ["/sbin/tini", "--", "/opt/recipes/boot.sh"]

README.md

@@ -23,6 +23,7 @@
 <a href="https://docs.tandoor.dev/install/docker/" target="_blank" rel="noopener noreferrer">Installation</a> •
 <a href="https://docs.tandoor.dev/" target="_blank" rel="noopener noreferrer">Docs</a> •
 <a href="https://app.tandoor.dev/accounts/login/?demo" target="_blank" rel="noopener noreferrer">Demo</a> •
+<a href="https://community.tandoor.dev" target="_blank" rel="noopener noreferrer">Community</a> •
 <a href="https://discord.gg/RhzBrfWgtp" target="_blank" rel="noopener noreferrer">Discord</a>
 </p>
 
@@ -39,13 +40,13 @@
 
 - 🔍 Powerful & customizable **search** with fulltext support and [TrigramSimilarity](https://docs.djangoproject.com/en/3.0/ref/contrib/postgres/search/#trigram-similarity)
 - 🏷️ Create and search for **tags**, assign them in batch to all files matching certain filters
-- ↔️ Quickly merge and rename ingredients, tags and units 
+- ↔️ Quickly merge and rename ingredients, tags and units
 - 📥️ **Import recipes** from thousands of websites supporting [ld+json or microdata](https://schema.org/Recipe)
 - ➗ Support for **fractions** or decimals
 - 🐳 Easy setup with **Docker** and included examples for **Kubernetes**, **Unraid** and **Synology**
 - 🎨 Customize your interface with **themes**
 - 📦 **Sync** files with Dropbox and Nextcloud
-  
+
 ## All the must haves
 
 - 📱Optimized for use on **mobile** devices
@@ -54,7 +55,7 @@
 - ➕ Many more like recipe scaling, image compression, printing views and supermarkets
 
 This application is meant for people with a collection of recipes they want to share with family and friends or simply
-store them in a nicely organized way. A basic permission system exists but this application is not meant to be run as 
+store them in a nicely organized way. A basic permission system exists but this application is not meant to be run as
 a public page.
 
 ## Docs
@@ -62,16 +63,16 @@ a public page.
 Documentation can be found [here](https://docs.tandoor.dev/).
 
 ## Support our work
-Tandoor is developed by volunteers in their free time just because its fun. That said earning 
+Tandoor is developed by volunteers in their free time just because its fun. That said earning
 some money with the project allows us to spend more time on it and thus make improvements we otherwise couldn't.
 Because of that there are several ways you can support us
 
 - **GitHub Sponsors** You can sponsor contributors of this project on GitHub: [vabene1111](https://github.com/sponsors/vabene1111)
 - **Host at Hetzner** We have been very happy customers of Hetzner for multiple years for all of our projects. If you want to get into self-hosting or are tired of the expensive big providers, their cloud servers are a great place to get started. When you sign up via our [referral link](https://hetzner.cloud/?ref=ISdlrLmr9kGj) you will get 20€ worth of cloud credits and we get a small kickback too.
-- **Let us host for you** We are offering a [hosted version](https://app.tandoor.dev) where all profits support us and the development of tandoor (currently only available in germany).  
+- **Let us host for you** We are offering a [hosted version](https://app.tandoor.dev) where all profits support us and the development of tandoor (currently only available in germany).
 
 ## Contributing
-Contributions are welcome but please read [this](https://docs.tandoor.dev/contribute/#contributing-code) **BEFORE** contributing anything!
+Contributions are welcome but please read [this](https://docs.tandoor.dev/contribute/guidelines/) **BEFORE** contributing anything!
 
 ## Your Feedback
 
@@ -81,13 +82,13 @@ Share some information on how you use Tandoor to help me improve the application
 
 <table>
   <tr>
-    <td><a href="https://discord.gg/RhzBrfWgtp">Discord</a></td>
-    <td>We have a public Discord server that anyone can join. This is where all our developers and contributors hang out and where we make announcements</td>
+    <td><a href="https://community.tandoor.dev">Community</a></td>
+    <td>Get support, share best practices, discuss feature ideas, and meet other Tandoor users.</td>
   </tr>
 
   <tr>
-    <td><a href="https://twitter.com/TandoorRecipes">Twitter</a></td>
-    <td>You can follow our Twitter account to get updates on new features or releases</td>
+    <td><a href="https://discord.gg/RhzBrfWgtp">Discord</a></td>
+    <td>We have a public Discord server that anyone can join. This is where all our developers and contributors hang out and where we make announcements</td>
   </tr>
 </table>
 
@@ -100,7 +101,7 @@ Beginning with version 0.10.0 the code in this repository is licensed under the
 > I am in the process of getting some professional legal advice to sort out these issues. 
 > Please also see [Issue 238](https://github.com/vabene1111/recipes/issues/238) for some discussion and **reasoning** regarding the topic.
 
-**Reasoning**  
+**Reasoning**
 **This software and *all* its features are and will always be free for everyone to use and enjoy.**
 
 The reason for the selling exception is that a significant amount of time was spend over multiple years to develop this software.

boot.sh

@@ -1,11 +1,19 @@
 #!/bin/sh
 source venv/bin/activate
 
-TANDOOR_PORT="${TANDOOR_PORT:-8080}"
+# these are envsubst in the nginx config, make sure they default to something sensible when unset
+export TANDOOR_PORT="${TANDOOR_PORT:-8080}"
+export MEDIA_ROOT=${MEDIA_ROOT:-/opt/recipes/mediafiles};
+export STATIC_ROOT=${STATIC_ROOT:-/opt/recipes/staticfiles};
+
 GUNICORN_WORKERS="${GUNICORN_WORKERS:-3}"
 GUNICORN_THREADS="${GUNICORN_THREADS:-2}"
 GUNICORN_LOG_LEVEL="${GUNICORN_LOG_LEVEL:-'info'}"
-NGINX_CONF_FILE=/opt/recipes/nginx/conf.d/Recipes.conf
+
+if [ "${TANDOOR_PORT}" -eq 80 ]; then
+    echo "TANDOOR_PORT set to 8080 because 80 is now taken by the integrated nginx"
+    TANDOOR_PORT=8080
+fi
 
 display_warning() {
     echo "[WARNING]"
@@ -14,11 +22,6 @@ display_warning() {
 
 echo "Checking configuration..."
 
-# Nginx config file must exist if gunicorn is not active
-if [ ! -f "$NGINX_CONF_FILE" ] && [ $GUNICORN_MEDIA -eq 0 ]; then
-    display_warning "Nginx configuration file could not be found at the default location!\nPath: ${NGINX_CONF_FILE}"
-fi
-
 # SECRET_KEY (or a valid file at SECRET_KEY_FILE) must be set in .env file
 
 if [ -f "${SECRET_KEY_FILE}" ]; then
@@ -29,6 +32,21 @@ if [ -z "${SECRET_KEY}" ]; then
     display_warning "The environment variable 'SECRET_KEY' (or 'SECRET_KEY_FILE' that points to an existing file) is not set but REQUIRED for running Tandoor!"
 fi
 
+if [ -f "${AUTH_LDAP_BIND_PASSWORD_FILE}" ]; then
+    export AUTH_LDAP_BIND_PASSWORD=$(cat "$AUTH_LDAP_BIND_PASSWORD_FILE")
+fi
+
+if [ -f "${EMAIL_HOST_PASSWORD_FILE}" ]; then
+    export EMAIL_HOST_PASSWORD=$(cat "$EMAIL_HOST_PASSWORD_FILE")
+fi
+
+if [ -f "${SOCIALACCOUNT_PROVIDERS_FILE}" ]; then
+    export SOCIALACCOUNT_PROVIDERS=$(cat "$SOCIALACCOUNT_PROVIDERS_FILE")
+fi
+
+if [ -f "${S3_SECRET_ACCESS_KEY_FILE}" ]; then
+    export S3_SECRET_ACCESS_KEY=$(cat "$S3_SECRET_ACCESS_KEY_FILE")
+fi
 
 echo "Waiting for database to be ready..."
 
@@ -67,13 +85,27 @@ echo "Migrating database"
 
 python manage.py migrate
 
-echo "Generating static files"
+echo "Collecting static files, this may take a while..."
 
-python manage.py collectstatic_js_reverse
 python manage.py collectstatic --noinput
 
 echo "Done"
 
-chmod -R 755 /opt/recipes/mediafiles
+chmod -R 755 ${MEDIA_ROOT:-/opt/recipes/mediafiles}
 
-exec gunicorn -b "[::]:$TANDOOR_PORT" --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --access-logfile - --error-logfile - --log-level $GUNICORN_LOG_LEVEL recipes.wsgi
+ipv6_disable=$(cat /sys/module/ipv6/parameters/disable)
+
+# prepare nginx config
+envsubst '$MEDIA_ROOT $STATIC_ROOT $TANDOOR_PORT' < /opt/recipes/http.d/Recipes.conf.template > /opt/recipes/http.d/Recipes.conf
+
+# start nginx
+echo "Starting nginx"
+nginx
+
+echo "Starting gunicorn"
+# Check if IPv6 is enabled, only then run gunicorn with ipv6 support
+if [ "$ipv6_disable" -eq 0 ]; then
+    exec gunicorn -b "[::]:$TANDOOR_PORT" --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --access-logfile - --error-logfile - --log-level $GUNICORN_LOG_LEVEL recipes.wsgi
+else
+    exec gunicorn -b ":$TANDOOR_PORT" --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --access-logfile - --error-logfile - --log-level $GUNICORN_LOG_LEVEL recipes.wsgi
+fi

cookbook/admin.py

@@ -7,16 +7,19 @@ from django.utils import translation
 from django_scopes import scopes_disabled
 from treebeard.admin import TreeAdmin
 from treebeard.forms import movenodeform_factory
+from allauth.account.decorators import secure_admin_login
 
 from cookbook.managers import DICTIONARY
 
-from .models import (BookmarkletImport, Comment, CookLog, Food, ImportLog, Ingredient, InviteLink,
+from .models import (BookmarkletImport, Comment, CookLog, CustomFilter, Food, ImportLog, Ingredient, InviteLink,
                      Keyword, MealPlan, MealType, NutritionInformation, Property, PropertyType,
                      Recipe, RecipeBook, RecipeBookEntry, RecipeImport, SearchPreference, ShareLink,
-                     ShoppingList, ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage,
+                     ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage,
                      Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog,
                      TelegramBot, Unit, UnitConversion, UserFile, UserPreference, UserSpace,
-                     ViewLog)
+                     ViewLog, ConnectorConfig)
+
+admin.site.login = secure_admin_login(admin.site.login)
 
 
 class CustomUserAdmin(UserAdmin):
@@ -95,6 +98,21 @@ class StorageAdmin(admin.ModelAdmin):
 admin.site.register(Storage, StorageAdmin)
 
 
+class ConnectorConfigAdmin(admin.ModelAdmin):
+    list_display = ('id', 'name', 'type', 'enabled', 'url')
+    search_fields = ('name', 'url')
+
+
+admin.site.register(ConnectorConfig, ConnectorConfigAdmin)
+
+
+class CustomFilterAdmin(admin.ModelAdmin):
+    list_display = ('id', 'type', 'name')
+
+
+admin.site.register(CustomFilter, CustomFilterAdmin)
+
+
 class SyncAdmin(admin.ModelAdmin):
     list_display = ('storage', 'path', 'active', 'last_checked')
     search_fields = ('storage__name', 'path')
@@ -177,7 +195,7 @@ class StepAdmin(admin.ModelAdmin):
     @admin.display(description="Name")
     def recipe_and_name(obj):
         if not obj.recipe_set.exists():
-            return f"Orphaned Step{'':s if not obj.name else f': {obj.name}'}"
+            return "Orphaned Step" + ('' if not obj.name else f': {obj.name}')
         return f"{obj.recipe_set.first().name}: {obj.name}" if obj.name else obj.recipe_set.first().name
 
 
@@ -315,8 +333,8 @@ admin.site.register(MealPlan, MealPlanAdmin)
 
 
 class MealTypeAdmin(admin.ModelAdmin):
-    list_display = ('name', 'created_by', 'order')
-    search_fields = ('name', 'created_by__username')
+    list_display = ('name', 'space', 'created_by', 'order')
+    search_fields = ('name', 'space', 'created_by__username')
 
 
 admin.site.register(MealType, MealTypeAdmin)
@@ -361,13 +379,6 @@ class ShoppingListEntryAdmin(admin.ModelAdmin):
 admin.site.register(ShoppingListEntry, ShoppingListEntryAdmin)
 
 
-# class ShoppingListAdmin(admin.ModelAdmin):
-#     list_display = ('id', 'created_by', 'created_at')
-
-
-# admin.site.register(ShoppingList, ShoppingListAdmin)
-
-
 class ShareLinkAdmin(admin.ModelAdmin):
     list_display = ('recipe', 'created_by', 'uuid', 'created_at',)
 
@@ -375,10 +386,17 @@ class ShareLinkAdmin(admin.ModelAdmin):
 admin.site.register(ShareLink, ShareLinkAdmin)
 
 
+@admin.action(description='Delete all properties with type')
+def delete_properties_with_type(modeladmin, request, queryset):
+    for pt in queryset:
+        Property.objects.filter(property_type=pt).delete()
+
+
 class PropertyTypeAdmin(admin.ModelAdmin):
-    search_fields = ('space',)
+    search_fields = ('name',)
 
     list_display = ('id', 'space', 'name', 'fdc_id')
+    actions = [delete_properties_with_type]
 
 
 admin.site.register(PropertyType, PropertyTypeAdmin)

cookbook/apps.py

@@ -3,6 +3,7 @@ import traceback
 from django.apps import AppConfig
 from django.conf import settings
 from django.db import OperationalError, ProgrammingError
+from django.db.models.signals import post_save, post_delete
 from django_scopes import scopes_disabled
 
 from recipes.settings import DEBUG
@@ -14,6 +15,12 @@ class CookbookConfig(AppConfig):
     def ready(self):
         import cookbook.signals  # noqa
 
+        if not settings.DISABLE_EXTERNAL_CONNECTORS:
+            from cookbook.connectors.connector_manager import ConnectorManager  # Needs to be here to prevent loading race condition of oauth2 modules in models.py
+            handler = ConnectorManager()
+            post_save.connect(handler, dispatch_uid="post_save-connector_manager")
+            post_delete.connect(handler, dispatch_uid="post_delete-connector_manager")
+
         # if not settings.DISABLE_TREE_FIX_STARTUP:
         #     # when starting up run fix_tree to:
         #     #   a) make sure that nodes are sorted when switching between sort modes
@@ -34,4 +41,4 @@ class CookbookConfig(AppConfig):
         #         except Exception:
         #             if DEBUG:
         #                 traceback.print_exc()
-        #             pass  # dont break startup just because fix could not run, need to investigate cases when this happens
+        #             pass  # dont break startup just because fix could not run, need to investigate cases when this happens

cookbook/connectors/connector.py

@@ -0,0 +1,64 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Optional
+
+from cookbook.models import ShoppingListEntry, User, ConnectorConfig
+
+
+@dataclass
+class UserDTO:
+    username: str
+    first_name: Optional[str]
+
+    @staticmethod
+    def create_from_user(instance: User) -> 'UserDTO':
+        return UserDTO(
+            username=instance.username,
+            first_name=instance.first_name if instance.first_name else None
+        )
+
+
+@dataclass
+class ShoppingListEntryDTO:
+    food_name: str
+    amount: Optional[float]
+    base_unit: Optional[str]
+    unit_name: Optional[str]
+    created_by: UserDTO
+
+    @staticmethod
+    def try_create_from_entry(instance: ShoppingListEntry) -> Optional['ShoppingListEntryDTO']:
+        if instance.food is None or instance.created_by is None:
+            return None
+
+        return ShoppingListEntryDTO(
+            food_name=instance.food.name,
+            amount=instance.amount if instance.amount else None,
+            unit_name=instance.unit.name if instance.unit else None,
+            base_unit=instance.unit.base_unit if instance.unit and instance.unit.base_unit else None,
+            created_by=UserDTO.create_from_user(instance.created_by),
+        )
+
+
+# A Connector is 'destroyed' & recreated each time 'any' ConnectorConfig in a space changes.
+class Connector(ABC):
+    @abstractmethod
+    def __init__(self, config: ConnectorConfig):
+        pass
+
+    @abstractmethod
+    async def on_shopping_list_entry_created(self, instance: ShoppingListEntryDTO) -> None:
+        pass
+
+    # This method might not trigger on 'direct' entry updates: https://stackoverflow.com/a/35238823
+    @abstractmethod
+    async def on_shopping_list_entry_updated(self, instance: ShoppingListEntryDTO) -> None:
+        pass
+
+    @abstractmethod
+    async def on_shopping_list_entry_deleted(self, instance: ShoppingListEntryDTO) -> None:
+        pass
+
+    @abstractmethod
+    async def close(self) -> None:
+        pass

cookbook/connectors/connector_manager.py

@@ -0,0 +1,226 @@
+import asyncio
+import logging
+import queue
+import threading
+from asyncio import Task
+from dataclasses import dataclass
+from enum import Enum
+from logging import Logger
+from types import UnionType
+from typing import List, Any, Dict, Optional, Type
+
+from django.conf import settings
+from django_scopes import scope
+
+from cookbook.connectors.connector import Connector, ShoppingListEntryDTO
+from cookbook.connectors.homeassistant import HomeAssistant
+from cookbook.models import ShoppingListEntry, Space, ConnectorConfig
+
+REGISTERED_CLASSES: UnionType | Type = ShoppingListEntry
+
+
+class ActionType(Enum):
+    CREATED = 1
+    UPDATED = 2
+    DELETED = 3
+
+
+@dataclass
+class Work:
+    instance: REGISTERED_CLASSES | ConnectorConfig
+    actionType: ActionType
+
+
+class Singleton(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+
+# The way ConnectionManager works is as follows:
+# 1. On init, it starts a worker & creates a queue for 'Work'
+# 2. Then any time its called, it verifies the type of action (create/update/delete) and if the item is of interest, pushes the Work (non-blocking) to the queue.
+# 3. The worker consumes said work from the queue.
+# 3.1 If the work is of type ConnectorConfig, it flushes its cache of known connectors (per space.id)
+# 3.2 If work is of type REGISTERED_CLASSES, it asynchronously fires of all connectors and wait for them to finish (runtime should depend on the 'slowest' connector)
+# 4. Work is marked as consumed, and next entry of the queue is consumed.
+# Each 'Work' is processed in sequential by the worker, so the throughput is about [workers * the slowest connector]
+# The Singleton class is used for ConnectorManager to have a self-reference and so Python does not garbage collect it
+class ConnectorManager(metaclass=Singleton):
+    _logger: Logger
+    _queue: queue.Queue
+    _listening_to_classes = REGISTERED_CLASSES | ConnectorConfig
+
+    def __init__(self):
+        self._logger = logging.getLogger("recipes.connector")
+        self._logger.debug("ConnectorManager initializing")
+        self._queue = queue.Queue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE)
+        self._worker = threading.Thread(target=self.worker, args=(0, self._queue,), daemon=True)
+        self._worker.start()
+
+    # Called by post save & post delete signals
+    def __call__(self, instance: Any, **kwargs) -> None:
+        action_type: ActionType
+        if "created" in kwargs and kwargs["created"]:
+            action_type = ActionType.CREATED
+        elif "created" in kwargs and not kwargs["created"]:
+            action_type = ActionType.UPDATED
+        elif "origin" in kwargs:
+            action_type = ActionType.DELETED
+        else:
+            return
+
+        self._add_work(action_type, instance)
+
+    def _add_work(self, action_type: ActionType, *instances: REGISTERED_CLASSES):
+        for instance in instances:
+            if not isinstance(instance, self._listening_to_classes) or not hasattr(instance, "space"):
+                continue
+            try:
+                _force_load_instance(instance)
+                self._queue.put_nowait(Work(instance, action_type))
+            except queue.Full:
+                self._logger.info(f"queue was full, so skipping {action_type} of type {type(instance)}")
+
+    def stop(self):
+        self._queue.join()
+        self._worker.join()
+
+    @classmethod
+    def is_initialized(cls):
+        return cls in cls._instances
+
+    @staticmethod
+    def add_work(action_type: ActionType, *instances: REGISTERED_CLASSES):
+        """
+        Manually inject work that failed to come in through the __call__ (aka Django signal)
+        Before the work is processed, we check if the connectionManager is initialized, because if it's not, we don't want to accidentally initialize it.
+        Be careful calling it, because it might result in a instance being processed twice.
+        """
+        if not ConnectorManager.is_initialized():
+            return
+        ConnectorManager()._add_work(action_type, *instances)
+
+    @staticmethod
+    def worker(worker_id: int, worker_queue: queue.Queue):
+        logger = logging.getLogger("recipes.connector.worker")
+
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+        logger.info(f"started ConnectionManager worker {worker_id}")
+
+        # When multiple workers are used, please make sure the cache is shared across all threads, otherwise it might lead to un-expected behavior.
+        _connectors_cache: Dict[int, List[Connector]] = dict()
+
+        while True:
+            try:
+                item: Optional[Work] = worker_queue.get()
+            except KeyboardInterrupt:
+                break
+
+            if item is None:
+                break
+
+            logger.debug(f"received {item.instance=} with {item.actionType=}")
+
+            # If a Connector was changed/updated, refresh connector from the database for said space
+            refresh_connector_cache = isinstance(item.instance, ConnectorConfig)
+
+            space: Space = item.instance.space
+            connectors: Optional[List[Connector]] = _connectors_cache.get(space.id)
+
+            if connectors is None or refresh_connector_cache:
+                if connectors is not None:
+                    loop.run_until_complete(_close_connectors(connectors))
+
+                with scope(space=space):
+                    connectors: List[Connector] = list()
+                    for config in space.connectorconfig_set.all():
+                        config: ConnectorConfig = config
+                        if not config.enabled:
+                            continue
+
+                        try:
+                            connector: Optional[Connector] = ConnectorManager.get_connected_for_config(config)
+                        except BaseException:
+                            logger.exception(f"failed to initialize {config.name}")
+                            continue
+
+                        if connector is not None:
+                            connectors.append(connector)
+
+                    _connectors_cache[space.id] = connectors
+
+            if len(connectors) == 0 or refresh_connector_cache:
+                worker_queue.task_done()
+                continue
+
+            logger.debug(f"running {len(connectors)} connectors for {item.instance=} with {item.actionType=}")
+
+            loop.run_until_complete(run_connectors(connectors, item.instance, item.actionType))
+            worker_queue.task_done()
+
+        logger.info(f"terminating ConnectionManager worker {worker_id}")
+
+        asyncio.set_event_loop(None)
+        loop.close()
+
+    @staticmethod
+    def get_connected_for_config(config: ConnectorConfig) -> Optional[Connector]:
+        match config.type:
+            case ConnectorConfig.HOMEASSISTANT:
+                return HomeAssistant(config)
+            case _:
+                return None
+
+
+def _force_load_instance(instance: REGISTERED_CLASSES):
+    if isinstance(instance, ShoppingListEntry):
+        _ = instance.food  # Force load food
+        _ = instance.unit  # Force load unit
+        _ = instance.created_by  # Force load created_by
+
+
+async def _close_connectors(connectors: List[Connector]):
+    tasks: List[Task] = [asyncio.create_task(connector.close()) for connector in connectors]
+
+    if len(tasks) == 0:
+        return
+
+    try:
+        await asyncio.gather(*tasks, return_exceptions=False)
+    except BaseException:
+        logging.exception("received an exception while closing one of the connectors")
+
+
+async def run_connectors(connectors: List[Connector], instance: REGISTERED_CLASSES, action_type: ActionType):
+    tasks: List[Task] = list()
+
+    if isinstance(instance, ShoppingListEntry):
+        shopping_list_entry = ShoppingListEntryDTO.try_create_from_entry(instance)
+        if shopping_list_entry is None:
+            return
+
+        match action_type:
+            case ActionType.CREATED:
+                for connector in connectors:
+                    tasks.append(asyncio.create_task(connector.on_shopping_list_entry_created(shopping_list_entry)))
+            case ActionType.UPDATED:
+                for connector in connectors:
+                    tasks.append(asyncio.create_task(connector.on_shopping_list_entry_updated(shopping_list_entry)))
+            case ActionType.DELETED:
+                for connector in connectors:
+                    tasks.append(asyncio.create_task(connector.on_shopping_list_entry_deleted(shopping_list_entry)))
+
+    if len(tasks) == 0:
+        return
+
+    try:
+        # Wait for all async tasks to finish, if one fails, the others still continue.
+        await asyncio.gather(*tasks, return_exceptions=False)
+    except BaseException:
+        logging.exception("received an exception from one of the connectors")

cookbook/connectors/homeassistant.py

@@ -0,0 +1,101 @@
+import logging
+from logging import Logger
+from typing import Dict, Tuple
+from urllib.parse import urljoin
+
+from aiohttp import request, ClientResponseError
+
+from cookbook.connectors.connector import Connector, ShoppingListEntryDTO
+from cookbook.models import ConnectorConfig
+
+
+class HomeAssistant(Connector):
+    _config: ConnectorConfig
+    _logger: Logger
+
+    def __init__(self, config: ConnectorConfig):
+        if not config.token or not config.url or not config.todo_entity:
+            raise ValueError("config for HomeAssistantConnector in incomplete")
+
+        self._logger = logging.getLogger(f"recipes.connector.homeassistant.{config.name}")
+
+        if config.url[-1] != "/":
+            config.url += "/"
+        self._config = config
+
+    async def homeassistant_api_call(self, method: str, path: str, data: Dict) -> str:
+        headers = {
+            "Authorization": f"Bearer {self._config.token}",
+            "Content-Type": "application/json"
+        }
+        async with request(method, urljoin(self._config.url, path), headers=headers, json=data) as response:
+            response.raise_for_status()
+            return await response.json()
+
+    async def on_shopping_list_entry_created(self, shopping_list_entry: ShoppingListEntryDTO) -> None:
+        if not self._config.on_shopping_list_entry_created_enabled:
+            return
+
+        item, description = _format_shopping_list_entry(shopping_list_entry)
+
+        self._logger.debug(f"adding {item=} with {description=} to {self._config.todo_entity}")
+
+        data = {
+            "entity_id": self._config.todo_entity,
+            "item": item,
+        }
+
+        if self._config.supports_description_field:
+            data["description"] = description
+
+        try:
+            await self.homeassistant_api_call("POST", "services/todo/add_item", data)
+        except ClientResponseError as err:
+            self._logger.warning(f"received an exception from the api: {err.request_info.url=}, {err.request_info.method=}, {err.status=}, {err.message=}, {type(err)=}")
+
+    async def on_shopping_list_entry_updated(self, shopping_list_entry: ShoppingListEntryDTO) -> None:
+        if not self._config.on_shopping_list_entry_updated_enabled:
+            return
+        pass
+
+    async def on_shopping_list_entry_deleted(self, shopping_list_entry: ShoppingListEntryDTO) -> None:
+        if not self._config.on_shopping_list_entry_deleted_enabled:
+            return
+
+        item, _ = _format_shopping_list_entry(shopping_list_entry)
+
+        self._logger.debug(f"removing {item=} from {self._config.todo_entity}")
+
+        data = {
+            "entity_id": self._config.todo_entity,
+            "item": item,
+        }
+
+        try:
+            await self.homeassistant_api_call("POST", "services/todo/remove_item", data)
+        except ClientResponseError as err:
+            # This error will always trigger if the item is not present/found
+            self._logger.debug(f"received an exception from the api: {err.request_info.url=}, {err.request_info.method=}, {err.status=}, {err.message=}, {type(err)=}")
+
+    async def close(self) -> None:
+        pass
+
+
+def _format_shopping_list_entry(shopping_list_entry: ShoppingListEntryDTO) -> Tuple[str, str]:
+    item = shopping_list_entry.food_name
+    if shopping_list_entry.amount:
+        item += f" ({shopping_list_entry.amount:.2f}".rstrip('0').rstrip('.')
+        if shopping_list_entry.base_unit:
+            item += f" {shopping_list_entry.base_unit})"
+        elif shopping_list_entry.unit_name:
+            item += f" {shopping_list_entry.unit_name})"
+        else:
+            item += ")"
+
+    description = "From TandoorRecipes"
+    if shopping_list_entry.created_by.first_name:
+        description += f", by {shopping_list_entry.created_by.first_name}"
+    else:
+        description += f", by {shopping_list_entry.created_by.username}"
+
+    return item, description

cookbook/forms.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 
 from allauth.account.forms import ResetPasswordForm, SignupForm
+from allauth.socialaccount.forms import SignupForm as SocialSignupForm
 from django import forms
 from django.conf import settings
 from django.core.exceptions import ValidationError
@@ -10,11 +11,7 @@ from django_scopes import scopes_disabled
 from django_scopes.forms import SafeModelChoiceField, SafeModelMultipleChoiceField
 from hcaptcha.fields import hCaptchaField
 
-
-
-from .models import (Comment, Food, InviteLink, Keyword, Recipe, RecipeBook, RecipeBookEntry,
-                     SearchPreference, Space, Storage, Sync, User, UserPreference)
-
+from .models import Comment, InviteLink, Keyword, Recipe, SearchPreference, Space, Storage, Sync, User, UserPreference, ConnectorConfig
 
 class SelectWidget(widgets.Select):
     class Media:
@@ -24,60 +21,6 @@ class SelectWidget(widgets.Select):
 class MultiSelectWidget(widgets.SelectMultiple):
     class Media:
         js = ('custom/js/form_multiselect.js',)
-
-
-# Yes there are some stupid browsers that still dont support this but
-# I dont support people using these browsers.
-class DateWidget(forms.DateInput):
-    input_type = 'date'
-
-    def __init__(self, **kwargs):
-        kwargs["format"] = "%Y-%m-%d"
-        super().__init__(**kwargs)
-
-
-class UserNameForm(forms.ModelForm):
-    prefix = 'name'
-
-    class Meta:
-        model = User
-        fields = ('first_name', 'last_name')
-
-        help_texts = {
-            'first_name': _('Both fields are optional. If none are given the username will be displayed instead')
-        }
-
-
-class ExternalRecipeForm(forms.ModelForm):
-    file_path = forms.CharField(disabled=True, required=False)
-    file_uid = forms.CharField(disabled=True, required=False)
-
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['keywords'].queryset = Keyword.objects.filter(space=space).all()
-
-    class Meta:
-        model = Recipe
-        fields = (
-            'name', 'description', 'servings', 'working_time', 'waiting_time',
-            'file_path', 'file_uid', 'keywords'
-        )
-
-        labels = {
-            'name': _('Name'),
-            'keywords': _('Keywords'),
-            'working_time': _('Preparation time in minutes'),
-            'waiting_time': _('Waiting time (cooking/baking) in minutes'),
-            'file_path': _('Path'),
-            'file_uid': _('Storage UID'),
-        }
-        widgets = {'keywords': MultiSelectWidget}
-        field_classes = {
-            'keywords': SafeModelMultipleChoiceField,
-        }
-
-
 class ImportExportBase(forms.Form):
     DEFAULT = 'DEFAULT'
     PAPRIKA = 'PAPRIKA'
@@ -101,22 +44,20 @@ class ImportExportBase(forms.Form):
     COOKMATE = 'COOKMATE'
     REZEPTSUITEDE = 'REZEPTSUITEDE'
     PDF = 'PDF'
+    GOURMET = 'GOURMET'
 
-    type = forms.ChoiceField(choices=(
-        (DEFAULT, _('Default')), (PAPRIKA, 'Paprika'), (NEXTCLOUD, 'Nextcloud Cookbook'),
-        (MEALIE, 'Mealie'), (CHOWDOWN, 'Chowdown'), (SAFFRON, 'Saffron'), (CHEFTAP, 'ChefTap'),
-        (PEPPERPLATE, 'Pepperplate'), (RECETTETEK, 'RecetteTek'), (RECIPESAGE, 'Recipe Sage'), (DOMESTICA, 'Domestica'),
-        (MEALMASTER, 'MealMaster'), (REZKONV, 'RezKonv'), (OPENEATS, 'Openeats'), (RECIPEKEEPER, 'Recipe Keeper'),
-        (PLANTOEAT, 'Plantoeat'), (COOKBOOKAPP, 'CookBookApp'), (COPYMETHAT, 'CopyMeThat'), (PDF, 'PDF'), (MELARECIPES, 'Melarecipes'),
-        (COOKMATE, 'Cookmate'), (REZEPTSUITEDE, 'Recipesuite.de')
-    ))
-
+    type = forms.ChoiceField(choices=((DEFAULT, _('Default')), (PAPRIKA, 'Paprika'), (NEXTCLOUD, 'Nextcloud Cookbook'), (MEALIE, 'Mealie'), (CHOWDOWN, 'Chowdown'),
+                                      (SAFFRON, 'Saffron'), (CHEFTAP, 'ChefTap'), (PEPPERPLATE, 'Pepperplate'), (RECETTETEK, 'RecetteTek'), (RECIPESAGE, 'Recipe Sage'),
+                                      (DOMESTICA, 'Domestica'), (MEALMASTER, 'MealMaster'), (REZKONV, 'RezKonv'), (OPENEATS, 'Openeats'), (RECIPEKEEPER, 'Recipe Keeper'),
+                                      (PLANTOEAT, 'Plantoeat'), (COOKBOOKAPP, 'CookBookApp'), (COPYMETHAT, 'CopyMeThat'), (PDF, 'PDF'), (MELARECIPES, 'Melarecipes'),
+                                      (COOKMATE, 'Cookmate'), (REZEPTSUITEDE, 'Recipesuite.de'), (GOURMET, 'Gourmet')))
 
 class MultipleFileInput(forms.ClearableFileInput):
     allow_multiple_selected = True
 
 
 class MultipleFileField(forms.FileField):
+
     def __init__(self, *args, **kwargs):
         kwargs.setdefault("widget", MultipleFileInput())
         super().__init__(*args, **kwargs)
@@ -132,11 +73,8 @@ class MultipleFileField(forms.FileField):
 
 class ImportForm(ImportExportBase):
     files = MultipleFileField(required=True)
-    duplicates = forms.BooleanField(help_text=_(
-        'To prevent duplicates recipes with the same name as existing ones are ignored. Check this box to import everything.'),
-        required=False)
-
-
+    duplicates = forms.BooleanField(help_text=_('To prevent duplicates recipes with the same name as existing ones are ignored. Check this box to import everything.'),
+                                    required=False)
 class ExportForm(ImportExportBase):
     recipes = forms.ModelMultipleChoiceField(widget=MultiSelectWidget, queryset=Recipe.objects.none(), required=False)
     all = forms.BooleanField(required=False)
@@ -147,128 +85,11 @@ class ExportForm(ImportExportBase):
         super().__init__(*args, **kwargs)
         self.fields['recipes'].queryset = Recipe.objects.filter(space=space).all()
 
-
-class CommentForm(forms.ModelForm):
-    prefix = 'comment'
-
-    class Meta:
-        model = Comment
-        fields = ('text',)
-
-        labels = {
-            'text': _('Add your comment: '),
-        }
-        widgets = {
-            'text': forms.Textarea(attrs={'rows': 2, 'cols': 15}),
-        }
-
-
-class StorageForm(forms.ModelForm):
-    username = forms.CharField(
-        widget=forms.TextInput(attrs={'autocomplete': 'new-password'}),
-        required=False
-    )
-    password = forms.CharField(
-        widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}),
-        required=False,
-        help_text=_('Leave empty for dropbox and enter app password for nextcloud.')
-    )
-    token = forms.CharField(
-        widget=forms.TextInput(
-            attrs={'autocomplete': 'new-password', 'type': 'password'}
-        ),
-        required=False,
-        help_text=_('Leave empty for nextcloud and enter api token for dropbox.')
-    )
-
-    class Meta:
-        model = Storage
-        fields = ('name', 'method', 'username', 'password', 'token', 'url', 'path')
-
-        help_texts = {
-            'url': _(
-                'Leave empty for dropbox and enter only base url for nextcloud (<code>/remote.php/webdav/</code> is added automatically)'),
-        }
-
-
-# TODO: Deprecate
-class RecipeBookEntryForm(forms.ModelForm):
-    prefix = 'bookmark'
-
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['book'].queryset = RecipeBook.objects.filter(space=space).all()
-
-    class Meta:
-        model = RecipeBookEntry
-        fields = ('book',)
-
-        field_classes = {
-            'book': SafeModelChoiceField,
-        }
-
-
-class SyncForm(forms.ModelForm):
-
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['storage'].queryset = Storage.objects.filter(space=space).all()
-
-    class Meta:
-        model = Sync
-        fields = ('storage', 'path', 'active')
-
-        field_classes = {
-            'storage': SafeModelChoiceField,
-        }
-
-        labels = {
-            'storage': _('Storage'),
-            'path': _('Path'),
-            'active': _('Active')
-        }
-
-
-# TODO deprecate
-class BatchEditForm(forms.Form):
-    search = forms.CharField(label=_('Search String'))
-    keywords = forms.ModelMultipleChoiceField(
-        queryset=Keyword.objects.none(),
-        required=False,
-        widget=MultiSelectWidget
-    )
-
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['keywords'].queryset = Keyword.objects.filter(space=space).all().order_by('id')
-
-
-class ImportRecipeForm(forms.ModelForm):
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['keywords'].queryset = Keyword.objects.filter(space=space).all()
-
-    class Meta:
-        model = Recipe
-        fields = ('name', 'keywords', 'file_path', 'file_uid')
-
-        labels = {
-            'name': _('Name'),
-            'keywords': _('Keywords'),
-            'file_path': _('Path'),
-            'file_uid': _('File ID'),
-        }
-        widgets = {'keywords': MultiSelectWidget}
-        field_classes = {
-            'keywords': SafeModelChoiceField,
-        }
+from .models import InviteLink, SearchPreference, Space, User, UserPreference
 
 
 class InviteLinkForm(forms.ModelForm):
+
     def __init__(self, *args, **kwargs):
         user = kwargs.pop('user')
         super().__init__(*args, **kwargs)
@@ -276,8 +97,8 @@ class InviteLinkForm(forms.ModelForm):
 
     def clean(self):
         space = self.cleaned_data['space']
-        if space.max_users != 0 and (UserPreference.objects.filter(space=space).count() +
-                                     InviteLink.objects.filter(valid_until__gte=datetime.today(), used_by=None, space=space).count()) >= space.max_users:
+        if space.max_users != 0 and (UserPreference.objects.filter(space=space).count()
+                                     + InviteLink.objects.filter(valid_until__gte=datetime.today(), used_by=None, space=space).count()) >= space.max_users:
             raise ValidationError(_('Maximum number of users for this space reached.'))
 
     def clean_email(self):
@@ -291,12 +112,8 @@ class InviteLinkForm(forms.ModelForm):
     class Meta:
         model = InviteLink
         fields = ('email', 'group', 'valid_until', 'space')
-        help_texts = {
-            'email': _('An email address is not required but if present the invite link will be sent to the user.'),
-        }
-        field_classes = {
-            'space': SafeModelChoiceField,
-        }
+        help_texts = {'email': _('An email address is not required but if present the invite link will be sent to the user.'), }
+        field_classes = {'space': SafeModelChoiceField, }
 
 
 class SpaceCreateForm(forms.Form):
@@ -331,6 +148,18 @@ class AllAuthSignupForm(SignupForm):
         pass
 
 
+class AllAuthSocialSignupForm(SocialSignupForm):
+    terms = forms.BooleanField(label=_('Accept Terms and Privacy'))
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        if settings.PRIVACY_URL == '' and settings.TERMS_URL == '':
+            self.fields.pop('terms')
+
+    def signup(self, request, user):
+        pass
+
+
 class CustomPasswordResetForm(ResetPasswordForm):
     captcha = hCaptchaField()
 
@@ -342,133 +171,5 @@ class CustomPasswordResetForm(ResetPasswordForm):
 
 class UserCreateForm(forms.Form):
     name = forms.CharField(label='Username')
-    password = forms.CharField(
-        widget=forms.TextInput(
-            attrs={'autocomplete': 'new-password', 'type': 'password'}
-        )
-    )
-    password_confirm = forms.CharField(
-        widget=forms.TextInput(
-            attrs={'autocomplete': 'new-password', 'type': 'password'}
-        )
-    )
-
-
-class SearchPreferenceForm(forms.ModelForm):
-    prefix = 'search'
-    trigram_threshold = forms.DecimalField(min_value=0.01, max_value=1, decimal_places=2,
-                                           widget=NumberInput(attrs={'class': "form-control-range", 'type': 'range'}),
-                                           help_text=_(
-                                               'Determines how fuzzy a search is if it uses trigram similarity matching (e.g. low values mean more typos are ignored).'))
-    preset = forms.CharField(widget=forms.HiddenInput(), required=False)
-
-    class Meta:
-        model = SearchPreference
-        fields = (
-            'search', 'lookup', 'unaccent', 'icontains', 'istartswith', 'trigram', 'fulltext', 'trigram_threshold')
-
-        help_texts = {
-            'search': _(
-                'Select type method of search.  Click <a href="/docs/search/">here</a> for full description of choices.'),
-            'lookup': _('Use fuzzy matching on units, keywords and ingredients when editing and importing recipes.'),
-            'unaccent': _(
-                'Fields to search ignoring accents.  Selecting this option can improve or degrade search quality depending on language'),
-            'icontains': _(
-                "Fields to search for partial matches.  (e.g. searching for 'Pie' will return 'pie' and 'piece' and 'soapie')"),
-            'istartswith': _(
-                "Fields to search for beginning of word matches. (e.g. searching for 'sa' will return 'salad' and 'sandwich')"),
-            'trigram': _(
-                "Fields to 'fuzzy' search. (e.g. searching for 'recpie' will find 'recipe'.)  Note: this option will conflict with 'web' and 'raw' methods of search."),
-            'fulltext': _(
-                "Fields to full text search.  Note: 'web', 'phrase', and 'raw' search methods only function with fulltext fields."),
-        }
-
-        labels = {
-            'search': _('Search Method'),
-            'lookup': _('Fuzzy Lookups'),
-            'unaccent': _('Ignore Accent'),
-            'icontains': _("Partial Match"),
-            'istartswith': _("Starts With"),
-            'trigram': _("Fuzzy Search"),
-            'fulltext': _("Full Text")
-        }
-
-        widgets = {
-            'search': SelectWidget,
-            'unaccent': MultiSelectWidget,
-            'icontains': MultiSelectWidget,
-            'istartswith': MultiSelectWidget,
-            'trigram': MultiSelectWidget,
-            'fulltext': MultiSelectWidget,
-        }
-
-
-class ShoppingPreferenceForm(forms.ModelForm):
-    prefix = 'shopping'
-
-    class Meta:
-        model = UserPreference
-
-        fields = (
-            'shopping_share', 'shopping_auto_sync', 'mealplan_autoadd_shopping', 'mealplan_autoexclude_onhand',
-            'mealplan_autoinclude_related', 'shopping_add_onhand', 'default_delay', 'filter_to_supermarket', 'shopping_recent_days', 'csv_delim', 'csv_prefix'
-        )
-
-        help_texts = {
-            'shopping_share': _('Users will see all items you add to your shopping list.  They must add you to see items on their list.'),
-            'shopping_auto_sync': _(
-                'Setting to 0 will disable auto sync. When viewing a shopping list the list is updated every set seconds to sync changes someone else might have made. Useful when shopping with multiple people but might use a little bit '
-                'of mobile data. If lower than instance limit it is reset when saving.'
-            ),
-            'mealplan_autoadd_shopping': _('Automatically add meal plan ingredients to shopping list.'),
-            'mealplan_autoinclude_related': _('When adding a meal plan to the shopping list (manually or automatically), include all related recipes.'),
-            'mealplan_autoexclude_onhand': _('When adding a meal plan to the shopping list (manually or automatically), exclude ingredients that are on hand.'),
-            'default_delay': _('Default number of hours to delay a shopping list entry.'),
-            'filter_to_supermarket': _('Filter shopping list to only include supermarket categories.'),
-            'shopping_recent_days': _('Days of recent shopping list entries to display.'),
-            'shopping_add_onhand': _("Mark food 'On Hand' when checked off shopping list."),
-            'csv_delim': _('Delimiter to use for CSV exports.'),
-            'csv_prefix': _('Prefix to add when copying list to the clipboard.'),
-
-        }
-        labels = {
-            'shopping_share': _('Share Shopping List'),
-            'shopping_auto_sync': _('Autosync'),
-            'mealplan_autoadd_shopping': _('Auto Add Meal Plan'),
-            'mealplan_autoexclude_onhand': _('Exclude On Hand'),
-            'mealplan_autoinclude_related': _('Include Related'),
-            'default_delay': _('Default Delay Hours'),
-            'filter_to_supermarket': _('Filter to Supermarket'),
-            'shopping_recent_days': _('Recent Days'),
-            'csv_delim': _('CSV Delimiter'),
-            "csv_prefix_label": _("List Prefix"),
-            'shopping_add_onhand': _("Auto On Hand"),
-        }
-
-        widgets = {
-            'shopping_share': MultiSelectWidget
-        }
-
-
-class SpacePreferenceForm(forms.ModelForm):
-    prefix = 'space'
-    reset_food_inherit = forms.BooleanField(label=_("Reset Food Inheritance"), initial=False, required=False,
-                                            help_text=_("Reset all food to inherit the fields configured."))
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)  # populates the post
-        self.fields['food_inherit'].queryset = Food.inheritable_fields
-
-    class Meta:
-        model = Space
-
-        fields = ('food_inherit', 'reset_food_inherit',)
-
-        help_texts = {
-            'food_inherit': _('Fields on food that should be inherited by default.'),
-            'use_plural': _('Use the plural form for units and food inside this space.'),
-        }
-
-        widgets = {
-            'food_inherit': MultiSelectWidget
-        }
+    password = forms.CharField(widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}))
+    password_confirm = forms.CharField(widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}))

cookbook/helper/CustomTestRunner.py

@@ -1,8 +0,0 @@
-from django.test.runner import DiscoverRunner
-from django_scopes import scopes_disabled
-
-
-class CustomTestRunner(DiscoverRunner):
-    def run_tests(self, *args, **kwargs):
-        with scopes_disabled():
-            return super().run_tests(*args, **kwargs)

cookbook/helper/HelperFunctions.py

@@ -1,4 +1,12 @@
+import socket
+from urllib.parse import urlparse
+
+from django.core.exceptions import ValidationError
+from django.core.validators import URLValidator
 from django.db.models import Func
+from ipaddress import ip_address
+
+from recipes import settings
 
 
 class Round(Func):
@@ -11,3 +19,30 @@ def str2bool(v):
         return v
     else:
         return v.lower() in ("yes", "true", "1")
+
+
+"""
+validates an url that is supposed to be imported
+checks that the protocol used is http(s) and that no local address is accessed
+@:param url to test
+@:return true if url is valid, false otherwise
+"""
+
+
+def validate_import_url(url):
+    try:
+        validator = URLValidator(schemes=['http', 'https'])
+        validator(url)
+    except ValidationError:
+        # if schema is not http or https, consider url invalid
+        return False
+
+    # resolve IP address of url
+    try:
+        url_ip_address = ip_address(str(socket.gethostbyname(urlparse(url).hostname)))
+    except (ValueError, AttributeError, TypeError, Exception) as e:
+        # if ip cannot be parsed, consider url invalid
+        return False
+
+    # validate that IP is neither private nor any other special address
+    return not any([url_ip_address.is_private, url_ip_address.is_reserved, url_ip_address.is_loopback,  url_ip_address.is_multicast, url_ip_address.is_link_local, ])

cookbook/helper/__init__.py

@@ -1,6 +1,4 @@
-import cookbook.helper.dal
 from cookbook.helper.AllAuthCustomAdapter import AllAuthCustomAdapter
 
 __all__ = [
-    'dal',
 ]

cookbook/helper/automation_helper.py

@@ -98,7 +98,7 @@ class AutomationEngine:
             try:
                 return self.food_aliases[food.lower()]
             except KeyError:
-                return food
+                return self.apply_regex_replace_automation(food, Automation.FOOD_REPLACE)
         else:
             if automation := Automation.objects.filter(space=self.request.space, type=Automation.FOOD_ALIAS, param_1__iexact=food, disabled=False).order_by('order').first():
                 return automation.param_2
@@ -109,7 +109,7 @@ class AutomationEngine:
         Moves a string that should never be treated as a unit to next token and optionally replaced with default unit
         e.g. NEVER_UNIT: param1: egg, param2: None would modify ['1', 'egg', 'white'] to ['1', '', 'egg', 'white']
         or NEVER_UNIT: param1: egg, param2: pcs would modify ['1', 'egg', 'yolk'] to ['1', 'pcs', 'egg', 'yolk']
-        :param1 string: string that should never be considered a unit, will be moved to token[2]
+        :param1 tokens: string that should never be considered a unit, will be moved to token[2]
         :param2 (optional) unit as string: will insert unit string into token[1]
         :return: unit as string (possibly changed by automation)
         """
@@ -135,7 +135,7 @@ class AutomationEngine:
                 new_unit = self.never_unit[tokens[1].lower()]
                 never_unit = True
             except KeyError:
-                return tokens
+                return tokens, never_unit
         else:
             if a := Automation.objects.annotate(param_1_lower=Lower('param_1')).filter(space=self.request.space, type=Automation.NEVER_UNIT, param_1_lower__in=[
                     tokens[1].lower(), alt_unit.lower()], disabled=False).order_by('order').first():
@@ -144,7 +144,7 @@ class AutomationEngine:
 
         if never_unit:
             tokens.insert(1, new_unit)
-        return tokens
+        return tokens, never_unit
 
     def apply_transpose_automation(self, string):
         """

cookbook/helper/context_processors.py

@@ -11,4 +11,5 @@ def context_settings(request):
         'PRIVACY_URL': settings.PRIVACY_URL,
         'IMPRINT_URL': settings.IMPRINT_URL,
         'SHOPPING_MIN_AUTOSYNC_INTERVAL': settings.SHOPPING_MIN_AUTOSYNC_INTERVAL,
+        'DISABLE_EXTERNAL_CONNECTORS': settings.DISABLE_EXTERNAL_CONNECTORS,
     }

cookbook/helper/dal.py

@@ -1,34 +0,0 @@
-from cookbook.models import Food, Keyword, Recipe, Unit
-
-from dal import autocomplete
-
-
-class BaseAutocomplete(autocomplete.Select2QuerySetView):
-    model = None
-
-    def get_queryset(self):
-        if not self.request.user.is_authenticated:
-            return self.model.objects.none()
-
-        qs = self.model.objects.filter(space=self.request.space).all()
-
-        if self.q:
-            qs = qs.filter(name__icontains=self.q)
-
-        return qs
-
-
-class KeywordAutocomplete(BaseAutocomplete):
-    model = Keyword
-
-
-class IngredientsAutocomplete(BaseAutocomplete):
-    model = Food
-
-
-class RecipeAutocomplete(BaseAutocomplete):
-    model = Recipe
-
-
-class UnitAutocomplete(BaseAutocomplete):
-    model = Unit

cookbook/helper/drf_spectacular_hooks.py

@@ -0,0 +1,102 @@
+# custom processing for schema
+# reason: DRF writable nested needs ID's to decide if a nested object should be created or updated
+# the API schema/client make ID's read only by default and strips them entirely in request objects (with COMPONENT_SPLIT_REQUEST enabled)
+# change the schema to make IDs optional but writable so they are included in the request
+
+def custom_postprocessing_hook(result, generator, request, public):
+    for c in result['components']['schemas'].keys():
+        # handle schemas used by the client to do requests on the server
+        if 'properties' in result['components']['schemas'][c] and 'id' in result['components']['schemas'][c]['properties']:
+            # make ID field not read only so it's not stripped from the request on the client
+            result['components']['schemas'][c]['properties']['id']['readOnly'] = False
+            # make ID field not required
+            if 'required' in result['components']['schemas'][c] and 'id' in result['components']['schemas'][c]['required']:
+                result['components']['schemas'][c]['required'].remove('id')
+
+    return result
+
+
+# TODO remove below once legacy API has been fully deprecated
+from drf_spectacular.openapi import AutoSchema  # noqa: E402 isort: skip
+import functools  # noqa: E402 isort: skip
+import re  # noqa: E402 isort: skip
+
+
+class LegacySchema(AutoSchema):
+    operation_id_base = None
+
+    @functools.cached_property
+    def path(self):
+        path = re.sub(pattern=self.path_prefix, repl='', string=self.path, flags=re.IGNORECASE)
+        # remove path variables
+        return re.sub(pattern=r'\{[\w\-]+\}', repl='', string=path)
+
+    def get_operation_id(self):
+        """
+        Compute an operation ID from the view type and get_operation_id_base method.
+        """
+        method_name = getattr(self.view, 'action', self.method.lower())
+        if self._is_list_view():
+            action = 'list'
+        elif method_name not in self.method_mapping:
+            action = self._to_camel_case(method_name)
+        else:
+            action = self.method_mapping[self.method.lower()]
+
+        name = self.get_operation_id_base(action)
+
+        return action + name
+
+    def get_operation_id_base(self, action):
+        """
+        Compute the base part for operation ID from the model, serializer or view name.
+        """
+        model = getattr(getattr(self.view, 'queryset', None), 'model', None)
+
+        if self.operation_id_base is not None:
+            name = self.operation_id_base
+
+        # Try to deduce the ID from the view's model
+        elif model is not None:
+            name = model.__name__
+
+        # Try with the serializer class name
+        elif self.get_serializer() is not None:
+            name = self.get_serializer().__class__.__name__
+            if name.endswith('Serializer'):
+                name = name[:-10]
+
+        # Fallback to the view name
+        else:
+            name = self.view.__class__.__name__
+            if name.endswith('APIView'):
+                name = name[:-7]
+            elif name.endswith('View'):
+                name = name[:-4]
+
+            # Due to camel-casing of classes and `action` being lowercase, apply title in order to find if action truly
+            # comes at the end of the name
+            if name.endswith(action.title()):  # ListView, UpdateAPIView, ThingDelete ...
+                name = name[:-len(action)]
+
+        if action == 'list' and not name.endswith('s'):  # listThings instead of listThing
+            name += 's'
+
+        return name
+
+    def get_serializer(self):
+        view = self.view
+
+        if not hasattr(view, 'get_serializer'):
+            return None
+
+        try:
+            return view.get_serializer()
+        except Exception:
+            return None
+
+    def _to_camel_case(self, snake_str):
+        components = snake_str.split('_')
+        # We capitalize the first letter of each component except the first one
+        # with the 'title' method and join them together.
+        return components[0] + ''.join(x.title() for x in components[1:])

cookbook/helper/image_processing.py

@@ -35,6 +35,33 @@ def get_filetype(name):
         return '.jpeg'
 
 
+def is_file_type_allowed(filename, image_only=False):
+    is_file_allowed = False
+    allowed_file_types = ['.pdf', '.docx', '.xlsx', '.css']
+    allowed_image_types = ['.png', '.jpg', '.jpeg', '.gif', '.webp']
+    check_list = allowed_image_types
+    if not image_only:
+        check_list += allowed_file_types
+
+    for file_type in check_list:
+        if filename.lower().endswith(file_type):
+            is_file_allowed = True
+
+    return is_file_allowed
+
+
+def strip_image_meta(image_object, file_format):
+    image_object = Image.open(image_object)
+
+    data = list(image_object.getdata())
+    image_without_exif = Image.new(image_object.mode, image_object.size)
+    image_without_exif.putdata(data)
+
+    im_io = BytesIO()
+    image_without_exif.save(im_io, file_format)
+    return im_io
+
+
 # TODO this whole file needs proper documentation, refactoring, and testing
 # TODO also add env variable to define which images sizes should be compressed
 # filetype argument can not be optional, otherwise this function will treat all images as if they were a jpeg
@@ -45,9 +72,19 @@ def handle_image(request, image_object, filetype):
     except Exception:
         return None
 
+    file_format = None
+    if filetype == '.jpeg' or filetype == '.jpg':
+        file_format = 'JPEG'
+    if filetype == '.png':
+        file_format = 'PNG'
+
     if (image_object.size / 1000) > 500:  # if larger than 500 kb compress
         if filetype == '.jpeg' or filetype == '.jpg':
             return rescale_image_jpeg(image_object)
         if filetype == '.png':
             return rescale_image_png(image_object)
+    else:
+        return strip_image_meta(image_object, file_format)
+
+    # TODO webp and gifs bypass the scaling and metadata checks, fix
     return image_object

cookbook/helper/ingredient_parser.py

@@ -118,7 +118,7 @@ class IngredientParser:
         note = ''
         start = 0
         # search for first occurrence of an argument ending in a comma
-        while start < len(tokens) and not tokens[start].endswith(','):
+        while start < len(tokens) and not tokens[start].endswith((',', ';', ':')):
             start += 1
         if start == len(tokens):
             # no token ending in a comma found -> use everything as food
@@ -176,7 +176,6 @@ class IngredientParser:
         # if something like this is detected move it to the beginning so the parser can handle it
         if len(ingredient) < 1000 and re.search(r'^([^\W\d_])+(.)*[1-9](\d)*\s*([^\W\d_])+', ingredient):
             match = re.search(r'[1-9](\d)*\s*([^\W\d_])+', ingredient)
-            print(f'reording from {ingredient} to {ingredient[match.start():match.end()] + " " + ingredient.replace(ingredient[match.start():match.end()], "")}')
             ingredient = ingredient[match.start():match.end()] + ' ' + ingredient.replace(ingredient[match.start():match.end()], '')
 
         # if the string contains parenthesis early on remove it and place it at the end
@@ -211,39 +210,46 @@ class IngredientParser:
                 # three arguments if it already has a unit there can't be
                 # a fraction for the amount
                 if len(tokens) > 2:
+                    never_unit_applied = False
                     if not self.ignore_rules:
-                        tokens = self.automation.apply_never_unit_automation(tokens)
-                    try:
-                        if unit is not None:
-                            # a unit is already found, no need to try the second argument for a fraction
-                            # probably not the best method to do it, but I didn't want to make an if check and paste the exact same thing in the else as already is in the except
-                            raise ValueError
-                        # try to parse second argument as amount and add that, in case of '2 1/2' or '2 ½'
-                        amount += self.parse_fraction(tokens[1])
-                        # assume that units can't end with a comma
-                        if len(tokens) > 3 and not tokens[2].endswith(','):
-                            # try to use third argument as unit and everything else as food, use everything as food if it fails
-                            try:
-                                food, note = self.parse_food(tokens[3:])
-                                unit = tokens[2]
-                            except ValueError:
+                        tokens, never_unit_applied = self.automation.apply_never_unit_automation(tokens)
+
+                    if never_unit_applied:
+                        unit = tokens[1]
+                        food, note = self.parse_food(tokens[2:])
+                    else:
+                        try:
+                            if unit is not None:
+                                # a unit is already found, no need to try the second argument for a fraction
+                                # probably not the best method to do it, but I didn't want to make an if check and paste the exact same thing in the else as already is in the except
+                                raise ValueError
+                            # try to parse second argument as amount and add that, in case of '2 1/2' or '2 ½'
+                            if tokens[1]:
+                                amount += self.parse_fraction(tokens[1])
+                            # assume that units can't end with a comma
+                            if len(tokens) > 3 and not tokens[2].endswith(','):
+                                # try to use third argument as unit and everything else as food, use everything as food if it fails
+                                try:
+                                    food, note = self.parse_food(tokens[3:])
+                                    unit = tokens[2]
+                                except ValueError:
+                                    food, note = self.parse_food(tokens[2:])
+                            else:
                                 food, note = self.parse_food(tokens[2:])
-                        else:
-                            food, note = self.parse_food(tokens[2:])
-                    except ValueError:
-                        # assume that units can't end with a comma
-                        if not tokens[1].endswith(','):
-                            # try to use second argument as unit and everything else as food, use everything as food if it fails
-                            try:
-                                food, note = self.parse_food(tokens[2:])
-                                if unit is None:
-                                    unit = tokens[1]
-                                else:
-                                    note = tokens[1]
-                            except ValueError:
+                        except ValueError:
+                            # assume that units can't end with a comma
+                            if not tokens[1].endswith(','):
+                                # try to use second argument as unit and everything else as food, use everything as food if it fails
+                                try:
+                                    food, note = self.parse_food(tokens[2:])
+                                    if unit is None:
+                                        unit = tokens[1]
+                                    else:
+                                        note = tokens[1]
+                                except ValueError:
+                                    food, note = self.parse_food(tokens[1:])
+                            else:
                                 food, note = self.parse_food(tokens[1:])
-                        else:
-                            food, note = self.parse_food(tokens[1:])
                 else:
                     # only two arguments, first one is the amount
                     # which means this is the food
@@ -264,6 +270,7 @@ class IngredientParser:
 
         if food and not self.ignore_rules:
             food = self.automation.apply_food_automation(food)
+
         if len(food) > Food._meta.get_field('name').max_length:  # test if food name is to long
             # try splitting it at a space and taking only the first arg
             if len(food.split()) > 1 and len(food.split()[0]) < Food._meta.get_field('name').max_length:

cookbook/helper/mdx_attributes.py

@@ -7,7 +7,9 @@ class StyleTreeprocessor(Treeprocessor):
     def run_processor(self, node):
         for child in node:
             if child.tag == "table":
-                child.set("class", "table table-bordered")
+                child.set("class", "markdown-table")
+            if child.tag == "th" or child.tag == "td":
+                child.set("class", "markdown-table-cell")
             if child.tag == "img":
                 child.set("class", "img-fluid")
             self.run_processor(child)

cookbook/helper/open_data_importer.py

@@ -1,5 +1,22 @@
+import traceback
+from collections import defaultdict
+from decimal import Decimal
+
 from cookbook.models import (Food, FoodProperty, Property, PropertyType, Supermarket,
                              SupermarketCategory, SupermarketCategoryRelation, Unit, UnitConversion)
+import re
+
+from recipes.settings import DEBUG
+
+
+class OpenDataImportResponse:
+    total_created = 0
+    total_updated = 0
+    total_untouched = 0
+    total_errored = 0
+
+    def to_dict(self):
+        return {'total_created': self.total_created, 'total_updated': self.total_updated, 'total_untouched': self.total_untouched, 'total_errored': self.total_errored}
 
 
 class OpenDataImporter:
@@ -18,69 +35,269 @@ class OpenDataImporter:
     def _update_slug_cache(self, object_class, datatype):
         self.slug_id_cache[datatype] = dict(object_class.objects.filter(space=self.request.space, open_data_slug__isnull=False).values_list('open_data_slug', 'id', ))
 
-    def import_units(self):
-        datatype = 'unit'
+    @staticmethod
+    def _is_obj_identical(field_list, obj, existing_obj):
+        """
+        checks if the obj meant for import is identical to an already existing one
+        :param field_list: list of field names to check
+        :type field_list: list[str]
+        :param obj: object meant for import
+        :type obj: Object
+        :param existing_obj: object already in DB
+        :type existing_obj: Object
+        :return: if objects are identical
+        :rtype: bool
+        """
+        for field in field_list:
+            if isinstance(getattr(obj, field), float) or isinstance(getattr(obj, field), Decimal):
+                if abs(float(getattr(obj, field)) - float(existing_obj[field])) > 0.001:  # convert both to float and check if basically equal
+                    print(f'comparing FLOAT {obj} failed because field {field} is not equal ({getattr(obj, field)} != {existing_obj[field]})')
+                    return False
+            elif getattr(obj, field) != existing_obj[field]:
+                print(f'comparing {obj} failed because field {field} is not equal ({getattr(obj, field)} != {existing_obj[field]})')
+                return False
+        return True
+
+    @staticmethod
+    def _merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+        """
+        sometimes there might be two objects conflicting for open data import (one has the slug, the other the name)
+        this function checks if that is the case and merges the two objects if possible
+        :param model_type: type of model to check/merge
+        :type model_type: Model
+        :param obj: object that should be created/updated
+        :type obj: Model
+        :param existing_data_slugs: dict of open data slugs mapped to objects
+        :type existing_data_slugs: dict
+        :param existing_data_names: dict of names mapped to objects
+        :type existing_data_names: dict
+        :return: true if merge was successful or not necessary else false
+        :rtype: bool
+        """
+        if obj.open_data_slug in existing_data_slugs and obj.name in existing_data_names and existing_data_slugs[obj.open_data_slug]['pk'] != existing_data_names[obj.name]['pk']:
+            try:
+                source_obj = model_type.objects.get(pk=existing_data_slugs[obj.open_data_slug]['pk'])
+                del existing_data_slugs[obj.open_data_slug]
+                source_obj.merge_into(model_type.objects.get(pk=existing_data_names[obj.name]['pk']))
+                return True
+            except RuntimeError:
+                return False  # in the edge case (e.g. parent/child) that an object cannot be merged don't update it for now
+        else:
+            return True
+
+    @staticmethod
+    def _get_existing_obj(obj, existing_data_slugs, existing_data_names):
+        """
+        gets the existing object from slug or name cache
+        :param obj: object that should be found
+        :type obj: Model
+        :param existing_data_slugs: dict of open data slugs mapped to objects
+        :type existing_data_slugs: dict
+        :param existing_data_names: dict of names mapped to objects
+        :type existing_data_names: dict
+        :return: existing object
+        :rtype: dict
+        """
+        existing_obj = None
+        if obj.open_data_slug in existing_data_slugs:
+            existing_obj = existing_data_slugs[obj.open_data_slug]
+        elif obj.name in existing_data_names:
+            existing_obj = existing_data_names[obj.name]
+
+        return existing_obj
+
+    def import_units(self):
+        od_response = OpenDataImportResponse()
+        datatype = 'unit'
+        model_type = Unit
+        field_list = ['name', 'plural_name', 'base_unit', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
 
-        insert_list = []
         for u in list(self.data[datatype].keys()):
-            insert_list.append(Unit(
+            obj = model_type(
                 name=self.data[datatype][u]['name'],
                 plural_name=self.data[datatype][u]['plural_name'],
-                base_unit=self.data[datatype][u]['base_unit'] if self.data[datatype][u]['base_unit'] != '' else None,
+                base_unit=self.data[datatype][u]['base_unit'].lower() if self.data[datatype][u]['base_unit'] != '' else None,
                 open_data_slug=u,
                 space=self.request.space
-            ))
+            )
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
 
-        if self.update_existing:
-            return Unit.objects.bulk_create(insert_list, update_conflicts=True, update_fields=(
-                'name', 'plural_name', 'base_unit', 'open_data_slug'), unique_fields=('space', 'name',))
-        else:
-            return Unit.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        return od_response
 
     def import_category(self):
+        od_response = OpenDataImportResponse()
         datatype = 'category'
+        model_type = SupermarketCategory
+        field_list = ['name', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
 
-        insert_list = []
         for k in list(self.data[datatype].keys()):
-            insert_list.append(SupermarketCategory(
+            obj = model_type(
                 name=self.data[datatype][k]['name'],
                 open_data_slug=k,
                 space=self.request.space
-            ))
+            )
 
-        return SupermarketCategory.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        return od_response
 
     def import_property(self):
+        od_response = OpenDataImportResponse()
         datatype = 'property'
+        model_type = PropertyType
+        field_list = ['name', 'unit', 'fdc_id', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
 
-        insert_list = []
         for k in list(self.data[datatype].keys()):
-            insert_list.append(PropertyType(
+            obj = model_type(
                 name=self.data[datatype][k]['name'],
                 unit=self.data[datatype][k]['unit'],
+                fdc_id=self.data[datatype][k]['fdc_id'],
                 open_data_slug=k,
                 space=self.request.space
-            ))
+            )
 
-        return PropertyType.objects.bulk_create(insert_list, update_conflicts=True, update_fields=('open_data_slug',), unique_fields=('space', 'name',))
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        return od_response
 
     def import_supermarket(self):
+        od_response = OpenDataImportResponse()
         datatype = 'store'
+        model_type = Supermarket
+        field_list = ['name', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
 
         self._update_slug_cache(SupermarketCategory, 'category')
-        insert_list = []
         for k in list(self.data[datatype].keys()):
-            insert_list.append(Supermarket(
+            obj = model_type(
                 name=self.data[datatype][k]['name'],
                 open_data_slug=k,
                 space=self.request.space
-            ))
+            )
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
 
         # always add open data slug if matching supermarket is found, otherwise relation might fail
-        supermarkets = Supermarket.objects.bulk_create(insert_list, unique_fields=('space', 'name',), update_conflicts=True, update_fields=('open_data_slug',))
         self._update_slug_cache(Supermarket, 'store')
 
-        insert_list = []
         for k in list(self.data[datatype].keys()):
             relations = []
             order = 0
@@ -96,115 +313,202 @@ class OpenDataImporter:
 
             SupermarketCategoryRelation.objects.bulk_create(relations, ignore_conflicts=True, unique_fields=('supermarket', 'category',))
 
-        return supermarkets
+        return od_response
 
     def import_food(self):
-        identifier_list = []
+        od_response = OpenDataImportResponse()
         datatype = 'food'
-        for k in list(self.data[datatype].keys()):
-            identifier_list.append(self.data[datatype][k]['name'])
-            identifier_list.append(self.data[datatype][k]['plural_name'])
+        model_type = Food
+        field_list = ['name', 'open_data_slug']
 
-        existing_objects_flat = []
-        existing_objects = {}
-        for f in Food.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('id', 'name', 'plural_name'):
-            existing_objects_flat.append(f[1])
-            existing_objects_flat.append(f[2])
-            existing_objects[f[1]] = f
-            existing_objects[f[2]] = f
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
 
         self._update_slug_cache(Unit, 'unit')
         self._update_slug_cache(PropertyType, 'property')
+        self._update_slug_cache(SupermarketCategory, 'category')
+
+        unit_g = Unit.objects.filter(space=self.request.space, base_unit__iexact='g').first()
 
-        insert_list = []
-        insert_list_flat = []
-        update_list = []
-        update_field_list = []
         for k in list(self.data[datatype].keys()):
-            if not (self.data[datatype][k]['name'] in existing_objects_flat or self.data[datatype][k]['plural_name'] in existing_objects_flat):
-                if not (self.data[datatype][k]['name'] in insert_list_flat or self.data[datatype][k]['plural_name'] in insert_list_flat):
-                    insert_list.append({'data': {
-                        'name': self.data[datatype][k]['name'],
-                        'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
-                        'supermarket_category_id': self.slug_id_cache['category'][self.data[datatype][k]['store_category']],
-                        'fdc_id': self.data[datatype][k]['fdc_id'] if self.data[datatype][k]['fdc_id'] != '' else None,
-                        'open_data_slug': k,
-                        'space': self.request.space.id,
-                    }})
-                    # build a fake second flat array to prevent duplicate foods from being inserted.
-                    # trying to insert a duplicate would throw a db error :(
-                    insert_list_flat.append(self.data[datatype][k]['name'])
-                    insert_list_flat.append(self.data[datatype][k]['plural_name'])
+
+            obj_dict = {
+                'name': self.data[datatype][k]['name'],
+                'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
+                'supermarket_category_id': self.slug_id_cache['category'][self.data[datatype][k]['store_category']] if self.data[datatype][k]['store_category'] in self.slug_id_cache['category'] else None,
+                'fdc_id': re.sub(r'\D', '', self.data[datatype][k]['fdc_id']) if self.data[datatype][k]['fdc_id'] != '' else None,
+                'open_data_slug': k,
+                'properties_food_unit_id': None,
+                'space_id': self.request.space.id,
+            }
+
+            if unit_g:
+                obj_dict['properties_food_unit_id'] = unit_g.id
+
+            obj = model_type(**obj_dict)
+
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
             else:
-                if self.data[datatype][k]['name'] in existing_objects:
-                    existing_food_id = existing_objects[self.data[datatype][k]['name']][0]
-                else:
-                    existing_food_id = existing_objects[self.data[datatype][k]['plural_name']][0]
+                create_list.append({'data': obj_dict})
 
-                if self.update_existing:
-                    update_field_list = ['name', 'plural_name', 'preferred_unit_id', 'preferred_shopping_unit_id', 'supermarket_category_id', 'fdc_id', 'open_data_slug', ]
-                    update_list.append(Food(
-                        id=existing_food_id,
-                        name=self.data[datatype][k]['name'],
-                        plural_name=self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
-                        supermarket_category_id=self.slug_id_cache['category'][self.data[datatype][k]['store_category']],
-                        fdc_id=self.data[datatype][k]['fdc_id'] if self.data[datatype][k]['fdc_id'] != '' else None,
-                        open_data_slug=k,
-                    ))
-                else:
-                    update_field_list = ['open_data_slug', ]
-                    update_list.append(Food(id=existing_food_id, open_data_slug=k, ))
+        if self.update_existing and len(update_list) > 0:
+            try:
+                model_type.objects.bulk_update(update_list, field_list)
+                od_response.total_updated += len(update_list)
+            except Exception:
+                if DEBUG:
+                    print('========= LOAD FOOD FAILED ============')
+                    print(update_list)
+                    print(existing_data_names)
+                    print(existing_data_slugs)
+                    traceback.print_exc()
 
-        Food.load_bulk(insert_list, None)
-        if len(update_list) > 0:
-            Food.objects.bulk_update(update_list, update_field_list)
+        if len(create_list) > 0:
+            try:
+                Food.load_bulk(create_list, None)
+                od_response.total_created += len(create_list)
+            except Exception:
+                if DEBUG:
+                    print('========= LOAD FOOD FAILED ============')
+                    print(create_list)
+                    print(existing_data_names)
+                    print(existing_data_slugs)
+                    traceback.print_exc()
+
+        # --------------- PROPERTY STUFF -----------------------
+        model_type = Property
+        field_list = ['property_type_id', 'property_amount', 'open_data_food_slug']
+
+        existing_data_slugs = {}
+        existing_data_property_types = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_food_slug']] = obj
+            existing_data_property_types[obj['property_type_id']] = obj
+
+        update_list = []
+        create_list = []
 
         self._update_slug_cache(Food, 'food')
 
-        food_property_list = []
-        # alias_list = []
+        for k in list(self.data['food'].keys()):
+            for fp in self.data['food'][k]['properties']['type_values']:
+                obj = model_type(
+                    property_type_id=self.slug_id_cache['property'][fp['property_type']],
+                    property_amount=fp['property_value'],
+                    open_data_food_slug=k,
+                    space=self.request.space,
+                )
 
-        for k in list(self.data[datatype].keys()):
-            for fp in self.data[datatype][k]['properties']['type_values']:
-                # try catch here because somettimes key "k" is not set for he food cache
-                try:
-                    food_property_list.append(Property(
-                        property_type_id=self.slug_id_cache['property'][fp['property_type']],
-                        property_amount=fp['property_value'],
-                        import_food_id=self.slug_id_cache['food'][k],
-                        space=self.request.space,
-                    ))
-                except KeyError:
-                    print(str(k) + ' is not in self.slug_id_cache["food"]')
+                if obj.open_data_food_slug in existing_data_slugs and obj.property_type_id in existing_data_property_types and existing_data_slugs[obj.open_data_food_slug] == existing_data_property_types[obj.property_type_id]:
+                    existing_obj = existing_data_slugs[obj.open_data_food_slug]
 
-        Property.objects.bulk_create(food_property_list, ignore_conflicts=True, unique_fields=('space', 'import_food_id', 'property_type',))
+                    if not self._is_obj_identical(field_list, obj, existing_obj):
+                        obj.pk = existing_obj['pk']
+                        update_list.append(obj)
+                else:
+                    create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, ignore_conflicts=True, unique_fields=('space', 'open_data_food_slug', 'property_type',))
+
+        linked_properties = list(FoodProperty.objects.filter(food__space=self.request.space).values_list('property_id', flat=True).all())
 
         property_food_relation_list = []
-        for p in Property.objects.filter(space=self.request.space, import_food_id__isnull=False).values_list('import_food_id', 'id', ):
-            property_food_relation_list.append(Food.properties.through(food_id=p[0], property_id=p[1]))
+        for p in model_type.objects.filter(space=self.request.space, open_data_food_slug__isnull=False).values_list('open_data_food_slug', 'id', ):
+            if p[1] == 147:
+                pass
+            # slug_id_cache should always exist, don't create relations for already linked properties (ignore_conflicts would do that as well but this is more performant)
+            if p[0] in self.slug_id_cache['food'] and p[1] not in linked_properties:
+                property_food_relation_list.append(Food.properties.through(food_id=self.slug_id_cache['food'][p[0]], property_id=p[1]))
 
-        FoodProperty.objects.bulk_create(property_food_relation_list, ignore_conflicts=True, unique_fields=('food_id', 'property_id',))
+        FoodProperty.objects.bulk_create(property_food_relation_list, unique_fields=('food_id', 'property_id',))
 
-        return insert_list + update_list
+        return od_response
 
     def import_conversion(self):
+        od_response = OpenDataImportResponse()
         datatype = 'conversion'
+        model_type = UnitConversion
+        field_list = ['base_amount', 'base_unit_id', 'converted_amount', 'converted_unit_id', 'food_id', 'open_data_slug']
+
+        self._update_slug_cache(Food, 'food')
+        self._update_slug_cache(Unit, 'unit')
+
+        existing_data_slugs = {}
+        existing_data_foods = defaultdict(list)
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_foods[obj['food_id']].append(obj)
+
+        update_list = []
+        create_list = []
 
-        insert_list = []
         for k in list(self.data[datatype].keys()):
-            # try catch here because sometimes key "k" is not set for he food cache
+            # try catch here because sometimes key "k" is not set for the food cache
             try:
-                insert_list.append(UnitConversion(
-                    base_amount=self.data[datatype][k]['base_amount'],
+                obj = model_type(
+                    base_amount=Decimal(self.data[datatype][k]['base_amount']),
                     base_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['base_unit']],
-                    converted_amount=self.data[datatype][k]['converted_amount'],
+                    converted_amount=Decimal(self.data[datatype][k]['converted_amount']),
                     converted_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['converted_unit']],
                     food_id=self.slug_id_cache['food'][self.data[datatype][k]['food']],
                     open_data_slug=k,
                     space=self.request.space,
-                    created_by=self.request.user,
-                ))
-            except KeyError:
-                print(str(k) + ' is not in self.slug_id_cache["food"]')
+                    created_by_id=self.request.user.id,
+                )
 
-        return UnitConversion.objects.bulk_create(insert_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug'))
+                if obj.open_data_slug in existing_data_slugs:
+                    existing_obj = existing_data_slugs[obj.open_data_slug]
+                    if not self._is_obj_identical(field_list, obj, existing_obj):
+                        obj.pk = existing_obj['pk']
+                        update_list.append(obj)
+                    else:
+                        od_response.total_untouched += 1
+                else:
+                    matching_existing_found = False
+                    if obj.food_id in existing_data_foods:
+                        for edf in existing_data_foods[obj.food_id]:
+                            if obj.base_unit_id == edf['base_unit_id'] and obj.converted_unit_id == edf['converted_unit_id']:
+                                matching_existing_found = True
+                                if not self._is_obj_identical(field_list, obj, edf):
+                                    obj.pk = edf['pk']
+                                    update_list.append(obj)
+                                else:
+                                    od_response.total_untouched += 1
+                    if not matching_existing_found:
+                        create_list.append(obj)
+            except KeyError as e:
+                traceback.print_exc()
+                od_response.total_errored += 1
+                print(self.data[datatype][k]['food'] + ' is not in self.slug_id_cache["food"]')
+
+        if self.update_existing and len(update_list) > 0:
+            od_response.total_updated = model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_errored += len(update_list) - od_response.total_updated
+
+        if len(create_list) > 0:
+            objs_created = model_type.objects.bulk_create(create_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug'))
+            od_response.total_created = len(objs_created)
+            od_response.total_errored += len(create_list) - od_response.total_created
+
+        return od_response

cookbook/helper/permission_helper.py

@@ -75,7 +75,7 @@ def is_object_owner(user, obj):
     if not user.is_authenticated:
         return False
     try:
-        return obj.get_owner() == user
+        return obj.get_owner() == 'orphan' or obj.get_owner() == user
     except Exception:
         return False
 
@@ -160,18 +160,15 @@ class GroupRequiredMixin(object):
     def dispatch(self, request, *args, **kwargs):
         if not has_group_permission(request.user, self.groups_required):
             if not request.user.is_authenticated:
-                messages.add_message(request, messages.ERROR,
-                                     _('You are not logged in and therefore cannot view this page!'))
+                messages.add_message(request, messages.ERROR, _('You are not logged in and therefore cannot view this page!'))
                 return HttpResponseRedirect(reverse_lazy('account_login') + '?next=' + request.path)
             else:
-                messages.add_message(request, messages.ERROR,
-                                     _('You do not have the required permissions to view this page!'))
+                messages.add_message(request, messages.ERROR, _('You do not have the required permissions to view this page!'))
                 return HttpResponseRedirect(reverse_lazy('index'))
         try:
             obj = self.get_object()
             if obj.get_space() != request.space:
-                messages.add_message(request, messages.ERROR,
-                                     _('You do not have the required permissions to view this page!'))
+                messages.add_message(request, messages.ERROR, _('You do not have the required permissions to view this page!'))
                 return HttpResponseRedirect(reverse_lazy('index'))
         except AttributeError:
             pass

cookbook/helper/property_helper.py

@@ -44,13 +44,17 @@ class FoodPropertyHelper:
             if i.food is not None:
                 conversions = uch.get_conversions(i)
                 for pt in property_types:
+                    # if a property could be calculated with an actual value
                     found_property = False
-                    if i.food.properties_food_amount == 0 or i.food.properties_food_unit is None:
-                        computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': 0}
-                        computed_properties[pt.id]['missing_value'] = i.food.properties_food_unit is None
+                    # if food has a value for the given property type (no matter if conversion is possible)
+                    has_property_value = False
+                    if i.food.properties_food_amount == 0 or i.food.properties_food_unit is None and not (i.amount == 0 or i.no_amount):  # if food is configured incorrectly
+                        computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': {'id': i.food.id, 'name': i.food.name}, 'value': None}
+                        computed_properties[pt.id]['missing_value'] = True
                     else:
                         for p in i.food.properties.all():
-                            if p.property_type == pt:
+                            if p.property_type == pt and p.property_amount is not None:
+                                has_property_value = True
                                 for c in conversions:
                                     if c.unit == i.food.properties_food_unit:
                                         found_property = True
@@ -58,17 +62,28 @@ class FoodPropertyHelper:
                                         computed_properties[pt.id]['food_values'] = self.add_or_create(
                                             computed_properties[p.property_type.id]['food_values'], c.food.id, (c.amount / i.food.properties_food_amount) * p.property_amount, c.food)
                     if not found_property:
-                        computed_properties[pt.id]['missing_value'] = True
-                        computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': 0}
+                        # if no amount and food does not exist yet add it but don't count as missing
+                        if i.amount == 0 or i.no_amount and i.food.id not in computed_properties[pt.id]['food_values']:
+                            computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': {'id': i.food.id, 'name': i.food.name}, 'value': 0}
+                        # if amount is present but unit is missing indicate it in the result
+                        elif i.unit is None:
+                            if i.food.id not in computed_properties[pt.id]['food_values']:
+                                computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': {'id': i.food.id, 'name': i.food.name}, 'value': 0}
+                            computed_properties[pt.id]['food_values'][i.food.id]['missing_unit'] = True
+                        else:
+                            computed_properties[pt.id]['missing_value'] = True
+                            computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': {'id': i.food.id, 'name': i.food.name}, 'value': None}
+                            if has_property_value and i.unit is not None:
+                                computed_properties[pt.id]['food_values'][i.food.id]['missing_conversion'] = {'base_unit': {'id': i.unit.id, 'name': i.unit.name}, 'converted_unit': {'id': i.food.properties_food_unit.id, 'name': i.food.properties_food_unit.name}}
 
         return computed_properties
 
     # small dict helper to add to existing key or create new, probably a better way of doing this
-    # TODO move to central helper ?
+    # TODO move to central helper ? --> use defaultdict
     @staticmethod
     def add_or_create(d, key, value, food):
-        if key in d:
+        if key in d and d[key]['value']:
             d[key]['value'] += value
         else:
-            d[key] = {'id': food.id, 'food': food.name, 'value': value}
+            d[key] = {'id': food.id, 'food': {'id': food.id, 'name': food.name}, 'value': value}
         return d

cookbook/helper/recipe_search.py

@@ -49,7 +49,11 @@ class RecipeSearch():
             self._search_prefs = SearchPreference()
         self._string = self._params.get('query').strip(
         ) if self._params.get('query', None) else None
+
         self._rating = self._params.get('rating', None)
+        self._rating_gte = self._params.get('rating_gte', None)
+        self._rating_lte = self._params.get('rating_lte', None)
+
         self._keywords = {
             'or': self._params.get('keywords_or', None) or self._params.get('keywords', None),
             'and': self._params.get('keywords_and', None),
@@ -70,20 +74,36 @@ class RecipeSearch():
         }
         self._steps = self._params.get('steps', None)
         self._units = self._params.get('units', None)
-        # TODO add created by
-        # TODO image exists
-        self._sort_order = self._params.get('sort_order', None)
         self._internal = str2bool(self._params.get('internal', None))
-        self._random = str2bool(self._params.get('random', False))
+        self._sort_order = self._params.get('sort_order', None)
+        if self._sort_order == 'random':
+            self._random = True
+            self.sort_order = None
+        else:
+            self._random = str2bool(self._params.get('random', False))
         self._new = str2bool(self._params.get('new', False))
         self._num_recent = int(self._params.get('num_recent', 0))
         self._include_children = str2bool(
             self._params.get('include_children', None))
         self._timescooked = self._params.get('timescooked', None)
-        self._cookedon = self._params.get('cookedon', None)
+        self._timescooked_gte = self._params.get('timescooked_gte', None)
+        self._timescooked_lte = self._params.get('timescooked_lte', None)
+
         self._createdon = self._params.get('createdon', None)
+        self._createdon_gte = self._params.get('createdon_gte', None)
+        self._createdon_lte = self._params.get('createdon_lte', None)
+
         self._updatedon = self._params.get('updatedon', None)
-        self._viewedon = self._params.get('viewedon', None)
+        self._updatedon_gte = self._params.get('updatedon_gte', None)
+        self._updatedon_lte = self._params.get('updatedon_lte', None)
+
+        self._viewedon_gte = self._params.get('viewedon_gte', None)
+        self._viewedon_lte = self._params.get('viewedon_lte', None)
+
+        self._cookedon_gte = self._params.get('cookedon_gte', None)
+        self._cookedon_lte = self._params.get('cookedon_lte', None)
+
+        self._createdby = self._params.get('createdby', None)
         self._makenow = self._params.get('makenow', None)
         # this supports hidden feature to find recipes missing X ingredients
         if isinstance(self._makenow, bool) and self._makenow == True:
@@ -130,16 +150,19 @@ class RecipeSearch():
 
         self._build_sort_order()
         self._recently_viewed(num_recent=self._num_recent)
-        self._cooked_on_filter(cooked_date=self._cookedon)
-        self._created_on_filter(created_date=self._createdon)
-        self._updated_on_filter(updated_date=self._updatedon)
-        self._viewed_on_filter(viewed_date=self._viewedon)
-        self._favorite_recipes(times_cooked=self._timescooked)
+
+        self._cooked_on_filter()
+        self._created_on_filter()
+        self._updated_on_filter()
+        self._viewed_on_filter()
+
+        self._created_by_filter(created_by_user_id=self._createdby)
+        self._favorite_recipes()
         self._new_recipes()
         self.keyword_filters(**self._keywords)
         self.food_filters(**self._foods)
         self.book_filters(**self._books)
-        self.rating_filter(rating=self._rating)
+        self.rating_filter()
         self.internal_filter(internal=self._internal)
         self.step_filters(steps=self._steps)
         self.unit_filters(units=self._units)
@@ -186,9 +209,9 @@ class RecipeSearch():
             else:
                 order += default_order
             order[:] = [Lower('name').asc() if x ==
-                        'name' else x for x in order]
+                                               'name' else x for x in order]
             order[:] = [Lower('name').desc() if x ==
-                        '-name' else x for x in order]
+                                                '-name' else x for x in order]
             self.orderby = order
 
     def string_filters(self, string=None):
@@ -227,9 +250,9 @@ class RecipeSearch():
                 query_filter |= Q(**{"%s" % f: self._string})
             self._queryset = self._queryset.filter(query_filter).distinct()
 
-    def _cooked_on_filter(self, cooked_date=None):
-        if self._sort_includes('lastcooked') or cooked_date:
-            lessthan = self._sort_includes('-lastcooked') or '-' in (cooked_date or [])[:1]
+    def _cooked_on_filter(self):
+        if self._sort_includes('lastcooked') or self._cookedon_gte or self._cookedon_lte:
+            lessthan = self._sort_includes('-lastcooked') or self._cookedon_lte
             if lessthan:
                 default = timezone.now() - timedelta(days=100000)
             else:
@@ -237,51 +260,44 @@ class RecipeSearch():
             self._queryset = self._queryset.annotate(
                 lastcooked=Coalesce(Max(Case(When(cooklog__created_by=self._request.user, cooklog__space=self._request.space, then='cooklog__created_at'))), Value(default))
             )
-        if cooked_date is None:
-            return
 
-        cooked_date = date(*[int(x)for x in cooked_date.split('-') if x != ''])
-
-        if lessthan:
-            self._queryset = self._queryset.filter(lastcooked__date__lte=cooked_date).exclude(lastcooked=default)
-        else:
-            self._queryset = self._queryset.filter(lastcooked__date__gte=cooked_date).exclude(lastcooked=default)
-
-    def _created_on_filter(self, created_date=None):
-        if created_date is None:
-            return
-        lessthan = '-' in created_date[:1]
-        created_date = date(*[int(x) for x in created_date.split('-') if x != ''])
-        if lessthan:
-            self._queryset = self._queryset.filter(created_at__date__lte=created_date)
-        else:
-            self._queryset = self._queryset.filter(created_at__date__gte=created_date)
-
-    def _updated_on_filter(self, updated_date=None):
-        if updated_date is None:
-            return
-        lessthan = '-' in updated_date[:1]
-        updated_date = date(*[int(x)for x in updated_date.split('-') if x != ''])
-        if lessthan:
-            self._queryset = self._queryset.filter(updated_at__date__lte=updated_date)
-        else:
-            self._queryset = self._queryset.filter(updated_at__date__gte=updated_date)
+        if self._cookedon_lte:
+            self._queryset = self._queryset.filter(lastcooked__date__lte=self._cookedon_lte).exclude(lastcooked=default)
+        elif self._cookedon_gte:
+            self._queryset = self._queryset.filter(lastcooked__date__gte=self._cookedon_gte).exclude(lastcooked=default)
 
     def _viewed_on_filter(self, viewed_date=None):
-        if self._sort_includes('lastviewed') or viewed_date:
+        if self._sort_includes('lastviewed') or self._viewedon_gte or self._viewedon_lte:
             longTimeAgo = timezone.now() - timedelta(days=100000)
             self._queryset = self._queryset.annotate(
                 lastviewed=Coalesce(Max(Case(When(viewlog__created_by=self._request.user, viewlog__space=self._request.space, then='viewlog__created_at'))), Value(longTimeAgo))
             )
-        if viewed_date is None:
-            return
-        lessthan = '-' in viewed_date[:1]
-        viewed_date = date(*[int(x)for x in viewed_date.split('-') if x != ''])
 
-        if lessthan:
-            self._queryset = self._queryset.filter(lastviewed__date__lte=viewed_date).exclude(lastviewed=longTimeAgo)
-        else:
-            self._queryset = self._queryset.filter(lastviewed__date__gte=viewed_date).exclude(lastviewed=longTimeAgo)
+        if self._viewedon_lte:
+            self._queryset = self._queryset.filter(lastviewed__date__lte=self._viewedon_lte).exclude(lastviewed=longTimeAgo)
+        elif self._viewedon_gte:
+            self._queryset = self._queryset.filter(lastviewed__date__gte=self._viewedon_gte).exclude(lastviewed=longTimeAgo)
+
+    def _created_on_filter(self):
+        if self._createdon:
+            self._queryset = self._queryset.filter(created_at__date=self._createdon)
+        elif self._createdon_lte:
+            self._queryset = self._queryset.filter(created_at__date__lte=self._createdon_lte)
+        elif self._createdon_gte:
+            self._queryset = self._queryset.filter(created_at__date__gte=self._createdon_gte)
+
+    def _updated_on_filter(self):
+        if self._updatedon:
+            self._queryset = self._queryset.filter(updated_at__date__date=self._updatedon)
+        elif self._updatedon_lte:
+            self._queryset = self._queryset.filter(updated_at__date__lte=self._updatedon_lte)
+        elif self._updatedon_gte:
+            self._queryset = self._queryset.filter(updated_at__date__gte=self._updatedon_gte)
+
+    def _created_by_filter(self, created_by_user_id=None):
+        if created_by_user_id is None:
+            return
+        self._queryset = self._queryset.filter(created_by__id=created_by_user_id)
 
     def _new_recipes(self, new_days=7):
         # TODO make new days a user-setting
@@ -307,9 +323,9 @@ class RecipeSearch():
         )
         self._queryset = self._queryset.annotate(recent=Coalesce(Max(Case(When(pk__in=num_recent_recipes.values('recipe'), then='viewlog__pk'))), Value(0)))
 
-    def _favorite_recipes(self, times_cooked=None):
-        if self._sort_includes('favorite') or times_cooked:
-            less_than = '-' in (str(times_cooked) or []) and not self._sort_includes('-favorite')
+    def _favorite_recipes(self):
+        if self._sort_includes('favorite') or self._timescooked or self._timescooked_gte or self._timescooked_lte:
+            less_than = self._timescooked_lte and not self._sort_includes('-favorite')
             if less_than:
                 default = 1000
             else:
@@ -321,15 +337,13 @@ class RecipeSearch():
                 .values('count')
             )
             self._queryset = self._queryset.annotate(favorite=Coalesce(Subquery(favorite_recipes), default))
-        if times_cooked is None:
-            return
 
-        if times_cooked == '0':
+        if self._timescooked:
             self._queryset = self._queryset.filter(favorite=0)
-        elif less_than:
-            self._queryset = self._queryset.filter(favorite__lte=int(times_cooked.replace('-', ''))).exclude(favorite=0)
-        else:
-            self._queryset = self._queryset.filter(favorite__gte=int(times_cooked))
+        elif self._timescooked_lte:
+            self._queryset = self._queryset.filter(favorite__lte=int(self._timescooked_lte)).exclude(favorite=0)
+        elif self._timescooked_gte:
+            self._queryset = self._queryset.filter(favorite__gte=int(self._timescooked_gte))
 
     def keyword_filters(self, **kwargs):
         if all([kwargs[x] is None for x in kwargs]):
@@ -407,25 +421,16 @@ class RecipeSearch():
             units = [units]
         self._queryset = self._queryset.filter(steps__ingredients__unit__in=units)
 
-    def rating_filter(self, rating=None):
-        if rating or self._sort_includes('rating'):
-            lessthan = '-' in (rating or [])
-            reverse = 'rating' in (self._sort_order or []) and '-rating' not in (self._sort_order or [])
-            if lessthan or reverse:
-                default = 100
-            else:
-                default = 0
-            # TODO make ratings a settings user-only vs all-users
-            self._queryset = self._queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=self._request.user, then='cooklog__rating'), default=default))))
-        if rating is None:
-            return
+    def rating_filter(self):
+        if self._rating or self._rating_lte or self._rating_gte or self._sort_includes('rating'):
+            self._queryset = self._queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=self._request.user, then='cooklog__rating'), default=0))))
 
-        if rating == '0':
-            self._queryset = self._queryset.filter(rating=0)
-        elif lessthan:
-            self._queryset = self._queryset.filter(rating__lte=int(rating[1:])).exclude(rating=0)
-        else:
-            self._queryset = self._queryset.filter(rating__gte=int(rating))
+        if self._rating:
+            self._queryset = self._queryset.filter(rating=round(int(self._rating)))
+        elif self._rating_gte:
+            self._queryset = self._queryset.filter(rating__gte=int(self._rating_gte))
+        elif self._rating_lte:
+            self._queryset = self._queryset.filter(rating__gte=int(self._rating_lte)).exclude(rating=0)
 
     def internal_filter(self, internal=None):
         if not internal:
@@ -535,11 +540,11 @@ class RecipeSearch():
         shopping_users = [*self._request.user.get_shopping_share(), self._request.user]
 
         onhand_filter = (
-            Q(steps__ingredients__food__onhand_users__in=shopping_users)  # food onhand
-            # or substitute food onhand
-            | Q(steps__ingredients__food__substitute__onhand_users__in=shopping_users)
-            | Q(steps__ingredients__food__in=self.__children_substitute_filter(shopping_users))
-            | Q(steps__ingredients__food__in=self.__sibling_substitute_filter(shopping_users))
+                Q(steps__ingredients__food__onhand_users__in=shopping_users)  # food onhand
+                # or substitute food onhand
+                | Q(steps__ingredients__food__substitute__onhand_users__in=shopping_users)
+                | Q(steps__ingredients__food__in=self.__children_substitute_filter(shopping_users))
+                | Q(steps__ingredients__food__in=self.__sibling_substitute_filter(shopping_users))
         )
         makenow_recipes = Recipe.objects.annotate(
             count_food=Count('steps__ingredients__food__pk', filter=Q(steps__ingredients__food__isnull=False), distinct=True),

cookbook/helper/recipe_url_import.py

@@ -6,7 +6,7 @@ from django.utils.dateparse import parse_duration
 from django.utils.translation import gettext as _
 from isodate import parse_duration as iso_parse_duration
 from isodate.isoerror import ISO8601Error
-from pytube import YouTube
+from pytubefix import YouTube
 from recipe_scrapers._utils import get_host_name, get_minutes
 
 from cookbook.helper.automation_helper import AutomationEngine
@@ -15,12 +15,9 @@ from cookbook.models import Automation, Keyword, PropertyType
 
 
 def get_from_scraper(scrape, request):
-    # converting the scrape_me object to the existing json format based on ld+json
+    # converting the scrape_html object to the existing json format based on ld+json
 
-    recipe_json = {
-        'steps': [],
-        'internal': True
-    }
+    recipe_json = {'steps': [], 'internal': True}
     keywords = []
 
     # assign source URL
@@ -31,7 +28,9 @@ def get_from_scraper(scrape, request):
             source_url = scrape.url
         except Exception:
             pass
-    if source_url:
+    if source_url == "https://urlnotfound.none" or not source_url:
+        recipe_json['source_url'] = ''
+    else:
         recipe_json['source_url'] = source_url
         try:
             keywords.append(source_url.replace('http://', '').replace('https://', '').split('/')[0])
@@ -107,14 +106,14 @@ def get_from_scraper(scrape, request):
 
     # assign image
     try:
-        recipe_json['image'] = parse_image(scrape.image()) or None
+        recipe_json['image_url'] = parse_image(scrape.image()) or None
     except Exception:
-        recipe_json['image'] = None
-    if not recipe_json['image']:
+        recipe_json['image_url'] = None
+    if not recipe_json['image_url']:
         try:
-            recipe_json['image'] = parse_image(scrape.schema.data.get('image')) or ''
+            recipe_json['image_url'] = parse_image(scrape.schema.data.get('image')) or ''
         except Exception:
-            recipe_json['image'] = ''
+            recipe_json['image_url'] = ''
 
     # assign keywords
     try:
@@ -157,11 +156,18 @@ def get_from_scraper(scrape, request):
     # assign steps
     try:
         for i in parse_instructions(scrape.instructions()):
-            recipe_json['steps'].append({'instruction': i, 'ingredients': [], 'show_ingredients_table': request.user.userpreference.show_step_ingredients, })
+            recipe_json['steps'].append({
+                'instruction': i,
+                'ingredients': [],
+                'show_ingredients_table': request.user.userpreference.show_step_ingredients,
+            })
     except Exception:
         pass
     if len(recipe_json['steps']) == 0:
-        recipe_json['steps'].append({'instruction': '', 'ingredients': [], })
+        recipe_json['steps'].append({
+            'instruction': '',
+            'ingredients': [],
+        })
 
     recipe_json['description'] = recipe_json['description'][:512]
     if len(recipe_json['description']) > 256:  # split at 256 as long descriptions don't look good on recipe cards
@@ -182,23 +188,24 @@ def get_from_scraper(scrape, request):
                         'original_text': x
                     }
                     if unit:
-                        ingredient['unit'] = {'name': unit, }
+                        ingredient['unit'] = {
+                            'name': unit,
+                        }
                     recipe_json['steps'][0]['ingredients'].append(ingredient)
                 except Exception:
-                    recipe_json['steps'][0]['ingredients'].append(
-                        {
-                            'amount': 0,
-                            'unit': None,
-                            'food': {
-                                'name': x,
-                            },
-                            'note': '',
-                            'original_text': x
-                        }
-                    )
+                    recipe_json['steps'][0]['ingredients'].append({
+                        'amount': 0,
+                        'unit': None,
+                        'food': {
+                            'name': x,
+                        },
+                        'note': '',
+                        'original_text': x
+                    })
     except Exception:
         pass
 
+    recipe_json['properties'] = []
     try:
         recipe_json['properties'] = get_recipe_properties(request.space, scrape.schema.nutrients())
         print(recipe_json['properties'])
@@ -221,6 +228,13 @@ def get_recipe_properties(space, property_data):
         "property-proteins": "proteinContent",
         "property-fats": "fatContent",
     }
+
+    serving_size = 1
+    try:
+        serving_size = parse_servings(property_data['servingSize'])
+    except KeyError:
+        pass
+
     recipe_properties = []
     for pt in PropertyType.objects.filter(space=space, open_data_slug__in=list(properties.keys())).all():
         for p in list(properties.keys()):
@@ -231,7 +245,7 @@ def get_recipe_properties(space, property_data):
                             'id': pt.id,
                             'name': pt.name,
                         },
-                        'property_amount': parse_servings(property_data[properties[p]]) / float(property_data['servingSize']),
+                        'property_amount': parse_servings(property_data[properties[p]]) / serving_size,
                     })
 
     return recipe_properties
@@ -248,14 +262,16 @@ def get_from_youtube_scraper(url, request):
         'working_time': 0,
         'waiting_time': 0,
         'image': "",
-        'keywords': [{'name': kw.name, 'label': kw.name, 'id': kw.pk}],
+        'keywords': [{
+            'name': kw.name,
+            'label': kw.name,
+            'id': kw.pk
+        }],
         'source_url': url,
-        'steps': [
-            {
-                'ingredients': [],
-                'instruction': ''
-            }
-        ]
+        'steps': [{
+            'ingredients': [],
+            'instruction': ''
+        }]
     }
 
     try:
@@ -266,9 +282,8 @@ def get_from_youtube_scraper(url, request):
         default_recipe_json['image'] = video.thumbnail_url
         if video.description:
             default_recipe_json['steps'][0]['instruction'] = automation_engine.apply_regex_replace_automation(video.description, Automation.INSTRUCTION_REPLACE)
-
     except Exception:
-        pass
+        traceback.print_exc()
 
     return default_recipe_json
 
@@ -367,8 +382,8 @@ def parse_servings(servings):
             servings = 1
     elif isinstance(servings, list):
         try:
-            servings = int(re.findall(r'\b\d+\b', servings[0])[0])
-        except KeyError:
+            servings = int(re.findall(r'\b\d+\b', str(servings[0]))[0])
+        except (KeyError, IndexError):
             servings = 1
     return servings
 
@@ -417,9 +432,9 @@ def parse_keywords(keyword_json, request):
         if len(kw) != 0:
             kw = automation_engine.apply_keyword_automation(kw)
             if k := Keyword.objects.filter(name__iexact=kw, space=request.space).first():
-                keywords.append({'label': str(k), 'name': k.name, 'id': k.id})
+                keywords.append({'label': str(k), 'name': k.name, 'id': k.id, 'import_keyword': True})
             else:
-                keywords.append({'label': kw, 'name': kw})
+                keywords.append({'label': kw, 'name': kw, 'import_keyword': False})
 
     return keywords
 
@@ -452,10 +467,7 @@ def normalize_string(string):
 
 
 def iso_duration_to_minutes(string):
-    match = re.match(
-        r'P((?P<years>\d+)Y)?((?P<months>\d+)M)?((?P<weeks>\d+)W)?((?P<days>\d+)D)?T((?P<hours>\d+)H)?((?P<minutes>\d+)M)?((?P<seconds>\d+)S)?',
-        string
-    ).groupdict()
+    match = re.match(r'P((?P<years>\d+)Y)?((?P<months>\d+)M)?((?P<weeks>\d+)W)?((?P<days>\d+)D)?T((?P<hours>\d+)H)?((?P<minutes>\d+)M)?((?P<seconds>\d+)S)?', string).groupdict()
     return int(match['days'] or 0) * 24 * 60 + int(match['hours'] or 0) * 60 + int(match['minutes'] or 0)
 
 
@@ -479,7 +491,11 @@ def get_images_from_soup(soup, url):
         u = u.split('?')[0]
         filename = re.search(r'/([\w_-]+[.](jpg|jpeg|gif|png))$', u)
         if filename:
-            if (('http' not in u) and (url)):
+            if u.startswith('//'):
+                # urls from e.g. ottolenghi.co.uk start with //
+                u = 'https:' + u
+            if ('http' not in u) and url:
+                print(f'rewriting URL {u}')
                 # sometimes an image source can be relative
                 # if it is provide the base url
                 u = '{}://{}{}'.format(prot, site, u)

cookbook/helper/scope_middleware.py

@@ -12,7 +12,7 @@ class ScopeMiddleware:
         self.get_response = get_response
 
     def __call__(self, request):
-        prefix = settings.JS_REVERSE_SCRIPT_PREFIX or ''
+        prefix = settings.SCRIPT_NAME or ''
 
         # need to disable scopes for writing requests into userpref and enable for loading ?
         if request.path.startswith(prefix + '/api/user-preference/'):

cookbook/helper/scrapers/cooksillustrated.py

@@ -1,68 +0,0 @@
-import json
-from recipe_scrapers._abstract import AbstractScraper
-
-
-class CooksIllustrated(AbstractScraper):
-    @classmethod
-    def host(cls, site='cooksillustrated'):
-        return {
-            'cooksillustrated': f"{site}.com",
-            'americastestkitchen': f"{site}.com",
-            'cookscountry': f"{site}.com",
-        }.get(site)
-
-    def title(self):
-        return self.schema.title()
-
-    def image(self):
-        return self.schema.image()
-
-    def total_time(self):
-        if not self.recipe:
-            self.get_recipe()
-        return self.recipe['recipeTimeNote']
-
-    def yields(self):
-        if not self.recipe:
-            self.get_recipe()
-        return self.recipe['yields']
-
-    def ingredients(self):
-        if not self.recipe:
-            self.get_recipe()
-        ingredients = []
-        for group in self.recipe['ingredientGroups']:
-            ingredients += group['fields']['recipeIngredientItems']
-        return [
-            "{} {} {}{}".format(
-                i['fields']['qty'] or '',
-                i['fields']['measurement'] or '',
-                i['fields']['ingredient']['fields']['title'] or '',
-                i['fields']['postText'] or ''
-            )
-            for i in ingredients
-        ]
-
-    def instructions(self):
-        if not self.recipe:
-            self.get_recipe()
-        if self.recipe.get('headnote', False):
-            i = ['Note: ' + self.recipe.get('headnote', '')]
-        else:
-            i = []
-        return "\n".join(
-            i
-            + [self.recipe.get('whyThisWorks', '')]
-            + [
-                instruction['fields']['content']
-                for instruction in self.recipe['instructions']
-            ]
-        )
-
-    def nutrients(self):
-        raise NotImplementedError("This should be implemented.")
-
-    def get_recipe(self):
-        j = json.loads(self.soup.find(type='application/json').string)
-        name = list(j['props']['initialState']['content']['documents'])[0]
-        self.recipe = j['props']['initialState']['content']['documents'][name]

cookbook/helper/scrapers/scrapers.py

@@ -1,43 +0,0 @@
-from json import JSONDecodeError
-
-from bs4 import BeautifulSoup
-from recipe_scrapers import SCRAPERS, get_host_name
-from recipe_scrapers._factory import SchemaScraperFactory
-from recipe_scrapers._schemaorg import SchemaOrg
-
-from .cooksillustrated import CooksIllustrated
-
-CUSTOM_SCRAPERS = {
-    CooksIllustrated.host(site="cooksillustrated"): CooksIllustrated,
-    CooksIllustrated.host(site="americastestkitchen"): CooksIllustrated,
-    CooksIllustrated.host(site="cookscountry"): CooksIllustrated,
-}
-SCRAPERS.update(CUSTOM_SCRAPERS)
-
-
-def text_scraper(text, url=None):
-    domain = None
-    if url:
-        domain = get_host_name(url)
-    if domain in SCRAPERS:
-        scraper_class = SCRAPERS[domain]
-    else:
-        scraper_class = SchemaScraperFactory.SchemaScraper
-
-    class TextScraper(scraper_class):
-        def __init__(
-            self,
-            html=None,
-            url=None,
-        ):
-            self.wild_mode = False
-            self.meta_http_equiv = False
-            self.soup = BeautifulSoup(html, "html.parser")
-            self.url = url
-            self.recipe = None
-            try:
-                self.schema = SchemaOrg(html)
-            except (JSONDecodeError, AttributeError):
-                pass
-
-    return TextScraper(url=url, html=text)

cookbook/helper/shopping_helper.py

@@ -1,9 +1,8 @@
-from datetime import timedelta
+
 from decimal import Decimal
 
 from django.db.models import F, OuterRef, Q, Subquery, Value
 from django.db.models.functions import Coalesce
-from django.utils import timezone
 from django.utils.translation import gettext as _
 
 from cookbook.models import (Ingredient, MealPlan, Recipe, ShoppingListEntry, ShoppingListRecipe,
@@ -27,9 +26,6 @@ def shopping_helper(qs, request):
     elif checked in ['true', 1, '1']:
         qs = qs.filter(checked=True)
     elif checked in ['recent']:
-        today_start = timezone.now().replace(hour=0, minute=0, second=0)
-        week_ago = today_start - timedelta(days=user.userpreference.shopping_recent_days)
-        qs = qs.filter(Q(checked=False) | Q(completed_at__gte=week_ago))
         supermarket_order = ['checked'] + supermarket_order
 
     return qs.distinct().order_by(*supermarket_order).select_related('unit', 'food', 'ingredient', 'created_by', 'list_recipe', 'list_recipe__mealplan', 'list_recipe__recipe')
@@ -79,10 +75,8 @@ class RecipeShoppingEditor():
 
     @staticmethod
     def get_shopping_list_recipe(id, user, space):
-        return ShoppingListRecipe.objects.filter(id=id).filter(Q(shoppinglist__space=space) | Q(entries__space=space)).filter(
-            Q(shoppinglist__created_by=user)
-            | Q(shoppinglist__shared=user)
-            | Q(entries__created_by=user)
+        return ShoppingListRecipe.objects.filter(id=id).filter(entries__space=space).filter(
+            Q(entries__created_by=user)
             | Q(entries__created_by__in=list(user.get_shopping_share()))
         ).prefetch_related('entries').first()
 
@@ -119,7 +113,7 @@ class RecipeShoppingEditor():
         if not self.servings:
             self.servings = getattr(self.mealplan, 'servings', None) or getattr(self.recipe, 'servings', 1.0)
 
-        self._shopping_list_recipe = ShoppingListRecipe.objects.create(recipe=self.recipe, mealplan=self.mealplan, servings=self.servings)
+        self._shopping_list_recipe = ShoppingListRecipe.objects.create(recipe=self.recipe, mealplan=self.mealplan, servings=self.servings, space=self.space, created_by=self.created_by)
 
         if ingredients:
             self._add_ingredients(ingredients=ingredients)
@@ -159,8 +153,9 @@ class RecipeShoppingEditor():
             return True
 
         for sle in ShoppingListEntry.objects.filter(list_recipe=self._shopping_list_recipe):
-            sle.amount = sle.ingredient.amount * Decimal(self._servings_factor)
-            sle.save()
+            if sle.ingredient: # TODO temporarily dont scale manual entries until ingredient_amount or some other base amount has been migrated to SLE
+                sle.amount = sle.ingredient.amount * Decimal(self._servings_factor)
+                sle.save()
         self._shopping_list_recipe.servings = self.servings
         self._shopping_list_recipe.save()
         return True

cookbook/helper/template_helper.py

@@ -3,6 +3,8 @@ from gettext import gettext as _
 import bleach
 import markdown as md
 from jinja2 import Template, TemplateSyntaxError, UndefinedError
+from jinja2.exceptions import SecurityError
+from jinja2.sandbox import SandboxedEnvironment
 from markdown.extensions.tables import TableExtension
 
 from cookbook.helper.mdx_attributes import MarkdownFormatExtension
@@ -68,7 +70,7 @@ def render_instructions(step):  # TODO deduplicate markdown cleanup code
     parsed_md = md.markdown(
         instructions,
         extensions=[
-            'markdown.extensions.fenced_code', TableExtension(),
+            'markdown.extensions.fenced_code', 'markdown.extensions.sane_lists', 'markdown.extensions.nl2br', TableExtension(),
             UrlizeExtension(), MarkdownFormatExtension()
         ]
     )
@@ -89,11 +91,13 @@ def render_instructions(step):  # TODO deduplicate markdown cleanup code
         return f"<scalable-number v-bind:number='{bleach.clean(str(number))}' v-bind:factor='ingredient_factor'></scalable-number>"
 
     try:
-        template = Template(instructions)
-        instructions = template.render(ingredients=ingredients, scale=scale)
+        env = SandboxedEnvironment()
+        instructions = env.from_string(instructions).render(ingredients=ingredients, scale=scale)
     except TemplateSyntaxError:
         return _('Could not parse template code.') + ' Error: Template Syntax broken'
     except UndefinedError:
         return _('Could not parse template code.') + ' Error: Undefined Error'
+    except SecurityError:
+        return _('Could not parse template code.') + ' Error: Security Error'
 
     return instructions

cookbook/helper/unit_conversion_helper.py

@@ -20,6 +20,7 @@ CONVERSION_TABLE = {
         'gallon': 0.264172,
         'tbsp': 67.628,
         'tsp': 202.884,
+        'us_cup': 4.22675,
         'imperial_fluid_ounce': 35.1951,
         'imperial_pint': 1.75975,
         'imperial_quart': 0.879877,

cookbook/integration/cookbookapp.py

@@ -2,12 +2,12 @@ import re
 from io import BytesIO
 
 import requests
-import validators
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import (get_from_scraper, get_images_from_soup,
                                                iso_duration_to_minutes)
-from cookbook.helper.scrapers.scrapers import text_scraper
+from recipe_scrapers import scrape_html
 from cookbook.integration.integration import Integration
 from cookbook.models import Ingredient, Recipe, Step
 
@@ -20,7 +20,7 @@ class CookBookApp(Integration):
     def get_recipe_from_file(self, file):
         recipe_html = file.getvalue().decode("utf-8")
 
-        scrape = text_scraper(text=recipe_html)
+        scrape = scrape_html(html=recipe_html, org_url="https://cookbookapp.import", supported_only=False)
         recipe_json = get_from_scraper(scrape, self.request)
         images = list(dict.fromkeys(get_images_from_soup(scrape.soup, None)))
 
@@ -60,14 +60,15 @@ class CookBookApp(Integration):
                 food=f, unit=u, amount=ingredient.get('amount', None), note=ingredient.get('note', None), original_text=ingredient.get('original_text', None), space=self.request.space,
             ))
 
-        if len(images) > 0:
-            try:
-                url = images[0]
-                if validators.url(url, public=True):
+        try:
+            for url in images:
+                # import the first valid image which is not cookbookapp branding
+                if validate_import_url(url) and not url.startswith("https://media.cookbookmanager.com/brand/"):
                     response = requests.get(url)
                     self.import_recipe_image(recipe, BytesIO(response.content))
-            except Exception as e:
-                print('failed to import image ', str(e))
+                    break
+        except Exception as e:
+            print('failed to import image ', str(e))
 
         recipe.save()
         return recipe

cookbook/integration/cookmate.py

@@ -1,8 +1,8 @@
 from io import BytesIO
 
 import requests
-import validators
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
@@ -69,7 +69,7 @@ class Cookmate(Integration):
         if recipe_xml.find('imageurl') is not None:
             try:
                 url = recipe_xml.find('imageurl').text.strip()
-                if validators.url(url, public=True):
+                if validate_import_url(url):
                     response = requests.get(url)
                 self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e:

cookbook/integration/gourmet.py

@@ -0,0 +1,211 @@
+import base64
+from io import BytesIO
+from lxml import etree
+import requests
+from pathlib import Path
+
+from bs4 import BeautifulSoup, Tag
+
+from cookbook.helper.HelperFunctions import validate_import_url
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time, iso_duration_to_minutes
+from cookbook.integration.integration import Integration
+from cookbook.models import Ingredient, Recipe, Step, Keyword
+from recipe_scrapers import scrape_html
+
+
+class Gourmet(Integration):
+
+    def split_recipe_file(self, file):
+        encoding = 'utf-8'
+        byte_string = file.read()
+        text_obj = byte_string.decode(encoding, errors="ignore")
+        soup = BeautifulSoup(text_obj, "html.parser")
+        return soup.find_all("div", {"class": "recipe"})
+
+    def get_ingredients_recursive(self, step, ingredients, ingredient_parser):
+        if isinstance(ingredients, Tag):
+            for ingredient in ingredients.children:
+                if not isinstance(ingredient, Tag):
+                    continue
+
+                if ingredient.name in ["li"]:
+                    step_name = "".join(ingredient.findAll(text=True, recursive=False)).strip().rstrip(":")
+
+                    step.ingredients.add(Ingredient.objects.create(
+                        is_header=True,
+                        note=step_name[:256],
+                        original_text=step_name,
+                        space=self.request.space,
+                    ))
+                    next_ingrediets = ingredient.find("ul", {"class": "ing"})
+                    self.get_ingredients_recursive(step, next_ingrediets, ingredient_parser)
+
+                else:
+                    try:
+                        amount, unit, food, note = ingredient_parser.parse(ingredient.text.strip())
+                        f = ingredient_parser.get_food(food)
+                        u = ingredient_parser.get_unit(unit)
+                        step.ingredients.add(
+                            Ingredient.objects.create(
+                                food=f,
+                                unit=u,
+                                amount=amount,
+                                note=note,
+                                original_text=ingredient.text.strip(),
+                                space=self.request.space,
+                            )
+                        )
+                    except ValueError:
+                        pass
+
+    def get_recipe_from_file(self, file):
+        # 'file' comes is as a beautifulsoup object
+
+        source_url = None
+        for item in file.find_all('a'):
+            if item.has_attr('href'):
+                source_url = item.get("href")
+                break
+
+        name = file.find("p", {"class": "title"}).find("span", {"itemprop": "name"}).text.strip()
+
+        recipe = Recipe.objects.create(
+            name=name[:128],
+            source_url=source_url,
+            created_by=self.request.user,
+            internal=True,
+            space=self.request.space,
+        )
+
+        for category in file.find_all("span", {"itemprop": "recipeCategory"}):
+            keyword, created = Keyword.objects.get_or_create(name=category.text, space=self.request.space)
+            recipe.keywords.add(keyword)
+
+        try:
+            recipe.servings = parse_servings(file.find("span", {"itemprop": "recipeYield"}).text.strip())
+        except AttributeError:
+            pass
+
+        try:
+            prep_time = file.find("span", {"itemprop": "prepTime"}).text.strip().split()
+            prep_time[0] = prep_time[0].replace(',', '.')
+            if prep_time[1].lower() in ['stunde', 'stunden', 'hour', 'hours']:
+                prep_time_min = int(float(prep_time[0]) * 60)
+            elif prep_time[1].lower() in ['tag', 'tage', 'day', 'days']:
+                prep_time_min = int(float(prep_time[0]) * 60 * 24)
+            else:
+                prep_time_min = int(prep_time[0])
+            recipe.waiting_time = prep_time_min
+        except AttributeError:
+            pass
+
+        try:
+            cook_time = file.find("span", {"itemprop": "cookTime"}).text.strip().split()
+            cook_time[0] = cook_time[0].replace(',', '.')
+            if cook_time[1].lower() in ['stunde', 'stunden', 'hour', 'hours']:
+                cook_time_min = int(float(cook_time[0]) * 60)
+            elif cook_time[1].lower() in ['tag', 'tage', 'day', 'days']:
+                cook_time_min = int(float(cook_time[0]) * 60 * 24)
+            else:
+                cook_time_min = int(cook_time[0])
+
+            recipe.working_time = cook_time_min
+        except AttributeError:
+            pass
+
+        for cuisine in file.find_all('span', {'itemprop': 'recipeCuisine'}):
+            cuisine_name = cuisine.text
+            keyword = Keyword.objects.get_or_create(space=self.request.space, name=cuisine_name)
+            if len(keyword):
+                recipe.keywords.add(keyword[0])
+
+        for category in file.find_all('span', {'itemprop': 'recipeCategory'}):
+            category_name = category.text
+            keyword = Keyword.objects.get_or_create(space=self.request.space, name=category_name)
+            if len(keyword):
+                recipe.keywords.add(keyword[0])
+
+        step = Step.objects.create(
+            instruction='',
+            space=self.request.space,
+            show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+        )
+
+        ingredient_parser = IngredientParser(self.request, True)
+
+        ingredients = file.find("ul", {"class": "ing"})
+        self.get_ingredients_recursive(step, ingredients, ingredient_parser)
+
+        instructions = file.find("div", {"class": "instructions"})
+        if isinstance(instructions, Tag):
+            for instruction in instructions.children:
+                if not isinstance(instruction, Tag) or instruction.text == "":
+                    continue
+                if instruction.name == "h3":
+                    if step.instruction:
+                        step.save()
+                        recipe.steps.add(step)
+                        step = Step.objects.create(
+                            instruction='',
+                            space=self.request.space,
+                        )
+
+                    step.name = instruction.text.strip()[:128]
+                else:
+                    if instruction.name == "div":
+                        for instruction_step in instruction.children:
+                            for br in instruction_step.find_all("br"):
+                                br.replace_with("\n")
+                            step.instruction += instruction_step.text.strip() + ' \n\n'
+
+        notes = file.find("div", {"class": "modifications"})
+        if notes:
+            for n in notes.children:
+                if n.text == "":
+                    continue
+                if n.name == "h3":
+                    step.instruction += f'*{n.text.strip()}:* \n\n'
+                else:
+                    for br in n.find_all("br"):
+                        br.replace_with("\n")
+
+                    step.instruction += '*' + n.text.strip() + '* \n\n'
+
+        description = ''
+        try:
+            description = file.find("div", {"id": "description"}).text.strip()
+        except AttributeError:
+            pass
+        if len(description) <= 512:
+            recipe.description = description
+        else:
+            recipe.description = description[:480] + ' ... (full description below)'
+            step.instruction += '*Description:* \n\n*' + description + '* \n\n'
+
+        step.save()
+        recipe.steps.add(step)
+
+        # import the Primary recipe image that is stored in the Zip
+        try:
+            image_path = file.find("img").get("src")
+            image_filename = image_path.split("\\")[1]
+
+            for f in self.import_zip.filelist:
+                zip_file_name = Path(f.filename).name
+                if image_filename == zip_file_name:
+                    image_file = self.import_zip.read(f)
+                    image_bytes = BytesIO(image_file)
+                    self.import_recipe_image(recipe, image_bytes, filetype='.jpeg')
+                    break
+        except Exception as e:
+            print(recipe.name, ': failed to import image ', str(e))
+
+        recipe.save()
+        return recipe
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        raise NotImplementedError('Method not implemented in storage integration')
+
+    def get_file_from_recipe(self, recipe):
+        raise NotImplementedError('Method not implemented in storage integration')

cookbook/integration/integration.py

@@ -153,6 +153,19 @@ class Integration:
                             il.total_recipes = len(new_file_list)
                             file_list = new_file_list
 
+                        if isinstance(self, cookbook.integration.gourmet.Gourmet):
+                            self.import_zip = import_zip
+                            new_file_list = []
+                            for file in file_list:
+                                if file.file_size == 0:
+                                    next
+                                if file.filename.startswith("index.htm"):
+                                    next
+                                if file.filename.endswith(".htm"):
+                                    new_file_list += self.split_recipe_file(BytesIO(import_zip.read(file.filename)))
+                            il.total_recipes = len(new_file_list)
+                            file_list = new_file_list
+
                         for z in file_list:
                             try:
                                 if not hasattr(z, 'filename') or isinstance(z, Tag):

cookbook/integration/mealie.py

@@ -72,14 +72,14 @@ class Mealie(Integration):
             )
             recipe.steps.add(step)
 
-        if 'recipe_yield' in recipe_json:
+        if 'recipe_yield' in recipe_json and recipe_json['recipe_yield'] is not None:
             recipe.servings = parse_servings(recipe_json['recipe_yield'])
             recipe.servings_text = parse_servings_text(recipe_json['recipe_yield'])
 
         if 'total_time' in recipe_json and recipe_json['total_time'] is not None:
             recipe.working_time = parse_time(recipe_json['total_time'])
 
-        if 'org_url' in recipe_json:
+        if 'org_url' in recipe_json and recipe_json['org_url'] is not None:
             recipe.source_url = recipe_json['org_url']
 
         recipe.save()

cookbook/integration/mealmaster.py

@@ -22,7 +22,7 @@ class MealMaster(Integration):
                         if 'Yield:' in line:
                             servings_text = line.replace('Yield:', '').strip()
                         else:
-                            if re.match('\s{2,}([0-9])+', line):
+                            if re.match(r'\s{2,}([0-9])+', line):
                                 ingredients.append(line.strip())
                             else:
                                 directions.append(line.strip())

cookbook/integration/nextcloud_cookbook.py

@@ -60,20 +60,20 @@ class NextcloudCookbook(Integration):
                 step = Step.objects.create(
                     instruction=s, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
                 )
-            if not ingredients_added:
-                if len(recipe_json['description'].strip()) > 500:
-                    step.instruction = recipe_json['description'].strip() + '\n\n' + step.instruction
 
-                ingredients_added = True
-
-                ingredient_parser = IngredientParser(self.request, True)
+            ingredient_parser = IngredientParser(self.request, True)
+            if ingredients_added == False:
                 for ingredient in recipe_json['recipeIngredient']:
-                    amount, unit, food, note = ingredient_parser.parse(ingredient)
-                    f = ingredient_parser.get_food(food)
-                    u = ingredient_parser.get_unit(unit)
-                    step.ingredients.add(Ingredient.objects.create(
-                        food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
-                    ))
+                    ingredients_added = True
+                    if ingredient.startswith('##'):
+                        subheader = ingredient.replace('##', '', 1)
+                        step.ingredients.add(Ingredient.objects.create(note=subheader, is_header=True, no_amount=True, space=self.request.space))
+                    else:
+                        amount, unit, food, note = ingredient_parser.parse(ingredient)
+                        f = ingredient_parser.get_food(food)
+                        u = ingredient_parser.get_unit(unit)
+                        step.ingredients.add(Ingredient.objects.create(
+                            food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,))
             recipe.steps.add(step)
 
         if 'nutrition' in recipe_json:

cookbook/integration/paprika.py

@@ -6,8 +6,8 @@ from gettext import gettext as _
 from io import BytesIO
 
 import requests
-import validators
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text
 from cookbook.integration.integration import Integration
@@ -84,13 +84,23 @@ class Paprika(Integration):
 
             recipe.steps.add(step)
 
+            # Paprika exports can have images in either of image_url, or photo_data.
+            # If a user takes an image himself, only photo_data will be set.
+            # If a user imports an image, both will be set. But the photo_data will be a center-cropped square resized version, so the image_url is preferred.
+            
+            # Try to download image if possible
             try:
                 if recipe_json.get("image_url", None):
                     url = recipe_json.get("image_url", None)
-                    if validators.url(url, public=True):
+                    if validate_import_url(url):
                         response = requests.get(url)
-                        self.import_recipe_image(recipe, BytesIO(response.content))
+                        if response.status_code == 200 and len(response.content) > 0:
+                            self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception:
+                pass
+
+            # If no image downloaded, try to extract from photo_data
+            if not recipe.image:
                 if recipe_json.get("photo_data", None):
                     self.import_recipe_image(recipe, BytesIO(base64.b64decode(recipe_json['photo_data'])), filetype='.jpeg')
 

cookbook/integration/plantoeat.py

@@ -1,9 +1,10 @@
 from io import BytesIO
 
 import requests
-import validators
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
 from cookbook.models import Ingredient, Keyword, Recipe, Step
 
@@ -18,32 +19,38 @@ class Plantoeat(Integration):
         tags = None
         ingredients = []
         directions = []
-        description = ''
+        fields = {}
         for line in file.replace('\r', '').split('\n'):
             if line.strip() != '':
                 if 'Title:' in line:
-                    title = line.replace('Title:', '').replace('"', '').strip()
+                    fields['name'] = line.replace('Title:', '').replace('"', '').strip()
                 if 'Description:' in line:
-                    description = line.replace('Description:', '').strip()
-                if 'Source:' in line or 'Serves:' in line or 'Prep Time:' in line or 'Cook Time:' in line:
-                    directions.append(line.strip() + '\n')
+                    fields['description'] = line.replace('Description:', '').strip()
+                if 'Serves:' in line:
+                    fields['servings'] = parse_servings(line.replace('Serves:', '').strip())
+                if 'Source:' in line:
+                    fields['source_url'] = line.replace('Source:', '').strip()
                 if 'Photo Url:' in line:
                     image_url = line.replace('Photo Url:', '').strip()
+                if 'Prep Time:' in line:
+                    fields['working_time'] = parse_time(line.replace('Prep Time:', '').strip())
+                if 'Cook Time:' in line:
+                    fields['waiting_time'] = parse_time(line.replace('Cook Time:', '').strip())
                 if 'Tags:' in line:
                     tags = line.replace('Tags:', '').strip()
-                if ingredient_mode:
-                    if len(line) > 2 and 'Instructions:' not in line:
-                        ingredients.append(line.strip())
-                if direction_mode:
-                    if len(line) > 2:
-                        directions.append(line.strip() + '\n')
                 if 'Ingredients:' in line:
                     ingredient_mode = True
                 if 'Directions:' in line:
                     ingredient_mode = False
                     direction_mode = True
+                if ingredient_mode:
+                    if len(line) > 2 and 'Ingredients:' not in line:
+                        ingredients.append(line.strip())
+                if direction_mode:
+                    if len(line) > 2:
+                        directions.append(line.strip() + '\n')                
 
-        recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space)
+        recipe = Recipe.objects.create(**fields, created_by=self.request.user, internal=True, space=self.request.space)
 
         step = Step.objects.create(
             instruction='\n'.join(directions) + '\n\n', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
@@ -68,7 +75,7 @@ class Plantoeat(Integration):
 
         if image_url:
             try:
-                if validators.url(image_url, public=True):
+                if validate_import_url(image_url):
                     response = requests.get(image_url)
                     self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e:

cookbook/integration/recettetek.py

@@ -1,13 +1,14 @@
-import imghdr
 import json
 import re
 from io import BytesIO
 from zipfile import ZipFile
 
 import requests
-import validators
+from PIL import Image
 
 from django.utils.translation import gettext as _
+
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.image_processing import get_filetype
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
@@ -125,9 +126,9 @@ class RecetteTek(Integration):
             else:
                 if file['originalPicture'] != '':
                     url = file['originalPicture']
-                    if validators.url(url, public=True):
+                    if validate_import_url(url):
                         response = requests.get(url)
-                        if imghdr.what(BytesIO(response.content)) is not None:
+                        if Image.open(BytesIO(response.content)).verify():
                             self.import_recipe_image(recipe, BytesIO(response.content), filetype=get_filetype(file['originalPicture']))
                         else:
                             raise Exception("Original image failed to download.")

cookbook/integration/recipesage.py

@@ -2,8 +2,8 @@ import json
 from io import BytesIO
 
 import requests
-import validators
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
@@ -56,7 +56,7 @@ class RecipeSage(Integration):
         if len(file['image']) > 0:
             try:
                 url = file['image'][0]
-                if validators.url(url, public=True):
+                if validate_import_url(url):
                     response = requests.get(url)
                     self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e:

cookbook/integration/rezeptsuitede.py

@@ -1,6 +1,6 @@
 import base64
 from io import BytesIO
-from xml import etree
+from lxml import etree
 
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text
@@ -53,7 +53,10 @@ class Rezeptsuitede(Integration):
                 u = ingredient_parser.get_unit(ingredient.attrib['unit'])
                 amount = 0
                 if ingredient.attrib['qty'].strip() != '':
-                    amount, unit, note = ingredient_parser.parse_amount(ingredient.attrib['qty'])
+                    try:
+                        amount, unit, note = ingredient_parser.parse_amount(ingredient.attrib['qty'])
+                    except ValueError:  # sometimes quantities contain words which cant be parsed
+                        pass
                 ingredient_step.ingredients.add(Ingredient.objects.create(food=f, unit=u, amount=amount, space=self.request.space, ))
 
         try: