Merge branch 'develop'

Update unit_conversion_helper.py

.coveragerc

@@ -0,0 +1,11 @@
+[run]
+omit =
+  */apps.py,
+  */migrations/*,
+  */settings*,
+  */test*,
+  */tests/*,
+  *urls.py,
+  */wsgi*,
+  manage.py,
+  *__init__*

.devcontainer/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.10-alpine3.18
+
+#Install all dependencies.
+RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg libwebp libxml2-dev libxslt-dev openldap git yarn
+
+#Print all logs without buffering it.
+ENV PYTHONUNBUFFERED 1
+
+#This port will be used by gunicorn.
+EXPOSE 8000
+
+#This port will be used by vue
+EXPOSE 8080
+
+#Install all python dependencies to the image
+COPY requirements.txt /tmp/pip-tmp/
+
+RUN \
+    if [ `apk --print-arch` = "armv7" ]; then \
+    printf "[global]\nextra-index-url=https://www.piwheels.org/simple\n" > /etc/pip.conf ; \
+    fi
+RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-dev jpeg-dev libwebp-dev openssl-dev libffi-dev cargo openldap-dev python3-dev && \
+    echo -n "INPUT ( libldap.so )" > /usr/lib/libldap_r.so && \
+    pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt && \
+    rm -rf /tmp/pip-tmp && \
+    apk --purge del .build-deps

.devcontainer/devcontainer.json

@@ -0,0 +1,27 @@
+// For format details, see https://aka.ms/devcontainer.json.
+{
+	"name": "Tandoor Dev Container",
+	"build": { "context": "..", "dockerfile": "Dockerfile" },
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [8000, 8080],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "pip3 install --user -r requirements.txt"
+
+	// Configure tool-specific properties.
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.debugpy",
+				"ms-python.python"
+			]
+		}
+	}
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.dockerignore

@@ -1,9 +1,8 @@
-node_modules
+**/node_modules
 npm-debug.log
 Dockerfile*
 docker-compose*
 .dockerignore
-.git
 .gitignore
 README.md
 LICENSE
@@ -12,6 +11,21 @@ LICENSE
 .env.template
 .github
 .idea
+.prettierignore
 LICENSE.md
 docs
-update.sh
+update.sh
+.pytest_cache
+cookbook/tests
+mediafiles
+staticfiles
+db.sqlite3
+pytest.ini
+vue/**/*.vue
+vue/**/*.ts
+**/.openapi-generator
+mkdocs.yml
+vue/babel.config*
+vue/package.json
+vue/tsconfig.json
+vue/src/utils/openapi

.env.template

@@ -1,132 +1,18 @@
-# only set this to true when testing/debugging
-# when unset: 1 (true) - dont unset this, just for development
-DEBUG=0
-SQL_DEBUG=0
-
-# hosts the application can run under e.g. recipes.mydomain.com,cooking.mydomain.com,...
-ALLOWED_HOSTS=*
+# ---------------------------------------------------------------------------
+# This template contains only required options.
+# Visit the docs to find more https://docs.tandoor.dev/system/configuration/
+# ---------------------------------------------------------------------------
 
 # random secret key, use for example `base64 /dev/urandom | head -c50` to generate one
 SECRET_KEY=
 
-# your default timezone See https://timezonedb.com/time-zones for a list of timezones
-TIMEZONE=Europe/Berlin
+# allowed hosts (see documentation), should be set to your hostname(s) but might be * (default) for some proxies/providers
+# ALLOWED_HOSTS=recipes.mydomain.com
 
 # add only a database password if you want to run with the default postgres, otherwise change settings accordingly
 DB_ENGINE=django.db.backends.postgresql
-# DB_OPTIONS= {} # e.g. {"sslmode":"require"} to enable ssl
 POSTGRES_HOST=db_recipes
+POSTGRES_DB=djangodb
 POSTGRES_PORT=5432
 POSTGRES_USER=djangouser
 POSTGRES_PASSWORD=
-POSTGRES_DB=djangodb
-
-# database connection string, when used overrides other database settings.
-# format might vary depending on backend
-# DATABASE_URL = engine://username:password@host:port/dbname
-
-# the default value for the user preference 'fractions' (enable/disable fraction support)
-# default: disabled=0
-FRACTION_PREF_DEFAULT=0
-
-# the default value for the user preference 'comments' (enable/disable commenting system)
-# default comments enabled=1
-COMMENT_PREF_DEFAULT=1
-
-# Users can set a amount of time after which the shopping list is refreshed when they are in viewing mode
-# This is the minimum interval users can set. Setting this to low will allow users to refresh very frequently which
-# might cause high load on the server. (Technically they can obviously refresh as often as they want with their own scripts)
-SHOPPING_MIN_AUTOSYNC_INTERVAL=5
-
-# Default for user setting sticky navbar
-# STICKY_NAV_PREF_DEFAULT=1
-
-# If staticfiles are stored at a different location uncomment and change accordingly
-# STATIC_URL=/static/
-
-# If mediafiles are stored at a different location uncomment and change accordingly
-# MEDIA_URL=/media/
-
-# Serve mediafiles directly using gunicorn. Basically everyone recommends not doing this. Please use any of the examples
-# provided that include an additional nxginx container to handle media file serving.
-# If you know what you are doing turn this back on (1) to serve media files using djangos serve() method.
-# when unset: 1 (true) - this is temporary until an appropriate amount of time has passed for everyone to migrate
-GUNICORN_MEDIA=0
-
-# S3 Media settings: store mediafiles in s3 or any compatible storage backend (e.g. minio)
-# as long as S3_ACCESS_KEY is not set S3 features are disabled
-# S3_ACCESS_KEY=
-# S3_SECRET_ACCESS_KEY=
-# S3_BUCKET_NAME=
-# S3_REGION_NAME= # default none, set your region might be required
-# S3_QUERYSTRING_AUTH=1 # default true, set to 0 to serve media from a public bucket without signed urls
-# S3_QUERYSTRING_EXPIRE=3600 # number of seconds querystring are valid for
-# S3_ENDPOINT_URL= # when using a custom endpoint like minio
-
-# Email Settings, see https://docs.djangoproject.com/en/3.2/ref/settings/#email-host
-# Required for email confirmation and password reset (automatically activates if host is set)
-# EMAIL_HOST=
-# EMAIL_PORT=
-# EMAIL_HOST_USER=
-# EMAIL_HOST_PASSWORD=
-# EMAIL_USE_TLS=0
-# EMAIL_USE_SSL=0
-# DEFAULT_FROM_EMAIL= # email sender address (default 'webmaster@localhost')
-# ACCOUNT_EMAIL_SUBJECT_PREFIX= # prefix used for account related emails (default "[Tandoor Recipes] ")
-
-# allow authentication via reverse proxy (e.g. authelia), leave off if you dont know what you are doing
-# see docs for more information https://vabene1111.github.io/recipes/features/authentication/
-# when unset: 0 (false)
-REVERSE_PROXY_AUTH=0
-
-# If base URL is something other than just /  (you are serving a subfolder in your proxy for instance http://recipe_app/recipes/)
-# SCRIPT_NAME=/recipes
-# Default settings for spaces, apply per space and can be changed in the admin view
-# SPACE_DEFAULT_MAX_RECIPES=0 # 0=unlimited recipes
-# SPACE_DEFAULT_MAX_USERS=0 # 0=unlimited users per space
-# SPACE_DEFAULT_MAX_FILES=0 # Maximum file storage for space in MB. 0 for unlimited, -1 to disable file upload.
-# SPACE_DEFAULT_ALLOW_SHARING=1 # Allow users to share recipes with public links
-
-# allow people to create accounts on your application instance (without an invite link)
-# when unset: 0 (false)
-# ENABLE_SIGNUP=0
-
-# If signup is enabled you might want to add a captcha to it to prevent spam
-# HCAPTCHA_SITEKEY=
-# HCAPTCHA_SECRET=
-
-# if signup is enabled you might want to provide urls to data protection policies or terms and conditions
-# TERMS_URL=
-# PRIVACY_URL=
-# IMPRINT_URL=
-
-# enable serving of prometheus metrics under the /metrics path
-# ATTENTION: view is not secured (as per the prometheus default way) so make sure to secure it
-# trough your web server (or leave it open of you dont care if the stats are exposed)
-# ENABLE_METRICS=0
-
-# allows you to setup OAuth providers
-# see docs for more information https://vabene1111.github.io/recipes/features/authentication/
-# SOCIAL_PROVIDERS = allauth.socialaccount.providers.github, allauth.socialaccount.providers.nextcloud,
-
-# Should a newly created user from a social provider get assigned to the default space and given permission by default ?
-# ATTENTION: This feature might be deprecated in favor of a space join and public viewing system in the future
-# default 0 (false), when 1 (true) users will be assigned space and group
-# SOCIAL_DEFAULT_ACCESS = 1
-
-# if SOCIAL_DEFAULT_ACCESS is used, which group should be added
-# SOCIAL_DEFAULT_GROUP=guest
-
-# Django session cookie settings. Can be changed to allow a single django application to authenticate several applications
-# when running under the same database
-# SESSION_COOKIE_DOMAIN=.example.com
-# SESSION_COOKIE_NAME=sessionid # use this only to not interfere with non unified django applications under the same top level domain
-
-
-# by default SORT_TREE_BY_NAME is disabled this will store all Keywords and Food in the order they are created
-# enabling this setting makes saving new keywords and foods very slow, which doesn't matter in most usecases.
-# however, when doing large imports of recipes that will create new objects, can increase total run time by 10-15x
-# Keywords and Food can be manually sorted by name in Admin
-# This value can also be temporarily changed in Admin, it will revert the next time the application is started
-# This will be fixed/changed in the future by changing the implementation or finding a better workaround for sorting
-# SORT_TREE_BY_NAME=0

.flake8

@@ -0,0 +1,22 @@
+[flake8]
+extend-ignore =
+    # Whitespace before ':' - Required for black compatibility
+    E203,
+    # Line break occurred before a binary operator - Required for black compatibility
+    W503,
+    # Comparison to False should be 'if cond is False:' or 'if not cond:'
+    E712
+exclude =
+    .git,
+    **/__pycache__,
+        **/.git,
+        **/.svn,
+        **/.hg,
+        **/CVS,
+        **/.DS_Store,
+        .vscode,
+        **/*.pyc
+per-file-ignores=
+    cookbook/apps.py:F401
+max-line-length = 179
+

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: [vabene1111]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,15 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
----
-
-### Version
-Please provide your current version (can be found on the system page since v0.8.4)
-Version: 
-
-### Bug description
-A clear and concise description of what the bug is.

.github/ISSUE_TEMPLATE/bug_report.md.bak

@@ -0,0 +1,81 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+## Version
+<!-- Please provide your current version (can be found on the system page since v0.8.4). -->
+**Tandoor-Version:** 
+
+## Setup configuration
+<!--Please tick all boxes which apply to your configuration. Feel free to provide additional information below.  
+To tick boxes here, simply put an X inside the brackets below -->
+
+### Setup
+- [ ] Docker / Docker-Compose
+- [ ] Unraid
+- [ ] Synology
+- [ ] Kubernetes
+- [ ] Manual setup
+- [ ] Others (please state below)
+
+### Reverse Proxy
+- [ ] No reverse proxy
+- [ ] jwilder's nginx proxy
+- [ ] Nginx proxy manager (NPM)
+- [ ] SWAG
+- [ ] Caddy
+- [ ] Traefik
+- [ ] Others (please state below)
+
+<!-- Please provide additional information if possible -->
+**Additional information:** 
+
+## Bug description
+A clear and concise description of what the bug is.
+
+
+
+## Logs
+<!-- *(Remove this section entirely if no logs are available or necessary for your issue)*  
+To get the most information about your issue, set DEBUG=1 (e.g. in your `.env` file if using docker-compose) and try to reproduce the issue afterwards.
+
+Please put your logs into the expandable section below and use code quotation for all logs! Usage: Put three backticks in front and after the log, like this:  
+` ``` <Many lines of log messages ``` ` 
+
+Feel free to remove parts if you don't fill them out.
+-->
+
+<details>
+  <summary>Web-Container-Logs</summary>
+    
+  <!-- *Put your logs inside here (leave the code quotations in takt):* -->
+  
+  ```
+  Replace me with logs
+  ```
+</details>
+
+<details>
+  <summary>DB-Container-Logs</summary>
+    
+  <!-- *Put your logs inside here (leave the code quotations in takt):* -->
+  
+  ```
+  Replace me with logs
+  ```
+</details>
+
+<details>
+  <summary>Nginx-Container-Logs <!-- if you use one --></summary>
+    
+  <!-- *Put your logs inside here (leave the code quotations in takt):* -->
+  
+  ```
+  Replace me with logs
+  ```
+</details>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,64 @@
+name: Bug Report
+description: "Create a report to help us improve"
+#title: ""
+#labels: ["Bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: version
+    attributes:
+      label: Tandoor Version
+      description: "What version of Tandoor are you using? (can be found on the system page since v0.8.4)"
+    validations:
+      required: true
+  - type: dropdown
+    id: setup
+    attributes:
+      label: Setup
+      description: "How is your Tandoor instance set up?"
+      options:
+        - Docker / Docker-Compose
+        - Unraid
+        - Synology
+        - Kubernetes
+        - Manual Setup
+        - Others (please state below)
+    validations:
+      required: true
+  - type: dropdown
+    id: reverse-proxy
+    attributes:
+      label: "Reverse Proxy"
+      description: "What reverse proxy do you use with Tandoor?"
+      options:
+        - No reverse proxy
+        - jwilder's nginx proxy
+        - Nginx Proxy Manager (NPM)
+        - SWAG
+        - Caddy
+        - Traefik
+        - Apache2
+        - Others (please state below)
+    validations:
+      required: true
+  - type: input
+    id: other
+    attributes:
+      label: Other
+      description: "In case you chose 'Others' above, please provide more info here."
+  - type: textarea
+    id: bug-descr
+    attributes:
+      label: Bug description
+      description: "Please accurately describe the bug you encountered."
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant logs
+      description: Please copy and paste any relevant logs. This will be automatically formatted into code, so no need for backticks.
+      render: shell

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: FAQs
+    url: https://docs.tandoor.dev/faq/
+    about: Please take a look at the FAQs before creating a bug ticket.

.github/ISSUE_TEMPLATE/doc_issue.yml

@@ -0,0 +1,40 @@
+name: Documentation Issue
+description: "Create a report to help us improve"
+#title: ""
+labels: ["documentation"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this documentation issue report!
+  - type: input
+    id: docs-link
+    attributes:
+      label: Documentation link
+      description: "Please provide a link to the corresponding documentation site on docs.tandoor.dev"
+  - type: dropdown
+    id: section
+    attributes:
+      label: Affected section
+      description: "What part of the documentation is the issue about?"
+      options:
+        - Installation
+        - Features
+        - System
+        - FAQ
+        - Does not exist yet
+        - Other (please state below)
+    validations:
+      required: true
+  - type: input
+    id: other
+    attributes:
+      label: Other
+      description: "In case you chose 'Other' above, please provide more info here."
+  - type: textarea
+    id: descr
+    attributes:
+      label: Issue description
+      description: "Please accurately describe the documentation issue you are seeing."
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,31 @@
+name: Feature Request
+description: "Suggest an idea for this project"
+#title: ""
+labels: ["enhancement"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this feature request!
+  - type: textarea
+    id: problem
+    attributes:
+      label: "Is your feature request related to a problem? Please describe."
+      description: "A clear and concise description of what the problem is. Ex. I'm always frustrated when..."
+  - type: textarea
+    id: solution
+    attributes:
+      label: "Describe the solution you'd like"
+      description: "A clear and concise description of what you want to happen."
+    validations:
+      required: true
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: "Describe alternatives you've considered"
+      description: "A clear and concise description of any alternative solutions or features you've considered."
+  - type: textarea
+    id: additional
+    attributes:
+      label: "Additional context"
+      description: "Add any other context or screenshots about the feature request here."

.github/ISSUE_TEMPLATE/help_request.yml

@@ -0,0 +1,82 @@
+name: Help request
+description: "If there is anything wrong with your setup"
+#title: ""
+labels: ["setup issue"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this help request!
+  - type: textarea
+    id: issue
+    attributes:
+      label: Issue
+      description: "Please describe your problem here."
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: Tandoor Version
+      description: "What version of Tandoor are you using? (can be found on the system page since v0.8.4)"
+    validations:
+      required: true
+  - type: input
+    id: os
+    attributes:
+      label: OS Version
+      description: "E.g. Ubuntu 20.02"
+    validations:
+      required: true
+  - type: dropdown
+    id: setup
+    attributes:
+      label: Setup
+      description: "How is your Tandoor instance set up?"
+      options:
+        - Docker / Docker-Compose
+        - Unraid
+        - Synology
+        - Kubernetes
+        - Manual Setup
+        - Others (please state below)
+    validations:
+      required: true
+  - type: dropdown
+    id: reverse-proxy
+    attributes:
+      label: "Reverse Proxy"
+      description: "What reverse proxy do you use with Tandoor?"
+      options:
+        - No reverse proxy
+        - jwilder's nginx proxy
+        - Nginx Proxy Manager (NPM)
+        - SWAG
+        - Caddy
+        - Traefik
+        - Others (please state below)
+    validations:
+      required: true
+  - type: input
+    id: other
+    attributes:
+      label: Other
+      description: "In case you chose 'Others' above or have more info, please provide additional details here."
+  - type: textarea
+    id: env
+    attributes:
+      label: Environment file
+      description: "Please include your `.env` config file (**make sure to remove/replace all secrets**)"
+      render: shell
+  - type: textarea
+    id: docker-compose
+    attributes:
+      label: Docker-Compose file
+      description: "When running with docker compose please provide your `docker-compose.yml`"
+      render: shell
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant logs
+      description: "If you feel like there is anything interesting please post the output of `docker-compose logs` at container startup and when the issue happens."
+      render: shell

.github/ISSUE_TEMPLATE/website_import.yml

@@ -0,0 +1,36 @@
+name: Website Import
+description: "Anything related to website imports"
+#title: ""
+#labels: ["enhancement"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this website import form!
+  - type: input
+    id: version
+    attributes:
+      label: Tandoor Version
+      description: "What version of Tandoor are you using? (can be found on the system page since v0.8.4)"
+    validations:
+      required: true
+  - type: input
+    id: url
+    attributes:
+      label: Import URL
+      description: "Exact URL you are trying to import from."
+    validations:
+      required: true
+  - type: textarea
+    id: bug-descr
+    attributes:
+      label: "When did the issue happen?"
+      description: "When pressing the search button with the url / when importing after the page has loaded / ..."
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Response / message shown
+      description: Please copy and paste any relevant logs or responses / messages which are shown in Tandoor. This will be automatically formatted into code, so no need for backticks.
+      render: shell

.github/dependabot.yml

@@ -1,10 +1,15 @@
 # To get started with Dependabot version updates, you'll need to specify which
 # package ecosystems to update and where the package manifests are located.
 # Please see the documentation for all configuration options:
-# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
 
 version: 2
 updates:
+  - package-ecosystem: "devcontainers"
+    directory: "/"
+    schedule:
+      interval: weekly
+
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
@@ -14,3 +19,8 @@ updates:
     directory: "/vue/"
     schedule:
       interval: "monthly"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"

.github/workflows/build-docker-open-data.yml

@@ -0,0 +1,110 @@
+name: Build Docker Container with open data plugin installed
+
+on: push
+
+jobs:
+  build-container:
+    name: Build ${{ matrix.name }} Container
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'TandoorRecipes'
+    continue-on-error: ${{ matrix.continue-on-error }}
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      matrix:
+        include:
+          # Standard build config
+          - name: Standard
+            dockerfile: Dockerfile
+            platforms: linux/amd64,linux/arm64
+            suffix: ""
+            continue-on-error: false
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get version number
+        id: get_version
+        run: |
+          if [[ "$GITHUB_REF" = refs/tags/* ]]; then
+            echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
+          elif [[ "$GITHUB_REF" = refs/heads/beta ]]; then
+            echo VERSION=beta >> $GITHUB_OUTPUT
+          else
+            echo VERSION=develop >> $GITHUB_OUTPUT
+          fi
+
+      # clone open data plugin
+      - name: clone open data plugin repo
+        uses: actions/checkout@master
+        with:
+          repository: TandoorRecipes/open_data_plugin
+          ref: master
+          path: ./recipes/plugins/open_data_plugin
+
+      # Build Vue frontend
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+          cache: yarn
+          cache-dependency-path: vue/yarn.lock
+      - name: Install dependencies
+        working-directory: ./vue
+        run: yarn install --frozen-lockfile
+      - name: Build dependencies
+        working-directory: ./vue
+        run: yarn build
+
+      - name: Setup Open Data Plugin Links
+        working-directory: ./recipes/plugins/open_data_plugin
+        run: python setup_repo.py
+        
+      - name: Build Open Data Frontend
+        working-directory: ./recipes/plugins/open_data_plugin/vue
+        run: yarn build
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        if: github.secret_source == 'Actions'
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        if: github.secret_source == 'Actions'
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            vabene1111/recipes
+            ghcr.io/TandoorRecipes/recipes
+          flavor: |
+            latest=false
+            suffix=${{ matrix.suffix }}
+          tags: |
+            type=raw,value=latest,suffix=-open-data-plugin,enable=${{ startsWith(github.ref, 'refs/tags/') }}
+            type=semver,suffix=-open-data-plugin,pattern={{version}}
+            type=semver,suffix=-open-data-plugin,pattern={{major}}.{{minor}}
+            type=semver,suffix=-open-data-plugin,pattern={{major}}
+            type=ref,suffix=-open-data-plugin,event=branch
+      - name: Build and Push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.dockerfile }}
+          pull: true
+          push: ${{ github.secret_source == 'Actions' }}
+          platforms: ${{ matrix.platforms }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/build-docker.yml

@@ -0,0 +1,126 @@
+name: Build Docker Container
+
+on: push
+
+jobs:
+  build-container:
+    name: Build ${{ matrix.name }} Container
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'TandoorRecipes'
+    continue-on-error: ${{ matrix.continue-on-error }}
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      matrix:
+        include:
+          # Standard build config
+          - name: Standard
+            dockerfile: Dockerfile
+            platforms: linux/amd64,linux/arm64,linux/arm/v7
+            suffix: ""
+            continue-on-error: false
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get version number
+        id: get_version
+        run: |
+          if [[ "$GITHUB_REF" = refs/tags/* ]]; then
+            echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
+          elif [[ "$GITHUB_REF" = refs/heads/beta ]]; then
+            echo VERSION=beta >> $GITHUB_OUTPUT
+          else
+            echo VERSION=develop >> $GITHUB_OUTPUT
+          fi
+
+      # Build Vue frontend
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+          cache: yarn
+          cache-dependency-path: vue/yarn.lock
+      - name: Install dependencies
+        working-directory: ./vue
+        run: yarn install --frozen-lockfile
+      - name: Build dependencies
+        working-directory: ./vue
+        run: yarn build
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        if: github.secret_source == 'Actions'
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        if: github.secret_source == 'Actions'
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            vabene1111/recipes
+            ghcr.io/TandoorRecipes/recipes
+          flavor: |
+            latest=false
+            suffix=${{ matrix.suffix }}
+          tags: |
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') }}
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=ref,event=branch
+      - name: Build and Push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ matrix.dockerfile }}
+          pull: true
+          push: ${{ github.secret_source == 'Actions' }}
+          platforms: ${{ matrix.platforms }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  notify-stable:
+    name: Notify Stable
+    runs-on: ubuntu-latest
+    needs: build-container
+    if: startsWith(github.ref, 'refs/tags/')
+    steps:
+      - name: Set tag name
+        run: |
+          # Strip "refs/tags/" prefix
+          echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+      # Send stable discord notification
+      - name: Discord notification
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.DISCORD_RELEASE_WEBHOOK }}
+        uses: Ilshidur/action-discord@0.3.2
+        with:
+          args: '🚀 Version {{ VERSION }} of tandoor has been released 🥳 Check it out https://github.com/vabene1111/recipes/releases/tag/{{ VERSION }}'
+
+  notify-beta:
+    name: Notify Beta
+    runs-on: ubuntu-latest
+    needs: build-container
+    if: github.ref == 'refs/heads/beta'
+    steps:
+      # Send beta discord notification
+      - name: Discord notification
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.DISCORD_BETA_WEBHOOK }}
+        uses: Ilshidur/action-discord@0.3.2
+        with:
+          args: '🚀 The BETA Image has been updated! 🥳'

.github/workflows/ci.yml

@@ -1,38 +1,81 @@
-name: Continous Integration
+name: Continuous Integration
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
-  build:
-    if: github.repository_owner == 'vabene1111'
-    runs-on: ubuntu-latest
-    strategy:
-      max-parallel: 4
-      matrix:
-        python-version: [3.9]
+    build:
+        if: github.repository_owner == 'TandoorRecipes'
+        runs-on: ubuntu-latest
+        strategy:
+            max-parallel: 4
+            matrix:
+                python-version: ["3.10"]
+                node-version: ["18"]
 
-    steps:
-    - uses: actions/checkout@v1
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.9
-    # Build Vue frontend
-    - uses: actions/setup-node@v2
-      with:
-        node-version: '14'
-    - name: Install dependencies
-      working-directory: ./vue
-      run: yarn install
-    - name: Build dependencies
-      working-directory: ./vue
-      run: yarn build
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        python3 manage.py collectstatic --noinput
-        python3 manage.py collectstatic_js_reverse
-    - name: Django Testing project
-      run: |
-        pytest
+        steps:
+            - uses: actions/checkout@v4
+            - uses: awalsh128/cache-apt-pkgs-action@v1.4.2
+              with:
+                  packages: libsasl2-dev python3-dev libldap2-dev libssl-dev
+                  version: 1.0
+
+            # Setup python & dependencies
+            - name: Set up Python ${{ matrix.python-version }}
+              uses: actions/setup-python@v5
+              with:
+                  python-version: ${{ matrix.python-version }}
+                  cache: "pip"
+
+            - name: Install Python Dependencies
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install -r requirements.txt
+
+            - name: Cache StaticFiles
+              uses: actions/cache@v4
+              id: django_cache
+              with:
+                  path: |
+                      ./cookbook/static
+                      ./vue/webpack-stats.json
+                      ./staticfiles
+                  key: |
+                      ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }}
+
+            # Build Vue frontend & Dependencies
+            - name: Set up Node ${{ matrix.node-version }}
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              uses: actions/setup-node@v4
+              with:
+                  node-version: ${{ matrix.node-version }}
+                  cache: "yarn"
+                  cache-dependency-path: ./vue/yarn.lock
+
+            - name: Install Vue dependencies
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              working-directory: ./vue
+              run: yarn install
+
+            - name: Build Vue dependencies
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              working-directory: ./vue
+              run: yarn build
+
+            - name: Compile Django StaticFiles
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              run: |
+                  python3 manage.py collectstatic --noinput
+                  python3 manage.py collectstatic_js_reverse
+
+            - uses: actions/cache/save@v4
+              if: steps.django_cache.outputs.cache-hit != 'true'
+              with:
+                  path: |
+                      ./cookbook/static
+                      ./vue/webpack-stats.json
+                      ./staticfiles
+                  key: |
+                      ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.node-version }}-collectstatic-${{ hashFiles('**/*.css', '**/*.js', 'vue/src/*') }}
+
+            - name: Django Testing project
+              run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml

.github/workflows/codeql-analysis.yml

@@ -8,11 +8,11 @@ on:
 
 jobs:
   CodeQL-Build:
-    if: github.repository_owner == 'vabene1111'
+    if: github.repository_owner == 'TandoorRecipes'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           # We must fetch at least the immediate parents so that if this is
           # a pull request then we can checkout the head.
@@ -25,7 +25,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v1
+        uses: github/codeql-action/init@v3
         # Override language selection by uncommenting this and choosing your languages
         with:
           languages: python, javascript
@@ -47,6 +47,6 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v1
+        uses: github/codeql-action/analyze@v3
         with:
           languages: javascript, python

.github/workflows/docker-publish-beta.yml

@@ -1,46 +0,0 @@
-name: publish beta image docker
-on:
-  push:
-    branches:
-      - 'beta'
-jobs:
-  build:
-    if: github.repository_owner == 'vabene1111'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-      # Update Version number
-      - name: Update version file
-        uses: DamianReeves/write-file-action@v1.0
-        with:
-          path: recipes/version.py
-          contents: |
-            VERSION_NUMBER = 'beta'
-            BUILD_REF = '${{ github.sha }}'
-          write-mode: overwrite
-      # Build Vue frontend
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-      - name: Install dependencies
-        working-directory: ./vue
-        run: yarn install
-      - name: Build dependencies
-        working-directory: ./vue
-        run: yarn build
-      # Build container
-      - name: Build and publish image
-        uses: ilteoood/docker_buildx@master
-        with:
-          publish: true
-          imageName: vabene1111/recipes
-          tag: beta
-          dockerHubUser: ${{ secrets.DOCKER_USERNAME }}
-          dockerHubPassword: ${{ secrets.DOCKER_PASSWORD }}
-      # Send discord notification
-      - name: Discord notification
-        env:
-          DISCORD_WEBHOOK: ${{ secrets.DISCORD_BETA_WEBHOOK }}
-        uses: Ilshidur/action-discord@0.3.2
-        with:
-          args: '🚀 The BETA Image has been updated! 🥳'

.github/workflows/docker-publish-dev.yml

@@ -1,39 +0,0 @@
-name: publish dev image docker
-on:
-  push:
-    branches:
-      - '*'
-      - '*/*'
-      - '!master'
-jobs:
-  build:
-    if: github.repository_owner == 'vabene1111'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-      # Update Version number
-      - name: Update version file
-        uses: DamianReeves/write-file-action@v1.0
-        with:
-          path: recipes/version.py
-          contents: |
-            VERSION_NUMBER = 'develop'
-            BUILD_REF = '${{ github.sha }}'
-          write-mode: overwrite
-      # Build Vue frontend
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-      - name: Install dependencies
-        working-directory: ./vue
-        run: yarn install
-      - name: Build dependencies
-        working-directory: ./vue
-        run: yarn build
-      # Build container
-      - name: Publish to Registry
-        uses: elgohr/Publish-Docker-Github-Action@2.13
-        with:
-          name: vabene1111/recipes
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/docker-publish-latest.yml

@@ -1,43 +0,0 @@
-name: publish latest image docker
-on:
-  push:
-    tags:
-    - '*'
-
-jobs:
-  build:
-    if: github.repository_owner == 'vabene1111'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-      - name: Get version number
-        id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
-      # Update Version number
-      - name: Update version file
-        uses: DamianReeves/write-file-action@v1.0
-        with:
-          path: recipes/version.py
-          contents: |
-            VERSION_NUMBER = '${{ steps.get_version.outputs.VERSION }}'
-            BUILD_REF = '${{ github.sha }}'
-          write-mode: overwrite
-      # Build Vue frontend
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-      - name: Install dependencies
-        working-directory: ./vue
-        run: yarn install
-      - name: Build dependencies
-        working-directory: ./vue
-        run: yarn build
-      # Build container
-      - name: Build and publish image
-        uses: ilteoood/docker_buildx@master
-        with:
-          publish: true
-          imageName: vabene1111/recipes
-          tag: latest
-          dockerHubUser: ${{ secrets.DOCKER_USERNAME }}
-          dockerHubPassword: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/docker-publish-release.yml

@@ -1,53 +0,0 @@
-name: publish tagged release docker
-
-on:
-  push:
-    tags:
-    - '*'
-
-jobs:
-  build:
-    if: github.repository_owner == 'vabene1111'
-    runs-on: ubuntu-latest
-    name: Build image job
-    steps:
-      - name: Checkout master
-        uses: actions/checkout@master
-      - name: Get version number
-        id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
-      # Update Version number
-      - name: Update version file
-        uses: DamianReeves/write-file-action@v1.0
-        with:
-          path: recipes/version.py
-          contents: |
-            VERSION_NUMBER = '${{ steps.get_version.outputs.VERSION }}'
-            BUILD_REF = '${{ github.sha }}'
-          write-mode: overwrite
-      # Build Vue frontend
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-      - name: Install dependencies
-        working-directory: ./vue
-        run: yarn install
-      - name: Build dependencies
-        working-directory: ./vue
-        run: yarn build
-      # Build container
-      - name: Build and publish image
-        uses: ilteoood/docker_buildx@master
-        with:
-          publish: true
-          imageName: vabene1111/recipes
-          tag: ${{ steps.get_version.outputs.VERSION }}
-          dockerHubUser: ${{ secrets.DOCKER_USERNAME }}
-          dockerHubPassword: ${{ secrets.DOCKER_PASSWORD }}
-      # Send discord notification
-      - name: Discord notification
-        env:
-          DISCORD_WEBHOOK: ${{ secrets.DISCORD_RELEASE_WEBHOOK }}
-        uses: Ilshidur/action-discord@0.3.2
-        with:
-          args: '🚀 A new Version of tandoor has been released 🥳 \n https://github.com/vabene1111/recipes/releases/tag/{{ steps.get_version.outputs.VERSION }}'

.github/workflows/docs.yml

@@ -1,18 +1,20 @@
 name: Make Docs
 on:
-  push:
-    branches:
-      - master
-      - develop
+  # the 1st condition
+  workflow_run:
+    workflows: ["Continuous Integration"]
+    branches: [master]
+    types:
+      - completed
 
 jobs:
   deploy:
-    if: github.repository_owner == 'vabene1111'
+    if: github.repository_owner == 'TandoorRecipes' && ${{ github.event.workflow_run.conclusion == 'success' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.x
-      - run: pip install mkdocs-material
-      - run: mkdocs gh-deploy --force
+      - run: pip install mkdocs-material mkdocs-include-markdown-plugin
+      - run: mkdocs gh-deploy --force

.gitignore

@@ -43,9 +43,15 @@ htmlcov/
 nosetests.xml
 coverage.xml
 *,cover
+docs/reports/**
 
 # Django stuff:
 *.log
+mediafiles/
+*.sqlite3*
+staticfiles/
+postgresql/
+data/
 
 # Sphinx documentation
 docs/_build/
@@ -54,33 +60,27 @@ docs/_build/
 target/
 
 \.idea/dataSources/
-
 \.idea/dataSources\.xml
 
 \.idea/dataSources\.local\.xml
 
-venv/
-
-mediafiles/
-
-*.sqlite3*
-
 \.idea/workspace\.xml
 
 \.idea/misc\.xml
 
 # Deployment
-
 \.env
-staticfiles/
-postgresql/
 
-
-/docker-compose.override.yml
-vue/node_modules
-.vscode/
-vue/yarn.lock
-vetur.config.js
 cookbook/static/vue
 vue/webpack-stats.json
+/docker-compose.override.yml
+vue/node_modules
+plugins
+vue3/node_modules
 cookbook/templates/sw.js
+vue/.yarn
+vue3/.vite
+
+# Configs
+vetur.config.js
+venv/

.idea/dictionaries/vaben.xml

@@ -0,0 +1,10 @@
+<component name="ProjectDictionaryState">
+  <dictionary name="vaben">
+    <words>
+      <w>mealplan</w>
+      <w>pinia</w>
+      <w>selfhosted</w>
+      <w>unapplied</w>
+    </words>
+  </dictionary>
+</component>

.idea/dictionaries/vabene1111_PC.xml

@@ -6,6 +6,7 @@
       <w>csrftoken</w>
       <w>gunicorn</w>
       <w>ical</w>
+      <w>invitelink</w>
       <w>mealie</w>
       <w>pepperplate</w>
       <w>safron</w>

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,5 +1,6 @@
 <component name="InspectionProjectProfileManager">
   <settings>
+    <option name="PROJECT_PROFILE" value="Default" />
     <option name="USE_PROJECT_PROFILE" value="false" />
     <version value="1.0" />
   </settings>

.idea/recipes.iml

@@ -18,7 +18,7 @@
       <excludeFolder url="file://$MODULE_DIR$/staticfiles" />
       <excludeFolder url="file://$MODULE_DIR$/venv" />
     </content>
-    <orderEntry type="jdk" jdkName="Python 3.9 (recipes)" jdkType="Python SDK" />
+    <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="TemplatesService">

.idea/vcs.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+    <mapping directory="" vcs="Git" />
   </component>
 </project>

.idea/watcherTasks.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectTasksOptions">
+    <TaskOptions isEnabled="false">
+      <option name="arguments" value="-m flake8 $FilePath$ --config $ContentRoot$\.flake8" />
+      <option name="checkSyntaxErrors" value="true" />
+      <option name="description" />
+      <option name="exitCodeBehavior" value="ALWAYS" />
+      <option name="fileExtension" value="py" />
+      <option name="immediateSync" value="false" />
+      <option name="name" value="Flake8 Watcher" />
+      <option name="output" value="$FilePath$" />
+      <option name="outputFilters">
+        <array>
+          <FilterInfo>
+            <option name="description" value="" />
+            <option name="name" value="" />
+            <option name="regExp" value="$FILE_PATH$:$LINE$:$COLUMN$: $MESSAGE$" />
+          </FilterInfo>
+        </array>
+      </option>
+      <option name="outputFromStdout" value="false" />
+      <option name="program" value="$PyInterpreterDirectory$/python" />
+      <option name="runOnExternalChanges" value="false" />
+      <option name="scopeName" value="Current File" />
+      <option name="trackOnlyRoot" value="false" />
+      <option name="workingDir" value="" />
+      <envs />
+    </TaskOptions>
+  </component>
+</project>

.prettierignore

@@ -0,0 +1,13 @@
+# generated files
+api.ts
+vue/src/apps/*.js
+vue/node_modules
+staticfiles/
+docs/reports/
+/vue3/src/openapi/
+
+# ignored files - prettier interferes with django templates and github actions
+*.html
+*.yml
+*.yaml
+

.prettierrc

@@ -0,0 +1,7 @@
+{
+    "printWidth": 179,
+    "trailingComma": "es5",
+    "tabWidth": 2,
+    "semi": false,
+    "experimentalTernaries": true
+}

.vscode/launch.json

@@ -0,0 +1,33 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+      {
+        "name": "Python Debugger: Django",
+        "type": "debugpy",
+        "request": "launch",
+        "program": "${workspaceFolder}/manage.py",
+        "args": ["runserver"],
+        "django": true,
+        "justMyCode": true
+      },
+    {
+      "name": "Python: Debug Tests",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "purpose": [
+        "debug-test"
+      ],
+      "console": "integratedTerminal",
+      "env": {
+        // coverage and pytest can't both be running at the same time
+        "PYTEST_ADDOPTS": "--no-cov"
+      },
+      "django": true,
+      "justMyCode": true
+    },
+    ]
+  }

.vscode/settings.json

@@ -0,0 +1,14 @@
+{
+    "python.testing.pytestArgs": [
+        "cookbook/tests"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true,
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "[python]": {
+        "editor.defaultFormatter": "eeyore.yapf",
+    },
+    "yapf.args": [],
+    "isort.args": []
+}

.vscode/tasks.json

@@ -0,0 +1,75 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+      {
+        "label": "Run Migrations",
+        "type": "shell",
+        "command": "python3 manage.py migrate",
+      },
+      {
+        "label": "Collect Static Files",
+        "type": "shell",
+        "command": "python3 manage.py collectstatic",
+        "dependsOn": ["Yarn Build"],
+      },
+      {
+        "label": "Setup Dev Server",
+        "dependsOn": ["Run Migrations", "Yarn Build"],
+      },
+      {
+        "label": "Run Dev Server",
+        "type": "shell",
+        "dependsOn": ["Setup Dev Server"],
+        "command": "python3 manage.py runserver",
+      },
+      {
+        "label": "Yarn Install",
+        "type": "shell",
+        "command": "yarn install",
+        "options": {
+          "cwd": "${workspaceFolder}/vue"
+        }
+      },
+      {
+        "label": "Yarn Serve",
+        "type": "shell",
+        "command": "yarn serve",
+        "dependsOn": ["Yarn Install"],
+        "options": {
+          "cwd": "${workspaceFolder}/vue"
+        }
+      },
+      {
+        "label": "Yarn Build",
+        "type": "shell",
+        "command": "yarn build",
+        "dependsOn": ["Yarn Install"],
+        "options": {
+          "cwd": "${workspaceFolder}/vue"
+        },
+        "group": "build",
+      },
+      {
+        "label": "Setup Tests",
+        "dependsOn": ["Run Migrations", "Collect Static Files"],
+      },
+      {
+        "label": "Run all pytests",
+        "type": "shell",
+        "command": "python3 -m pytest cookbook/tests",
+        "dependsOn": ["Setup Tests"],
+        "group": "test",
+      },
+      {
+        "label": "Setup Documentation Dependencies",
+        "type": "shell",
+        "command": "pip install mkdocs-material mkdocs-include-markdown-plugin",
+      },
+      {
+        "label": "Serve Documentation",
+        "type": "shell",
+        "command": "mkdocs serve",
+        "dependsOn": ["Setup Documentation Dependencies"],
+      }
+    ]
+}

CONTRIBUTERS.md

@@ -1,58 +1,84 @@
+# Contributers
+
 Many thanks to everyone who contributed to this project! If you add something or help out feel free to add yourself
 to this list.
 
 ## Code/Features
+
 Please have a look at the [list of pull requests](https://github.com/vabene1111/recipes/pulls) for 
 a complete list of contributions.
 Below are some of the larger contributions made yet.
 
-
-- @tourn provided the serving feature and **several** other improvements!
-- @l0c4lh057 provided a much improved ingredient text parser in [#277](https://github.com/vabene1111/recipes/pull/277)
-- @sebimarkgraf added nutritional information [#199](https://github.com/vabene1111/recipes/pull/199)
-- @cazier added reverse proxy authentication [#88](https://github.com/vabene1111/recipes/pull/88)
+- [vabene1111]
+- [Kaibu]
+- [smilerz]
+- [MaxJa4] Docker builds and other improvements
+- [tourn] provided the serving feature and **several** other improvements!
+- [l0c4lh057] provided a much improved ingredient text parser in [#277](https://github.com/vabene1111/recipes/pull/277)
+- [sebimarkgraf] added nutritional information [#199](https://github.com/vabene1111/recipes/pull/199)
+- [cazier] added reverse proxy authentication [#88](https://github.com/vabene1111/recipes/pull/88)
+- [murphy83] added support for IPv6 #1490
+- [TheHaf] added custom serving size component #1411
+- [lostlont] added LDAP support #960
+- [c0mputerguru] added devcontainers for ease of development
 
 ## Translations
 
-### Catalan 
+### Catalan
+
 [Rubenix](https://www.transifex.com/user/profile/rubenix/)
 
 ### Dutch
-[D0T1X](https://www.transifex.com/user/profile/D0T1X/)  
-[ikbenfrank](https://www.transifex.com/user/profile/ikbenfrank/)  
-[kampsj](https://www.transifex.com/user/profile/kampsj/)  
+
+[D0T1X](https://www.transifex.com/user/profile/D0T1X/)
+[ikbenfrank](https://www.transifex.com/user/profile/ikbenfrank/)
+[kampsj](https://www.transifex.com/user/profile/kampsj/)
 
 ### French
-[jt117](https://www.transifex.com/user/profile/jt117/)  
-[nerdinator](https://www.transifex.com/user/profile/nerdinator/)  
-[agaume](https://www.transifex.com/user/profile/agaume/)  
+
+[jt117](https://www.transifex.com/user/profile/jt117/)
+[nerdinator](https://www.transifex.com/user/profile/nerdinator/)
+[agaume](https://www.transifex.com/user/profile/agaume/)
 
 ### German
+
 [eTaurus](https://www.transifex.com/user/profile/eTaurus/)
 [l0c4lh057](https://www.transifex.com/user/profile/l0c4lh057/)
+[hyperbit00](https://github.com/hyperbit00)
 
 ### Hungarian
+
 [igazka](https://www.transifex.com/user/profile/igazka/)
 
 ### Italian
-[SK3LA](https://www.transifex.com/user/profile/SK3LA/)  
-[auanasgheps](https://www.transifex.com/user/profile/auanasgheps/)  
+
+[SK3LA](https://www.transifex.com/user/profile/SK3LA/)
+[auanasgheps](https://www.transifex.com/user/profile/auanasgheps/)
 
 ### Latvian
+
 [melkypie](https://github.com/melkypie)
 
 ### Portuguese
 
-[hds](https://www.transifex.com/user/profile/hds/)  
-[mlopezifu](https://www.transifex.com/user/profile/mlopezifu/)  
-[stormsz](https://www.transifex.com/user/profile/stormsz/)  
+[hds](https://www.transifex.com/user/profile/hds/)
+[mlopezifu](https://www.transifex.com/user/profile/mlopezifu/)
+[stormsz](https://www.transifex.com/user/profile/stormsz/)
+
+### Russian
+
+[amillerr](https://github.com/amillerr)
 
 ### Spanish
 
-[albertocp](https://www.transifex.com/user/profile/albertocp/)  
-[alfa5](https://www.transifex.com/user/profile/alfa5/)  
-[mlopezifu](https://www.transifex.com/user/profile/mlopezifu/)  
-[sergio.laya](https://www.transifex.com/user/profile/sergio.laya/)  
+[albertocp](https://www.transifex.com/user/profile/albertocp/)
+[alfa5](https://www.transifex.com/user/profile/alfa5/)
+[mlopezifu](https://www.transifex.com/user/profile/mlopezifu/)
+[sergio.laya](https://www.transifex.com/user/profile/sergio.laya/)
+
+### Swedish
+
+[makanz](https://github.com/makanz)
 
 ### Turkish
 
@@ -60,4 +86,4 @@ Below are some of the larger contributions made yet.
 
 ### Vietnamese
 
-[vuongtrunghieu](https://www.transifex.com/user/profile/vuongtrunghieu/)
+[vuongtrunghieu](https://www.transifex.com/user/profile/vuongtrunghieu/)

Dockerfile

@@ -1,11 +1,13 @@
-FROM python:3.9-alpine3.12
+FROM python:3.12-alpine3.19
 
 #Install all dependencies.
-RUN apk add --no-cache postgresql-libs gettext zlib libjpeg libxml2-dev libxslt-dev py-cryptography
+RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg libwebp libxml2-dev libxslt-dev openldap git
 
 #Print all logs without buffering it.
 ENV PYTHONUNBUFFERED 1
 
+ENV DOCKER true
+
 #This port will be used by gunicorn.
 EXPOSE 8080
 
@@ -15,14 +17,28 @@ WORKDIR /opt/recipes
 
 COPY requirements.txt ./
 
-RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-dev jpeg-dev libressl-dev libffi-dev cargo && \
+RUN \
+    if [ `apk --print-arch` = "armv7" ]; then \
+    printf "[global]\nextra-index-url=https://www.piwheels.org/simple\n" > /etc/pip.conf ; \
+    fi
+# remove Development dependencies from requirements.txt
+RUN sed -i '/# Development/,$d' requirements.txt
+RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-dev jpeg-dev libwebp-dev openssl-dev libffi-dev cargo openldap-dev python3-dev && \
+    echo -n "INPUT ( libldap.so )" > /usr/lib/libldap_r.so && \
     python -m venv venv && \
     /opt/recipes/venv/bin/python -m pip install --upgrade pip && \
-    venv/bin/pip install wheel==0.36.2 && \
+    venv/bin/pip install wheel==0.42.0 && \
+    venv/bin/pip install setuptools_rust==1.9.0 && \
     venv/bin/pip install -r requirements.txt --no-cache-dir &&\
     apk --purge del .build-deps
 
 #Copy project and execute it.
 COPY . ./
+
+# collect information from git repositories
+RUN /opt/recipes/venv/bin/python version.py
+# delete git repositories to reduce image size
+RUN find . -type d -name ".git" | xargs rm -rf
+
 RUN chmod +x boot.sh
-ENTRYPOINT ["/opt/recipes/boot.sh"]
+ENTRYPOINT ["/opt/recipes/boot.sh"]

README.md

@@ -1,6 +1,6 @@
 <h1 align="center">
   <br>
-  <a href="https://app.tandoor.dev"><img src="https://github.com/vabene1111/recipes/raw/develop/docs/logo_color.svg" height="256px" width="256px"></a>
+  <a href="https://tandoor.dev"><img src="https://github.com/vabene1111/recipes/raw/develop/docs/logo_color.svg" height="256px" width="256px"></a>
   <br>
   Tandoor Recipes
   <br>
@@ -9,67 +9,103 @@
 <h4 align="center">The recipe manager that allows you to manage your ever growing collection of digital recipes.</h4>
 
 <p align="center">
-<a href="https://github.com/vabene1111/recipes/actions" target="_blank" rel="noopener noreferrer"><img src="https://github.com/vabene1111/recipes/workflows/Continous%20Integration/badge.svg?branch=master" ></a>
+<a href="https://github.com/vabene1111/recipes/actions" target="_blank" rel="noopener noreferrer"><img src="https://github.com/vabene1111/recipes/workflows/Continuous%20Integration/badge.svg?branch=master" ></a>
 <a href="https://github.com/vabene1111/recipes/stargazers" target="_blank" rel="noopener noreferrer"><img src="https://img.shields.io/github/stars/vabene1111/recipes" ></a>
 <a href="https://github.com/vabene1111/recipes/network/members" target="_blank" rel="noopener noreferrer"><img src="https://img.shields.io/github/forks/vabene1111/recipes" ></a>
 <a href="https://discord.gg/RhzBrfWgtp" target="_blank" rel="noopener noreferrer"><img src="https://badgen.net/badge/icon/discord?icon=discord&label" ></a>
 <a href="https://hub.docker.com/r/vabene1111/recipes" target="_blank" rel="noopener noreferrer"><img src="https://img.shields.io/docker/pulls/vabene1111/recipes" ></a>
 <a href="https://github.com/vabene1111/recipes/releases/latest" rel="noopener noreferrer"><img src="https://img.shields.io/github/v/release/vabene1111/recipes" ></a>
+<a href="https://app.tandoor.dev/accounts/login/?demo" rel="noopener noreferrer"><img src="https://img.shields.io/badge/demo-available-success" ></a>
 </p>
 
 <p align="center">
+<a href="https://tandoor.dev" target="_blank" rel="noopener noreferrer">Website</a> •
 <a href="https://docs.tandoor.dev/install/docker/" target="_blank" rel="noopener noreferrer">Installation</a> •
-<a href="https://docs.tandoor.dev/" target="_blank" rel="noopener noreferrer">Documentation</a> •
+<a href="https://docs.tandoor.dev/" target="_blank" rel="noopener noreferrer">Docs</a> •
 <a href="https://app.tandoor.dev/accounts/login/?demo" target="_blank" rel="noopener noreferrer">Demo</a> •
-<a href="https://discord.gg/RhzBrfWgtp" target="_blank" rel="noopener noreferrer">Discord server</a>
+<a href="https://discord.gg/RhzBrfWgtp" target="_blank" rel="noopener noreferrer">Discord</a>
 </p>
 
 ![Preview](docs/preview.png)
 
-# Your Feedback
+## Core Features
+- 🥗 **Manage your recipes** - Manage your ever growing recipe collection
+- 📆 **Plan** - multiple meals for each day
+- 🛒 **Shopping lists** - via the meal plan or straight from recipes
+- 📚 **Cookbooks** - collect recipes into books
+- 👪 **Share and collaborate** on recipes with friends and family
+
+## Made by and for power users
+
+- 🔍 Powerful & customizable **search** with fulltext support and [TrigramSimilarity](https://docs.djangoproject.com/en/3.0/ref/contrib/postgres/search/#trigram-similarity)
+- 🏷️ Create and search for **tags**, assign them in batch to all files matching certain filters
+- ↔️ Quickly merge and rename ingredients, tags and units
+- 📥️ **Import recipes** from thousands of websites supporting [ld+json or microdata](https://schema.org/Recipe)
+- ➗ Support for **fractions** or decimals
+- 🐳 Easy setup with **Docker** and included examples for **Kubernetes**, **Unraid** and **Synology**
+- 🎨 Customize your interface with **themes**
+- 📦 **Sync** files with Dropbox and Nextcloud
+
+## All the must haves
+
+- 📱Optimized for use on **mobile** devices
+- 🌍 localized in many languages thanks to the awesome community
+- 📥️ **Import your collection** from many other [recipe managers](https://docs.tandoor.dev/features/import_export/)
+- ➕ Many more like recipe scaling, image compression, printing views and supermarkets
+
+This application is meant for people with a collection of recipes they want to share with family and friends or simply
+store them in a nicely organized way. A basic permission system exists but this application is not meant to be run as
+a public page.
+
+## Docs
+
+Documentation can be found [here](https://docs.tandoor.dev/).
+
+## Support our work
+Tandoor is developed by volunteers in their free time just because its fun. That said earning
+some money with the project allows us to spend more time on it and thus make improvements we otherwise couldn't.
+Because of that there are several ways you can support us
+
+- **GitHub Sponsors** You can sponsor contributors of this project on GitHub: [vabene1111](https://github.com/sponsors/vabene1111)
+- **Host at Hetzner** We have been very happy customers of Hetzner for multiple years for all of our projects. If you want to get into self-hosting or are tired of the expensive big providers, their cloud servers are a great place to get started. When you sign up via our [referral link](https://hetzner.cloud/?ref=ISdlrLmr9kGj) you will get 20€ worth of cloud credits and we get a small kickback too.
+- **Let us host for you** We are offering a [hosted version](https://app.tandoor.dev) where all profits support us and the development of tandoor (currently only available in germany).
+
+## Contributing
+Contributions are welcome but please read [this](https://docs.tandoor.dev/contribute/guidelines/) **BEFORE** contributing anything!
+
+## Your Feedback
 
 Share some information on how you use Tandoor to help me improve the application [Google Survey](https://forms.gle/qNfLK2tWTeWHe9Qd7)
 
-## Features
+## Get in touch
 
-- 📦 **Sync** files with Dropbox and Nextcloud (more can easily be added)
-- 🔍 Powerful **search** with Djangos [TrigramSimilarity](https://docs.djangoproject.com/en/3.0/ref/contrib/postgres/search/#trigram-similarity)
-- 🏷️ Create and search for **tags**, assign them in batch to all files matching certain filters
-- 📄 **Create recipes** locally within a nice, standardized web interface
-- ⬇️ **Import recipes** from thousands of websites supporting [ld+json or microdata](https://schema.org/Recipe)
-- 📱 Optimized for use on **mobile** devices like phones and tablets
-- 🛒 Generate **shopping** lists from recipes
-- 📆 Create a **Plan** on what to eat when
-- 👪 **Share** recipes with friends and comment on them to suggest or remember changes you made
-- ➗ automatically convert decimal units to **fractions** for those who like this
-- 🐳 Easy setup with **Docker** and included examples for Kubernetes, Unraid and Synology
-- 🎨 Customize your interface with **themes**
-- ✉️ Export and import recipes from other users
-- 🌍 localized in many languages thanks to the awesome community
-- ➕ Many more like recipe scaling, image compression, cookbooks, printing views, ...
+<table>
+  <tr>
+    <td><a href="https://discord.gg/RhzBrfWgtp">Discord</a></td>
+    <td>We have a public Discord server that anyone can join. This is where all our developers and contributors hang out and where we make announcements</td>
+  </tr>
 
-This application is meant for people with a collection of recipes they want to share with family and friends or simply
-store them in a nicely organized way. A basic permission system exists but this application is not meant to be run as 
-a public page.
-Documentation can be found [here](https://docs.tandoor.dev/).
+  <tr>
+    <td><a href="https://twitter.com/TandoorRecipes">Twitter</a></td>
+    <td>You can follow our Twitter account to get updates on new features or releases</td>
+  </tr>
+</table>
 
-While this application has been around for a while and is actively used by many (including myself), it is still considered
-**beta** software that has a lot of rough edges and unpolished parts.
 ## License
 
-Beginning with version 0.10.0 the code in this repository is licensed under the [GNU AGPL v3](https://www.gnu.org/licenses/agpl-3.0.de.html) license with an
+Beginning with version 0.10.0 the code in this repository is licensed under the [GNU AGPL v3](https://www.gnu.org/licenses/agpl-3.0.de.html) license with a
 [common clause](https://commonsclause.com/) selling exception. See [LICENSE.md](https://github.com/vabene1111/recipes/blob/develop/LICENSE.md) for details.
 
-> NOTE: There appears to be a whole range of legal issues with licensing anything else then the standard completely open licenses.
+> NOTE: There appears to be a whole range of legal issues with licensing anything other than the standard completely open licenses.
 > I am in the process of getting some professional legal advice to sort out these issues. 
 > Please also see [Issue 238](https://github.com/vabene1111/recipes/issues/238) for some discussion and **reasoning** regarding the topic.
 
-**Reasoning**  
+**Reasoning**
 **This software and *all* its features are and will always be free for everyone to use and enjoy.**
 
 The reason for the selling exception is that a significant amount of time was spend over multiple years to develop this software.
-A payed hosted version which will be identical in features and code base to the software offered in this repository will
+A paid hosted version which will be identical in features and code base to the software offered in this repository will
 likely be released in the future (including all features needed to sell a hosted version as they might also be useful for personal use).
-This will not only benefit me personally but also everyone who self-hosts this software as any profits made trough selling the hosted option
+This will not only benefit me personally but also everyone who self-hosts this software as any profits made through selling the hosted option
 allow me to spend more time developing and improving the software for everyone. Selling exceptions are [approved by Richard Stallman](http://www.gnu.org/philosophy/selling-exceptions.en.html) and the
 common clause license is very permissive (see the [FAQ](https://commonsclause.com/)).

SECURITY.md

@@ -6,5 +6,4 @@ Since this software is still considered beta/WIP support is always only given fo
 
 ## Reporting a Vulnerability
 
-Please open a normal public issue if you have any security related concerns. If you feel like the issue should not be discussed in 
-public just open a generic issue and we will discuss further communitcation there (since GitHub does not allow everyone to create a security advisory :/).
+Please use GitHub Security Advisories to report any kind of security vulnerabilities.

boot.sh

@@ -1,12 +1,86 @@
 #!/bin/sh
 source venv/bin/activate
 
-echo "Updating database"
+TANDOOR_PORT="${TANDOOR_PORT:-8080}"
+GUNICORN_WORKERS="${GUNICORN_WORKERS:-3}"
+GUNICORN_THREADS="${GUNICORN_THREADS:-2}"
+GUNICORN_LOG_LEVEL="${GUNICORN_LOG_LEVEL:-'info'}"
+NGINX_CONF_FILE=/opt/recipes/nginx/conf.d/Recipes.conf
+
+display_warning() {
+    echo "[WARNING]"
+    echo -e "$1"
+}
+
+echo "Checking configuration..."
+
+# Nginx config file must exist if gunicorn is not active
+if [ ! -f "$NGINX_CONF_FILE" ] && [ $GUNICORN_MEDIA -eq 0 ]; then
+    display_warning "Nginx configuration file could not be found at the default location!\nPath: ${NGINX_CONF_FILE}"
+fi
+
+# SECRET_KEY (or a valid file at SECRET_KEY_FILE) must be set in .env file
+
+if [ -f "${SECRET_KEY_FILE}" ]; then
+    export SECRET_KEY=$(cat "$SECRET_KEY_FILE")
+fi
+
+if [ -z "${SECRET_KEY}" ]; then
+    display_warning "The environment variable 'SECRET_KEY' (or 'SECRET_KEY_FILE' that points to an existing file) is not set but REQUIRED for running Tandoor!"
+fi
+
+
+echo "Waiting for database to be ready..."
+
+attempt=0
+max_attempts=20
+
+if [ "${DB_ENGINE}" == 'django.db.backends.postgresql' ] || [ "${DATABASE_URL}" == 'postgres'* ]; then
+
+  # POSTGRES_PASSWORD (or a valid file at POSTGRES_PASSWORD_FILE) must be set in .env file
+
+  if [ -f "${POSTGRES_PASSWORD_FILE}" ]; then
+    export POSTGRES_PASSWORD=$(cat "$POSTGRES_PASSWORD_FILE")
+  fi
+
+  if [ -z "${POSTGRES_PASSWORD}" ]; then
+      display_warning "The environment variable 'POSTGRES_PASSWORD' (or 'POSTGRES_PASSWORD_FILE' that points to an existing file) is not set but REQUIRED for running Tandoor!"
+  fi
+
+  while pg_isready --host=${POSTGRES_HOST} --port=${POSTGRES_PORT} --user=${POSTGRES_USER} -q; status=$?; attempt=$((attempt+1)); [ $status -ne 0 ] && [ $attempt -le $max_attempts ]; do
+      sleep 5
+  done
+fi
+
+if [ $attempt -gt $max_attempts ]; then
+    echo -e "\nDatabase not reachable. Maximum attempts exceeded."
+    echo "Please check logs above - misconfiguration is very likely."
+    echo "Make sure the DB container is up and POSTGRES_HOST is set properly."
+    echo "Shutting down container."
+    exit 1 # exit with error to make the container stop
+fi
+
+echo "Database is ready"
+
+echo "Migrating database"
+
+
 python manage.py migrate
+
+echo "Collecting static files, this may take a while..."
+
 python manage.py collectstatic_js_reverse
 python manage.py collectstatic --noinput
+
 echo "Done"
 
 chmod -R 755 /opt/recipes/mediafiles
 
-exec gunicorn -b :8080 --access-logfile - --error-logfile - --log-level INFO recipes.wsgi
+ipv6_disable=$(cat /sys/module/ipv6/parameters/disable)
+
+# Check if IPv6 is enabled, only then run gunicorn with ipv6 support
+if [ "$ipv6_disable" -eq 0 ]; then
+    exec gunicorn -b "[::]:$TANDOOR_PORT" --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --access-logfile - --error-logfile - --log-level $GUNICORN_LOG_LEVEL recipes.wsgi
+else
+    exec gunicorn -b ":$TANDOOR_PORT" --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --access-logfile - --error-logfile - --log-level $GUNICORN_LOG_LEVEL recipes.wsgi
+fi

cookbook/admin.py

@@ -1,23 +1,23 @@
 from django.conf import settings
 from django.contrib import admin
+from django.contrib.auth.admin import UserAdmin
+from django.contrib.auth.models import Group, User
 from django.contrib.postgres.search import SearchVector
+from django.utils import translation
+from django_scopes import scopes_disabled
 from treebeard.admin import TreeAdmin
 from treebeard.forms import movenodeform_factory
-from django.contrib.auth.admin import UserAdmin
-from django.contrib.auth.models import User, Group
-from django_scopes import scopes_disabled
-from django.utils import translation
-
-from .models import (Comment, CookLog, Food, Ingredient, InviteLink, Keyword,
-                     MealPlan, MealType, NutritionInformation, Recipe,
-                     RecipeBook, RecipeBookEntry, RecipeImport, ShareLink,
-                     ShoppingList, ShoppingListEntry, ShoppingListRecipe,
-                     Space, Step, Storage, Sync, SyncLog, Unit, UserPreference,
-                     ViewLog, Supermarket, SupermarketCategory, SupermarketCategoryRelation,
-                     ImportLog, TelegramBot, BookmarkletImport, UserFile, SearchPreference)
 
 from cookbook.managers import DICTIONARY
 
+from .models import (BookmarkletImport, Comment, CookLog, Food, ImportLog, Ingredient, InviteLink,
+                     Keyword, MealPlan, MealType, NutritionInformation, Property, PropertyType,
+                     Recipe, RecipeBook, RecipeBookEntry, RecipeImport, SearchPreference, ShareLink,
+                     ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage,
+                     Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog,
+                     TelegramBot, Unit, UnitConversion, UserFile, UserPreference, UserSpace,
+                     ViewLog, ConnectorConfig)
+
 
 class CustomUserAdmin(UserAdmin):
     def has_add_permission(self, request, obj=None):
@@ -30,25 +30,45 @@ admin.site.register(User, CustomUserAdmin)
 admin.site.unregister(Group)
 
 
+@admin.action(description='Delete all data from a space')
+def delete_space_action(modeladmin, request, queryset):
+    for space in queryset:
+        space.safe_delete()
+
+
 class SpaceAdmin(admin.ModelAdmin):
     list_display = ('name', 'created_by', 'max_recipes', 'max_users', 'max_file_storage_mb', 'allow_sharing')
     search_fields = ('name', 'created_by__username')
+    autocomplete_fields = ('created_by',)
+    filter_horizontal = ('food_inherit',)
     list_filter = ('max_recipes', 'max_users', 'max_file_storage_mb', 'allow_sharing')
     date_hierarchy = 'created_at'
+    actions = [delete_space_action]
 
 
 admin.site.register(Space, SpaceAdmin)
 
 
+class UserSpaceAdmin(admin.ModelAdmin):
+    list_display = ('user', 'space',)
+    search_fields = ('user__username', 'space__name',)
+    filter_horizontal = ('groups',)
+    autocomplete_fields = ('user', 'space',)
+
+
+admin.site.register(UserSpace, UserSpaceAdmin)
+
+
 class UserPreferenceAdmin(admin.ModelAdmin):
-    list_display = ('name', 'space', 'theme', 'nav_color', 'default_page', 'search_style',)  # TODO add new fields
-    search_fields = ('user__username', 'space__name')
-    list_filter = ('theme', 'nav_color', 'default_page', 'search_style')
+    list_display = ('name', 'theme', 'default_page')
+    search_fields = ('user__username',)
+    list_filter = ('theme', 'default_page',)
     date_hierarchy = 'created_at'
+    filter_horizontal = ('plan_share', 'shopping_share',)
 
     @staticmethod
     def name(obj):
-        return obj.user.get_user_name()
+        return obj.user.get_user_display_name()
 
 
 admin.site.register(UserPreference, UserPreferenceAdmin)
@@ -61,7 +81,7 @@ class SearchPreferenceAdmin(admin.ModelAdmin):
 
     @staticmethod
     def name(obj):
-        return obj.user.get_user_name()
+        return obj.user.get_user_display_name()
 
 
 admin.site.register(SearchPreference, SearchPreferenceAdmin)
@@ -75,6 +95,14 @@ class StorageAdmin(admin.ModelAdmin):
 admin.site.register(Storage, StorageAdmin)
 
 
+class ConnectorConfigAdmin(admin.ModelAdmin):
+    list_display = ('id', 'name', 'type', 'enabled', 'url')
+    search_fields = ('name', 'url')
+
+
+admin.site.register(ConnectorConfig, ConnectorConfigAdmin)
+
+
 class SyncAdmin(admin.ModelAdmin):
     list_display = ('storage', 'path', 'active', 'last_checked')
     search_fields = ('storage__name', 'path')
@@ -88,11 +116,16 @@ class SupermarketCategoryInline(admin.TabularInline):
 
 
 class SupermarketAdmin(admin.ModelAdmin):
+    list_display = ('name', 'space',)
     inlines = (SupermarketCategoryInline,)
 
 
+class SupermarketCategoryAdmin(admin.ModelAdmin):
+    list_display = ('name', 'space',)
+
+
 admin.site.register(Supermarket, SupermarketAdmin)
-admin.site.register(SupermarketCategory)
+admin.site.register(SupermarketCategory, SupermarketCategoryAdmin)
 
 
 class SyncLogAdmin(admin.ModelAdmin):
@@ -129,15 +162,31 @@ def sort_tree(modeladmin, request, queryset):
 class KeywordAdmin(TreeAdmin):
     form = movenodeform_factory(Keyword)
     ordering = ('space', 'path',)
+    search_fields = ('name',)
     actions = [sort_tree, enable_tree_sorting, disable_tree_sorting]
 
 
 admin.site.register(Keyword, KeywordAdmin)
 
 
+@admin.action(description='Delete Steps not part of a Recipe.')
+def delete_unattached_steps(modeladmin, request, queryset):
+    with scopes_disabled():
+        Step.objects.filter(recipe=None).delete()
+
+
 class StepAdmin(admin.ModelAdmin):
-    list_display = ('name', 'type', 'order')
-    search_fields = ('name', 'type')
+    list_display = ('recipe_and_name', 'order', 'space')
+    ordering = ('recipe__name', 'name', 'space',)
+    search_fields = ('name', 'recipe__name')
+    actions = [delete_unattached_steps]
+
+    @staticmethod
+    @admin.display(description="Name")
+    def recipe_and_name(obj):
+        if not obj.recipe_set.exists():
+            return "Orphaned Step" + ('' if not obj.name else f': {obj.name}')
+        return f"{obj.recipe_set.first().name}: {obj.name}" if obj.name else obj.recipe_set.first().name
 
 
 admin.site.register(Step, StepAdmin)
@@ -155,49 +204,82 @@ def rebuild_index(modeladmin, request, queryset):
 
 
 class RecipeAdmin(admin.ModelAdmin):
-    list_display = ('name', 'internal', 'created_by', 'storage')
+    list_display = ('name', 'internal', 'created_by', 'storage', 'space')
     search_fields = ('name', 'created_by__username')
+    ordering = ('name', 'created_by__username',)
     list_filter = ('internal',)
     date_hierarchy = 'created_at'
 
     @staticmethod
     def created_by(obj):
-        return obj.created_by.get_user_name()
+        return obj.created_by.get_user_display_name()
 
-    if settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2', 'django.db.backends.postgresql']:
+    if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql':
         actions = [rebuild_index]
 
 
 admin.site.register(Recipe, RecipeAdmin)
 
-admin.site.register(Unit)
+
+class UnitAdmin(admin.ModelAdmin):
+    list_display = ('name', 'space')
+    ordering = ('name', 'space',)
+    search_fields = ('name',)
+
+
+admin.site.register(Unit, UnitAdmin)
+
+
+# admin.site.register(FoodInheritField)
 
 
 class FoodAdmin(TreeAdmin):
     form = movenodeform_factory(Keyword)
     ordering = ('space', 'path',)
+    search_fields = ('name',)
     actions = [sort_tree, enable_tree_sorting, disable_tree_sorting]
 
 
 admin.site.register(Food, FoodAdmin)
 
 
-class IngredientAdmin(admin.ModelAdmin):
-    list_display = ('food', 'amount', 'unit')
+class UnitConversionAdmin(admin.ModelAdmin):
+    list_display = ('base_amount', 'base_unit', 'food', 'converted_amount', 'converted_unit')
     search_fields = ('food__name', 'unit__name')
 
 
+admin.site.register(UnitConversion, UnitConversionAdmin)
+
+
+@admin.action(description='Delete Ingredients not part of a Recipe.')
+def delete_unattached_ingredients(modeladmin, request, queryset):
+    with scopes_disabled():
+        Ingredient.objects.filter(step__recipe=None).delete()
+
+
+class IngredientAdmin(admin.ModelAdmin):
+    list_display = ('recipe_name', 'amount', 'unit', 'food', 'space')
+    search_fields = ('food__name', 'unit__name', 'step__recipe__name')
+    actions = [delete_unattached_ingredients]
+
+    @staticmethod
+    @admin.display(description="Recipe")
+    def recipe_name(obj):
+        recipes = obj.step_set.first().recipe_set.all() if obj.step_set.exists() else None
+        return recipes.first().name if recipes else 'Orphaned Ingredient'
+
+
 admin.site.register(Ingredient, IngredientAdmin)
 
 
 class CommentAdmin(admin.ModelAdmin):
     list_display = ('recipe', 'name', 'created_at')
-    search_fields = ('text', 'user__username')
+    search_fields = ('text', 'created_by__username')
     date_hierarchy = 'created_at'
 
     @staticmethod
     def name(obj):
-        return obj.created_by.get_user_name()
+        return obj.created_by.get_user_display_name()
 
 
 admin.site.register(Comment, CommentAdmin)
@@ -211,12 +293,12 @@ admin.site.register(RecipeImport, RecipeImportAdmin)
 
 
 class RecipeBookAdmin(admin.ModelAdmin):
-    list_display = ('name', 'user_name')
+    list_display = ('name', 'user_name', 'space')
     search_fields = ('name', 'created_by__username')
 
     @staticmethod
     def user_name(obj):
-        return obj.created_by.get_user_name()
+        return obj.created_by.get_user_display_name()
 
 
 admin.site.register(RecipeBook, RecipeBookAdmin)
@@ -230,19 +312,19 @@ admin.site.register(RecipeBookEntry, RecipeBookEntryAdmin)
 
 
 class MealPlanAdmin(admin.ModelAdmin):
-    list_display = ('user', 'recipe', 'meal_type', 'date')
+    list_display = ('user', 'recipe', 'meal_type', 'from_date', 'to_date')
 
     @staticmethod
     def user(obj):
-        return obj.created_by.get_user_name()
+        return obj.created_by.get_user_display_name()
 
 
 admin.site.register(MealPlan, MealPlanAdmin)
 
 
 class MealTypeAdmin(admin.ModelAdmin):
-    list_display = ('name', 'created_by', 'order')
-    search_fields = ('name', 'created_by__username')
+    list_display = ('name', 'space', 'created_by', 'order')
+    search_fields = ('name', 'space', 'created_by__username')
 
 
 admin.site.register(MealType, MealTypeAdmin)
@@ -257,7 +339,7 @@ admin.site.register(ViewLog, ViewLogAdmin)
 
 class InviteLinkAdmin(admin.ModelAdmin):
     list_display = (
-        'group', 'valid_until',
+        'group', 'valid_until', 'space',
         'created_by', 'created_at', 'used_by'
     )
 
@@ -267,6 +349,7 @@ admin.site.register(InviteLink, InviteLinkAdmin)
 
 class CookLogAdmin(admin.ModelAdmin):
     list_display = ('recipe', 'created_by', 'created_at', 'rating', 'servings')
+    search_fields = ('recipe__name', 'space__name',)
 
 
 admin.site.register(CookLog, CookLogAdmin)
@@ -280,19 +363,12 @@ admin.site.register(ShoppingListRecipe, ShoppingListRecipeAdmin)
 
 
 class ShoppingListEntryAdmin(admin.ModelAdmin):
-    list_display = ('id', 'food', 'unit', 'list_recipe', 'checked')
+    list_display = ('id', 'food', 'unit', 'list_recipe', 'created_by', 'created_at', 'checked')
 
 
 admin.site.register(ShoppingListEntry, ShoppingListEntryAdmin)
 
 
-class ShoppingListAdmin(admin.ModelAdmin):
-    list_display = ('id', 'created_by', 'created_at')
-
-
-admin.site.register(ShoppingList, ShoppingListAdmin)
-
-
 class ShareLinkAdmin(admin.ModelAdmin):
     list_display = ('recipe', 'created_by', 'uuid', 'created_at',)
 
@@ -300,6 +376,29 @@ class ShareLinkAdmin(admin.ModelAdmin):
 admin.site.register(ShareLink, ShareLinkAdmin)
 
 
+@admin.action(description='Delete all properties with type')
+def delete_properties_with_type(modeladmin, request, queryset):
+    for pt in queryset:
+        Property.objects.filter(property_type=pt).delete()
+
+
+class PropertyTypeAdmin(admin.ModelAdmin):
+    search_fields = ('name',)
+
+    list_display = ('id', 'space', 'name', 'fdc_id')
+    actions = [delete_properties_with_type]
+
+
+admin.site.register(PropertyType, PropertyTypeAdmin)
+
+
+class PropertyAdmin(admin.ModelAdmin):
+    list_display = ('property_amount', 'property_type')
+
+
+admin.site.register(Property, PropertyAdmin)
+
+
 class NutritionInformationAdmin(admin.ModelAdmin):
     list_display = ('id',)
 

cookbook/apps.py

@@ -1,26 +1,48 @@
+import traceback
+
 from django.apps import AppConfig
 from django.conf import settings
 from django.db import OperationalError, ProgrammingError
+from django.db.models.signals import post_save, post_delete
 from django_scopes import scopes_disabled
 
+from recipes.settings import DEBUG
+
 
 class CookbookConfig(AppConfig):
     name = 'cookbook'
 
     def ready(self):
-        # post_save signal is only necessary if using full-text search on postgres
-        if settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2', 'django.db.backends.postgresql']:
-            import cookbook.signals  # noqa
+        import cookbook.signals  # noqa
 
-        # when starting up run fix_tree to:
-        #   a) make sure that nodes are sorted when switching between sort modes
-        #   b) fix problems, if any, with tree consistency
-        with scopes_disabled():
+        if not settings.DISABLE_EXTERNAL_CONNECTORS:
             try:
-                from cookbook.models import Keyword, Food
-                Keyword.fix_tree(fix_paths=True)
-                Food.fix_tree(fix_paths=True)
-            except OperationalError:
-                pass  # if model does not exist there is no need to fix it
-            except ProgrammingError:
-                pass  # if migration has not been run database cannot be fixed yet
+                from cookbook.connectors.connector_manager import ConnectorManager  # Needs to be here to prevent loading race condition of oauth2 modules in models.py
+                handler = ConnectorManager()
+                post_save.connect(handler, dispatch_uid="connector_manager")
+                post_delete.connect(handler, dispatch_uid="connector_manager")
+            except Exception as e:
+                traceback.print_exc()
+                print('Failed to initialize connectors')
+                pass
+        # if not settings.DISABLE_TREE_FIX_STARTUP:
+        #     # when starting up run fix_tree to:
+        #     #   a) make sure that nodes are sorted when switching between sort modes
+        #     #   b) fix problems, if any, with tree consistency
+        #     with scopes_disabled():
+        #         try:
+        #             from cookbook.models import Food, Keyword
+        #             Keyword.fix_tree(fix_paths=True)
+        #             Food.fix_tree(fix_paths=True)
+        #         except OperationalError:
+        #             if DEBUG:
+        #                 traceback.print_exc()
+        #             pass  # if model does not exist there is no need to fix it
+        #         except ProgrammingError:
+        #             if DEBUG:
+        #                 traceback.print_exc()
+        #             pass  # if migration has not been run database cannot be fixed yet
+        #         except Exception:
+        #             if DEBUG:
+        #                 traceback.print_exc()
+        #             pass  # dont break startup just because fix could not run, need to investigate cases when this happens

cookbook/connectors/connector.py

@@ -0,0 +1,29 @@
+from abc import ABC, abstractmethod
+
+from cookbook.models import ShoppingListEntry, Space, ConnectorConfig
+
+
+# A Connector is 'destroyed' & recreated each time 'any' ConnectorConfig in a space changes.
+class Connector(ABC):
+    @abstractmethod
+    def __init__(self, config: ConnectorConfig):
+        pass
+
+    @abstractmethod
+    async def on_shopping_list_entry_created(self, space: Space, instance: ShoppingListEntry) -> None:
+        pass
+
+    # This method might not trigger on 'direct' entry updates: https://stackoverflow.com/a/35238823
+    @abstractmethod
+    async def on_shopping_list_entry_updated(self, space: Space, instance: ShoppingListEntry) -> None:
+        pass
+
+    @abstractmethod
+    async def on_shopping_list_entry_deleted(self, space: Space, instance: ShoppingListEntry) -> None:
+        pass
+
+    @abstractmethod
+    async def close(self) -> None:
+        pass
+
+    # TODO: Add Recipes & possibly Meal Place listeners/hooks (And maybe more?)

cookbook/connectors/connector_manager.py

@@ -0,0 +1,188 @@
+import asyncio
+import logging
+import queue
+import threading
+from asyncio import Task
+from dataclasses import dataclass
+from enum import Enum
+from logging import Logger
+from types import UnionType
+from typing import List, Any, Dict, Optional, Type
+
+from django.conf import settings
+from django_scopes import scope
+
+from cookbook.connectors.connector import Connector
+from cookbook.connectors.homeassistant import HomeAssistant
+from cookbook.models import ShoppingListEntry, Space, ConnectorConfig
+
+REGISTERED_CLASSES: UnionType | Type = ShoppingListEntry
+
+
+class ActionType(Enum):
+    CREATED = 1
+    UPDATED = 2
+    DELETED = 3
+
+
+@dataclass
+class Work:
+    instance: REGISTERED_CLASSES | ConnectorConfig
+    actionType: ActionType
+
+
+# The way ConnectionManager works is as follows:
+# 1. On init, it starts a worker & creates a queue for 'Work'
+# 2. Then any time its called, it verifies the type of action (create/update/delete) and if the item is of interest, pushes the Work (non-blocking) to the queue.
+# 3. The worker consumes said work from the queue.
+# 3.1 If the work is of type ConnectorConfig, it flushes its cache of known connectors (per space.id)
+# 3.2 If work is of type REGISTERED_CLASSES, it asynchronously fires of all connectors and wait for them to finish (runtime should depend on the 'slowest' connector)
+# 4. Work is marked as consumed, and next entry of the queue is consumed.
+# Each 'Work' is processed in sequential by the worker, so the throughput is about [workers * the slowest connector]
+class ConnectorManager:
+    _logger: Logger
+    _queue: queue.Queue
+    _listening_to_classes = REGISTERED_CLASSES | ConnectorConfig
+
+    def __init__(self):
+        self._logger = logging.getLogger("recipes.connector")
+        self._queue = queue.Queue(maxsize=settings.EXTERNAL_CONNECTORS_QUEUE_SIZE)
+        self._worker = threading.Thread(target=self.worker, args=(0, self._queue,), daemon=True)
+        self._worker.start()
+
+    # Called by post save & post delete signals
+    def __call__(self, instance: Any, **kwargs) -> None:
+        if not isinstance(instance, self._listening_to_classes) or not hasattr(instance, "space"):
+            return
+
+        action_type: ActionType
+        if "created" in kwargs and kwargs["created"]:
+            action_type = ActionType.CREATED
+        elif "created" in kwargs and not kwargs["created"]:
+            action_type = ActionType.UPDATED
+        elif "origin" in kwargs:
+            action_type = ActionType.DELETED
+        else:
+            return
+
+        try:
+            self._queue.put_nowait(Work(instance, action_type))
+        except queue.Full:
+            self._logger.info(f"queue was full, so skipping {action_type} of type {type(instance)}")
+            return
+
+    def stop(self):
+        self._queue.join()
+        self._worker.join()
+
+    @staticmethod
+    def worker(worker_id: int, worker_queue: queue.Queue):
+        logger = logging.getLogger("recipes.connector.worker")
+
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+        logger.info(f"started ConnectionManager worker {worker_id}")
+
+        # When multiple workers are used, please make sure the cache is shared across all threads, otherwise it might lead to un-expected behavior.
+        _connectors_cache: Dict[int, List[Connector]] = dict()
+
+        while True:
+            try:
+                item: Optional[Work] = worker_queue.get()
+            except KeyboardInterrupt:
+                break
+
+            if item is None:
+                break
+
+            logger.debug(f"received {item.instance=} with {item.actionType=}")
+
+            # If a Connector was changed/updated, refresh connector from the database for said space
+            refresh_connector_cache = isinstance(item.instance, ConnectorConfig)
+
+            space: Space = item.instance.space
+            connectors: Optional[List[Connector]] = _connectors_cache.get(space.id)
+
+            if connectors is None or refresh_connector_cache:
+                if connectors is not None:
+                    loop.run_until_complete(close_connectors(connectors))
+
+                with scope(space=space):
+                    connectors: List[Connector] = list()
+                    for config in space.connectorconfig_set.all():
+                        config: ConnectorConfig = config
+                        if not config.enabled:
+                            continue
+
+                        try:
+                            connector: Optional[Connector] = ConnectorManager.get_connected_for_config(config)
+                        except BaseException:
+                            logger.exception(f"failed to initialize {config.name}")
+                            continue
+
+                        if connector is not None:
+                            connectors.append(connector)
+
+                    _connectors_cache[space.id] = connectors
+
+            if len(connectors) == 0 or refresh_connector_cache:
+                worker_queue.task_done()
+                continue
+
+            logger.debug(f"running {len(connectors)} connectors for {item.instance=} with {item.actionType=}")
+
+            loop.run_until_complete(run_connectors(connectors, space, item.instance, item.actionType))
+            worker_queue.task_done()
+
+        logger.info(f"terminating ConnectionManager worker {worker_id}")
+
+        asyncio.set_event_loop(None)
+        loop.close()
+
+    @staticmethod
+    def get_connected_for_config(config: ConnectorConfig) -> Optional[Connector]:
+        match config.type:
+            case ConnectorConfig.HOMEASSISTANT:
+                return HomeAssistant(config)
+            case _:
+                return None
+
+
+async def close_connectors(connectors: List[Connector]):
+    tasks: List[Task] = [asyncio.create_task(connector.close()) for connector in connectors]
+
+    if len(tasks) == 0:
+        return
+
+    try:
+        await asyncio.gather(*tasks, return_exceptions=False)
+    except BaseException:
+        logging.exception("received an exception while closing one of the connectors")
+
+
+async def run_connectors(connectors: List[Connector], space: Space, instance: REGISTERED_CLASSES, action_type: ActionType):
+    tasks: List[Task] = list()
+
+    if isinstance(instance, ShoppingListEntry):
+        shopping_list_entry: ShoppingListEntry = instance
+
+        match action_type:
+            case ActionType.CREATED:
+                for connector in connectors:
+                    tasks.append(asyncio.create_task(connector.on_shopping_list_entry_created(space, shopping_list_entry)))
+            case ActionType.UPDATED:
+                for connector in connectors:
+                    tasks.append(asyncio.create_task(connector.on_shopping_list_entry_updated(space, shopping_list_entry)))
+            case ActionType.DELETED:
+                for connector in connectors:
+                    tasks.append(asyncio.create_task(connector.on_shopping_list_entry_deleted(space, shopping_list_entry)))
+
+    if len(tasks) == 0:
+        return
+
+    try:
+        # Wait for all async tasks to finish, if one fails, the others still continue.
+        await asyncio.gather(*tasks, return_exceptions=False)
+    except BaseException:
+        logging.exception("received an exception from one of the connectors")

cookbook/connectors/homeassistant.py

@@ -0,0 +1,104 @@
+import logging
+from logging import Logger
+from typing import Dict, Tuple
+from urllib.parse import urljoin
+
+from aiohttp import ClientError, request
+
+from cookbook.connectors.connector import Connector
+from cookbook.models import ShoppingListEntry, ConnectorConfig, Space
+
+
+class HomeAssistant(Connector):
+    _config: ConnectorConfig
+    _logger: Logger
+
+    def __init__(self, config: ConnectorConfig):
+        if not config.token or not config.url or not config.todo_entity:
+            raise ValueError("config for HomeAssistantConnector in incomplete")
+
+        self._logger = logging.getLogger(f"recipes.connector.homeassistant.{config.name}")
+
+        if config.url[-1] != "/":
+            config.url += "/"
+        self._config = config
+
+    async def homeassistant_api_call(self, method: str, path: str, data: Dict) -> str:
+        headers = {
+            "Authorization": f"Bearer {self._config.token}",
+            "Content-Type": "application/json"
+        }
+        async with request(method, urljoin(self._config.url, path), headers=headers, json=data) as response:
+            response.raise_for_status()
+            return await response.json()
+
+    async def on_shopping_list_entry_created(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None:
+        if not self._config.on_shopping_list_entry_created_enabled:
+            return
+
+        item, description = _format_shopping_list_entry(shopping_list_entry)
+
+        self._logger.debug(f"adding {item=}")
+
+        data = {
+            "entity_id": self._config.todo_entity,
+            "item": item,
+            "description": description,
+        }
+
+        try:
+            await self.homeassistant_api_call("POST", "services/todo/add_item", data)
+        except ClientError as err:
+            self._logger.warning(f"received an exception from the api: {err=}, {type(err)=}")
+
+    async def on_shopping_list_entry_updated(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None:
+        if not self._config.on_shopping_list_entry_updated_enabled:
+            return
+        pass
+
+    async def on_shopping_list_entry_deleted(self, space: Space, shopping_list_entry: ShoppingListEntry) -> None:
+        if not self._config.on_shopping_list_entry_deleted_enabled:
+            return
+
+        if not hasattr(shopping_list_entry._state.fields_cache, "food"):
+            # Sometimes the food foreign key is not loaded, and we cant load it from an async process
+            self._logger.debug("required property was not present in ShoppingListEntry")
+            return
+
+        item, _ = _format_shopping_list_entry(shopping_list_entry)
+
+        self._logger.debug(f"removing {item=}")
+
+        data = {
+            "entity_id": self._config.todo_entity,
+            "item": item,
+        }
+
+        try:
+            await self.homeassistant_api_call("POST", "services/todo/remove_item", data)
+        except ClientError as err:
+            # This error will always trigger if the item is not present/found
+            self._logger.debug(f"received an exception from the api: {err=}, {type(err)=}")
+
+    async def close(self) -> None:
+        pass
+
+
+def _format_shopping_list_entry(shopping_list_entry: ShoppingListEntry) -> Tuple[str, str]:
+    item = shopping_list_entry.food.name
+    if shopping_list_entry.amount > 0:
+        item += f" ({shopping_list_entry.amount:.2f}".rstrip('0').rstrip('.')
+        if shopping_list_entry.unit and shopping_list_entry.unit.base_unit and len(shopping_list_entry.unit.base_unit) > 0:
+            item += f" {shopping_list_entry.unit.base_unit})"
+        elif shopping_list_entry.unit and shopping_list_entry.unit.name and len(shopping_list_entry.unit.name) > 0:
+            item += f" {shopping_list_entry.unit.name})"
+        else:
+            item += ")"
+
+    description = "From TandoorRecipes"
+    if shopping_list_entry.created_by.first_name and len(shopping_list_entry.created_by.first_name) > 0:
+        description += f", by {shopping_list_entry.created_by.first_name}"
+    else:
+        description += f", by {shopping_list_entry.created_by.username}"
+
+    return item, description

cookbook/filters.py

@@ -1,81 +0,0 @@
-import django_filters
-from django.conf import settings
-from django.contrib.postgres.search import TrigramSimilarity
-from django.db.models import Q
-from django.utils.translation import gettext as _
-from django_scopes import scopes_disabled
-
-from cookbook.forms import MultiSelectWidget
-from cookbook.models import Food, Keyword, Recipe, ShoppingList
-
-with scopes_disabled():
-    class RecipeFilter(django_filters.FilterSet):
-        name = django_filters.CharFilter(method='filter_name')
-        keywords = django_filters.ModelMultipleChoiceFilter(
-            queryset=Keyword.objects.none(),
-            widget=MultiSelectWidget,
-            method='filter_keywords'
-        )
-        foods = django_filters.ModelMultipleChoiceFilter(
-            queryset=Food.objects.none(),
-            widget=MultiSelectWidget,
-            method='filter_foods',
-            label=_('Ingredients')
-        )
-
-        def __init__(self, data=None, *args, **kwargs):
-            space = kwargs.pop('space')
-            super().__init__(data, *args, **kwargs)
-            self.filters['foods'].queryset = Food.objects.filter(space=space).all()
-            self.filters['keywords'].queryset = Keyword.objects.filter(space=space).all()
-
-        @staticmethod
-        def filter_keywords(queryset, name, value):
-            if not name == 'keywords':
-                return queryset
-            for x in value:
-                queryset = queryset.filter(keywords=x)
-            return queryset
-
-        @staticmethod
-        def filter_foods(queryset, name, value):
-            if not name == 'foods':
-                return queryset
-            for x in value:
-                queryset = queryset.filter(steps__ingredients__food__name=x).distinct()
-            return queryset
-
-        @staticmethod
-        def filter_name(queryset, name, value):
-            if not name == 'name':
-                return queryset
-            if settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2',
-                                                           'django.db.backends.postgresql']:
-                queryset = queryset.annotate(similarity=TrigramSimilarity('name', value), ).filter(
-                    Q(similarity__gt=0.1) | Q(name__unaccent__icontains=value)).order_by('-similarity')
-            else:
-                queryset = queryset.filter(name__icontains=value)
-            return queryset
-
-        class Meta:
-            model = Recipe
-            fields = ['name', 'keywords', 'foods', 'internal']
-
-    # class FoodFilter(django_filters.FilterSet):
-    #     name = django_filters.CharFilter(lookup_expr='icontains')
-
-    #     class Meta:
-    #         model = Food
-    #         fields = ['name']
-
-    class ShoppingListFilter(django_filters.FilterSet):
-
-        def __init__(self, data=None, *args, **kwargs):
-            if data is not None:
-                data = data.copy()
-                data.setdefault("finished", False)
-            super().__init__(data, *args, **kwargs)
-
-        class Meta:
-            model = ShoppingList
-            fields = ['finished']

cookbook/forms.py

@@ -1,26 +1,28 @@
+from datetime import datetime
+
+from allauth.account.forms import ResetPasswordForm, SignupForm
 from django import forms
 from django.conf import settings
 from django.core.exceptions import ValidationError
-from django.forms import widgets, NumberInput
+from django.forms import NumberInput, widgets
 from django.utils.translation import gettext_lazy as _
 from django_scopes import scopes_disabled
 from django_scopes.forms import SafeModelChoiceField, SafeModelMultipleChoiceField
 from hcaptcha.fields import hCaptchaField
 
-from .models import (Comment, InviteLink, Keyword, MealPlan, Recipe,
-                     RecipeBook, RecipeBookEntry, Storage, Sync, User,
-                     UserPreference, MealType, Space,
-                     SearchPreference)
+from .models import Comment, InviteLink, Keyword, Recipe, SearchPreference, Space, Storage, Sync, User, UserPreference, ConnectorConfig
 
 
 class SelectWidget(widgets.Select):
+
     class Media:
-        js = ('custom/js/form_select.js',)
+        js = ('custom/js/form_select.js', )
 
 
 class MultiSelectWidget(widgets.SelectMultiple):
+
     class Media:
-        js = ('custom/js/form_multiselect.js',)
+        js = ('custom/js/form_multiselect.js', )
 
 
 # Yes there are some stupid browsers that still dont support this but
@@ -33,43 +35,6 @@ class DateWidget(forms.DateInput):
         super().__init__(**kwargs)
 
 
-class UserPreferenceForm(forms.ModelForm):
-    prefix = 'preference'
-
-    class Meta:
-        model = UserPreference
-        fields = (
-            'default_unit', 'use_fractions', 'theme', 'nav_color',
-            'sticky_navbar', 'default_page', 'show_recent', 'search_style',
-            'plan_share', 'ingredient_decimals', 'shopping_auto_sync',
-            'comments'
-        )
-
-        help_texts = {
-            'nav_color': _('Color of the top navigation bar. Not all colors work with all themes, just try them out!'),
-            # noqa: E501
-            'default_unit': _('Default Unit to be used when inserting a new ingredient into a recipe.'),  # noqa: E501
-            'use_fractions': _(
-                'Enables support for fractions in ingredient amounts (e.g. convert decimals to fractions automatically)'),
-            # noqa: E501
-            'plan_share': _(
-                'Users with whom newly created meal plan/shopping list entries should be shared by default.'),
-            # noqa: E501
-            'show_recent': _('Show recently viewed recipes on search page.'),  # noqa: E501
-            'ingredient_decimals': _('Number of decimals to round ingredients.'),  # noqa: E501
-            'comments': _('If you want to be able to create and see comments underneath recipes.'),  # noqa: E501
-            'shopping_auto_sync': _(
-                'Setting to 0 will disable auto sync. When viewing a shopping list the list is updated every set seconds to sync changes someone else might have made. Useful when shopping with multiple people but might use a little bit '  # noqa: E501
-                'of mobile data. If lower than instance limit it is reset when saving.'  # noqa: E501
-            ),
-            'sticky_navbar': _('Makes the navbar stick to the top of the page.')  # noqa: E501
-        }
-
-        widgets = {
-            'plan_share': MultiSelectWidget
-        }
-
-
 class UserNameForm(forms.ModelForm):
     prefix = 'name'
 
@@ -77,9 +42,7 @@ class UserNameForm(forms.ModelForm):
         model = User
         fields = ('first_name', 'last_name')
 
-        help_texts = {
-            'first_name': _('Both fields are optional. If none are given the username will be displayed instead')
-        }
+        help_texts = {'first_name': _('Both fields are optional. If none are given the username will be displayed instead')}
 
 
 class ExternalRecipeForm(forms.ModelForm):
@@ -93,23 +56,14 @@ class ExternalRecipeForm(forms.ModelForm):
 
     class Meta:
         model = Recipe
-        fields = (
-            'name', 'description', 'servings', 'working_time', 'waiting_time',
-            'file_path', 'file_uid', 'keywords'
-        )
+        fields = ('name', 'description', 'servings', 'working_time', 'waiting_time', 'file_path', 'file_uid', 'keywords')
 
         labels = {
-            'name': _('Name'),
-            'keywords': _('Keywords'),
-            'working_time': _('Preparation time in minutes'),
-            'waiting_time': _('Waiting time (cooking/baking) in minutes'),
-            'file_path': _('Path'),
-            'file_uid': _('Storage UID'),
+            'name': _('Name'), 'keywords': _('Keywords'), 'working_time': _('Preparation time in minutes'), 'waiting_time': _('Waiting time (cooking/baking) in minutes'),
+            'file_path': _('Path'), 'file_uid': _('Storage UID'),
         }
         widgets = {'keywords': MultiSelectWidget}
-        field_classes = {
-            'keywords': SafeModelMultipleChoiceField,
-        }
+        field_classes = {'keywords': SafeModelMultipleChoiceField, }
 
 
 class ImportExportBase(forms.Form):
@@ -118,7 +72,7 @@ class ImportExportBase(forms.Form):
     NEXTCLOUD = 'NEXTCLOUD'
     MEALIE = 'MEALIE'
     CHOWDOWN = 'CHOWDOWN'
-    SAFRON = 'SAFRON'
+    SAFFRON = 'SAFFRON'
     CHEFTAP = 'CHEFTAP'
     PEPPERPLATE = 'PEPPERPLATE'
     RECIPEKEEPER = 'RECIPEKEEPER'
@@ -126,30 +80,52 @@ class ImportExportBase(forms.Form):
     RECIPESAGE = 'RECIPESAGE'
     DOMESTICA = 'DOMESTICA'
     MEALMASTER = 'MEALMASTER'
+    MELARECIPES = 'MELARECIPES'
     REZKONV = 'REZKONV'
     OPENEATS = 'OPENEATS'
     PLANTOEAT = 'PLANTOEAT'
     COOKBOOKAPP = 'COOKBOOKAPP'
+    COPYMETHAT = 'COPYMETHAT'
+    COOKMATE = 'COOKMATE'
+    REZEPTSUITEDE = 'REZEPTSUITEDE'
+    PDF = 'PDF'
 
-    type = forms.ChoiceField(choices=(
-        (DEFAULT, _('Default')), (PAPRIKA, 'Paprika'), (NEXTCLOUD, 'Nextcloud Cookbook'),
-        (MEALIE, 'Mealie'), (CHOWDOWN, 'Chowdown'), (SAFRON, 'Safron'), (CHEFTAP, 'ChefTap'),
-        (PEPPERPLATE, 'Pepperplate'), (RECETTETEK, 'RecetteTek'), (RECIPESAGE, 'Recipe Sage'), (DOMESTICA, 'Domestica'),
-        (MEALMASTER, 'MealMaster'), (REZKONV, 'RezKonv'), (OPENEATS, 'Openeats'), (RECIPEKEEPER, 'Recipe Keeper'),
-        (PLANTOEAT, 'Plantoeat'), (COOKBOOKAPP, 'CookBookApp'),
-    ))
+    type = forms.ChoiceField(choices=((DEFAULT, _('Default')), (PAPRIKA, 'Paprika'), (NEXTCLOUD, 'Nextcloud Cookbook'), (MEALIE, 'Mealie'), (CHOWDOWN, 'Chowdown'),
+                                      (SAFFRON, 'Saffron'), (CHEFTAP, 'ChefTap'), (PEPPERPLATE, 'Pepperplate'), (RECETTETEK, 'RecetteTek'), (RECIPESAGE, 'Recipe Sage'),
+                                      (DOMESTICA, 'Domestica'), (MEALMASTER, 'MealMaster'), (REZKONV, 'RezKonv'), (OPENEATS, 'Openeats'), (RECIPEKEEPER, 'Recipe Keeper'),
+                                      (PLANTOEAT, 'Plantoeat'), (COOKBOOKAPP, 'CookBookApp'), (COPYMETHAT, 'CopyMeThat'), (PDF, 'PDF'), (MELARECIPES, 'Melarecipes'),
+                                      (COOKMATE, 'Cookmate'), (REZEPTSUITEDE, 'Recipesuite.de')))
+
+
+class MultipleFileInput(forms.ClearableFileInput):
+    allow_multiple_selected = True
+
+
+class MultipleFileField(forms.FileField):
+
+    def __init__(self, *args, **kwargs):
+        kwargs.setdefault("widget", MultipleFileInput())
+        super().__init__(*args, **kwargs)
+
+    def clean(self, data, initial=None):
+        single_file_clean = super().clean
+        if isinstance(data, (list, tuple)):
+            result = [single_file_clean(d, initial) for d in data]
+        else:
+            result = single_file_clean(data, initial)
+        return result
 
 
 class ImportForm(ImportExportBase):
-    files = forms.FileField(required=True, widget=forms.ClearableFileInput(attrs={'multiple': True}))
-    duplicates = forms.BooleanField(help_text=_(
-        'To prevent duplicates recipes with the same name as existing ones are ignored. Check this box to import everything.'),
-        required=False)
+    files = MultipleFileField(required=True)
+    duplicates = forms.BooleanField(help_text=_('To prevent duplicates recipes with the same name as existing ones are ignored. Check this box to import everything.'),
+                                    required=False)
 
 
 class ExportForm(ImportExportBase):
-    recipes = forms.ModelMultipleChoiceField(widget=MultiSelectWidget, queryset=Recipe.objects.none())
+    recipes = forms.ModelMultipleChoiceField(widget=MultiSelectWidget, queryset=Recipe.objects.none(), required=False)
     all = forms.BooleanField(required=False)
+    custom_filter = forms.IntegerField(required=False)
 
     def __init__(self, *args, **kwargs):
         space = kwargs.pop('space')
@@ -162,58 +138,71 @@ class CommentForm(forms.ModelForm):
 
     class Meta:
         model = Comment
-        fields = ('text',)
+        fields = ('text', )
 
-        labels = {
-            'text': _('Add your comment: '),
-        }
-        widgets = {
-            'text': forms.Textarea(attrs={'rows': 2, 'cols': 15}),
-        }
+        labels = {'text': _('Add your comment: '), }
+        widgets = {'text': forms.Textarea(attrs={'rows': 2, 'cols': 15}), }
 
 
 class StorageForm(forms.ModelForm):
-    username = forms.CharField(
-        widget=forms.TextInput(attrs={'autocomplete': 'new-password'}),
-        required=False
-    )
-    password = forms.CharField(
-        widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}),
-        required=False,
-        help_text=_('Leave empty for dropbox and enter app password for nextcloud.')
-    )
-    token = forms.CharField(
-        widget=forms.TextInput(
-            attrs={'autocomplete': 'new-password', 'type': 'password'}
-        ),
-        required=False,
-        help_text=_('Leave empty for nextcloud and enter api token for dropbox.')
-    )
+    username = forms.CharField(widget=forms.TextInput(attrs={'autocomplete': 'new-password'}), required=False)
+    password = forms.CharField(widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}),
+                               required=False,
+                               help_text=_('Leave empty for dropbox and enter app password for nextcloud.'))
+    token = forms.CharField(widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}),
+                            required=False,
+                            help_text=_('Leave empty for nextcloud and enter api token for dropbox.'))
 
     class Meta:
         model = Storage
         fields = ('name', 'method', 'username', 'password', 'token', 'url', 'path')
 
-        help_texts = {
-            'url': _(
-                'Leave empty for dropbox and enter only base url for nextcloud (<code>/remote.php/webdav/</code> is added automatically)'),
-        }
+        help_texts = {'url': _('Leave empty for dropbox and enter only base url for nextcloud (<code>/remote.php/webdav/</code> is added automatically)'), }
 
 
-class RecipeBookEntryForm(forms.ModelForm):
-    prefix = 'bookmark'
 
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['book'].queryset = RecipeBook.objects.filter(space=space).all()
+class ConnectorConfigForm(forms.ModelForm):
+    enabled = forms.BooleanField(
+        help_text="Is the connector enabled",
+        required=False,
+    )
+
+    on_shopping_list_entry_created_enabled = forms.BooleanField(
+        help_text="Enable action for ShoppingListEntry created events",
+        required=False,
+    )
+
+    on_shopping_list_entry_updated_enabled = forms.BooleanField(
+        help_text="Enable action for ShoppingListEntry updated events",
+        required=False,
+    )
+
+    on_shopping_list_entry_deleted_enabled = forms.BooleanField(
+        help_text="Enable action for ShoppingListEntry deleted events",
+        required=False,
+    )
+
+    update_token = forms.CharField(
+        widget=forms.TextInput(attrs={'autocomplete': 'update-token', 'type': 'password'}),
+        required=False,
+        help_text=_('<a href="https://www.home-assistant.io/docs/authentication/#your-account-profile">Long Lived Access Token</a> for your HomeAssistant instance')
+    )
+
+    url = forms.URLField(
+        required=False,
+        help_text=_('Something like http://homeassistant.local:8123/api'),
+    )
 
     class Meta:
-        model = RecipeBookEntry
-        fields = ('book',)
+        model = ConnectorConfig
 
-        field_classes = {
-            'book': SafeModelChoiceField,
+        fields = (
+            'name', 'type', 'enabled', 'on_shopping_list_entry_created_enabled', 'on_shopping_list_entry_updated_enabled',
+            'on_shopping_list_entry_deleted_enabled', 'url', 'todo_entity',
+        )
+
+        help_texts = {
+            'url': _('http://homeassistant.local:8123/api for example'),
         }
 
 
@@ -228,18 +217,14 @@ class SyncForm(forms.ModelForm):
         model = Sync
         fields = ('storage', 'path', 'active')
 
-        field_classes = {
-            'storage': SafeModelChoiceField,
-        }
+        field_classes = {'storage': SafeModelChoiceField, }
+
+        labels = {'storage': _('Storage'), 'path': _('Path'), 'active': _('Active')}
 
 
 class BatchEditForm(forms.Form):
     search = forms.CharField(label=_('Search String'))
-    keywords = forms.ModelMultipleChoiceField(
-        queryset=Keyword.objects.none(),
-        required=False,
-        widget=MultiSelectWidget
-    )
+    keywords = forms.ModelMultipleChoiceField(queryset=Keyword.objects.none(), required=False, widget=MultiSelectWidget)
 
     def __init__(self, *args, **kwargs):
         space = kwargs.pop('space')
@@ -248,6 +233,7 @@ class BatchEditForm(forms.Form):
 
 
 class ImportRecipeForm(forms.ModelForm):
+
     def __init__(self, *args, **kwargs):
         space = kwargs.pop('space')
         super().__init__(*args, **kwargs)
@@ -257,62 +243,13 @@ class ImportRecipeForm(forms.ModelForm):
         model = Recipe
         fields = ('name', 'keywords', 'file_path', 'file_uid')
 
-        labels = {
-            'name': _('Name'),
-            'keywords': _('Keywords'),
-            'file_path': _('Path'),
-            'file_uid': _('File ID'),
-        }
+        labels = {'name': _('Name'), 'keywords': _('Keywords'), 'file_path': _('Path'), 'file_uid': _('File ID'), }
         widgets = {'keywords': MultiSelectWidget}
-        field_classes = {
-            'keywords': SafeModelChoiceField,
-        }
-
-
-class MealPlanForm(forms.ModelForm):
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['recipe'].queryset = Recipe.objects.filter(space=space).all()
-        self.fields['meal_type'].queryset = MealType.objects.filter(space=space).all()
-        self.fields['shared'].queryset = User.objects.filter(userpreference__space=space).all()
-
-    def clean(self):
-        cleaned_data = super(MealPlanForm, self).clean()
-
-        if cleaned_data['title'] == '' and cleaned_data['recipe'] is None:
-            raise forms.ValidationError(
-                _('You must provide at least a recipe or a title.')
-            )
-
-        return cleaned_data
-
-    class Meta:
-        model = MealPlan
-        fields = (
-            'recipe', 'title', 'meal_type', 'note',
-            'servings', 'date', 'shared'
-        )
-
-        help_texts = {
-            'shared': _('You can list default users to share recipes with in the settings.'),  # noqa: E501
-            'note': _('You can use markdown to format this field. See the <a href="/docs/markdown/">docs here</a>')
-            # noqa: E501
-        }
-
-        widgets = {
-            'recipe': SelectWidget,
-            'date': DateWidget,
-            'shared': MultiSelectWidget
-        }
-        field_classes = {
-            'recipe': SafeModelChoiceField,
-            'meal_type': SafeModelChoiceField,
-            'shared': SafeModelMultipleChoiceField,
-        }
+        field_classes = {'keywords': SafeModelChoiceField, }
 
 
 class InviteLinkForm(forms.ModelForm):
+
     def __init__(self, *args, **kwargs):
         user = kwargs.pop('user')
         super().__init__(*args, **kwargs)
@@ -320,7 +257,8 @@ class InviteLinkForm(forms.ModelForm):
 
     def clean(self):
         space = self.cleaned_data['space']
-        if space.max_users != 0 and (UserPreference.objects.filter(space=space).count() + InviteLink.objects.filter(space=space).count()) >= space.max_users:
+        if space.max_users != 0 and (UserPreference.objects.filter(space=space).count()
+                                     + InviteLink.objects.filter(valid_until__gte=datetime.today(), used_by=None, space=space).count()) >= space.max_users:
             raise ValidationError(_('Maximum number of users for this space reached.'))
 
     def clean_email(self):
@@ -334,12 +272,8 @@ class InviteLinkForm(forms.ModelForm):
     class Meta:
         model = InviteLink
         fields = ('email', 'group', 'valid_until', 'space')
-        help_texts = {
-            'email': _('An email address is not required but if present the invite link will be send to the user.'),
-        }
-        field_classes = {
-            'space': SafeModelChoiceField,
-        }
+        help_texts = {'email': _('An email address is not required but if present the invite link will be sent to the user.'), }
+        field_classes = {'space': SafeModelChoiceField, }
 
 
 class SpaceCreateForm(forms.Form):
@@ -359,12 +293,12 @@ class SpaceJoinForm(forms.Form):
     token = forms.CharField()
 
 
-class AllAuthSignupForm(forms.Form):
+class AllAuthSignupForm(SignupForm):
     captcha = hCaptchaField()
     terms = forms.BooleanField(label=_('Accept Terms and Privacy'))
 
     def __init__(self, **kwargs):
-        super(AllAuthSignupForm, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         if settings.PRIVACY_URL == '' and settings.TERMS_URL == '':
             self.fields.pop('terms')
         if settings.HCAPTCHA_SECRET == '':
@@ -374,55 +308,50 @@ class AllAuthSignupForm(forms.Form):
         pass
 
 
+class CustomPasswordResetForm(ResetPasswordForm):
+    captcha = hCaptchaField()
+
+    def __init__(self, **kwargs):
+        super(CustomPasswordResetForm, self).__init__(**kwargs)
+        if settings.HCAPTCHA_SECRET == '':
+            self.fields.pop('captcha')
+
+
 class UserCreateForm(forms.Form):
     name = forms.CharField(label='Username')
-    password = forms.CharField(
-        widget=forms.TextInput(
-            attrs={'autocomplete': 'new-password', 'type': 'password'}
-        )
-    )
-    password_confirm = forms.CharField(
-        widget=forms.TextInput(
-            attrs={'autocomplete': 'new-password', 'type': 'password'}
-        )
-    )
+    password = forms.CharField(widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}))
+    password_confirm = forms.CharField(widget=forms.TextInput(attrs={'autocomplete': 'new-password', 'type': 'password'}))
 
 
 class SearchPreferenceForm(forms.ModelForm):
     prefix = 'search'
-    trigram_threshold = forms.DecimalField(min_value=0.01, max_value=1, decimal_places=2, widget=NumberInput(attrs={'class': "form-control-range", 'type': 'range'}),
+    trigram_threshold = forms.DecimalField(min_value=0.01,
+                                           max_value=1,
+                                           decimal_places=2,
+                                           widget=NumberInput(attrs={'class': "form-control-range", 'type': 'range'}),
                                            help_text=_('Determines how fuzzy a search is if it uses trigram similarity matching (e.g. low values mean more typos are ignored).'))
-    preset = forms.CharField(widget=forms.HiddenInput(),required=False)
+    preset = forms.CharField(widget=forms.HiddenInput(), required=False)
 
     class Meta:
         model = SearchPreference
         fields = ('search', 'lookup', 'unaccent', 'icontains', 'istartswith', 'trigram', 'fulltext', 'trigram_threshold')
 
         help_texts = {
-            'search': _('Select type method of search.  Click <a href="/docs/search/">here</a> for full desciption of choices.'),
-            'lookup': _('Use fuzzy matching on units, keywords and ingredients when editing and importing recipes.'),
-            'unaccent': _('Fields to search ignoring accents.  Selecting this option can improve or degrade search quality depending on language'),
-            'icontains': _("Fields to search for partial matches.  (e.g. searching for 'Pie' will return 'pie' and 'piece' and 'soapie')"),
-            'istartswith': _("Fields to search for beginning of word matches. (e.g. searching for 'sa' will return 'salad' and 'sandwich')"),
-            'trigram': _("Fields to 'fuzzy' search. (e.g. searching for 'recpie' will find 'recipe'.)  Note: this option will conflict with 'web' and 'raw' methods of search."),
-            'fulltext': _("Fields to full text search.  Note: 'web', 'phrase', and 'raw' search methods only function with fulltext fields."),
+            'search': _('Select type method of search.  Click <a href="/docs/search/">here</a> for full description of choices.'), 'lookup':
+            _('Use fuzzy matching on units, keywords and ingredients when editing and importing recipes.'), 'unaccent':
+            _('Fields to search ignoring accents.  Selecting this option can improve or degrade search quality depending on language'), 'icontains':
+            _("Fields to search for partial matches.  (e.g. searching for 'Pie' will return 'pie' and 'piece' and 'soapie')"), 'istartswith':
+            _("Fields to search for beginning of word matches. (e.g. searching for 'sa' will return 'salad' and 'sandwich')"), 'trigram':
+            _("Fields to 'fuzzy' search. (e.g. searching for 'recpie' will find 'recipe'.)  Note: this option will conflict with 'web' and 'raw' methods of search."), 'fulltext':
+            _("Fields to full text search.  Note: 'web', 'phrase', and 'raw' search methods only function with fulltext fields."),
         }
 
         labels = {
-            'search': _('Search Method'),
-            'lookup': _('Fuzzy Lookups'),
-            'unaccent': _('Ignore Accent'),
-            'icontains': _("Partial Match"),
-            'istartswith': _("Starts Wtih"),
-            'trigram': _("Fuzzy Search"),
-            'fulltext': _("Full Text")
+            'search': _('Search Method'), 'lookup': _('Fuzzy Lookups'), 'unaccent': _('Ignore Accent'), 'icontains': _("Partial Match"), 'istartswith': _("Starts With"),
+            'trigram': _("Fuzzy Search"), 'fulltext': _("Full Text")
         }
 
         widgets = {
-            'search': SelectWidget,
-            'unaccent': MultiSelectWidget,
-            'icontains': MultiSelectWidget,
-            'istartswith': MultiSelectWidget,
-            'trigram': MultiSelectWidget,
-            'fulltext': MultiSelectWidget,
+            'search': SelectWidget, 'unaccent': MultiSelectWidget, 'icontains': MultiSelectWidget, 'istartswith': MultiSelectWidget, 'trigram': MultiSelectWidget, 'fulltext':
+            MultiSelectWidget,
         }

cookbook/helper/AllAuthCustomAdapter.py

@@ -1,11 +1,10 @@
 import datetime
-
-from django.conf import settings
+from gettext import gettext as _
 
 from allauth.account.adapter import DefaultAccountAdapter
+from django.conf import settings
 from django.contrib import messages
 from django.core.cache import caches
-from gettext import gettext as _
 
 from cookbook.models import InviteLink
 
@@ -14,13 +13,16 @@ class AllAuthCustomAdapter(DefaultAccountAdapter):
 
     def is_open_for_signup(self, request):
         """
-        Whether to allow sign ups.
+        Whether to allow sign-ups.
         """
         signup_token = False
-        if 'signup_token' in request.session and InviteLink.objects.filter(valid_until__gte=datetime.datetime.today(), used_by=None, uuid=request.session['signup_token']).exists():
+        if 'signup_token' in request.session and InviteLink.objects.filter(
+                valid_until__gte=datetime.datetime.today(), used_by=None, uuid=request.session['signup_token']).exists():
             signup_token = True
 
-        if (request.resolver_match.view_name == 'account_signup' or request.resolver_match.view_name == 'socialaccount_signup') and not settings.ENABLE_SIGNUP and not signup_token:
+        if request.resolver_match.view_name == 'account_signup' and not settings.ENABLE_SIGNUP and not signup_token:
+            return False
+        elif request.resolver_match.view_name == 'socialaccount_signup' and len(settings.SOCIAL_PROVIDERS) < 1:
             return False
         else:
             return super(AllAuthCustomAdapter, self).is_open_for_signup(request)
@@ -31,7 +33,10 @@ class AllAuthCustomAdapter(DefaultAccountAdapter):
             default = datetime.datetime.now()
             c = caches['default'].get_or_set(email, default, timeout=360)
             if c == default:
-                super(AllAuthCustomAdapter, self).send_mail(template_prefix, email, context)
+                try:
+                    super(AllAuthCustomAdapter, self).send_mail(template_prefix, email, context)
+                except Exception:  # dont fail signup just because confirmation mail could not be send
+                    pass
             else:
                 messages.add_message(self.request, messages.ERROR, _('In order to prevent spam, the requested email was not send. Please wait a few minutes and try again.'))
         else:

cookbook/helper/CustomTestRunner.py

@@ -1,8 +0,0 @@
-from django.test.runner import DiscoverRunner
-from django_scopes import scopes_disabled
-
-
-class CustomTestRunner(DiscoverRunner):
-    def run_tests(self, *args, **kwargs):
-        with scopes_disabled():
-            return super().run_tests(*args, **kwargs)

cookbook/helper/HelperFunctions.py

@@ -0,0 +1,48 @@
+import socket
+from urllib.parse import urlparse
+
+from django.core.exceptions import ValidationError
+from django.core.validators import URLValidator
+from django.db.models import Func
+from ipaddress import ip_address
+
+from recipes import settings
+
+
+class Round(Func):
+    function = 'ROUND'
+    template = '%(function)s(%(expressions)s, 0)'
+
+
+def str2bool(v):
+    if isinstance(v, bool) or v is None:
+        return v
+    else:
+        return v.lower() in ("yes", "true", "1")
+
+
+"""
+validates an url that is supposed to be imported
+checks that the protocol used is http(s) and that no local address is accessed
+@:param url to test
+@:return true if url is valid, false otherwise
+"""
+
+
+def validate_import_url(url):
+    try:
+        validator = URLValidator(schemes=['http', 'https'])
+        validator(url)
+    except ValidationError:
+        # if schema is not http or https, consider url invalid
+        return False
+
+    # resolve IP address of url
+    try:
+        url_ip_address = ip_address(str(socket.gethostbyname(urlparse(url).hostname)))
+    except (ValueError, AttributeError, TypeError, Exception) as e:
+        # if ip cannot be parsed, consider url invalid
+        return False
+
+    # validate that IP is neither private nor any other special address
+    return not any([url_ip_address.is_private, url_ip_address.is_reserved, url_ip_address.is_loopback,  url_ip_address.is_multicast, url_ip_address.is_link_local, ])

cookbook/helper/__init__.py

@@ -1,6 +1,4 @@
-import cookbook.helper.dal
 from cookbook.helper.AllAuthCustomAdapter import AllAuthCustomAdapter
 
 __all__ = [
-    'dal',
 ]

cookbook/helper/automation_helper.py

@@ -0,0 +1,227 @@
+import re
+
+from django.core.cache import caches
+from django.db.models.functions import Lower
+
+from cookbook.models import Automation
+
+
+class AutomationEngine:
+    request = None
+    source = None
+    use_cache = None
+    food_aliases = None
+    keyword_aliases = None
+    unit_aliases = None
+    never_unit = None
+    transpose_words = None
+    regex_replace = {
+        Automation.DESCRIPTION_REPLACE: None,
+        Automation.INSTRUCTION_REPLACE: None,
+        Automation.FOOD_REPLACE: None,
+        Automation.UNIT_REPLACE: None,
+        Automation.NAME_REPLACE: None,
+    }
+
+    def __init__(self, request, use_cache=True, source=None):
+        self.request = request
+        self.use_cache = use_cache
+        if not source:
+            self.source = "default_string_to_avoid_false_regex_match"
+        else:
+            self.source = source
+
+    def apply_keyword_automation(self, keyword):
+        keyword = keyword.strip()
+        if self.use_cache and self.keyword_aliases is None:
+            self.keyword_aliases = {}
+            KEYWORD_CACHE_KEY = f'automation_keyword_alias_{self.request.space.pk}'
+            if c := caches['default'].get(KEYWORD_CACHE_KEY, None):
+                self.keyword_aliases = c
+                caches['default'].touch(KEYWORD_CACHE_KEY, 30)
+            else:
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.KEYWORD_ALIAS).only('param_1', 'param_2').order_by('order').all():
+                    self.keyword_aliases[a.param_1.lower()] = a.param_2
+                caches['default'].set(KEYWORD_CACHE_KEY, self.keyword_aliases, 30)
+        else:
+            self.keyword_aliases = {}
+        if self.keyword_aliases:
+            try:
+                keyword = self.keyword_aliases[keyword.lower()]
+            except KeyError:
+                pass
+        else:
+            if automation := Automation.objects.filter(space=self.request.space, type=Automation.KEYWORD_ALIAS, param_1__iexact=keyword, disabled=False).order_by('order').first():
+                return automation.param_2
+        return keyword
+
+    def apply_unit_automation(self, unit):
+        unit = unit.strip()
+        if self.use_cache and self.unit_aliases is None:
+            self.unit_aliases = {}
+            UNIT_CACHE_KEY = f'automation_unit_alias_{self.request.space.pk}'
+            if c := caches['default'].get(UNIT_CACHE_KEY, None):
+                self.unit_aliases = c
+                caches['default'].touch(UNIT_CACHE_KEY, 30)
+            else:
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.UNIT_ALIAS).only('param_1', 'param_2').order_by('order').all():
+                    self.unit_aliases[a.param_1.lower()] = a.param_2
+                caches['default'].set(UNIT_CACHE_KEY, self.unit_aliases, 30)
+        else:
+            self.unit_aliases = {}
+        if self.unit_aliases:
+            try:
+                unit = self.unit_aliases[unit.lower()]
+            except KeyError:
+                pass
+        else:
+            if automation := Automation.objects.filter(space=self.request.space, type=Automation.UNIT_ALIAS, param_1__iexact=unit, disabled=False).order_by('order').first():
+                return automation.param_2
+        return self.apply_regex_replace_automation(unit, Automation.UNIT_REPLACE)
+
+    def apply_food_automation(self, food):
+        food = food.strip()
+        if self.use_cache and self.food_aliases is None:
+            self.food_aliases = {}
+            FOOD_CACHE_KEY = f'automation_food_alias_{self.request.space.pk}'
+            if c := caches['default'].get(FOOD_CACHE_KEY, None):
+                self.food_aliases = c
+                caches['default'].touch(FOOD_CACHE_KEY, 30)
+            else:
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.FOOD_ALIAS).only('param_1', 'param_2').order_by('order').all():
+                    self.food_aliases[a.param_1.lower()] = a.param_2
+                caches['default'].set(FOOD_CACHE_KEY, self.food_aliases, 30)
+        else:
+            self.food_aliases = {}
+
+        if self.food_aliases:
+            try:
+                return self.food_aliases[food.lower()]
+            except KeyError:
+                return self.apply_regex_replace_automation(food, Automation.FOOD_REPLACE)
+        else:
+            if automation := Automation.objects.filter(space=self.request.space, type=Automation.FOOD_ALIAS, param_1__iexact=food, disabled=False).order_by('order').first():
+                return automation.param_2
+        return self.apply_regex_replace_automation(food, Automation.FOOD_REPLACE)
+
+    def apply_never_unit_automation(self, tokens):
+        """
+        Moves a string that should never be treated as a unit to next token and optionally replaced with default unit
+        e.g. NEVER_UNIT: param1: egg, param2: None would modify ['1', 'egg', 'white'] to ['1', '', 'egg', 'white']
+        or NEVER_UNIT: param1: egg, param2: pcs would modify ['1', 'egg', 'yolk'] to ['1', 'pcs', 'egg', 'yolk']
+        :param1 string: string that should never be considered a unit, will be moved to token[2]
+        :param2 (optional) unit as string: will insert unit string into token[1]
+        :return: unit as string (possibly changed by automation)
+        """
+
+        if self.use_cache and self.never_unit is None:
+            self.never_unit = {}
+            NEVER_UNIT_CACHE_KEY = f'automation_never_unit_{self.request.space.pk}'
+            if c := caches['default'].get(NEVER_UNIT_CACHE_KEY, None):
+                self.never_unit = c
+                caches['default'].touch(NEVER_UNIT_CACHE_KEY, 30)
+            else:
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.NEVER_UNIT).only('param_1', 'param_2').order_by('order').all():
+                    self.never_unit[a.param_1.lower()] = a.param_2
+                caches['default'].set(NEVER_UNIT_CACHE_KEY, self.never_unit, 30)
+        else:
+            self.never_unit = {}
+
+        new_unit = None
+        alt_unit = self.apply_unit_automation(tokens[1])
+        never_unit = False
+        if self.never_unit:
+            try:
+                new_unit = self.never_unit[tokens[1].lower()]
+                never_unit = True
+            except KeyError:
+                return tokens
+        else:
+            if a := Automation.objects.annotate(param_1_lower=Lower('param_1')).filter(space=self.request.space, type=Automation.NEVER_UNIT, param_1_lower__in=[
+                    tokens[1].lower(), alt_unit.lower()], disabled=False).order_by('order').first():
+                new_unit = a.param_2
+                never_unit = True
+
+        if never_unit:
+            tokens.insert(1, new_unit)
+        return tokens
+
+    def apply_transpose_automation(self, string):
+        """
+        If two words (param_1 & param_2) are detected in sequence, swap their position in the ingredient string
+        :param 1: first word to detect
+        :param 2: second word to detect
+        return: new ingredient string
+        """
+        if self.use_cache and self.transpose_words is None:
+            self.transpose_words = {}
+            TRANSPOSE_WORDS_CACHE_KEY = f'automation_transpose_words_{self.request.space.pk}'
+            if c := caches['default'].get(TRANSPOSE_WORDS_CACHE_KEY, None):
+                self.transpose_words = c
+                caches['default'].touch(TRANSPOSE_WORDS_CACHE_KEY, 30)
+            else:
+                i = 0
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.TRANSPOSE_WORDS).only(
+                        'param_1', 'param_2').order_by('order').all()[:512]:
+                    self.transpose_words[i] = [a.param_1.lower(), a.param_2.lower()]
+                    i += 1
+                caches['default'].set(TRANSPOSE_WORDS_CACHE_KEY, self.transpose_words, 30)
+        else:
+            self.transpose_words = {}
+
+        tokens = [x.lower() for x in string.replace(',', ' ').split()]
+        if self.transpose_words:
+            for key, value in self.transpose_words.items():
+                if value[0] in tokens and value[1] in tokens:
+                    string = re.sub(rf"\b({value[0]})\W*({value[1]})\b", r"\2 \1", string, flags=re.IGNORECASE)
+        else:
+            for rule in Automation.objects.filter(space=self.request.space, type=Automation.TRANSPOSE_WORDS, disabled=False) \
+                    .annotate(param_1_lower=Lower('param_1'), param_2_lower=Lower('param_2')) \
+                    .filter(param_1_lower__in=tokens, param_2_lower__in=tokens).order_by('order')[:512]:
+                if rule.param_1 in tokens and rule.param_2 in tokens:
+                    string = re.sub(rf"\b({rule.param_1})\W*({rule.param_2})\b", r"\2 \1", string, flags=re.IGNORECASE)
+        return string
+
+    def apply_regex_replace_automation(self, string, automation_type):
+        # TODO add warning - maybe on SPACE page? when a max of 512 automations of a specific type is exceeded (ALIAS types excluded?)
+        """
+        Replaces strings in a recipe field that are from a matched source
+        field_type are Automation.type that apply regex replacements
+        Automation.DESCRIPTION_REPLACE
+        Automation.INSTRUCTION_REPLACE
+        Automation.FOOD_REPLACE
+        Automation.UNIT_REPLACE
+        Automation.NAME_REPLACE
+
+        regex replacment utilized the following fields from the Automation model
+        :param 1: source that should apply the automation in regex format ('.*' for all)
+        :param 2: regex pattern to match ()
+        :param 3: replacement string (leave blank to delete)
+        return: new string
+        """
+        if self.use_cache and self.regex_replace[automation_type] is None:
+            self.regex_replace[automation_type] = {}
+            REGEX_REPLACE_CACHE_KEY = f'automation_regex_replace_{self.request.space.pk}'
+            if c := caches['default'].get(REGEX_REPLACE_CACHE_KEY, None):
+                self.regex_replace[automation_type] = c[automation_type]
+                caches['default'].touch(REGEX_REPLACE_CACHE_KEY, 30)
+            else:
+                i = 0
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=automation_type).only(
+                        'param_1', 'param_2', 'param_3').order_by('order').all()[:512]:
+                    self.regex_replace[automation_type][i] = [a.param_1, a.param_2, a.param_3]
+                    i += 1
+                caches['default'].set(REGEX_REPLACE_CACHE_KEY, self.regex_replace, 30)
+        else:
+            self.regex_replace[automation_type] = {}
+
+        if self.regex_replace[automation_type]:
+            for rule in self.regex_replace[automation_type].values():
+                if re.match(rule[0], (self.source)[:512]):
+                    string = re.sub(rule[1], rule[2], string, flags=re.IGNORECASE)
+        else:
+            for rule in Automation.objects.filter(space=self.request.space, disabled=False, type=automation_type).only(
+                    'param_1', 'param_2', 'param_3').order_by('order').all()[:512]:
+                if re.match(rule.param_1, (self.source)[:512]):
+                    string = re.sub(rule.param_2, rule.param_3, string, flags=re.IGNORECASE)
+        return string

cookbook/helper/cache_helper.py

@@ -0,0 +1,11 @@
+class CacheHelper:
+    space = None
+
+    BASE_UNITS_CACHE_KEY = None
+    PROPERTY_TYPE_CACHE_KEY = None
+
+    def __init__(self, space):
+        self.space = space
+
+        self.BASE_UNITS_CACHE_KEY = f'SPACE_{space.id}_BASE_UNITS'
+        self.PROPERTY_TYPE_CACHE_KEY = f'SPACE_{space.id}_PROPERTY_TYPES'

cookbook/helper/context_processors.py

@@ -10,4 +10,6 @@ def context_settings(request):
         'TERMS_URL': settings.TERMS_URL,
         'PRIVACY_URL': settings.PRIVACY_URL,
         'IMPRINT_URL': settings.IMPRINT_URL,
+        'SHOPPING_MIN_AUTOSYNC_INTERVAL': settings.SHOPPING_MIN_AUTOSYNC_INTERVAL,
+        'DISABLE_EXTERNAL_CONNECTORS': settings.DISABLE_EXTERNAL_CONNECTORS,
     }

cookbook/helper/dal.py

@@ -1,34 +0,0 @@
-from cookbook.models import Food, Keyword, Recipe, Unit
-
-from dal import autocomplete
-
-
-class BaseAutocomplete(autocomplete.Select2QuerySetView):
-    model = None
-
-    def get_queryset(self):
-        if not self.request.user.is_authenticated:
-            return self.model.objects.none()
-
-        qs = self.model.objects.filter(space=self.request.space).all()
-
-        if self.q:
-            qs = qs.filter(name__icontains=self.q)
-
-        return qs
-
-
-class KeywordAutocomplete(BaseAutocomplete):
-    model = Keyword
-
-
-class IngredientsAutocomplete(BaseAutocomplete):
-    model = Food
-
-
-class RecipeAutocomplete(BaseAutocomplete):
-    model = Recipe
-
-
-class UnitAutocomplete(BaseAutocomplete):
-    model = Unit

cookbook/helper/fdc_helper.py

@@ -0,0 +1,19 @@
+import json
+
+
+def get_all_nutrient_types():
+    f = open('')  # <--- download the foundation food or any other dataset and retrieve all nutrition ID's from it https://fdc.nal.usda.gov/download-datasets.html
+    json_data = json.loads(f.read())
+
+    nutrients = {}
+    for food in json_data['FoundationFoods']:
+        for entry in food['foodNutrients']:
+            nutrients[entry['nutrient']['id']] = {'name': entry['nutrient']['name'], 'unit': entry['nutrient']['unitName']}
+
+    nutrient_ids = list(nutrients.keys())
+    nutrient_ids.sort()
+    for nid in nutrient_ids:
+        print('{', f'value: {nid}, text: "{nutrients[nid]["name"]} [{nutrients[nid]["unit"]}] ({nid})"', '},')
+
+
+get_all_nutrient_types()

cookbook/helper/image_processing.py

@@ -1,32 +1,31 @@
 import os
-import sys
-
-from PIL import Image
 from io import BytesIO
 
+from PIL import Image
 
-def rescale_image_jpeg(image_object, base_width=720):
+
+def rescale_image_jpeg(image_object, base_width=1020):
     img = Image.open(image_object)
     icc_profile = img.info.get('icc_profile')  # remember color profile to not mess up colors
     width_percent = (base_width / float(img.size[0]))
     height = int((float(img.size[1]) * float(width_percent)))
 
-    img = img.resize((base_width, height), Image.ANTIALIAS)
+    img = img.resize((base_width, height), Image.LANCZOS)
     img_bytes = BytesIO()
-    img.save(img_bytes, 'JPEG', quality=75, optimize=True, icc_profile=icc_profile)
+    img.save(img_bytes, 'JPEG', quality=90, optimize=True, icc_profile=icc_profile)
 
     return img_bytes
 
 
-def rescale_image_png(image_object, base_width=720):
-    basewidth = 720
-    wpercent = (basewidth / float(image_object.size[0]))
+def rescale_image_png(image_object, base_width=1020):
+    image_object = Image.open(image_object)
+    wpercent = (base_width / float(image_object.size[0]))
     hsize = int((float(image_object.size[1]) * float(wpercent)))
-    img = image_object.resize((basewidth, hsize), Image.ANTIALIAS)
+    img = image_object.resize((base_width, hsize), Image.LANCZOS)
 
     im_io = BytesIO()
-    img.save(im_io, 'PNG', quality=70)
-    return img
+    img.save(im_io, 'PNG', quality=90)
+    return im_io
 
 
 def get_filetype(name):
@@ -36,10 +35,19 @@ def get_filetype(name):
         return '.jpeg'
 
 
-def handle_image(request, image_object, filetype='.jpeg'):
-    if sys.getsizeof(image_object) / 8 > 500:
-        if filetype == '.jpeg':
-            return rescale_image_jpeg(image_object), filetype
+# TODO this whole file needs proper documentation, refactoring, and testing
+# TODO also add env variable to define which images sizes should be compressed
+# filetype argument can not be optional, otherwise this function will treat all images as if they were a jpeg
+# Because it's no longer optional, no reason to return it
+def handle_image(request, image_object, filetype):
+    try:
+        Image.open(image_object).verify()
+    except Exception:
+        return None
+
+    if (image_object.size / 1000) > 500:  # if larger than 500 kb compress
+        if filetype == '.jpeg' or filetype == '.jpg':
+            return rescale_image_jpeg(image_object)
         if filetype == '.png':
-            return rescale_image_png(image_object), filetype
-    return image_object, filetype
+            return rescale_image_png(image_object)
+    return image_object

cookbook/helper/ingredient_parser.py

@@ -2,18 +2,16 @@ import re
 import string
 import unicodedata
 
-from django.core.cache import caches
-
-from cookbook.models import Unit, Food, Automation
+from cookbook.helper.automation_helper import AutomationEngine
+from cookbook.models import Food, Ingredient, Unit
 
 
 class IngredientParser:
     request = None
     ignore_rules = False
-    food_aliases = {}
-    unit_aliases = {}
+    automation = None
 
-    def __init__(self, request, cache_mode, ignore_automations=False):
+    def __init__(self, request, cache_mode=True, ignore_automations=False):
         """
         Initialize ingredient parser
         :param request: request context (to control caching, rule ownership, etc.)
@@ -22,65 +20,8 @@ class IngredientParser:
         """
         self.request = request
         self.ignore_rules = ignore_automations
-        if cache_mode:
-            FOOD_CACHE_KEY = f'automation_food_alias_{self.request.space.pk}'
-            if c := caches['default'].get(FOOD_CACHE_KEY, None):
-                self.food_aliases = c
-                caches['default'].touch(FOOD_CACHE_KEY, 30)
-            else:
-                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.FOOD_ALIAS).only('param_1', 'param_2').all():
-                    self.food_aliases[a.param_1] = a.param_2
-                caches['default'].set(FOOD_CACHE_KEY, self.food_aliases, 30)
-
-            UNIT_CACHE_KEY = f'automation_unit_alias_{self.request.space.pk}'
-            if c := caches['default'].get(UNIT_CACHE_KEY, None):
-                self.unit_aliases = c
-                caches['default'].touch(UNIT_CACHE_KEY, 30)
-            else:
-                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.UNIT_ALIAS).only('param_1', 'param_2').all():
-                    self.unit_aliases[a.param_1] = a.param_2
-                caches['default'].set(UNIT_CACHE_KEY, self.unit_aliases, 30)
-        else:
-            self.food_aliases = {}
-            self.unit_aliases = {}
-
-    def apply_food_automation(self, food):
-        """
-        Apply food alias automations to passed foood
-        :param food: unit as string
-        :return: food as string (possibly changed by automation)
-        """
-        if self.ignore_rules:
-            return food
-        else:
-            if self.food_aliases:
-                try:
-                    return self.food_aliases[food]
-                except KeyError:
-                    return food
-            else:
-                if automation := Automation.objects.filter(space=self.request.space, type=Automation.FOOD_ALIAS, param_1=food, disabled=False).first():
-                    return automation.param_2
-        return food
-
-    def apply_unit_automation(self, unit):
-        """
-        Apply unit alias automations to passed unit
-        :param unit: unit as string
-        :return: unit as string (possibly changed by automation)
-        """
-        if self.ignore_rules:
-            return unit
-        else:
-            if self.unit_aliases:
-                try:
-                    return self.unit_aliases[unit]
-                except KeyError:
-                    return unit
-            else:
-                if automation := Automation.objects.filter(space=self.request.space, type=Automation.UNIT_ALIAS, param_1=unit, disabled=False).first():
-                    return automation.param_2
-        return unit
+        if not self.ignore_rules:
+            self.automation = AutomationEngine(self.request, use_cache=cache_mode)
 
     def get_unit(self, unit):
         """
@@ -91,7 +32,10 @@ class IngredientParser:
         if not unit:
             return None
         if len(unit) > 0:
-            u, created = Unit.objects.get_or_create(name=self.apply_unit_automation(unit), space=self.request.space)
+            if self.ignore_rules:
+                u, created = Unit.objects.get_or_create(name=unit.strip(), space=self.request.space)
+            else:
+                u, created = Unit.objects.get_or_create(name=self.automation.apply_unit_automation(unit), space=self.request.space)
             return u
         return None
 
@@ -104,7 +48,10 @@ class IngredientParser:
         if not food:
             return None
         if len(food) > 0:
-            f, created = Food.objects.get_or_create(name=self.apply_food_automation(food), space=self.request.space)
+            if self.ignore_rules:
+                f, created = Food.objects.get_or_create(name=food.strip(), space=self.request.space)
+            else:
+                f, created = Food.objects.get_or_create(name=self.automation.apply_food_automation(food), space=self.request.space)
             return f
         return None
 
@@ -124,17 +71,19 @@ class IngredientParser:
 
     def parse_amount(self, x):
         amount = 0
-        unit = ''
+        unit = None
         note = ''
+        if x.strip() == '':
+            return amount, unit, note
 
         did_check_frac = False
         end = 0
         while (end < len(x) and (x[end] in string.digits
                                  or (
-                                         (x[end] == '.' or x[end] == ',' or x[end] == '/')
-                                         and end + 1 < len(x)
-                                         and x[end + 1] in string.digits
-                                 ))):
+            (x[end] == '.' or x[end] == ',' or x[end] == '/')
+            and end + 1 < len(x)
+            and x[end + 1] in string.digits
+        ))):
             end += 1
         if end > 0:
             if "/" in x[:end]:
@@ -155,33 +104,37 @@ class IngredientParser:
                 except ValueError:
                     unit = x[end:]
 
-        if unit.startswith('(') or unit.startswith('-'):  # i dont know any unit that starts with ( or - so its likely an alternative like 1L (500ml) Water or 2-3
-            unit = ''
+        if unit is not None and unit.strip() == '':
+            unit = None
+
+        if unit is not None and (unit.startswith('(') or unit.startswith(
+                '-')):  # i dont know any unit that starts with ( or - so its likely an alternative like 1L (500ml) Water or 2-3
+            unit = None
             note = x
         return amount, unit, note
 
-    def parse_ingredient_with_comma(self, tokens):
-        ingredient = ''
+    def parse_food_with_comma(self, tokens):
+        food = ''
         note = ''
         start = 0
         # search for first occurrence of an argument ending in a comma
         while start < len(tokens) and not tokens[start].endswith(','):
             start += 1
         if start == len(tokens):
-            # no token ending in a comma found -> use everything as ingredient
-            ingredient = ' '.join(tokens)
+            # no token ending in a comma found -> use everything as food
+            food = ' '.join(tokens)
         else:
-            ingredient = ' '.join(tokens[:start + 1])[:-1]
+            food = ' '.join(tokens[:start + 1])[:-1]
             note = ' '.join(tokens[start + 1:])
-        return ingredient, note
+        return food, note
 
-    def parse_ingredient(self, tokens):
-        ingredient = ''
+    def parse_food(self, tokens):
+        food = ''
         note = ''
         if tokens[-1].endswith(')'):
             # Check if the matching opening bracket is in the same token
             if (not tokens[-1].startswith('(')) and ('(' in tokens[-1]):
-                return self.parse_ingredient_with_comma(tokens)
+                return self.parse_food_with_comma(tokens)
             # last argument ends with closing bracket -> look for opening bracket
             start = len(tokens) - 1
             while not tokens[start].startswith('(') and not start == 0:
@@ -191,33 +144,65 @@ class IngredientParser:
                 raise ValueError
             elif start < 0:
                 # no opening bracket anywhere -> just ignore the last bracket
-                ingredient, note = self.parse_ingredient_with_comma(tokens)
+                food, note = self.parse_food_with_comma(tokens)
             else:
-                # opening bracket found -> split in ingredient and note, remove brackets from note  # noqa: E501
+                # opening bracket found -> split in food and note, remove brackets from note  # noqa: E501
                 note = ' '.join(tokens[start:])[1:-1]
-                ingredient = ' '.join(tokens[:start])
+                food = ' '.join(tokens[:start])
         else:
-            ingredient, note = self.parse_ingredient_with_comma(tokens)
-        return ingredient, note
+            food, note = self.parse_food_with_comma(tokens)
+        return food, note
 
-    def parse(self, x):
+    def parse(self, ingredient):
+        """
+        Main parsing function, takes an ingredient string (e.g. '1 l Water') and extracts amount, unit, food, ...
+        :param ingredient: string ingredient
+        :return: amount, unit (can be None), food, note (can be empty)
+        """
         # initialize default values
         amount = 0
-        unit = ''
-        ingredient = ''
+        unit = None
+        food = ''
         note = ''
         unit_note = ''
 
+        if len(ingredient) == 0:
+            raise ValueError('string to parse cannot be empty')
+
+        if len(ingredient) > 512:
+            raise ValueError('cannot parse ingredients with more than 512 characters')
+
+        # some people/languages put amount and unit at the end of the ingredient string
+        # if something like this is detected move it to the beginning so the parser can handle it
+        if len(ingredient) < 1000 and re.search(r'^([^\W\d_])+(.)*[1-9](\d)*\s*([^\W\d_])+', ingredient):
+            match = re.search(r'[1-9](\d)*\s*([^\W\d_])+', ingredient)
+            print(f'reording from {ingredient} to {ingredient[match.start():match.end()] + " " + ingredient.replace(ingredient[match.start():match.end()], "")}')
+            ingredient = ingredient[match.start():match.end()] + ' ' + ingredient.replace(ingredient[match.start():match.end()], '')
+
         # if the string contains parenthesis early on remove it and place it at the end
         # because its likely some kind of note
-        if re.match('(.){1,6}\s\((.[^\(\)])+\)\s', x):
-            match = re.search('\((.[^\(])+\)', x)
-            x = x[:match.start()] + x[match.end():] + ' ' + x[match.start():match.end()]
+        if re.match('(.){1,6}\\s\\((.[^\\(\\)])+\\)\\s', ingredient):
+            match = re.search('\\((.[^\\(])+\\)', ingredient)
+            ingredient = ingredient[:match.start()] + ingredient[match.end():] + ' ' + ingredient[match.start():match.end()]
 
-        tokens = x.split()
+        # leading spaces before commas result in extra tokens, clean them out
+        ingredient = ingredient.replace(' ,', ',')
+
+        # handle "(from) - (to)" amounts by using the minimum amount and adding the range to the description
+        # "10.5 - 200 g XYZ" => "100 g XYZ (10.5 - 200)"
+        ingredient = re.sub("^(\\d+|\\d+[\\.,]\\d+) - (\\d+|\\d+[\\.,]\\d+) (.*)", "\\1 \\3 (\\1 - \\2)", ingredient)
+
+        # if amount and unit are connected add space in between
+        if re.match('([0-9])+([A-z])+\\s', ingredient):
+            ingredient = re.sub(r'(?<=([a-z])|\d)(?=(?(1)\d|[a-z]))', ' ', ingredient)
+
+        if not self.ignore_rules:
+            ingredient = self.automation.apply_transpose_automation(ingredient)
+
+        tokens = ingredient.split()  # split at each space into tokens
         if len(tokens) == 1:
-            # there only is one argument, that must be the ingredient
-            ingredient = tokens[0]
+            # there only is one argument, that must be the food
+            food = tokens[0]
         else:
             try:
                 # try to parse first argument as amount
@@ -226,49 +211,69 @@ class IngredientParser:
                 # three arguments if it already has a unit there can't be
                 # a fraction for the amount
                 if len(tokens) > 2:
+                    if not self.ignore_rules:
+                        tokens = self.automation.apply_never_unit_automation(tokens)
                     try:
-                        if not unit == '':
+                        if unit is not None:
                             # a unit is already found, no need to try the second argument for a fraction
-                            # probably not the best method to do it, but I didn't want to make an if check and paste the exact same thing in the else as already is in the except  # noqa: E501
+                            # probably not the best method to do it, but I didn't want to make an if check and paste the exact same thing in the else as already is in the except
                             raise ValueError
                         # try to parse second argument as amount and add that, in case of '2 1/2' or '2 ½'
                         amount += self.parse_fraction(tokens[1])
                         # assume that units can't end with a comma
                         if len(tokens) > 3 and not tokens[2].endswith(','):
-                            # try to use third argument as unit and everything else as ingredient, use everything as ingredient if it fails  # noqa: E501
+                            # try to use third argument as unit and everything else as food, use everything as food if it fails
                             try:
-                                ingredient, note = self.parse_ingredient(tokens[3:])
+                                food, note = self.parse_food(tokens[3:])
                                 unit = tokens[2]
                             except ValueError:
-                                ingredient, note = self.parse_ingredient(tokens[2:])
+                                food, note = self.parse_food(tokens[2:])
                         else:
-                            ingredient, note = self.parse_ingredient(tokens[2:])
+                            food, note = self.parse_food(tokens[2:])
                     except ValueError:
                         # assume that units can't end with a comma
                         if not tokens[1].endswith(','):
-                            # try to use second argument as unit and everything else as ingredient, use everything as ingredient if it fails  # noqa: E501
+                            # try to use second argument as unit and everything else as food, use everything as food if it fails
                             try:
-                                ingredient, note = self.parse_ingredient(tokens[2:])
-                                if unit == '':
+                                food, note = self.parse_food(tokens[2:])
+                                if unit is None:
                                     unit = tokens[1]
                                 else:
                                     note = tokens[1]
                             except ValueError:
-                                ingredient, note = self.parse_ingredient(tokens[1:])
+                                food, note = self.parse_food(tokens[1:])
                         else:
-                            ingredient, note = self.parse_ingredient(tokens[1:])
+                            food, note = self.parse_food(tokens[1:])
                 else:
                     # only two arguments, first one is the amount
-                    # which means this is the ingredient
-                    ingredient = tokens[1]
+                    # which means this is the food
+                    food = tokens[1]
             except ValueError:
                 try:
                     # can't parse first argument as amount
-                    # -> no unit -> parse everything as ingredient
-                    ingredient, note = self.parse_ingredient(tokens)
+                    # -> no unit -> parse everything as food
+                    food, note = self.parse_food(tokens)
                 except ValueError:
-                    ingredient = ' '.join(tokens[1:])
+                    food = ' '.join(tokens[1:])
 
         if unit_note not in note:
             note += ' ' + unit_note
-        return amount, self.apply_unit_automation(unit.strip()), self.apply_food_automation(ingredient.strip()), note.strip()
+
+        if unit and not self.ignore_rules:
+            unit = self.automation.apply_unit_automation(unit)
+
+        if food and not self.ignore_rules:
+            food = self.automation.apply_food_automation(food)
+        if len(food) > Food._meta.get_field('name').max_length:  # test if food name is to long
+            # try splitting it at a space and taking only the first arg
+            if len(food.split()) > 1 and len(food.split()[0]) < Food._meta.get_field('name').max_length:
+                note = ' '.join(food.split()[1:]) + ' ' + note
+                food = food.split()[0]
+            else:
+                note = food + ' ' + note
+                food = food[:Food._meta.get_field('name').max_length]
+
+        if len(food.strip()) == 0:
+            raise ValueError(f'Error parsing string {ingredient}, food cannot be empty')
+
+        return amount, unit, food, note[:Ingredient._meta.get_field('note').max_length].strip()

cookbook/helper/mdx_urlize.py

@@ -35,6 +35,7 @@ Negative examples:
 u'<p>del.icio.us</p>'
 
 """
+from xml.etree.ElementTree import Element
 
 import markdown
 
@@ -64,7 +65,7 @@ class UrlizePattern(markdown.inlinepatterns.Pattern):
             else:
                 url = 'http://' + url
 
-        el = markdown.util.etree.Element("a")
+        el = Element("a")
         el.set('href', url)
         el.text = markdown.util.AtomicString(text)
         return el
@@ -73,9 +74,9 @@ class UrlizePattern(markdown.inlinepatterns.Pattern):
 class UrlizeExtension(markdown.Extension):
     """ Urlize Extension for Python-Markdown. """
 
-    def extendMarkdown(self, md, md_globals):
+    def extendMarkdown(self, md):
         """ Replace autolink with UrlizePattern """
-        md.inlinePatterns['autolink'] = UrlizePattern(URLIZE_RE, md)
+        md.inlinePatterns.register(UrlizePattern(URLIZE_RE, md), 'autolink', 120)
 
 
 def makeExtension(*args, **kwargs):

cookbook/helper/open_data_importer.py

@@ -0,0 +1,496 @@
+import traceback
+from collections import defaultdict
+from decimal import Decimal
+
+from cookbook.models import (Food, FoodProperty, Property, PropertyType, Supermarket,
+                             SupermarketCategory, SupermarketCategoryRelation, Unit, UnitConversion)
+import re
+
+
+class OpenDataImportResponse:
+    total_created = 0
+    total_updated = 0
+    total_untouched = 0
+    total_errored = 0
+
+    def to_dict(self):
+        return {'total_created': self.total_created, 'total_updated': self.total_updated, 'total_untouched': self.total_untouched, 'total_errored': self.total_errored}
+
+
+class OpenDataImporter:
+    request = None
+    data = {}
+    slug_id_cache = {}
+    update_existing = False
+    use_metric = True
+
+    def __init__(self, request, data, update_existing=False, use_metric=True):
+        self.request = request
+        self.data = data
+        self.update_existing = update_existing
+        self.use_metric = use_metric
+
+    def _update_slug_cache(self, object_class, datatype):
+        self.slug_id_cache[datatype] = dict(object_class.objects.filter(space=self.request.space, open_data_slug__isnull=False).values_list('open_data_slug', 'id', ))
+
+    @staticmethod
+    def _is_obj_identical(field_list, obj, existing_obj):
+        """
+        checks if the obj meant for import is identical to an already existing one
+        :param field_list: list of field names to check
+        :type field_list: list[str]
+        :param obj: object meant for import
+        :type obj: Object
+        :param existing_obj: object already in DB
+        :type existing_obj: Object
+        :return: if objects are identical
+        :rtype: bool
+        """
+        for field in field_list:
+            if isinstance(getattr(obj, field), float) or isinstance(getattr(obj, field), Decimal):
+                if abs(float(getattr(obj, field)) - float(existing_obj[field])) > 0.001:  # convert both to float and check if basically equal
+                    print(f'comparing FLOAT {obj} failed because field {field} is not equal ({getattr(obj, field)} != {existing_obj[field]})')
+                    return False
+            elif getattr(obj, field) != existing_obj[field]:
+                print(f'comparing {obj} failed because field {field} is not equal ({getattr(obj, field)} != {existing_obj[field]})')
+                return False
+        return True
+
+    @staticmethod
+    def _merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+        """
+        sometimes there might be two objects conflicting for open data import (one has the slug, the other the name)
+        this function checks if that is the case and merges the two objects if possible
+        :param model_type: type of model to check/merge
+        :type model_type: Model
+        :param obj: object that should be created/updated
+        :type obj: Model
+        :param existing_data_slugs: dict of open data slugs mapped to objects
+        :type existing_data_slugs: dict
+        :param existing_data_names: dict of names mapped to objects
+        :type existing_data_names: dict
+        :return: true if merge was successful or not necessary else false
+        :rtype: bool
+        """
+        if obj.open_data_slug in existing_data_slugs and obj.name in existing_data_names and existing_data_slugs[obj.open_data_slug]['pk'] != existing_data_names[obj.name]['pk']:
+            try:
+                source_obj = model_type.objects.get(pk=existing_data_slugs[obj.open_data_slug]['pk'])
+                del existing_data_slugs[obj.open_data_slug]
+                source_obj.merge_into(model_type.objects.get(pk=existing_data_names[obj.name]['pk']))
+                return True
+            except RuntimeError:
+                return False  # in the edge case (e.g. parent/child) that an object cannot be merged don't update it for now
+        else:
+            return True
+
+    @staticmethod
+    def _get_existing_obj(obj, existing_data_slugs, existing_data_names):
+        """
+        gets the existing object from slug or name cache
+        :param obj: object that should be found
+        :type obj: Model
+        :param existing_data_slugs: dict of open data slugs mapped to objects
+        :type existing_data_slugs: dict
+        :param existing_data_names: dict of names mapped to objects
+        :type existing_data_names: dict
+        :return: existing object
+        :rtype: dict
+        """
+        existing_obj = None
+        if obj.open_data_slug in existing_data_slugs:
+            existing_obj = existing_data_slugs[obj.open_data_slug]
+        elif obj.name in existing_data_names:
+            existing_obj = existing_data_names[obj.name]
+
+        return existing_obj
+
+    def import_units(self):
+        od_response = OpenDataImportResponse()
+        datatype = 'unit'
+        model_type = Unit
+        field_list = ['name', 'plural_name', 'base_unit', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
+
+        for u in list(self.data[datatype].keys()):
+            obj = model_type(
+                name=self.data[datatype][u]['name'],
+                plural_name=self.data[datatype][u]['plural_name'],
+                base_unit=self.data[datatype][u]['base_unit'].lower() if self.data[datatype][u]['base_unit'] != '' else None,
+                open_data_slug=u,
+                space=self.request.space
+            )
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        return od_response
+
+    def import_category(self):
+        od_response = OpenDataImportResponse()
+        datatype = 'category'
+        model_type = SupermarketCategory
+        field_list = ['name', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
+
+        for k in list(self.data[datatype].keys()):
+            obj = model_type(
+                name=self.data[datatype][k]['name'],
+                open_data_slug=k,
+                space=self.request.space
+            )
+
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        return od_response
+
+    def import_property(self):
+        od_response = OpenDataImportResponse()
+        datatype = 'property'
+        model_type = PropertyType
+        field_list = ['name', 'unit', 'fdc_id', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
+
+        for k in list(self.data[datatype].keys()):
+            obj = model_type(
+                name=self.data[datatype][k]['name'],
+                unit=self.data[datatype][k]['unit'],
+                fdc_id=self.data[datatype][k]['fdc_id'],
+                open_data_slug=k,
+                space=self.request.space
+            )
+
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        return od_response
+
+    def import_supermarket(self):
+        od_response = OpenDataImportResponse()
+        datatype = 'store'
+        model_type = Supermarket
+        field_list = ['name', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
+
+        self._update_slug_cache(SupermarketCategory, 'category')
+        for k in list(self.data[datatype].keys()):
+            obj = model_type(
+                name=self.data[datatype][k]['name'],
+                open_data_slug=k,
+                space=self.request.space
+            )
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, update_conflicts=True, update_fields=field_list, unique_fields=('space', 'name',))
+            od_response.total_created += len(create_list)
+
+        # always add open data slug if matching supermarket is found, otherwise relation might fail
+        self._update_slug_cache(Supermarket, 'store')
+
+        for k in list(self.data[datatype].keys()):
+            relations = []
+            order = 0
+            for c in self.data[datatype][k]['categories']:
+                relations.append(
+                    SupermarketCategoryRelation(
+                        supermarket_id=self.slug_id_cache[datatype][k],
+                        category_id=self.slug_id_cache['category'][c],
+                        order=order,
+                    )
+                )
+                order += 1
+
+            SupermarketCategoryRelation.objects.bulk_create(relations, ignore_conflicts=True, unique_fields=('supermarket', 'category',))
+
+        return od_response
+
+    def import_food(self):
+        od_response = OpenDataImportResponse()
+        datatype = 'food'
+        model_type = Food
+        field_list = ['name', 'open_data_slug']
+
+        existing_data_slugs = {}
+        existing_data_names = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_names[obj['name']] = obj
+
+        update_list = []
+        create_list = []
+
+        self._update_slug_cache(Unit, 'unit')
+        self._update_slug_cache(PropertyType, 'property')
+        self._update_slug_cache(SupermarketCategory, 'category')
+
+        unit_g = Unit.objects.filter(space=self.request.space, base_unit__iexact='g').first()
+
+        for k in list(self.data[datatype].keys()):
+
+            obj_dict = {
+                'name': self.data[datatype][k]['name'],
+                'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None,
+                'supermarket_category_id': self.slug_id_cache['category'][self.data[datatype][k]['store_category']],
+                'fdc_id': re.sub(r'\D', '', self.data[datatype][k]['fdc_id']) if self.data[datatype][k]['fdc_id'] != '' else None,
+                'open_data_slug': k,
+                'properties_food_unit_id': None,
+                'space_id': self.request.space.id,
+            }
+
+            if unit_g:
+                obj_dict['properties_food_unit_id'] = unit_g.id
+
+            obj = model_type(**obj_dict)
+
+            if obj.open_data_slug in existing_data_slugs or obj.name in existing_data_names:
+                if not self._merge_if_conflicting(model_type, obj, existing_data_slugs, existing_data_names):
+                    od_response.total_errored += 1
+                    continue  # if conflicting objects exist and cannot be merged skip object
+
+                existing_obj = self._get_existing_obj(obj, existing_data_slugs, existing_data_names)
+
+                if not self._is_obj_identical(field_list, obj, existing_obj):
+                    obj.pk = existing_obj['pk']
+                    update_list.append(obj)
+                else:
+                    od_response.total_untouched += 1
+            else:
+                create_list.append({'data': obj_dict})
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_updated += len(update_list)
+
+        if len(create_list) > 0:
+            Food.load_bulk(create_list, None)
+            od_response.total_created += len(create_list)
+
+        # --------------- PROPERTY STUFF -----------------------
+        model_type = Property
+        field_list = ['property_type_id', 'property_amount', 'open_data_food_slug']
+
+        existing_data_slugs = {}
+        existing_data_property_types = {}
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_food_slug']] = obj
+            existing_data_property_types[obj['property_type_id']] = obj
+
+        update_list = []
+        create_list = []
+
+        self._update_slug_cache(Food, 'food')
+
+        for k in list(self.data['food'].keys()):
+            for fp in self.data['food'][k]['properties']['type_values']:
+                obj = model_type(
+                    property_type_id=self.slug_id_cache['property'][fp['property_type']],
+                    property_amount=fp['property_value'],
+                    open_data_food_slug=k,
+                    space=self.request.space,
+                )
+
+                if obj.open_data_food_slug in existing_data_slugs and obj.property_type_id in existing_data_property_types and existing_data_slugs[obj.open_data_food_slug] == existing_data_property_types[obj.property_type_id]:
+                    existing_obj = existing_data_slugs[obj.open_data_food_slug]
+
+                    if not self._is_obj_identical(field_list, obj, existing_obj):
+                        obj.pk = existing_obj['pk']
+                        update_list.append(obj)
+                else:
+                    create_list.append(obj)
+
+        if self.update_existing and len(update_list) > 0:
+            model_type.objects.bulk_update(update_list, field_list)
+
+        if len(create_list) > 0:
+            model_type.objects.bulk_create(create_list, ignore_conflicts=True, unique_fields=('space', 'open_data_food_slug', 'property_type',))
+
+        linked_properties = list(FoodProperty.objects.filter(food__space=self.request.space).values_list('property_id', flat=True).all())
+
+        property_food_relation_list = []
+        for p in model_type.objects.filter(space=self.request.space, open_data_food_slug__isnull=False).values_list('open_data_food_slug', 'id', ):
+            if p[1] == 147:
+                pass
+            # slug_id_cache should always exist, don't create relations for already linked properties (ignore_conflicts would do that as well but this is more performant)
+            if p[0] in self.slug_id_cache['food'] and p[1] not in linked_properties:
+                property_food_relation_list.append(Food.properties.through(food_id=self.slug_id_cache['food'][p[0]], property_id=p[1]))
+
+        FoodProperty.objects.bulk_create(property_food_relation_list, unique_fields=('food_id', 'property_id',))
+
+        return od_response
+
+    def import_conversion(self):
+        od_response = OpenDataImportResponse()
+        datatype = 'conversion'
+        model_type = UnitConversion
+        field_list = ['base_amount', 'base_unit_id', 'converted_amount', 'converted_unit_id', 'food_id', 'open_data_slug']
+
+        self._update_slug_cache(Food, 'food')
+        self._update_slug_cache(Unit, 'unit')
+
+        existing_data_slugs = {}
+        existing_data_foods = defaultdict(list)
+        for obj in model_type.objects.filter(space=self.request.space).values('pk', *field_list):
+            existing_data_slugs[obj['open_data_slug']] = obj
+            existing_data_foods[obj['food_id']].append(obj)
+
+        update_list = []
+        create_list = []
+
+        for k in list(self.data[datatype].keys()):
+            # try catch here because sometimes key "k" is not set for the food cache
+            try:
+                obj = model_type(
+                    base_amount=Decimal(self.data[datatype][k]['base_amount']),
+                    base_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['base_unit']],
+                    converted_amount=Decimal(self.data[datatype][k]['converted_amount']),
+                    converted_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['converted_unit']],
+                    food_id=self.slug_id_cache['food'][self.data[datatype][k]['food']],
+                    open_data_slug=k,
+                    space=self.request.space,
+                    created_by_id=self.request.user.id,
+                )
+
+                if obj.open_data_slug in existing_data_slugs:
+                    existing_obj = existing_data_slugs[obj.open_data_slug]
+                    if not self._is_obj_identical(field_list, obj, existing_obj):
+                        obj.pk = existing_obj['pk']
+                        update_list.append(obj)
+                    else:
+                        od_response.total_untouched += 1
+                else:
+                    matching_existing_found = False
+                    if obj.food_id in existing_data_foods:
+                        for edf in existing_data_foods[obj.food_id]:
+                            if obj.base_unit_id == edf['base_unit_id'] and obj.converted_unit_id == edf['converted_unit_id']:
+                                matching_existing_found = True
+                                if not self._is_obj_identical(field_list, obj, edf):
+                                    obj.pk = edf['pk']
+                                    update_list.append(obj)
+                                else:
+                                    od_response.total_untouched += 1
+                    if not matching_existing_found:
+                        create_list.append(obj)
+            except KeyError as e:
+                traceback.print_exc()
+                od_response.total_errored += 1
+                print(self.data[datatype][k]['food'] + ' is not in self.slug_id_cache["food"]')
+
+        if self.update_existing and len(update_list) > 0:
+            od_response.total_updated = model_type.objects.bulk_update(update_list, field_list)
+            od_response.total_errored += len(update_list) - od_response.total_updated
+
+        if len(create_list) > 0:
+            objs_created = model_type.objects.bulk_create(create_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug'))
+            od_response.total_created = len(objs_created)
+            od_response.total_errored += len(create_list) - od_response.total_created
+
+        return od_response

cookbook/helper/permission_helper.py

@@ -1,19 +1,20 @@
-"""
-Source: https://djangosnippets.org/snippets/1703/
-"""
-from django.conf import settings
-from django.core.cache import caches
+import inspect
 
-from cookbook.models import ShareLink
+from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.decorators import user_passes_test
-from django.core.exceptions import ValidationError
+from django.core.cache import cache
+from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.http import HttpResponseRedirect
 from django.urls import reverse, reverse_lazy
 from django.utils.translation import gettext as _
+from oauth2_provider.contrib.rest_framework import TokenHasReadWriteScope, TokenHasScope
+from oauth2_provider.models import AccessToken
 from rest_framework import permissions
 from rest_framework.permissions import SAFE_METHODS
 
+from cookbook.models import Recipe, ShareLink, UserSpace
+
 
 def get_allowed_groups(groups_required):
     """
@@ -30,11 +31,12 @@ def get_allowed_groups(groups_required):
     return groups_allowed
 
 
-def has_group_permission(user, groups):
+def has_group_permission(user, groups, no_cache=False):
     """
     Tests if a given user is member of a certain group (or any higher group)
     Superusers always bypass permission checks.
-    Unauthenticated users cant be member of any group thus always return false.
+    Unauthenticated users can't be member of any group thus always return false.
+    :param no_cache: (optional) do not return cached results, always check agains DB
     :param user: django auth user object
     :param groups: list or tuple of groups the user should be checked for
     :return: True if user is in allowed groups, false otherwise
@@ -42,10 +44,23 @@ def has_group_permission(user, groups):
     if not user.is_authenticated:
         return False
     groups_allowed = get_allowed_groups(groups)
+
+    CACHE_KEY = hash((inspect.stack()[0][3], (user.pk, user.username, user.email), groups_allowed))
+    if not no_cache:
+        cached_result = cache.get(CACHE_KEY, default=None)
+        if cached_result is not None:
+            return cached_result
+
+    result = False
     if user.is_authenticated:
-        if bool(user.groups.filter(name__in=groups_allowed)):
-            return True
-    return False
+        if user_space := user.userspace_set.filter(active=True):
+            if len(user_space) != 1:
+                result = False  # do not allow any group permission if more than one space is active, needs to be changed when simultaneous multi-space-tenancy is added
+            elif bool(user_space.first().groups.filter(name__in=groups_allowed)):
+                result = True
+
+    cache.set(CACHE_KEY, result, timeout=10)
+    return result
 
 
 def is_object_owner(user, obj):
@@ -53,7 +68,6 @@ def is_object_owner(user, obj):
     Tests if a given user is the owner of a given object
     test performed by checking user against the objects user
     and create_by field (if exists)
-    superusers bypass all checks, unauthenticated users cannot own anything
     :param user django auth user object
     :param obj any object that should be tested
     :return: true if user is owner of object, false otherwise
@@ -61,7 +75,22 @@ def is_object_owner(user, obj):
     if not user.is_authenticated:
         return False
     try:
-        return obj.get_owner() == user
+        return obj.get_owner() == 'orphan' or obj.get_owner() == user
+    except Exception:
+        return False
+
+
+def is_space_owner(user, obj):
+    """
+    Tests if a given user is the owner the space of a given object
+    :param user django auth user object
+    :param obj any object that should be tested
+    :return: true if user is owner of the objects space, false otherwise
+    """
+    if not user.is_authenticated:
+        return False
+    try:
+        return obj.get_space().get_owner() == user
     except Exception:
         return False
 
@@ -70,7 +99,6 @@ def is_object_shared(user, obj):
     """
     Tests if a given user is shared for a given object
     test performed by checking user against the objects shared table
-    superusers bypass all checks, unauthenticated users cannot own anything
     :param user django auth user object
     :param obj any object that should be tested
     :return: true if user is shared for object, false otherwise
@@ -91,15 +119,15 @@ def share_link_valid(recipe, share):
     """
     try:
         CACHE_KEY = f'recipe_share_{recipe.pk}_{share}'
-        if c := caches['default'].get(CACHE_KEY, False):
+        if c := cache.get(CACHE_KEY, False):
             return c
 
         if link := ShareLink.objects.filter(recipe=recipe, uuid=share, abuse_blocked=False).first():
-            if 0 < settings.SHARING_LIMIT < link.request_count:
+            if 0 < settings.SHARING_LIMIT < link.request_count and not link.space.no_sharing_limit:
                 return False
             link.request_count += 1
             link.save()
-            caches['default'].set(CACHE_KEY, True, timeout=3)
+            cache.set(CACHE_KEY, True, timeout=3)
             return True
         return False
     except ValidationError:
@@ -166,7 +194,7 @@ class OwnerRequiredMixin(object):
 
         try:
             obj = self.get_object()
-            if obj.get_space() != request.space:
+            if not request.user.userspace.filter(space=obj.get_space()).exists():
                 messages.add_message(request, messages.ERROR,
                                      _('You do not have the required permissions to view this page!'))
                 return HttpResponseRedirect(reverse_lazy('index'))
@@ -184,7 +212,7 @@ class CustomIsOwner(permissions.BasePermission):
     verifies user has ownership over object
     (either user or created_by or user is request user)
     """
-    message = _('You cannot interact with this object as it is not owned by you!')  # noqa: E501
+    message = _('You cannot interact with this object as it is not owned by you!')
 
     def has_permission(self, request, view):
         return request.user.is_authenticated
@@ -193,6 +221,28 @@ class CustomIsOwner(permissions.BasePermission):
         return is_object_owner(request.user, obj)
 
 
+class CustomIsOwnerReadOnly(CustomIsOwner):
+    def has_permission(self, request, view):
+        return super().has_permission(request, view) and request.method in SAFE_METHODS
+
+    def has_object_permission(self, request, view, obj):
+        return super().has_object_permission(request, view) and request.method in SAFE_METHODS
+
+
+class CustomIsSpaceOwner(permissions.BasePermission):
+    """
+    Custom permission class for django rest framework views
+    verifies if the user is the owner of the space the object belongs to
+    """
+    message = _('You cannot interact with this object as it is not owned by you!')
+
+    def has_permission(self, request, view):
+        return request.user.is_authenticated and request.space.created_by == request.user
+
+    def has_object_permission(self, request, view, obj):
+        return is_space_owner(request.user, obj)
+
+
 # TODO function duplicate/too similar name
 class CustomIsShared(permissions.BasePermission):
     """
@@ -259,3 +309,134 @@ class CustomIsShare(permissions.BasePermission):
         if share:
             return share_link_valid(obj, share)
         return False
+
+
+class CustomRecipePermission(permissions.BasePermission):
+    """
+    Custom permission class for recipe api endpoint
+    """
+    message = _('You do not have the required permissions to view this page!')
+
+    def has_permission(self, request, view):  # user is either at least a guest or a share link is given and the request is safe
+        share = request.query_params.get('share', None)
+        return ((has_group_permission(request.user, ['guest']) and request.method in SAFE_METHODS) or has_group_permission(
+            request.user, ['user'])) or (share and request.method in SAFE_METHODS and 'pk' in view.kwargs)
+
+    def has_object_permission(self, request, view, obj):
+        share = request.query_params.get('share', None)
+        if share:
+            return share_link_valid(obj, share)
+        else:
+            if obj.private:
+                return ((obj.created_by == request.user) or (request.user in obj.shared.all())) and obj.space == request.space
+            else:
+                return ((has_group_permission(request.user, ['guest']) and request.method in SAFE_METHODS)
+                        or has_group_permission(request.user, ['user'])) and obj.space == request.space
+
+
+class CustomUserPermission(permissions.BasePermission):
+    """
+    Custom permission class for user api endpoint
+    """
+    message = _('You do not have the required permissions to view this page!')
+
+    def has_permission(self, request, view):  # a space filtered user list is visible for everyone
+        return has_group_permission(request.user, ['guest'])
+
+    def has_object_permission(self, request, view, obj):  # object write permissions are only available for user
+        if request.method in SAFE_METHODS and 'pk' in view.kwargs and has_group_permission(request.user, ['guest']) and request.space in obj.userspace_set.all():
+            return True
+        elif request.user == obj:
+            return True
+        else:
+            return False
+
+
+class CustomTokenHasScope(TokenHasScope):
+    """
+    Custom implementation of Django OAuth Toolkit TokenHasScope class
+    Only difference: if any other authentication method except OAuth2Authentication is used the scope check is ignored
+    IMPORTANT: do not use this class without any other permission class as it will not check anything besides token scopes
+    """
+
+    def has_permission(self, request, view):
+        if isinstance(request.auth, AccessToken):
+            return super().has_permission(request, view)
+        else:
+            return request.user.is_authenticated
+
+
+class CustomTokenHasReadWriteScope(TokenHasReadWriteScope):
+    """
+    Custom implementation of Django OAuth Toolkit TokenHasReadWriteScope class
+    Only difference: if any other authentication method except OAuth2Authentication is used the scope check is ignored
+    IMPORTANT: do not use this class without any other permission class as it will not check anything besides token scopes
+    """
+
+    def has_permission(self, request, view):
+        if isinstance(request.auth, AccessToken):
+            return super().has_permission(request, view)
+        else:
+            return True
+
+
+def above_space_limit(space):  # TODO add file storage limit
+    """
+    Test if the space has reached any limit (e.g. max recipes, users, ..)
+    :param space: Space to test for limits
+    :return: Tuple (True if above or equal any limit else false, message)
+    """
+    r_limit, r_msg = above_space_recipe_limit(space)
+    u_limit, u_msg = above_space_user_limit(space)
+    return r_limit or u_limit, (r_msg + ' ' + u_msg).strip()
+
+
+def above_space_recipe_limit(space):
+    """
+    Test if a space has reached its recipe limit
+    :param space: Space to test for limits
+    :return: Tuple (True if above or equal limit else false, message)
+    """
+    limit = space.max_recipes != 0 and Recipe.objects.filter(space=space).count() >= space.max_recipes
+    if limit:
+        return True, _('You have reached the maximum number of recipes for your space.')
+    return False, ''
+
+
+def above_space_user_limit(space):
+    """
+    Test if a space has reached its user limit
+    :param space: Space to test for limits
+    :return: Tuple (True if above or equal limit else false, message)
+    """
+    limit = space.max_users != 0 and UserSpace.objects.filter(space=space).count() > space.max_users
+    if limit:
+        return True, _('You have more users than allowed in your space.')
+    return False, ''
+
+
+def switch_user_active_space(user, space):
+    """
+    Switch the currently active space of a user by setting all spaces to inactive and activating the one passed
+    :param user: user to change active space for
+    :param space: space to activate user for
+    :return user space object or none if not found/no permission
+    """
+    try:
+        us = UserSpace.objects.get(space=space, user=user)
+        if not us.active:
+            UserSpace.objects.filter(user=user).update(active=False)
+            us.active = True
+            us.save()
+            return us
+        else:
+            return us
+    except ObjectDoesNotExist:
+        return None
+
+
+class IsReadOnlyDRF(permissions.BasePermission):
+    message = 'You cannot interact with this object as it is not owned by you!'
+
+    def has_permission(self, request, view):
+        return request.method in SAFE_METHODS

cookbook/helper/property_helper.py

@@ -0,0 +1,78 @@
+from django.core.cache import caches
+
+from cookbook.helper.cache_helper import CacheHelper
+from cookbook.helper.unit_conversion_helper import UnitConversionHelper
+from cookbook.models import PropertyType
+
+
+class FoodPropertyHelper:
+    space = None
+
+    def __init__(self, space):
+        """
+        Helper to perform food property calculations
+        :param space: space to limit scope to
+        """
+        self.space = space
+
+    def calculate_recipe_properties(self, recipe):
+        """
+        Calculate all food properties for a given recipe.
+        :param recipe: recipe to calculate properties for
+        :return: dict of with property keys and total/food values for each property available
+        """
+        ingredients = []
+        computed_properties = {}
+
+        for s in recipe.steps.all():
+            ingredients += s.ingredients.all()
+
+        property_types = caches['default'].get(CacheHelper(self.space).PROPERTY_TYPE_CACHE_KEY, None)
+
+        if not property_types:
+            property_types = PropertyType.objects.filter(space=self.space).all()
+            # cache is cleared on property type save signal so long duration is fine
+            caches['default'].set(CacheHelper(self.space).PROPERTY_TYPE_CACHE_KEY, property_types, 60 * 60)
+
+        for fpt in property_types:
+            computed_properties[fpt.id] = {'id': fpt.id, 'name': fpt.name, 'description': fpt.description,
+                                           'unit': fpt.unit, 'order': fpt.order, 'food_values': {}, 'total_value': 0, 'missing_value': False}
+
+        uch = UnitConversionHelper(self.space)
+
+        for i in ingredients:
+            if i.food is not None:
+                conversions = uch.get_conversions(i)
+                for pt in property_types:
+                    found_property = False
+                    if i.food.properties_food_amount == 0 or i.food.properties_food_unit is None:  # if food is configured incorrectly
+                        computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': None}
+                        computed_properties[pt.id]['missing_value'] = True
+                    else:
+                        for p in i.food.properties.all():
+                            if p.property_type == pt and p.property_amount is not None:
+                                for c in conversions:
+                                    if c.unit == i.food.properties_food_unit:
+                                        found_property = True
+                                        computed_properties[pt.id]['total_value'] += (c.amount / i.food.properties_food_amount) * p.property_amount
+                                        computed_properties[pt.id]['food_values'] = self.add_or_create(
+                                            computed_properties[p.property_type.id]['food_values'], c.food.id, (c.amount / i.food.properties_food_amount) * p.property_amount, c.food)
+                    if not found_property:
+                        if i.amount == 0:  # don't count ingredients without an amount as missing
+                            computed_properties[pt.id]['missing_value'] = computed_properties[pt.id]['missing_value'] or False  # don't override if another food was already missing
+                            computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': 0}
+                        else:
+                            computed_properties[pt.id]['missing_value'] = True
+                            computed_properties[pt.id]['food_values'][i.food.id] = {'id': i.food.id, 'food': i.food.name, 'value': None}
+
+        return computed_properties
+
+    # small dict helper to add to existing key or create new, probably a better way of doing this
+    # TODO move to central helper ? --> use defaultdict
+    @staticmethod
+    def add_or_create(d, key, value, food):
+        if key in d and d[key]['value']:
+            d[key]['value'] += value
+        else:
+            d[key] = {'id': food.id, 'food': food.name, 'value': value}
+        return d

cookbook/helper/recipe_html_import.py

@@ -1,193 +0,0 @@
-import json
-import re
-
-from bs4 import BeautifulSoup
-from bs4.element import Tag
-from cookbook.helper import recipe_url_import as helper
-from cookbook.helper.scrapers.scrapers import text_scraper
-from json import JSONDecodeError
-from recipe_scrapers._utils import get_host_name, normalize_string
-from urllib.parse import unquote
-
-
-def get_recipe_from_source(text, url, request):
-    def build_node(k, v):
-        if isinstance(v, dict):
-            node = {
-                'name': k,
-                'value': k,
-                'children': get_children_dict(v)
-            }
-        elif isinstance(v, list):
-            node = {
-                'name': k,
-                'value': k,
-                'children': get_children_list(v)
-            }
-        else:
-            node = {
-                'name': k + ": " + normalize_string(str(v)),
-                'value': normalize_string(str(v))
-            }
-        return node
-
-    def get_children_dict(children):
-        kid_list = []
-        for k, v in children.items():
-            kid_list.append(build_node(k, v))
-        return kid_list
-
-    def get_children_list(children):
-        kid_list = []
-        for kid in children:
-            if type(kid) == list:
-                node = {
-                    'name': "unknown list",
-                    'value': "unknown list",
-                    'children': get_children_list(kid)
-                }
-                kid_list.append(node)
-            elif type(kid) == dict:
-                for k, v in kid.items():
-                    kid_list.append(build_node(k, v))
-            else:
-                kid_list.append({
-                    'name': normalize_string(str(kid)),
-                    'value': normalize_string(str(kid))
-                })
-        return kid_list
-
-    recipe_json = {
-                'name': '',
-                'url': '',
-                'description': '',
-                'image': '',
-                'keywords': [],
-                'recipeIngredient': [],
-                'recipeInstructions': '',
-                'servings': '',
-                'prepTime': '',
-                'cookTime': ''
-    }
-    recipe_tree = []
-    parse_list = []
-    html_data = []
-    images = []
-    text = unquote(text)
-
-    try:
-        parse_list.append(remove_graph(json.loads(text)))
-        if not url and 'url' in parse_list[0]:
-            url = parse_list[0]['url']
-        scrape = text_scraper("<script type='application/ld+json'>" + text + "</script>", url=url)
-
-    except JSONDecodeError:
-        soup = BeautifulSoup(text, "html.parser")
-        html_data = get_from_html(soup)
-        images += get_images_from_source(soup, url)
-        for el in soup.find_all('script', type='application/ld+json'):
-            el = remove_graph(el)
-            if not url and 'url' in el:
-                url = el['url']
-            if type(el) == list:
-                for le in el:
-                    parse_list.append(le)
-            elif type(el) == dict:
-                parse_list.append(el)
-        for el in soup.find_all(type='application/json'):
-            el = remove_graph(el)
-            if type(el) == list:
-                for le in el:
-                    parse_list.append(le)
-            elif type(el) == dict:
-                parse_list.append(el)
-        scrape = text_scraper(text, url=url)
-
-    recipe_json = helper.get_from_scraper(scrape, request)
-
-    for el in parse_list:
-        temp_tree = []
-        if isinstance(el, Tag):
-            try:
-                el = json.loads(el.string)
-            except TypeError:
-                continue
-
-        for k, v in el.items():
-            if isinstance(v, dict):
-                node = {
-                    'name': k,
-                    'value': k,
-                    'children': get_children_dict(v)
-                }
-            elif isinstance(v, list):
-                node = {
-                    'name': k,
-                    'value': k,
-                    'children': get_children_list(v)
-                }
-            else:
-                node = {
-                    'name': k + ": " + normalize_string(str(v)),
-                    'value': normalize_string(str(v))
-                }
-            temp_tree.append(node)
-
-        if '@type' in el and el['@type'] == 'Recipe':
-            recipe_tree += [{'name': 'ld+json', 'children': temp_tree}]
-        else:
-            recipe_tree += [{'name': 'json', 'children': temp_tree}]
-
-    return recipe_json, recipe_tree, html_data, images
-
-
-def get_from_html(soup):
-    INVISIBLE_ELEMS = ('style', 'script', 'head', 'title')
-    html = []
-    for s in soup.strings:
-        if ((s.parent.name not in INVISIBLE_ELEMS) and (len(s.strip()) > 0)):
-            html.append(s)
-    return html
-
-
-def get_images_from_source(soup, url):
-    sources = ['src', 'srcset', 'data-src']
-    images = []
-    img_tags = soup.find_all('img')
-    if url:
-        site = get_host_name(url)
-        prot = url.split(':')[0]
-
-    urls = []
-    for img in img_tags:
-        for src in sources:
-            try:
-                urls.append(img[src])
-            except KeyError:
-                pass
-
-    for u in urls:
-        u = u.split('?')[0]
-        filename = re.search(r'/([\w_-]+[.](jpg|jpeg|gif|png))$', u)
-        if filename:
-            if (('http' not in u) and (url)):
-                # sometimes an image source can be relative
-                # if it is provide the base url
-                u = '{}://{}{}'.format(prot, site, u)
-            if 'http' in u:
-                images.append(u)
-    return images
-
-
-def remove_graph(el):
-    # recipes type might be wrapped in @graph type
-    if isinstance(el, Tag):
-        try:
-            el = json.loads(el.string)
-            if '@graph' in el:
-                for x in el['@graph']:
-                    if '@type' in x and x['@type'] == 'Recipe':
-                        el = x
-        except TypeError:
-            pass
-    return el

cookbook/helper/recipe_search.py

@@ -1,397 +1,581 @@
-from collections import Counter
-from datetime import timedelta
+import json
+from datetime import date, timedelta
 
-from recipes import settings
-from django.contrib.postgres.search import (
-    SearchQuery, SearchRank, TrigramSimilarity
-)
-from django.core.cache import caches
-from django.db.models import Avg, Case, Count, Func, Max, Q, Subquery, Value, When
-from django.db.models.functions import Coalesce
+from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector, TrigramSimilarity
+from django.core.cache import cache
+from django.db.models import Avg, Case, Count, Exists, F, Max, OuterRef, Q, Subquery, Value, When
+from django.db.models.functions import Coalesce, Lower, Substr
 from django.utils import timezone, translation
 
+from cookbook.helper.HelperFunctions import Round, str2bool
 from cookbook.managers import DICTIONARY
-from cookbook.models import Food, Keyword, ViewLog, SearchPreference
+from cookbook.models import (CookLog, CustomFilter, Food, Keyword, Recipe, SearchFields,
+                             SearchPreference, ViewLog)
+from recipes import settings
 
 
-class Round(Func):
-    function = 'ROUND'
-    template = '%(function)s(%(expressions)s, 0)'
-
-
-def str2bool(v):
-    if type(v) == bool:
-        return v
-    else:
-        return v.lower() in ("yes", "true", "1")
-
-
-# TODO create extensive tests to make sure ORs ANDs and various filters, sorting, etc work as expected
 # TODO consider creating a simpleListRecipe API that only includes minimum of recipe info and minimal filtering
-def search_recipes(request, queryset, params):
-    if request.user.is_authenticated:
-        search_prefs = request.user.searchpreference
-    else:
-        search_prefs = SearchPreference()
-    search_string = params.get('query', '').strip()
-    search_rating = int(params.get('rating', 0))
-    search_keywords = params.getlist('keywords', [])
-    search_foods = params.getlist('foods', [])
-    search_books = params.getlist('books', [])
-    search_units = params.get('units', None)
+class RecipeSearch():
+    _postgres = settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql'
 
-    # TODO I think default behavior should be 'AND' which is how most sites operate with facet/filters based on results
-    search_keywords_or = str2bool(params.get('keywords_or', True))
-    search_foods_or = str2bool(params.get('foods_or', True))
-    search_books_or = str2bool(params.get('books_or', True))
-
-    search_internal = str2bool(params.get('internal', False))
-    search_random = str2bool(params.get('random', False))
-    search_new = str2bool(params.get('new', False))
-    search_last_viewed = int(params.get('last_viewed', 0))
-    orderby = []
-
-    # only sort by recent not otherwise filtering/sorting
-    if search_last_viewed > 0:
-        last_viewed_recipes = ViewLog.objects.filter(
-            created_by=request.user, space=request.space,
-            created_at__gte=timezone.now() - timedelta(days=14)  # TODO make recent days a setting
-        ).order_by('-pk').values_list('recipe__pk', flat=True)
-        last_viewed_recipes = list(dict.fromkeys(last_viewed_recipes))[:search_last_viewed]  # removes duplicates from list prior to slicing
-
-        # return queryset.annotate(last_view=Max('viewlog__pk')).annotate(new=Case(When(pk__in=last_viewed_recipes, then=('last_view')), default=Value(0))).filter(new__gt=0).order_by('-new')
-        # queryset that only annotates most recent view (higher pk = lastest view)
-        queryset = queryset.annotate(recent=Coalesce(Max('viewlog__pk'), Value(0)))
-        orderby += ['-recent']
-
-    # TODO create setting for default ordering - most cooked, rating,
-    # TODO create options for live sorting
-    # TODO make days of new recipe a setting
-    if search_new:
-        queryset = (
-            queryset.annotate(new_recipe=Case(
-                When(created_at__gte=(timezone.now() - timedelta(days=7)), then=('pk')), default=Value(0), ))
-        )
-        # only sort by new recipes if not otherwise filtering/sorting
-        orderby += ['-new_recipe']
-
-    search_type = search_prefs.search or 'plain'
-    if len(search_string) > 0:
-        unaccent_include = search_prefs.unaccent.values_list('field', flat=True)
-
-        icontains_include = [x + '__unaccent' if x in unaccent_include else x for x in search_prefs.icontains.values_list('field', flat=True)]
-        istartswith_include = [x + '__unaccent' if x in unaccent_include else x for x in search_prefs.istartswith.values_list('field', flat=True)]
-        trigram_include = [x + '__unaccent' if x in unaccent_include else x for x in search_prefs.trigram.values_list('field', flat=True)]
-        fulltext_include = search_prefs.fulltext.values_list('field', flat=True)  # fulltext doesn't use field name directly
-
-        # if no filters are configured use name__icontains as default
-        if len(icontains_include) + len(istartswith_include) + len(trigram_include) + len(fulltext_include) == 0:
-            filters = [Q(**{"name__icontains": search_string})]
-        else:
-            filters = []
-
-        # dynamically build array of filters that will be applied
-        for f in icontains_include:
-            filters += [Q(**{"%s__icontains" % f: search_string})]
-
-        for f in istartswith_include:
-            filters += [Q(**{"%s__istartswith" % f: search_string})]
-
-        if settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2', 'django.db.backends.postgresql']:
-            language = DICTIONARY.get(translation.get_language(), 'simple')
-            # django full text search https://docs.djangoproject.com/en/3.2/ref/contrib/postgres/search/#searchquery
-            # TODO can options install this extension to further enhance search query language https://github.com/caub/pg-tsquery
-            # trigram breaks full text search 'websearch' and 'raw' capabilities and will be ignored if those methods are chosen
-            if search_type in ['websearch', 'raw']:
-                search_trigram = False
+    def __init__(self, request, **params):
+        self._request = request
+        self._queryset = None
+        if f := params.get('filter', None):
+            custom_filter = (
+                CustomFilter.objects.filter(id=f, space=self._request.space)
+                .filter(Q(created_by=self._request.user) | Q(shared=self._request.user) | Q(recipebook__shared=self._request.user))
+                .first()
+            )
+            if custom_filter:
+                self._params = {**json.loads(custom_filter.search)}
+                self._original_params = {**(params or {})}
+                # json.loads casts rating as an integer, expecting string
+                if isinstance(self._params.get('rating', None), int):
+                    self._params['rating'] = str(self._params['rating'])
             else:
-                search_trigram = True
-            search_query = SearchQuery(
-                search_string,
-                search_type=search_type,
-                config=language,
-            )
-
-            # iterate through fields to use in trigrams generating a single trigram
-            if search_trigram and len(trigram_include) > 0:
-                trigram = None
-                for f in trigram_include:
-                    if trigram:
-                        trigram += TrigramSimilarity(f, search_string)
-                    else:
-                        trigram = TrigramSimilarity(f, search_string)
-                queryset = queryset.annotate(similarity=trigram)
-                filters += [Q(similarity__gt=search_prefs.trigram_threshold)]
-
-            if 'name' in fulltext_include:
-                filters += [Q(name_search_vector=search_query)]
-            if 'description' in fulltext_include:
-                filters += [Q(desc_search_vector=search_query)]
-            if 'instructions' in fulltext_include:
-                filters += [Q(steps__search_vector=search_query)]
-            if 'keywords' in fulltext_include:
-                filters += [Q(keywords__in=Subquery(Keyword.objects.filter(name__search=search_query).values_list('id', flat=True)))]
-            if 'foods' in fulltext_include:
-                filters += [Q(steps__ingredients__food__in=Subquery(Food.objects.filter(name__search=search_query).values_list('id', flat=True)))]
-            query_filter = None
-            for f in filters:
-                if query_filter:
-                    query_filter |= f
-                else:
-                    query_filter = f
-
-            # TODO add order by user settings - only do search rank and annotation if rank order is configured
-            search_rank = (
-                    SearchRank('name_search_vector', search_query, cover_density=True)
-                    + SearchRank('desc_search_vector', search_query, cover_density=True)
-                    + SearchRank('steps__search_vector', search_query, cover_density=True)
-            )
-            queryset = queryset.filter(query_filter).annotate(rank=search_rank)
-            orderby += ['-rank']
+                self._params = {**(params or {})}
         else:
-            queryset = queryset.filter(name__icontains=search_string)
-
-    if len(search_keywords) > 0:
-        if search_keywords_or:
-            # TODO creating setting to include descendants of keywords a setting
-            # for kw in Keyword.objects.filter(pk__in=search_keywords):
-            #     search_keywords += list(kw.get_descendants().values_list('pk', flat=True))
-            queryset = queryset.filter(keywords__id__in=search_keywords)
+            self._params = {**(params or {})}
+        if self._request.user.is_authenticated:
+            CACHE_KEY = f'search_pref_{request.user.id}'
+            cached_result = cache.get(CACHE_KEY, default=None)
+            if cached_result is not None:
+                self._search_prefs = cached_result
+            else:
+                self._search_prefs = request.user.searchpreference
+            cache.set(CACHE_KEY, self._search_prefs, timeout=10)
         else:
-            # when performing an 'and' search returned recipes should include a parent OR any of its descedants
-            # AND other keywords selected so filters are appended using keyword__id__in the list of keywords and descendants
-            for kw in Keyword.objects.filter(pk__in=search_keywords):
-                queryset = queryset.filter(keywords__id__in=list(kw.get_descendants_and_self().values_list('pk', flat=True)))
-
-    if len(search_foods) > 0:
-        if search_foods_or:
-            # TODO creating setting to include descendants of food a setting
-            queryset = queryset.filter(steps__ingredients__food__id__in=search_foods)
-        else:
-            # when performing an 'and' search returned recipes should include a parent OR any of its descedants
-            # AND other foods selected so filters are appended using steps__ingredients__food__id__in the list of foods and descendants
-            for fd in Food.objects.filter(pk__in=search_foods):
-                queryset = queryset.filter(steps__ingredients__food__id__in=list(fd.get_descendants_and_self().values_list('pk', flat=True)))
-
-    if len(search_books) > 0:
-        if search_books_or:
-            queryset = queryset.filter(recipebookentry__book__id__in=search_books)
-        else:
-            for k in search_books:
-                queryset = queryset.filter(recipebookentry__book__id=k)
-
-    if search_rating:
-        queryset = queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=request.user, then='cooklog__rating'), default=Value(0)))))
-        if search_rating == -1:
-            queryset = queryset.filter(rating=0)
-        else:
-            queryset = queryset.filter(rating__gte=search_rating)
-
-    # probably only useful in Unit list view, so keeping it simple
-    if search_units:
-        queryset = queryset.filter(steps__ingredients__unit__id=search_units)
-
-    if search_internal:
-        queryset = queryset.filter(internal=True)
-
-    queryset = queryset.distinct()
-
-    if search_random:
-        queryset = queryset.order_by("?")
-    else:
-        queryset = queryset.order_by(*orderby)
-    return queryset
-
-
-def get_facet(qs=None, request=None, use_cache=True, hash_key=None):
-    """
-    Gets an annotated list from a queryset.
-    :param qs:
-
-        recipe queryset to build facets from
-
-    :param request:
-
-        the web request that contains the necessary query parameters
-
-    :param use_cache:
-
-        will find results in cache, if any, and return them or empty list.
-        will save the list of recipes IDs in the cache for future processing
-
-    :param hash_key:
-
-        the cache key of the recipe list to process
-        only evaluated if the use_cache parameter is false
-    """
-    facets = {}
-    recipe_list = []
-    cache_timeout = 600
-
-    if use_cache:
-        qs_hash = hash(frozenset(qs.values_list('pk')))
-        facets['cache_key'] = str(qs_hash)
-        SEARCH_CACHE_KEY = f"recipes_filter_{qs_hash}"
-        if c := caches['default'].get(SEARCH_CACHE_KEY, None):
-            facets['Keywords'] = c['Keywords'] or []
-            facets['Foods'] = c['Foods'] or []
-            facets['Books'] = c['Books'] or []
-            facets['Ratings'] = c['Ratings'] or []
-            facets['Recent'] = c['Recent'] or []
-        else:
-            facets['Keywords'] = []
-            facets['Foods'] = []
-            facets['Books'] = []
-            rating_qs = qs.annotate(rating=Round(Avg(Case(When(cooklog__created_by=request.user, then='cooklog__rating'), default=Value(0)))))
-            facets['Ratings'] = dict(Counter(r.rating for r in rating_qs))
-            facets['Recent'] = ViewLog.objects.filter(
-                created_by=request.user, space=request.space,
-                created_at__gte=timezone.now() - timedelta(days=14)  # TODO make days of recent recipe a setting
-            ).values_list('recipe__pk', flat=True)
-
-        cached_search = {
-            'recipe_list': list(qs.values_list('id', flat=True)),
-            'keyword_list': request.query_params.getlist('keywords', []),
-            'food_list': request.query_params.getlist('foods', []),
-            'book_list': request.query_params.getlist('book', []),
-            'search_keywords_or': str2bool(request.query_params.get('keywords_or', True)),
-            'search_foods_or': str2bool(request.query_params.get('foods_or', True)),
-            'search_books_or': str2bool(request.query_params.get('books_or', True)),
-            'space': request.space,
-            'Ratings': facets['Ratings'],
-            'Recent': facets['Recent'],
-            'Keywords': facets['Keywords'],
-            'Foods': facets['Foods'],
-            'Books': facets['Books']
+            self._search_prefs = SearchPreference()
+        self._string = self._params.get('query').strip(
+        ) if self._params.get('query', None) else None
+        self._rating = self._params.get('rating', None)
+        self._keywords = {
+            'or': self._params.get('keywords_or', None) or self._params.get('keywords', None),
+            'and': self._params.get('keywords_and', None),
+            'or_not': self._params.get('keywords_or_not', None),
+            'and_not': self._params.get('keywords_and_not', None)
         }
-        caches['default'].set(SEARCH_CACHE_KEY, cached_search, cache_timeout)
-        return facets
-
-    SEARCH_CACHE_KEY = f'recipes_filter_{hash_key}'
-    if c := caches['default'].get(SEARCH_CACHE_KEY, None):
-        recipe_list = c['recipe_list']
-        keyword_list = c['keyword_list']
-        food_list = c['food_list']
-        book_list = c['book_list']
-        search_keywords_or = c['search_keywords_or']
-        search_foods_or = c['search_foods_or']
-        search_books_or = c['search_books_or']
-    else:
-        return {}
-
-    # if using an OR search, will annotate all keywords, otherwise, just those that appear in results
-    if search_keywords_or:
-        keywords = Keyword.objects.filter(space=request.space).annotate(recipe_count=Count('recipe'))
-    else:
-        keywords = Keyword.objects.filter(recipe__in=recipe_list, space=request.space).annotate(recipe_count=Count('recipe'))
-    # custom django-tree function annotates a queryset to make building a tree easier.
-    # see https://django-treebeard.readthedocs.io/en/latest/api.html#treebeard.models.Node.get_annotated_list_qs for details
-    kw_a = annotated_qs(keywords, root=True, fill=True)
-
-    # # if using an OR search, will annotate all keywords, otherwise, just those that appear in results
-    if search_foods_or:
-        foods = Food.objects.filter(space=request.space).annotate(recipe_count=Count('ingredient'))
-    else:
-        foods = Food.objects.filter(ingredient__step__recipe__in=recipe_list, space=request.space).annotate(recipe_count=Count('ingredient'))
-    food_a = annotated_qs(foods, root=True, fill=True)
-
-    # TODO add rating facet
-    facets['Keywords'] = fill_annotated_parents(kw_a, keyword_list)
-    facets['Foods'] = fill_annotated_parents(food_a, food_list)
-    # TODO add book facet
-    facets['Books'] = []
-    c['Keywords'] = facets['Keywords']
-    c['Foods'] = facets['Foods']
-    c['Books'] = facets['Books']
-    caches['default'].set(SEARCH_CACHE_KEY, c, cache_timeout)
-    return facets
-
-
-def fill_annotated_parents(annotation, filters):
-    tree_list = []
-    parent = []
-    i = 0
-    level = -1
-    for r in annotation:
-        expand = False
-
-        annotation[i][1]['id'] = r[0].id
-        annotation[i][1]['name'] = r[0].name
-        annotation[i][1]['count'] = getattr(r[0], 'recipe_count', 0)
-        annotation[i][1]['isDefaultExpanded'] = False
-
-        if str(r[0].id) in filters:
-            expand = True
-        if r[1]['level'] < level:
-            parent = parent[:r[1]['level'] - level]
-            parent[-1] = i
-            level = r[1]['level']
-        elif r[1]['level'] > level:
-            parent.extend([i])
-            level = r[1]['level']
+        self._foods = {
+            'or': self._params.get('foods_or', None) or self._params.get('foods', None),
+            'and': self._params.get('foods_and', None),
+            'or_not': self._params.get('foods_or_not', None),
+            'and_not': self._params.get('foods_and_not', None)
+        }
+        self._books = {
+            'or': self._params.get('books_or', None) or self._params.get('books', None),
+            'and': self._params.get('books_and', None),
+            'or_not': self._params.get('books_or_not', None),
+            'and_not': self._params.get('books_and_not', None)
+        }
+        self._steps = self._params.get('steps', None)
+        self._units = self._params.get('units', None)
+        # TODO add created by
+        # TODO image exists
+        self._sort_order = self._params.get('sort_order', None)
+        self._internal = str2bool(self._params.get('internal', None))
+        self._random = str2bool(self._params.get('random', False))
+        self._new = str2bool(self._params.get('new', False))
+        self._num_recent = int(self._params.get('num_recent', 0))
+        self._include_children = str2bool(
+            self._params.get('include_children', None))
+        self._timescooked = self._params.get('timescooked', None)
+        self._cookedon = self._params.get('cookedon', None)
+        self._createdon = self._params.get('createdon', None)
+        self._updatedon = self._params.get('updatedon', None)
+        self._viewedon = self._params.get('viewedon', None)
+        self._makenow = self._params.get('makenow', None)
+        # this supports hidden feature to find recipes missing X ingredients
+        if isinstance(self._makenow, bool) and self._makenow == True:
+            self._makenow = 0
+        elif isinstance(self._makenow, str) and self._makenow in ["yes", "true"]:
+            self._makenow = 0
         else:
-            parent[-1] = i
-        j = 0
+            try:
+                self._makenow = int(self._makenow)
+            except (ValueError, TypeError):
+                self._makenow = None
 
-        while j < level:
-            # this causes some double counting when a recipe has both a child and an ancestor
-            annotation[parent[j]][1]['count'] += getattr(r[0], 'recipe_count', 0)
-            if expand:
-                annotation[parent[j]][1]['isDefaultExpanded'] = True
-            j += 1
-        if level == 0:
-            tree_list.append(annotation[i][1])
-        elif level > 0:
-            annotation[parent[level - 1]][1].setdefault('children', []).append(annotation[i][1])
-        i += 1
-    return tree_list
+        self._search_type = self._search_prefs.search or 'plain'
+        if self._string:
+            if self._postgres:
+                self._unaccent_include = self._search_prefs.unaccent.values_list('field', flat=True)
+            else:
+                self._unaccent_include = []
+            self._icontains_include = [x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.icontains.values_list('field', flat=True)]
+            self._istartswith_include = [x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.istartswith.values_list('field', flat=True)]
+            self._trigram_include = None
+            self._fulltext_include = None
+        self._trigram = False
+        if self._postgres and self._string:
+            self._language = DICTIONARY.get(translation.get_language(), 'simple')
+            self._trigram_include = [x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.trigram.values_list('field', flat=True)]
+            self._fulltext_include = self._search_prefs.fulltext.values_list('field', flat=True) or None
 
+            if self._search_type not in ['websearch', 'raw'] and self._trigram_include:
+                self._trigram = True
+            self.search_query = SearchQuery(
+                self._string,
+                search_type=self._search_type,
+                config=self._language,
+            )
+            self.search_rank = None
+        self.orderby = []
+        self._filters = None
+        self._fuzzy_match = None
 
-def annotated_qs(qs, root=False, fill=False):
-    """
-    Gets an annotated list from a queryset.
-    :param root:
+    def get_queryset(self, queryset):
+        self._queryset = queryset
+        self._queryset = self._queryset.prefetch_related('keywords')
 
-        Will backfill in annotation to include all parents to root node.
+        self._build_sort_order()
+        self._recently_viewed(num_recent=self._num_recent)
+        self._cooked_on_filter(cooked_date=self._cookedon)
+        self._created_on_filter(created_date=self._createdon)
+        self._updated_on_filter(updated_date=self._updatedon)
+        self._viewed_on_filter(viewed_date=self._viewedon)
+        self._favorite_recipes(times_cooked=self._timescooked)
+        self._new_recipes()
+        self.keyword_filters(**self._keywords)
+        self.food_filters(**self._foods)
+        self.book_filters(**self._books)
+        self.rating_filter(rating=self._rating)
+        self.internal_filter(internal=self._internal)
+        self.step_filters(steps=self._steps)
+        self.unit_filters(units=self._units)
+        self._makenow_filter(missing=self._makenow)
+        self.string_filters(string=self._string)
+        return self._queryset.filter(space=self._request.space).order_by(*self.orderby)
 
-    :param fill:
-        Will fill in gaps in annotation where nodes between children
-        and ancestors are not included in the queryset.
-    """
+    def _sort_includes(self, *args):
+        for x in args:
+            if x in self.orderby:
+                return True
+            elif '-' + x in self.orderby:
+                return True
+        return False
 
-    result, info = [], {}
-    start_depth, prev_depth = (None, None)
-    nodes_list = list(qs.values_list('pk', flat=True))
-    for node in qs.order_by('path'):
-        node_queue = [node]
-        while len(node_queue) > 0:
-            dirty = False
-            current_node = node_queue[-1]
-            depth = current_node.get_depth()
-            # TODO if node is at the wrong depth for some reason this fails
-            # either create a 'fix node' page, or automatically move the node to the root
-            parent_id = current_node.parent
-            if root and depth > 1 and parent_id not in nodes_list:
-                parent_id = current_node.parent
-                nodes_list.append(parent_id)
-                node_queue.append(current_node.__class__.objects.get(pk=parent_id))
-                dirty = True
+    def _build_sort_order(self):
+        if self._random:
+            self.orderby = ['?']
+        else:
+            order = []
+            # TODO add userpreference for default sort order and replace '-favorite'
+            default_order = ['name']
+            # recent and new_recipe are always first; they float a few recipes to the top
+            if self._num_recent:
+                order += ['-recent']
+            if self._new:
+                order += ['-new_recipe']
 
-            if fill and depth > 1 and prev_depth and depth > prev_depth and parent_id not in nodes_list:
-                nodes_list.append(parent_id)
-                node_queue.append(current_node.__class__.objects.get(pk=parent_id))
-                dirty = True
+            # if a sort order is provided by user - use that order
+            if self._sort_order:
+                if not isinstance(self._sort_order, list):
+                    order += [self._sort_order]
+                else:
+                    order += self._sort_order
+                if not self._postgres or not self._string:
+                    if 'score' in order:
+                        order.remove('score')
+                    if '-score' in order:
+                        order.remove('-score')
+            # if no sort order provided prioritize text search, followed by the default search
+            elif self._postgres and self._string and (self._trigram or self._fulltext_include):
+                order += ['-score', *default_order]
+            # otherwise sort by the remaining order_by attributes or favorite by default
+            else:
+                order += default_order
+            order[:] = [Lower('name').asc() if x ==
+                        'name' else x for x in order]
+            order[:] = [Lower('name').desc() if x ==
+                        '-name' else x for x in order]
+            self.orderby = order
 
-            if not dirty:
-                working_node = node_queue.pop()
-                if start_depth is None:
-                    start_depth = depth
-                open = (depth and (prev_depth is None or depth > prev_depth))
-                if prev_depth is not None and depth < prev_depth:
-                    info['close'] = list(range(0, prev_depth - depth))
-                info = {'open': open, 'close': [], 'level': depth - start_depth}
-                result.append((working_node, info,))
-                prev_depth = depth
-    if start_depth and start_depth > 0:
-        info['close'] = list(range(0, prev_depth - start_depth + 1))
-    return result
+    def string_filters(self, string=None):
+        if not string:
+            return
+
+        self.build_text_filters(self._string)
+        if self._postgres:
+            self.build_fulltext_filters(self._string)
+            self.build_trigram(self._string)
+
+        query_filter = Q()
+        if self._filters:
+            for f in self._filters:
+                query_filter |= f
+
+            # this creates duplicate records which can screw up other aggregates, see makenow for workaround
+            self._queryset = self._queryset.filter(query_filter).distinct()
+            if self._fulltext_include:
+                if self._fuzzy_match is None:
+                    self._queryset = self._queryset.annotate(score=Coalesce(Max(self.search_rank), 0.0))
+                else:
+                    self._queryset = self._queryset.annotate(rank=Coalesce(Max(self.search_rank), 0.0))
+
+            if self._fuzzy_match is not None:
+                simularity = self._fuzzy_match.filter(pk=OuterRef('pk')).values('simularity')
+                if not self._fulltext_include:
+                    self._queryset = self._queryset.annotate(score=Coalesce(Subquery(simularity), 0.0))
+                else:
+                    self._queryset = self._queryset.annotate(simularity=Coalesce(Subquery(simularity), 0.0))
+            if self._sort_includes('score') and self._fulltext_include and self._fuzzy_match is not None:
+                self._queryset = self._queryset.annotate(score=F('rank') + F('simularity'))
+        else:
+            query_filter = Q()
+            for f in [x + '__unaccent__iexact' if x in self._unaccent_include else x + '__iexact' for x in SearchFields.objects.all().values_list('field', flat=True)]:
+                query_filter |= Q(**{"%s" % f: self._string})
+            self._queryset = self._queryset.filter(query_filter).distinct()
+
+    def _cooked_on_filter(self, cooked_date=None):
+        if self._sort_includes('lastcooked') or cooked_date:
+            lessthan = self._sort_includes('-lastcooked') or '-' in (cooked_date or [])[:1]
+            if lessthan:
+                default = timezone.now() - timedelta(days=100000)
+            else:
+                default = timezone.now()
+            self._queryset = self._queryset.annotate(
+                lastcooked=Coalesce(Max(Case(When(cooklog__created_by=self._request.user, cooklog__space=self._request.space, then='cooklog__created_at'))), Value(default))
+            )
+        if cooked_date is None:
+            return
+
+        cooked_date = date(*[int(x)for x in cooked_date.split('-') if x != ''])
+
+        if lessthan:
+            self._queryset = self._queryset.filter(lastcooked__date__lte=cooked_date).exclude(lastcooked=default)
+        else:
+            self._queryset = self._queryset.filter(lastcooked__date__gte=cooked_date).exclude(lastcooked=default)
+
+    def _created_on_filter(self, created_date=None):
+        if created_date is None:
+            return
+        lessthan = '-' in created_date[:1]
+        created_date = date(*[int(x) for x in created_date.split('-') if x != ''])
+        if lessthan:
+            self._queryset = self._queryset.filter(created_at__date__lte=created_date)
+        else:
+            self._queryset = self._queryset.filter(created_at__date__gte=created_date)
+
+    def _updated_on_filter(self, updated_date=None):
+        if updated_date is None:
+            return
+        lessthan = '-' in updated_date[:1]
+        updated_date = date(*[int(x)for x in updated_date.split('-') if x != ''])
+        if lessthan:
+            self._queryset = self._queryset.filter(updated_at__date__lte=updated_date)
+        else:
+            self._queryset = self._queryset.filter(updated_at__date__gte=updated_date)
+
+    def _viewed_on_filter(self, viewed_date=None):
+        if self._sort_includes('lastviewed') or viewed_date:
+            longTimeAgo = timezone.now() - timedelta(days=100000)
+            self._queryset = self._queryset.annotate(
+                lastviewed=Coalesce(Max(Case(When(viewlog__created_by=self._request.user, viewlog__space=self._request.space, then='viewlog__created_at'))), Value(longTimeAgo))
+            )
+        if viewed_date is None:
+            return
+        lessthan = '-' in viewed_date[:1]
+        viewed_date = date(*[int(x)for x in viewed_date.split('-') if x != ''])
+
+        if lessthan:
+            self._queryset = self._queryset.filter(lastviewed__date__lte=viewed_date).exclude(lastviewed=longTimeAgo)
+        else:
+            self._queryset = self._queryset.filter(lastviewed__date__gte=viewed_date).exclude(lastviewed=longTimeAgo)
+
+    def _new_recipes(self, new_days=7):
+        # TODO make new days a user-setting
+        if not self._new:
+            return
+        self._queryset = self._queryset.annotate(
+            new_recipe=Case(
+                When(created_at__gte=(timezone.now() - timedelta(days=new_days)), then=('pk')),
+                default=Value(0),
+            )
+        )
+
+    def _recently_viewed(self, num_recent=None):
+        if not num_recent:
+            if self._sort_includes('lastviewed'):
+                self._queryset = self._queryset.annotate(lastviewed=Coalesce(
+                    Max(Case(When(viewlog__created_by=self._request.user, viewlog__space=self._request.space, then='viewlog__pk'))), Value(0)))
+            return
+
+        num_recent_recipes = (
+            ViewLog.objects.filter(created_by=self._request.user, space=self._request.space)
+            .values('recipe').annotate(recent=Max('created_at')).order_by('-recent')[:num_recent]
+        )
+        self._queryset = self._queryset.annotate(recent=Coalesce(Max(Case(When(pk__in=num_recent_recipes.values('recipe'), then='viewlog__pk'))), Value(0)))
+
+    def _favorite_recipes(self, times_cooked=None):
+        if self._sort_includes('favorite') or times_cooked:
+            less_than = '-' in (str(times_cooked) or []) and not self._sort_includes('-favorite')
+            if less_than:
+                default = 1000
+            else:
+                default = 0
+            favorite_recipes = (
+                CookLog.objects.filter(created_by=self._request.user, space=self._request.space, recipe=OuterRef('pk'))
+                .values('recipe')
+                .annotate(count=Count('pk', distinct=True))
+                .values('count')
+            )
+            self._queryset = self._queryset.annotate(favorite=Coalesce(Subquery(favorite_recipes), default))
+        if times_cooked is None:
+            return
+
+        if times_cooked == '0':
+            self._queryset = self._queryset.filter(favorite=0)
+        elif less_than:
+            self._queryset = self._queryset.filter(favorite__lte=int(times_cooked.replace('-', ''))).exclude(favorite=0)
+        else:
+            self._queryset = self._queryset.filter(favorite__gte=int(times_cooked))
+
+    def keyword_filters(self, **kwargs):
+        if all([kwargs[x] is None for x in kwargs]):
+            return
+        for kw_filter in kwargs:
+            if not kwargs[kw_filter]:
+                continue
+            if not isinstance(kwargs[kw_filter], list):
+                kwargs[kw_filter] = [kwargs[kw_filter]]
+
+            keywords = Keyword.objects.filter(pk__in=kwargs[kw_filter])
+            if 'or' in kw_filter:
+                if self._include_children:
+                    f_or = Q(keywords__in=Keyword.include_descendants(keywords))
+                else:
+                    f_or = Q(keywords__in=keywords)
+                if 'not' in kw_filter:
+                    self._queryset = self._queryset.exclude(f_or)
+                else:
+                    self._queryset = self._queryset.filter(f_or)
+            elif 'and' in kw_filter:
+                recipes = Recipe.objects.all()
+                for kw in keywords:
+                    if self._include_children:
+                        f_and = Q(keywords__in=kw.get_descendants_and_self())
+                    else:
+                        f_and = Q(keywords=kw)
+                    if 'not' in kw_filter:
+                        recipes = recipes.filter(f_and)
+                    else:
+                        self._queryset = self._queryset.filter(f_and)
+                if 'not' in kw_filter:
+                    self._queryset = self._queryset.exclude(id__in=recipes.values('id'))
+
+    def food_filters(self, **kwargs):
+        if all([kwargs[x] is None for x in kwargs]):
+            return
+        for fd_filter in kwargs:
+            if not kwargs[fd_filter]:
+                continue
+            if not isinstance(kwargs[fd_filter], list):
+                kwargs[fd_filter] = [kwargs[fd_filter]]
+
+            foods = Food.objects.filter(pk__in=kwargs[fd_filter])
+            if 'or' in fd_filter:
+                if self._include_children:
+                    f_or = Q(steps__ingredients__food__in=Food.include_descendants(foods))
+                else:
+                    f_or = Q(steps__ingredients__food__in=foods)
+
+                if 'not' in fd_filter:
+                    self._queryset = self._queryset.exclude(f_or)
+                else:
+                    self._queryset = self._queryset.filter(f_or)
+            elif 'and' in fd_filter:
+                recipes = Recipe.objects.all()
+                for food in foods:
+                    if self._include_children:
+                        f_and = Q(steps__ingredients__food__in=food.get_descendants_and_self())
+                    else:
+                        f_and = Q(steps__ingredients__food=food)
+                    if 'not' in fd_filter:
+                        recipes = recipes.filter(f_and)
+                    else:
+                        self._queryset = self._queryset.filter(f_and)
+                if 'not' in fd_filter:
+                    self._queryset = self._queryset.exclude(id__in=recipes.values('id'))
+
+    def unit_filters(self, units=None, operator=True):
+        if operator != True:
+            raise NotImplementedError
+        if not units:
+            return
+        if not isinstance(units, list):
+            units = [units]
+        self._queryset = self._queryset.filter(steps__ingredients__unit__in=units)
+
+    def rating_filter(self, rating=None):
+        if rating or self._sort_includes('rating'):
+            lessthan = '-' in (rating or [])
+            reverse = 'rating' in (self._sort_order or []) and '-rating' not in (self._sort_order or [])
+            if lessthan or reverse:
+                default = 100
+            else:
+                default = 0
+            # TODO make ratings a settings user-only vs all-users
+            self._queryset = self._queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=self._request.user, then='cooklog__rating'), default=default))))
+        if rating is None:
+            return
+
+        if rating == '0':
+            self._queryset = self._queryset.filter(rating=0)
+        elif lessthan:
+            self._queryset = self._queryset.filter(rating__lte=int(rating[1:])).exclude(rating=0)
+        else:
+            self._queryset = self._queryset.filter(rating__gte=int(rating))
+
+    def internal_filter(self, internal=None):
+        if not internal:
+            return
+        self._queryset = self._queryset.filter(internal=internal)
+
+    def book_filters(self, **kwargs):
+        if all([kwargs[x] is None for x in kwargs]):
+            return
+        for bk_filter in kwargs:
+            if not kwargs[bk_filter]:
+                continue
+            if not isinstance(kwargs[bk_filter], list):
+                kwargs[bk_filter] = [kwargs[bk_filter]]
+
+            if 'or' in bk_filter:
+                f = Q(recipebookentry__book__id__in=kwargs[bk_filter])
+                if 'not' in bk_filter:
+                    self._queryset = self._queryset.exclude(f)
+                else:
+                    self._queryset = self._queryset.filter(f)
+            elif 'and' in bk_filter:
+                recipes = Recipe.objects.all()
+                for book in kwargs[bk_filter]:
+                    if 'not' in bk_filter:
+                        recipes = recipes.filter(recipebookentry__book__id=book)
+                    else:
+                        self._queryset = self._queryset.filter(recipebookentry__book__id=book)
+                if 'not' in bk_filter:
+                    self._queryset = self._queryset.exclude(id__in=recipes.values('id'))
+
+    def step_filters(self, steps=None, operator=True):
+        if operator != True:
+            raise NotImplementedError
+        if not steps:
+            return
+        if not isinstance(steps, list):
+            steps = [steps]
+        self._queryset = self._queryset.filter(steps__id__in=steps)
+
+    def build_fulltext_filters(self, string=None):
+        if not string:
+            return
+        if self._fulltext_include:
+            vectors = []
+            rank = []
+            if 'name' in self._fulltext_include:
+                vectors.append('name_search_vector')
+                rank.append(SearchRank('name_search_vector', self.search_query, cover_density=True))
+            if 'description' in self._fulltext_include:
+                vectors.append('desc_search_vector')
+                rank.append(SearchRank('desc_search_vector', self.search_query, cover_density=True))
+            if 'steps__instruction' in self._fulltext_include:
+                vectors.append('steps__search_vector')
+                rank.append(SearchRank('steps__search_vector', self.search_query, cover_density=True))
+            if 'keywords__name' in self._fulltext_include:
+                # explicitly settings unaccent on keywords and foods so that they behave the same as search_vector fields
+                vectors.append('keywords__name__unaccent')
+                rank.append(SearchRank('keywords__name__unaccent', self.search_query, cover_density=True))
+            if 'steps__ingredients__food__name' in self._fulltext_include:
+                vectors.append('steps__ingredients__food__name__unaccent')
+                rank.append(SearchRank('steps__ingredients__food__name', self.search_query, cover_density=True))
+
+            for r in rank:
+                if self.search_rank is None:
+                    self.search_rank = r
+                else:
+                    self.search_rank += r
+            # modifying queryset will annotation creates duplicate results
+            self._filters.append(Q(id__in=Recipe.objects.annotate(vector=SearchVector(*vectors)).filter(Q(vector=self.search_query))))
+
+    def build_text_filters(self, string=None):
+        if not string:
+            return
+
+        if not self._filters:
+            self._filters = []
+        # dynamically build array of filters that will be applied
+        for f in self._icontains_include:
+            self._filters += [Q(**{"%s__icontains" % f: self._string})]
+
+        for f in self._istartswith_include:
+            self._filters += [Q(**{"%s__istartswith" % f: self._string})]
+
+    def build_trigram(self, string=None):
+        if not string:
+            return
+        if self._trigram:
+            trigram = None
+            for f in self._trigram_include:
+                if trigram:
+                    trigram += TrigramSimilarity(f, self._string)
+                else:
+                    trigram = TrigramSimilarity(f, self._string)
+            self._fuzzy_match = (
+                Recipe.objects.annotate(trigram=trigram)
+                .distinct()
+                .annotate(simularity=Max('trigram'))
+                .values('id', 'simularity')
+                .filter(simularity__gt=self._search_prefs.trigram_threshold)
+            )
+            self._filters += [Q(pk__in=self._fuzzy_match.values('pk'))]
+
+    def _makenow_filter(self, missing=None):
+        if missing is None or (isinstance(missing, bool) and missing == False):
+            return
+        shopping_users = [*self._request.user.get_shopping_share(), self._request.user]
+
+        onhand_filter = (
+            Q(steps__ingredients__food__onhand_users__in=shopping_users)  # food onhand
+            # or substitute food onhand
+            | Q(steps__ingredients__food__substitute__onhand_users__in=shopping_users)
+            | Q(steps__ingredients__food__in=self.__children_substitute_filter(shopping_users))
+            | Q(steps__ingredients__food__in=self.__sibling_substitute_filter(shopping_users))
+        )
+        makenow_recipes = Recipe.objects.annotate(
+            count_food=Count('steps__ingredients__food__pk', filter=Q(steps__ingredients__food__isnull=False), distinct=True),
+            count_onhand=Count('steps__ingredients__food__pk', filter=onhand_filter, distinct=True),
+            count_ignore_shopping=Count(
+                'steps__ingredients__food__pk', filter=Q(steps__ingredients__food__ignore_shopping=True, steps__ingredients__food__recipe__isnull=True), distinct=True
+            ),
+            has_child_sub=Case(When(steps__ingredients__food__in=self.__children_substitute_filter(shopping_users), then=Value(1)), default=Value(0)),
+            has_sibling_sub=Case(When(steps__ingredients__food__in=self.__sibling_substitute_filter(shopping_users), then=Value(1)), default=Value(0))
+        ).annotate(missingfood=F('count_food') - F('count_onhand') - F('count_ignore_shopping')).filter(missingfood__lte=missing)
+        self._queryset = self._queryset.distinct().filter(id__in=makenow_recipes.values('id'))
+
+    @staticmethod
+    def __children_substitute_filter(shopping_users=None):
+        children_onhand_subquery = Food.objects.filter(path__startswith=OuterRef('path'), depth__gt=OuterRef('depth'), onhand_users__in=shopping_users)
+        return (
+            Food.objects.exclude(  # list of foods that are onhand and children of: foods that are not onhand and are set to use children as substitutes
+                Q(onhand_users__in=shopping_users) | Q(ignore_shopping=True, recipe__isnull=True) | Q(substitute__onhand_users__in=shopping_users)
+            )
+            .exclude(depth=1, numchild=0)
+            .filter(substitute_children=True)
+            .annotate(child_onhand_count=Exists(children_onhand_subquery))
+            .filter(child_onhand_count=True)
+        )
+
+    @staticmethod
+    def __sibling_substitute_filter(shopping_users=None):
+        sibling_onhand_subquery = Food.objects.filter(
+            path__startswith=Substr(OuterRef('path'), 1, Food.steplen * (OuterRef('depth') - 1)), depth=OuterRef('depth'), onhand_users__in=shopping_users
+        )
+        return (
+            Food.objects.exclude(  # list of foods that are onhand and siblings of: foods that are not onhand and are set to use siblings as substitutes
+                Q(onhand_users__in=shopping_users) | Q(ignore_shopping=True, recipe__isnull=True) | Q(substitute__onhand_users__in=shopping_users)
+            )
+            .exclude(depth=1, numchild=0)
+            .filter(substitute_siblings=True)
+            .annotate(sibling_onhand=Exists(sibling_onhand_subquery))
+            .filter(sibling_onhand=True)
+        )

cookbook/helper/recipe_url_import.py

@@ -1,20 +1,44 @@
-import random
 import re
+import traceback
+from html import unescape
+
+from django.utils.dateparse import parse_duration
+from django.utils.translation import gettext as _
 from isodate import parse_duration as iso_parse_duration
 from isodate.isoerror import ISO8601Error
+from pytube import YouTube
+from recipe_scrapers._utils import get_host_name, get_minutes
 
+from cookbook.helper.automation_helper import AutomationEngine
 from cookbook.helper.ingredient_parser import IngredientParser
-from cookbook.models import Keyword
-from django.utils.dateparse import parse_duration
-from html import unescape
-from recipe_scrapers._utils import get_minutes
+from cookbook.models import Automation, Keyword, PropertyType
 
 
 def get_from_scraper(scrape, request):
-    # converting the scrape_me object to the existing json format based on ld+json
-    recipe_json = {}
+    # converting the scrape_html object to the existing json format based on ld+json
+
+    recipe_json = {'steps': [], 'internal': True}
+    keywords = []
+
+    # assign source URL
     try:
-        recipe_json['name'] = parse_name(scrape.title() or None)
+        source_url = scrape.canonical_url()
+    except Exception:
+        try:
+            source_url = scrape.url
+        except Exception:
+            pass
+    if source_url:
+        recipe_json['source_url'] = source_url
+        try:
+            keywords.append(source_url.replace('http://', '').replace('https://', '').split('/')[0])
+        except Exception:
+            recipe_json['source_url'] = ''
+
+    automation_engine = AutomationEngine(request, source=recipe_json.get('source_url'))
+    # assign recipe name
+    try:
+        recipe_json['name'] = parse_name(scrape.title()[:128] or None)
     except Exception:
         recipe_json['name'] = None
     if not recipe_json['name']:
@@ -23,47 +47,62 @@ def get_from_scraper(scrape, request):
         except Exception:
             recipe_json['name'] = ''
 
+    if isinstance(recipe_json['name'], list) and len(recipe_json['name']) > 0:
+        recipe_json['name'] = recipe_json['name'][0]
+
+    recipe_json['name'] = automation_engine.apply_regex_replace_automation(recipe_json['name'], Automation.NAME_REPLACE)
+
+    # assign recipe description
+    # TODO notify user about limit if reached - >256 description will be truncated
     try:
-        description = scrape.schema.data.get("description") or ''
+        description = scrape.description() or None
     except Exception:
-        description = ''
+        description = None
+    if not description:
+        try:
+            description = scrape.schema.data.get("description") or ''
+        except Exception:
+            description = ''
 
     recipe_json['description'] = parse_description(description)
+    recipe_json['description'] = automation_engine.apply_regex_replace_automation(recipe_json['description'], Automation.DESCRIPTION_REPLACE)
 
+    # assign servings attributes
     try:
-        servings = scrape.yields() or None
+        # dont use scrape.yields() as this will always return "x servings" or "x items", should be improved in scrapers directly
+        servings = scrape.schema.data.get('recipeYield') or 1
+    except Exception:
+        servings = 1
+
+    recipe_json['servings'] = parse_servings(servings)
+    recipe_json['servings_text'] = parse_servings_text(servings)
+
+    # assign time attributes
+    try:
+        recipe_json['working_time'] = get_minutes(scrape.prep_time()) or 0
     except Exception:
-        servings = None
-    if not servings:
         try:
-            servings = scrape.schema.data.get('recipeYield') or 1
+            recipe_json['working_time'] = get_minutes(scrape.schema.data.get("prepTime")) or 0
         except Exception:
-            servings = 1
-    if type(servings) != int:
+            recipe_json['working_time'] = 0
+    try:
+        recipe_json['waiting_time'] = get_minutes(scrape.cook_time()) or 0
+    except Exception:
         try:
-            servings = int(re.findall(r'\b\d+\b', servings)[0])
+            recipe_json['waiting_time'] = get_minutes(scrape.schema.data.get("cookTime")) or 0
         except Exception:
-            servings = 1
-    recipe_json['servings'] = max(servings, 1)
+            recipe_json['waiting_time'] = 0
 
-    try:
-        recipe_json['prepTime'] = get_minutes(scrape.schema.data.get("prepTime")) or 0
-    except Exception:
-        recipe_json['prepTime'] = 0
-    try:
-        recipe_json['cookTime'] = get_minutes(scrape.schema.data.get("cookTime")) or 0
-    except Exception:
-        recipe_json['cookTime'] = 0
-
-    if recipe_json['cookTime'] + recipe_json['prepTime'] == 0:
+    if recipe_json['working_time'] + recipe_json['waiting_time'] == 0:
         try:
-            recipe_json['prepTime'] = get_minutes(scrape.total_time()) or 0
+            recipe_json['working_time'] = get_minutes(scrape.total_time()) or 0
         except Exception:
             try:
-                get_minutes(scrape.schema.data.get("totalTime")) or 0
+                recipe_json['working_time'] = get_minutes(scrape.schema.data.get("totalTime")) or 0
             except Exception:
                 pass
 
+    # assign image
     try:
         recipe_json['image'] = parse_image(scrape.image()) or None
     except Exception:
@@ -74,81 +113,174 @@ def get_from_scraper(scrape, request):
         except Exception:
             recipe_json['image'] = ''
 
-    keywords = []
+    # assign keywords
     try:
         if scrape.schema.data.get("keywords"):
             keywords += listify_keywords(scrape.schema.data.get("keywords"))
     except Exception:
         pass
     try:
-        if scrape.schema.data.get('recipeCategory'):
-            keywords += listify_keywords(scrape.schema.data.get("recipeCategory"))
+        if scrape.category():
+            keywords += listify_keywords(scrape.category())
+    except Exception:
+        try:
+            if scrape.schema.data.get('recipeCategory'):
+                keywords += listify_keywords(scrape.schema.data.get("recipeCategory"))
+        except Exception:
+            pass
+    try:
+        if scrape.cuisine():
+            keywords += listify_keywords(scrape.cuisine())
+    except Exception:
+        try:
+            if scrape.schema.data.get('recipeCuisine'):
+                keywords += listify_keywords(scrape.schema.data.get("recipeCuisine"))
+        except Exception:
+            pass
+
+    try:
+        if scrape.author():
+            keywords.append(scrape.author())
     except Exception:
         pass
+
     try:
-        if scrape.schema.data.get('recipeCuisine'):
-            keywords += listify_keywords(scrape.schema.data.get("recipeCuisine"))
-    except Exception:
-        pass
-    try:
-        recipe_json['keywords'] = parse_keywords(list(set(map(str.casefold, keywords))), request.space)
+        recipe_json['keywords'] = parse_keywords(list(set(map(str.casefold, keywords))), request)
     except AttributeError:
         recipe_json['keywords'] = keywords
 
     ingredient_parser = IngredientParser(request, True)
+
+    # assign steps
+    try:
+        for i in parse_instructions(scrape.instructions()):
+            recipe_json['steps'].append({
+                'instruction': i,
+                'ingredients': [],
+                'show_ingredients_table': request.user.userpreference.show_step_ingredients,
+            })
+    except Exception:
+        pass
+    if len(recipe_json['steps']) == 0:
+        recipe_json['steps'].append({
+            'instruction': '',
+            'ingredients': [],
+        })
+
+    recipe_json['description'] = recipe_json['description'][:512]
+    if len(recipe_json['description']) > 256:  # split at 256 as long descriptions don't look good on recipe cards
+        recipe_json['steps'][0]['instruction'] = f"*{recipe_json['description']}*  \n\n" + recipe_json['steps'][0]['instruction']
+
     try:
-        ingredients = []
         for x in scrape.ingredients():
-            try:
-                amount, unit, ingredient, note = ingredient_parser.parse(x)
-                ingredients.append(
-                    {
+            if x.strip() != '':
+                try:
+                    amount, unit, ingredient, note = ingredient_parser.parse(x)
+                    ingredient = {
                         'amount': amount,
-                        'unit': {
-                            'text': unit,
-                            'id': random.randrange(10000, 99999)
-                        },
-                        'ingredient': {
-                            'text': ingredient,
-                            'id': random.randrange(10000, 99999)
+                        'food': {
+                            'name': ingredient,
                         },
+                        'unit': None,
                         'note': note,
-                        'original': x
+                        'original_text': x
                     }
-                )
-            except Exception:
-                ingredients.append(
-                    {
+                    if unit:
+                        ingredient['unit'] = {
+                            'name': unit,
+                        }
+                    recipe_json['steps'][0]['ingredients'].append(ingredient)
+                except Exception:
+                    recipe_json['steps'][0]['ingredients'].append({
                         'amount': 0,
-                        'unit': {
-                            'text': '',
-                            'id': random.randrange(10000, 99999)
-                        },
-                        'ingredient': {
-                            'text': x,
-                            'id': random.randrange(10000, 99999)
+                        'unit': None,
+                        'food': {
+                            'name': x,
                         },
                         'note': '',
-                        'original': x
-                    }
-                )
-        recipe_json['recipeIngredient'] = ingredients
+                        'original_text': x
+                    })
     except Exception:
-        recipe_json['recipeIngredient'] = ingredients
+        pass
 
     try:
-        recipe_json['recipeInstructions'] = parse_instructions(scrape.instructions())
+        recipe_json['properties'] = get_recipe_properties(request.space, scrape.schema.nutrients())
+        print(recipe_json['properties'])
     except Exception:
-        recipe_json['recipeInstructions'] = ""
+        traceback.print_exc()
+        pass
+
+    for s in recipe_json['steps']:
+        s['instruction'] = automation_engine.apply_regex_replace_automation(s['instruction'], Automation.INSTRUCTION_REPLACE)
+        # re.sub(a.param_2, a.param_3, s['instruction'])
 
-    if scrape.url:
-        recipe_json['url'] = scrape.url
-        recipe_json['recipeInstructions'] += "\n\nImported from " + scrape.url
     return recipe_json
 
 
+def get_recipe_properties(space, property_data):
+    # {'servingSize': '1', 'calories': '302 kcal', 'proteinContent': '7,66g', 'fatContent': '11,56g', 'carbohydrateContent': '41,33g'}
+    properties = {
+        "property-calories": "calories",
+        "property-carbohydrates": "carbohydrateContent",
+        "property-proteins": "proteinContent",
+        "property-fats": "fatContent",
+    }
+    recipe_properties = []
+    for pt in PropertyType.objects.filter(space=space, open_data_slug__in=list(properties.keys())).all():
+        for p in list(properties.keys()):
+            if pt.open_data_slug == p:
+                if properties[p] in property_data:
+                    recipe_properties.append({
+                        'property_type': {
+                            'id': pt.id,
+                            'name': pt.name,
+                        },
+                        'property_amount': parse_servings(property_data[properties[p]]) / parse_servings(property_data['servingSize']),
+                    })
+
+    return recipe_properties
+
+
+def get_from_youtube_scraper(url, request):
+    """A YouTube Information Scraper."""
+    kw, created = Keyword.objects.get_or_create(name='YouTube', space=request.space)
+    default_recipe_json = {
+        'name': '',
+        'internal': True,
+        'description': '',
+        'servings': 1,
+        'working_time': 0,
+        'waiting_time': 0,
+        'image': "",
+        'keywords': [{
+            'name': kw.name,
+            'label': kw.name,
+            'id': kw.pk
+        }],
+        'source_url': url,
+        'steps': [{
+            'ingredients': [],
+            'instruction': ''
+        }]
+    }
+
+    try:
+        automation_engine = AutomationEngine(request, source=url)
+        video = YouTube(url)
+        video.streams.first()  # this is required to execute some kind of generator/web request that fetches the description
+        default_recipe_json['name'] = automation_engine.apply_regex_replace_automation(video.title, Automation.NAME_REPLACE)
+        default_recipe_json['image'] = video.thumbnail_url
+        if video.description:
+            default_recipe_json['steps'][0]['instruction'] = automation_engine.apply_regex_replace_automation(video.description, Automation.INSTRUCTION_REPLACE)
+
+    except Exception:
+        pass
+
+    return default_recipe_json
+
+
 def parse_name(name):
-    if type(name) == list:
+    if isinstance(name, list):
         try:
             name = name[0]
         except Exception:
@@ -156,110 +288,74 @@ def parse_name(name):
     return normalize_string(name)
 
 
-def parse_ingredients(ingredients):
-    # some pages have comma separated ingredients in a single array entry
-    try:
-        if type(ingredients[0]) == dict:
-            return ingredients
-    except (KeyError, IndexError):
-        pass
-
-    if (len(ingredients) == 1 and type(ingredients) == list):
-        ingredients = ingredients[0].split(',')
-    elif type(ingredients) == str:
-        ingredients = ingredients.split(',')
-
-    for x in ingredients:
-        if '\n' in x:
-            ingredients.remove(x)
-            for i in x.split('\n'):
-                ingredients.insert(0, i)
-
-    ingredient_list = []
-
-    for x in ingredients:
-        if x.replace(' ', '') != '':
-            x = x.replace('½', "0.5").replace('¼', "0.25").replace('¾', "0.75")
-            try:
-                amount, unit, ingredient, note = parse_single_ingredient(x)
-                if ingredient:
-                    ingredient_list.append(
-                        {
-                            'amount': amount,
-                            'unit': {
-                                'text': unit,
-                                'id': random.randrange(10000, 99999)
-                            },
-                            'ingredient': {
-                                'text': ingredient,
-                                'id': random.randrange(10000, 99999)
-                            },
-                            'note': note,
-                            'original': x
-                        }
-                    )
-            except Exception:
-                ingredient_list.append(
-                    {
-                        'amount': 0,
-                        'unit': {
-                            'text': '',
-                            'id': random.randrange(10000, 99999)
-                        },
-                        'ingredient': {
-                            'text': x,
-                            'id': random.randrange(10000, 99999)
-                        },
-                        'note': '',
-                        'original': x
-                    }
-                )
-
-            ingredients = ingredient_list
-        else:
-            ingredients = []
-    return ingredients
-
-
 def parse_description(description):
     return normalize_string(description)
 
 
-def parse_instructions(instructions):
-    instruction_text = ''
+def clean_instruction_string(instruction):
+    # handle HTML tags that can be converted to markup
+    normalized_string = instruction \
+        .replace("<nobr>", "**") \
+        .replace("</nobr>", "**") \
+        .replace("<strong>", "**") \
+        .replace("</strong>", "**")
+    normalized_string = normalize_string(normalized_string)
+    normalized_string = normalized_string.replace('\n', '  \n')
+    normalized_string = normalized_string.replace('  \n  \n', '\n\n')
 
-    # flatten instructions if they are in a list
-    if type(instructions) == list:
+    # handle unsupported, special UTF8 character in Thermomix-specific instructions,
+    # that happen in nearly every recipe on Cookidoo, Zaubertopf Club, Rezeptwelt
+    # and in Thermomix-specific recipes on many other sites
+    return normalized_string \
+        .replace("", _('reverse rotation')) \
+        .replace("", _('careful rotation')) \
+        .replace("", _('knead')) \
+        .replace("Andicken ", _('thicken')) \
+        .replace("Erwärmen ", _('warm up')) \
+        .replace("Fermentieren ", _('ferment')) \
+        .replace("Sous-vide ", _("sous-vide"))
+
+
+def parse_instructions(instructions):
+    """
+    Convert arbitrary instructions object from website import and turn it into a flat list of strings
+    :param instructions: any instructions object from import
+    :return: list of strings (from one to many elements depending on website)
+    """
+    instruction_list = []
+
+    if isinstance(instructions, list):
         for i in instructions:
-            if type(i) == str:
-                instruction_text += i
+            if isinstance(i, str):
+                instruction_list.append(clean_instruction_string(i))
             else:
                 if 'text' in i:
-                    instruction_text += i['text'] + '\n\n'
+                    instruction_list.append(clean_instruction_string(i['text']))
                 elif 'itemListElement' in i:
                     for ile in i['itemListElement']:
-                        if type(ile) == str:
-                            instruction_text += ile + '\n\n'
+                        if isinstance(ile, str):
+                            instruction_list.append(clean_instruction_string(ile))
                         elif 'text' in ile:
-                            instruction_text += ile['text'] + '\n\n'
+                            instruction_list.append(clean_instruction_string(ile['text']))
                 else:
-                    instruction_text += str(i)
-        instructions = instruction_text
+                    instruction_list.append(clean_instruction_string(str(i)))
+    else:
+        instruction_list.append(clean_instruction_string(instructions))
 
-    return normalize_string(instructions)
+    return instruction_list
 
 
 def parse_image(image):
     # check if list of images is returned, take first if so
     if not image:
         return None
-    if type(image) == list:
+    if isinstance(image, list):
         for pic in image:
-            if (type(pic) == str) and (pic[:4] == 'http'):
+            if (isinstance(pic, str)) and (pic[:4] == 'http'):
                 image = pic
             elif 'url' in pic:
                 image = pic['url']
-    elif type(image) == dict:
+    elif isinstance(image, dict):
         if 'url' in image:
             image = image['url']
 
@@ -270,65 +366,66 @@ def parse_image(image):
 
 
 def parse_servings(servings):
-    if type(servings) == str:
+    if isinstance(servings, str):
         try:
             servings = int(re.search(r'\d+', servings).group())
         except AttributeError:
             servings = 1
-    elif type(servings) == list:
+    elif isinstance(servings, list):
         try:
-            servings = int(re.findall(r'\b\d+\b', servings[0])[0])
-        except KeyError:
+            servings = int(re.findall(r'\b\d+\b', str(servings[0]))[0])
+        except (KeyError, IndexError):
             servings = 1
     return servings
 
 
-def parse_cooktime(cooktime):
-    if type(cooktime) not in [int, float]:
+def parse_servings_text(servings):
+    if isinstance(servings, str):
         try:
-            cooktime = float(re.search(r'\d+', cooktime).group())
+            servings = re.sub("\\d+", '', servings).strip()
+        except Exception:
+            servings = ''
+    if isinstance(servings, list):
+        try:
+            servings = parse_servings_text(servings[1])
+        except Exception:
+            pass
+    return str(servings)[:32]
+
+
+def parse_time(recipe_time):
+    if type(recipe_time) not in [int, float]:
+        try:
+            recipe_time = float(re.search(r'\d+', recipe_time).group())
         except (ValueError, AttributeError):
             try:
-                cooktime = round(iso_parse_duration(cooktime).seconds / 60)
+                recipe_time = round(iso_parse_duration(recipe_time).seconds / 60)
             except ISO8601Error:
                 try:
-                    if (type(cooktime) == list and len(cooktime) > 0):
-                        cooktime = cooktime[0]
-                    cooktime = round(parse_duration(cooktime).seconds / 60)
+                    if (isinstance(recipe_time, list) and len(recipe_time) > 0):
+                        recipe_time = recipe_time[0]
+                    recipe_time = round(parse_duration(recipe_time).seconds / 60)
                 except AttributeError:
-                    cooktime = 0
+                    recipe_time = 0
 
-    return cooktime
+    return recipe_time
 
 
-def parse_preptime(preptime):
-    if type(preptime) not in [int, float]:
-        try:
-            preptime = float(re.search(r'\d+', preptime).group())
-        except ValueError:
-            try:
-                preptime = round(iso_parse_duration(preptime).seconds / 60)
-            except ISO8601Error:
-                try:
-                    if (type(preptime) == list and len(preptime) > 0):
-                        preptime = preptime[0]
-                    preptime = round(parse_duration(preptime).seconds / 60)
-                except AttributeError:
-                    preptime = 0
-
-    return preptime
-
-
-def parse_keywords(keyword_json, space):
+def parse_keywords(keyword_json, request):
     keywords = []
+    automation_engine = AutomationEngine(request)
+
     # keywords as list
     for kw in keyword_json:
         kw = normalize_string(kw)
+        # if alias exists use that instead
+
         if len(kw) != 0:
-            if k := Keyword.objects.filter(name=kw, space=space).first():
-                keywords.append({'id': str(k.id), 'text': str(k)})
+            kw = automation_engine.apply_keyword_automation(kw)
+            if k := Keyword.objects.filter(name__iexact=kw, space=request.space).first():
+                keywords.append({'label': str(k), 'name': k.name, 'id': k.id})
             else:
-                keywords.append({'id': random.randrange(1111111, 9999999, 1), 'text': kw})
+                keywords.append({'label': kw, 'name': kw})
 
     return keywords
 
@@ -336,15 +433,15 @@ def parse_keywords(keyword_json, space):
 def listify_keywords(keyword_list):
     # keywords as string
     try:
-        if type(keyword_list[0]) == dict:
+        if isinstance(keyword_list[0], dict):
             return keyword_list
     except (KeyError, IndexError):
         pass
-    if type(keyword_list) == str:
+    if isinstance(keyword_list, str):
         keyword_list = keyword_list.split(',')
 
     # keywords as string in list
-    if (type(keyword_list) == list and len(keyword_list) == 1 and ',' in keyword_list[0]):
+    if (isinstance(keyword_list, list) and len(keyword_list) == 1 and ',' in keyword_list[0]):
         keyword_list = keyword_list[0].split(',')
     return [x.strip() for x in keyword_list]
 
@@ -361,8 +458,49 @@ def normalize_string(string):
 
 
 def iso_duration_to_minutes(string):
-    match = re.match(
-        r'P((?P<years>\d+)Y)?((?P<months>\d+)M)?((?P<weeks>\d+)W)?((?P<days>\d+)D)?T((?P<hours>\d+)H)?((?P<minutes>\d+)M)?((?P<seconds>\d+)S)?',
-        string
-    ).groupdict()
+    match = re.match(r'P((?P<years>\d+)Y)?((?P<months>\d+)M)?((?P<weeks>\d+)W)?((?P<days>\d+)D)?T((?P<hours>\d+)H)?((?P<minutes>\d+)M)?((?P<seconds>\d+)S)?', string).groupdict()
     return int(match['days'] or 0) * 24 * 60 + int(match['hours'] or 0) * 60 + int(match['minutes'] or 0)
+
+
+def get_images_from_soup(soup, url):
+    sources = ['src', 'srcset', 'data-src']
+    images = []
+    img_tags = soup.find_all('img')
+    if url:
+        site = get_host_name(url)
+        prot = url.split(':')[0]
+
+    urls = []
+    for img in img_tags:
+        for src in sources:
+            try:
+                urls.append(img[src])
+            except KeyError:
+                pass
+
+    for u in urls:
+        u = u.split('?')[0]
+        filename = re.search(r'/([\w_-]+[.](jpg|jpeg|gif|png))$', u)
+        if filename:
+            if (('http' not in u) and (url)):
+                # sometimes an image source can be relative
+                # if it is provide the base url
+                u = '{}://{}{}'.format(prot, site, u)
+            if 'http' in u:
+                images.append(u)
+    return images
+
+
+def clean_dict(input_dict, key):
+    if isinstance(input_dict, dict):
+        for x in list(input_dict):
+            if x == key:
+                del input_dict[x]
+            elif isinstance(input_dict[x], dict):
+                input_dict[x] = clean_dict(input_dict[x], key)
+            elif isinstance(input_dict[x], list):
+                temp_list = []
+                for e in input_dict[x]:
+                    temp_list.append(clean_dict(e, key))
+
+    return input_dict

cookbook/helper/scope_middleware.py

@@ -1,7 +1,10 @@
 from django.urls import reverse
 from django_scopes import scope, scopes_disabled
+from oauth2_provider.contrib.rest_framework import OAuth2Authentication
+from rest_framework.exceptions import AuthenticationFailed
 
 from cookbook.views import views
+from recipes import settings
 
 
 class ScopeMiddleware:
@@ -9,30 +12,56 @@ class ScopeMiddleware:
         self.get_response = get_response
 
     def __call__(self, request):
+        prefix = settings.JS_REVERSE_SCRIPT_PREFIX or ''
+
+        # need to disable scopes for writing requests into userpref and enable for loading ?
+        if request.path.startswith(prefix + '/api/user-preference/'):
+            with scopes_disabled():
+                return self.get_response(request)
+
         if request.user.is_authenticated:
 
-            if request.path.startswith('/admin/'):
+            if request.path.startswith(prefix + '/admin/'):
                 with scopes_disabled():
                     return self.get_response(request)
 
-            if request.path.startswith('/signup/') or request.path.startswith('/invite/'):
+            if request.path.startswith(prefix + '/signup/') or request.path.startswith(prefix + '/invite/'):
                 return self.get_response(request)
 
-            if request.path.startswith('/accounts/'):
+            if request.path.startswith(prefix + '/accounts/'):
+                return self.get_response(request)
+
+            if request.path.startswith(prefix + '/switch-space/'):
                 return self.get_response(request)
 
             with scopes_disabled():
-                if request.user.userpreference.space is None and not reverse('account_logout') in request.path:
-                    return views.no_space(request)
+                if request.user.userspace_set.count() == 0 and not reverse('account_logout') in request.path:
+                    return views.space_overview(request)
 
-            if request.user.groups.count() == 0 and not reverse('account_logout') in request.path:
+            # get active user space, if for some reason more than one space is active select first (group permission checks will fail, this is not intended at this point)
+            user_space = request.user.userspace_set.filter(active=True).first()
+
+            if not user_space:
+                return views.space_overview(request)
+
+            if user_space.groups.count() == 0 and not reverse('account_logout') in request.path:
                 return views.no_groups(request)
 
-            request.space = request.user.userpreference.space
-            # with scopes_disabled():
+            request.space = user_space.space
             with scope(space=request.space):
                 return self.get_response(request)
         else:
+            if request.path.startswith(prefix + '/api/'):
+                try:
+                    if auth := OAuth2Authentication().authenticate(request):
+                        user_space = auth[0].userspace_set.filter(active=True).first()
+                        if user_space:
+                            request.space = user_space.space
+                            with scope(space=request.space):
+                                return self.get_response(request)
+                except AuthenticationFailed:
+                    pass
+
             with scopes_disabled():
                 request.space = None
                 return self.get_response(request)

cookbook/helper/scrapers/cooksillustrated.py

@@ -1,68 +0,0 @@
-import json
-from recipe_scrapers._abstract import AbstractScraper
-
-
-class CooksIllustrated(AbstractScraper):
-    @classmethod
-    def host(cls, site='cooksillustrated'):
-        return {
-            'cooksillustrated': f"{site}.com",
-            'americastestkitchen': f"{site}.com",
-            'cookscountry': f"{site}.com",
-        }.get(site)
-
-    def title(self):
-        return self.schema.title()
-
-    def image(self):
-        return self.schema.image()
-
-    def total_time(self):
-        if not self.recipe:
-            self.get_recipe()
-        return self.recipe['recipeTimeNote']
-
-    def yields(self):
-        if not self.recipe:
-            self.get_recipe()
-        return self.recipe['yields']
-
-    def ingredients(self):
-        if not self.recipe:
-            self.get_recipe()
-        ingredients = []
-        for group in self.recipe['ingredientGroups']:
-            ingredients += group['fields']['recipeIngredientItems']
-        return [
-            "{} {} {}{}".format(
-                i['fields']['qty'] or '',
-                i['fields']['measurement'] or '',
-                i['fields']['ingredient']['fields']['title'] or '',
-                i['fields']['postText'] or ''
-            )
-            for i in ingredients
-        ]
-
-    def instructions(self):
-        if not self.recipe:
-            self.get_recipe()
-        if self.recipe.get('headnote', False):
-            i = ['Note: ' + self.recipe.get('headnote', '')]
-        else:
-            i = []
-        return "\n".join(
-            i
-            + [self.recipe.get('whyThisWorks', '')]
-            + [
-                instruction['fields']['content']
-                for instruction in self.recipe['instructions']
-            ]
-        )
-
-    def nutrients(self):
-        raise NotImplementedError("This should be implemented.")
-
-    def get_recipe(self):
-        j = json.loads(self.soup.find(type='application/json').string)
-        name = list(j['props']['initialState']['content']['documents'])[0]
-        self.recipe = j['props']['initialState']['content']['documents'][name]

cookbook/helper/scrapers/scrapers.py

@@ -1,42 +0,0 @@
-from bs4 import BeautifulSoup
-from json import JSONDecodeError
-from recipe_scrapers import SCRAPERS, get_host_name
-from recipe_scrapers._factory import SchemaScraperFactory
-from recipe_scrapers._schemaorg import SchemaOrg
-
-from .cooksillustrated import CooksIllustrated
-
-CUSTOM_SCRAPERS = {
-    CooksIllustrated.host(site="cooksillustrated"): CooksIllustrated,
-    CooksIllustrated.host(site="americastestkitchen"): CooksIllustrated,
-    CooksIllustrated.host(site="cookscountry"): CooksIllustrated,
-}
-SCRAPERS.update(CUSTOM_SCRAPERS)
-
-
-def text_scraper(text, url=None):
-    domain = None
-    if url:
-        domain = get_host_name(url)
-    if domain in SCRAPERS:
-        scraper_class = SCRAPERS[domain]
-    else:
-        scraper_class = SchemaScraperFactory.SchemaScraper
-
-    class TextScraper(scraper_class):
-        def __init__(
-            self,
-            page_data,
-            url=None
-        ):
-            self.wild_mode = False
-            self.meta_http_equiv = False
-            self.soup = BeautifulSoup(page_data, "html.parser")
-            self.url = url
-            self.recipe = None
-            try:
-                self.schema = SchemaOrg(page_data)
-            except (JSONDecodeError, AttributeError):
-                pass
-
-    return TextScraper(text, url)

cookbook/helper/shopping_helper.py

@@ -0,0 +1,194 @@
+
+from decimal import Decimal
+
+from django.db.models import F, OuterRef, Q, Subquery, Value
+from django.db.models.functions import Coalesce
+from django.utils.translation import gettext as _
+
+from cookbook.models import (Ingredient, MealPlan, Recipe, ShoppingListEntry, ShoppingListRecipe,
+                             SupermarketCategoryRelation)
+
+
+def shopping_helper(qs, request):
+    supermarket = request.query_params.get('supermarket', None)
+    checked = request.query_params.get('checked', 'recent')
+    user = request.user
+    supermarket_order = [F('food__supermarket_category__name').asc(nulls_first=True), 'food__name']
+
+    # TODO created either scheduled task or startup task to delete very old shopping list entries
+    # TODO create user preference to define 'very old'
+    if supermarket:
+        supermarket_categories = SupermarketCategoryRelation.objects.filter(supermarket=supermarket, category=OuterRef('food__supermarket_category'))
+        qs = qs.annotate(supermarket_order=Coalesce(Subquery(supermarket_categories.values('order')), Value(9999)))
+        supermarket_order = ['supermarket_order'] + supermarket_order
+    if checked in ['false', 0, '0']:
+        qs = qs.filter(checked=False)
+    elif checked in ['true', 1, '1']:
+        qs = qs.filter(checked=True)
+    elif checked in ['recent']:
+        supermarket_order = ['checked'] + supermarket_order
+
+    return qs.distinct().order_by(*supermarket_order).select_related('unit', 'food', 'ingredient', 'created_by', 'list_recipe', 'list_recipe__mealplan', 'list_recipe__recipe')
+
+
+class RecipeShoppingEditor():
+    def __init__(self, user, space, **kwargs):
+        self.created_by = user
+        self.space = space
+        self._kwargs = {**kwargs}
+
+        self.mealplan = self._kwargs.get('mealplan', None)
+        if type(self.mealplan) in [int, float]:
+            self.mealplan = MealPlan.objects.filter(id=self.mealplan, space=self.space)
+        if isinstance(self.mealplan, dict):
+            self.mealplan = MealPlan.objects.filter(id=self.mealplan['id'], space=self.space).first()
+        self.id = self._kwargs.get('id', None)
+
+        self._shopping_list_recipe = self.get_shopping_list_recipe(self.id, self.created_by, self.space)
+
+        if self._shopping_list_recipe:
+            # created_by needs to be sticky to original creator as it is 'their' shopping list
+            # changing shopping list created_by can shift some items to new owner which may not share in the other direction
+            self.created_by = getattr(self._shopping_list_recipe.entries.first(), 'created_by', self.created_by)
+
+        self.recipe = getattr(self._shopping_list_recipe, 'recipe', None) or self._kwargs.get('recipe', None) or getattr(self.mealplan, 'recipe', None)
+        if type(self.recipe) in [int, float]:
+            self.recipe = Recipe.objects.filter(id=self.recipe, space=self.space)
+
+        try:
+            self.servings = float(self._kwargs.get('servings', None))
+        except (ValueError, TypeError):
+            self.servings = getattr(self._shopping_list_recipe, 'servings', None) or getattr(self.mealplan, 'servings', None) or getattr(self.recipe, 'servings', None)
+
+    @property
+    def _recipe_servings(self):
+        return getattr(self.recipe, 'servings', None) or getattr(getattr(self.mealplan, 'recipe', None), 'servings',
+                                                                 None) or getattr(getattr(self._shopping_list_recipe, 'recipe', None), 'servings', None)
+
+    @property
+    def _servings_factor(self):
+        return Decimal(self.servings) / Decimal(self._recipe_servings)
+
+    @property
+    def _shared_users(self):
+        return [*list(self.created_by.get_shopping_share()), self.created_by]
+
+    @staticmethod
+    def get_shopping_list_recipe(id, user, space):
+        return ShoppingListRecipe.objects.filter(id=id).filter(entries__space=space).filter(
+            Q(entries__created_by=user)
+            | Q(entries__created_by__in=list(user.get_shopping_share()))
+        ).prefetch_related('entries').first()
+
+    def get_recipe_ingredients(self, id, exclude_onhand=False):
+        if exclude_onhand:
+            return Ingredient.objects.filter(step__recipe__id=id, food__ignore_shopping=False, space=self.space).exclude(
+                food__onhand_users__id__in=[x.id for x in self._shared_users])
+        else:
+            return Ingredient.objects.filter(step__recipe__id=id, food__ignore_shopping=False, space=self.space)
+
+    @property
+    def _include_related(self):
+        return self.created_by.userpreference.mealplan_autoinclude_related
+
+    @property
+    def _exclude_onhand(self):
+        return self.created_by.userpreference.mealplan_autoexclude_onhand
+
+    def create(self, **kwargs):
+        ingredients = kwargs.get('ingredients', None)
+        exclude_onhand = not ingredients and self._exclude_onhand
+        if servings := kwargs.get('servings', None):
+            self.servings = float(servings)
+
+        if mealplan := kwargs.get('mealplan', None):
+            if isinstance(mealplan, dict):
+                self.mealplan = MealPlan.objects.filter(id=mealplan['id'], space=self.space).first()
+            else:
+                self.mealplan = mealplan
+            self.recipe = mealplan.recipe
+        elif recipe := kwargs.get('recipe', None):
+            self.recipe = recipe
+
+        if not self.servings:
+            self.servings = getattr(self.mealplan, 'servings', None) or getattr(self.recipe, 'servings', 1.0)
+
+        self._shopping_list_recipe = ShoppingListRecipe.objects.create(recipe=self.recipe, mealplan=self.mealplan, servings=self.servings)
+
+        if ingredients:
+            self._add_ingredients(ingredients=ingredients)
+        else:
+            if self._include_related:
+                related = self.recipe.get_related_recipes()
+                self._add_ingredients(self.get_recipe_ingredients(self.recipe.id, exclude_onhand=exclude_onhand).exclude(food__recipe__in=related))
+                for r in related:
+                    self._add_ingredients(self.get_recipe_ingredients(r.id, exclude_onhand=exclude_onhand).exclude(food__recipe__in=related))
+            else:
+                self._add_ingredients(self.get_recipe_ingredients(self.recipe.id, exclude_onhand=exclude_onhand))
+
+        return True
+
+    def add(self, **kwargs):
+        return
+
+    def edit(self, servings=None, ingredients=None, **kwargs):
+        if servings:
+            self.servings = servings
+
+        self._delete_ingredients(ingredients=ingredients)
+        if self.servings != self._shopping_list_recipe.servings:
+            self.edit_servings()
+        self._add_ingredients(ingredients=ingredients)
+        return True
+
+    def edit_servings(self, servings=None, **kwargs):
+        if servings:
+            self.servings = servings
+        if id := kwargs.get('id', None):
+            self._shopping_list_recipe = self.get_shopping_list_recipe(id, self.created_by, self.space)
+        if not self.servings:
+            raise ValueError(_("You must supply a servings size"))
+
+        if self._shopping_list_recipe.servings == self.servings:
+            return True
+
+        for sle in ShoppingListEntry.objects.filter(list_recipe=self._shopping_list_recipe):
+            sle.amount = sle.ingredient.amount * Decimal(self._servings_factor)
+            sle.save()
+        self._shopping_list_recipe.servings = self.servings
+        self._shopping_list_recipe.save()
+        return True
+
+    def delete(self, **kwargs):
+        try:
+            self._shopping_list_recipe.delete()
+            return True
+        except BaseException:
+            return False
+
+    def _add_ingredients(self, ingredients=None):
+        if not ingredients:
+            return
+        elif isinstance(ingredients, list):
+            ingredients = Ingredient.objects.filter(id__in=ingredients, food__ignore_shopping=False)
+        existing = self._shopping_list_recipe.entries.filter(ingredient__in=ingredients).values_list('ingredient__pk', flat=True)
+        add_ingredients = ingredients.exclude(id__in=existing)
+
+        for i in [x for x in add_ingredients if x.food]:
+            ShoppingListEntry.objects.create(
+                list_recipe=self._shopping_list_recipe,
+                food=i.food,
+                unit=i.unit,
+                ingredient=i,
+                amount=i.amount * Decimal(self._servings_factor),
+                created_by=self.created_by,
+                space=self.space,
+            )
+
+    # deletes shopping list entries not in ingredients list
+    def _delete_ingredients(self, ingredients=None):
+        if not ingredients:
+            return
+        to_delete = self._shopping_list_recipe.entries.exclude(ingredient__in=ingredients)
+        ShoppingListEntry.objects.filter(id__in=to_delete).delete()
+        self._shopping_list_recipe = self.get_shopping_list_recipe(self.id, self.created_by, self.space)

cookbook/helper/template_helper.py

@@ -1,10 +1,12 @@
+from gettext import gettext as _
+
 import bleach
 import markdown as md
-from bleach_allowlist import markdown_attrs, markdown_tags
+from jinja2 import Template, TemplateSyntaxError, UndefinedError
+from markdown.extensions.tables import TableExtension
+
 from cookbook.helper.mdx_attributes import MarkdownFormatExtension
 from cookbook.helper.mdx_urlize import UrlizeExtension
-from jinja2 import Template, TemplateSyntaxError, UndefinedError
-from gettext import gettext as _
 
 
 class IngredientObject(object):
@@ -12,17 +14,34 @@ class IngredientObject(object):
     unit = ""
     food = ""
     note = ""
+    numeric_amount = 0
 
     def __init__(self, ingredient):
         if ingredient.no_amount:
             self.amount = ""
         else:
             self.amount = f"<scalable-number v-bind:number='{bleach.clean(str(ingredient.amount))}' v-bind:factor='ingredient_factor'></scalable-number>"
+            self.numeric_amount = float(ingredient.amount)
         if ingredient.unit:
-            self.unit = bleach.clean(str(ingredient.unit))
+            if ingredient.unit.plural_name in (None, ""):
+                self.unit = bleach.clean(str(ingredient.unit))
+            else:
+                if ingredient.always_use_plural_unit or ingredient.amount > 1 and not ingredient.no_amount:
+                    self.unit = bleach.clean(ingredient.unit.plural_name)
+                else:
+                    self.unit = bleach.clean(str(ingredient.unit))
         else:
             self.unit = ""
-        self.food = bleach.clean(str(ingredient.food))
+        if ingredient.food:
+            if ingredient.food.plural_name in (None, ""):
+                self.food = bleach.clean(str(ingredient.food))
+            else:
+                if ingredient.always_use_plural_food or ingredient.amount > 1 and not ingredient.no_amount:
+                    self.food = bleach.clean(str(ingredient.food.plural_name))
+                else:
+                    self.food = bleach.clean(str(ingredient.food))
+        else:
+            self.food = ""
         self.note = bleach.clean(str(ingredient.note))
 
     def __str__(self):
@@ -35,17 +54,29 @@ class IngredientObject(object):
 def render_instructions(step):  # TODO deduplicate markdown cleanup code
     instructions = step.instruction
 
-    tags = markdown_tags + [
+    tags = {
+        "h1", "h2", "h3", "h4", "h5", "h6",
+        "b", "i", "strong", "em", "tt",
+        "p", "br",
+        "span", "div", "blockquote", "code", "pre", "hr",
+        "ul", "ol", "li", "dd", "dt",
+        "img",
+        "a",
+        "sub", "sup",
         'pre', 'table', 'td', 'tr', 'th', 'tbody', 'style', 'thead'
-    ]
+    }
     parsed_md = md.markdown(
         instructions,
         extensions=[
-            'markdown.extensions.fenced_code', 'tables',
+            'markdown.extensions.fenced_code', TableExtension(),
             UrlizeExtension(), MarkdownFormatExtension()
         ]
     )
-    markdown_attrs['*'] = markdown_attrs['*'] + ['class']
+    markdown_attrs = {
+        "*": ["id", "class", 'width', 'height'],
+        "img": ["src", "alt", "title"],
+        "a": ["href", "alt", "title"],
+    }
 
     instructions = bleach.clean(parsed_md, tags, markdown_attrs)
 
@@ -54,9 +85,12 @@ def render_instructions(step):  # TODO deduplicate markdown cleanup code
     for i in step.ingredients.all():
         ingredients.append(IngredientObject(i))
 
+    def scale(number):
+        return f"<scalable-number v-bind:number='{bleach.clean(str(number))}' v-bind:factor='ingredient_factor'></scalable-number>"
+
     try:
         template = Template(instructions)
-        instructions = template.render(ingredients=ingredients)
+        instructions = template.render(ingredients=ingredients, scale=scale)
     except TemplateSyntaxError:
         return _('Could not parse template code.') + ' Error: Template Syntax broken'
     except UndefinedError:

cookbook/helper/unit_conversion_helper.py

@@ -0,0 +1,142 @@
+from django.core.cache import caches
+from decimal import Decimal
+
+from cookbook.helper.cache_helper import CacheHelper
+from cookbook.models import Ingredient, Unit
+
+CONVERSION_TABLE = {
+    'weight': {
+        'g': 1000,
+        'kg': 1,
+        'ounce': 35.274,
+        'pound': 2.20462
+    },
+    'volume': {
+        'ml': 1000,
+        'l': 1,
+        'fluid_ounce': 33.814,
+        'pint': 2.11338,
+        'quart': 1.05669,
+        'gallon': 0.264172,
+        'tbsp': 67.628,
+        'tsp': 202.884,
+        'us_cup': 4.22675,
+        'imperial_fluid_ounce': 35.1951,
+        'imperial_pint': 1.75975,
+        'imperial_quart': 0.879877,
+        'imperial_gallon': 0.219969,
+        'imperial_tbsp': 56.3121,
+        'imperial_tsp': 168.936,
+    },
+}
+
+BASE_UNITS_WEIGHT = list(CONVERSION_TABLE['weight'].keys())
+BASE_UNITS_VOLUME = list(CONVERSION_TABLE['volume'].keys())
+
+
+class ConversionException(Exception):
+    pass
+
+
+class UnitConversionHelper:
+    space = None
+
+    def __init__(self, space):
+        """
+        Initializes unit conversion helper
+        :param space: space to perform conversions on
+        """
+        self.space = space
+
+    @staticmethod
+    def convert_from_to(from_unit, to_unit, amount):
+        """
+        Convert from one base unit to another. Throws ConversionException if trying to convert between different systems (weight/volume) or if units are not supported.
+        :param from_unit: str unit to convert from
+        :param to_unit: str unit to convert to
+        :param amount: amount to convert
+        :return: Decimal converted amount
+        """
+        system = None
+        if from_unit in BASE_UNITS_WEIGHT and to_unit in BASE_UNITS_WEIGHT:
+            system = 'weight'
+        if from_unit in BASE_UNITS_VOLUME and to_unit in BASE_UNITS_VOLUME:
+            system = 'volume'
+
+        if not system:
+            raise ConversionException('Trying to convert units not existing or not in one unit system (weight/volume)')
+
+        return Decimal(amount / Decimal(CONVERSION_TABLE[system][from_unit] / CONVERSION_TABLE[system][to_unit]))
+
+    def base_conversions(self, ingredient_list):
+        """
+        Calculates all possible base unit conversions for each ingredient give.
+        Converts to all common base units IF they exist in the unit database of the space.
+        For useful results all ingredients passed should be of the same food, otherwise filtering afterwards might be required.
+        :param ingredient_list: list of ingredients to convert
+        :return: ingredient list with appended conversions
+        """
+        base_conversion_ingredient_list = ingredient_list.copy()
+        for i in ingredient_list:
+            try:
+                conversion_unit = i.unit.name
+                if i.unit.base_unit:
+                    conversion_unit = i.unit.base_unit
+
+                # TODO allow setting which units to convert to? possibly only once conversions become visible
+                units = caches['default'].get(CacheHelper(self.space).BASE_UNITS_CACHE_KEY, None)
+                if not units:
+                    units = Unit.objects.filter(space=self.space, base_unit__in=(BASE_UNITS_VOLUME + BASE_UNITS_WEIGHT)).all()
+                    caches['default'].set(CacheHelper(self.space).BASE_UNITS_CACHE_KEY, units, 60 * 60)  # cache is cleared on unit save signal so long duration is fine
+
+                for u in units:
+                    try:
+                        ingredient = Ingredient(amount=self.convert_from_to(conversion_unit, u.base_unit, i.amount), unit=u, food=ingredient_list[0].food, )
+                        if not any((x.unit.name == ingredient.unit.name or x.unit.base_unit == ingredient.unit.name) for x in base_conversion_ingredient_list):
+                            base_conversion_ingredient_list.append(ingredient)
+                    except ConversionException:
+                        pass
+            except Exception:
+                pass
+
+        return base_conversion_ingredient_list
+
+    def get_conversions(self, ingredient):
+        """
+        Converts an ingredient to all possible conversions based on the custom unit conversion database.
+        After that passes conversion to UnitConversionHelper.base_conversions() to get all base conversions possible.
+        :param ingredient: Ingredient object
+        :return: list of ingredients with all possible custom and base conversions
+        """
+        conversions = [ingredient]
+        if ingredient.unit:
+            for c in ingredient.unit.unit_conversion_base_relation.all():
+                if c.space == self.space:
+                    r = self._uc_convert(c, ingredient.amount, ingredient.unit, ingredient.food)
+                    if r and r not in conversions:
+                        conversions.append(r)
+            for c in ingredient.unit.unit_conversion_converted_relation.all():
+                if c.space == self.space:
+                    r = self._uc_convert(c, ingredient.amount, ingredient.unit, ingredient.food)
+                    if r and r not in conversions:
+                        conversions.append(r)
+
+        conversions = self.base_conversions(conversions)
+
+        return conversions
+
+    def _uc_convert(self, uc, amount, unit, food):
+        """
+        Helper to calculate values for custom unit conversions.
+        Converts given base values using the passed UnitConversion object into a converted Ingredient
+        :param uc: UnitConversion object
+        :param amount: base amount
+        :param unit: base unit
+        :param food: base food
+        :return: converted ingredient object from base amount/unit/food
+        """
+        if uc.food is None or uc.food == food:
+            if unit == uc.base_unit:
+                return Ingredient(amount=amount * (uc.converted_amount / uc.base_amount), unit=uc.converted_unit, food=food, space=self.space)
+            else:
+                return Ingredient(amount=amount * (uc.base_amount / uc.converted_amount), unit=uc.base_unit, food=food, space=self.space)

cookbook/integration/cheftap.py

@@ -2,14 +2,14 @@ import re
 
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient
+from cookbook.models import Ingredient, Recipe, Step
 
 
 class ChefTap(Integration):
 
     def import_file_name_filter(self, zip_info_object):
         print("testing", zip_info_object.filename)
-        return re.match(r'^cheftap_export/([A-Za-z\d\w\s-])+.txt$', zip_info_object.filename) or re.match(r'^([A-Za-z\d\w\s-])+.txt$', zip_info_object.filename)
+        return re.match(r'^cheftap_export/([A-Za-z\d\s\-_()\[\]\u00C0-\u017F])+.txt$', zip_info_object.filename) or re.match(r'^([A-Za-z\d\s\-_()\[\]\u00C0-\u017F])+.txt$', zip_info_object.filename)
 
     def get_recipe_from_file(self, file):
         source_url = ''
@@ -36,7 +36,7 @@ class ChefTap(Integration):
 
         recipe = Recipe.objects.create(name=title, created_by=self.request.user, internal=True, space=self.request.space, )
 
-        step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space,)
+        step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,)
 
         if source_url != '':
             step.instruction += '\n' + source_url
@@ -45,11 +45,11 @@ class ChefTap(Integration):
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
             if len(ingredient.strip()) > 0:
-                amount, unit, ingredient, note = ingredient_parser.parse(ingredient)
-                f = ingredient_parser.get_food(ingredient)
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
+                f = ingredient_parser.get_food(food)
                 u = ingredient_parser.get_unit(unit)
                 step.ingredients.add(Ingredient.objects.create(
-                    food=f, unit=u, amount=amount, note=note, space=self.request.space,
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                 ))
         recipe.steps.add(step)
 

cookbook/integration/chowdown.py

@@ -4,21 +4,26 @@ from zipfile import ZipFile
 
 from cookbook.helper.image_processing import get_filetype
 from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient, Keyword
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class Chowdown(Integration):
 
     def import_file_name_filter(self, zip_info_object):
         print("testing", zip_info_object.filename)
-        return re.match(r'^(_)*recipes/([A-Za-z\d\s-])+.md$', zip_info_object.filename)
+        return re.match(r'^(_)*recipes/([A-Za-z\d\s\-_()\[\]\u00C0-\u017F])+.md$', zip_info_object.filename)
 
     def get_recipe_from_file(self, file):
         ingredient_mode = False
         direction_mode = False
         description_mode = False
 
+        description = None
+        prep_time = None
+        serving = None
+
         ingredients = []
         directions = []
         descriptions = []
@@ -26,6 +31,12 @@ class Chowdown(Integration):
             line = fl.decode("utf-8")
             if 'title:' in line:
                 title = line.replace('title:', '').replace('"', '').strip()
+            if 'description:' in line:
+                description = line.replace('description:', '').replace('"', '').strip()
+            if 'prep_time:' in line:
+                prep_time = line.replace('prep_time:', '').replace('"', '').strip()
+            if 'yield:' in line:
+                serving = line.replace('yield:', '').replace('"', '').strip()
             if 'image:' in line:
                 image = line.replace('image:', '').strip()
             if 'tags:' in line:
@@ -48,24 +59,53 @@ class Chowdown(Integration):
                 descriptions.append(line)
 
         recipe = Recipe.objects.create(name=title, created_by=self.request.user, internal=True, space=self.request.space)
+        if description:
+            recipe.description = description
 
         for k in tags.split(','):
-            print(f'adding keyword {k.strip()}')
             keyword, created = Keyword.objects.get_or_create(name=k.strip(), space=self.request.space)
             recipe.keywords.add(keyword)
 
-        step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n' + '\n'.join(descriptions), space=self.request.space,
-        )
+        ingredients_added = False
+        for direction in directions:
+            if len(direction.strip()) > 0:
+                step = Step.objects.create(
+                    instruction=direction, name='', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+                )
+            else:
+                step = Step.objects.create(
+                    instruction=direction, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+                )
+            if not ingredients_added:
+                ingredients_added = True
+
+                ingredient_parser = IngredientParser(self.request, True)
+                for ingredient in ingredients:
+                    if len(ingredient.strip()) > 0:
+                        amount, unit, food, note = ingredient_parser.parse(ingredient)
+                        f = ingredient_parser.get_food(food)
+                        u = ingredient_parser.get_unit(unit)
+                        step.ingredients.add(Ingredient.objects.create(
+                            food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
+                        ))
+            recipe.steps.add(step)
+
+        if serving:
+            recipe.servings = parse_servings(serving)
+            recipe.servings_text = 'servings'
+
+        if prep_time:
+            recipe.working_time = parse_time(prep_time)
 
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
-            amount, unit, ingredient, note = ingredient_parser.parse(ingredient)
-            f = ingredient_parser.get_food(ingredient)
-            u = ingredient_parser.get_unit(unit)
-            step.ingredients.add(Ingredient.objects.create(
-                food=f, unit=u, amount=amount, note=note, space=self.request.space,
-            ))
+            if len(ingredient.strip()) > 0:
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
+                f = ingredient_parser.get_food(food)
+                u = ingredient_parser.get_unit(unit)
+                step.ingredients.add(Ingredient.objects.create(
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
+                ))
         recipe.steps.add(step)
 
         for f in self.files:
@@ -75,6 +115,7 @@ class Chowdown(Integration):
                     if re.match(f'^images/{image}$', z.filename):
                         self.import_recipe_image(recipe, BytesIO(import_zip.read(z.filename)), filetype=get_filetype(z.filename))
 
+        recipe.save()
         return recipe
 
     def get_file_from_recipe(self, recipe):

cookbook/integration/cookbookapp.py

@@ -1,67 +1,73 @@
-import base64
-import gzip
-import json
 import re
 from io import BytesIO
 
-import yaml
+import requests
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import (get_from_scraper, get_images_from_soup,
+                                               iso_duration_to_minutes)
+from recipe_scrapers import scrape_html
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient, Keyword
-from gettext import gettext as _
+from cookbook.models import Ingredient, Recipe, Step
 
 
 class CookBookApp(Integration):
 
     def import_file_name_filter(self, zip_info_object):
-        return zip_info_object.filename.endswith('.yml')
+        return zip_info_object.filename.endswith('.html')
 
     def get_recipe_from_file(self, file):
-        recipe_yml = yaml.safe_load(file.getvalue().decode("utf-8"))
+        recipe_html = file.getvalue().decode("utf-8")
+
+        scrape = scrape_html(html=recipe_html, org_url="https://cookbookapp.import", supported_only=False)
+        recipe_json = get_from_scraper(scrape, self.request)
+        images = list(dict.fromkeys(get_images_from_soup(scrape.soup, None)))
 
         recipe = Recipe.objects.create(
-            name=recipe_yml['name'].strip(),
+            name=recipe_json['name'].strip(),
             created_by=self.request.user, internal=True,
             space=self.request.space)
 
         try:
-            recipe.servings = re.findall('([0-9])+', recipe_yml['recipeYield'])[0]
-        except Exception as e:
-            pass
-
-        try:
-            recipe.working_time = recipe_yml['prep_time'].replace(' minutes', '')
-            recipe.waiting_time = recipe_yml['cook_time'].replace(' minutes', '')
+            recipe.servings = re.findall('([0-9])+', recipe_json['recipeYield'])[0]
         except Exception:
             pass
 
-        if recipe_yml['on_favorites']:
-            recipe.keywords.add(Keyword.objects.get_or_create(name=_('Favorites'), space=self.request.space))
+        try:
+            recipe.working_time = iso_duration_to_minutes(recipe_json['prepTime'])
+            recipe.waiting_time = iso_duration_to_minutes(recipe_json['cookTime'])
+        except Exception:
+            pass
 
-        step = Step.objects.create(instruction=recipe_yml['directions'], space=self.request.space, )
+        # assuming import files only contain single step
+        step = Step.objects.create(instruction=recipe_json['steps'][0]['instruction'], space=self.request.space,
+                                   show_ingredients_table=self.request.user.userpreference.show_step_ingredients, )
 
-        if 'notes' in recipe_yml and recipe_yml['notes'].strip() != '':
-            step.instruction = step.instruction + '\n\n' + recipe_yml['notes']
-
-        if 'nutritional_info' in recipe_yml:
-            step.instruction = step.instruction + '\n\n' + recipe_yml['nutritional_info']
-
-        if 'source' in recipe_yml and recipe_yml['source'].strip() != '':
-            step.instruction = step.instruction + '\n\n' + recipe_yml['source']
+        if 'nutrition' in recipe_json:
+            step.instruction = step.instruction + '\n\n' + recipe_json['nutrition']
 
         step.save()
         recipe.steps.add(step)
 
         ingredient_parser = IngredientParser(self.request, True)
-        for ingredient in recipe_yml['ingredients'].split('\n'):
-            if ingredient.strip() != '':
-                amount, unit, ingredient, note = ingredient_parser.parse(ingredient)
-                f = ingredient_parser.get_food(ingredient)
-                u = ingredient_parser.get_unit(unit)
-                step.ingredients.add(Ingredient.objects.create(
-                    food=f, unit=u, amount=amount, note=note, space=self.request.space,
-                ))
+        for ingredient in recipe_json['steps'][0]['ingredients']:
+            f = ingredient_parser.get_food(ingredient['food']['name'])
+            u = None
+            if unit := ingredient.get('unit', None):
+                u = ingredient_parser.get_unit(unit.get('name', None))
+            step.ingredients.add(Ingredient.objects.create(
+                food=f, unit=u, amount=ingredient.get('amount', None), note=ingredient.get('note', None), original_text=ingredient.get('original_text', None), space=self.request.space,
+            ))
+
+        if len(images) > 0:
+            try:
+                url = images[0]
+                if validate_import_url(url):
+                    response = requests.get(url)
+                    self.import_recipe_image(recipe, BytesIO(response.content))
+            except Exception as e:
+                print('failed to import image ', str(e))
 
         recipe.save()
         return recipe

cookbook/integration/cookmate.py

@@ -0,0 +1,83 @@
+from io import BytesIO
+
+import requests
+
+from cookbook.helper.HelperFunctions import validate_import_url
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
+from cookbook.integration.integration import Integration
+from cookbook.models import Ingredient, Recipe, Step
+
+
+class Cookmate(Integration):
+
+    def import_file_name_filter(self, zip_info_object):
+        return zip_info_object.filename.endswith('.xml')
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        raise NotImplementedError('Method not implemented in storage integration')
+
+    def get_recipe_from_file(self, file):
+        recipe_xml = file
+
+        recipe = Recipe.objects.create(
+            name=recipe_xml.find('title').text.strip(),
+            created_by=self.request.user, internal=True, space=self.request.space)
+
+        if recipe_xml.find('preptime') is not None and recipe_xml.find('preptime').text is not None:
+            recipe.working_time = parse_time(recipe_xml.find('preptime').text.strip())
+
+        if recipe_xml.find('cooktime') is not None and recipe_xml.find('cooktime').text is not None:
+            recipe.waiting_time = parse_time(recipe_xml.find('cooktime').text.strip())
+
+        if recipe_xml.find('quantity') is not None and recipe_xml.find('quantity').text is not None:
+            recipe.servings = parse_servings(recipe_xml.find('quantity').text.strip())
+            recipe.servings_text = parse_servings_text(recipe_xml.find('quantity').text.strip())
+
+        if recipe_xml.find('url') is not None and recipe_xml.find('url').text is not None:
+            recipe.source_url = recipe_xml.find('url').text.strip()
+
+        if recipe_xml.find('description') is not None:  # description is a list of <li>'s with text
+            if len(recipe_xml.find('description')) > 0:
+                recipe.description = recipe_xml.find('description')[0].text[:512]
+
+        if recipe_text := recipe_xml.find('recipetext'):
+            for step in recipe_text.getchildren():
+                if step.text:
+                    step = Step.objects.create(
+                        instruction=step.text.strip(), space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+                    )
+                    recipe.steps.add(step)
+
+        ingredient_parser = IngredientParser(self.request, True)
+
+        if recipe_ingredients := recipe_xml.find('ingredient'):
+            ingredient_step = recipe.steps.first()
+            if ingredient_step is None:
+                ingredient_step = Step.objects.create(space=self.request.space, instruction='')
+
+            for ingredient in recipe_ingredients.getchildren():
+                if ingredient.text:
+                    if ingredient.text.strip() != '':
+                        amount, unit, food, note = ingredient_parser.parse(ingredient.text.strip())
+                        f = ingredient_parser.get_food(food)
+                        u = ingredient_parser.get_unit(unit)
+                        ingredient_step.ingredients.add(Ingredient.objects.create(
+                            food=f, unit=u, amount=amount, note=note, original_text=ingredient.text.strip(), space=self.request.space,
+                        ))
+
+        if recipe_xml.find('imageurl') is not None:
+            try:
+                url = recipe_xml.find('imageurl').text.strip()
+                if validate_import_url(url):
+                    response = requests.get(url)
+                self.import_recipe_image(recipe, BytesIO(response.content))
+            except Exception as e:
+                print('failed to import image ', str(e))
+
+        recipe.save()
+
+        return recipe
+
+    def get_file_from_recipe(self, recipe):
+        raise NotImplementedError('Method not implemented in storage integration')

cookbook/integration/copymethat.py

@@ -0,0 +1,130 @@
+from io import BytesIO
+from zipfile import ZipFile
+
+from bs4 import BeautifulSoup, Tag
+from django.utils.translation import gettext as _
+
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import iso_duration_to_minutes, parse_servings
+from cookbook.integration.integration import Integration
+from cookbook.models import Ingredient, Keyword, Recipe, Step
+from recipes.settings import DEBUG
+
+
+class CopyMeThat(Integration):
+
+    def import_file_name_filter(self, zip_info_object):
+        if DEBUG:
+            print("testing", zip_info_object.filename, zip_info_object.filename == 'recipes.html')
+        return zip_info_object.filename == 'recipes.html'
+
+    def get_recipe_from_file(self, file):
+        # 'file' comes is as a beautifulsoup object
+        try:
+            source = file.find("a", {"id": "original_link"}).text
+        except AttributeError:
+            source = None
+
+        recipe = Recipe.objects.create(name=file.find("div", {"id": "name"}).text.strip(
+        )[:128], source_url=source, created_by=self.request.user, internal=True, space=self.request.space, )
+
+        for category in file.find_all("span", {"class": "recipeCategory"}):
+            keyword, created = Keyword.objects.get_or_create(name=category.text, space=self.request.space)
+            recipe.keywords.add(keyword)
+
+        try:
+            recipe.servings = parse_servings(file.find("a", {"id": "recipeYield"}).text.strip())
+            recipe.working_time = iso_duration_to_minutes(file.find("span", {"meta": "prepTime"}).text.strip())
+            recipe.waiting_time = iso_duration_to_minutes(file.find("span", {"meta": "cookTime"}).text.strip())
+        except AttributeError:
+            pass
+
+        try:
+            if len(file.find("span", {"id": "starred"}).text.strip()) > 0:
+                recipe.keywords.add(Keyword.objects.get_or_create(space=self.request.space, name=_('Favorite'))[0])
+        except AttributeError:
+            pass
+
+        try:
+            if len(file.find("span", {"id": "made_this"}).text.strip()) > 0:
+                recipe.keywords.add(Keyword.objects.get_or_create(space=self.request.space, name=_('I made this'))[0])
+        except AttributeError:
+            pass
+
+        step = Step.objects.create(instruction='', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients, )
+
+        ingredient_parser = IngredientParser(self.request, True)
+
+        ingredients = file.find("ul", {"id": "recipeIngredients"})
+        if isinstance(ingredients, Tag):
+            for ingredient in ingredients.children:
+                if not isinstance(ingredient, Tag) or not ingredient.text.strip() or "recipeIngredient_spacer" in ingredient['class']:
+                    continue
+                if any(x in ingredient['class'] for x in ["recipeIngredient_subheader", "recipeIngredient_note"]):
+                    step.ingredients.add(
+                        Ingredient.objects.create(
+                            is_header=True,
+                            note=ingredient.text.strip()[
+                                :256],
+                            original_text=ingredient.text.strip(),
+                            space=self.request.space,
+                        ))
+                else:
+                    amount, unit, food, note = ingredient_parser.parse(ingredient.text.strip())
+                    f = ingredient_parser.get_food(food)
+                    u = ingredient_parser.get_unit(unit)
+                    step.ingredients.add(Ingredient.objects.create(food=f, unit=u, amount=amount, note=note, original_text=ingredient.text.strip(), space=self.request.space, ))
+
+        instructions = file.find("ol", {"id": "recipeInstructions"})
+        if isinstance(instructions, Tag):
+            for instruction in instructions.children:
+                if not isinstance(instruction, Tag) or instruction.text == "":
+                    continue
+                if "instruction_subheader" in instruction['class']:
+                    if step.instruction:
+                        step.save()
+                        recipe.steps.add(step)
+                        step = Step.objects.create(instruction='', space=self.request.space, )
+
+                    step.name = instruction.text.strip()[:128]
+                else:
+                    step.instruction += instruction.text.strip() + ' \n\n'
+
+        notes = file.find_all("li", {"class": "recipeNote"})
+        if notes:
+            step.instruction += '*Notes:* \n\n'
+
+            for n in notes:
+                if n.text == "":
+                    continue
+                step.instruction += '*' + n.text.strip() + '* \n\n'
+
+        description = ''
+        try:
+            description = file.find("div", {"id": "description"}).text.strip()
+        except AttributeError:
+            pass
+        if len(description) <= 512:
+            recipe.description = description
+        else:
+            recipe.description = description[:480] + ' ... (full description below)'
+            step.instruction += '*Description:* \n\n*' + description + '* \n\n'
+
+        step.save()
+        recipe.steps.add(step)
+
+        # import the Primary recipe image that is stored in the Zip
+        try:
+            for f in self.files:
+                if '.zip' in f['name']:
+                    import_zip = ZipFile(f['file'])
+                    self.import_recipe_image(recipe, BytesIO(import_zip.read(file.find("img", class_="recipeImage").get("src"))), filetype='.jpeg')
+        except Exception as e:
+            print(recipe.name, ': failed to import image ', str(e))
+
+        recipe.save()
+        return recipe
+
+    def split_recipe_file(self, file):
+        soup = BeautifulSoup(file, "html.parser")
+        return soup.find_all("div", {"class": "recipe"})

cookbook/integration/default.py

@@ -1,5 +1,6 @@
 import json
-from io import BytesIO
+import traceback
+from io import BytesIO, StringIO
 from re import match
 from zipfile import ZipFile
 
@@ -19,7 +20,10 @@ class Default(Integration):
         recipe = self.decode_recipe(recipe_string)
         images = list(filter(lambda v: match('image.*', v), recipe_zip.namelist()))
         if images:
-            self.import_recipe_image(recipe, BytesIO(recipe_zip.read(images[0])), filetype=get_filetype(images[0]))
+            try:
+                self.import_recipe_image(recipe, BytesIO(recipe_zip.read(images[0])), filetype=get_filetype(images[0]))
+            except AttributeError:
+                traceback.print_exc()
         return recipe
 
     def decode_recipe(self, string):
@@ -32,6 +36,39 @@ class Default(Integration):
         return None
 
     def get_file_from_recipe(self, recipe):
+
         export = RecipeExportSerializer(recipe).data
 
         return 'recipe.json', JSONRenderer().render(export).decode("utf-8")
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        export_zip_stream = BytesIO()
+        export_zip_obj = ZipFile(export_zip_stream, 'w')
+
+        for r in recipes:
+            if r.internal and r.space == self.request.space:
+                recipe_zip_stream = BytesIO()
+                recipe_zip_obj = ZipFile(recipe_zip_stream, 'w')
+
+                recipe_stream = StringIO()
+                filename, data = self.get_file_from_recipe(r)
+                recipe_stream.write(data)
+                recipe_zip_obj.writestr(filename, recipe_stream.getvalue())
+                recipe_stream.close()
+
+                try:
+                    recipe_zip_obj.writestr(f'image{get_filetype(r.image.file.name)}', r.image.file.read())
+                except (ValueError, FileNotFoundError):
+                    pass
+
+                recipe_zip_obj.close()
+
+                export_zip_obj.writestr(str(r.pk) + '.zip', recipe_zip_stream.getvalue())
+
+            el.exported_recipes += 1
+            el.msg += self.get_recipe_processed_msg(r)
+            el.save()
+
+        export_zip_obj.close()
+
+        return [[self.get_export_file_name(), export_zip_stream.getvalue()]]

cookbook/integration/domestica.py

@@ -4,7 +4,7 @@ from io import BytesIO
 
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient
+from cookbook.models import Ingredient, Recipe, Step
 
 
 class Domestica(Integration):
@@ -28,7 +28,7 @@ class Domestica(Integration):
         recipe.save()
 
         step = Step.objects.create(
-            instruction=file['directions'], space=self.request.space,
+            instruction=file['directions'], space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
         )
 
         if file['source'] != '':
@@ -37,11 +37,11 @@ class Domestica(Integration):
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in file['ingredients'].split('\n'):
             if len(ingredient.strip()) > 0:
-                amount, unit, ingredient, note = ingredient_parser.parse(ingredient)
-                f = ingredient_parser.get_food(ingredient)
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
+                f = ingredient_parser.get_food(food)
                 u = ingredient_parser.get_unit(unit)
                 step.ingredients.add(Ingredient.objects.create(
-                    food=f, unit=u, amount=amount, note=note, space=self.request.space,
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                 ))
         recipe.steps.add(step)
 

cookbook/integration/integration.py

@@ -1,10 +1,11 @@
 import datetime
-import json
 import traceback
 import uuid
-from io import BytesIO, StringIO
-from zipfile import ZipFile, BadZipFile
+from io import BytesIO
+from zipfile import BadZipFile, ZipFile
 
+from bs4 import Tag
+from django.core.cache import cache
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.files import File
 from django.db import IntegrityError
@@ -12,11 +13,11 @@ from django.http import HttpResponse
 from django.utils.formats import date_format
 from django.utils.translation import gettext as _
 from django_scopes import scope
+from lxml import etree
 
-from cookbook.forms import ImportExportBase
-from cookbook.helper.image_processing import get_filetype
+from cookbook.helper.image_processing import handle_image
 from cookbook.models import Keyword, Recipe
-from recipes.settings import DATABASES, DEBUG
+from recipes.settings import DEBUG, EXPORT_FILE_CACHE_DURATION
 
 
 class Integration:
@@ -35,13 +36,12 @@ class Integration:
         self.export_type = export_type
         self.ignored_recipes = []
 
-        description = f'Imported by {request.user.get_user_name()} at {date_format(datetime.datetime.now(), "DATETIME_FORMAT")}. Type: {export_type}'
-        icon = '📥'
+        description = f'Imported by {request.user.get_user_display_name()} at {date_format(datetime.datetime.now(), "DATETIME_FORMAT")}. Type: {export_type}'
 
         try:
             last_kw = Keyword.objects.filter(name__regex=r'^(Import [0-9]+)', space=request.space).latest('created_at')
             name = f'Import {int(last_kw.name.replace("Import ", "")) + 1}'
-        except ObjectDoesNotExist:
+        except (ObjectDoesNotExist, ValueError):
             name = 'Import 1'
 
         parent, created = Keyword.objects.get_or_create(name='Import', space=request.space)
@@ -49,60 +49,48 @@ class Integration:
             self.keyword = parent.add_child(
                 name=name,
                 description=description,
-                icon=icon,
                 space=request.space
             )
-        except IntegrityError: # in case, for whatever reason, the name does exist append UUID to it. Not nice but works for now.
+        except (IntegrityError, ValueError):  # in case, for whatever reason, the name does exist append UUID to it. Not nice but works for now.
             self.keyword = parent.add_child(
                 name=f'{name} {str(uuid.uuid4())[0:8]}',
                 description=description,
-                icon=icon,
                 space=request.space
             )
 
-    def do_export(self, recipes):
-        """
-        Perform the export based on a list of recipes
-        :param recipes: list of recipe objects
-        :return: HttpResponse with a ZIP file that is directly downloaded
-        """
+    def do_export(self, recipes, el):
 
-        # TODO this is temporary, find a better solution for different export formats when doing other exporters
-        if self.export_type != ImportExportBase.RECIPESAGE:
-            export_zip_stream = BytesIO()
-            export_zip_obj = ZipFile(export_zip_stream, 'w')
+        with scope(space=self.request.space):
+            el.total_recipes = len(recipes)
+            el.cache_duration = EXPORT_FILE_CACHE_DURATION
+            el.save()
 
-            for r in recipes:
-                if r.internal and r.space == self.request.space:
-                    recipe_zip_stream = BytesIO()
-                    recipe_zip_obj = ZipFile(recipe_zip_stream, 'w')
+            files = self.get_files_from_recipes(recipes, el, self.request.COOKIES)
 
-                    recipe_stream = StringIO()
-                    filename, data = self.get_file_from_recipe(r)
-                    recipe_stream.write(data)
-                    recipe_zip_obj.writestr(filename, recipe_stream.getvalue())
-                    recipe_stream.close()
-                    try:
-                        recipe_zip_obj.writestr(f'image{get_filetype(r.image.file.name)}', r.image.file.read())
-                    except ValueError:
-                        pass
+            if len(files) == 1:
+                filename, file = files[0]
+                export_filename = filename
+                export_file = file
 
-                    recipe_zip_obj.close()
-                    export_zip_obj.writestr(str(r.pk) + '.zip', recipe_zip_stream.getvalue())
+            else:
+                # zip the files if there is more then one file
+                export_filename = self.get_export_file_name()
+                export_stream = BytesIO()
+                export_obj = ZipFile(export_stream, 'w')
 
-            export_zip_obj.close()
+                for filename, file in files:
+                    export_obj.writestr(filename, file)
 
-            response = HttpResponse(export_zip_stream.getvalue(), content_type='application/force-download')
-            response['Content-Disposition'] = 'attachment; filename="export.zip"'
-            return response
-        else:
-            json_list = []
-            for r in recipes:
-                json_list.append(self.get_file_from_recipe(r))
+                export_obj.close()
+                export_file = export_stream.getvalue()
 
-            response = HttpResponse(json.dumps(json_list), content_type='application/force-download')
-            response['Content-Disposition'] = 'attachment; filename="recipes.json"'
-            return response
+            cache.set('export_file_' + str(el.pk), {'filename': export_filename, 'file': export_file}, EXPORT_FILE_CACHE_DURATION)
+            el.running = False
+            el.save()
+
+        response = HttpResponse(export_file, content_type='application/force-download')
+        response['Content-Disposition'] = 'attachment; filename="' + export_filename + '"'
+        return response
 
     def import_file_name_filter(self, zip_info_object):
         """
@@ -123,8 +111,6 @@ class Integration:
         :return: HttpResponseRedirect to the recipe search showing all imported recipes
         """
         with scope(space=self.request.space):
-            self.keyword.name = _('Import') + ' ' + str(il.pk)
-            self.keyword.save()
 
             try:
                 self.files = files
@@ -142,12 +128,12 @@ class Integration:
                             for d in data_list:
                                 recipe = self.get_recipe_from_file(d)
                                 recipe.keywords.add(self.keyword)
-                                il.msg += f'{recipe.pk} - {recipe.name} \n'
+                                il.msg += self.get_recipe_processed_msg(recipe)
                                 self.handle_duplicates(recipe, import_duplicates)
                                 il.imported_recipes += 1
                                 il.save()
                         import_zip.close()
-                    elif '.zip' in f['name'] or '.paprikarecipes' in f['name']:
+                    elif '.zip' in f['name'] or '.paprikarecipes' in f['name'] or '.mcb' in f['name']:
                         import_zip = ZipFile(f['file'])
                         file_list = []
                         for z in import_zip.filelist:
@@ -155,11 +141,26 @@ class Integration:
                                 file_list.append(z)
                         il.total_recipes += len(file_list)
 
+                        import cookbook
+                        if isinstance(self, cookbook.integration.copymethat.CopyMeThat):
+                            file_list = self.split_recipe_file(BytesIO(import_zip.read('recipes.html')))
+                            il.total_recipes += len(file_list)
+
+                        if isinstance(self, cookbook.integration.cookmate.Cookmate):
+                            new_file_list = []
+                            for file in file_list:
+                                new_file_list += etree.parse(BytesIO(import_zip.read(file.filename))).getroot().getchildren()
+                            il.total_recipes = len(new_file_list)
+                            file_list = new_file_list
+
                         for z in file_list:
                             try:
-                                recipe = self.get_recipe_from_file(BytesIO(import_zip.read(z.filename)))
+                                if not hasattr(z, 'filename') or isinstance(z, Tag):
+                                    recipe = self.get_recipe_from_file(z)
+                                else:
+                                    recipe = self.get_recipe_from_file(BytesIO(import_zip.read(z.filename)))
                                 recipe.keywords.add(self.keyword)
-                                il.msg += f'{recipe.pk} - {recipe.name} \n'
+                                il.msg += self.get_recipe_processed_msg(recipe)
                                 self.handle_duplicates(recipe, import_duplicates)
                                 il.imported_recipes += 1
                                 il.save()
@@ -167,14 +168,14 @@ class Integration:
                                 traceback.print_exc()
                                 self.handle_exception(e, log=il, message=f'-------------------- \nERROR \n{e}\n--------------------\n')
                         import_zip.close()
-                    elif '.json' in f['name'] or '.txt' in f['name'] or '.mmf' in f['name']:
+                    elif '.json' in f['name'] or '.xml' in f['name'] or '.txt' in f['name'] or '.mmf' in f['name'] or '.rk' in f['name'] or '.melarecipe' in f['name']:
                         data_list = self.split_recipe_file(f['file'])
                         il.total_recipes += len(data_list)
                         for d in data_list:
                             try:
                                 recipe = self.get_recipe_from_file(d)
                                 recipe.keywords.add(self.keyword)
-                                il.msg += f'{recipe.pk} - {recipe.name} \n'
+                                il.msg += self.get_recipe_processed_msg(recipe)
                                 self.handle_duplicates(recipe, import_duplicates)
                                 il.imported_recipes += 1
                                 il.save()
@@ -191,7 +192,7 @@ class Integration:
                                     try:
                                         recipe = self.get_recipe_from_file(d)
                                         recipe.keywords.add(self.keyword)
-                                        il.msg += f'{recipe.pk} - {recipe.name} \n'
+                                        il.msg += self.get_recipe_processed_msg(recipe)
                                         self.handle_duplicates(recipe, import_duplicates)
                                         il.imported_recipes += 1
                                         il.save()
@@ -201,7 +202,7 @@ class Integration:
                     else:
                         recipe = self.get_recipe_from_file(f['file'])
                         recipe.keywords.add(self.keyword)
-                        il.msg += f'{recipe.pk} - {recipe.name} \n'
+                        il.msg += self.get_recipe_processed_msg(recipe)
                         self.handle_duplicates(recipe, import_duplicates)
             except BadZipFile:
                 il.msg += 'ERROR ' + _(
@@ -231,15 +232,14 @@ class Integration:
             self.ignored_recipes.append(recipe.name)
             recipe.delete()
 
-    @staticmethod
-    def import_recipe_image(recipe, image_file, filetype='.jpeg'):
+    def import_recipe_image(self, recipe, image_file, filetype='.jpeg'):
         """
         Adds an image to a recipe naming it correctly
         :param recipe: Recipe object
         :param image_file: ByteIO stream containing the image
         :param filetype: type of file to write bytes to, default to .jpeg if unknown
         """
-        recipe.image = File(image_file, name=f'{uuid.uuid4()}_{recipe.pk}{filetype}')
+        recipe.image = File(handle_image(self.request, File(image_file, name='image'), filetype=filetype), name=f'{uuid.uuid4()}_{recipe.pk}{filetype}')
         recipe.save()
 
     def get_recipe_from_file(self, file):
@@ -269,7 +269,18 @@ class Integration:
         """
         raise NotImplementedError('Method not implemented in integration')
 
-    def handle_exception(self, exception, log=None, message=''):
+    def get_files_from_recipes(self, recipes, el, cookie):
+        """
+        Takes a list of recipe object and converts it to a array containing each file.
+        Each file is represented as an array [filename, data] where data is a string of the content of the file.
+        :param recipe: Recipe object that should be converted
+        :returns:
+            [[filename, data], ...]
+        """
+        raise NotImplementedError('Method not implemented in integration')
+
+    @staticmethod
+    def handle_exception(exception, log=None, message=''):
         if log:
             if message:
                 log.msg += message
@@ -277,3 +288,9 @@ class Integration:
                 log.msg += exception.msg
         if DEBUG:
             traceback.print_exc()
+
+    def get_export_file_name(self, format='zip'):
+        return "export_{}.{}".format(datetime.datetime.now().strftime("%Y-%m-%d"), format)
+
+    def get_recipe_processed_msg(self, recipe):
+        return f'{recipe.pk} - {recipe.name} \n'

cookbook/integration/mealie.py

@@ -5,14 +5,15 @@ from zipfile import ZipFile
 
 from cookbook.helper.image_processing import get_filetype
 from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class Mealie(Integration):
 
     def import_file_name_filter(self, zip_info_object):
-        return re.match(r'^recipes/([A-Za-z\d-])+/([A-Za-z\d-])+.json$', zip_info_object.filename)
+        return re.match(r'^recipes/([A-Za-z\d\s\-_()\[\]\u00C0-\u017F])+/([A-Za-z\d\s\-_()\[\]\u00C0-\u017F])+.json$', zip_info_object.filename)
 
     def get_recipe_from_file(self, file):
         recipe_json = json.loads(file.getvalue().decode("utf-8"))
@@ -23,44 +24,72 @@ class Mealie(Integration):
             name=recipe_json['name'].strip(), description=description,
             created_by=self.request.user, internal=True, space=self.request.space)
 
-        # TODO parse times (given in PT2H3M )
-        # @vabene check recipe_url_import.iso_duration_to_minutes  I think it does what you are looking for
-
-        ingredients_added = False
         for s in recipe_json['recipe_instructions']:
-            step = Step.objects.create(
-                instruction=s['text'], space=self.request.space,
-            )
-            if not ingredients_added:
-                ingredients_added = True
-
-                if len(recipe_json['description'].strip()) > 500:
-                    step.instruction = recipe_json['description'].strip() + '\n\n' + step.instruction
-
-                ingredient_parser = IngredientParser(self.request, True)
-                for ingredient in recipe_json['recipe_ingredient']:
-                    try:
-                        if ingredient['food']:
-                            f = ingredient_parser.get_food(ingredient['food'])
-                            u = ingredient_parser.get_unit(ingredient['unit'])
-                            amount = ingredient['quantity']
-                            note = ingredient['note']
-                        else:
-                            amount, unit, ingredient, note = ingredient_parser.parse(ingredient['note'])
-                            f = ingredient_parser.get_food(ingredient)
-                            u = ingredient_parser.get_unit(unit)
-                        step.ingredients.add(Ingredient.objects.create(
-                            food=f, unit=u, amount=amount, note=note, space=self.request.space,
-                        ))
-                    except Exception:
-                        pass
+            step = Step.objects.create(instruction=s['text'], space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients, )
             recipe.steps.add(step)
 
+        step = recipe.steps.first()
+        if not step:  # if there is no step in the exported data
+            step = Step.objects.create(instruction='', space=self.request.space, )
+            recipe.steps.add(step)
+
+        if len(recipe_json['description'].strip()) > 500:
+            step.instruction = recipe_json['description'].strip() + '\n\n' + step.instruction
+
+        ingredient_parser = IngredientParser(self.request, True)
+        for ingredient in recipe_json['recipe_ingredient']:
+            try:
+                if ingredient['food']:
+                    f = ingredient_parser.get_food(ingredient['food'])
+                    u = ingredient_parser.get_unit(ingredient['unit'])
+                    amount = ingredient['quantity']
+                    note = ingredient['note']
+                    original_text = None
+                else:
+                    amount, unit, food, note = ingredient_parser.parse(ingredient['note'])
+                    f = ingredient_parser.get_food(food)
+                    u = ingredient_parser.get_unit(unit)
+                    original_text = ingredient['note']
+                step.ingredients.add(Ingredient.objects.create(
+                    food=f, unit=u, amount=amount, note=note, original_text=original_text, space=self.request.space,
+                ))
+            except Exception:
+                pass
+
+        if 'tags' in recipe_json and len(recipe_json['tags']) > 0:
+            for k in recipe_json['tags']:
+                if 'name' in k:
+                    keyword, created = Keyword.objects.get_or_create(name=k['name'].strip(), space=self.request.space)
+                    recipe.keywords.add(keyword)
+
+        if 'notes' in recipe_json and len(recipe_json['notes']) > 0:
+            notes_text = "#### Notes  \n\n"
+            for n in recipe_json['notes']:
+                notes_text += f'{n["text"]}  \n'
+
+            step = Step.objects.create(
+                instruction=notes_text, space=self.request.space,
+            )
+            recipe.steps.add(step)
+
+        if 'recipe_yield' in recipe_json:
+            recipe.servings = parse_servings(recipe_json['recipe_yield'])
+            recipe.servings_text = parse_servings_text(recipe_json['recipe_yield'])
+
+        if 'total_time' in recipe_json and recipe_json['total_time'] is not None:
+            recipe.working_time = parse_time(recipe_json['total_time'])
+
+        if 'org_url' in recipe_json:
+            recipe.source_url = recipe_json['org_url']
+
+        recipe.save()
+
         for f in self.files:
             if '.zip' in f['name']:
                 import_zip = ZipFile(f['file'])
                 try:
-                    self.import_recipe_image(recipe, BytesIO(import_zip.read(f'recipes/{recipe_json["slug"]}/images/min-original.webp')), filetype=get_filetype(f'recipes/{recipe_json["slug"]}/images/original'))
+                    self.import_recipe_image(recipe, BytesIO(import_zip.read(f'recipes/{recipe_json["slug"]}/images/min-original.webp')),
+                                             filetype=get_filetype(f'recipes/{recipe_json["slug"]}/images/original'))
                 except Exception:
                     pass
 

cookbook/integration/mealmaster.py

@@ -2,7 +2,7 @@ import re
 
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient, Keyword
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class MealMaster(Integration):
@@ -22,7 +22,7 @@ class MealMaster(Integration):
                         if 'Yield:' in line:
                             servings_text = line.replace('Yield:', '').strip()
                         else:
-                            if re.match('\s{2,}([0-9])+', line):
+                            if re.match(r'\s{2,}([0-9])+', line):
                                 ingredients.append(line.strip())
                             else:
                                 directions.append(line.strip())
@@ -39,17 +39,17 @@ class MealMaster(Integration):
             recipe.keywords.add(keyword)
 
         step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n', space=self.request.space,
+            instruction='\n'.join(directions) + '\n\n', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
         )
 
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
             if len(ingredient.strip()) > 0:
-                amount, unit, ingredient, note = ingredient_parser.parse(ingredient)
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
                 f = ingredient_parser.get_food(ingredient)
                 u = ingredient_parser.get_unit(unit)
                 step.ingredients.add(Ingredient.objects.create(
-                    food=f, unit=u, amount=amount, note=note, space=self.request.space,
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                 ))
         recipe.steps.add(step)
 
@@ -63,7 +63,7 @@ class MealMaster(Integration):
         current_recipe = ''
 
         for fl in file.readlines():
-            line = fl.decode("ANSI")
+            line = fl.decode("windows-1250")
             if (line.startswith('MMMMM') or line.startswith('-----')) and 'meal-master' in line.lower():
                 if current_recipe != '':
                     recipe_list.append(current_recipe)

cookbook/integration/melarecipes.py

@@ -0,0 +1,83 @@
+import base64
+import json
+from io import BytesIO
+
+from gettext import gettext as _
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_time
+from cookbook.integration.integration import Integration
+from cookbook.models import Ingredient, Keyword, Recipe, Step
+
+
+class MelaRecipes(Integration):
+
+    def split_recipe_file(self, file):
+        return [json.loads(file.getvalue().decode("utf-8"))]
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        raise NotImplementedError('Method not implemented in storage integration')
+
+    def get_recipe_from_file(self, file):
+        recipe_json = file
+
+        recipe = Recipe.objects.create(
+            name=recipe_json['title'].strip(),
+            created_by=self.request.user, internal=True, space=self.request.space)
+
+        if 'yield' in recipe_json:
+            recipe.servings = parse_servings(recipe_json['yield'])
+
+        if 'cookTime' in recipe_json:
+            recipe.waiting_time = parse_time(recipe_json['cookTime'])
+
+        if 'prepTime' in recipe_json:
+            recipe.working_time = parse_time(recipe_json['prepTime'])
+
+        if 'favorite' in recipe_json and recipe_json['favorite']:
+            recipe.keywords.add(Keyword.objects.get_or_create(space=self.request.space, name=_('Favorite'))[0])
+
+        if 'categories' in recipe_json:
+            try:
+                for x in recipe_json['categories']:
+                    recipe.keywords.add(Keyword.objects.get_or_create(space=self.request.space, name=x)[0])
+            except Exception:
+                pass
+
+        instruction = ''
+        if 'text' in recipe_json:
+            instruction += f'*{recipe_json["text"].strip()}*  \n'
+
+        if 'instructions' in recipe_json:
+            instruction += recipe_json["instructions"].strip() + '  \n'
+
+        if 'notes' in recipe_json:
+            instruction += recipe_json["notes"].strip() + '  \n'
+
+        if 'link' in recipe_json:
+            recipe.source_url = recipe_json['link']
+
+        step = Step.objects.create(
+            instruction=instruction, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients
+        )
+
+        ingredient_parser = IngredientParser(self.request, True)
+        for ingredient in recipe_json['ingredients'].split('\n'):
+            if ingredient.strip() != '':
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
+                f = ingredient_parser.get_food(food)
+                u = ingredient_parser.get_unit(unit)
+                step.ingredients.add(Ingredient.objects.create(
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
+                ))
+        recipe.steps.add(step)
+
+        if recipe_json.get("images", None):
+            try:
+                self.import_recipe_image(recipe, BytesIO(base64.b64decode(recipe_json['images'][0])), filetype='.jpeg')
+            except Exception:
+                pass
+
+        return recipe
+
+    def get_file_from_recipe(self, recipe):
+        raise NotImplementedError('Method not implemented in storage integration')

cookbook/integration/nextcloud_cookbook.py

@@ -1,12 +1,15 @@
 import json
 import re
-from io import BytesIO
+from io import BytesIO, StringIO
 from zipfile import ZipFile
 
+from PIL import Image
+
 from cookbook.helper.image_processing import get_filetype
 from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import iso_duration_to_minutes
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient
+from cookbook.models import Ingredient, Keyword, NutritionInformation, Recipe, Step
 
 
 class NextcloudCookbook(Integration):
@@ -24,39 +27,166 @@ class NextcloudCookbook(Integration):
             created_by=self.request.user, internal=True,
             servings=recipe_json['recipeYield'], space=self.request.space)
 
-        # TODO parse times (given in PT2H3M )
-        # @vabene check recipe_url_import.iso_duration_to_minutes  I think it does what you are looking for
-        # TODO parse keywords
+        try:
+            recipe.working_time = iso_duration_to_minutes(recipe_json['prepTime'])
+            recipe.waiting_time = iso_duration_to_minutes(recipe_json['cookTime'])
+        except Exception:
+            pass
+
+        if 'url' in recipe_json:
+            recipe.source_url = recipe_json['url'].strip()
+
+        if 'recipeCategory' in recipe_json:
+            try:
+                recipe.keywords.add(Keyword.objects.get_or_create(space=self.request.space, name=recipe_json['recipeCategory'])[0])
+            except Exception:
+                pass
+
+        if 'keywords' in recipe_json:
+            try:
+                for x in recipe_json['keywords'].split(','):
+                    if x.strip() != '':
+                        recipe.keywords.add(Keyword.objects.get_or_create(space=self.request.space, name=x)[0])
+            except Exception:
+                pass
 
         ingredients_added = False
         for s in recipe_json['recipeInstructions']:
-            step = Step.objects.create(
-                instruction=s, space=self.request.space,
-            )
-            if not ingredients_added:
-                if len(recipe_json['description'].strip()) > 500:
-                    step.instruction = recipe_json['description'].strip() + '\n\n' + step.instruction
+            if 'text' in s:
+                step = Step.objects.create(
+                    instruction=s['text'], name=s['name'], space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+                )
+            else:
+                step = Step.objects.create(
+                    instruction=s, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+                )
 
-                ingredients_added = True
-
-                ingredient_parser = IngredientParser(self.request, True)
+            ingredient_parser = IngredientParser(self.request, True)
+            if ingredients_added == False:
                 for ingredient in recipe_json['recipeIngredient']:
-                    amount, unit, ingredient, note = parse(ingredient)
-                    f = get_food(ingredient, self.request.space)
-                    u = get_unit(unit, self.request.space)
-                    step.ingredients.add(Ingredient.objects.create(
-                        food=f, unit=u, amount=amount, note=note, space=self.request.space,
-                    ))
+                    ingredients_added = True
+                    if ingredient.startswith('##'):
+                        subheader = ingredient.replace('##', '', 1)
+                        step.ingredients.add(Ingredient.objects.create(note=subheader, is_header=True, no_amount=True, space=self.request.space))
+                    else:
+                        amount, unit, food, note = ingredient_parser.parse(ingredient)
+                        f = ingredient_parser.get_food(food)
+                        u = ingredient_parser.get_unit(unit)
+                        step.ingredients.add(Ingredient.objects.create(
+                            food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,))
             recipe.steps.add(step)
 
+        if 'nutrition' in recipe_json:
+            nutrition = {}
+            try:
+                if 'calories' in recipe_json['nutrition']:
+                    nutrition['calories'] = int(re.search(r'\d+', recipe_json['nutrition']['calories']).group())
+                if 'proteinContent' in recipe_json['nutrition']:
+                    nutrition['proteins'] = int(re.search(r'\d+', recipe_json['nutrition']['proteinContent']).group())
+                if 'fatContent' in recipe_json['nutrition']:
+                    nutrition['fats'] = int(re.search(r'\d+', recipe_json['nutrition']['fatContent']).group())
+                if 'carbohydrateContent' in recipe_json['nutrition']:
+                    nutrition['carbohydrates'] = int(re.search(r'\d+', recipe_json['nutrition']['carbohydrateContent']).group())
+
+                if nutrition != {}:
+                    recipe.nutrition = NutritionInformation.objects.create(**nutrition, space=self.request.space)
+                    recipe.save()
+            except Exception:
+                pass
+
         for f in self.files:
             if '.zip' in f['name']:
                 import_zip = ZipFile(f['file'])
                 for z in import_zip.filelist:
-                    if re.match(f'^Recipes/{recipe.name}/full.jpg$', z.filename):
+                    if re.match(f'^(.)+{recipe.name}/full.jpg$', z.filename):
                         self.import_recipe_image(recipe, BytesIO(import_zip.read(z.filename)), filetype=get_filetype(z.filename))
 
         return recipe
 
+    def formatTime(self, min):
+        h = min // 60
+        m = min % 60
+        return f'PT{h}H{m}M0S'
+
     def get_file_from_recipe(self, recipe):
-        raise NotImplementedError('Method not implemented in storage integration')
+
+        export = {}
+        export['name'] = recipe.name
+        export['description'] = recipe.description
+        export['url'] = recipe.source_url
+        export['prepTime'] = self.formatTime(recipe.working_time)
+        export['cookTime'] = self.formatTime(recipe.waiting_time)
+        export['totalTime'] = self.formatTime(recipe.working_time + recipe.waiting_time)
+        export['recipeYield'] = recipe.servings
+        export['image'] = f'/Recipes/{recipe.name}/full.jpg'
+        export['imageUrl'] = f'/Recipes/{recipe.name}/full.jpg'
+
+        recipeKeyword = []
+        for k in recipe.keywords.all():
+            recipeKeyword.append(k.name)
+
+        export['keywords'] = recipeKeyword
+
+        recipeInstructions = []
+        recipeIngredient = []
+        for s in recipe.steps.all():
+            recipeInstructions.append(s.instruction)
+
+            for i in s.ingredients.all():
+                recipeIngredient.append(f'{float(i.amount)} {i.unit} {i.food}')
+
+        export['recipeIngredient'] = recipeIngredient
+        export['recipeInstructions'] = recipeInstructions
+
+        return "recipe.json", json.dumps(export)
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        export_zip_stream = BytesIO()
+        export_zip_obj = ZipFile(export_zip_stream, 'w')
+
+        for recipe in recipes:
+            if recipe.internal and recipe.space == self.request.space:
+
+                recipe_stream = StringIO()
+                filename, data = self.get_file_from_recipe(recipe)
+                recipe_stream.write(data)
+                export_zip_obj.writestr(f'{recipe.name}/{filename}', recipe_stream.getvalue())
+                recipe_stream.close()
+
+                try:
+                    imageByte = recipe.image.file.read()
+                    export_zip_obj.writestr(f'{recipe.name}/full.jpg', self.getJPEG(imageByte))
+                    export_zip_obj.writestr(f'{recipe.name}/thumb.jpg', self.getThumb(171, imageByte))
+                    export_zip_obj.writestr(f'{recipe.name}/thumb16.jpg', self.getThumb(16, imageByte))
+                except ValueError:
+                    pass
+
+            el.exported_recipes += 1
+            el.msg += self.get_recipe_processed_msg(recipe)
+            el.save()
+
+        export_zip_obj.close()
+
+        return [[self.get_export_file_name(), export_zip_stream.getvalue()]]
+
+    def getJPEG(self, imageByte):
+        image = Image.open(BytesIO(imageByte))
+        image = image.convert('RGB')
+
+        bytes = BytesIO()
+        image.save(bytes, "JPEG")
+        return bytes.getvalue()
+
+    def getThumb(self, size, imageByte):
+        image = Image.open(BytesIO(imageByte))
+
+        w, h = image.size
+        m = min(w, h)
+
+        image = image.crop(((w - m) // 2, (h - m) // 2, (w + m) // 2, (h + m) // 2))
+        image = image.resize([size, size], Image.Resampling.LANCZOS)
+        image = image.convert('RGB')
+
+        bytes = BytesIO()
+        image.save(bytes, "JPEG")
+        return bytes.getvalue()

cookbook/integration/openeats.py

@@ -1,27 +1,59 @@
 import json
 
+from django.utils.translation import gettext as _
+
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient
+from cookbook.models import Comment, CookLog, Ingredient, Keyword, Recipe, Step
 
 
 class OpenEats(Integration):
 
     def get_recipe_from_file(self, file):
-        recipe = Recipe.objects.create(name=file['name'].strip(), created_by=self.request.user, internal=True,
+
+        description = file['info']
+        description_max_length = Recipe._meta.get_field('description').max_length
+        if len(description) > description_max_length:
+            description = description[0:description_max_length]
+
+        recipe = Recipe.objects.create(name=file['name'].strip(), description=description, created_by=self.request.user, internal=True,
                                        servings=file['servings'], space=self.request.space, waiting_time=file['cook_time'], working_time=file['prep_time'])
 
         instructions = ''
-        if file["info"] != '':
-            instructions += file["info"]
 
         if file["directions"] != '':
             instructions += file["directions"]
 
         if file["source"] != '':
-            instructions += file["source"]
+            instructions += '\n' + _('Recipe source:') + f'[{file["source"]}]({file["source"]})'
 
-        step = Step.objects.create(instruction=instructions, space=self.request.space,)
+        cuisine_keyword, created = Keyword.objects.get_or_create(name="Cuisine", space=self.request.space)
+        if file["cuisine"] != '':
+            keyword, created = Keyword.objects.get_or_create(name=file["cuisine"].strip(), space=self.request.space)
+            if created:
+                keyword.move(cuisine_keyword, pos="last-child")
+            recipe.keywords.add(keyword)
+
+        course_keyword, created = Keyword.objects.get_or_create(name="Course", space=self.request.space)
+        if file["course"] != '':
+            keyword, created = Keyword.objects.get_or_create(name=file["course"].strip(), space=self.request.space)
+            if created:
+                keyword.move(course_keyword, pos="last-child")
+            recipe.keywords.add(keyword)
+
+        for tag in file["tags"]:
+            keyword, created = Keyword.objects.get_or_create(name=tag.strip(), space=self.request.space)
+            recipe.keywords.add(keyword)
+
+        for comment in file['comments']:
+            Comment.objects.create(recipe=recipe, text=comment['text'], created_by=self.request.user)
+            CookLog.objects.create(recipe=recipe, rating=comment['rating'], created_by=self.request.user, space=self.request.space)
+
+        if file["photo"] != '':
+            recipe.image = f'recipes/openeats-import/{file["photo"]}'
+            recipe.save()
+
+        step = Step.objects.create(instruction=instructions, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,)
 
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in file['ingredients']:
@@ -38,6 +70,9 @@ class OpenEats(Integration):
         recipe_json = json.loads(file.read())
         recipe_dict = {}
         ingredient_group_dict = {}
+        cuisine_group_dict = {}
+        course_group_dict = {}
+        tag_group_dict = {}
 
         for o in recipe_json:
             if o['model'] == 'recipe.recipe':
@@ -50,11 +85,27 @@ class OpenEats(Integration):
                     'cook_time': o['fields']['cook_time'],
                     'servings': o['fields']['servings'],
                     'ingredients': [],
+                    'photo': o['fields']['photo'],
+                    'cuisine': o['fields']['cuisine'],
+                    'course': o['fields']['course'],
+                    'tags': o['fields']['tags'],
+                    'comments': [],
                 }
             if o['model'] == 'ingredient.ingredientgroup':
                 ingredient_group_dict[o['pk']] = o['fields']['recipe']
+            if o['model'] == 'recipe_groups.cuisine':
+                cuisine_group_dict[o['pk']] = o['fields']['title']
+            if o['model'] == 'recipe_groups.course':
+                course_group_dict[o['pk']] = o['fields']['title']
+            if o['model'] == 'recipe_groups.tag':
+                tag_group_dict[o['pk']] = o['fields']['title']
 
         for o in recipe_json:
+            if o['model'] == 'rating.rating':
+                recipe_dict[o['fields']['recipe']]["comments"].append({
+                    "text": o['fields']['comment'],
+                    "rating": o['fields']['rating']
+                })
             if o['model'] == 'ingredient.ingredient':
                 ingredient = {
                     'food': o['fields']['title'],
@@ -63,6 +114,15 @@ class OpenEats(Integration):
                 }
                 recipe_dict[ingredient_group_dict[o['fields']['ingredient_group']]]['ingredients'].append(ingredient)
 
+        for k, r in recipe_dict.items():
+            if r["cuisine"] in cuisine_group_dict:
+                r["cuisine"] = cuisine_group_dict[r["cuisine"]]
+            if r["course"] in course_group_dict:
+                r["course"] = course_group_dict[r["course"]]
+            for index in range(len(r["tags"])):
+                if r["tags"][index] in tag_group_dict:
+                    r["tags"][index] = tag_group_dict[r["tags"][index]]
+
         return list(recipe_dict.values())
 
     def get_file_from_recipe(self, recipe):

cookbook/integration/paprika.py

@@ -2,12 +2,16 @@ import base64
 import gzip
 import json
 import re
+from gettext import gettext as _
 from io import BytesIO
 
+import requests
+
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient, Keyword
-from gettext import gettext as _
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class Paprika(Integration):
@@ -26,10 +30,9 @@ class Paprika(Integration):
                 recipe.description = '' if len(recipe_json['description'].strip()) > 500 else recipe_json['description'].strip()
 
             try:
-                if re.match(r'([0-9])+\s(.)*', recipe_json['servings']):
-                    s = recipe_json['servings'].split(' ')
-                    recipe.servings = s[0]
-                    recipe.servings_text = s[1]
+                if 'servings' in recipe_json:
+                    recipe.servings = parse_servings(recipe_json['servings'])
+                    recipe.servings_text = parse_servings_text(recipe_json['servings'])
 
                 if len(recipe_json['cook_time'].strip()) > 0:
                     recipe.waiting_time = re.findall(r'\d+', recipe_json['cook_time'])[0]
@@ -55,7 +58,7 @@ class Paprika(Integration):
                 pass
 
             step = Step.objects.create(
-                instruction=instructions, space=self.request.space,
+                instruction=instructions, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
             )
 
             if 'description' in recipe_json and len(recipe_json['description'].strip()) > 500:
@@ -70,18 +73,25 @@ class Paprika(Integration):
             try:
                 for ingredient in recipe_json['ingredients'].split('\n'):
                     if len(ingredient.strip()) > 0:
-                        amount, unit, ingredient, note = ingredient_parser.parse(ingredient)
-                        f = ingredient_parser.get_food(ingredient)
+                        amount, unit, food, note = ingredient_parser.parse(ingredient)
+                        f = ingredient_parser.get_food(food)
                         u = ingredient_parser.get_unit(unit)
                         step.ingredients.add(Ingredient.objects.create(
-                            food=f, unit=u, amount=amount, note=note, space=self.request.space,
+                            food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                         ))
             except AttributeError:
                 pass
 
             recipe.steps.add(step)
 
-            if recipe_json.get("photo_data", None):
-                self.import_recipe_image(recipe, BytesIO(base64.b64decode(recipe_json['photo_data'])), filetype='.jpeg')
+            try:
+                if recipe_json.get("image_url", None):
+                    url = recipe_json.get("image_url", None)
+                    if validate_import_url(url):
+                        response = requests.get(url)
+                        self.import_recipe_image(recipe, BytesIO(response.content))
+            except Exception:
+                if recipe_json.get("photo_data", None):
+                    self.import_recipe_image(recipe, BytesIO(base64.b64decode(recipe_json['photo_data'])), filetype='.jpeg')
 
             return recipe

cookbook/integration/pdfexport.py

@@ -0,0 +1,56 @@
+import asyncio
+
+import django.core.management.commands.runserver as runserver
+from asgiref.sync import sync_to_async
+from pyppeteer import launch
+
+from cookbook.integration.integration import Integration
+
+
+class PDFexport(Integration):
+
+    def get_recipe_from_file(self, file):
+        raise NotImplementedError('Method not implemented in storage integration')
+
+    async def get_files_from_recipes_async(self, recipes, el, cookie):
+        cmd = runserver.Command()
+
+        browser = await launch(
+            handleSIGINT=False,
+            handleSIGTERM=False,
+            handleSIGHUP=False,
+            ignoreHTTPSErrors=True,
+        )
+
+        cookies = {'domain': cmd.default_addr, 'name': 'sessionid', 'value': cookie['sessionid'], }
+        options = {'format': 'letter',
+                   'margin': {
+                       'top': '0.75in',
+                       'bottom': '0.75in',
+                       'left': '0.75in',
+                       'right': '0.75in',
+                   }
+                   }
+
+        files = []
+        for recipe in recipes:
+
+            page = await browser.newPage()
+            await page.emulateMedia('print')
+            await page.setCookie(cookies)
+
+            await page.goto('http://' + cmd.default_addr + ':' + cmd.default_port + '/view/recipe/' + str(recipe.id), {'waitUntil': 'domcontentloaded'})
+            await page.waitForSelector('#printReady')
+
+            files.append([recipe.name + '.pdf', await page.pdf(options)])
+            await page.close()
+
+            el.exported_recipes += 1
+            el.msg += self.get_recipe_processed_msg(recipe)
+            await sync_to_async(el.save, thread_sensitive=True)()
+
+        await browser.close()
+        return files
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        return asyncio.run(self.get_files_from_recipes_async(recipes, el, cookie))

cookbook/integration/pepperplate.py

@@ -1,6 +1,6 @@
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient
+from cookbook.models import Ingredient, Recipe, Step
 
 
 class Pepperplate(Integration):
@@ -35,17 +35,17 @@ class Pepperplate(Integration):
         recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space)
 
         step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n', space=self.request.space,
+            instruction='\n'.join(directions) + '\n\n', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
         )
 
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
             if len(ingredient.strip()) > 0:
-                amount, unit, ingredient, note = ingredient_parser.parse(ingredient)
-                f = ingredient_parser.get_food(ingredient)
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
+                f = ingredient_parser.get_food(food)
                 u = ingredient_parser.get_unit(unit)
                 step.ingredients.add(Ingredient.objects.create(
-                    food=f, unit=u, amount=amount, note=note, space=self.request.space,
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                 ))
         recipe.steps.add(step)
 

cookbook/integration/plantoeat.py

@@ -2,9 +2,11 @@ from io import BytesIO
 
 import requests
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient, Keyword
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class Plantoeat(Integration):
@@ -17,38 +19,45 @@ class Plantoeat(Integration):
         tags = None
         ingredients = []
         directions = []
-        description = ''
+        fields = {}
         for line in file.replace('\r', '').split('\n'):
             if line.strip() != '':
                 if 'Title:' in line:
-                    title = line.replace('Title:', '').replace('"', '').strip()
+                    fields['name'] = line.replace('Title:', '').replace('"', '').strip()
                 if 'Description:' in line:
-                    description = line.replace('Description:', '').strip()
-                if 'Source:' in line or 'Serves:' in line or 'Prep Time:' in line or 'Cook Time:' in line:
-                    directions.append(line.strip() + '\n')
+                    fields['description'] = line.replace('Description:', '').strip()
+                if 'Serves:' in line:
+                    fields['servings'] = parse_servings(line.replace('Serves:', '').strip())
+                if 'Source:' in line:
+                    fields['source_url'] = line.replace('Source:', '').strip()
                 if 'Photo Url:' in line:
                     image_url = line.replace('Photo Url:', '').strip()
+                if 'Prep Time:' in line:
+                    fields['working_time'] = parse_time(line.replace('Prep Time:', '').strip())
+                if 'Cook Time:' in line:
+                    fields['waiting_time'] = parse_time(line.replace('Cook Time:', '').strip())
                 if 'Tags:' in line:
                     tags = line.replace('Tags:', '').strip()
-                if ingredient_mode:
-                    if len(line) > 2 and 'Instructions:' not in line:
-                        ingredients.append(line.strip())
-                if direction_mode:
-                    if len(line) > 2:
-                        directions.append(line.strip() + '\n')
                 if 'Ingredients:' in line:
                     ingredient_mode = True
                 if 'Directions:' in line:
                     ingredient_mode = False
                     direction_mode = True
+                if ingredient_mode:
+                    if len(line) > 2 and 'Ingredients:' not in line:
+                        ingredients.append(line.strip())
+                if direction_mode:
+                    if len(line) > 2:
+                        directions.append(line.strip() + '\n')                
 
-        recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space)
+        recipe = Recipe.objects.create(**fields, created_by=self.request.user, internal=True, space=self.request.space)
 
         step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n', space=self.request.space,
+            instruction='\n'.join(directions) + '\n\n', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
         )
 
         if tags:
+            tags = tags.replace('^',',')
             for k in tags.split(','):
                 keyword, created = Keyword.objects.get_or_create(name=k.strip(), space=self.request.space)
                 recipe.keywords.add(keyword)
@@ -56,18 +65,19 @@ class Plantoeat(Integration):
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
             if len(ingredient.strip()) > 0:
-                amount, unit, ingredient, note = ingredient_parser.parse(ingredient)
-                f = ingredient_parser.get_food(ingredient)
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
+                f = ingredient_parser.get_food(food)
                 u = ingredient_parser.get_unit(unit)
                 step.ingredients.add(Ingredient.objects.create(
-                    food=f, unit=u, amount=amount, note=note, space=self.request.space,
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                 ))
         recipe.steps.add(step)
 
         if image_url:
             try:
-                response = requests.get(image_url)
-                self.import_recipe_image(recipe, BytesIO(response.content))
+                if validate_import_url(image_url):
+                    response = requests.get(image_url)
+                    self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e:
                 print('failed to import image ', str(e))
 
@@ -78,7 +88,11 @@ class Plantoeat(Integration):
         current_recipe = ''
 
         for fl in file.readlines():
-            line = fl.decode("ANSI")
+            try:
+                line = fl.decode("utf-8")
+            except UnicodeDecodeError:
+                line = fl.decode("windows-1250")
+
             if line.startswith('--------------'):
                 if current_recipe != '':
                     recipe_list.append(current_recipe)

cookbook/integration/recettetek.py

@@ -1,14 +1,18 @@
-import re
+import imghdr
 import json
-import requests
+import re
 from io import BytesIO
 from zipfile import ZipFile
-import imghdr
 
+import requests
+
+from django.utils.translation import gettext as _
+
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.image_processing import get_filetype
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient, Keyword
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class RecetteTek(Integration):
@@ -27,10 +31,10 @@ class RecetteTek(Integration):
 
     def get_recipe_from_file(self, file):
 
-        # Create initial recipe with just a title and a decription
+        # Create initial recipe with just a title and a description
         recipe = Recipe.objects.create(name=file['title'], created_by=self.request.user, internal=True, space=self.request.space, )
 
-        # set the description as an empty string for later use for the source URL, incase there is no description text.
+        # set the description as an empty string for later use for the source URL, in case there is no description text.
         recipe.description = ''
 
         try:
@@ -43,12 +47,12 @@ class RecetteTek(Integration):
         if not instructions:
             instructions = ''
 
-        step = Step.objects.create(instruction=instructions, space=self.request.space,)
+        step = Step.objects.create(instruction=instructions, space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,)
 
         # Append the original import url to the step (if it exists)
         try:
             if file['url'] != '':
-                step.instruction += '\n\nImported from: ' + file['url']
+                step.instruction += '\n\n' + _('Imported from') + ': ' + file['url']
                 step.save()
         except Exception as e:
             print(recipe.name, ': failed to import source url ', str(e))
@@ -58,11 +62,11 @@ class RecetteTek(Integration):
             ingredient_parser = IngredientParser(self.request, True)
             for ingredient in file['ingredients'].split('\n'):
                 if len(ingredient.strip()) > 0:
-                    amount, unit, ingredient, note = ingredient_parser.parse(ingredient)
+                    amount, unit, food, note = ingredient_parser.parse(ingredient.strip())
                     f = ingredient_parser.get_food(ingredient)
                     u = ingredient_parser.get_unit(unit)
                     step.ingredients.add(Ingredient.objects.create(
-                        food=f, unit=u, amount=amount, note=note, space=self.request.space,
+                        food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                     ))
         except Exception as e:
             print(recipe.name, ': failed to parse recipe ingredients ', str(e))
@@ -121,11 +125,13 @@ class RecetteTek(Integration):
                         self.import_recipe_image(recipe, BytesIO(import_zip.read(image_file_name)), filetype=get_filetype(image_file_name))
             else:
                 if file['originalPicture'] != '':
-                    response = requests.get(file['originalPicture'])
-                    if imghdr.what(BytesIO(response.content)) is not None:
-                        self.import_recipe_image(recipe, BytesIO(response.content), filetype=get_filetype(file['originalPicture']))
-                    else:
-                        raise Exception("Original image failed to download.")
+                    url = file['originalPicture']
+                    if validate_import_url(url):
+                        response = requests.get(url)
+                        if imghdr.what(BytesIO(response.content)) is not None:
+                            self.import_recipe_image(recipe, BytesIO(response.content), filetype=get_filetype(file['originalPicture']))
+                        else:
+                            raise Exception("Original image failed to download.")
         except Exception as e:
             print(recipe.name, ': failed to import image ', str(e))
 

cookbook/integration/recipekeeper.py

@@ -1,12 +1,14 @@
 import re
-from bs4 import BeautifulSoup
 from io import BytesIO
 from zipfile import ZipFile
 
+from bs4 import BeautifulSoup
+
+from django.utils.translation import gettext as _
 from cookbook.helper.ingredient_parser import IngredientParser
-from cookbook.helper.recipe_url_import import parse_servings, iso_duration_to_minutes
+from cookbook.helper.recipe_url_import import iso_duration_to_minutes, parse_servings
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient, Keyword
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class RecipeKeeper(Integration):
@@ -39,26 +41,33 @@ class RecipeKeeper(Integration):
         except AttributeError:
             pass
 
-        step = Step.objects.create(instruction='', space=self.request.space,)
+        step = Step.objects.create(instruction='', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients, )
 
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in file.find("div", {"itemprop": "recipeIngredients"}).findChildren("p"):
             if ingredient.text == "":
                 continue
-            amount, unit, ingredient, note = ingredient_parser.parse(ingredient.text.strip())
-            f = ingredient_parser.get_food(ingredient)
+            amount, unit, food, note = ingredient_parser.parse(ingredient.text.strip())
+            f = ingredient_parser.get_food(food)
             u = ingredient_parser.get_unit(unit)
             step.ingredients.add(Ingredient.objects.create(
-                food=f, unit=u, amount=amount, note=note, space=self.request.space,
+                food=f, unit=u, amount=amount, note=note, original_text=str(ingredient).replace('<p>', '').replace('</p>', ''), space=self.request.space,
             ))
 
         for s in file.find("div", {"itemprop": "recipeDirections"}).find_all("p"):
             if s.text == "":
                 continue
             step.instruction += s.text + ' \n'
+            step.save()
+
+        for s in file.find("div", {"itemprop": "recipeNotes"}).find_all("p"):
+            if s.text == "":
+                continue
+            step.instruction += s.text + ' \n'
+            step.save()
 
         if file.find("span", {"itemprop": "recipeSource"}).text != '':
-            step.instruction += "\n\nImported from: " + file.find("span", {"itemprop": "recipeSource"}).text
+            step.instruction += "\n\n" + _("Imported from") + ": " + file.find("span", {"itemprop": "recipeSource"}).text
             step.save()
 
         recipe.steps.add(step)

cookbook/integration/recipesage.py

@@ -3,9 +3,11 @@ from io import BytesIO
 
 import requests
 
+from cookbook.helper.HelperFunctions import validate_import_url
 from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient
+from cookbook.models import Ingredient, Recipe, Step
 
 
 class RecipeSage(Integration):
@@ -17,42 +19,46 @@ class RecipeSage(Integration):
             created_by=self.request.user, internal=True,
             space=self.request.space)
 
+        if file['recipeYield'] != '':
+            recipe.servings = parse_servings(file['recipeYield'])
+            recipe.servings_text = parse_servings_text(file['recipeYield'])
+
         try:
-            if file['recipeYield'] != '':
-                recipe.servings = int(file['recipeYield'])
+            if 'totalTime' in file and file['totalTime'] != '':
+                recipe.working_time = parse_time(file['totalTime'])
 
-            if file['totalTime'] != '':
-                recipe.waiting_time = int(file['totalTime']) - int(file['timePrep'])
-
-            if file['prepTime'] != '':
-                recipe.working_time = int(file['timePrep'])
-
-            recipe.save()
+            if 'timePrep' in file and file['prepTime'] != '':
+                recipe.working_time = parse_time(file['timePrep'])
+                recipe.waiting_time = parse_time(file['totalTime']) - parse_time(file['timePrep'])
         except Exception as e:
-            print('failed to parse yield or time ', str(e))
+            print('failed to parse time ', str(e))
 
-        ingredient_parser = IngredientParser(self.request,True)
+        recipe.save()
+
+        ingredient_parser = IngredientParser(self.request, True)
         ingredients_added = False
         for s in file['recipeInstructions']:
             step = Step.objects.create(
-                instruction=s['text'], space=self.request.space,
+                instruction=s['text'], space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
             )
             if not ingredients_added:
                 ingredients_added = True
 
                 for ingredient in file['recipeIngredient']:
-                    amount, unit, ingredient, note = ingredient_parser.parse(ingredient)
-                    f = ingredient_parser.get_food(ingredient)
+                    amount, unit, food, note = ingredient_parser.parse(ingredient)
+                    f = ingredient_parser.get_food(food)
                     u = ingredient_parser.get_unit(unit)
                     step.ingredients.add(Ingredient.objects.create(
-                        food=f, unit=u, amount=amount, note=note, space=self.request.space,
+                        food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                     ))
             recipe.steps.add(step)
 
         if len(file['image']) > 0:
             try:
-                response = requests.get(file['image'][0])
-                self.import_recipe_image(recipe, BytesIO(response.content))
+                url = file['image'][0]
+                if validate_import_url(url):
+                    response = requests.get(url)
+                    self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e:
                 print('failed to import image ', str(e))
 
@@ -77,16 +83,26 @@ class RecipeSage(Integration):
         }
 
         for s in recipe.steps.all():
-            if s.type != Step.TIME:
-                data['recipeInstructions'].append({
-                    '@type': 'HowToStep',
-                    'text': s.instruction
-                })
+            data['recipeInstructions'].append({
+                '@type': 'HowToStep',
+                'text': s.instruction
+            })
 
-                for i in s.ingredients.all():
-                    data['recipeIngredient'].append(f'{float(i.amount)} {i.unit} {i.food}')
+            for i in s.ingredients.all():
+                data['recipeIngredient'].append(f'{float(i.amount)} {i.unit} {i.food}')
 
         return data
 
+    def get_files_from_recipes(self, recipes, el, cookie):
+        json_list = []
+        for r in recipes:
+            json_list.append(self.get_file_from_recipe(r))
+
+            el.exported_recipes += 1
+            el.msg += self.get_recipe_processed_msg(r)
+            el.save()
+
+        return [[self.get_export_file_name('json'), json.dumps(json_list)]]
+
     def split_recipe_file(self, file):
         return json.loads(file.read().decode("utf-8"))

cookbook/integration/rezeptsuitede.py

@@ -0,0 +1,78 @@
+import base64
+from io import BytesIO
+from lxml import etree
+
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text
+from cookbook.integration.integration import Integration
+from cookbook.models import Ingredient, Keyword, Recipe, Step
+
+
+class Rezeptsuitede(Integration):
+
+    def split_recipe_file(self, file):
+        return etree.parse(file).getroot().getchildren()
+
+    def get_recipe_from_file(self, file):
+        recipe_xml = file
+
+        recipe = Recipe.objects.create(
+            name=recipe_xml.find('head').attrib['title'].strip(),
+            created_by=self.request.user, internal=True, space=self.request.space)
+
+        try:
+            if recipe_xml.find('head').attrib['servingtype']:
+                recipe.servings = parse_servings(recipe_xml.find('head').attrib['servingtype'].strip())
+                recipe.servings_text = parse_servings_text(recipe_xml.find('head').attrib['servingtype'].strip())
+        except KeyError:
+            pass
+
+        if recipe_xml.find('remark') is not None:  # description is a list of <li>'s with text
+            if recipe_xml.find('remark').find('line') is not None:
+                recipe.description = recipe_xml.find('remark').find('line').text[:512]
+
+        for prep in recipe_xml.findall('preparation'):
+            try:
+                if prep.find('step').text:
+                    step = Step.objects.create(
+                        instruction=prep.find('step').text.strip(), space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
+                    )
+                    recipe.steps.add(step)
+            except Exception:
+                pass
+
+        ingredient_parser = IngredientParser(self.request, True)
+
+        if recipe_xml.find('part').find('ingredient') is not None:
+            ingredient_step = recipe.steps.first()
+            if ingredient_step is None:
+                ingredient_step = Step.objects.create(space=self.request.space, instruction='')
+
+            for ingredient in recipe_xml.find('part').findall('ingredient'):
+                f = ingredient_parser.get_food(ingredient.attrib['item'])
+                u = ingredient_parser.get_unit(ingredient.attrib['unit'])
+                amount = 0
+                if ingredient.attrib['qty'].strip() != '':
+                    try:
+                        amount, unit, note = ingredient_parser.parse_amount(ingredient.attrib['qty'])
+                    except ValueError:  # sometimes quantities contain words which cant be parsed
+                        pass
+                ingredient_step.ingredients.add(Ingredient.objects.create(food=f, unit=u, amount=amount, space=self.request.space, ))
+
+        try:
+            k, created = Keyword.objects.get_or_create(name=recipe_xml.find('head').find('cat').text.strip(), space=self.request.space)
+            recipe.keywords.add(k)
+        except Exception:
+            pass
+
+        recipe.save()
+
+        try:
+            self.import_recipe_image(recipe, BytesIO(base64.b64decode(recipe_xml.find('head').find('picbin').text)), filetype='.jpeg')
+        except BaseException:
+            pass
+
+        return recipe
+
+    def get_file_from_recipe(self, recipe):
+        raise NotImplementedError('Method not implemented in storage integration')

cookbook/integration/rezkonv.py

@@ -1,6 +1,6 @@
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient, Keyword
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class RezKonv(Integration):
@@ -12,43 +12,43 @@ class RezKonv(Integration):
 
         ingredients = []
         directions = []
-        for line in file.replace('\r', '').split('\n'):
+        for line in file.replace('\r', '').replace('\n\n', '\n').split('\n'):
             if 'Titel:' in line:
                 title = line.replace('Titel:', '').strip()
             if 'Kategorien:' in line:
                 tags = line.replace('Kategorien:', '').strip()
-            if ingredient_mode and ('quelle' in line.lower() or 'source' in line.lower()):
+            if ingredient_mode and (
+                    'quelle' in line.lower() or 'source' in line.lower() or (line == '' and len(ingredients) > 0)):
                 ingredient_mode = False
+                direction_mode = True
             if ingredient_mode:
                 if line != '' and '===' not in line and 'Zubereitung' not in line:
                     ingredients.append(line.strip())
             if direction_mode:
                 if line.strip() != '' and line.strip() != '=====':
                     directions.append(line.strip())
-            if 'Zutaten:' in line:
+            if 'Zutaten:' in line or 'Ingredients' in line or 'Menge:' in line:
                 ingredient_mode = True
-            if 'Zubereitung:' in line:
-                ingredient_mode = False
-                direction_mode = True
 
-        recipe = Recipe.objects.create(name=title, created_by=self.request.user, internal=True, space=self.request.space)
+        recipe = Recipe.objects.create(name=title, created_by=self.request.user, internal=True,
+                                       space=self.request.space)
 
         for k in tags.split(','):
             keyword, created = Keyword.objects.get_or_create(name=k.strip(), space=self.request.space)
             recipe.keywords.add(keyword)
 
         step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n', space=self.request.space,
+            instruction='  \n'.join(directions) + '\n\n', space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients,
         )
 
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
             if len(ingredient.strip()) > 0:
-                amount, unit, ingredient, note = ingredient_parser.parse(ingredient)
-                f = ingredient_parser.get_food(ingredient)
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
+                f = ingredient_parser.get_food(food)
                 u = ingredient_parser.get_unit(unit)
                 step.ingredients.add(Ingredient.objects.create(
-                    food=f, unit=u, amount=amount, note=note, space=self.request.space,
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                 ))
         recipe.steps.add(step)
 
@@ -60,9 +60,15 @@ class RezKonv(Integration):
     def split_recipe_file(self, file):
         recipe_list = []
         current_recipe = ''
-
+        # TODO build algorithm to try trough encodings and fail if none work, use for all importers
+        # encoding_list = ['windows-1250', 'latin-1']
+        encoding = 'windows-1250'
         for fl in file.readlines():
-            line = fl.decode("ANSI")
+            try:
+                line = fl.decode(encoding)
+            except UnicodeDecodeError:
+                encoding = 'latin-1'
+                line = fl.decode(encoding)
             if line.startswith('=====') and 'rezkonv' in line.lower():
                 if current_recipe != '':
                     recipe_list.append(current_recipe)

cookbook/integration/saffron.py

@@ -2,10 +2,10 @@ from django.utils.translation import gettext as _
 
 from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient
+from cookbook.models import Ingredient, Recipe, Step
 
 
-class Safron(Integration):
+class Saffron(Integration):
 
     def get_recipe_from_file(self, file):
         ingredient_mode = False
@@ -43,19 +43,57 @@ class Safron(Integration):
 
         recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space, )
 
-        step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space, )
+        step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space, show_ingredients_table=self.request.user.userpreference.show_step_ingredients, )
 
         ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
-            amount, unit, ingredient, note = ingredient_parser.parse(ingredient)
-            f = ingredient_parser.get_food(ingredient)
+            amount, unit, food, note = ingredient_parser.parse(ingredient)
+            f = ingredient_parser.get_food(food)
             u = ingredient_parser.get_unit(unit)
             step.ingredients.add(Ingredient.objects.create(
-                food=f, unit=u, amount=amount, note=note, space=self.request.space,
+                food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
             ))
         recipe.steps.add(step)
 
         return recipe
 
     def get_file_from_recipe(self, recipe):
-        raise NotImplementedError('Method not implemented in storage integration')
+
+        data = "Title: " + recipe.name if recipe.name else "" + "\n"
+        data += "Description: " + recipe.description if recipe.description else "" + "\n"
+        data += "Source: \n"
+        data += "Original URL: \n"
+        data += "Yield: " + str(recipe.servings) + "\n"
+        data += "Cookbook: \n"
+        data += "Section: \n"
+        data += "Image: \n"
+
+        recipeInstructions = []
+        recipeIngredient = []
+        for s in recipe.steps.all():
+            recipeInstructions.append(s.instruction)
+
+            for i in s.ingredients.all():
+                recipeIngredient.append(f'{float(i.amount)} {i.unit} {i.food}')
+
+        data += "Ingredients: \n"
+        for ingredient in recipeIngredient:
+            data += ingredient + "\n"
+
+        data += "Instructions: \n"
+        for instruction in recipeInstructions:
+            data += instruction + "\n"
+
+        return recipe.name + '.txt', data
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        files = []
+        for r in recipes:
+            filename, data = self.get_file_from_recipe(r)
+            files.append([filename, data])
+
+            el.exported_recipes += 1
+            el.msg += self.get_recipe_processed_msg(r)
+            el.save()
+
+        return files