Merge branch 'develop'

Currently translated at 10.4% (48 of 460 strings)

Translation: Tandoor/Recipes Frontend
Translate-URL: http://translate.tandoor.dev/projects/tandoor/recipes-frontend/pt/

.dockerignore

@@ -1,4 +1,4 @@
-node_modules
+**/node_modules
 npm-debug.log
 Dockerfile*
 docker-compose*
@@ -12,6 +12,21 @@ LICENSE
 .env.template
 .github
 .idea
+.prettierignore
 LICENSE.md
 docs
-update.sh
+update.sh
+.pytest_cache
+cookbook/tests
+mediafiles
+staticfiles
+db.sqlite3
+pytest.ini
+vue/**/*.vue
+vue/**/*.ts
+**/.openapi-generator
+mkdocs.yml
+vue/babel.config*
+vue/package.json
+vue/tsconfig.json
+vue/src/utils/openapi

.env.template

@@ -1,12 +1,18 @@
 # only set this to true when testing/debugging
 # when unset: 1 (true) - dont unset this, just for development
 DEBUG=0
+SQL_DEBUG=0
+
+# HTTP port to bind to
+# TANDOOR_PORT=8080
 
 # hosts the application can run under e.g. recipes.mydomain.com,cooking.mydomain.com,...
 ALLOWED_HOSTS=*
 
 # random secret key, use for example `base64 /dev/urandom | head -c50` to generate one
+# ---------------------------- REQUIRED -------------------------
 SECRET_KEY=
+# ---------------------------------------------------------------
 
 # your default timezone See https://timezonedb.com/time-zones for a list of timezones
 TIMEZONE=Europe/Berlin
@@ -17,7 +23,9 @@ DB_ENGINE=django.db.backends.postgresql
 POSTGRES_HOST=db_recipes
 POSTGRES_PORT=5432
 POSTGRES_USER=djangouser
+# ---------------------------- REQUIRED -------------------------
 POSTGRES_PASSWORD=
+# ---------------------------------------------------------------
 POSTGRES_DB=djangodb
 
 # database connection string, when used overrides other database settings.
@@ -40,10 +48,18 @@ SHOPPING_MIN_AUTOSYNC_INTERVAL=5
 # Default for user setting sticky navbar
 # STICKY_NAV_PREF_DEFAULT=1
 
-# If staticfiles are stored at a different location uncomment and change accordingly
+# If base URL is something other than just / (you are serving a subfolder in your proxy for instance http://recipe_app/recipes/)
+# Be sure to not have a trailing slash: e.g. '/recipes' instead of '/recipes/'
+# SCRIPT_NAME=/recipes
+
+# If staticfiles are stored at a different location uncomment and change accordingly, MUST END IN /
+# this is not required if you are just using a subfolder
+# This can either be a relative path from the applications base path or the url of an external host
 # STATIC_URL=/static/
 
-# If mediafiles are stored at a different location uncomment and change accordingly
+# If mediafiles are stored at a different location uncomment and change accordingly, MUST END IN /
+# this is not required if you are just using a subfolder
+# This can either be a relative path from the applications base path or the url of an external host
 # MEDIA_URL=/media/
 
 # Serve mediafiles directly using gunicorn. Basically everyone recommends not doing this. Please use any of the examples
@@ -52,6 +68,10 @@ SHOPPING_MIN_AUTOSYNC_INTERVAL=5
 # when unset: 1 (true) - this is temporary until an appropriate amount of time has passed for everyone to migrate
 GUNICORN_MEDIA=0
 
+# GUNICORN SERVER RELATED SETTINGS (see https://docs.gunicorn.org/en/stable/design.html#how-many-workers for recommended settings)
+# GUNICORN_WORKERS=1
+# GUNICORN_THREADS=1
+
 # S3 Media settings: store mediafiles in s3 or any compatible storage backend (e.g. minio)
 # as long as S3_ACCESS_KEY is not set S3 features are disabled
 # S3_ACCESS_KEY=
@@ -61,6 +81,7 @@ GUNICORN_MEDIA=0
 # S3_QUERYSTRING_AUTH=1 # default true, set to 0 to serve media from a public bucket without signed urls
 # S3_QUERYSTRING_EXPIRE=3600 # number of seconds querystring are valid for
 # S3_ENDPOINT_URL= # when using a custom endpoint like minio
+# S3_CUSTOM_DOMAIN= # when using a CDN/proxy to S3 (see https://github.com/TandoorRecipes/recipes/issues/1943)
 
 # Email Settings, see https://docs.djangoproject.com/en/3.2/ref/settings/#email-host
 # Required for email confirmation and password reset (automatically activates if host is set)
@@ -70,8 +91,10 @@ GUNICORN_MEDIA=0
 # EMAIL_HOST_PASSWORD=
 # EMAIL_USE_TLS=0
 # EMAIL_USE_SSL=0
-# DEFAULT_FROM_EMAIL= # email sender address (default 'webmaster@localhost')
-# ACCOUNT_EMAIL_SUBJECT_PREFIX= # prefix used for account related emails (default "[Tandoor Recipes] ")
+# email sender address (default 'webmaster@localhost')
+# DEFAULT_FROM_EMAIL=
+# prefix used for account related emails (default "[Tandoor Recipes] ")
+# ACCOUNT_EMAIL_SUBJECT_PREFIX=
 
 # allow authentication via reverse proxy (e.g. authelia), leave off if you dont know what you are doing
 # see docs for more information https://vabene1111.github.io/recipes/features/authentication/
@@ -81,7 +104,8 @@ REVERSE_PROXY_AUTH=0
 # Default settings for spaces, apply per space and can be changed in the admin view
 # SPACE_DEFAULT_MAX_RECIPES=0 # 0=unlimited recipes
 # SPACE_DEFAULT_MAX_USERS=0 # 0=unlimited users per space
-# SPACE_DEFAULT_FILES=1 # 1=can upload files (images, etc.) NOT IMPLEMENTED YET
+# SPACE_DEFAULT_MAX_FILES=0 # Maximum file storage for space in MB. 0 for unlimited, -1 to disable file upload.
+# SPACE_DEFAULT_ALLOW_SHARING=1 # Allow users to share recipes with public links
 
 # allow people to create accounts on your application instance (without an invite link)
 # when unset: 0 (false)
@@ -111,4 +135,33 @@ REVERSE_PROXY_AUTH=0
 # SOCIAL_DEFAULT_ACCESS = 1
 
 # if SOCIAL_DEFAULT_ACCESS is used, which group should be added
-# SOCIAL_DEFAULT_GROUP=guest
+# SOCIAL_DEFAULT_GROUP=guest
+
+# Django session cookie settings. Can be changed to allow a single django application to authenticate several applications
+# when running under the same database
+# SESSION_COOKIE_DOMAIN=.example.com
+# SESSION_COOKIE_NAME=sessionid # use this only to not interfere with non unified django applications under the same top level domain
+
+# by default SORT_TREE_BY_NAME is disabled this will store all Keywords and Food in the order they are created
+# enabling this setting makes saving new keywords and foods very slow, which doesn't matter in most usecases.
+# however, when doing large imports of recipes that will create new objects, can increase total run time by 10-15x
+# Keywords and Food can be manually sorted by name in Admin
+# This value can also be temporarily changed in Admin, it will revert the next time the application is started
+# This will be fixed/changed in the future by changing the implementation or finding a better workaround for sorting
+# SORT_TREE_BY_NAME=0
+# LDAP authentication
+# default 0 (false), when 1 (true) list of allowed users will be fetched from LDAP server
+#LDAP_AUTH=
+#AUTH_LDAP_SERVER_URI=
+#AUTH_LDAP_BIND_DN=
+#AUTH_LDAP_BIND_PASSWORD=
+#AUTH_LDAP_USER_SEARCH_BASE_DN=
+#AUTH_LDAP_TLS_CACERTFILE=
+
+# Enables exporting PDF (see export docs)
+# Disabled by default, uncomment to enable
+# ENABLE_PDF_EXPORT=1
+
+# Recipe exports are cached for a certain time by default, adjust time if needed
+# EXPORT_FILE_CACHE_DURATION=600
+

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: [vabene1111]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,15 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
----
-
-### Version
-Please provide your current version (can be found on the system page since v0.8.4)
-Version: 
-
-### Bug description
-A clear and concise description of what the bug is.

.github/ISSUE_TEMPLATE/bug_report.md.bak

@@ -0,0 +1,81 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+## Version
+<!-- Please provide your current version (can be found on the system page since v0.8.4). -->
+**Tandoor-Version:** 
+
+## Setup configuration
+<!--Please tick all boxes which apply to your configuration. Feel free to provide additional information below.  
+To tick boxes here, simply put an X inside the brackets below -->
+
+### Setup
+- [ ] Docker / Docker-Compose
+- [ ] Unraid
+- [ ] Synology
+- [ ] Kubernetes
+- [ ] Manual setup
+- [ ] Others (please state below)
+
+### Reverse Proxy
+- [ ] No reverse proxy
+- [ ] jwilder's nginx proxy
+- [ ] Nginx proxy manager (NPM)
+- [ ] SWAG
+- [ ] Caddy
+- [ ] Traefik
+- [ ] Others (please state below)
+
+<!-- Please provide additional information if possible -->
+**Additional information:** 
+
+## Bug description
+A clear and concise description of what the bug is.
+
+
+
+## Logs
+<!-- *(Remove this section entirely if no logs are available or necessary for your issue)*  
+To get the most information about your issue, set DEBUG=1 (e.g. in your `.env` file if using docker-compose) and try to reproduce the issue afterwards.
+
+Please put your logs into the expandable section below and use code quotation for all logs! Usage: Put three backticks in front and after the log, like this:  
+` ``` <Many lines of log messages ``` ` 
+
+Feel free to remove parts if you don't fill them out.
+-->
+
+<details>
+  <summary>Web-Container-Logs</summary>
+    
+  <!-- *Put your logs inside here (leave the code quotations in takt):* -->
+  
+  ```
+  Replace me with logs
+  ```
+</details>
+
+<details>
+  <summary>DB-Container-Logs</summary>
+    
+  <!-- *Put your logs inside here (leave the code quotations in takt):* -->
+  
+  ```
+  Replace me with logs
+  ```
+</details>
+
+<details>
+  <summary>Nginx-Container-Logs <!-- if you use one --></summary>
+    
+  <!-- *Put your logs inside here (leave the code quotations in takt):* -->
+  
+  ```
+  Replace me with logs
+  ```
+</details>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,64 @@
+name: Bug Report
+description: "Create a report to help us improve"
+#title: ""
+#labels: ["Bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: version
+    attributes:
+      label: Tandoor Version
+      description: "What version of Tandoor are you using? (can be found on the system page since v0.8.4)"
+    validations:
+      required: true
+  - type: dropdown
+    id: setup
+    attributes:
+      label: Setup
+      description: "How is your Tandoor instance set up?"
+      options:
+        - Docker / Docker-Compose
+        - Unraid
+        - Synology
+        - Kubernetes
+        - Manual Setup
+        - Others (please state below)
+    validations:
+      required: true
+  - type: dropdown
+    id: reverse-proxy
+    attributes:
+      label: "Reverse Proxy"
+      description: "What reverse proxy do you use with Tandoor?"
+      options:
+        - No reverse proxy
+        - jwilder's nginx proxy
+        - Nginx Proxy Manager (NPM)
+        - SWAG
+        - Caddy
+        - Traefik
+        - Apache2
+        - Others (please state below)
+    validations:
+      required: true
+  - type: input
+    id: other
+    attributes:
+      label: Other
+      description: "In case you chose 'Others' above, please provide more info here."
+  - type: textarea
+    id: bug-descr
+    attributes:
+      label: Bug description
+      description: "Please accurately describe the bug you encountered."
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant logs
+      description: Please copy and paste any relevant logs. This will be automatically formatted into code, so no need for backticks.
+      render: shell

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: FAQs
+    url: https://docs.tandoor.dev/faq/
+    about: Please take a look at the FAQs before creating a bug ticket.

.github/ISSUE_TEMPLATE/doc_issue.yml

@@ -0,0 +1,40 @@
+name: Documentation Issue
+description: "Create a report to help us improve"
+#title: ""
+labels: ["documentation"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this documentation issue report!
+  - type: input
+    id: docs-link
+    attributes:
+      label: Documentation link
+      description: "Please provide a link to the corresponding documentation site on docs.tandoor.dev"
+  - type: dropdown
+    id: section
+    attributes:
+      label: Affected section
+      description: "What part of the documentation is the issue about?"
+      options:
+        - Installation
+        - Features
+        - System
+        - FAQ
+        - Does not exist yet
+        - Other (please state below)
+    validations:
+      required: true
+  - type: input
+    id: other
+    attributes:
+      label: Other
+      description: "In case you chose 'Other' above, please provide more info here."
+  - type: textarea
+    id: descr
+    attributes:
+      label: Issue description
+      description: "Please accurately describe the documentation issue you are seeing."
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,31 @@
+name: Feature Request
+description: "Suggest an idea for this project"
+#title: ""
+labels: ["enhancement"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this feature request!
+  - type: textarea
+    id: problem
+    attributes:
+      label: "Is your feature request related to a problem? Please describe."
+      description: "A clear and concise description of what the problem is. Ex. I'm always frustrated when..."
+  - type: textarea
+    id: solution
+    attributes:
+      label: "Describe the solution you'd like"
+      description: "A clear and concise description of what you want to happen."
+    validations:
+      required: true
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: "Describe alternatives you've considered"
+      description: "A clear and concise description of any alternative solutions or features you've considered."
+  - type: textarea
+    id: additional
+    attributes:
+      label: "Additional context"
+      description: "Add any other context or screenshots about the feature request here."

.github/ISSUE_TEMPLATE/help_request.yml

@@ -0,0 +1,82 @@
+name: Help request
+description: "If there is anything wrong with your setup"
+#title: ""
+labels: ["setup issue"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this help request!
+  - type: textarea
+    id: issue
+    attributes:
+      label: Issue
+      description: "Please describe your problem here."
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: Tandoor Version
+      description: "What version of Tandoor are you using? (can be found on the system page since v0.8.4)"
+    validations:
+      required: true
+  - type: input
+    id: os
+    attributes:
+      label: OS Version
+      description: "E.g. Ubuntu 20.02"
+    validations:
+      required: true
+  - type: dropdown
+    id: setup
+    attributes:
+      label: Setup
+      description: "How is your Tandoor instance set up?"
+      options:
+        - Docker / Docker-Compose
+        - Unraid
+        - Synology
+        - Kubernetes
+        - Manual Setup
+        - Others (please state below)
+    validations:
+      required: true
+  - type: dropdown
+    id: reverse-proxy
+    attributes:
+      label: "Reverse Proxy"
+      description: "What reverse proxy do you use with Tandoor?"
+      options:
+        - No reverse proxy
+        - jwilder's nginx proxy
+        - Nginx Proxy Manager (NPM)
+        - SWAG
+        - Caddy
+        - Traefik
+        - Others (please state below)
+    validations:
+      required: true
+  - type: input
+    id: other
+    attributes:
+      label: Other
+      description: "In case you chose 'Others' above or have more info, please provide additional details here."
+  - type: textarea
+    id: env
+    attributes:
+      label: Environment file
+      description: "Please include your `.env` config file (**make sure to remove/replace all secrets**)"
+      render: shell
+  - type: textarea
+    id: docker-compose
+    attributes:
+      label: Docker-Compose file
+      description: "When running with docker compose please provide your `docker-compose.yml`"
+      render: shell
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant logs
+      description: "If you feel like there is anything interesting please post the output of `docker-compose logs` at container startup and when the issue happens."
+      render: shell

.github/ISSUE_TEMPLATE/website_import.yml

@@ -0,0 +1,36 @@
+name: Website Import
+description: "Anything related to website imports"
+#title: ""
+#labels: ["enhancement"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this website import form!
+  - type: input
+    id: version
+    attributes:
+      label: Tandoor Version
+      description: "What version of Tandoor are you using? (can be found on the system page since v0.8.4)"
+    validations:
+      required: true
+  - type: input
+    id: url
+    attributes:
+      label: Import URL
+      description: "Exact URL you are trying to import from."
+    validations:
+      required: true
+  - type: textarea
+    id: bug-descr
+    attributes:
+      label: "When did the issue happen?"
+      description: "When pressing the search button with the url / when importing after the page has loaded / ..."
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Response / message shown
+      description: Please copy and paste any relevant logs or responses / messages which are shown in Tandoor. This will be automatically formatted into code, so no need for backticks.
+      render: shell

.github/dependabot.yml

@@ -5,7 +5,12 @@
 
 version: 2
 updates:
-  - package-ecosystem: "pip" # See documentation for possible values
-    directory: "/" # Location of package manifests
+  - package-ecosystem: "pip"
+    directory: "/"
     schedule:
-      interval: "daily"
+      interval: "monthly"
+
+  - package-ecosystem: "npm"
+    directory: "/vue/"
+    schedule:
+      interval: "monthly"

.github/workflows/ci.yml

@@ -1,24 +1,36 @@
-name: Continous Integration
+name: Continuous Integration
 
 on: [push]
 
 jobs:
   build:
-
+    if: github.repository_owner == 'TandoorRecipes'
     runs-on: ubuntu-latest
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.9]
+        python-version: ['3.10']
 
     steps:
     - uses: actions/checkout@v1
-    - name: Set up Python 3.9
+    - name: Set up Python 3.10
       uses: actions/setup-python@v1
       with:
-        python-version: 3.9
-    - name: Install dependencies
+        python-version: '3.10'
+    # Build Vue frontend
+    - uses: actions/setup-node@v2
+      with:
+        node-version: '14'
+    - name: Install Vue dependencies
+      working-directory: ./vue
+      run: yarn install
+    - name: Build Vue dependencies
+      working-directory: ./vue
+      run: yarn build
+    - name: Install Django dependencies
       run: |
+        sudo apt-get -y update
+        sudo apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev
         python -m pip install --upgrade pip
         pip install -r requirements.txt
         python3 manage.py collectstatic --noinput

.github/workflows/codeql-analysis.yml

@@ -8,9 +8,8 @@ on:
 
 jobs:
   CodeQL-Build:
-
+    if: github.repository_owner == 'TandoorRecipes'
     runs-on: ubuntu-latest
-
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2

.github/workflows/docker-publish-beta-raspi.yml

@@ -0,0 +1,48 @@
+name: publish beta raspi image docker
+on:
+  push:
+    branches:
+      - 'beta'
+jobs:
+  build:
+    if: github.repository_owner == 'TandoorRecipes'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      # Update Version number
+      - name: Update version file
+        uses: DamianReeves/write-file-action@v1.0
+        with:
+          path: recipes/version.py
+          contents: |
+            VERSION_NUMBER = 'beta'
+            BUILD_REF = '${{ github.sha }}'
+          write-mode: overwrite
+      # Build Vue frontend
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - name: Install dependencies
+        working-directory: ./vue
+        run: yarn install
+      - name: Build dependencies
+        working-directory: ./vue
+        run: yarn build
+      # Build container
+      - name: Build and publish image
+        uses: ilteoood/docker_buildx@master
+        with:
+          publish: true
+          imageName: vabene1111/recipes
+          tag: beta-raspi
+          dockerFile: Dockerfile-raspi
+          platform: linux/arm/v7
+          dockerUser: ${{ secrets.DOCKER_USERNAME }}
+          dockerPassword: ${{ secrets.DOCKER_PASSWORD }}
+      # Send discord notification
+      - name: Discord notification
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.DISCORD_BETA_WEBHOOK }}
+        uses: Ilshidur/action-discord@0.3.2
+        with:
+          args: '🚀 The BETA Image has been updated! 🥳'

.github/workflows/docker-publish-beta.yml

@@ -5,9 +5,11 @@ on:
       - 'beta'
 jobs:
   build:
+    if: github.repository_owner == 'TandoorRecipes'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@master
+      # Update Version number
       - name: Update version file
         uses: DamianReeves/write-file-action@v1.0
         with:
@@ -16,11 +18,30 @@ jobs:
             VERSION_NUMBER = 'beta'
             BUILD_REF = '${{ github.sha }}'
           write-mode: overwrite
+      # Build Vue frontend
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - name: Install dependencies
+        working-directory: ./vue
+        run: yarn install
+      - name: Build dependencies
+        working-directory: ./vue
+        run: yarn build
+      # Build container
       - name: Build and publish image
         uses: ilteoood/docker_buildx@master
         with:
           publish: true
           imageName: vabene1111/recipes
           tag: beta
-          dockerHubUser: ${{ secrets.DOCKER_USERNAME }}
-          dockerHubPassword: ${{ secrets.DOCKER_PASSWORD }}
+          platform: linux/amd64,linux/arm64
+          dockerUser: ${{ secrets.DOCKER_USERNAME }}
+          dockerPassword: ${{ secrets.DOCKER_PASSWORD }}
+      # Send discord notification
+      - name: Discord notification
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.DISCORD_BETA_WEBHOOK }}
+        uses: Ilshidur/action-discord@0.3.2
+        with:
+          args: '🚀 The BETA Image has been updated! 🥳'

.github/workflows/docker-publish-dev.yml

@@ -7,9 +7,11 @@ on:
       - '!master'
 jobs:
   build:
+    if: github.repository_owner == 'TandoorRecipes'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@master
+      # Update Version number
       - name: Update version file
         uses: DamianReeves/write-file-action@v1.0
         with:
@@ -18,6 +20,20 @@ jobs:
             VERSION_NUMBER = 'develop'
             BUILD_REF = '${{ github.sha }}'
           write-mode: overwrite
+      # Build Vue frontend
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - name: Clear Cache
+        working-directory: ./vue
+        run: yarn cache clean --all
+      - name: Install dependencies
+        working-directory: ./vue
+        run: yarn install
+      - name: Build dependencies
+        working-directory: ./vue
+        run: yarn build
+      # Build container
       - name: Publish to Registry
         uses: elgohr/Publish-Docker-Github-Action@2.13
         with:

.github/workflows/docker-publish-latest-raspi.yml

@@ -0,0 +1,45 @@
+name: publish latest raspi image docker
+on:
+  push:
+    tags:
+    - '*'
+
+jobs:
+  build:
+    if: github.repository_owner == 'TandoorRecipes'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: Get version number
+        id: get_version
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}-raspi
+      # Update Version number
+      - name: Update version file
+        uses: DamianReeves/write-file-action@v1.0
+        with:
+          path: recipes/version.py
+          contents: |
+            VERSION_NUMBER = '${{ steps.get_version.outputs.VERSION }}-raspi'
+            BUILD_REF = '${{ github.sha }}'
+          write-mode: overwrite
+      # Build Vue frontend
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - name: Install dependencies
+        working-directory: ./vue
+        run: yarn install
+      - name: Build dependencies
+        working-directory: ./vue
+        run: yarn build
+      # Build container
+      - name: Build and publish image
+        uses: ilteoood/docker_buildx@master
+        with:
+          publish: true
+          imageName: vabene1111/recipes
+          dockerFile: Dockerfile-raspi
+          platform: linux/arm/v7
+          tag: latest-raspi
+          dockerUser: ${{ secrets.DOCKER_USERNAME }}
+          dockerPassword: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/docker-publish-latest.yml

@@ -6,12 +6,14 @@ on:
 
 jobs:
   build:
+    if: github.repository_owner == 'TandoorRecipes'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@master
       - name: Get version number
         id: get_version
         run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+      # Update Version number
       - name: Update version file
         uses: DamianReeves/write-file-action@v1.0
         with:
@@ -20,11 +22,23 @@ jobs:
             VERSION_NUMBER = '${{ steps.get_version.outputs.VERSION }}'
             BUILD_REF = '${{ github.sha }}'
           write-mode: overwrite
+      # Build Vue frontend
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - name: Install dependencies
+        working-directory: ./vue
+        run: yarn install
+      - name: Build dependencies
+        working-directory: ./vue
+        run: yarn build
+      # Build container
       - name: Build and publish image
         uses: ilteoood/docker_buildx@master
         with:
           publish: true
           imageName: vabene1111/recipes
+          platform: linux/amd64,linux/arm64
           tag: latest
-          dockerHubUser: ${{ secrets.DOCKER_USERNAME }}
-          dockerHubPassword: ${{ secrets.DOCKER_PASSWORD }}
+          dockerUser: ${{ secrets.DOCKER_USERNAME }}
+          dockerPassword: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/docker-publish-release-raspi.yml

@@ -0,0 +1,47 @@
+name: publish tagged raspi release docker
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    if: github.repository_owner == 'TandoorRecipes'
+    runs-on: ubuntu-latest
+    name: Build image job
+    steps:
+      - name: Checkout master
+        uses: actions/checkout@master
+      - name: Get version number
+        id: get_version
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+      # Update Version number
+      - name: Update version file
+        uses: DamianReeves/write-file-action@v1.0
+        with:
+          path: recipes/version.py
+          contents: |
+            VERSION_NUMBER = '${{ steps.get_version.outputs.VERSION }}'
+            BUILD_REF = '${{ github.sha }}'
+          write-mode: overwrite
+      # Build Vue frontend
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - name: Install dependencies
+        working-directory: ./vue
+        run: yarn install
+      - name: Build dependencies
+        working-directory: ./vue
+        run: yarn build
+      # Build container
+      - name: Build and publish image
+        uses: ilteoood/docker_buildx@master
+        with:
+          publish: true
+          imageName: vabene1111/recipes
+          dockerFile: Dockerfile-raspi
+          platform: linux/arm/v7
+          tag: ${{ steps.get_version.outputs.VERSION }}-raspi
+          dockerUser: ${{ secrets.DOCKER_USERNAME }}
+          dockerPassword: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/docker-publish-release.yml

@@ -1,12 +1,12 @@
 name: publish tagged release docker
 
 on:
-  push:
-    tags:
-    - '*'
+  release:
+    types: [published]
 
 jobs:
   build:
+    if: github.repository_owner == 'TandoorRecipes'
     runs-on: ubuntu-latest
     name: Build image job
     steps:
@@ -15,6 +15,7 @@ jobs:
       - name: Get version number
         id: get_version
         run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+      # Update Version number
       - name: Update version file
         uses: DamianReeves/write-file-action@v1.0
         with:
@@ -23,11 +24,30 @@ jobs:
             VERSION_NUMBER = '${{ steps.get_version.outputs.VERSION }}'
             BUILD_REF = '${{ github.sha }}'
           write-mode: overwrite
+      # Build Vue frontend
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - name: Install dependencies
+        working-directory: ./vue
+        run: yarn install
+      - name: Build dependencies
+        working-directory: ./vue
+        run: yarn build
+      # Build container
       - name: Build and publish image
         uses: ilteoood/docker_buildx@master
         with:
           publish: true
           imageName: vabene1111/recipes
+          platform: linux/amd64,linux/arm64
           tag: ${{ steps.get_version.outputs.VERSION }}
-          dockerHubUser: ${{ secrets.DOCKER_USERNAME }}
-          dockerHubPassword: ${{ secrets.DOCKER_PASSWORD }}
+          dockerUser: ${{ secrets.DOCKER_USERNAME }}
+          dockerPassword: ${{ secrets.DOCKER_PASSWORD }}
+      # Send discord notification
+      - name: Discord notification
+        env:
+          DISCORD_WEBHOOK: ${{ secrets.DISCORD_RELEASE_WEBHOOK }}
+        uses: Ilshidur/action-discord@0.3.2
+        with:
+          args: '🚀 Version {{ EVENT_PAYLOAD.release.tag_name }} of tandoor has been released 🥳 Check it out https://github.com/vabene1111/recipes/releases/tag/{{ EVENT_PAYLOAD.release.tag_name }}'

.github/workflows/docs.yml

@@ -3,15 +3,15 @@ on:
   push:
     branches:
       - master
-      - develop
 
 jobs:
   deploy:
+    if: github.repository_owner == 'TandoorRecipes'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
           python-version: 3.x
-      - run: pip install mkdocs-material
+      - run: pip install mkdocs-material mkdocs-include-markdown-plugin
       - run: mkdocs gh-deploy --force

.gitignore

@@ -79,3 +79,8 @@ postgresql/
 /docker-compose.override.yml
 vue/node_modules
 .vscode/
+vetur.config.js
+cookbook/static/vue
+vue/webpack-stats.json
+cookbook/templates/sw.js
+.prettierignore

.idea/dictionaries/vabene1111_PC.xml

@@ -6,6 +6,7 @@
       <w>csrftoken</w>
       <w>gunicorn</w>
       <w>ical</w>
+      <w>invitelink</w>
       <w>mealie</w>
       <w>pepperplate</w>
       <w>safron</w>

.pre-commit-config.yaml

@@ -1,31 +0,0 @@
-# See https://pre-commit.com for more information
-# See https://pre-commit.com/hooks.html for more hooks
-repos:
-  - repo: local
-    hooks:
-      - id: pre-commit-yarn-build
-        name: Build javascript files
-        entry: yarn --cwd ./vue build
-        always_run: true
-        language: system
-        types: [ python ]
-        pass_filenames: false
-
-#- id: pre-commit-django-migrations
-#        name: Check django migrations
-#        entry: bash -c './venv/bin/activate && ./manage.py makemigrations --check'
-#        language: system
-#        types: [ python ]
-#        pass_filenames: false
-#      - id: pre-commit-django-make-messages
-#        name: Make messages if necessary
-#        entry: ./manage.py makemessages -i venv -a
-#        language: system
-#        types: [ python ]
-#        pass_filenames: false
-#      - id: pre-commit-django-compile-messages
-#        name: Compile messages if necessary
-#        entry: ./manage.py compilemessages -i venv
-#        language: system
-#        types: [ python ]
-#        pass_filenames: false

CONTRIBUTERS.md

@@ -1,58 +1,83 @@
+# Contributers
+
 Many thanks to everyone who contributed to this project! If you add something or help out feel free to add yourself
 to this list.
 
 ## Code/Features
+
 Please have a look at the [list of pull requests](https://github.com/vabene1111/recipes/pulls) for 
 a complete list of contributions.
 Below are some of the larger contributions made yet.
 
-
-- @tourn provided the serving feature and **several** other improvements!
-- @l0c4lh057 provided a much improved ingredient text parser in [#277](https://github.com/vabene1111/recipes/pull/277)
-- @sebimarkgraf added nutritional information [#199](https://github.com/vabene1111/recipes/pull/199)
-- @cazier added reverse proxy authentication [#88](https://github.com/vabene1111/recipes/pull/88)
+- [vabene1111]
+- [Kaibu]
+- [smilerz]
+- [MaxJa4] Docker builds and other improvements
+- [tourn] provided the serving feature and **several** other improvements!
+- [l0c4lh057] provided a much improved ingredient text parser in [#277](https://github.com/vabene1111/recipes/pull/277)
+- [sebimarkgraf] added nutritional information [#199](https://github.com/vabene1111/recipes/pull/199)
+- [cazier] added reverse proxy authentication [#88](https://github.com/vabene1111/recipes/pull/88)
+- [murphy83] added support for IPv6 #1490
+- [TheHaf] added custom serving size component #1411
+- [lostlont] added LDAP support #960
 
 ## Translations
 
-### Catalan 
+### Catalan
+
 [Rubenix](https://www.transifex.com/user/profile/rubenix/)
 
 ### Dutch
-[D0T1X](https://www.transifex.com/user/profile/D0T1X/)  
-[ikbenfrank](https://www.transifex.com/user/profile/ikbenfrank/)  
-[kampsj](https://www.transifex.com/user/profile/kampsj/)  
+
+[D0T1X](https://www.transifex.com/user/profile/D0T1X/)
+[ikbenfrank](https://www.transifex.com/user/profile/ikbenfrank/)
+[kampsj](https://www.transifex.com/user/profile/kampsj/)
 
 ### French
-[jt117](https://www.transifex.com/user/profile/jt117/)  
-[nerdinator](https://www.transifex.com/user/profile/nerdinator/)  
-[agaume](https://www.transifex.com/user/profile/agaume/)  
+
+[jt117](https://www.transifex.com/user/profile/jt117/)
+[nerdinator](https://www.transifex.com/user/profile/nerdinator/)
+[agaume](https://www.transifex.com/user/profile/agaume/)
 
 ### German
+
 [eTaurus](https://www.transifex.com/user/profile/eTaurus/)
 [l0c4lh057](https://www.transifex.com/user/profile/l0c4lh057/)
+[hyperbit00](https://github.com/hyperbit00)
 
 ### Hungarian
+
 [igazka](https://www.transifex.com/user/profile/igazka/)
 
 ### Italian
-[SK3LA](https://www.transifex.com/user/profile/SK3LA/)  
-[auanasgheps](https://www.transifex.com/user/profile/auanasgheps/)  
+
+[SK3LA](https://www.transifex.com/user/profile/SK3LA/)
+[auanasgheps](https://www.transifex.com/user/profile/auanasgheps/)
 
 ### Latvian
+
 [melkypie](https://github.com/melkypie)
 
 ### Portuguese
 
-[hds](https://www.transifex.com/user/profile/hds/)  
-[mlopezifu](https://www.transifex.com/user/profile/mlopezifu/)  
-[stormsz](https://www.transifex.com/user/profile/stormsz/)  
+[hds](https://www.transifex.com/user/profile/hds/)
+[mlopezifu](https://www.transifex.com/user/profile/mlopezifu/)
+[stormsz](https://www.transifex.com/user/profile/stormsz/)
+
+### Russian
+
+[amillerr](https://github.com/amillerr)
 
 ### Spanish
 
-[albertocp](https://www.transifex.com/user/profile/albertocp/)  
-[alfa5](https://www.transifex.com/user/profile/alfa5/)  
-[mlopezifu](https://www.transifex.com/user/profile/mlopezifu/)  
-[sergio.laya](https://www.transifex.com/user/profile/sergio.laya/)  
+[albertocp](https://www.transifex.com/user/profile/albertocp/)
+[alfa5](https://www.transifex.com/user/profile/alfa5/)
+[mlopezifu](https://www.transifex.com/user/profile/mlopezifu/)
+[sergio.laya](https://www.transifex.com/user/profile/sergio.laya/)
+
+### Swedish
+
+[makanz](https://github.com/makanz)
 
 ### Turkish
 
@@ -60,4 +85,4 @@ Below are some of the larger contributions made yet.
 
 ### Vietnamese
 
-[vuongtrunghieu](https://www.transifex.com/user/profile/vuongtrunghieu/)
+[vuongtrunghieu](https://www.transifex.com/user/profile/vuongtrunghieu/)

Dockerfile

@@ -1,7 +1,7 @@
-FROM python:3.9-alpine3.12
+FROM python:3.10-alpine3.15
 
 #Install all dependencies.
-RUN apk add --no-cache postgresql-libs gettext zlib libjpeg libxml2-dev libxslt-dev py-cryptography
+RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg libwebp libxml2-dev libxslt-dev py-cryptography openldap
 
 #Print all logs without buffering it.
 ENV PYTHONUNBUFFERED 1
@@ -15,14 +15,16 @@ WORKDIR /opt/recipes
 
 COPY requirements.txt ./
 
-RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-dev jpeg-dev libressl-dev libffi-dev cargo && \
+RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-dev jpeg-dev libwebp-dev openssl-dev libffi-dev cargo openldap-dev python3-dev git && \
+    echo -n "INPUT ( libldap.so )" > /usr/lib/libldap_r.so && \
     python -m venv venv && \
     /opt/recipes/venv/bin/python -m pip install --upgrade pip && \
-    venv/bin/pip install wheel==0.36.2 && \
+    venv/bin/pip install wheel==0.37.1 && \
+    venv/bin/pip install setuptools_rust==1.1.2 && \
     venv/bin/pip install -r requirements.txt --no-cache-dir &&\
     apk --purge del .build-deps
 
 #Copy project and execute it.
 COPY . ./
 RUN chmod +x boot.sh
-ENTRYPOINT ["/opt/recipes/boot.sh"]
+ENTRYPOINT ["/opt/recipes/boot.sh"]

Dockerfile-raspi

@@ -0,0 +1,33 @@
+# builds of cryptography for raspberry pi (or better arm v7) fail for some
+FROM python:3.9-alpine3.15
+
+#Install all dependencies.
+RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg libwebp libxml2-dev libxslt-dev py-cryptography openldap gcompat
+
+#Print all logs without buffering it.
+ENV PYTHONUNBUFFERED 1
+
+#This port will be used by gunicorn.
+EXPOSE 8080
+
+#Create app dir and install requirements.
+RUN mkdir /opt/recipes
+WORKDIR /opt/recipes
+
+COPY requirements.txt ./
+RUN \
+    if [ `apk --print-arch` = "armv7" ]; then \
+    printf "[global]\nextra-index-url=https://www.piwheels.org/simple\n" > /etc/pip.conf ; \
+    fi
+RUN apk add --no-cache --virtual .build-deps gcc musl-dev zlib-dev jpeg-dev libwebp-dev python3-dev git && \
+    echo -n "INPUT ( libldap.so )" > /usr/lib/libldap_r.so && \
+    python -m venv venv && \
+    /opt/recipes/venv/bin/python -m pip install --upgrade pip && \
+    venv/bin/pip install wheel==0.37.1 && \
+    venv/bin/pip install -r requirements.txt --no-cache-dir --no-binary=Pillow && \
+    apk --purge del .build-deps
+
+#Copy project and execute it.
+COPY . ./
+RUN chmod +x boot.sh
+ENTRYPOINT ["/opt/recipes/boot.sh"]

README.md

@@ -1,6 +1,6 @@
 <h1 align="center">
   <br>
-  <a href="https://app.tandoor.dev"><img src="https://github.com/vabene1111/recipes/raw/develop/docs/logo_color.svg" height="256px" width="256px"></a>
+  <a href="https://tandoor.dev"><img src="https://github.com/vabene1111/recipes/raw/develop/docs/logo_color.svg" height="256px" width="256px"></a>
   <br>
   Tandoor Recipes
   <br>
@@ -9,54 +9,92 @@
 <h4 align="center">The recipe manager that allows you to manage your ever growing collection of digital recipes.</h4>
 
 <p align="center">
-
-<img src="https://github.com/vabene1111/recipes/workflows/Continous%20Integration/badge.svg?branch=develop" >
-<img src="https://img.shields.io/github/stars/vabene1111/recipes" >
-<img src="https://img.shields.io/github/forks/vabene1111/recipes" >
-<img src="https://img.shields.io/docker/pulls/vabene1111/recipes" >
-
+<a href="https://github.com/vabene1111/recipes/actions" target="_blank" rel="noopener noreferrer"><img src="https://github.com/vabene1111/recipes/workflows/Continuous%20Integration/badge.svg?branch=master" ></a>
+<a href="https://github.com/vabene1111/recipes/stargazers" target="_blank" rel="noopener noreferrer"><img src="https://img.shields.io/github/stars/vabene1111/recipes" ></a>
+<a href="https://github.com/vabene1111/recipes/network/members" target="_blank" rel="noopener noreferrer"><img src="https://img.shields.io/github/forks/vabene1111/recipes" ></a>
+<a href="https://discord.gg/RhzBrfWgtp" target="_blank" rel="noopener noreferrer"><img src="https://badgen.net/badge/icon/discord?icon=discord&label" ></a>
+<a href="https://hub.docker.com/r/vabene1111/recipes" target="_blank" rel="noopener noreferrer"><img src="https://img.shields.io/docker/pulls/vabene1111/recipes" ></a>
+<a href="https://github.com/vabene1111/recipes/releases/latest" rel="noopener noreferrer"><img src="https://img.shields.io/github/v/release/vabene1111/recipes" ></a>
+<a href="https://app.tandoor.dev/accounts/login/?demo" rel="noopener noreferrer"><img src="https://img.shields.io/badge/demo-available-success" ></a>
 </p>
 
 <p align="center">
+<a href="https://tandoor.dev" target="_blank" rel="noopener noreferrer">Website</a> •
 <a href="https://docs.tandoor.dev/install/docker/" target="_blank" rel="noopener noreferrer">Installation</a> •
-<a href="https://docs.tandoor.dev/" target="_blank" rel="noopener noreferrer">Documentation</a> •
-<a href="https://app.tandoor.dev/" target="_blank" rel="noopener noreferrer">Demo</a>
+<a href="https://docs.tandoor.dev/" target="_blank" rel="noopener noreferrer">Docs</a> •
+<a href="https://app.tandoor.dev/accounts/login/?demo" target="_blank" rel="noopener noreferrer">Demo</a> •
+<a href="https://discord.gg/RhzBrfWgtp" target="_blank" rel="noopener noreferrer">Discord</a>
 </p>
 
 ![Preview](docs/preview.png)
 
-# Your Feedback
+## Core Features
+- 🥗 **Manage your recipes** - Manage your ever growing recipe collection
+- 📆 **Plan** - multiple meals for each day
+- 🛒 **Shopping lists** - via the meal plan or straight from recipes
+- 📚 **Cookbooks** - collect recipes into books
+- 👪 **Share and collaborate** on recipes with friends and family
 
-Share some information on how you use Tandoor to help me improve the application [Google Survey](https://forms.gle/qNfLK2tWTeWHe9Qd7)
+## Made by and for power users
 
-## Features
-
-- 📦 **Sync** files with Dropbox and Nextcloud (more can easily be added)
-- 🔍 Powerful **search** with Djangos [TrigramSimilarity](https://docs.djangoproject.com/en/3.0/ref/contrib/postgres/search/#trigram-similarity)
+- 🔍 Powerful & customizable **search** with fulltext support and [TrigramSimilarity](https://docs.djangoproject.com/en/3.0/ref/contrib/postgres/search/#trigram-similarity)
 - 🏷️ Create and search for **tags**, assign them in batch to all files matching certain filters
-- 📄 **Create recipes** locally within a nice, standardized web interface
-- ⬇️ **Import recipes** from thousands of websites supporting [ld+json or microdata](https://schema.org/Recipe)
-- 📱 Optimized for use on **mobile** devices like phones and tablets
-- 🛒 Generate **shopping** lists from recipes
-- 📆 Create a **Plan** on what to eat when
-- 👪 **Share** recipes with friends and comment on them to suggest or remember changes you made
-- ➗ automatically convert decimal units to **fractions** for those who like this
-- 🐳 Easy setup with **Docker** and included examples for Kubernetes, Unraid and Synology
+- ↔️ Quickly merge and rename ingredients, tags and units 
+- 📥️ **Import recipes** from thousands of websites supporting [ld+json or microdata](https://schema.org/Recipe)
+- ➗ Support for **fractions** or decimals
+- 🐳 Easy setup with **Docker** and included examples for **Kubernetes**, **Unraid** and **Synology**
 - 🎨 Customize your interface with **themes**
-- ✉️ Export and import recipes from other users
+- 📦 **Sync** files with Dropbox and Nextcloud
+  
+## All the must haves
+
+- 📱Optimized for use on **mobile** devices
 - 🌍 localized in many languages thanks to the awesome community
-- ➕ Many more like recipe scaling, image compression, cookbooks, printing views, ...
+- 📥️ **Import your collection** from many other [recipe managers](https://docs.tandoor.dev/features/import_export/)
+- ➕ Many more like recipe scaling, image compression, printing views and supermarkets
 
 This application is meant for people with a collection of recipes they want to share with family and friends or simply
 store them in a nicely organized way. A basic permission system exists but this application is not meant to be run as 
 a public page.
+
+## Docs
+
 Documentation can be found [here](https://docs.tandoor.dev/).
 
-While this application has been around for a while and is actively used by many (including myself), it is still considered
-**beta** software that has a lot of rough edges and unpolished parts.
+## Support our work
+Tandoor is developed by volunteers in their free time just because its fun. That said earning 
+some money with the project allows us to spend more time on it and thus make improvements we otherwise couldn't.
+Because of that there are several ways you can support us
+
+- **GitHub Sponsors** You can sponsor contributors of this project on GitHub: [vabene1111](https://github.com/sponsors/vabene1111)
+- **Host at Hetzner** We have been very happy customers of Hetzner for multiple years for all of our projects. If you want to get into self-hosting or are tired of the expensive big providers, their cloud servers are a great place to get started. When you sign up via our [referral link](https://hetzner.cloud/?ref=ISdlrLmr9kGj) you will get 20€ worth of cloud credits and we get a small kickback too.
+- **Let us host for you** We are offering a [hosted version](https://app.tandoor.dev) where all profits support us and the development of tandoor (currently only available in germany).  
+
+## Contributing
+
+You can help out with the ongoing development by looking for potential bugs in our code base, or by contributing new features. We are always welcoming new pull requests containing bug fixes, refactors and new features. We have a list of tasks and bugs on our issue tracker on Github. Please comment on issues if you want to contribute with, to avoid duplicating effort.
+
+## Your Feedback
+
+Share some information on how you use Tandoor to help me improve the application [Google Survey](https://forms.gle/qNfLK2tWTeWHe9Qd7)
+
+## Get in touch
+
+<table>
+  <tr>
+    <td><a href="https://discord.gg/RhzBrfWgtp">Discord</a></td>
+    <td>We have a public Discord server that anyone can join. This is where all our developers and contributors hang out and where we make announcements</td>
+  </tr>
+
+  <tr>
+    <td><a href="https://twitter.com/TandoorRecipes">Twitter</a></td>
+    <td>You can follow our Twitter account to get updates on new features or releases</td>
+  </tr>
+</table>
+
 ## License
 
-Beginning with version 0.10.0 the code in this repository is licensed under the [GNU AGPL v3](https://www.gnu.org/licenses/agpl-3.0.de.html) license with an
+Beginning with version 0.10.0 the code in this repository is licensed under the [GNU AGPL v3](https://www.gnu.org/licenses/agpl-3.0.de.html) license with a
 [common clause](https://commonsclause.com/) selling exception. See [LICENSE.md](https://github.com/vabene1111/recipes/blob/develop/LICENSE.md) for details.
 
 > NOTE: There appears to be a whole range of legal issues with licensing anything else then the standard completely open licenses.
@@ -67,8 +105,8 @@ Beginning with version 0.10.0 the code in this repository is licensed under the
 **This software and *all* its features are and will always be free for everyone to use and enjoy.**
 
 The reason for the selling exception is that a significant amount of time was spend over multiple years to develop this software.
-A payed hosted version which will be identical in features and code base to the software offered in this repository will
+A paid hosted version which will be identical in features and code base to the software offered in this repository will
 likely be released in the future (including all features needed to sell a hosted version as they might also be useful for personal use).
-This will not only benefit me personally but also everyone who self-hosts this software as any profits made trough selling the hosted option
+This will not only benefit me personally but also everyone who self-hosts this software as any profits made through selling the hosted option
 allow me to spend more time developing and improving the software for everyone. Selling exceptions are [approved by Richard Stallman](http://www.gnu.org/philosophy/selling-exceptions.en.html) and the
 common clause license is very permissive (see the [FAQ](https://commonsclause.com/)).

SECURITY.md

@@ -7,4 +7,4 @@ Since this software is still considered beta/WIP support is always only given fo
 ## Reporting a Vulnerability
 
 Please open a normal public issue if you have any security related concerns. If you feel like the issue should not be discussed in 
-public just open a generic issue and we will discuss further communitcation there (since GitHub does not allow everyone to create a security advisory :/).
+public just open a generic issue and we will discuss further communication there (since GitHub does not allow everyone to create a security advisory :/).

boot.sh

@@ -1,12 +1,68 @@
 #!/bin/sh
 source venv/bin/activate
 
-echo "Updating database"
+TANDOOR_PORT="${TANDOOR_PORT:-8080}"
+GUNICORN_WORKERS="${GUNICORN_WORKERS:-3}"
+GUNICORN_THREADS="${GUNICORN_THREADS:-2}"
+NGINX_CONF_FILE=/opt/recipes/nginx/conf.d/Recipes.conf
+
+display_warning() {
+    echo "[WARNING]"
+    echo -e "$1"
+}
+
+echo "Checking configuration..."
+
+# Nginx config file must exist if gunicorn is not active
+if [ ! -f "$NGINX_CONF_FILE" ] && [ $GUNICORN_MEDIA -eq 0 ]; then
+    display_warning "Nginx configuration file could not be found at the default location!\nPath: ${NGINX_CONF_FILE}"
+fi
+
+# SECRET_KEY must be set in .env file
+if [ -z "${SECRET_KEY}" ]; then
+    display_warning "The environment variable 'SECRET_KEY' is not set but REQUIRED for running Tandoor!"
+fi
+
+
+echo "Waiting for database to be ready..."
+
+attempt=0
+max_attempts=20
+
+if [ "${DB_ENGINE}" != 'django.db.backends.sqlite3' ]; then
+
+  # POSTGRES_PASSWORD must be set in .env file
+  if [ -z "${POSTGRES_PASSWORD}" ]; then
+      display_warning "The environment variable 'POSTGRES_PASSWORD' is not set but REQUIRED for running Tandoor!"
+  fi
+
+  while pg_isready --host=${POSTGRES_HOST} --port=${POSTGRES_PORT} --user=${POSTGRES_USER} -q; status=$?; attempt=$((attempt+1)); [ $status -ne 0 ] && [ $attempt -le $max_attempts ]; do
+      sleep 5
+  done
+fi
+
+if [ $attempt -gt $max_attempts ]; then
+    echo -e "\nDatabase not reachable. Maximum attempts exceeded."
+    echo "Please check logs above - misconfiguration is very likely."
+    echo "Make sure the DB container is up and POSTGRES_HOST is set properly."
+    echo "Shutting down container."
+    exit 1 # exit with error to make the container stop
+fi
+
+echo "Database is ready"
+
+echo "Migrating database"
+
+
 python manage.py migrate
+
+echo "Generating static files"
+
 python manage.py collectstatic_js_reverse
 python manage.py collectstatic --noinput
+
 echo "Done"
 
 chmod -R 755 /opt/recipes/mediafiles
 
-exec gunicorn -b :8080 --access-logfile - --error-logfile - --log-level INFO recipes.wsgi
+exec gunicorn -b :$TANDOOR_PORT --workers $GUNICORN_WORKERS --threads $GUNICORN_THREADS --access-logfile - --error-logfile - --log-level INFO recipes.wsgi

cookbook/admin.py

@@ -1,14 +1,21 @@
+from django.conf import settings
 from django.contrib import admin
 from django.contrib.auth.admin import UserAdmin
-from django.contrib.auth.models import User, Group
+from django.contrib.auth.models import Group, User
+from django.contrib.postgres.search import SearchVector
+from django.utils import translation
+from django_scopes import scopes_disabled
+from treebeard.admin import TreeAdmin
+from treebeard.forms import movenodeform_factory
 
-from .models import (Comment, CookLog, Food, Ingredient, InviteLink, Keyword,
-                     MealPlan, MealType, NutritionInformation, Recipe,
-                     RecipeBook, RecipeBookEntry, RecipeImport, ShareLink,
-                     ShoppingList, ShoppingListEntry, ShoppingListRecipe,
-                     Space, Step, Storage, Sync, SyncLog, Unit, UserPreference,
-                     ViewLog, Supermarket, SupermarketCategory, SupermarketCategoryRelation,
-                     ImportLog, TelegramBot, BookmarkletImport, UserFile)
+from cookbook.managers import DICTIONARY
+
+from .models import (BookmarkletImport, Comment, CookLog, Food, FoodInheritField, ImportLog,
+                     Ingredient, InviteLink, Keyword, MealPlan, MealType, NutritionInformation,
+                     Recipe, RecipeBook, RecipeBookEntry, RecipeImport, SearchPreference, ShareLink,
+                     ShoppingList, ShoppingListEntry, ShoppingListRecipe, Space, Step, Storage,
+                     Supermarket, SupermarketCategory, SupermarketCategoryRelation, Sync, SyncLog,
+                     TelegramBot, Unit, UserFile, UserPreference, ViewLog, Automation, UserSpace)
 
 
 class CustomUserAdmin(UserAdmin):
@@ -22,26 +29,61 @@ admin.site.register(User, CustomUserAdmin)
 admin.site.unregister(Group)
 
 
+@admin.action(description='Delete all data from a space')
+def delete_space_action(modeladmin, request, queryset):
+    for space in queryset:
+        space.save()
+
+
 class SpaceAdmin(admin.ModelAdmin):
-    list_display = ('name', 'created_by', 'message')
+    list_display = ('name', 'created_by', 'max_recipes', 'max_users', 'max_file_storage_mb', 'allow_sharing')
+    search_fields = ('name', 'created_by__username')
+    list_filter = ('max_recipes', 'max_users', 'max_file_storage_mb', 'allow_sharing')
+    date_hierarchy = 'created_at'
+    actions = [delete_space_action]
 
 
 admin.site.register(Space, SpaceAdmin)
 
 
+class UserSpaceAdmin(admin.ModelAdmin):
+    list_display = ('user', 'space',)
+    search_fields = ('user__username', 'space__name',)
+
+
+admin.site.register(UserSpace, UserSpaceAdmin)
+
+
 class UserPreferenceAdmin(admin.ModelAdmin):
-    list_display = ('name', 'space', 'theme', 'nav_color', 'default_page', 'search_style',)
+    list_display = ('name', 'theme', 'nav_color', 'default_page',)
+    search_fields = ('user__username',)
+    list_filter = ('theme', 'nav_color', 'default_page',)
+    date_hierarchy = 'created_at'
 
     @staticmethod
     def name(obj):
-        return obj.user.get_user_name()
+        return obj.user.get_user_display_name()
 
 
 admin.site.register(UserPreference, UserPreferenceAdmin)
 
 
+class SearchPreferenceAdmin(admin.ModelAdmin):
+    list_display = ('name', 'search', 'trigram_threshold',)
+    search_fields = ('user__username',)
+    list_filter = ('search',)
+
+    @staticmethod
+    def name(obj):
+        return obj.user.get_user_display_name()
+
+
+admin.site.register(SearchPreference, SearchPreferenceAdmin)
+
+
 class StorageAdmin(admin.ModelAdmin):
     list_display = ('name', 'method')
+    search_fields = ('name',)
 
 
 admin.site.register(Storage, StorageAdmin)
@@ -49,6 +91,7 @@ admin.site.register(Storage, StorageAdmin)
 
 class SyncAdmin(admin.ModelAdmin):
     list_display = ('storage', 'path', 'active', 'last_checked')
+    search_fields = ('storage__name', 'path')
 
 
 admin.site.register(Sync, SyncAdmin)
@@ -72,32 +115,95 @@ class SyncLogAdmin(admin.ModelAdmin):
 
 admin.site.register(SyncLog, SyncLogAdmin)
 
-admin.site.register(Keyword)
+
+@admin.action(description='Temporarily ENABLE sorting on Foods and Keywords.')
+def enable_tree_sorting(modeladmin, request, queryset):
+    Food.node_order_by = ['name']
+    Keyword.node_order_by = ['name']
+    with scopes_disabled():
+        Food.fix_tree(fix_paths=True)
+        Keyword.fix_tree(fix_paths=True)
+
+
+@admin.action(description='Temporarily DISABLE sorting on Foods and Keywords.')
+def disable_tree_sorting(modeladmin, request, queryset):
+    Food.node_order_by = []
+    Keyword.node_order_by = []
+
+
+@admin.action(description='Fix problems and sort tree by name')
+def sort_tree(modeladmin, request, queryset):
+    orginal_value = modeladmin.model.node_order_by[:]
+    modeladmin.model.node_order_by = ['name']
+    with scopes_disabled():
+        modeladmin.model.fix_tree(fix_paths=True)
+    modeladmin.model.node_order_by = orginal_value
+
+
+class KeywordAdmin(TreeAdmin):
+    form = movenodeform_factory(Keyword)
+    ordering = ('space', 'path',)
+    search_fields = ('name',)
+    actions = [sort_tree, enable_tree_sorting, disable_tree_sorting]
+
+
+admin.site.register(Keyword, KeywordAdmin)
 
 
 class StepAdmin(admin.ModelAdmin):
-    list_display = ('name', 'type', 'order')
+    list_display = ('name', 'order',)
+    search_fields = ('name',)
 
 
 admin.site.register(Step, StepAdmin)
 
 
+@admin.action(description='Rebuild index for selected recipes')
+def rebuild_index(modeladmin, request, queryset):
+    language = DICTIONARY.get(translation.get_language(), 'simple')
+    with scopes_disabled():
+        Recipe.objects.all().update(
+            name_search_vector=SearchVector('name__unaccent', weight='A', config=language),
+            desc_search_vector=SearchVector('description__unaccent', weight='B', config=language)
+        )
+        Step.objects.all().update(search_vector=SearchVector('instruction__unaccent', weight='B', config=language))
+
+
 class RecipeAdmin(admin.ModelAdmin):
     list_display = ('name', 'internal', 'created_by', 'storage')
+    search_fields = ('name', 'created_by__username')
+    list_filter = ('internal',)
+    date_hierarchy = 'created_at'
 
     @staticmethod
     def created_by(obj):
-        return obj.created_by.get_user_name()
+        return obj.created_by.get_user_display_name()
+
+    if settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2', 'django.db.backends.postgresql']:
+        actions = [rebuild_index]
 
 
 admin.site.register(Recipe, RecipeAdmin)
 
 admin.site.register(Unit)
-admin.site.register(Food)
+
+
+# admin.site.register(FoodInheritField)
+
+
+class FoodAdmin(TreeAdmin):
+    form = movenodeform_factory(Keyword)
+    ordering = ('space', 'path',)
+    search_fields = ('name',)
+    actions = [sort_tree, enable_tree_sorting, disable_tree_sorting]
+
+
+admin.site.register(Food, FoodAdmin)
 
 
 class IngredientAdmin(admin.ModelAdmin):
     list_display = ('food', 'amount', 'unit')
+    search_fields = ('food__name', 'unit__name')
 
 
 admin.site.register(Ingredient, IngredientAdmin)
@@ -105,10 +211,12 @@ admin.site.register(Ingredient, IngredientAdmin)
 
 class CommentAdmin(admin.ModelAdmin):
     list_display = ('recipe', 'name', 'created_at')
+    search_fields = ('text', 'created_by__username')
+    date_hierarchy = 'created_at'
 
     @staticmethod
     def name(obj):
-        return obj.created_by.get_user_name()
+        return obj.created_by.get_user_display_name()
 
 
 admin.site.register(Comment, CommentAdmin)
@@ -123,10 +231,11 @@ admin.site.register(RecipeImport, RecipeImportAdmin)
 
 class RecipeBookAdmin(admin.ModelAdmin):
     list_display = ('name', 'user_name')
+    search_fields = ('name', 'created_by__username')
 
     @staticmethod
     def user_name(obj):
-        return obj.created_by.get_user_name()
+        return obj.created_by.get_user_display_name()
 
 
 admin.site.register(RecipeBook, RecipeBookAdmin)
@@ -144,7 +253,7 @@ class MealPlanAdmin(admin.ModelAdmin):
 
     @staticmethod
     def user(obj):
-        return obj.created_by.get_user_name()
+        return obj.created_by.get_user_display_name()
 
 
 admin.site.register(MealPlan, MealPlanAdmin)
@@ -152,6 +261,7 @@ admin.site.register(MealPlan, MealPlanAdmin)
 
 class MealTypeAdmin(admin.ModelAdmin):
     list_display = ('name', 'created_by', 'order')
+    search_fields = ('name', 'created_by__username')
 
 
 admin.site.register(MealType, MealTypeAdmin)
@@ -166,7 +276,7 @@ admin.site.register(ViewLog, ViewLogAdmin)
 
 class InviteLinkAdmin(admin.ModelAdmin):
     list_display = (
-        'group', 'valid_until',
+        'group', 'valid_until', 'space',
         'created_by', 'created_at', 'used_by'
     )
 
@@ -189,7 +299,7 @@ admin.site.register(ShoppingListRecipe, ShoppingListRecipeAdmin)
 
 
 class ShoppingListEntryAdmin(admin.ModelAdmin):
-    list_display = ('id', 'food', 'unit', 'list_recipe', 'checked')
+    list_display = ('id', 'food', 'unit', 'list_recipe', 'created_by', 'created_at', 'checked')
 
 
 admin.site.register(ShoppingListEntry, ShoppingListEntryAdmin)

cookbook/apps.py

@@ -1,5 +1,37 @@
+import traceback
+
 from django.apps import AppConfig
+from django.conf import settings
+from django.db import OperationalError, ProgrammingError
+from django_scopes import scopes_disabled
+
+from recipes.settings import DEBUG
 
 
 class CookbookConfig(AppConfig):
     name = 'cookbook'
+
+    def ready(self):
+        import cookbook.signals  # noqa
+
+        # if not settings.DISABLE_TREE_FIX_STARTUP:
+        #     # when starting up run fix_tree to:
+        #     #   a) make sure that nodes are sorted when switching between sort modes
+        #     #   b) fix problems, if any, with tree consistency
+        #     with scopes_disabled():
+        #         try:
+        #             from cookbook.models import Food, Keyword
+        #             Keyword.fix_tree(fix_paths=True)
+        #             Food.fix_tree(fix_paths=True)
+        #         except OperationalError:
+        #             if DEBUG:
+        #                 traceback.print_exc()
+        #             pass  # if model does not exist there is no need to fix it
+        #         except ProgrammingError:
+        #             if DEBUG:
+        #                 traceback.print_exc()
+        #             pass  # if migration has not been run database cannot be fixed yet
+        #         except Exception:
+        #             if DEBUG:
+        #                 traceback.print_exc()
+        #             pass  # dont break startup just because fix could not run, need to investigate cases when this happens

cookbook/filters.py

@@ -1,83 +0,0 @@
-import django_filters
-from django.conf import settings
-from django.contrib.postgres.search import TrigramSimilarity
-from django.db.models import Q
-from django.utils.translation import gettext as _
-from django_scopes import scopes_disabled
-
-from cookbook.forms import MultiSelectWidget
-from cookbook.models import Food, Keyword, Recipe, ShoppingList
-
-with scopes_disabled():
-    class RecipeFilter(django_filters.FilterSet):
-        name = django_filters.CharFilter(method='filter_name')
-        keywords = django_filters.ModelMultipleChoiceFilter(
-            queryset=Keyword.objects.none(),
-            widget=MultiSelectWidget,
-            method='filter_keywords'
-        )
-        foods = django_filters.ModelMultipleChoiceFilter(
-            queryset=Food.objects.none(),
-            widget=MultiSelectWidget,
-            method='filter_foods',
-            label=_('Ingredients')
-        )
-
-        def __init__(self, data=None, *args, **kwargs):
-            space = kwargs.pop('space')
-            super().__init__(data, *args, **kwargs)
-            self.filters['foods'].queryset = Food.objects.filter(space=space).all()
-            self.filters['keywords'].queryset = Keyword.objects.filter(space=space).all()
-
-        @staticmethod
-        def filter_keywords(queryset, name, value):
-            if not name == 'keywords':
-                return queryset
-            for x in value:
-                queryset = queryset.filter(keywords=x)
-            return queryset
-
-        @staticmethod
-        def filter_foods(queryset, name, value):
-            if not name == 'foods':
-                return queryset
-            for x in value:
-                queryset = queryset.filter(steps__ingredients__food__name=x).distinct()
-            return queryset
-
-        @staticmethod
-        def filter_name(queryset, name, value):
-            if not name == 'name':
-                return queryset
-            if settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2',
-                                                           'django.db.backends.postgresql']:
-                queryset = queryset.annotate(similarity=TrigramSimilarity('name', value), ).filter(
-                    Q(similarity__gt=0.1) | Q(name__unaccent__icontains=value)).order_by('-similarity')
-            else:
-                queryset = queryset.filter(name__icontains=value)
-            return queryset
-
-        class Meta:
-            model = Recipe
-            fields = ['name', 'keywords', 'foods', 'internal']
-
-
-    class FoodFilter(django_filters.FilterSet):
-        name = django_filters.CharFilter(lookup_expr='icontains')
-
-        class Meta:
-            model = Food
-            fields = ['name']
-
-
-    class ShoppingListFilter(django_filters.FilterSet):
-
-        def __init__(self, data=None, *args, **kwargs):
-            if data is not None:
-                data = data.copy()
-                data.setdefault("finished", False)
-            super().__init__(data, *args, **kwargs)
-
-        class Meta:
-            model = ShoppingList
-            fields = ['finished']

cookbook/forms.py

@@ -1,16 +1,16 @@
+from datetime import datetime
+
 from django import forms
 from django.conf import settings
 from django.core.exceptions import ValidationError
-from django.forms import widgets
+from django.forms import NumberInput, widgets
 from django.utils.translation import gettext_lazy as _
 from django_scopes import scopes_disabled
 from django_scopes.forms import SafeModelChoiceField, SafeModelMultipleChoiceField
-from emoji_picker.widgets import EmojiPickerTextInput
 from hcaptcha.fields import hCaptchaField
 
-from .models import (Comment, Food, InviteLink, Keyword, MealPlan, Recipe,
-                     RecipeBook, RecipeBookEntry, Storage, Sync, Unit, User,
-                     UserPreference, SupermarketCategory, MealType, Space)
+from .models import (Comment, Food, InviteLink, Keyword, MealPlan, MealType, Recipe, RecipeBook,
+                     RecipeBookEntry, SearchPreference, Space, Storage, Sync, User, UserPreference)
 
 
 class SelectWidget(widgets.Select):
@@ -36,37 +36,58 @@ class DateWidget(forms.DateInput):
 class UserPreferenceForm(forms.ModelForm):
     prefix = 'preference'
 
+    def __init__(self, *args, **kwargs):
+        space = kwargs.pop('space')
+        super().__init__(*args, **kwargs)
+        self.fields['plan_share'].queryset = User.objects.filter(userspace__space=space).all()
+
     class Meta:
         model = UserPreference
         fields = (
-            'default_unit', 'use_fractions', 'theme', 'nav_color',
-            'sticky_navbar', 'default_page', 'show_recent', 'search_style',
-            'plan_share', 'ingredient_decimals', 'shopping_auto_sync',
-            'comments'
+            'default_unit', 'use_fractions', 'use_kj', 'theme', 'nav_color',
+            'sticky_navbar', 'default_page', 'plan_share', 'ingredient_decimals', 'comments', 'left_handed',
         )
 
+        labels = {
+            'default_unit': _('Default unit'),
+            'use_fractions': _('Use fractions'),
+            'use_kj': _('Use KJ'),
+            'theme': _('Theme'),
+            'nav_color': _('Navbar color'),
+            'sticky_navbar': _('Sticky navbar'),
+            'default_page': _('Default page'),
+            'plan_share': _('Plan sharing'),
+            'ingredient_decimals': _('Ingredient decimal places'),
+            'shopping_auto_sync': _('Shopping list auto sync period'),
+            'comments': _('Comments'),
+            'left_handed': _('Left-handed mode')
+        }
+
         help_texts = {
             'nav_color': _('Color of the top navigation bar. Not all colors work with all themes, just try them out!'),
-            # noqa: E501
-            'default_unit': _('Default Unit to be used when inserting a new ingredient into a recipe.'),  # noqa: E501
+            
+            'default_unit': _('Default Unit to be used when inserting a new ingredient into a recipe.'),  
             'use_fractions': _(
                 'Enables support for fractions in ingredient amounts (e.g. convert decimals to fractions automatically)'),
-            # noqa: E501
-            'plan_share': _(
-                'Users with whom newly created meal plan/shopping list entries should be shared by default.'),
-            # noqa: E501
-            'show_recent': _('Show recently viewed recipes on search page.'),  # noqa: E501
-            'ingredient_decimals': _('Number of decimals to round ingredients.'),  # noqa: E501
-            'comments': _('If you want to be able to create and see comments underneath recipes.'),  # noqa: E501
+            
+            'use_kj': _('Display nutritional energy amounts in joules instead of calories'),  
+            'plan_share': _('Users with whom newly created meal plans should be shared by default.'),
+            'shopping_share': _('Users with whom to share shopping lists.'),
+            'ingredient_decimals': _('Number of decimals to round ingredients.'),  
+            'comments': _('If you want to be able to create and see comments underneath recipes.'),  
             'shopping_auto_sync': _(
-                'Setting to 0 will disable auto sync. When viewing a shopping list the list is updated every set seconds to sync changes someone else might have made. Useful when shopping with multiple people but might use a little bit '  # noqa: E501
-                'of mobile data. If lower than instance limit it is reset when saving.'  # noqa: E501
+                'Setting to 0 will disable auto sync. When viewing a shopping list the list is updated every set seconds to sync changes someone else might have made. Useful when shopping with multiple people but might use a little bit '  
+                'of mobile data. If lower than instance limit it is reset when saving.'  
             ),
-            'sticky_navbar': _('Makes the navbar stick to the top of the page.')  # noqa: E501
+            'sticky_navbar': _('Makes the navbar stick to the top of the page.'),  
+            'mealplan_autoadd_shopping': _('Automatically add meal plan ingredients to shopping list.'),
+            'mealplan_autoexclude_onhand': _('Exclude ingredients that are on hand.'),
+            'left_handed': _('Will optimize the UI for use with your left hand.')
         }
 
         widgets = {
-            'plan_share': MultiSelectWidget
+            'plan_share': MultiSelectWidget,
+            'shopping_share': MultiSelectWidget,
         }
 
 
@@ -118,7 +139,7 @@ class ImportExportBase(forms.Form):
     NEXTCLOUD = 'NEXTCLOUD'
     MEALIE = 'MEALIE'
     CHOWDOWN = 'CHOWDOWN'
-    SAFRON = 'SAFRON'
+    SAFFRON = 'SAFFRON'
     CHEFTAP = 'CHEFTAP'
     PEPPERPLATE = 'PEPPERPLATE'
     RECIPEKEEPER = 'RECIPEKEEPER'
@@ -126,15 +147,22 @@ class ImportExportBase(forms.Form):
     RECIPESAGE = 'RECIPESAGE'
     DOMESTICA = 'DOMESTICA'
     MEALMASTER = 'MEALMASTER'
+    MELARECIPES = 'MELARECIPES'
     REZKONV = 'REZKONV'
     OPENEATS = 'OPENEATS'
+    PLANTOEAT = 'PLANTOEAT'
+    COOKBOOKAPP = 'COOKBOOKAPP'
+    COPYMETHAT = 'COPYMETHAT'
+    COOKMATE = 'COOKMATE'
+    PDF = 'PDF'
 
     type = forms.ChoiceField(choices=(
         (DEFAULT, _('Default')), (PAPRIKA, 'Paprika'), (NEXTCLOUD, 'Nextcloud Cookbook'),
-        (MEALIE, 'Mealie'), (CHOWDOWN, 'Chowdown'), (SAFRON, 'Safron'), (CHEFTAP, 'ChefTap'),
+        (MEALIE, 'Mealie'), (CHOWDOWN, 'Chowdown'), (SAFFRON, 'Saffron'), (CHEFTAP, 'ChefTap'),
         (PEPPERPLATE, 'Pepperplate'), (RECETTETEK, 'RecetteTek'), (RECIPESAGE, 'Recipe Sage'), (DOMESTICA, 'Domestica'),
         (MEALMASTER, 'MealMaster'), (REZKONV, 'RezKonv'), (OPENEATS, 'Openeats'), (RECIPEKEEPER, 'Recipe Keeper'),
-
+        (PLANTOEAT, 'Plantoeat'), (COOKBOOKAPP, 'CookBookApp'), (COPYMETHAT, 'CopyMeThat'), (PDF, 'PDF'), (MELARECIPES, 'Melarecipes'),
+        (COOKMATE, 'Cookmate')
     ))
 
 
@@ -146,8 +174,9 @@ class ImportForm(ImportExportBase):
 
 
 class ExportForm(ImportExportBase):
-    recipes = forms.ModelMultipleChoiceField(widget=MultiSelectWidget, queryset=Recipe.objects.none())
+    recipes = forms.ModelMultipleChoiceField(widget=MultiSelectWidget, queryset=Recipe.objects.none(), required=False)
     all = forms.BooleanField(required=False)
+    custom_filter = forms.IntegerField(required=False)
 
     def __init__(self, *args, **kwargs):
         space = kwargs.pop('space')
@@ -155,52 +184,6 @@ class ExportForm(ImportExportBase):
         self.fields['recipes'].queryset = Recipe.objects.filter(space=space).all()
 
 
-class UnitMergeForm(forms.Form):
-    prefix = 'unit'
-
-    new_unit = SafeModelChoiceField(
-        queryset=Unit.objects.none(),
-        widget=SelectWidget,
-        label=_('New Unit'),
-        help_text=_('New unit that other gets replaced by.'),
-    )
-    old_unit = SafeModelChoiceField(
-        queryset=Unit.objects.none(),
-        widget=SelectWidget,
-        label=_('Old Unit'),
-        help_text=_('Unit that should be replaced.'),
-    )
-
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['new_unit'].queryset = Unit.objects.filter(space=space).all()
-        self.fields['old_unit'].queryset = Unit.objects.filter(space=space).all()
-
-
-class FoodMergeForm(forms.Form):
-    prefix = 'food'
-
-    new_food = SafeModelChoiceField(
-        queryset=Food.objects.none(),
-        widget=SelectWidget,
-        label=_('New Food'),
-        help_text=_('New food that other gets replaced by.'),
-    )
-    old_food = SafeModelChoiceField(
-        queryset=Food.objects.none(),
-        widget=SelectWidget,
-        label=_('Old Food'),
-        help_text=_('Food that should be replaced.'),
-    )
-
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['new_food'].queryset = Food.objects.filter(space=space).all()
-        self.fields['old_food'].queryset = Food.objects.filter(space=space).all()
-
-
 class CommentForm(forms.ModelForm):
     prefix = 'comment'
 
@@ -216,32 +199,6 @@ class CommentForm(forms.ModelForm):
         }
 
 
-class KeywordForm(forms.ModelForm):
-    class Meta:
-        model = Keyword
-        fields = ('name', 'icon', 'description')
-        widgets = {'icon': EmojiPickerTextInput}
-
-
-class FoodForm(forms.ModelForm):
-
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['recipe'].queryset = Recipe.objects.filter(space=space).all()
-        self.fields['supermarket_category'].queryset = SupermarketCategory.objects.filter(space=space).all()
-
-    class Meta:
-        model = Food
-        fields = ('name', 'description', 'ignore_shopping', 'recipe', 'supermarket_category')
-        widgets = {'recipe': SelectWidget}
-
-        field_classes = {
-            'recipe': SafeModelChoiceField,
-            'supermarket_category': SafeModelChoiceField,
-        }
-
-
 class StorageForm(forms.ModelForm):
     username = forms.CharField(
         widget=forms.TextInput(attrs={'autocomplete': 'new-password'}),
@@ -270,6 +227,7 @@ class StorageForm(forms.ModelForm):
         }
 
 
+# TODO: Deprecate
 class RecipeBookEntryForm(forms.ModelForm):
     prefix = 'bookmark'
 
@@ -302,7 +260,14 @@ class SyncForm(forms.ModelForm):
             'storage': SafeModelChoiceField,
         }
 
+        labels = {
+            'storage': _('Storage'),
+            'path': _('Path'),
+            'active': _('Active')
+        }
 
+
+# TODO deprecate
 class BatchEditForm(forms.Form):
     search = forms.CharField(label=_('Search String'))
     keywords = forms.ModelMultipleChoiceField(
@@ -339,21 +304,7 @@ class ImportRecipeForm(forms.ModelForm):
         }
 
 
-class RecipeBookForm(forms.ModelForm):
-    def __init__(self, *args, **kwargs):
-        space = kwargs.pop('space')
-        super().__init__(*args, **kwargs)
-        self.fields['shared'].queryset = User.objects.filter(userpreference__space=space).all()
-
-    class Meta:
-        model = RecipeBook
-        fields = ('name', 'icon', 'description', 'shared')
-        widgets = {'icon': EmojiPickerTextInput, 'shared': MultiSelectWidget}
-        field_classes = {
-            'shared': SafeModelMultipleChoiceField,
-        }
-
-
+# TODO deprecate
 class MealPlanForm(forms.ModelForm):
     def __init__(self, *args, **kwargs):
         space = kwargs.pop('space')
@@ -380,9 +331,9 @@ class MealPlanForm(forms.ModelForm):
         )
 
         help_texts = {
-            'shared': _('You can list default users to share recipes with in the settings.'),  # noqa: E501
+            'shared': _('You can list default users to share recipes with in the settings.'),  
             'note': _('You can use markdown to format this field. See the <a href="/docs/markdown/">docs here</a>')
-            # noqa: E501
+            
         }
 
         widgets = {
@@ -405,7 +356,8 @@ class InviteLinkForm(forms.ModelForm):
 
     def clean(self):
         space = self.cleaned_data['space']
-        if space.max_users != 0 and (UserPreference.objects.filter(space=space).count() + InviteLink.objects.filter(space=space).count()) >= space.max_users:
+        if space.max_users != 0 and (UserPreference.objects.filter(space=space).count() +
+                                     InviteLink.objects.filter(valid_until__gte=datetime.today(), used_by=None, space=space).count()) >= space.max_users:
             raise ValidationError(_('Maximum number of users for this space reached.'))
 
     def clean_email(self):
@@ -420,7 +372,7 @@ class InviteLinkForm(forms.ModelForm):
         model = InviteLink
         fields = ('email', 'group', 'valid_until', 'space')
         help_texts = {
-            'email': _('An email address is not required but if present the invite link will be send to the user.'),
+            'email': _('An email address is not required but if present the invite link will be sent to the user.'),
         }
         field_classes = {
             'space': SafeModelChoiceField,
@@ -471,3 +423,122 @@ class UserCreateForm(forms.Form):
             attrs={'autocomplete': 'new-password', 'type': 'password'}
         )
     )
+
+
+class SearchPreferenceForm(forms.ModelForm):
+    prefix = 'search'
+    trigram_threshold = forms.DecimalField(min_value=0.01, max_value=1, decimal_places=2,
+                                           widget=NumberInput(attrs={'class': "form-control-range", 'type': 'range'}),
+                                           help_text=_(
+                                               'Determines how fuzzy a search is if it uses trigram similarity matching (e.g. low values mean more typos are ignored).'))
+    preset = forms.CharField(widget=forms.HiddenInput(), required=False)
+
+    class Meta:
+        model = SearchPreference
+        fields = (
+            'search', 'lookup', 'unaccent', 'icontains', 'istartswith', 'trigram', 'fulltext', 'trigram_threshold')
+
+        help_texts = {
+            'search': _(
+                'Select type method of search.  Click <a href="/docs/search/">here</a> for full description of choices.'),
+            'lookup': _('Use fuzzy matching on units, keywords and ingredients when editing and importing recipes.'),
+            'unaccent': _(
+                'Fields to search ignoring accents.  Selecting this option can improve or degrade search quality depending on language'),
+            'icontains': _(
+                "Fields to search for partial matches.  (e.g. searching for 'Pie' will return 'pie' and 'piece' and 'soapie')"),
+            'istartswith': _(
+                "Fields to search for beginning of word matches. (e.g. searching for 'sa' will return 'salad' and 'sandwich')"),
+            'trigram': _(
+                "Fields to 'fuzzy' search. (e.g. searching for 'recpie' will find 'recipe'.)  Note: this option will conflict with 'web' and 'raw' methods of search."),
+            'fulltext': _(
+                "Fields to full text search.  Note: 'web', 'phrase', and 'raw' search methods only function with fulltext fields."),
+        }
+
+        labels = {
+            'search': _('Search Method'),
+            'lookup': _('Fuzzy Lookups'),
+            'unaccent': _('Ignore Accent'),
+            'icontains': _("Partial Match"),
+            'istartswith': _("Starts With"),
+            'trigram': _("Fuzzy Search"),
+            'fulltext': _("Full Text")
+        }
+
+        widgets = {
+            'search': SelectWidget,
+            'unaccent': MultiSelectWidget,
+            'icontains': MultiSelectWidget,
+            'istartswith': MultiSelectWidget,
+            'trigram': MultiSelectWidget,
+            'fulltext': MultiSelectWidget,
+        }
+
+
+class ShoppingPreferenceForm(forms.ModelForm):
+    prefix = 'shopping'
+
+    class Meta:
+        model = UserPreference
+
+        fields = (
+            'shopping_share', 'shopping_auto_sync', 'mealplan_autoadd_shopping', 'mealplan_autoexclude_onhand',
+            'mealplan_autoinclude_related', 'shopping_add_onhand', 'default_delay', 'filter_to_supermarket', 'shopping_recent_days', 'csv_delim', 'csv_prefix'
+        )
+
+        help_texts = {
+            'shopping_share': _('Users will see all items you add to your shopping list.  They must add you to see items on their list.'),
+            'shopping_auto_sync': _(
+                'Setting to 0 will disable auto sync. When viewing a shopping list the list is updated every set seconds to sync changes someone else might have made. Useful when shopping with multiple people but might use a little bit '  
+                'of mobile data. If lower than instance limit it is reset when saving.'  
+            ),
+            'mealplan_autoadd_shopping': _('Automatically add meal plan ingredients to shopping list.'),
+            'mealplan_autoinclude_related': _('When adding a meal plan to the shopping list (manually or automatically), include all related recipes.'),
+            'mealplan_autoexclude_onhand': _('When adding a meal plan to the shopping list (manually or automatically), exclude ingredients that are on hand.'),
+            'default_delay': _('Default number of hours to delay a shopping list entry.'),
+            'filter_to_supermarket': _('Filter shopping list to only include supermarket categories.'),
+            'shopping_recent_days': _('Days of recent shopping list entries to display.'),
+            'shopping_add_onhand': _("Mark food 'On Hand' when checked off shopping list."),
+            'csv_delim': _('Delimiter to use for CSV exports.'),
+            'csv_prefix': _('Prefix to add when copying list to the clipboard.'),
+
+        }
+        labels = {
+            'shopping_share': _('Share Shopping List'),
+            'shopping_auto_sync': _('Autosync'),
+            'mealplan_autoadd_shopping': _('Auto Add Meal Plan'),
+            'mealplan_autoexclude_onhand': _('Exclude On Hand'),
+            'mealplan_autoinclude_related': _('Include Related'),
+            'default_delay': _('Default Delay Hours'),
+            'filter_to_supermarket': _('Filter to Supermarket'),
+            'shopping_recent_days': _('Recent Days'),
+            'csv_delim': _('CSV Delimiter'),
+            "csv_prefix_label": _("List Prefix"),
+            'shopping_add_onhand': _("Auto On Hand"),
+        }
+
+        widgets = {
+            'shopping_share': MultiSelectWidget
+        }
+
+
+class SpacePreferenceForm(forms.ModelForm):
+    prefix = 'space'
+    reset_food_inherit = forms.BooleanField(label=_("Reset Food Inheritance"), initial=False, required=False,
+                                            help_text=_("Reset all food to inherit the fields configured."))
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)  # populates the post
+        self.fields['food_inherit'].queryset = Food.inheritable_fields
+
+    class Meta:
+        model = Space
+
+        fields = ('food_inherit', 'reset_food_inherit', 'show_facet_count')
+
+        help_texts = {
+            'food_inherit': _('Fields on food that should be inherited by default.'),
+            'show_facet_count': _('Show recipe counts on search filters'), }
+
+        widgets = {
+            'food_inherit': MultiSelectWidget
+        }

cookbook/helper/AllAuthCustomAdapter.py

@@ -7,14 +7,20 @@ from django.contrib import messages
 from django.core.cache import caches
 from gettext import gettext as _
 
+from cookbook.models import InviteLink
+
 
 class AllAuthCustomAdapter(DefaultAccountAdapter):
 
     def is_open_for_signup(self, request):
         """
-        Whether to allow sign ups.
+        Whether to allow sign-ups.
         """
-        if request.resolver_match.view_name == 'account_signup' and not settings.ENABLE_SIGNUP:
+        signup_token = False
+        if 'signup_token' in request.session and InviteLink.objects.filter(valid_until__gte=datetime.datetime.today(), used_by=None, uuid=request.session['signup_token']).exists():
+            signup_token = True
+
+        if (request.resolver_match.view_name == 'account_signup' or request.resolver_match.view_name == 'socialaccount_signup') and not settings.ENABLE_SIGNUP and not signup_token:
             return False
         else:
             return super(AllAuthCustomAdapter, self).is_open_for_signup(request)
@@ -25,7 +31,10 @@ class AllAuthCustomAdapter(DefaultAccountAdapter):
             default = datetime.datetime.now()
             c = caches['default'].get_or_set(email, default, timeout=360)
             if c == default:
-                super(AllAuthCustomAdapter, self).send_mail(template_prefix, email, context)
+                try:
+                    super(AllAuthCustomAdapter, self).send_mail(template_prefix, email, context)
+                except Exception: # dont fail signup just because confirmation mail could not be send
+                    pass
             else:
                 messages.add_message(self.request, messages.ERROR, _('In order to prevent spam, the requested email was not send. Please wait a few minutes and try again.'))
         else:

cookbook/helper/HelperFunctions.py

@@ -0,0 +1,13 @@
+from django.db.models import Func
+
+
+class Round(Func):
+    function = 'ROUND'
+    template = '%(function)s(%(expressions)s, 0)'
+
+
+def str2bool(v):
+    if type(v) == bool or v is None:
+        return v
+    else:
+        return v.lower() in ("yes", "true", "1")

cookbook/helper/context_processors.py

@@ -10,4 +10,5 @@ def context_settings(request):
         'TERMS_URL': settings.TERMS_URL,
         'PRIVACY_URL': settings.PRIVACY_URL,
         'IMPRINT_URL': settings.IMPRINT_URL,
+        'SHOPPING_MIN_AUTOSYNC_INTERVAL': settings.SHOPPING_MIN_AUTOSYNC_INTERVAL,
     }

cookbook/helper/image_processing.py

@@ -5,7 +5,7 @@ from PIL import Image
 from io import BytesIO
 
 
-def rescale_image_jpeg(image_object, base_width=720):
+def rescale_image_jpeg(image_object, base_width=1020):
     img = Image.open(image_object)
     icc_profile = img.info.get('icc_profile')  # remember color profile to not mess up colors
     width_percent = (base_width / float(img.size[0]))
@@ -13,33 +13,37 @@ def rescale_image_jpeg(image_object, base_width=720):
 
     img = img.resize((base_width, height), Image.ANTIALIAS)
     img_bytes = BytesIO()
-    img.save(img_bytes, 'JPEG', quality=75, optimize=True, icc_profile=icc_profile)
+    img.save(img_bytes, 'JPEG', quality=90, optimize=True, icc_profile=icc_profile)
 
     return img_bytes
 
 
-def rescale_image_png(image_object, base_width=720):
-    basewidth = 720
-    wpercent = (basewidth / float(image_object.size[0]))
+def rescale_image_png(image_object, base_width=1020):
+    image_object = Image.open(image_object)
+    wpercent = (base_width / float(image_object.size[0]))
     hsize = int((float(image_object.size[1]) * float(wpercent)))
-    img = image_object.resize((basewidth, hsize), Image.ANTIALIAS)
+    img = image_object.resize((base_width, hsize), Image.ANTIALIAS)
 
     im_io = BytesIO()
-    img.save(im_io, 'PNG', quality=70)
-    return img
+    img.save(im_io, 'PNG', quality=90)
+    return im_io
 
 
 def get_filetype(name):
     try:
         return os.path.splitext(name)[1]
-    except:
+    except Exception:
         return '.jpeg'
 
 
-def handle_image(request, image_object, filetype='.jpeg'):
-    if sys.getsizeof(image_object) / 8 > 500:
-        if filetype == '.jpeg':
-            return rescale_image_jpeg(image_object), filetype
+# TODO this whole file needs proper documentation, refactoring, and testing
+# TODO also add env variable to define which images sizes should be compressed
+# filetype argument can not be optional, otherwise this function will treat all images as if they were a jpeg
+# Because it's no longer optional, no reason to return it
+def handle_image(request, image_object, filetype): 
+    if (image_object.size / 1000) > 500:  # if larger than 500 kb compress
+        if filetype == '.jpeg' or filetype == '.jpg':
+            return rescale_image_jpeg(image_object)
         if filetype == '.png':
-            return rescale_image_png(image_object), filetype
-    return image_object, filetype
+            return rescale_image_png(image_object)
+    return image_object

cookbook/helper/ingredient_parser.py

@@ -1,180 +1,312 @@
+import re
 import string
 import unicodedata
 
-from cookbook.models import Unit, Food
+from django.core.cache import caches
+
+from cookbook.models import Unit, Food, Automation, Ingredient
 
 
-def parse_fraction(x):
-    if len(x) == 1 and 'fraction' in unicodedata.decomposition(x):
-        frac_split = unicodedata.decomposition(x[-1:]).split()
-        return (float((frac_split[1]).replace('003', ''))
-                / float((frac_split[3]).replace('003', '')))
-    else:
-        frac_split = x.split('/')
-        if not len(frac_split) == 2:
-            raise ValueError
-        try:
-            return int(frac_split[0]) / int(frac_split[1])
-        except ZeroDivisionError:
-            raise ValueError
+class IngredientParser:
+    request = None
+    ignore_rules = False
+    food_aliases = {}
+    unit_aliases = {}
 
-
-def parse_amount(x):
-    amount = 0
-    unit = ''
-
-    did_check_frac = False
-    end = 0
-    while (
-            end < len(x)
-            and (
-                    x[end] in string.digits
-                    or (
-                            (x[end] == '.' or x[end] == ',' or x[end] == '/')
-                            and end + 1 < len(x)
-                            and x[end + 1] in string.digits
-                    )
-            )
-    ):
-        end += 1
-    if end > 0:
-        if "/" in x[:end]:
-            amount = parse_fraction(x[:end])
-        else:
-            amount = float(x[:end].replace(',', '.'))
-    else:
-        amount = parse_fraction(x[0])
-        end += 1
-        did_check_frac = True
-    if end < len(x):
-        if did_check_frac:
-            unit = x[end:]
-        else:
-            try:
-                amount += parse_fraction(x[end])
-                unit = x[end + 1:]
-            except ValueError:
-                unit = x[end:]
-    return amount, unit
-
-
-def parse_ingredient_with_comma(tokens):
-    ingredient = ''
-    note = ''
-    start = 0
-    # search for first occurrence of an argument ending in a comma
-    while start < len(tokens) and not tokens[start].endswith(','):
-        start += 1
-    if start == len(tokens):
-        # no token ending in a comma found -> use everything as ingredient
-        ingredient = ' '.join(tokens)
-    else:
-        ingredient = ' '.join(tokens[:start + 1])[:-1]
-        note = ' '.join(tokens[start + 1:])
-    return ingredient, note
-
-
-def parse_ingredient(tokens):
-    ingredient = ''
-    note = ''
-    if tokens[-1].endswith(')'):
-        # Check if the matching opening bracket is in the same token
-        if (not tokens[-1].startswith('(')) and ('(' in tokens[-1]):
-            return parse_ingredient_with_comma(tokens)
-        # last argument ends with closing bracket -> look for opening bracket
-        start = len(tokens) - 1
-        while not tokens[start].startswith('(') and not start == 0:
-            start -= 1
-        if start == 0:
-            # the whole list is wrapped in brackets -> assume it is an error (e.g. assumed first argument was the unit)  # noqa: E501
-            raise ValueError
-        elif start < 0:
-            # no opening bracket anywhere -> just ignore the last bracket
-            ingredient, note = parse_ingredient_with_comma(tokens)
-        else:
-            # opening bracket found -> split in ingredient and note, remove brackets from note  # noqa: E501
-            note = ' '.join(tokens[start:])[1:-1]
-            ingredient = ' '.join(tokens[:start])
-    else:
-        ingredient, note = parse_ingredient_with_comma(tokens)
-    return ingredient, note
-
-
-def parse(x):
-    # initialize default values
-    amount = 0
-    unit = ''
-    ingredient = ''
-    note = ''
-
-    tokens = x.split()
-    if len(tokens) == 1:
-        # there only is one argument, that must be the ingredient
-        ingredient = tokens[0]
-    else:
-        try:
-            # try to parse first argument as amount
-            amount, unit = parse_amount(tokens[0])
-            # only try to parse second argument as amount if there are at least
-            # three arguments if it already has a unit there can't be
-            # a fraction for the amount
-            if len(tokens) > 2:
-                try:
-                    if not unit == '':
-                        # a unit is already found, no need to try the second argument for a fraction  # noqa: E501
-                        # probably not the best method to do it, but I didn't want to make an if check and paste the exact same thing in the else as already is in the except  # noqa: E501
-                        raise ValueError
-                    # try to parse second argument as amount and add that, in case of '2 1/2' or '2 ½'  # noqa: E501
-                    amount += parse_fraction(tokens[1])
-                    # assume that units can't end with a comma
-                    if len(tokens) > 3 and not tokens[2].endswith(','):
-                        # try to use third argument as unit and everything else as ingredient, use everything as ingredient if it fails  # noqa: E501
-                        try:
-                            ingredient, note = parse_ingredient(tokens[3:])
-                            unit = tokens[2]
-                        except ValueError:
-                            ingredient, note = parse_ingredient(tokens[2:])
-                    else:
-                        ingredient, note = parse_ingredient(tokens[2:])
-                except ValueError:
-                    # assume that units can't end with a comma
-                    if not tokens[1].endswith(','):
-                        # try to use second argument as unit and everything else as ingredient, use everything as ingredient if it fails  # noqa: E501
-                        try:
-                            ingredient, note = parse_ingredient(tokens[2:])
-                            unit = tokens[1]
-                        except ValueError:
-                            ingredient, note = parse_ingredient(tokens[1:])
-                    else:
-                        ingredient, note = parse_ingredient(tokens[1:])
+    def __init__(self, request, cache_mode, ignore_automations=False):
+        """
+        Initialize ingredient parser
+        :param request: request context (to control caching, rule ownership, etc.)
+        :param cache_mode: defines if all rules should be loaded on initialization (good when parser is used many times) or if they should be retrieved every time (good when parser is not used many times in a row)
+        :param ignore_automations: ignore automation rules, allows to use ingredient parser without database access/request (request can be None)
+        """
+        self.request = request
+        self.ignore_rules = ignore_automations
+        if cache_mode:
+            FOOD_CACHE_KEY = f'automation_food_alias_{self.request.space.pk}'
+            if c := caches['default'].get(FOOD_CACHE_KEY, None):
+                self.food_aliases = c
+                caches['default'].touch(FOOD_CACHE_KEY, 30)
             else:
-                # only two arguments, first one is the amount
-                # which means this is the ingredient
-                ingredient = tokens[1]
-        except ValueError:
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.FOOD_ALIAS).only('param_1', 'param_2').all():
+                    self.food_aliases[a.param_1] = a.param_2
+                caches['default'].set(FOOD_CACHE_KEY, self.food_aliases, 30)
+
+            UNIT_CACHE_KEY = f'automation_unit_alias_{self.request.space.pk}'
+            if c := caches['default'].get(UNIT_CACHE_KEY, None):
+                self.unit_aliases = c
+                caches['default'].touch(UNIT_CACHE_KEY, 30)
+            else:
+                for a in Automation.objects.filter(space=self.request.space, disabled=False, type=Automation.UNIT_ALIAS).only('param_1', 'param_2').all():
+                    self.unit_aliases[a.param_1] = a.param_2
+                caches['default'].set(UNIT_CACHE_KEY, self.unit_aliases, 30)
+        else:
+            self.food_aliases = {}
+            self.unit_aliases = {}
+
+    def apply_food_automation(self, food):
+        """
+        Apply food alias automations to passed food
+        :param food: unit as string
+        :return: food as string (possibly changed by automation)
+        """
+        if self.ignore_rules:
+            return food
+        else:
+            if self.food_aliases:
+                try:
+                    return self.food_aliases[food]
+                except KeyError:
+                    return food
+            else:
+                if automation := Automation.objects.filter(space=self.request.space, type=Automation.FOOD_ALIAS, param_1=food, disabled=False).first():
+                    return automation.param_2
+        return food
+
+    def apply_unit_automation(self, unit):
+        """
+        Apply unit alias automations to passed unit
+        :param unit: unit as string
+        :return: unit as string (possibly changed by automation)
+        """
+        if self.ignore_rules:
+            return unit
+        else:
+            if self.unit_aliases:
+                try:
+                    return self.unit_aliases[unit]
+                except KeyError:
+                    return unit
+            else:
+                if automation := Automation.objects.filter(space=self.request.space, type=Automation.UNIT_ALIAS, param_1=unit, disabled=False).first():
+                    return automation.param_2
+        return unit
+
+    def get_unit(self, unit):
+        """
+        Get or create a unit for given space respecting possible automations
+        :param unit: string unit
+        :return: None if unit passed is invalid, Unit object otherwise
+        """
+        if not unit:
+            return None
+        if len(unit) > 0:
+            u, created = Unit.objects.get_or_create(name=self.apply_unit_automation(unit), space=self.request.space)
+            return u
+        return None
+
+    def get_food(self, food):
+        """
+        Get or create a food for given space respecting possible automations
+        :param food: string food
+        :return: None if food passed is invalid, Food object otherwise
+        """
+        if not food:
+            return None
+        if len(food) > 0:
+            f, created = Food.objects.get_or_create(name=self.apply_food_automation(food), space=self.request.space)
+            return f
+        return None
+
+    def parse_fraction(self, x):
+        if len(x) == 1 and 'fraction' in unicodedata.decomposition(x):
+            frac_split = unicodedata.decomposition(x[-1:]).split()
+            return (float((frac_split[1]).replace('003', ''))
+                    / float((frac_split[3]).replace('003', '')))
+        else:
+            frac_split = x.split('/')
+            if not len(frac_split) == 2:
+                raise ValueError
             try:
-                # can't parse first argument as amount
-                # -> no unit -> parse everything as ingredient
-                ingredient, note = parse_ingredient(tokens)
+                return int(frac_split[0]) / int(frac_split[1])
+            except ZeroDivisionError:
+                raise ValueError
+
+    def parse_amount(self, x):
+        amount = 0
+        unit = None
+        note = ''
+
+        did_check_frac = False
+        end = 0
+        while (end < len(x) and (x[end] in string.digits
+                                 or (
+                                         (x[end] == '.' or x[end] == ',' or x[end] == '/')
+                                         and end + 1 < len(x)
+                                         and x[end + 1] in string.digits
+                                 ))):
+            end += 1
+        if end > 0:
+            if "/" in x[:end]:
+                amount = self.parse_fraction(x[:end])
+            else:
+                amount = float(x[:end].replace(',', '.'))
+        else:
+            amount = self.parse_fraction(x[0])
+            end += 1
+            did_check_frac = True
+        if end < len(x):
+            if did_check_frac:
+                unit = x[end:]
+            else:
+                try:
+                    amount += self.parse_fraction(x[end])
+                    unit = x[end + 1:]
+                except ValueError:
+                    unit = x[end:]
+
+        if unit is not None and unit.strip() == '':
+            unit = None
+
+        if unit is not None and (unit.startswith('(') or unit.startswith('-')):  # i dont know any unit that starts with ( or - so its likely an alternative like 1L (500ml) Water or 2-3
+            unit = None
+            note = x
+        return amount, unit, note
+
+    def parse_food_with_comma(self, tokens):
+        food = ''
+        note = ''
+        start = 0
+        # search for first occurrence of an argument ending in a comma
+        while start < len(tokens) and not tokens[start].endswith(','):
+            start += 1
+        if start == len(tokens):
+            # no token ending in a comma found -> use everything as food
+            food = ' '.join(tokens)
+        else:
+            food = ' '.join(tokens[:start + 1])[:-1]
+            note = ' '.join(tokens[start + 1:])
+        return food, note
+
+    def parse_food(self, tokens):
+        food = ''
+        note = ''
+        if tokens[-1].endswith(')'):
+            # Check if the matching opening bracket is in the same token
+            if (not tokens[-1].startswith('(')) and ('(' in tokens[-1]):
+                return self.parse_food_with_comma(tokens)
+            # last argument ends with closing bracket -> look for opening bracket
+            start = len(tokens) - 1
+            while not tokens[start].startswith('(') and not start == 0:
+                start -= 1
+            if start == 0:
+                # the whole list is wrapped in brackets -> assume it is an error (e.g. assumed first argument was the unit)  # noqa: E501
+                raise ValueError
+            elif start < 0:
+                # no opening bracket anywhere -> just ignore the last bracket
+                food, note = self.parse_food_with_comma(tokens)
+            else:
+                # opening bracket found -> split in food and note, remove brackets from note  # noqa: E501
+                note = ' '.join(tokens[start:])[1:-1]
+                food = ' '.join(tokens[:start])
+        else:
+            food, note = self.parse_food_with_comma(tokens)
+        return food, note
+
+    def parse(self, ingredient):
+        """
+        Main parsing function, takes an ingredient string (e.g. '1 l Water') and extracts amount, unit, food, ...
+        :param ingredient: string ingredient
+        :return: amount, unit (can be None), food, note (can be empty)
+        """
+        # initialize default values
+        amount = 0
+        unit = None
+        food = ''
+        note = ''
+        unit_note = ''
+
+        if len(ingredient) == 0:
+            raise ValueError('string to parse cannot be empty')
+
+        # some people/languages put amount and unit at the end of the ingredient string
+        # if something like this is detected move it to the beginning so the parser can handle it
+        if len(ingredient) < 1000 and re.search(r'^([^\W\d_])+(.)*[1-9](\d)*\s*([^\W\d_])+', ingredient):
+            match = re.search(r'[1-9](\d)*\s*([^\W\d_])+', ingredient)
+            print(f'reording from {ingredient} to {ingredient[match.start():match.end()] + " " + ingredient.replace(ingredient[match.start():match.end()], "")}')
+            ingredient = ingredient[match.start():match.end()] + ' ' + ingredient.replace(ingredient[match.start():match.end()], '')
+
+        # if the string contains parenthesis early on remove it and place it at the end
+        # because its likely some kind of note
+        if re.match('(.){1,6}\s\((.[^\(\)])+\)\s', ingredient):
+            match = re.search('\((.[^\(])+\)', ingredient)
+            ingredient = ingredient[:match.start()] + ingredient[match.end():] + ' ' + ingredient[match.start():match.end()]
+
+        # leading spaces before commas result in extra tokens, clean them out
+        ingredient = ingredient.replace(' ,', ',')
+
+        tokens = ingredient.split()  # split at each space into tokens
+        if len(tokens) == 1:
+            # there only is one argument, that must be the food
+            food = tokens[0]
+        else:
+            try:
+                # try to parse first argument as amount
+                amount, unit, unit_note = self.parse_amount(tokens[0])
+                # only try to parse second argument as amount if there are at least
+                # three arguments if it already has a unit there can't be
+                # a fraction for the amount
+                if len(tokens) > 2:
+                    try:
+                        if unit is not None:
+                            # a unit is already found, no need to try the second argument for a fraction
+                            # probably not the best method to do it, but I didn't want to make an if check and paste the exact same thing in the else as already is in the except
+                            raise ValueError
+                        # try to parse second argument as amount and add that, in case of '2 1/2' or '2 ½'
+                        amount += self.parse_fraction(tokens[1])
+                        # assume that units can't end with a comma
+                        if len(tokens) > 3 and not tokens[2].endswith(','):
+                            # try to use third argument as unit and everything else as food, use everything as food if it fails
+                            try:
+                                food, note = self.parse_food(tokens[3:])
+                                unit = tokens[2]
+                            except ValueError:
+                                food, note = self.parse_food(tokens[2:])
+                        else:
+                            food, note = self.parse_food(tokens[2:])
+                    except ValueError:
+                        # assume that units can't end with a comma
+                        if not tokens[1].endswith(','):
+                            # try to use second argument as unit and everything else as food, use everything as food if it fails
+                            try:
+                                food, note = self.parse_food(tokens[2:])
+                                if unit is None:
+                                    unit = tokens[1]
+                                else:
+                                    note = tokens[1]
+                            except ValueError:
+                                food, note = self.parse_food(tokens[1:])
+                        else:
+                            food, note = self.parse_food(tokens[1:])
+                else:
+                    # only two arguments, first one is the amount
+                    # which means this is the food
+                    food = tokens[1]
             except ValueError:
-                ingredient = ' '.join(tokens[1:])
-    return amount, unit.strip(), ingredient.strip(), note.strip()
+                try:
+                    # can't parse first argument as amount
+                    # -> no unit -> parse everything as food
+                    food, note = self.parse_food(tokens)
+                except ValueError:
+                    food = ' '.join(tokens[1:])
 
+        if unit_note not in note:
+            note += ' ' + unit_note
 
-# small utility functions to prevent emtpy unit/food creation
-def get_unit(unit, space):
-    if not unit:
-        return None
-    if len(unit) > 0:
-        u, created = Unit.objects.get_or_create(name=unit, space=space)
-        return u
-    return None
+        if unit:
+            unit = self.apply_unit_automation(unit.strip())
 
+        food = self.apply_food_automation(food.strip())
+        if len(food) > Food._meta.get_field('name').max_length:  # test if food name is to long
+            # try splitting it at a space and taking only the first arg
+            if len(food.split()) > 1 and len(food.split()[0]) < Food._meta.get_field('name').max_length:
+                note = ' '.join(food.split()[1:]) + ' ' + note
+                food = food.split()[0]
+            else:
+                note = food + ' ' + note
+                food = food[:Food._meta.get_field('name').max_length]
 
-def get_food(food, space):
-    if not food:
-        return None
-    if len(food) > 0:
-        f, created = Food.objects.get_or_create(name=food, space=space)
-        return f
-    return None
+        if len(food.strip()) == 0:
+            raise ValueError(f'Error parsing string {ingredient}, food cannot be empty')
+
+        return amount, unit, food, note[:Ingredient._meta.get_field('note').max_length].strip()

cookbook/helper/mdx_urlize.py

@@ -73,9 +73,9 @@ class UrlizePattern(markdown.inlinepatterns.Pattern):
 class UrlizeExtension(markdown.Extension):
     """ Urlize Extension for Python-Markdown. """
 
-    def extendMarkdown(self, md, md_globals):
+    def extendMarkdown(self, md):
         """ Replace autolink with UrlizePattern """
-        md.inlinePatterns['autolink'] = UrlizePattern(URLIZE_RE, md)
+        md.inlinePatterns.register(UrlizePattern(URLIZE_RE, md), 'autolink', 120)
 
 
 def makeExtension(*args, **kwargs):

cookbook/helper/permission_helper.py

@@ -1,21 +1,20 @@
-"""
-Source: https://djangosnippets.org/snippets/1703/
-"""
-from django.conf import settings
-from django.core.cache import caches
-from django.views.generic.detail import SingleObjectTemplateResponseMixin
-from django.views.generic.edit import ModelFormMixin
+import inspect
 
-from cookbook.models import ShareLink
+from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.decorators import user_passes_test
-from django.core.exceptions import ValidationError
+from django.core.cache import cache
+from django.core.exceptions import ValidationError, ObjectDoesNotExist
 from django.http import HttpResponseRedirect
 from django.urls import reverse, reverse_lazy
 from django.utils.translation import gettext as _
+from oauth2_provider.contrib.rest_framework import TokenHasScope, TokenHasReadWriteScope
+from oauth2_provider.models import AccessToken
 from rest_framework import permissions
 from rest_framework.permissions import SAFE_METHODS
 
+from cookbook.models import ShareLink, Recipe, UserSpace
+
 
 def get_allowed_groups(groups_required):
     """
@@ -32,11 +31,12 @@ def get_allowed_groups(groups_required):
     return groups_allowed
 
 
-def has_group_permission(user, groups):
+def has_group_permission(user, groups, no_cache=False):
     """
     Tests if a given user is member of a certain group (or any higher group)
     Superusers always bypass permission checks.
-    Unauthenticated users cant be member of any group thus always return false.
+    Unauthenticated users can't be member of any group thus always return false.
+    :param no_cache: (optional) do not return cached results, always check agains DB
     :param user: django auth user object
     :param groups: list or tuple of groups the user should be checked for
     :return: True if user is in allowed groups, false otherwise
@@ -44,10 +44,24 @@ def has_group_permission(user, groups):
     if not user.is_authenticated:
         return False
     groups_allowed = get_allowed_groups(groups)
+
+    CACHE_KEY = hash((inspect.stack()[0][3], (user.pk, user.username, user.email), groups_allowed))
+    if not no_cache:
+        cached_result = cache.get(CACHE_KEY, default=None)
+        if cached_result is not None:
+            return cached_result
+
+    result = False
+    print('running check', user, groups_allowed)
     if user.is_authenticated:
-        if bool(user.groups.filter(name__in=groups_allowed)):
-            return True
-    return False
+        if user_space := user.userspace_set.filter(active=True):
+            if len(user_space) != 1:
+                result = False  # do not allow any group permission if more than one space is active, needs to be changed when simultaneous multi-space-tenancy is added
+            elif bool(user_space.first().groups.filter(name__in=groups_allowed)):
+                result = True
+
+    cache.set(CACHE_KEY, result, timeout=10)
+    return result
 
 
 def is_object_owner(user, obj):
@@ -55,7 +69,6 @@ def is_object_owner(user, obj):
     Tests if a given user is the owner of a given object
     test performed by checking user against the objects user
     and create_by field (if exists)
-    superusers bypass all checks, unauthenticated users cannot own anything
     :param user django auth user object
     :param obj any object that should be tested
     :return: true if user is owner of object, false otherwise
@@ -64,7 +77,22 @@ def is_object_owner(user, obj):
         return False
     try:
         return obj.get_owner() == user
-    except:
+    except Exception:
+        return False
+
+
+def is_space_owner(user, obj):
+    """
+    Tests if a given user is the owner the space of a given object
+    :param user django auth user object
+    :param obj any object that should be tested
+    :return: true if user is owner of the objects space, false otherwise
+    """
+    if not user.is_authenticated:
+        return False
+    try:
+        return obj.get_space().get_owner() == user
+    except Exception:
         return False
 
 
@@ -72,7 +100,6 @@ def is_object_shared(user, obj):
     """
     Tests if a given user is shared for a given object
     test performed by checking user against the objects shared table
-    superusers bypass all checks, unauthenticated users cannot own anything
     :param user django auth user object
     :param obj any object that should be tested
     :return: true if user is shared for object, false otherwise
@@ -93,7 +120,7 @@ def share_link_valid(recipe, share):
     """
     try:
         CACHE_KEY = f'recipe_share_{recipe.pk}_{share}'
-        if c := caches['default'].get(CACHE_KEY, False):
+        if c := cache.get(CACHE_KEY, False):
             return c
 
         if link := ShareLink.objects.filter(recipe=recipe, uuid=share, abuse_blocked=False).first():
@@ -101,7 +128,7 @@ def share_link_valid(recipe, share):
                 return False
             link.request_count += 1
             link.save()
-            caches['default'].set(CACHE_KEY, True, timeout=3)
+            cache.set(CACHE_KEY, True, timeout=3)
             return True
         return False
     except ValidationError:
@@ -168,7 +195,7 @@ class OwnerRequiredMixin(object):
 
         try:
             obj = self.get_object()
-            if obj.get_space() != request.space:
+            if not request.user.userspace.filter(space=obj.get_space()).exists():
                 messages.add_message(request, messages.ERROR,
                                      _('You do not have the required permissions to view this page!'))
                 return HttpResponseRedirect(reverse_lazy('index'))
@@ -186,7 +213,7 @@ class CustomIsOwner(permissions.BasePermission):
     verifies user has ownership over object
     (either user or created_by or user is request user)
     """
-    message = _('You cannot interact with this object as it is not owned by you!')  # noqa: E501
+    message = _('You cannot interact with this object as it is not owned by you!')
 
     def has_permission(self, request, view):
         return request.user.is_authenticated
@@ -195,6 +222,28 @@ class CustomIsOwner(permissions.BasePermission):
         return is_object_owner(request.user, obj)
 
 
+class CustomIsOwnerReadOnly(CustomIsOwner):
+    def has_permission(self, request, view):
+        return super().has_permission(request, view) and request.method in SAFE_METHODS
+
+    def has_object_permission(self, request, view, obj):
+        return super().has_object_permission(request, view) and request.method in SAFE_METHODS
+
+
+class CustomIsSpaceOwner(permissions.BasePermission):
+    """
+    Custom permission class for django rest framework views
+    verifies if the user is the owner of the space the object belongs to
+    """
+    message = _('You cannot interact with this object as it is not owned by you!')
+
+    def has_permission(self, request, view):
+        return request.user.is_authenticated and request.space.created_by == request.user
+
+    def has_object_permission(self, request, view, obj):
+        return is_space_owner(request.user, obj)
+
+
 # TODO function duplicate/too similar name
 class CustomIsShared(permissions.BasePermission):
     """
@@ -207,6 +256,9 @@ class CustomIsShared(permissions.BasePermission):
         return request.user.is_authenticated
 
     def has_object_permission(self, request, view, obj):
+        # # temporary hack to make old shopping list work with new shopping list
+        # if obj.__class__.__name__ in ['ShoppingList', 'ShoppingListEntry']:
+        #     return is_object_shared(request.user, obj) or obj.created_by in list(request.user.get_shopping_share())
         return is_object_shared(request.user, obj)
 
 
@@ -261,3 +313,125 @@ class CustomIsShare(permissions.BasePermission):
         if share:
             return share_link_valid(obj, share)
         return False
+
+
+class CustomRecipePermission(permissions.BasePermission):
+    """
+    Custom permission class for recipe api endpoint
+    """
+    message = _('You do not have the required permissions to view this page!')
+
+    def has_permission(self, request, view):  # user is either at least a guest or a share link is given and the request is safe
+        share = request.query_params.get('share', None)
+        return has_group_permission(request.user, ['guest']) or (share and request.method in SAFE_METHODS and 'pk' in view.kwargs)
+
+    def has_object_permission(self, request, view, obj):
+        share = request.query_params.get('share', None)
+        if share:
+            return share_link_valid(obj, share)
+        else:
+            if obj.private:
+                return ((obj.created_by == request.user) or (request.user in obj.shared.all())) and obj.space == request.space
+            else:
+                return has_group_permission(request.user, ['guest']) and obj.space == request.space
+
+
+class CustomUserPermission(permissions.BasePermission):
+    """
+    Custom permission class for user api endpoint
+    """
+    message = _('You do not have the required permissions to view this page!')
+
+    def has_permission(self, request, view):  # a space filtered user list is visible for everyone
+        return has_group_permission(request.user, ['guest'])
+
+    def has_object_permission(self, request, view, obj):  # object write permissions are only available for user
+        if request.method in SAFE_METHODS and 'pk' in view.kwargs and has_group_permission(request.user, ['guest']) and request.space in obj.userspace_set.all():
+            return True
+        elif request.user == obj:
+            return True
+        else:
+            return False
+
+
+class CustomTokenHasScope(TokenHasScope):
+    """
+    Custom implementation of Django OAuth Toolkit TokenHasScope class
+    Only difference: if any other authentication method except OAuth2Authentication is used the scope check is ignored
+    IMPORTANT: do not use this class without any other permission class as it will not check anything besides token scopes
+    """
+
+    def has_permission(self, request, view):
+        if type(request.auth) == AccessToken:
+            return super().has_permission(request, view)
+        else:
+            return request.user.is_authenticated
+
+
+class CustomTokenHasReadWriteScope(TokenHasReadWriteScope):
+    """
+    Custom implementation of Django OAuth Toolkit TokenHasReadWriteScope class
+    Only difference: if any other authentication method except OAuth2Authentication is used the scope check is ignored
+    IMPORTANT: do not use this class without any other permission class as it will not check anything besides token scopes
+    """
+
+    def has_permission(self, request, view):
+        if type(request.auth) == AccessToken:
+            return super().has_permission(request, view)
+        else:
+            return True
+
+
+def above_space_limit(space):  # TODO add file storage limit
+    """
+    Test if the space has reached any limit (e.g. max recipes, users, ..)
+    :param space: Space to test for limits
+    :return: Tuple (True if above or equal any limit else false, message)
+    """
+    r_limit, r_msg = above_space_recipe_limit(space)
+    u_limit, u_msg = above_space_user_limit(space)
+    return r_limit or u_limit, (r_msg + ' ' + u_msg).strip()
+
+
+def above_space_recipe_limit(space):
+    """
+    Test if a space has reached its recipe limit
+    :param space: Space to test for limits
+    :return: Tuple (True if above or equal limit else false, message)
+    """
+    limit = space.max_recipes != 0 and Recipe.objects.filter(space=space).count() >= space.max_recipes
+    if limit:
+        return True, _('You have reached the maximum number of recipes for your space.')
+    return False, ''
+
+
+def above_space_user_limit(space):
+    """
+    Test if a space has reached its user limit
+    :param space: Space to test for limits
+    :return: Tuple (True if above or equal limit else false, message)
+    """
+    limit = space.max_users != 0 and UserSpace.objects.filter(space=space).count() > space.max_users
+    if limit:
+        return True, _('You have more users than allowed in your space.')
+    return False, ''
+
+
+def switch_user_active_space(user, space):
+    """
+    Switch the currently active space of a user by setting all spaces to inactive and activating the one passed
+    :param user: user to change active space for
+    :param space: space to activate user for
+    :return user space object or none if not found/no permission
+    """
+    try:
+        us = UserSpace.objects.get(space=space, user=user)
+        if not us.active:
+            UserSpace.objects.filter(user=user).update(active=False)
+            us.active = True
+            us.save()
+            return us
+        else:
+            return us
+    except ObjectDoesNotExist:
+        return None

cookbook/helper/recipe_html_import.py

@@ -1,193 +1,191 @@
-import json
-import re
+# import json
+# import re
+# from json import JSONDecodeError
+# from urllib.parse import unquote
 
-from bs4 import BeautifulSoup
-from bs4.element import Tag
-from cookbook.helper import recipe_url_import as helper
-from cookbook.helper.scrapers.scrapers import text_scraper
-from json import JSONDecodeError
-from recipe_scrapers._utils import get_host_name, normalize_string
-from urllib.parse import unquote
+# from bs4 import BeautifulSoup
+# from bs4.element import Tag
+# from recipe_scrapers import scrape_html, scrape_me
+# from recipe_scrapers._exceptions import NoSchemaFoundInWildMode
+# from recipe_scrapers._utils import get_host_name, normalize_string
+
+# from cookbook.helper import recipe_url_import as helper
+# from cookbook.helper.scrapers.scrapers import text_scraper
 
 
-def get_recipe_from_source(text, url, space):
-    def build_node(k, v):
-        if isinstance(v, dict):
-            node = {
-                'name': k,
-                'value': k,
-                'children': get_children_dict(v)
-            }
-        elif isinstance(v, list):
-            node = {
-                'name': k,
-                'value': k,
-                'children': get_children_list(v)
-            }
-        else:
-            node = {
-                'name': k + ": " + normalize_string(str(v)),
-                'value': normalize_string(str(v))
-            }
-        return node
+# def get_recipe_from_source(text, url, request):
+#     def build_node(k, v):
+#         if isinstance(v, dict):
+#             node = {
+#                 'name': k,
+#                 'value': k,
+#                 'children': get_children_dict(v)
+#             }
+#         elif isinstance(v, list):
+#             node = {
+#                 'name': k,
+#                 'value': k,
+#                 'children': get_children_list(v)
+#             }
+#         else:
+#             node = {
+#                 'name': k + ": " + normalize_string(str(v)),
+#                 'value': normalize_string(str(v))
+#             }
+#         return node
 
-    def get_children_dict(children):
-        kid_list = []
-        for k, v in children.items():
-            kid_list.append(build_node(k, v))
-        return kid_list
+#     def get_children_dict(children):
+#         kid_list = []
+#         for k, v in children.items():
+#             kid_list.append(build_node(k, v))
+#         return kid_list
 
-    def get_children_list(children):
-        kid_list = []
-        for kid in children:
-            if type(kid) == list:
-                node = {
-                    'name': "unknown list",
-                    'value': "unknown list",
-                    'children': get_children_list(kid)
-                }
-                kid_list.append(node)
-            elif type(kid) == dict:
-                for k, v in kid.items():
-                    kid_list.append(build_node(k, v))
-            else:
-                kid_list.append({
-                    'name': normalize_string(str(kid)),
-                    'value': normalize_string(str(kid))
-                })
-        return kid_list
+#     def get_children_list(children):
+#         kid_list = []
+#         for kid in children:
+#             if type(kid) == list:
+#                 node = {
+#                     'name': "unknown list",
+#                     'value': "unknown list",
+#                     'children': get_children_list(kid)
+#                 }
+#                 kid_list.append(node)
+#             elif type(kid) == dict:
+#                 for k, v in kid.items():
+#                     kid_list.append(build_node(k, v))
+#             else:
+#                 kid_list.append({
+#                     'name': normalize_string(str(kid)),
+#                     'value': normalize_string(str(kid))
+#                 })
+#         return kid_list
 
-    recipe_json = {
-                'name': '',
-                'url': '',
-                'description': '',
-                'image': '',
-                'keywords': [],
-                'recipeIngredient': [],
-                'recipeInstructions': '',
-                'servings': '',
-                'prepTime': '',
-                'cookTime': ''
-    }
-    recipe_tree = []
-    parse_list = []
-    html_data = []
-    images = []
-    text = unquote(text)
+#     recipe_tree = []
+#     parse_list = []
+#     soup = BeautifulSoup(text, "html.parser")
+#     html_data = get_from_html(soup)
+#     images = get_images_from_source(soup, url)
+#     text = unquote(text)
+#     scrape = None
 
-    try:
-        parse_list.append(remove_graph(json.loads(text)))
-        if not url and 'url' in parse_list[0]:
-            url = parse_list[0]['url']
-        scrape = text_scraper("<script type='application/ld+json'>" + text + "</script>", url=url)
+#     if url and not text:
+#         try:
+#             scrape = scrape_me(url_path=url, wild_mode=True)
+#         except(NoSchemaFoundInWildMode):
+#             pass
 
-    except JSONDecodeError:
-        soup = BeautifulSoup(text, "html.parser")
-        html_data = get_from_html(soup)
-        images += get_images_from_source(soup, url)
-        for el in soup.find_all('script', type='application/ld+json'):
-            el = remove_graph(el)
-            if not url and 'url' in el:
-                url = el['url']
-            if type(el) == list:
-                for le in el:
-                    parse_list.append(le)
-            elif type(el) == dict:
-                parse_list.append(el)
-        for el in soup.find_all(type='application/json'):
-            el = remove_graph(el)
-            if type(el) == list:
-                for le in el:
-                    parse_list.append(le)
-            elif type(el) == dict:
-                parse_list.append(el)
-        scrape = text_scraper(text, url=url)
+#     if not scrape:
+#         try:
+#             parse_list.append(remove_graph(json.loads(text)))
+#             if not url and 'url' in parse_list[0]:
+#                 url = parse_list[0]['url']
+#             scrape = text_scraper("<script type='application/ld+json'>" + text + "</script>", url=url)
 
-    recipe_json = helper.get_from_scraper(scrape, space)
+#         except JSONDecodeError:
+#             for el in soup.find_all('script', type='application/ld+json'):
+#                 el = remove_graph(el)
+#                 if not url and 'url' in el:
+#                     url = el['url']
+#                 if type(el) == list:
+#                     for le in el:
+#                         parse_list.append(le)
+#                 elif type(el) == dict:
+#                     parse_list.append(el)
+#             for el in soup.find_all(type='application/json'):
+#                 el = remove_graph(el)
+#                 if type(el) == list:
+#                     for le in el:
+#                         parse_list.append(le)
+#                 elif type(el) == dict:
+#                     parse_list.append(el)
+#             scrape = text_scraper(text, url=url)
 
-    for el in parse_list:
-        temp_tree = []
-        if isinstance(el, Tag):
-            try:
-                el = json.loads(el.string)
-            except TypeError:
-                continue
+#     recipe_json = helper.get_from_scraper(scrape, request)
 
-        for k, v in el.items():
-            if isinstance(v, dict):
-                node = {
-                    'name': k,
-                    'value': k,
-                    'children': get_children_dict(v)
-                }
-            elif isinstance(v, list):
-                node = {
-                    'name': k,
-                    'value': k,
-                    'children': get_children_list(v)
-                }
-            else:
-                node = {
-                    'name': k + ": " + normalize_string(str(v)),
-                    'value': normalize_string(str(v))
-                }
-            temp_tree.append(node)
+#     # TODO: DEPRECATE recipe_tree & html_data.  first validate it isn't used anywhere
+#     for el in parse_list:
+#         temp_tree = []
+#         if isinstance(el, Tag):
+#             try:
+#                 el = json.loads(el.string)
+#             except TypeError:
+#                 continue
 
-        if '@type' in el and el['@type'] == 'Recipe':
-            recipe_tree += [{'name': 'ld+json', 'children': temp_tree}]
-        else:
-            recipe_tree += [{'name': 'json', 'children': temp_tree}]
+#         for k, v in el.items():
+#             if isinstance(v, dict):
+#                 node = {
+#                     'name': k,
+#                     'value': k,
+#                     'children': get_children_dict(v)
+#                 }
+#             elif isinstance(v, list):
+#                 node = {
+#                     'name': k,
+#                     'value': k,
+#                     'children': get_children_list(v)
+#                 }
+#             else:
+#                 node = {
+#                     'name': k + ": " + normalize_string(str(v)),
+#                     'value': normalize_string(str(v))
+#                 }
+#             temp_tree.append(node)
 
-    return recipe_json, recipe_tree, html_data, images
+#         if '@type' in el and el['@type'] == 'Recipe':
+#             recipe_tree += [{'name': 'ld+json', 'children': temp_tree}]
+#         else:
+#             recipe_tree += [{'name': 'json', 'children': temp_tree}]
+
+#     return recipe_json, recipe_tree, html_data, images
 
 
-def get_from_html(soup):
-    INVISIBLE_ELEMS = ('style', 'script', 'head', 'title')
-    html = []
-    for s in soup.strings:
-        if ((s.parent.name not in INVISIBLE_ELEMS) and (len(s.strip()) > 0)):
-            html.append(s)
-    return html
+# def get_from_html(soup):
+#     INVISIBLE_ELEMS = ('style', 'script', 'head', 'title')
+#     html = []
+#     for s in soup.strings:
+#         if ((s.parent.name not in INVISIBLE_ELEMS) and (len(s.strip()) > 0)):
+#             html.append(s)
+#     return html
 
 
-def get_images_from_source(soup, url):
-    sources = ['src', 'srcset', 'data-src']
-    images = []
-    img_tags = soup.find_all('img')
-    if url:
-        site = get_host_name(url)
-        prot = url.split(':')[0]
+# def get_images_from_source(soup, url):
+#     sources = ['src', 'srcset', 'data-src']
+#     images = []
+#     img_tags = soup.find_all('img')
+#     if url:
+#         site = get_host_name(url)
+#         prot = url.split(':')[0]
 
-    urls = []
-    for img in img_tags:
-        for src in sources:
-            try:
-                urls.append(img[src])
-            except KeyError:
-                pass
+#     urls = []
+#     for img in img_tags:
+#         for src in sources:
+#             try:
+#                 urls.append(img[src])
+#             except KeyError:
+#                 pass
 
-    for u in urls:
-        u = u.split('?')[0]
-        filename = re.search(r'/([\w_-]+[.](jpg|jpeg|gif|png))$', u)
-        if filename:
-            if (('http' not in u) and (url)):
-                # sometimes an image source can be relative
-                # if it is provide the base url
-                u = '{}://{}{}'.format(prot, site, u)
-            if 'http' in u:
-                images.append(u)
-    return images
+#     for u in urls:
+#         u = u.split('?')[0]
+#         filename = re.search(r'/([\w_-]+[.](jpg|jpeg|gif|png))$', u)
+#         if filename:
+#             if (('http' not in u) and (url)):
+#                 # sometimes an image source can be relative
+#                 # if it is provide the base url
+#                 u = '{}://{}{}'.format(prot, site, u)
+#             if 'http' in u:
+#                 images.append(u)
+#     return images
 
 
-def remove_graph(el):
-    # recipes type might be wrapped in @graph type
-    if isinstance(el, Tag):
-        try:
-            el = json.loads(el.string)
-            if '@graph' in el:
-                for x in el['@graph']:
-                    if '@type' in x and x['@type'] == 'Recipe':
-                        el = x
-        except TypeError:
-            pass
-    return el
+# def remove_graph(el):
+#     # recipes type might be wrapped in @graph type
+#     if isinstance(el, Tag):
+#         try:
+#             el = json.loads(el.string)
+#             if '@graph' in el:
+#                 for x in el['@graph']:
+#                     if '@type' in x and x['@type'] == 'Recipe':
+#                         el = x
+#         except (TypeError, JSONDecodeError):
+#             pass
+#     return el

cookbook/helper/recipe_search.py

@@ -1,72 +1,765 @@
-from datetime import datetime, timedelta
-from functools import reduce
+import json
+from collections import Counter
+from datetime import date, timedelta
 
-from django.contrib.postgres.search import TrigramSimilarity
-from django.db.models import Q, Case, When, Value
-from django.forms import IntegerField
+from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector, TrigramSimilarity
+from django.core.cache import cache
+from django.core.cache import caches
+from django.db.models import (Avg, Case, Count, Exists, F, Func, Max, OuterRef, Q, Subquery, Value, When, FilteredRelation)
+from django.db.models.functions import Coalesce, Lower, Substr
+from django.utils import timezone, translation
+from django.utils.translation import gettext as _
 
-from cookbook.models import ViewLog
+from cookbook.helper.HelperFunctions import Round, str2bool
+from cookbook.managers import DICTIONARY
+from cookbook.models import (CookLog, CustomFilter, Food, Keyword, Recipe, SearchFields,
+                             SearchPreference, ViewLog)
 from recipes import settings
 
 
-def search_recipes(request, queryset, params):
-    search_string = params.get('query', '')
-    search_keywords = params.getlist('keywords', [])
-    search_foods = params.getlist('foods', [])
-    search_books = params.getlist('books', [])
+# TODO create extensive tests to make sure ORs ANDs and various filters, sorting, etc work as expected
+# TODO consider creating a simpleListRecipe API that only includes minimum of recipe info and minimal filtering
+class RecipeSearch():
+    _postgres = settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2', 'django.db.backends.postgresql']
 
-    search_keywords_or = params.get('keywords_or', True)
-    search_foods_or = params.get('foods_or', True)
-    search_books_or = params.get('books_or', True)
-
-    search_internal = params.get('internal', None)
-    search_random = params.get('random', False)
-    search_last_viewed = int(params.get('last_viewed', 0))
-
-    if search_last_viewed > 0:
-        last_viewed_recipes = ViewLog.objects.filter(created_by=request.user, space=request.space,
-                                                     created_at__gte=datetime.now() - timedelta(days=14)).order_by('pk').values_list('recipe__pk', flat=True).distinct()
-
-        return queryset.filter(pk__in=last_viewed_recipes[len(last_viewed_recipes) - min(len(last_viewed_recipes), search_last_viewed):])
-
-    queryset = queryset.annotate(
-        new_recipe=Case(When(created_at__gte=(datetime.now() - timedelta(days=7)), then=Value(100)),
-                        default=Value(0), )).order_by('-new_recipe', 'name')
-
-    if settings.DATABASES['default']['ENGINE'] in ['django.db.backends.postgresql_psycopg2',
-                                                   'django.db.backends.postgresql']:
-        queryset = queryset.annotate(similarity=TrigramSimilarity('name', search_string), ).filter(
-            Q(similarity__gt=0.1) | Q(name__unaccent__icontains=search_string)).order_by('-similarity')
-    else:
-        queryset = queryset.filter(name__icontains=search_string)
-
-    if len(search_keywords) > 0:
-        if search_keywords_or == 'true':
-            queryset = queryset.filter(keywords__id__in=search_keywords)
+    def __init__(self, request, **params):
+        self._request = request
+        self._queryset = None
+        if f := params.get('filter', None):
+            custom_filter = CustomFilter.objects.filter(id=f, space=self._request.space).filter(Q(created_by=self._request.user) |
+                                                                                                Q(shared=self._request.user) | Q(recipebook__shared=self._request.user)).first()
+            if custom_filter:
+                self._params = {**json.loads(custom_filter.search)}
+                self._original_params = {**(params or {})}
+            else:
+                self._params = {**(params or {})}
         else:
-            for k in search_keywords:
-                queryset = queryset.filter(keywords__id=k)
-
-    if len(search_foods) > 0:
-        if search_foods_or == 'true':
-            queryset = queryset.filter(steps__ingredients__food__id__in=search_foods)
+            self._params = {**(params or {})}
+        if self._request.user.is_authenticated:
+            CACHE_KEY = f'search_pref_{request.user.id}'
+            cached_result = cache.get(CACHE_KEY, default=None)
+            if cached_result is not None:
+                self._search_prefs = cached_result
+            else:
+                self._search_prefs = request.user.searchpreference
+            cache.set(CACHE_KEY, self._search_prefs, timeout=10)
         else:
-            for k in search_foods:
-                queryset = queryset.filter(steps__ingredients__food__id=k)
-
-    if len(search_books) > 0:
-        if search_books_or == 'true':
-            queryset = queryset.filter(recipebookentry__book__id__in=search_books)
+            self._search_prefs = SearchPreference()
+        self._string = self._params.get('query').strip() if self._params.get('query', None) else None
+        self._rating = self._params.get('rating', None)
+        self._keywords = {
+            'or': self._params.get('keywords_or', None) or self._params.get('keywords', None),
+            'and': self._params.get('keywords_and', None),
+            'or_not': self._params.get('keywords_or_not', None),
+            'and_not': self._params.get('keywords_and_not', None)
+        }
+        self._foods = {
+            'or': self._params.get('foods_or', None) or self._params.get('foods', None),
+            'and': self._params.get('foods_and', None),
+            'or_not': self._params.get('foods_or_not', None),
+            'and_not': self._params.get('foods_and_not', None)
+        }
+        self._books = {
+            'or': self._params.get('books_or', None) or self._params.get('books', None),
+            'and': self._params.get('books_and', None),
+            'or_not': self._params.get('books_or_not', None),
+            'and_not': self._params.get('books_and_not', None)
+        }
+        self._steps = self._params.get('steps', None)
+        self._units = self._params.get('units', None)
+        # TODO add created by
+        # TODO image exists
+        self._sort_order = self._params.get('sort_order', None)
+        self._internal = str2bool(self._params.get('internal', None))
+        self._random = str2bool(self._params.get('random', False))
+        self._new = str2bool(self._params.get('new', False))
+        self._num_recent = int(self._params.get('num_recent', 0))
+        self._include_children = str2bool(self._params.get('include_children', None))
+        self._timescooked = self._params.get('timescooked', None)
+        self._cookedon = self._params.get('cookedon', None)
+        self._createdon = self._params.get('createdon', None)
+        self._updatedon = self._params.get('updatedon', None)
+        self._viewedon = self._params.get('viewedon', None)
+        self._makenow = self._params.get('makenow', None)
+        # this supports hidden feature to find recipes missing X ingredients
+        if type(self._makenow) == bool and self._makenow == True:
+            self._makenow = 0
+        elif type(self._makenow) == str and self._makenow in ["yes", "true"]:
+            self._makenow = 0
         else:
-            for k in search_books:
-                queryset = queryset.filter(recipebookentry__book__id=k)
+            try:
+                self._makenow = int(self._makenow)
+            except (ValueError, TypeError):
+                self._makenow = None
 
-    queryset = queryset.distinct()
+        self._search_type = self._search_prefs.search or 'plain'
+        if self._string:
+            if self._postgres:
+                self._unaccent_include = self._search_prefs.unaccent.values_list('field', flat=True)
+            else:
+                self._unaccent_include = []
+            self._icontains_include = [x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.icontains.values_list('field', flat=True)]
+            self._istartswith_include = [x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.istartswith.values_list('field', flat=True)]
+            self._trigram_include = None
+            self._fulltext_include = None
+        self._trigram = False
+        if self._postgres and self._string:
+            self._language = DICTIONARY.get(translation.get_language(), 'simple')
+            self._trigram_include = [x + '__unaccent' if x in self._unaccent_include else x for x in self._search_prefs.trigram.values_list('field', flat=True)]
+            self._fulltext_include = self._search_prefs.fulltext.values_list('field', flat=True) or None
 
-    if search_internal == 'true':
-        queryset = queryset.filter(internal=True)
+            if self._search_type not in ['websearch', 'raw'] and self._trigram_include:
+                self._trigram = True
+            self.search_query = SearchQuery(
+                self._string,
+                search_type=self._search_type,
+                config=self._language,
+            )
+            self.search_rank = None
+        self.orderby = []
+        self._filters = None
+        self._fuzzy_match = None
 
-    if search_random == 'true':
-        queryset = queryset.order_by("?")
+    def get_queryset(self, queryset):
+        self._queryset = queryset
+        self._queryset = self._queryset.prefetch_related('keywords')
 
-    return queryset
+        self._build_sort_order()
+        self._recently_viewed(num_recent=self._num_recent)
+        self._cooked_on_filter(cooked_date=self._cookedon)
+        self._created_on_filter(created_date=self._createdon)
+        self._updated_on_filter(updated_date=self._updatedon)
+        self._viewed_on_filter(viewed_date=self._viewedon)
+        self._favorite_recipes(times_cooked=self._timescooked)
+        self._new_recipes()
+        self.keyword_filters(**self._keywords)
+        self.food_filters(**self._foods)
+        self.book_filters(**self._books)
+        self.rating_filter(rating=self._rating)
+        self.internal_filter(internal=self._internal)
+        self.step_filters(steps=self._steps)
+        self.unit_filters(units=self._units)
+        self._makenow_filter(missing=self._makenow)
+        self.string_filters(string=self._string)
+        return self._queryset.filter(space=self._request.space).distinct().order_by(*self.orderby)
+
+    def _sort_includes(self, *args):
+        for x in args:
+            if x in self.orderby:
+                return True
+            elif '-' + x in self.orderby:
+                return True
+        return False
+
+    def _build_sort_order(self):
+        if self._random:
+            self.orderby = ['?']
+        else:
+            order = []
+            # TODO add userpreference for default sort order and replace '-favorite'
+            default_order = ['-name']
+            # recent and new_recipe are always first; they float a few recipes to the top
+            if self._num_recent:
+                order += ['-recent']
+            if self._new:
+                order += ['-new_recipe']
+
+            # if a sort order is provided by user - use that order
+            if self._sort_order:
+
+                if not isinstance(self._sort_order, list):
+                    order += [self._sort_order]
+                else:
+                    order += self._sort_order
+                if not self._postgres or not self._string:
+                    if 'score' in order:
+                        order.remove('score')
+                    if '-score' in order:
+                        order.remove('-score')
+            # if no sort order provided prioritize text search, followed by the default search
+            elif self._postgres and self._string and (self._trigram or self._fulltext_include):
+                order += ['-score', *default_order]
+            # otherwise sort by the remaining order_by attributes or favorite by default
+            else:
+                order += default_order
+            order[:] = [Lower('name').asc() if x == 'name' else x for x in order]
+            order[:] = [Lower('name').desc() if x == '-name' else x for x in order]
+            self.orderby = order
+
+    def string_filters(self, string=None):
+        if not string:
+            return
+
+        self.build_text_filters(self._string)
+        if self._postgres:
+            self.build_fulltext_filters(self._string)
+            self.build_trigram(self._string)
+
+        query_filter = Q()
+        if self._filters:
+            for f in self._filters:
+                query_filter |= f
+
+            self._queryset = self._queryset.filter(query_filter).distinct()  # this creates duplicate records which can screw up other aggregates, see makenow for workaround
+            if self._fulltext_include:
+                if self._fuzzy_match is None:
+                    self._queryset = self._queryset.annotate(score=Coalesce(Max(self.search_rank), 0.0))
+                else:
+                    self._queryset = self._queryset.annotate(rank=Coalesce(Max(self.search_rank), 0.0))
+
+            if self._fuzzy_match is not None:
+                simularity = self._fuzzy_match.filter(pk=OuterRef('pk')).values('simularity')
+                if not self._fulltext_include:
+                    self._queryset = self._queryset.annotate(score=Coalesce(Subquery(simularity), 0.0))
+                else:
+                    self._queryset = self._queryset.annotate(simularity=Coalesce(Subquery(simularity), 0.0))
+            if self._sort_includes('score') and self._fulltext_include and self._fuzzy_match is not None:
+                self._queryset = self._queryset.annotate(score=F('rank') + F('simularity'))
+        else:
+            query_filter = Q()
+            for f in [x + '__unaccent__iexact' if x in self._unaccent_include else x + '__iexact' for x in SearchFields.objects.all().values_list('field', flat=True)]:
+                query_filter |= Q(**{"%s" % f: self._string})
+            self._queryset = self._queryset.filter(query_filter).distinct()
+
+    def _cooked_on_filter(self, cooked_date=None):
+        if self._sort_includes('lastcooked') or cooked_date:
+            lessthan = self._sort_includes('-lastcooked') or '-' in (cooked_date or [])[:1]
+            if lessthan:
+                default = timezone.now() - timedelta(days=100000)
+            else:
+                default = timezone.now()
+            self._queryset = self._queryset.annotate(lastcooked=Coalesce(
+                Max(Case(When(cooklog__created_by=self._request.user, cooklog__space=self._request.space, then='cooklog__created_at'))), Value(default)))
+        if cooked_date is None:
+            return
+
+        cooked_date = date(*[int(x) for x in cooked_date.split('-') if x != ''])
+
+        if lessthan:
+            self._queryset = self._queryset.filter(lastcooked__date__lte=cooked_date).exclude(lastcooked=default)
+        else:
+            self._queryset = self._queryset.filter(lastcooked__date__gte=cooked_date).exclude(lastcooked=default)
+
+    def _created_on_filter(self, created_date=None):
+        if created_date is None:
+            return
+        lessthan = '-' in created_date[:1]
+        created_date = date(*[int(x) for x in created_date.split('-') if x != ''])
+        if lessthan:
+            self._queryset = self._queryset.filter(created_at__date__lte=created_date)
+        else:
+            self._queryset = self._queryset.filter(created_at__date__gte=created_date)
+
+    def _updated_on_filter(self, updated_date=None):
+        if updated_date is None:
+            return
+        lessthan = '-' in updated_date[:1]
+        updated_date = date(*[int(x) for x in updated_date.split('-') if x != ''])
+        if lessthan:
+            self._queryset = self._queryset.filter(updated_at__date__lte=updated_date)
+        else:
+            self._queryset = self._queryset.filter(updated_at__date__gte=updated_date)
+
+    def _viewed_on_filter(self, viewed_date=None):
+        if self._sort_includes('lastviewed') or viewed_date:
+            longTimeAgo = timezone.now() - timedelta(days=100000)
+            self._queryset = self._queryset.annotate(lastviewed=Coalesce(
+                Max(Case(When(viewlog__created_by=self._request.user, viewlog__space=self._request.space, then='viewlog__created_at'))), Value(longTimeAgo)))
+        if viewed_date is None:
+            return
+        lessthan = '-' in viewed_date[:1]
+        viewed_date = date(*[int(x) for x in viewed_date.split('-') if x != ''])
+
+        if lessthan:
+            self._queryset = self._queryset.filter(lastviewed__date__lte=viewed_date).exclude(lastviewed=longTimeAgo)
+        else:
+            self._queryset = self._queryset.filter(lastviewed__date__gte=viewed_date).exclude(lastviewed=longTimeAgo)
+
+    def _new_recipes(self, new_days=7):
+        # TODO make new days a user-setting
+        if not self._new:
+            return
+        self._queryset = (
+            self._queryset.annotate(new_recipe=Case(
+                When(created_at__gte=(timezone.now() - timedelta(days=new_days)), then=('pk')), default=Value(0), ))
+        )
+
+    def _recently_viewed(self, num_recent=None):
+        if not num_recent:
+            if self._sort_includes('lastviewed'):
+                self._queryset = self._queryset.annotate(lastviewed=Coalesce(
+                    Max(Case(When(viewlog__created_by=self._request.user, viewlog__space=self._request.space, then='viewlog__pk'))), Value(0)))
+            return
+
+        num_recent_recipes = ViewLog.objects.filter(created_by=self._request.user, space=self._request.space).values(
+            'recipe').annotate(recent=Max('created_at')).order_by('-recent')[:num_recent]
+        self._queryset = self._queryset.annotate(recent=Coalesce(Max(Case(When(pk__in=num_recent_recipes.values('recipe'), then='viewlog__pk'))), Value(0)))
+
+    def _favorite_recipes(self, times_cooked=None):
+        if self._sort_includes('favorite') or times_cooked:
+            less_than = '-' in (times_cooked or []) or not self._sort_includes('-favorite')
+            if less_than:
+                default = 1000
+            else:
+                default = 0
+            favorite_recipes = CookLog.objects.filter(created_by=self._request.user, space=self._request.space, recipe=OuterRef('pk')
+                                                      ).values('recipe').annotate(count=Count('pk', distinct=True)).values('count')
+            self._queryset = self._queryset.annotate(favorite=Coalesce(Subquery(favorite_recipes), default))
+        if times_cooked is None:
+            return
+
+        if times_cooked == '0':
+            self._queryset = self._queryset.filter(favorite=0)
+        elif less_than:
+            self._queryset = self._queryset.filter(favorite__lte=int(times_cooked[1:])).exclude(favorite=0)
+        else:
+            self._queryset = self._queryset.filter(favorite__gte=int(times_cooked))
+
+    def keyword_filters(self, **kwargs):
+        if all([kwargs[x] is None for x in kwargs]):
+            return
+        for kw_filter in kwargs:
+            if not kwargs[kw_filter]:
+                continue
+            if not isinstance(kwargs[kw_filter], list):
+                kwargs[kw_filter] = [kwargs[kw_filter]]
+
+            keywords = Keyword.objects.filter(pk__in=kwargs[kw_filter])
+            if 'or' in kw_filter:
+                if self._include_children:
+                    f_or = Q(keywords__in=Keyword.include_descendants(keywords))
+                else:
+                    f_or = Q(keywords__in=keywords)
+                if 'not' in kw_filter:
+                    self._queryset = self._queryset.exclude(f_or)
+                else:
+                    self._queryset = self._queryset.filter(f_or)
+            elif 'and' in kw_filter:
+                recipes = Recipe.objects.all()
+                for kw in keywords:
+                    if self._include_children:
+                        f_and = Q(keywords__in=kw.get_descendants_and_self())
+                    else:
+                        f_and = Q(keywords=kw)
+                    if 'not' in kw_filter:
+                        recipes = recipes.filter(f_and)
+                    else:
+                        self._queryset = self._queryset.filter(f_and)
+                if 'not' in kw_filter:
+                    self._queryset = self._queryset.exclude(id__in=recipes.values('id'))
+
+    def food_filters(self, **kwargs):
+        if all([kwargs[x] is None for x in kwargs]):
+            return
+        for fd_filter in kwargs:
+            if not kwargs[fd_filter]:
+                continue
+            if not isinstance(kwargs[fd_filter], list):
+                kwargs[fd_filter] = [kwargs[fd_filter]]
+
+            foods = Food.objects.filter(pk__in=kwargs[fd_filter])
+            if 'or' in fd_filter:
+                if self._include_children:
+                    f_or = Q(steps__ingredients__food__in=Food.include_descendants(foods))
+                else:
+                    f_or = Q(steps__ingredients__food__in=foods)
+
+                if 'not' in fd_filter:
+                    self._queryset = self._queryset.exclude(f_or)
+                else:
+                    self._queryset = self._queryset.filter(f_or)
+            elif 'and' in fd_filter:
+                recipes = Recipe.objects.all()
+                for food in foods:
+                    if self._include_children:
+                        f_and = Q(steps__ingredients__food__in=food.get_descendants_and_self())
+                    else:
+                        f_and = Q(steps__ingredients__food=food)
+                    if 'not' in fd_filter:
+                        recipes = recipes.filter(f_and)
+                    else:
+                        self._queryset = self._queryset.filter(f_and)
+                if 'not' in fd_filter:
+                    self._queryset = self._queryset.exclude(id__in=recipes.values('id'))
+
+    def unit_filters(self, units=None, operator=True):
+        if operator != True:
+            raise NotImplementedError
+        if not units:
+            return
+        if not isinstance(units, list):
+            units = [units]
+        self._queryset = self._queryset.filter(steps__ingredients__unit__in=units)
+
+    def rating_filter(self, rating=None):
+        if rating or self._sort_includes('rating'):
+            lessthan = self._sort_includes('-rating') or '-' in (rating or [])
+            if lessthan:
+                default = 100
+            else:
+                default = 0
+            # TODO make ratings a settings user-only vs all-users
+            self._queryset = self._queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=self._request.user, then='cooklog__rating'), default=default))))
+        if rating is None:
+            return
+
+        if rating == '0':
+            self._queryset = self._queryset.filter(rating=0)
+        elif lessthan:
+            self._queryset = self._queryset.filter(rating__lte=int(rating[1:])).exclude(rating=0)
+        else:
+            self._queryset = self._queryset.filter(rating__gte=int(rating))
+
+    def internal_filter(self, internal=None):
+        if not internal:
+            return
+        self._queryset = self._queryset.filter(internal=internal)
+
+    def book_filters(self, **kwargs):
+        if all([kwargs[x] is None for x in kwargs]):
+            return
+        for bk_filter in kwargs:
+            if not kwargs[bk_filter]:
+                continue
+            if not isinstance(kwargs[bk_filter], list):
+                kwargs[bk_filter] = [kwargs[bk_filter]]
+
+            if 'or' in bk_filter:
+                f = Q(recipebookentry__book__id__in=kwargs[bk_filter])
+                if 'not' in bk_filter:
+                    self._queryset = self._queryset.exclude(f)
+                else:
+                    self._queryset = self._queryset.filter(f)
+            elif 'and' in bk_filter:
+                recipes = Recipe.objects.all()
+                for book in kwargs[bk_filter]:
+                    if 'not' in bk_filter:
+                        recipes = recipes.filter(recipebookentry__book__id=book)
+                    else:
+                        self._queryset = self._queryset.filter(recipebookentry__book__id=book)
+                if 'not' in bk_filter:
+                    self._queryset = self._queryset.exclude(id__in=recipes.values('id'))
+
+    def step_filters(self, steps=None, operator=True):
+        if operator != True:
+            raise NotImplementedError
+        if not steps:
+            return
+        if not isinstance(steps, list):
+            steps = [unistepsts]
+        self._queryset = self._queryset.filter(steps__id__in=steps)
+
+    def build_fulltext_filters(self, string=None):
+        if not string:
+            return
+        if self._fulltext_include:
+            vectors = []
+            rank = []
+            if 'name' in self._fulltext_include:
+                vectors.append('name_search_vector')
+                rank.append(SearchRank('name_search_vector', self.search_query, cover_density=True))
+            if 'description' in self._fulltext_include:
+                vectors.append('desc_search_vector')
+                rank.append(SearchRank('desc_search_vector', self.search_query, cover_density=True))
+            if 'steps__instruction' in self._fulltext_include:
+                vectors.append('steps__search_vector')
+                rank.append(SearchRank('steps__search_vector', self.search_query, cover_density=True))
+            if 'keywords__name' in self._fulltext_include:
+                # explicitly settings unaccent on keywords and foods so that they behave the same as search_vector fields
+                vectors.append('keywords__name__unaccent')
+                rank.append(SearchRank('keywords__name__unaccent', self.search_query, cover_density=True))
+            if 'steps__ingredients__food__name' in self._fulltext_include:
+                vectors.append('steps__ingredients__food__name__unaccent')
+                rank.append(SearchRank('steps__ingredients__food__name', self.search_query, cover_density=True))
+
+            for r in rank:
+                if self.search_rank is None:
+                    self.search_rank = r
+                else:
+                    self.search_rank += r
+            # modifying queryset will annotation creates duplicate results
+            self._filters.append(Q(id__in=Recipe.objects.annotate(vector=SearchVector(*vectors)).filter(Q(vector=self.search_query))))
+
+    def build_text_filters(self, string=None):
+        if not string:
+            return
+
+        if not self._filters:
+            self._filters = []
+        # dynamically build array of filters that will be applied
+        for f in self._icontains_include:
+            self._filters += [Q(**{"%s__icontains" % f: self._string})]
+
+        for f in self._istartswith_include:
+            self._filters += [Q(**{"%s__istartswith" % f: self._string})]
+
+    def build_trigram(self, string=None):
+        if not string:
+            return
+        if self._trigram:
+            trigram = None
+            for f in self._trigram_include:
+                if trigram:
+                    trigram += TrigramSimilarity(f, self._string)
+                else:
+                    trigram = TrigramSimilarity(f, self._string)
+            self._fuzzy_match = Recipe.objects.annotate(trigram=trigram).distinct(
+            ).annotate(simularity=Max('trigram')).values('id', 'simularity').filter(simularity__gt=self._search_prefs.trigram_threshold)
+            self._filters += [Q(pk__in=self._fuzzy_match.values('pk'))]
+
+    def _makenow_filter(self, missing=None):
+        if missing is None or (type(missing) == bool and missing == False):
+            return
+        shopping_users = [*self._request.user.get_shopping_share(), self._request.user]
+
+        onhand_filter = (
+                Q(steps__ingredients__food__onhand_users__in=shopping_users)  # food onhand
+                | Q(steps__ingredients__food__substitute__onhand_users__in=shopping_users)  # or substitute food onhand
+                | Q(steps__ingredients__food__in=self.__children_substitute_filter(shopping_users))
+                | Q(steps__ingredients__food__in=self.__sibling_substitute_filter(shopping_users))
+        )
+        makenow_recipes = Recipe.objects.annotate(
+            count_food=Count('steps__ingredients__food__pk', filter=Q(steps__ingredients__food__isnull=False), distinct=True),
+            count_onhand=Count('steps__ingredients__food__pk', filter=onhand_filter, distinct=True),
+            count_ignore_shopping=Count('steps__ingredients__food__pk', filter=Q(steps__ingredients__food__ignore_shopping=True,
+                                                                                 steps__ingredients__food__recipe__isnull=True), distinct=True),
+            has_child_sub=Case(When(steps__ingredients__food__in=self.__children_substitute_filter(shopping_users), then=Value(1)), default=Value(0)),
+            has_sibling_sub=Case(When(steps__ingredients__food__in=self.__sibling_substitute_filter(shopping_users), then=Value(1)), default=Value(0))
+        ).annotate(missingfood=F('count_food') - F('count_onhand') - F('count_ignore_shopping')).filter(missingfood=missing)
+        self._queryset = self._queryset.distinct().filter(id__in=makenow_recipes.values('id'))
+
+    @staticmethod
+    def __children_substitute_filter(shopping_users=None):
+        children_onhand_subquery = Food.objects.filter(
+            path__startswith=OuterRef('path'),
+            depth__gt=OuterRef('depth'),
+            onhand_users__in=shopping_users
+        )
+        return Food.objects.exclude(  # list of foods that are onhand and children of: foods that are not onhand and are set to use children as substitutes
+            Q(onhand_users__in=shopping_users)
+            | Q(ignore_shopping=True, recipe__isnull=True)
+            | Q(substitute__onhand_users__in=shopping_users)
+        ).exclude(depth=1, numchild=0
+                  ).filter(substitute_children=True
+                           ).annotate(child_onhand_count=Exists(children_onhand_subquery)
+                                      ).filter(child_onhand_count=True)
+
+    @staticmethod
+    def __sibling_substitute_filter(shopping_users=None):
+        sibling_onhand_subquery = Food.objects.filter(
+            path__startswith=Substr(OuterRef('path'), 1, Food.steplen * (OuterRef('depth') - 1)),
+            depth=OuterRef('depth'),
+            onhand_users__in=shopping_users
+        )
+        return Food.objects.exclude(  # list of foods that are onhand and siblings of: foods that are not onhand and are set to use siblings as substitutes
+            Q(onhand_users__in=shopping_users)
+            | Q(ignore_shopping=True, recipe__isnull=True)
+            | Q(substitute__onhand_users__in=shopping_users)
+        ).exclude(depth=1, numchild=0
+                  ).filter(substitute_siblings=True
+                           ).annotate(sibling_onhand=Exists(sibling_onhand_subquery)
+                                      ).filter(sibling_onhand=True)
+
+
+class RecipeFacet():
+    class CacheEmpty(Exception):
+        pass
+
+    def __init__(self, request, queryset=None, hash_key=None, cache_timeout=3600):
+        if hash_key is None and queryset is None:
+            raise ValueError(_("One of queryset or hash_key must be provided"))
+
+        self._request = request
+        self._queryset = queryset
+        self.hash_key = hash_key or str(hash(self._queryset.query))
+        self._SEARCH_CACHE_KEY = f"recipes_filter_{self.hash_key}"
+        self._cache_timeout = cache_timeout
+        self._cache = caches['default'].get(self._SEARCH_CACHE_KEY, {})
+        if self._cache is None and self._queryset is None:
+            raise self.CacheEmpty("No queryset provided and cache empty")
+
+        self.Keywords = self._cache.get('Keywords', None)
+        self.Foods = self._cache.get('Foods', None)
+        self.Books = self._cache.get('Books', None)
+        self.Ratings = self._cache.get('Ratings', None)
+        # TODO Move Recent to recipe annotation/serializer:  requrires change in RecipeSearch(), RecipeSearchView.vue and serializer
+        self.Recent = self._cache.get('Recent', None)
+
+        if self._queryset is not None:
+            self._recipe_list = list(self._queryset.values_list('id', flat=True))
+            self._search_params = {
+                'keyword_list': self._request.query_params.getlist('keywords', []),
+                'food_list': self._request.query_params.getlist('foods', []),
+                'book_list': self._request.query_params.getlist('book', []),
+                'search_keywords_or': str2bool(self._request.query_params.get('keywords_or', True)),
+                'search_foods_or': str2bool(self._request.query_params.get('foods_or', True)),
+                'search_books_or': str2bool(self._request.query_params.get('books_or', True)),
+                'space': self._request.space,
+            }
+        elif self.hash_key is not None:
+            self._recipe_list = self._cache.get('recipe_list', [])
+            self._search_params = {
+                'keyword_list': self._cache.get('keyword_list', None),
+                'food_list': self._cache.get('food_list', None),
+                'book_list': self._cache.get('book_list', None),
+                'search_keywords_or': self._cache.get('search_keywords_or', None),
+                'search_foods_or': self._cache.get('search_foods_or', None),
+                'search_books_or': self._cache.get('search_books_or', None),
+                'space': self._cache.get('space', None),
+            }
+
+        self._cache = {
+            **self._search_params,
+            'recipe_list': self._recipe_list,
+            'Ratings': self.Ratings,
+            'Recent': self.Recent,
+            'Keywords': self.Keywords,
+            'Foods': self.Foods,
+            'Books': self.Books
+
+        }
+        caches['default'].set(self._SEARCH_CACHE_KEY, self._cache, self._cache_timeout)
+
+    def get_facets(self, from_cache=False):
+        if from_cache:
+            return {
+                'cache_key': self.hash_key or '',
+                'Ratings': self.Ratings or {},
+                'Recent': self.Recent or [],
+                'Keywords': self.Keywords or [],
+                'Foods': self.Foods or [],
+                'Books': self.Books or []
+            }
+        return {
+            'cache_key': self.hash_key,
+            'Ratings': self.get_ratings(),
+            'Recent': self.get_recent(),
+            'Keywords': self.get_keywords(),
+            'Foods': self.get_foods(),
+            'Books': self.get_books()
+        }
+
+    def set_cache(self, key, value):
+        self._cache = {**self._cache, key: value}
+        caches['default'].set(
+            self._SEARCH_CACHE_KEY,
+            self._cache,
+            self._cache_timeout
+        )
+
+    def get_books(self):
+        if self.Books is None:
+            self.Books = []
+        return self.Books
+
+    def get_keywords(self):
+        if self.Keywords is None:
+            if self._search_params['search_keywords_or']:
+                keywords = Keyword.objects.filter(space=self._request.space).distinct()
+            else:
+                keywords = Keyword.objects.filter(Q(recipe__in=self._recipe_list) | Q(depth=1)).filter(space=self._request.space).distinct()
+
+            # set keywords to root objects only
+            keywords = self._keyword_queryset(keywords)
+            self.Keywords = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(keywords)]
+            self.set_cache('Keywords', self.Keywords)
+        return self.Keywords
+
+    def get_foods(self):
+        if self.Foods is None:
+            # # if using an OR search, will annotate all keywords, otherwise, just those that appear in results
+            if self._search_params['search_foods_or']:
+                foods = Food.objects.filter(space=self._request.space).distinct()
+            else:
+                foods = Food.objects.filter(Q(ingredient__step__recipe__in=self._recipe_list) | Q(depth=1)).filter(space=self._request.space).distinct()
+
+            # set keywords to root objects only
+            foods = self._food_queryset(foods)
+
+            self.Foods = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(foods)]
+            self.set_cache('Foods', self.Foods)
+        return self.Foods
+
+    def get_books(self):
+        if self.Books is None:
+            self.Books = []
+        return self.Books
+
+    def get_ratings(self):
+        if self.Ratings is None:
+            if not self._request.space.demo and self._request.space.show_facet_count:
+                if self._queryset is None:
+                    self._queryset = Recipe.objects.filter(id__in=self._recipe_list)
+                rating_qs = self._queryset.annotate(rating=Round(Avg(Case(When(cooklog__created_by=self._request.user, then='cooklog__rating'), default=Value(0)))))
+                self.Ratings = dict(Counter(r.rating for r in rating_qs))
+            else:
+                self.Rating = {}
+            self.set_cache('Ratings', self.Ratings)
+        return self.Ratings
+
+    def get_recent(self):
+        if self.Recent is None:
+            # TODO make days of recent recipe a setting
+            recent_recipes = ViewLog.objects.filter(created_by=self._request.user, space=self._request.space, created_at__gte=timezone.now() - timedelta(days=14)
+                                                    ).values_list('recipe__pk', flat=True)
+            self.Recent = list(recent_recipes)
+            self.set_cache('Recent', self.Recent)
+        return self.Recent
+
+    def add_food_children(self, id):
+        try:
+            food = Food.objects.get(id=id)
+            nodes = food.get_ancestors()
+        except Food.DoesNotExist:
+            return self.get_facets()
+        foods = self._food_queryset(food.get_children(), food)
+        deep_search = self.Foods
+        for node in nodes:
+            index = next((i for i, x in enumerate(deep_search) if x["id"] == node.id), None)
+            deep_search = deep_search[index]['children']
+        index = next((i for i, x in enumerate(deep_search) if x["id"] == food.id), None)
+        deep_search[index]['children'] = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(foods)]
+        self.set_cache('Foods', self.Foods)
+        return self.get_facets()
+
+    def add_keyword_children(self, id):
+        try:
+            keyword = Keyword.objects.get(id=id)
+            nodes = keyword.get_ancestors()
+        except Keyword.DoesNotExist:
+            return self.get_facets()
+        keywords = self._keyword_queryset(keyword.get_children(), keyword)
+        deep_search = self.Keywords
+        for node in nodes:
+            index = next((i for i, x in enumerate(deep_search) if x["id"] == node.id), None)
+            deep_search = deep_search[index]['children']
+        index = next((i for i, x in enumerate(deep_search) if x["id"] == keyword.id), None)
+        deep_search[index]['children'] = [{**x, 'children': None} if x['numchild'] > 0 else x for x in list(keywords)]
+        self.set_cache('Keywords', self.Keywords)
+        return self.get_facets()
+
+    def _recipe_count_queryset(self, field, depth=1, steplen=4):
+        return Recipe.objects.filter(**{f'{field}__path__startswith': OuterRef('path'), f'{field}__depth__gte': depth}, id__in=self._recipe_list, space=self._request.space
+                                     ).annotate(count=Coalesce(Func('pk', function='Count'), 0)).values('count')
+
+    def _keyword_queryset(self, queryset, keyword=None):
+        depth = getattr(keyword, 'depth', 0) + 1
+        steplen = depth * Keyword.steplen
+
+        if not self._request.space.demo and self._request.space.show_facet_count:
+            return queryset.annotate(count=Coalesce(Subquery(self._recipe_count_queryset('keywords', depth, steplen)), 0)
+                                     ).filter(depth=depth, count__gt=0
+                                              ).values('id', 'name', 'count', 'numchild').order_by(Lower('name').asc())[:200]
+        else:
+            return queryset.filter(depth=depth).values('id', 'name', 'numchild').order_by(Lower('name').asc())
+
+    def _food_queryset(self, queryset, food=None):
+        depth = getattr(food, 'depth', 0) + 1
+        steplen = depth * Food.steplen
+
+        if not self._request.space.demo and self._request.space.show_facet_count:
+            return queryset.annotate(count=Coalesce(Subquery(self._recipe_count_queryset('steps__ingredients__food', depth, steplen)), 0)
+                                     ).filter(depth__lte=depth, count__gt=0
+                                              ).values('id', 'name', 'count', 'numchild').order_by(Lower('name').asc())[:200]
+        else:
+            return queryset.filter(depth__lte=depth).values('id', 'name', 'numchild').order_by(Lower('name').asc())

cookbook/helper/recipe_url_import.py

@@ -1,22 +1,27 @@
 import random
 import re
+from html import unescape
+from unicodedata import decomposition
+
+from django.utils.dateparse import parse_duration
+from django.utils.translation import gettext as _
 from isodate import parse_duration as iso_parse_duration
 from isodate.isoerror import ISO8601Error
-from recipe_scrapers._exceptions import ElementNotFoundInHtml
+from pytube import YouTube
+from recipe_scrapers._utils import get_host_name, get_minutes
 
-from cookbook.helper.ingredient_parser import parse as parse_single_ingredient
+from cookbook.helper import recipe_url_import as helper
+from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.models import Keyword
-from django.utils.dateparse import parse_duration
-from html import unescape
-from recipe_scrapers._schemaorg import SchemaOrgException
-from recipe_scrapers._utils import get_minutes
+
+# from recipe_scrapers._utils import get_minutes  ## temporary until/unless upstream incorporates get_minutes() PR
 
 
-def get_from_scraper(scrape, space):
+def get_from_scraper(scrape, request):
     # converting the scrape_me object to the existing json format based on ld+json
     recipe_json = {}
     try:
-        recipe_json['name'] = parse_name(scrape.title() or None)
+        recipe_json['name'] = parse_name(scrape.title()[:128] or None)
     except Exception:
         recipe_json['name'] = None
     if not recipe_json['name']:
@@ -26,42 +31,46 @@ def get_from_scraper(scrape, space):
             recipe_json['name'] = ''
 
     try:
-        description = scrape.schema.data.get("description") or ''
+        description = scrape.description() or None
     except Exception:
-        description = ''
-
-    recipe_json['description'] = parse_description(description)
-
-    try:
-        servings = scrape.yields() or None
-    except Exception:
-        servings = None
-    if not servings:
+        description = None
+    if not description:
         try:
-            servings = scrape.schema.data.get('recipeYield') or 1
+            description = scrape.schema.data.get("description") or ''
         except Exception:
-            servings = 1
-    if type(servings) != int:
-        try:
-            servings = int(re.findall(r'\b\d+\b', servings)[0])
-        except Exception:
-            servings = 1
-    recipe_json['servings'] = max(servings, 1)
+            description = ''
+
+    recipe_json['internal'] = True
 
     try:
-        recipe_json['prepTime'] = get_minutes(scrape.schema.data.get("prepTime")) or 0
+        servings = scrape.schema.data.get('recipeYield') or 1  # dont use scrape.yields() as this will always return "x servings" or "x items", should be improved in scrapers directly
     except Exception:
-        recipe_json['prepTime'] = 0
+        servings = 1
+
+    recipe_json['servings'] = parse_servings(servings)
+    recipe_json['servings_text'] = parse_servings_text(servings)
+
     try:
-        recipe_json['cookTime'] = get_minutes(scrape.schema.data.get("cookTime")) or 0
+        recipe_json['working_time'] = get_minutes(scrape.prep_time()) or 0
     except Exception:
-        recipe_json['cookTime'] = 0
-    if recipe_json['cookTime'] + recipe_json['prepTime'] == 0:
         try:
-            recipe_json['prepTime'] = get_minutes(scrape.total_time()) or 0
+            recipe_json['working_time'] = get_minutes(scrape.schema.data.get("prepTime")) or 0
+        except Exception:
+            recipe_json['working_time'] = 0
+    try:
+        recipe_json['waiting_time'] = get_minutes(scrape.cook_time()) or 0
+    except Exception:
+        try:
+            recipe_json['waiting_time'] = get_minutes(scrape.schema.data.get("cookTime")) or 0
+        except Exception:
+            recipe_json['waiting_time'] = 0
+
+    if recipe_json['working_time'] + recipe_json['waiting_time'] == 0:
+        try:
+            recipe_json['working_time'] = get_minutes(scrape.total_time()) or 0
         except Exception:
             try:
-                get_minutes(scrape.schema.data.get("totalTime")) or 0
+                recipe_json['working_time'] = get_minutes(scrape.schema.data.get("totalTime")) or 0
             except Exception:
                 pass
 
@@ -82,69 +91,123 @@ def get_from_scraper(scrape, space):
     except Exception:
         pass
     try:
-        if scrape.schema.data.get('recipeCategory'):
-            keywords += listify_keywords(scrape.schema.data.get("recipeCategory"))
+        if scrape.category():
+            keywords += listify_keywords(scrape.category())
     except Exception:
-        pass
+        try:
+            if scrape.schema.data.get('recipeCategory'):
+                keywords += listify_keywords(scrape.schema.data.get("recipeCategory"))
+        except Exception:
+            pass
     try:
-        if scrape.schema.data.get('recipeCuisine'):
-            keywords += listify_keywords(scrape.schema.data.get("recipeCuisine"))
+        if scrape.cuisine():
+            keywords += listify_keywords(scrape.cuisine())
     except Exception:
-        pass
+        try:
+            if scrape.schema.data.get('recipeCuisine'):
+                keywords += listify_keywords(scrape.schema.data.get("recipeCuisine"))
+        except Exception:
+            pass
+
     try:
-        recipe_json['keywords'] = parse_keywords(list(set(map(str.casefold, keywords))), space)
+        source_url = scrape.canonical_url()
+    except Exception:
+        try:
+            source_url = scrape.url
+        except Exception:
+            pass
+    if source_url:
+        recipe_json['source_url'] = source_url
+        try:
+            keywords.append(source_url.replace('http://', '').replace('https://', '').split('/')[0])
+        except Exception:
+            pass
+
+    try:
+        recipe_json['keywords'] = parse_keywords(list(set(map(str.casefold, keywords))), request.space)
     except AttributeError:
         recipe_json['keywords'] = keywords
 
+    ingredient_parser = IngredientParser(request, True)
+
+    recipe_json['steps'] = []
+    try:
+        for i in parse_instructions(scrape.instructions()):
+            recipe_json['steps'].append({'instruction': i, 'ingredients': [], })
+    except Exception:
+        pass
+    if len(recipe_json['steps']) == 0:
+        recipe_json['steps'].append({'instruction': '', 'ingredients': [], })
+
+    if len(parse_description(description)) > 256:  # split at 256 as long descriptions dont look good on recipe cards
+        recipe_json['steps'][0]['instruction'] = f'*{parse_description(description)}*  \n\n' + recipe_json['steps'][0]['instruction']
+    else:
+        recipe_json['description'] = parse_description(description)[:512]
+
     try:
-        ingredients = []
         for x in scrape.ingredients():
             try:
-                amount, unit, ingredient, note = parse_single_ingredient(x)
-                ingredients.append(
-                    {
-                        'amount': amount,
-                        'unit': {
-                            'text': unit,
-                            'id': random.randrange(10000, 99999)
-                        },
-                        'ingredient': {
-                            'text': ingredient,
-                            'id': random.randrange(10000, 99999)
-                        },
-                        'note': note,
-                        'original': x
-                    }
-                )
+                amount, unit, ingredient, note = ingredient_parser.parse(x)
+                ingredient = {
+                    'amount': amount,
+                    'food': {
+                        'name': ingredient,
+                    },
+                    'unit': None,
+                    'note': note,
+                    'original_text': x
+                }
+                if unit:
+                    ingredient['unit'] = {'name': unit, }
+                recipe_json['steps'][0]['ingredients'].append(ingredient)
             except Exception:
-                ingredients.append(
+                recipe_json['steps'][0]['ingredients'].append(
                     {
                         'amount': 0,
-                        'unit': {
-                            'text': '',
-                            'id': random.randrange(10000, 99999)
-                        },
-                        'ingredient': {
-                            'text': x,
-                            'id': random.randrange(10000, 99999)
+                        'unit': None,
+                        'food': {
+                            'name': x,
                         },
                         'note': '',
-                        'original': x
+                        'original_text': x
                     }
                 )
-        recipe_json['recipeIngredient'] = ingredients
     except Exception:
-        recipe_json['recipeIngredient'] = ingredients
+        pass
+
+    return recipe_json
+
+
+def get_from_youtube_scraper(url, request):
+    """A YouTube Information Scraper."""
+    kw, created = Keyword.objects.get_or_create(name='YouTube', space=request.space)
+    default_recipe_json = {
+        'name': '',
+        'internal': True,
+        'description': '',
+        'servings': 1,
+        'working_time': 0,
+        'waiting_time': 0,
+        'image': "",
+        'keywords': [{'name': kw.name, 'label': kw.name, 'id': kw.pk}],
+        'source_url': url,
+        'steps': [
+            {
+                'ingredients': [],
+                'instruction': ''
+            }
+        ]
+    }
 
     try:
-        recipe_json['recipeInstructions'] = parse_instructions(scrape.instructions())
+        video = YouTube(url=url)
+        default_recipe_json['name'] = video.title
+        default_recipe_json['image'] = video.thumbnail_url
+        default_recipe_json['steps'][0]['instruction'] = video.description
     except Exception:
-        recipe_json['recipeInstructions'] = ""
+        pass
 
-    if scrape.url:
-        recipe_json['url'] = scrape.url
-        recipe_json['recipeInstructions'] += "\n\nImported from " + scrape.url
-    return recipe_json
+    return default_recipe_json
 
 
 def parse_name(name):
@@ -156,97 +219,44 @@ def parse_name(name):
     return normalize_string(name)
 
 
-def parse_ingredients(ingredients):
-    # some pages have comma separated ingredients in a single array entry
-    try:
-        if type(ingredients[0]) == dict:
-            return ingredients
-    except (KeyError, IndexError):
-        pass
-
-    if (len(ingredients) == 1 and type(ingredients) == list):
-        ingredients = ingredients[0].split(',')
-    elif type(ingredients) == str:
-        ingredients = ingredients.split(',')
-
-    for x in ingredients:
-        if '\n' in x:
-            ingredients.remove(x)
-            for i in x.split('\n'):
-                ingredients.insert(0, i)
-
-    ingredient_list = []
-
-    for x in ingredients:
-        if x.replace(' ', '') != '':
-            x = x.replace('½', "0.5").replace('¼', "0.25").replace('¾', "0.75")
-            try:
-                amount, unit, ingredient, note = parse_single_ingredient(x)
-                if ingredient:
-                    ingredient_list.append(
-                        {
-                            'amount': amount,
-                            'unit': {
-                                'text': unit,
-                                'id': random.randrange(10000, 99999)
-                            },
-                            'ingredient': {
-                                'text': ingredient,
-                                'id': random.randrange(10000, 99999)
-                            },
-                            'note': note,
-                            'original': x
-                        }
-                    )
-            except Exception:
-                ingredient_list.append(
-                    {
-                        'amount': 0,
-                        'unit': {
-                            'text': '',
-                            'id': random.randrange(10000, 99999)
-                        },
-                        'ingredient': {
-                            'text': x,
-                            'id': random.randrange(10000, 99999)
-                        },
-                        'note': '',
-                        'original': x
-                    }
-                )
-
-            ingredients = ingredient_list
-        else:
-            ingredients = []
-    return ingredients
-
-
 def parse_description(description):
     return normalize_string(description)
 
 
-def parse_instructions(instructions):
-    instruction_text = ''
+def clean_instruction_string(instruction):
+    normalized_string = normalize_string(instruction)
+    normalized_string = normalized_string.replace('\n', '  \n')
+    normalized_string = normalized_string.replace('  \n  \n', '\n\n')
+    return normalized_string
+
+
+def parse_instructions(instructions):
+    """
+    Convert arbitrary instructions object from website import and turn it into a flat list of strings
+    :param instructions: any instructions object from import
+    :return: list of strings (from one to many elements depending on website)
+    """
+    instruction_list = []
 
-    # flatten instructions if they are in a list
     if type(instructions) == list:
         for i in instructions:
             if type(i) == str:
-                instruction_text += i
+                instruction_list.append(clean_instruction_string(i))
             else:
                 if 'text' in i:
-                    instruction_text += i['text'] + '\n\n'
+                    instruction_list.append(clean_instruction_string(i['text']))
                 elif 'itemListElement' in i:
                     for ile in i['itemListElement']:
                         if type(ile) == str:
-                            instruction_text += ile + '\n\n'
+                            instruction_list.append(clean_instruction_string(ile))
                         elif 'text' in ile:
-                            instruction_text += ile['text'] + '\n\n'
+                            instruction_list.append(clean_instruction_string(ile['text']))
                 else:
-                    instruction_text += str(i)
-        instructions = instruction_text
+                    instruction_list.append(clean_instruction_string(str(i)))
+    else:
+        instruction_list.append(clean_instruction_string(instructions))
 
-    return normalize_string(instructions)
+    return instruction_list
 
 
 def parse_image(image):
@@ -283,40 +293,31 @@ def parse_servings(servings):
     return servings
 
 
-def parse_cooktime(cooktime):
-    if type(cooktime) not in [int, float]:
+def parse_servings_text(servings):
+    if type(servings) == str:
         try:
-            cooktime = float(re.search(r'\d+', cooktime).group())
+            servings = re.sub("\d+", '', servings).strip()
+        except Exception:
+            servings = ''
+    return str(servings)[:32]
+
+
+def parse_time(recipe_time):
+    if type(recipe_time) not in [int, float]:
+        try:
+            recipe_time = float(re.search(r'\d+', recipe_time).group())
         except (ValueError, AttributeError):
             try:
-                cooktime = round(iso_parse_duration(cooktime).seconds / 60)
+                recipe_time = round(iso_parse_duration(recipe_time).seconds / 60)
             except ISO8601Error:
                 try:
-                    if (type(cooktime) == list and len(cooktime) > 0):
-                        cooktime = cooktime[0]
-                    cooktime = round(parse_duration(cooktime).seconds / 60)
+                    if (type(recipe_time) == list and len(recipe_time) > 0):
+                        recipe_time = recipe_time[0]
+                    recipe_time = round(parse_duration(recipe_time).seconds / 60)
                 except AttributeError:
-                    cooktime = 0
+                    recipe_time = 0
 
-    return cooktime
-
-
-def parse_preptime(preptime):
-    if type(preptime) not in [int, float]:
-        try:
-            preptime = float(re.search(r'\d+', preptime).group())
-        except ValueError:
-            try:
-                preptime = round(iso_parse_duration(preptime).seconds / 60)
-            except ISO8601Error:
-                try:
-                    if (type(preptime) == list and len(preptime) > 0):
-                        preptime = preptime[0]
-                    preptime = round(parse_duration(preptime).seconds / 60)
-                except AttributeError:
-                    preptime = 0
-
-    return preptime
+    return recipe_time
 
 
 def parse_keywords(keyword_json, space):
@@ -326,9 +327,9 @@ def parse_keywords(keyword_json, space):
         kw = normalize_string(kw)
         if len(kw) != 0:
             if k := Keyword.objects.filter(name=kw, space=space).first():
-                keywords.append({'id': str(k.id), 'text': str(k)})
+                keywords.append({'label': str(k), 'name': k.name, 'id': k.id})
             else:
-                keywords.append({'id': random.randrange(1111111, 9999999, 1), 'text': kw})
+                keywords.append({'label': kw, 'name': kw})
 
     return keywords
 
@@ -366,3 +367,32 @@ def iso_duration_to_minutes(string):
         string
     ).groupdict()
     return int(match['days'] or 0) * 24 * 60 + int(match['hours'] or 0) * 60 + int(match['minutes'] or 0)
+
+
+def get_images_from_soup(soup, url):
+    sources = ['src', 'srcset', 'data-src']
+    images = []
+    img_tags = soup.find_all('img')
+    if url:
+        site = get_host_name(url)
+        prot = url.split(':')[0]
+
+    urls = []
+    for img in img_tags:
+        for src in sources:
+            try:
+                urls.append(img[src])
+            except KeyError:
+                pass
+
+    for u in urls:
+        u = u.split('?')[0]
+        filename = re.search(r'/([\w_-]+[.](jpg|jpeg|gif|png))$', u)
+        if filename:
+            if (('http' not in u) and (url)):
+                # sometimes an image source can be relative
+                # if it is provide the base url
+                u = '{}://{}{}'.format(prot, site, u)
+            if 'http' in u:
+                images.append(u)
+    return images

cookbook/helper/scope_middleware.py

@@ -1,8 +1,12 @@
-from django.shortcuts import redirect
 from django.urls import reverse
 from django_scopes import scope, scopes_disabled
+from oauth2_provider.contrib.rest_framework import OAuth2Authentication
+from rest_framework.authentication import TokenAuthentication
+from rest_framework.authtoken.models import Token
+from rest_framework.exceptions import AuthenticationFailed
 
 from cookbook.views import views
+from recipes import settings
 
 
 class ScopeMiddleware:
@@ -10,30 +14,57 @@ class ScopeMiddleware:
         self.get_response = get_response
 
     def __call__(self, request):
+        prefix = settings.JS_REVERSE_SCRIPT_PREFIX or ''
+
+        # need to disable scopes for writing requests into userpref and enable for loading ?
+        if request.path.startswith(prefix + '/api/user-preference/'):
+            with scopes_disabled():
+                return self.get_response(request)
+
         if request.user.is_authenticated:
 
-            if request.path.startswith('/admin/'):
+            if request.path.startswith(prefix + '/admin/'):
                 with scopes_disabled():
                     return self.get_response(request)
 
-            if request.path.startswith('/signup/') or request.path.startswith('/invite/'):
+            if request.path.startswith(prefix + '/signup/') or request.path.startswith(prefix + '/invite/'):
                 return self.get_response(request)
 
-            if request.path.startswith('/accounts/'):
+            if request.path.startswith(prefix + '/accounts/'):
+                return self.get_response(request)
+
+            if request.path.startswith(prefix + '/switch-space/'):
                 return self.get_response(request)
 
             with scopes_disabled():
-                if request.user.userpreference.space is None and not reverse('account_logout') in request.path:
-                    return views.no_space(request)
+                if request.user.userspace_set.count() == 0 and not reverse('account_logout') in request.path:
+                    return views.space_overview(request)
 
-            if request.user.groups.count() == 0 and not reverse('account_logout') in request.path:
+            # get active user space, if for some reason more than one space is active select first (group permission checks will fail, this is not intended at this point)
+            user_space = request.user.userspace_set.filter(active=True).first()
+
+            if not user_space:
+                return views.space_overview(request)
+
+            if user_space.groups.count() == 0 and not reverse('account_logout') in request.path:
                 return views.no_groups(request)
 
-            request.space = request.user.userpreference.space
+            request.space = user_space.space
             # with scopes_disabled():
             with scope(space=request.space):
                 return self.get_response(request)
         else:
+            if request.path.startswith(prefix + '/api/'):
+                try:
+                    if auth := OAuth2Authentication().authenticate(request):
+                        user_space = auth[0].userspace_set.filter(active=True).first()
+                        if user_space:
+                            request.space = user_space.space
+                            with scope(space=request.space):
+                                return self.get_response(request)
+                except AuthenticationFailed:
+                    pass
+
             with scopes_disabled():
                 request.space = None
                 return self.get_response(request)

cookbook/helper/scrapers/scrapers.py

@@ -1,5 +1,6 @@
-from bs4 import BeautifulSoup
 from json import JSONDecodeError
+
+from bs4 import BeautifulSoup
 from recipe_scrapers import SCRAPERS, get_host_name
 from recipe_scrapers._factory import SchemaScraperFactory
 from recipe_scrapers._schemaorg import SchemaOrg
@@ -26,18 +27,17 @@ def text_scraper(text, url=None):
     class TextScraper(scraper_class):
         def __init__(
             self,
-            page_data,
-            url=None
+            html=None,
+            url=None,
         ):
             self.wild_mode = False
-            # self.exception_handling = None  # TODO add new method here, old one was deprecated
             self.meta_http_equiv = False
-            self.soup = BeautifulSoup(page_data, "html.parser")
+            self.soup = BeautifulSoup(html, "html.parser")
             self.url = url
             self.recipe = None
             try:
-                self.schema = SchemaOrg(page_data)
+                self.schema = SchemaOrg(html)
             except (JSONDecodeError, AttributeError):
                 pass
 
-    return TextScraper(text, url)
+    return TextScraper(url=url, html=text)

cookbook/helper/shopping_helper.py

@@ -0,0 +1,313 @@
+from datetime import timedelta
+from decimal import Decimal
+
+from django.contrib.postgres.aggregates import ArrayAgg
+from django.db.models import F, OuterRef, Q, Subquery, Value
+from django.db.models.functions import Coalesce
+from django.utils import timezone
+from django.utils.translation import gettext as _
+
+from cookbook.helper.HelperFunctions import Round, str2bool
+from cookbook.models import (Ingredient, MealPlan, Recipe, ShoppingListEntry, ShoppingListRecipe,
+                             SupermarketCategoryRelation)
+from recipes import settings
+
+
+def shopping_helper(qs, request):
+    supermarket = request.query_params.get('supermarket', None)
+    checked = request.query_params.get('checked', 'recent')
+    user = request.user
+    supermarket_order = [F('food__supermarket_category__name').asc(nulls_first=True), 'food__name']
+
+    # TODO created either scheduled task or startup task to delete very old shopping list entries
+    # TODO create user preference to define 'very old'
+    if supermarket:
+        supermarket_categories = SupermarketCategoryRelation.objects.filter(supermarket=supermarket, category=OuterRef('food__supermarket_category'))
+        qs = qs.annotate(supermarket_order=Coalesce(Subquery(supermarket_categories.values('order')), Value(9999)))
+        supermarket_order = ['supermarket_order'] + supermarket_order
+    if checked in ['false', 0, '0']:
+        qs = qs.filter(checked=False)
+    elif checked in ['true', 1, '1']:
+        qs = qs.filter(checked=True)
+    elif checked in ['recent']:
+        today_start = timezone.now().replace(hour=0, minute=0, second=0)
+        week_ago = today_start - timedelta(days=user.userpreference.shopping_recent_days)
+        qs = qs.filter(Q(checked=False) | Q(completed_at__gte=week_ago))
+        supermarket_order = ['checked'] + supermarket_order
+
+    return qs.distinct().order_by(*supermarket_order).select_related('unit', 'food', 'ingredient', 'created_by', 'list_recipe', 'list_recipe__mealplan', 'list_recipe__recipe')
+
+
+class RecipeShoppingEditor():
+    def __init__(self, user, space, **kwargs):
+        self.created_by = user
+        self.space = space
+        self._kwargs = {**kwargs}
+
+        self.mealplan = self._kwargs.get('mealplan', None)
+        if type(self.mealplan) in [int, float]:
+            self.mealplan = MealPlan.objects.filter(id=self.mealplan, space=self.space)
+        self.id = self._kwargs.get('id', None)
+
+        self._shopping_list_recipe = self.get_shopping_list_recipe(self.id, self.created_by, self.space)
+
+        if self._shopping_list_recipe:
+            # created_by needs to be sticky to original creator as it is 'their' shopping list
+            # changing shopping list created_by can shift some items to new owner which may not share in the other direction
+            self.created_by = getattr(self._shopping_list_recipe.entries.first(), 'created_by', self.created_by)
+
+        self.recipe = getattr(self._shopping_list_recipe, 'recipe', None) or self._kwargs.get('recipe', None) or getattr(self.mealplan, 'recipe', None)
+        if type(self.recipe) in [int, float]:
+            self.recipe = Recipe.objects.filter(id=self.recipe, space=self.space)
+
+        try:
+            self.servings = float(self._kwargs.get('servings', None))
+        except (ValueError, TypeError):
+            self.servings = getattr(self._shopping_list_recipe, 'servings', None) or getattr(self.mealplan, 'servings', None) or getattr(self.recipe, 'servings', None)
+
+    @property
+    def _recipe_servings(self):
+        return getattr(self.recipe, 'servings', None) or getattr(getattr(self.mealplan, 'recipe', None), 'servings', None) or getattr(getattr(self._shopping_list_recipe, 'recipe', None), 'servings', None)
+
+    @property
+    def _servings_factor(self):
+        return Decimal(self.servings)/Decimal(self._recipe_servings)
+
+    @property
+    def _shared_users(self):
+        return [*list(self.created_by.get_shopping_share()), self.created_by]
+
+    @staticmethod
+    def get_shopping_list_recipe(id, user, space):
+        return ShoppingListRecipe.objects.filter(id=id).filter(Q(shoppinglist__space=space) | Q(entries__space=space)).filter(
+            Q(shoppinglist__created_by=user)
+            | Q(shoppinglist__shared=user)
+            | Q(entries__created_by=user)
+            | Q(entries__created_by__in=list(user.get_shopping_share()))
+        ).prefetch_related('entries').first()
+
+    def get_recipe_ingredients(self, id, exclude_onhand=False):
+        if exclude_onhand:
+            return Ingredient.objects.filter(step__recipe__id=id,  food__ignore_shopping=False,  space=self.space).exclude(food__onhand_users__id__in=[x.id for x in self._shared_users])
+        else:
+            return Ingredient.objects.filter(step__recipe__id=id,  food__ignore_shopping=False,  space=self.space)
+
+    @property
+    def _include_related(self):
+        return self.created_by.userpreference.mealplan_autoinclude_related
+
+    @property
+    def _exclude_onhand(self):
+        return self.created_by.userpreference.mealplan_autoexclude_onhand
+
+    def create(self, **kwargs):
+        ingredients = kwargs.get('ingredients', None)
+        exclude_onhand = not ingredients and self._exclude_onhand
+        if servings := kwargs.get('servings', None):
+            self.servings = float(servings)
+
+        if mealplan := kwargs.get('mealplan', None):
+            self.mealplan = mealplan
+            self.recipe = mealplan.recipe
+        elif recipe := kwargs.get('recipe', None):
+            self.recipe = recipe
+
+        if not self.servings:
+            self.servings = getattr(self.mealplan, 'servings', None) or getattr(self.recipe, 'servings', 1.0)
+
+        self._shopping_list_recipe = ShoppingListRecipe.objects.create(recipe=self.recipe, mealplan=self.mealplan, servings=self.servings)
+
+        if ingredients:
+            self._add_ingredients(ingredients=ingredients)
+        else:
+            if self._include_related:
+                related = self.recipe.get_related_recipes()
+                self._add_ingredients(self.get_recipe_ingredients(self.recipe.id, exclude_onhand=exclude_onhand).exclude(food__recipe__in=related))
+                for r in related:
+                    self._add_ingredients(self.get_recipe_ingredients(r.id, exclude_onhand=exclude_onhand).exclude(food__recipe__in=related))
+            else:
+                self._add_ingredients(self.get_recipe_ingredients(self.recipe.id, exclude_onhand=exclude_onhand))
+
+        return True
+
+    def add(self, **kwargs):
+        return
+
+    def edit(self, servings=None, ingredients=None, **kwargs):
+        if servings:
+            self.servings = servings
+
+        self._delete_ingredients(ingredients=ingredients)
+        if self.servings != self._shopping_list_recipe.servings:
+            self.edit_servings()
+        self._add_ingredients(ingredients=ingredients)
+        return True
+
+    def edit_servings(self, servings=None, **kwargs):
+        if servings:
+            self.servings = servings
+        if id := kwargs.get('id', None):
+            self._shopping_list_recipe = self.get_shopping_list_recipe(id, self.created_by, self.space)
+        if not self.servings:
+            raise ValueError(_("You must supply a servings size"))
+
+        if self._shopping_list_recipe.servings == self.servings:
+            return True
+
+        for sle in ShoppingListEntry.objects.filter(list_recipe=self._shopping_list_recipe):
+            sle.amount = sle.ingredient.amount * Decimal(self._servings_factor)
+            sle.save()
+        self._shopping_list_recipe.servings = self.servings
+        self._shopping_list_recipe.save()
+        return True
+
+    def delete(self, **kwargs):
+        try:
+            self._shopping_list_recipe.delete()
+            return True
+        except:
+            return False
+
+    def _add_ingredients(self, ingredients=None):
+        if not ingredients:
+            return
+        elif type(ingredients) == list:
+            ingredients = Ingredient.objects.filter(id__in=ingredients)
+        existing = self._shopping_list_recipe.entries.filter(ingredient__in=ingredients).values_list('ingredient__pk', flat=True)
+        add_ingredients = ingredients.exclude(id__in=existing)
+
+        for i in [x for x in add_ingredients if x.food]:
+            ShoppingListEntry.objects.create(
+                list_recipe=self._shopping_list_recipe,
+                food=i.food,
+                unit=i.unit,
+                ingredient=i,
+                amount=i.amount * Decimal(self._servings_factor),
+                created_by=self.created_by,
+                space=self.space,
+            )
+
+    # deletes shopping list entries not in ingredients list
+    def _delete_ingredients(self, ingredients=None):
+        if not ingredients:
+            return
+        to_delete = self._shopping_list_recipe.entries.exclude(ingredient__in=ingredients)
+        ShoppingListEntry.objects.filter(id__in=to_delete).delete()
+        self._shopping_list_recipe = self.get_shopping_list_recipe(self.id, self.created_by, self.space)
+
+
+# # TODO refactor as class
+# def list_from_recipe(list_recipe=None, recipe=None, mealplan=None, servings=None, ingredients=None, created_by=None, space=None, append=False):
+#     """
+#     Creates ShoppingListRecipe and associated ShoppingListEntrys from a recipe or a meal plan with a recipe
+#     :param list_recipe: Modify an existing ShoppingListRecipe
+#     :param recipe: Recipe to use as list of ingredients.  One of [recipe, mealplan] are required
+#     :param mealplan: alternatively use a mealplan recipe as source of ingredients
+#     :param servings: Optional: Number of servings to use to scale shoppinglist.  If servings = 0 an existing recipe list will be deleted
+#     :param ingredients: Ingredients, list of ingredient IDs to include on the shopping list.  When not provided all ingredients will be used
+#     :param append: If False will remove any entries not included with ingredients, when True will append ingredients to the shopping list
+#     """
+#     r = recipe or getattr(mealplan, 'recipe', None) or getattr(list_recipe, 'recipe', None)
+#     if not r:
+#         raise ValueError(_("You must supply a recipe or mealplan"))
+
+#     created_by = created_by or getattr(ShoppingListEntry.objects.filter(list_recipe=list_recipe).first(), 'created_by', None)
+#     if not created_by:
+#         raise ValueError(_("You must supply a created_by"))
+
+#     try:
+#         servings = float(servings)
+#     except (ValueError, TypeError):
+#         servings = getattr(mealplan, 'servings', 1.0)
+
+#     servings_factor = servings / r.servings
+
+#     shared_users = list(created_by.get_shopping_share())
+#     shared_users.append(created_by)
+#     if list_recipe:
+#         created = False
+#     else:
+#         list_recipe = ShoppingListRecipe.objects.create(recipe=r, mealplan=mealplan, servings=servings)
+#         created = True
+
+#     related_step_ing = []
+#     if servings == 0 and not created:
+#         list_recipe.delete()
+#         return []
+#     elif ingredients:
+#         ingredients = Ingredient.objects.filter(pk__in=ingredients, space=space)
+#     else:
+#         ingredients = Ingredient.objects.filter(step__recipe=r,  food__ignore_shopping=False, space=space)
+
+#         if exclude_onhand := created_by.userpreference.mealplan_autoexclude_onhand:
+#             ingredients = ingredients.exclude(food__onhand_users__id__in=[x.id for x in shared_users])
+
+#         if related := created_by.userpreference.mealplan_autoinclude_related:
+#             # TODO: add levels of related recipes (related recipes of related recipes) to use when auto-adding mealplans
+#             related_recipes = r.get_related_recipes()
+
+#             for x in related_recipes:
+#                 # related recipe is a Step serving size is driven by recipe serving size
+#                 # TODO once/if Steps can have a serving size this needs to be refactored
+#                 if exclude_onhand:
+#                     # if steps are used more than once in a recipe or subrecipe - I don' think this results in the desired behavior
+#                     related_step_ing += Ingredient.objects.filter(step__recipe=x, space=space).exclude(food__onhand_users__id__in=[x.id for x in shared_users]).values_list('id', flat=True)
+#                 else:
+#                     related_step_ing += Ingredient.objects.filter(step__recipe=x, space=space).values_list('id', flat=True)
+
+#                 x_ing = []
+#                 if ingredients.filter(food__recipe=x).exists():
+#                     for ing in ingredients.filter(food__recipe=x):
+#                         if exclude_onhand:
+#                             x_ing = Ingredient.objects.filter(step__recipe=x,  food__ignore_shopping=False, space=space).exclude(food__onhand_users__id__in=[x.id for x in shared_users])
+#                         else:
+#                             x_ing = Ingredient.objects.filter(step__recipe=x,  food__ignore_shopping=False, space=space).exclude(food__ignore_shopping=True)
+#                         for i in [x for x in x_ing]:
+#                             ShoppingListEntry.objects.create(
+#                                 list_recipe=list_recipe,
+#                                 food=i.food,
+#                                 unit=i.unit,
+#                                 ingredient=i,
+#                                 amount=i.amount * Decimal(servings_factor),
+#                                 created_by=created_by,
+#                                 space=space,
+#                             )
+#                 # dont' add food to the shopping list that are actually recipes that will be added as ingredients
+#                 ingredients = ingredients.exclude(food__recipe=x)
+
+#     add_ingredients = list(ingredients.values_list('id', flat=True)) + related_step_ing
+#     if not append:
+#         existing_list = ShoppingListEntry.objects.filter(list_recipe=list_recipe)
+#         # delete shopping list entries not included in ingredients
+#         existing_list.exclude(ingredient__in=ingredients).delete()
+#         # add shopping list entries that did not previously exist
+#         add_ingredients = set(add_ingredients) - set(existing_list.values_list('ingredient__id', flat=True))
+#     add_ingredients = Ingredient.objects.filter(id__in=add_ingredients, space=space)
+
+#     # if servings have changed, update the ShoppingListRecipe and existing Entries
+#     if servings <= 0:
+#         servings = 1
+
+#     if not created and list_recipe.servings != servings:
+#         update_ingredients = set(ingredients.values_list('id', flat=True)) - set(add_ingredients.values_list('id', flat=True))
+#         list_recipe.servings = servings
+#         list_recipe.save()
+#         for sle in ShoppingListEntry.objects.filter(list_recipe=list_recipe, ingredient__id__in=update_ingredients):
+#             sle.amount = sle.ingredient.amount * Decimal(servings_factor)
+#             sle.save()
+
+#     # add any missing Entries
+#     for i in [x for x in add_ingredients if x.food]:
+
+#         ShoppingListEntry.objects.create(
+#             list_recipe=list_recipe,
+#             food=i.food,
+#             unit=i.unit,
+#             ingredient=i,
+#             amount=i.amount * Decimal(servings_factor),
+#             created_by=created_by,
+#             space=space,
+#         )
+
+#     # return all shopping list items
+#     return list_recipe

cookbook/helper/template_helper.py

@@ -1,10 +1,14 @@
+from gettext import gettext as _
+
 import bleach
 import markdown as md
 from bleach_allowlist import markdown_attrs, markdown_tags
+from jinja2 import Template, TemplateSyntaxError, UndefinedError
+from markdown.extensions.tables import TableExtension
+
 from cookbook.helper.mdx_attributes import MarkdownFormatExtension
 from cookbook.helper.mdx_urlize import UrlizeExtension
-from jinja2 import Template, TemplateSyntaxError, UndefinedError
-from gettext import gettext as _
+
 
 class IngredientObject(object):
     amount = ""
@@ -35,16 +39,16 @@ def render_instructions(step):  # TODO deduplicate markdown cleanup code
     instructions = step.instruction
 
     tags = markdown_tags + [
-        'pre', 'table', 'td', 'tr', 'th', 'tbody', 'style', 'thead'
+        'pre', 'table', 'td', 'tr', 'th', 'tbody', 'style', 'thead', 'img'
     ]
     parsed_md = md.markdown(
         instructions,
         extensions=[
-            'markdown.extensions.fenced_code', 'tables',
+            'markdown.extensions.fenced_code', TableExtension(),
             UrlizeExtension(), MarkdownFormatExtension()
         ]
     )
-    markdown_attrs['*'] = markdown_attrs['*'] + ['class']
+    markdown_attrs['*'] = markdown_attrs['*'] + ['class', 'width', 'height']
 
     instructions = bleach.clean(parsed_md, tags, markdown_attrs)
 

cookbook/integration/cheftap.py

@@ -1,17 +1,15 @@
 import re
 
-from django.utils.translation import gettext as _
-
-from cookbook.helper.ingredient_parser import parse, get_food, get_unit
+from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Food, Unit, Ingredient
+from cookbook.models import Ingredient, Recipe, Step
 
 
 class ChefTap(Integration):
 
     def import_file_name_filter(self, zip_info_object):
         print("testing", zip_info_object.filename)
-        return re.match(r'^cheftap_export/([A-Za-z\d\w\s-])+.txt$', zip_info_object.filename) or re.match(r'^([A-Za-z\d\w\s-])+.txt$', zip_info_object.filename)
+        return re.match(r'^cheftap_export/([A-Za-z\d\s\-_()\[\]\u00C0-\u017F])+.txt$', zip_info_object.filename) or re.match(r'^([A-Za-z\d\s\-_()\[\]\u00C0-\u017F])+.txt$', zip_info_object.filename)
 
     def get_recipe_from_file(self, file):
         source_url = ''
@@ -38,19 +36,20 @@ class ChefTap(Integration):
 
         recipe = Recipe.objects.create(name=title, created_by=self.request.user, internal=True, space=self.request.space, )
 
-        step = Step.objects.create(instruction='\n'.join(directions))
+        step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space,)
 
         if source_url != '':
             step.instruction += '\n' + source_url
             step.save()
 
+        ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
             if len(ingredient.strip()) > 0:
-                amount, unit, ingredient, note = parse(ingredient)
-                f = get_food(ingredient, self.request.space)
-                u = get_unit(unit, self.request.space)
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
+                f = ingredient_parser.get_food(food)
+                u = ingredient_parser.get_unit(unit)
                 step.ingredients.add(Ingredient.objects.create(
-                    food=f, unit=u, amount=amount, note=note
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                 ))
         recipe.steps.add(step)
 

cookbook/integration/chowdown.py

@@ -1,19 +1,18 @@
-import json
 import re
 from io import BytesIO
 from zipfile import ZipFile
 
 from cookbook.helper.image_processing import get_filetype
-from cookbook.helper.ingredient_parser import parse, get_food, get_unit
+from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Food, Unit, Ingredient, Keyword
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class Chowdown(Integration):
 
     def import_file_name_filter(self, zip_info_object):
         print("testing", zip_info_object.filename)
-        return re.match(r'^(_)*recipes/([A-Za-z\d\s-])+.md$', zip_info_object.filename)
+        return re.match(r'^(_)*recipes/([A-Za-z\d\s\-_()\[\]\u00C0-\u017F])+.md$', zip_info_object.filename)
 
     def get_recipe_from_file(self, file):
         ingredient_mode = False
@@ -51,20 +50,23 @@ class Chowdown(Integration):
         recipe = Recipe.objects.create(name=title, created_by=self.request.user, internal=True, space=self.request.space)
 
         for k in tags.split(','):
+            print(f'adding keyword {k.strip()}')
             keyword, created = Keyword.objects.get_or_create(name=k.strip(), space=self.request.space)
             recipe.keywords.add(keyword)
 
         step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n' + '\n'.join(descriptions)
+            instruction='\n'.join(directions) + '\n\n' + '\n'.join(descriptions), space=self.request.space,
         )
 
+        ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
-            amount, unit, ingredient, note = parse(ingredient)
-            f = get_food(ingredient, self.request.space)
-            u = get_unit(unit, self.request.space)
-            step.ingredients.add(Ingredient.objects.create(
-                food=f, unit=u, amount=amount, note=note
-            ))
+            if len(ingredient.strip()) > 0:
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
+                f = ingredient_parser.get_food(food)
+                u = ingredient_parser.get_unit(unit)
+                step.ingredients.add(Ingredient.objects.create(
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
+                ))
         recipe.steps.add(step)
 
         for f in self.files:

cookbook/integration/cookbookapp.py

@@ -0,0 +1,78 @@
+import base64
+import gzip
+import json
+import re
+from gettext import gettext as _
+from io import BytesIO
+
+import requests
+import validators
+import yaml
+
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import (get_from_scraper, get_images_from_soup,
+                                               iso_duration_to_minutes)
+from cookbook.helper.scrapers.scrapers import text_scraper
+from cookbook.integration.integration import Integration
+from cookbook.models import Ingredient, Keyword, Recipe, Step
+
+
+class CookBookApp(Integration):
+
+    def import_file_name_filter(self, zip_info_object):
+        return zip_info_object.filename.endswith('.html')
+
+    def get_recipe_from_file(self, file):
+        recipe_html = file.getvalue().decode("utf-8")
+
+        # recipe_json, recipe_tree, html_data, images = get_recipe_from_source(recipe_html, 'CookBookApp', self.request)
+        scrape = text_scraper(text=recipe_html)
+        recipe_json = get_from_scraper(scrape, self.request)
+        images = list(dict.fromkeys(get_images_from_soup(scrape.soup, None)))
+
+        recipe = Recipe.objects.create(
+            name=recipe_json['name'].strip(),
+            created_by=self.request.user, internal=True,
+            space=self.request.space)
+
+        try:
+            recipe.servings = re.findall('([0-9])+', recipe_json['recipeYield'])[0]
+        except Exception as e:
+            pass
+
+        try:
+            recipe.working_time = iso_duration_to_minutes(recipe_json['prepTime'])
+            recipe.waiting_time = iso_duration_to_minutes(recipe_json['cookTime'])
+        except Exception:
+            pass
+
+        # assuming import files only contain single step
+        step = Step.objects.create(instruction=recipe_json['steps'][0]['instruction'], space=self.request.space, )
+
+        if 'nutrition' in recipe_json:
+            step.instruction = step.instruction + '\n\n' + recipe_json['nutrition']
+
+        step.save()
+        recipe.steps.add(step)
+
+        ingredient_parser = IngredientParser(self.request, True)
+        for ingredient in recipe_json['steps'][0]['ingredients']:
+            f = ingredient_parser.get_food(ingredient['food']['name'])
+            u = None
+            if unit := ingredient.get('unit', None):
+                u = ingredient_parser.get_unit(unit.get('name', None))
+            step.ingredients.add(Ingredient.objects.create(
+                food=f, unit=u, amount=ingredient.get('amount', None), note=ingredient.get('note', None),  original_text=ingredient.get('original_text', None), space=self.request.space,
+            ))
+
+        if len(images) > 0:
+            try:
+                url = images[0]
+                if validators.url(url, public=True):
+                    response = requests.get(url)
+                    self.import_recipe_image(recipe, BytesIO(response.content))
+            except Exception as e:
+                print('failed to import image ', str(e))
+
+        recipe.save()
+        return recipe

cookbook/integration/cookmate.py

@@ -0,0 +1,88 @@
+import base64
+import json
+from io import BytesIO
+
+from gettext import gettext as _
+
+import requests
+import validators
+from lxml import etree
+
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_time, parse_servings_text
+from cookbook.integration.integration import Integration
+from cookbook.models import Ingredient, Keyword, Recipe, Step
+
+
+class Cookmate(Integration):
+
+    def import_file_name_filter(self, zip_info_object):
+        return zip_info_object.filename.endswith('.xml')
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        raise NotImplementedError('Method not implemented in storage integration')
+
+    def get_recipe_from_file(self, file):
+        recipe_xml = file
+
+        recipe = Recipe.objects.create(
+            name=recipe_xml.find('title').text.strip(),
+            created_by=self.request.user, internal=True, space=self.request.space)
+
+        if recipe_xml.find('preptime') is not None and recipe_xml.find('preptime').text is not None:
+            recipe.working_time = parse_time(recipe_xml.find('preptime').text.strip())
+
+        if recipe_xml.find('cooktime') is not None and recipe_xml.find('cooktime').text is not None:
+            recipe.waiting_time = parse_time(recipe_xml.find('cooktime').text.strip())
+
+        if recipe_xml.find('quantity') is not None and recipe_xml.find('quantity').text is not None:
+            recipe.servings = parse_servings(recipe_xml.find('quantity').text.strip())
+            recipe.servings_text = parse_servings_text(recipe_xml.find('quantity').text.strip())
+
+        if recipe_xml.find('url') is not None and recipe_xml.find('url').text is not None:
+            recipe.source_url = recipe_xml.find('url').text.strip()
+
+        if recipe_xml.find('description') is not None:  # description is a list of <li>'s with text
+            if len(recipe_xml.find('description')) > 0:
+                recipe.description = recipe_xml.find('description')[0].text[:512]
+
+        if recipe_text := recipe_xml.find('recipetext'):
+            for step in recipe_text.getchildren():
+                if step.text:
+                    step = Step.objects.create(
+                        instruction=step.text.strip(), space=self.request.space,
+                    )
+                    recipe.steps.add(step)
+
+        ingredient_parser = IngredientParser(self.request, True)
+
+        if recipe_ingredients := recipe_xml.find('ingredient'):
+            ingredient_step = recipe.steps.first()
+            if ingredient_step is None:
+                ingredient_step = Step.objects.create(space=self.request.space, instruction='')
+
+            for ingredient in recipe_ingredients.getchildren():
+                if ingredient.text:
+                    if ingredient.text.strip() != '':
+                        amount, unit, food, note = ingredient_parser.parse(ingredient.text.strip())
+                        f = ingredient_parser.get_food(food)
+                        u = ingredient_parser.get_unit(unit)
+                        ingredient_step.ingredients.add(Ingredient.objects.create(
+                            food=f, unit=u, amount=amount, note=note, original_text=ingredient.text.strip(), space=self.request.space,
+                        ))
+
+        if recipe_xml.find('imageurl') is not None:
+            try:
+                url = recipe_xml.find('imageurl').text.strip()
+                if validators.url(url, public=True):
+                    response = requests.get(url)
+                self.import_recipe_image(recipe, BytesIO(response.content))
+            except Exception as e:
+                print('failed to import image ', str(e))
+
+        recipe.save()
+
+        return recipe
+
+    def get_file_from_recipe(self, recipe):
+        raise NotImplementedError('Method not implemented in storage integration')

cookbook/integration/copymethat.py

@@ -0,0 +1,123 @@
+import re
+from io import BytesIO
+from zipfile import ZipFile
+
+from bs4 import BeautifulSoup, Tag
+from django.utils.translation import gettext as _
+
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import iso_duration_to_minutes, parse_servings
+from cookbook.integration.integration import Integration
+from cookbook.models import Ingredient, Keyword, Recipe, Step
+from recipes.settings import DEBUG
+
+
+class CopyMeThat(Integration):
+
+    def import_file_name_filter(self, zip_info_object):
+        if DEBUG:
+            print("testing", zip_info_object.filename, zip_info_object.filename == 'recipes.html')
+        return zip_info_object.filename == 'recipes.html'
+
+    def get_recipe_from_file(self, file):
+        # 'file' comes is as a beautifulsoup object
+        try:
+            source = file.find("a", {"id": "original_link"}).text
+        except AttributeError:
+            source = None
+
+        recipe = Recipe.objects.create(name=file.find("div", {"id": "name"}).text.strip()[:128], source_url=source, created_by=self.request.user, internal=True, space=self.request.space, )
+
+        for category in file.find_all("span", {"class": "recipeCategory"}):
+            keyword, created = Keyword.objects.get_or_create(name=category.text, space=self.request.space)
+            recipe.keywords.add(keyword)
+        
+        try:
+            recipe.servings = parse_servings(file.find("a", {"id": "recipeYield"}).text.strip())
+            recipe.working_time = iso_duration_to_minutes(file.find("span", {"meta": "prepTime"}).text.strip())
+            recipe.waiting_time = iso_duration_to_minutes(file.find("span", {"meta": "cookTime"}).text.strip())
+        except AttributeError:
+            pass
+
+        try:
+            if len(file.find("span", {"id": "starred"}).text.strip()) > 0:
+                recipe.keywords.add(Keyword.objects.get_or_create(space=self.request.space, name=_('Favorite'))[0])
+        except AttributeError:
+            pass
+
+        try:
+            if len(file.find("span", {"id": "made_this"}).text.strip()) > 0:
+                recipe.keywords.add(Keyword.objects.get_or_create(space=self.request.space, name=_('I made this'))[0])
+        except AttributeError:
+            pass
+
+        step = Step.objects.create(instruction='', space=self.request.space, )
+
+        ingredient_parser = IngredientParser(self.request, True)
+
+        ingredients = file.find("ul", {"id": "recipeIngredients"})
+        if isinstance(ingredients, Tag):
+            for ingredient in ingredients.children:
+                if not isinstance(ingredient, Tag) or not ingredient.text.strip() or "recipeIngredient_spacer" in ingredient['class']:
+                    continue
+                if any(x in ingredient['class'] for x in ["recipeIngredient_subheader", "recipeIngredient_note"]):
+                    step.ingredients.add(Ingredient.objects.create(is_header=True, note=ingredient.text.strip()[:256], original_text=ingredient.text.strip(), space=self.request.space, ))
+                else:
+                    amount, unit, food, note = ingredient_parser.parse(ingredient.text.strip())
+                    f = ingredient_parser.get_food(food)
+                    u = ingredient_parser.get_unit(unit)
+                    step.ingredients.add(Ingredient.objects.create(food=f, unit=u, amount=amount, note=note, original_text=ingredient.text.strip(), space=self.request.space, ))
+
+        instructions = file.find("ol", {"id": "recipeInstructions"})
+        if isinstance(instructions, Tag):
+            for instruction in instructions.children:
+                if not isinstance(instruction, Tag) or instruction.text == "":
+                    continue
+                if "instruction_subheader" in instruction['class']:
+                    if step.instruction:
+                        step.save()
+                        recipe.steps.add(step)
+                        step = Step.objects.create(instruction='', space=self.request.space, )
+                    
+                    step.name = instruction.text.strip()[:128]
+                else:
+                    step.instruction += instruction.text.strip() + ' \n\n'
+
+        notes = file.find_all("li", {"class": "recipeNote"})
+        if notes:
+            step.instruction += '*Notes:* \n\n'
+
+            for n in notes:
+                if n.text == "":
+                    continue
+                step.instruction += '*' + n.text.strip() + '* \n\n'
+
+        description = ''
+        try:
+            description = file.find("div", {"id": "description"}).text.strip()
+        except AttributeError:
+            pass
+        if len(description) <= 512:
+            recipe.description = description
+        else:
+            recipe.description = description[:480] + ' ... (full description below)'
+            step.instruction += '*Description:* \n\n*' + description + '* \n\n'
+
+        step.save()
+        recipe.steps.add(step)
+
+        # import the Primary recipe image that is stored in the Zip
+        try:
+            for f in self.files:
+                if '.zip' in f['name']:
+                    import_zip = ZipFile(f['file'])
+                    self.import_recipe_image(recipe, BytesIO(import_zip.read(file.find("img", class_="recipeImage").get("src"))), filetype='.jpeg')
+        except Exception as e:
+            print(recipe.name, ': failed to import image ', str(e))
+
+        recipe.save()
+        return recipe
+
+    def split_recipe_file(self, file):
+        soup = BeautifulSoup(file, "html.parser")
+        return soup.find_all("div", {"class": "recipe"})

cookbook/integration/default.py

@@ -1,5 +1,5 @@
 import json
-from io import BytesIO
+from io import BytesIO, StringIO
 from re import match
 from zipfile import ZipFile
 
@@ -32,6 +32,39 @@ class Default(Integration):
         return None
 
     def get_file_from_recipe(self, recipe):
+
         export = RecipeExportSerializer(recipe).data
 
         return 'recipe.json', JSONRenderer().render(export).decode("utf-8")
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        export_zip_stream = BytesIO()
+        export_zip_obj = ZipFile(export_zip_stream, 'w')
+
+        for r in recipes:
+            if r.internal and r.space == self.request.space:
+                recipe_zip_stream = BytesIO()
+                recipe_zip_obj = ZipFile(recipe_zip_stream, 'w')
+
+                recipe_stream = StringIO()
+                filename, data = self.get_file_from_recipe(r)
+                recipe_stream.write(data)
+                recipe_zip_obj.writestr(filename, recipe_stream.getvalue())
+                recipe_stream.close()
+
+                try:
+                    recipe_zip_obj.writestr(f'image{get_filetype(r.image.file.name)}', r.image.file.read())
+                except ValueError:
+                    pass
+
+                recipe_zip_obj.close()
+
+                export_zip_obj.writestr(str(r.pk) + '.zip', recipe_zip_stream.getvalue())
+
+            el.exported_recipes += 1
+            el.msg += self.get_recipe_processed_msg(r)
+            el.save()
+
+        export_zip_obj.close()
+
+        return [[ self.get_export_file_name(), export_zip_stream.getvalue() ]]

cookbook/integration/domestica.py

@@ -2,9 +2,9 @@ import base64
 import json
 from io import BytesIO
 
-from cookbook.helper.ingredient_parser import parse, get_food, get_unit
+from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient
+from cookbook.models import Ingredient, Recipe, Step
 
 
 class Domestica(Integration):
@@ -28,19 +28,20 @@ class Domestica(Integration):
         recipe.save()
 
         step = Step.objects.create(
-            instruction=file['directions']
+            instruction=file['directions'], space=self.request.space,
         )
 
         if file['source'] != '':
             step.instruction += '\n' + file['source']
 
+        ingredient_parser = IngredientParser(self.request, True)
         for ingredient in file['ingredients'].split('\n'):
             if len(ingredient.strip()) > 0:
-                amount, unit, ingredient, note = parse(ingredient)
-                f = get_food(ingredient, self.request.space)
-                u = get_unit(unit, self.request.space)
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
+                f = ingredient_parser.get_food(food)
+                u = ingredient_parser.get_unit(unit)
                 step.ingredients.add(Ingredient.objects.create(
-                    food=f, unit=u, amount=amount, note=note
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                 ))
         recipe.steps.add(step)
 

cookbook/integration/integration.py

@@ -1,20 +1,30 @@
+import time
 import datetime
 import json
-import os
-import re
+import traceback
 import uuid
 from io import BytesIO, StringIO
-from zipfile import ZipFile, BadZipFile
+from zipfile import BadZipFile, ZipFile
 
+import lxml
+from django.core.cache import cache
+import datetime
+
+from bs4 import Tag
+from django.core.exceptions import ObjectDoesNotExist
 from django.core.files import File
+from django.db import IntegrityError
 from django.http import HttpResponse
 from django.utils.formats import date_format
 from django.utils.translation import gettext as _
 from django_scopes import scope
+from lxml import etree
 
 from cookbook.forms import ImportExportBase
-from cookbook.helper.image_processing import get_filetype
+from cookbook.helper.image_processing import get_filetype, handle_image
 from cookbook.models import Keyword, Recipe
+from recipes.settings import DEBUG
+from recipes.settings import EXPORT_FILE_CACHE_DURATION
 
 
 class Integration:
@@ -31,56 +41,70 @@ class Integration:
         """
         self.request = request
         self.export_type = export_type
-        self.keyword = Keyword.objects.create(
-            name=f'Import {export_type} {date_format(datetime.datetime.now(), "DATETIME_FORMAT")}.{datetime.datetime.now().strftime("%S")}',
-            description=f'Imported by {request.user.get_user_name()} at {date_format(datetime.datetime.now(), "DATETIME_FORMAT")}. Type: {export_type}',
-            icon='📥',
-            space=request.space
-        )
+        self.ignored_recipes = []
 
-    def do_export(self, recipes):
-        """
-        Perform the export based on a list of recipes
-        :param recipes: list of recipe objects
-        :return: HttpResponse with a ZIP file that is directly downloaded
-        """
+        description = f'Imported by {request.user.get_user_display_name()} at {date_format(datetime.datetime.now(), "DATETIME_FORMAT")}. Type: {export_type}'
+        icon = '📥'
 
-        # TODO this is temporary, find a better solution for different export formats when doing other exporters
-        if self.export_type != ImportExportBase.RECIPESAGE:
-            export_zip_stream = BytesIO()
-            export_zip_obj = ZipFile(export_zip_stream, 'w')
+        try:
+            last_kw = Keyword.objects.filter(name__regex=r'^(Import [0-9]+)', space=request.space).latest('created_at')
+            name = f'Import {int(last_kw.name.replace("Import ", "")) + 1}'
+        except (ObjectDoesNotExist, ValueError):
+            name = 'Import 1'
 
-            for r in recipes:
-                if r.internal and r.space == self.request.space:
-                    recipe_zip_stream = BytesIO()
-                    recipe_zip_obj = ZipFile(recipe_zip_stream, 'w')
+        parent, created = Keyword.objects.get_or_create(name='Import', space=request.space)
+        try:
+            self.keyword = parent.add_child(
+                name=name,
+                description=description,
+                icon=icon,
+                space=request.space
+            )
+        except (IntegrityError, ValueError):  # in case, for whatever reason, the name does exist append UUID to it. Not nice but works for now.
+            self.keyword = parent.add_child(
+                name=f'{name} {str(uuid.uuid4())[0:8]}',
+                description=description,
+                icon=icon,
+                space=request.space
+            )
 
-                    recipe_stream = StringIO()
-                    filename, data = self.get_file_from_recipe(r)
-                    recipe_stream.write(data)
-                    recipe_zip_obj.writestr(filename, recipe_stream.getvalue())
-                    recipe_stream.close()
-                    try:
-                        recipe_zip_obj.writestr(f'image{get_filetype(r.image.file.name)}', r.image.file.read())
-                    except ValueError:
-                        pass
 
-                    recipe_zip_obj.close()
-                    export_zip_obj.writestr(str(r.pk) + '.zip', recipe_zip_stream.getvalue())
 
-            export_zip_obj.close()
+    def do_export(self, recipes, el):
 
-            response = HttpResponse(export_zip_stream.getvalue(), content_type='application/force-download')
-            response['Content-Disposition'] = 'attachment; filename="export.zip"'
-            return response
-        else:
-            json_list = []
-            for r in recipes:
-                json_list.append(self.get_file_from_recipe(r))
+        with scope(space=self.request.space):
+            el.total_recipes = len(recipes) 
+            el.cache_duration = EXPORT_FILE_CACHE_DURATION
+            el.save()
+
+            files = self.get_files_from_recipes(recipes, el, self.request.COOKIES)
+
+            if len(files) == 1:
+                filename, file = files[0]
+                export_filename = filename
+                export_file = file
+
+            else:
+                #zip the files if there is more then one file
+                export_filename = self.get_export_file_name()
+                export_stream = BytesIO()
+                export_obj = ZipFile(export_stream, 'w')
+
+                for filename, file in files:
+                    export_obj.writestr(filename, file)
+
+                export_obj.close()
+                export_file = export_stream.getvalue()
+
+
+            cache.set('export_file_'+str(el.pk), {'filename': export_filename, 'file': export_file}, EXPORT_FILE_CACHE_DURATION)
+            el.running = False
+            el.save()
+
+        response = HttpResponse(export_file, content_type='application/force-download')
+        response['Content-Disposition'] = 'attachment; filename="' + export_filename + '"'
+        return response
 
-            response = HttpResponse(json.dumps(json_list), content_type='application/force-download')
-            response['Content-Disposition'] = 'attachment; filename="recipes.json"'
-            return response
 
     def import_file_name_filter(self, zip_info_object):
         """
@@ -101,70 +125,106 @@ class Integration:
         :return: HttpResponseRedirect to the recipe search showing all imported recipes
         """
         with scope(space=self.request.space):
-            self.keyword.name = _('Import') + ' ' + str(il.pk)
-            self.keyword.save()
 
             try:
                 self.files = files
                 for f in files:
                     if 'RecipeKeeper' in f['name']:
                         import_zip = ZipFile(f['file'])
+                        file_list = []
                         for z in import_zip.filelist:
                             if self.import_file_name_filter(z):
-                                data_list = self.split_recipe_file(import_zip.read(z.filename).decode('utf-8'))
-                                for d in data_list:
-                                    recipe = self.get_recipe_from_file(d)
-                                    recipe.keywords.add(self.keyword)
-                                    il.msg += f'{recipe.pk} - {recipe.name} \n'
-                                    self.handle_duplicates(recipe, import_duplicates)
+                                file_list.append(z)
+                        il.total_recipes += len(file_list)
+
+                        for z in file_list:
+                            data_list = self.split_recipe_file(import_zip.read(z.filename).decode('utf-8'))
+                            for d in data_list:
+                                recipe = self.get_recipe_from_file(d)
+                                recipe.keywords.add(self.keyword)
+                                il.msg += self.get_recipe_processed_msg(recipe)
+                                self.handle_duplicates(recipe, import_duplicates)
+                                il.imported_recipes += 1
+                                il.save()
                         import_zip.close()
-                    elif '.zip' in f['name'] or '.paprikarecipes' in f['name']:
+                    elif '.zip' in f['name'] or '.paprikarecipes' in f['name'] or '.mcb' in f['name']:
                         import_zip = ZipFile(f['file'])
+                        file_list = []
                         for z in import_zip.filelist:
                             if self.import_file_name_filter(z):
-                                try:
+                                file_list.append(z)
+                        il.total_recipes += len(file_list)
+
+                        import cookbook
+                        if isinstance(self, cookbook.integration.copymethat.CopyMeThat):
+                            file_list = self.split_recipe_file(BytesIO(import_zip.read('recipes.html')))
+                            il.total_recipes += len(file_list)
+
+                        if isinstance(self, cookbook.integration.cookmate.Cookmate):
+                            new_file_list = []
+                            for file in file_list:
+                                new_file_list += etree.parse(BytesIO(import_zip.read(file.filename))).getroot().getchildren()
+                            il.total_recipes = len(new_file_list)
+                            file_list = new_file_list
+
+                        for z in file_list:
+                            try:
+                                if not hasattr(z, 'filename') or type(z) == Tag:
+                                    recipe = self.get_recipe_from_file(z)
+                                else:
                                     recipe = self.get_recipe_from_file(BytesIO(import_zip.read(z.filename)))
-                                    recipe.keywords.add(self.keyword)
-                                    il.msg += f'{recipe.pk} - {recipe.name} \n'
-                                    self.handle_duplicates(recipe, import_duplicates)
-                                except Exception as e:
-                                    il.msg += f'-------------------- \n ERROR \n{e}\n--------------------\n'
+                                recipe.keywords.add(self.keyword)
+                                il.msg += self.get_recipe_processed_msg(recipe)
+                                self.handle_duplicates(recipe, import_duplicates)
+                                il.imported_recipes += 1
+                                il.save()
+                            except Exception as e:
+                                traceback.print_exc()
+                                self.handle_exception(e, log=il, message=f'-------------------- \nERROR \n{e}\n--------------------\n')
                         import_zip.close()
-                    elif '.json' in f['name'] or '.txt' in f['name']:
+                    elif '.json' in f['name'] or '.txt' in f['name'] or '.mmf' in f['name'] or '.rk' in f['name'] or '.melarecipe' in f['name']:
                         data_list = self.split_recipe_file(f['file'])
+                        il.total_recipes += len(data_list)
                         for d in data_list:
                             try:
                                 recipe = self.get_recipe_from_file(d)
                                 recipe.keywords.add(self.keyword)
-                                il.msg += f'{recipe.pk} - {recipe.name} \n'
+                                il.msg += self.get_recipe_processed_msg(recipe)
                                 self.handle_duplicates(recipe, import_duplicates)
+                                il.imported_recipes += 1
+                                il.save()
                             except Exception as e:
-                                il.msg += f'-------------------- \n ERROR \n{e}\n--------------------\n'
+                                self.handle_exception(e, log=il, message=f'-------------------- \nERROR \n{e}\n--------------------\n')
                     elif '.rtk' in f['name']:
                         import_zip = ZipFile(f['file'])
                         for z in import_zip.filelist:
                             if self.import_file_name_filter(z):
                                 data_list = self.split_recipe_file(import_zip.read(z.filename).decode('utf-8'))
+                                il.total_recipes += len(data_list)
+
                                 for d in data_list:
                                     try:
                                         recipe = self.get_recipe_from_file(d)
                                         recipe.keywords.add(self.keyword)
-                                        il.msg += f'{recipe.pk} - {recipe.name} \n'
+                                        il.msg += self.get_recipe_processed_msg(recipe)
                                         self.handle_duplicates(recipe, import_duplicates)
+                                        il.imported_recipes += 1
+                                        il.save()
                                     except Exception as e:
-                                        il.msg += f'-------------------- \n ERROR \n{e}\n--------------------\n'
+                                        self.handle_exception(e, log=il, message=f'-------------------- \nERROR \n{e}\n--------------------\n')
                         import_zip.close()
                     else:
                         recipe = self.get_recipe_from_file(f['file'])
                         recipe.keywords.add(self.keyword)
-                        il.msg += f'{recipe.pk} - {recipe.name} \n'
+                        il.msg += self.get_recipe_processed_msg(recipe)
                         self.handle_duplicates(recipe, import_duplicates)
             except BadZipFile:
                 il.msg += 'ERROR ' + _(
                     'Importer expected a .zip file. Did you choose the correct importer type for your data ?') + '\n'
-            except:
-                il.msg += 'ERROR ' + _(
+            except Exception as e:
+                msg = 'ERROR ' + _(
                     'An unexpected error occurred during the import. Please make sure you have uploaded a valid file.') + '\n'
+                self.handle_exception(e, log=il, message=msg)
 
             if len(self.ignored_recipes) > 0:
                 il.msg += '\n' + _(
@@ -183,18 +243,17 @@ class Integration:
         :param import_duplicates: if duplicates should be imported
         """
         if Recipe.objects.filter(space=self.request.space, name=recipe.name).count() > 1 and not import_duplicates:
-            recipe.delete()
             self.ignored_recipes.append(recipe.name)
+            recipe.delete()
 
-    @staticmethod
-    def import_recipe_image(recipe, image_file, filetype='.jpeg'):
+    def import_recipe_image(self, recipe, image_file, filetype='.jpeg'):
         """
         Adds an image to a recipe naming it correctly
         :param recipe: Recipe object
         :param image_file: ByteIO stream containing the image
         :param filetype: type of file to write bytes to, default to .jpeg if unknown
         """
-        recipe.image = File(image_file, name=f'{uuid.uuid4()}_{recipe.pk}{filetype}')
+        recipe.image = File(handle_image(self.request, File(image_file, name='image'), filetype=filetype), name=f'{uuid.uuid4()}_{recipe.pk}{filetype}')
         recipe.save()
 
     def get_recipe_from_file(self, file):
@@ -223,3 +282,30 @@ class Integration:
             - data - string content for file to get created in export zip
         """
         raise NotImplementedError('Method not implemented in integration')
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        """
+        Takes a list of recipe object and converts it to a array containing each file.
+        Each file is represented as an array [filename, data] where data is a string of the content of the file.
+        :param recipe: Recipe object that should be converted
+        :returns:
+            [[filename, data], ...]
+        """
+        raise NotImplementedError('Method not implemented in integration')
+
+    @staticmethod
+    def handle_exception(exception, log=None, message=''):
+        if log:
+            if message:
+                log.msg += message
+            else:
+                log.msg += exception.msg
+        if DEBUG:
+            traceback.print_exc()
+
+
+    def get_export_file_name(self, format='zip'):
+        return "export_{}.{}".format(datetime.datetime.now().strftime("%Y-%m-%d"), format)
+
+    def get_recipe_processed_msg(self, recipe):
+        return f'{recipe.pk} - {recipe.name} \n'

cookbook/integration/mealie.py

@@ -4,15 +4,15 @@ from io import BytesIO
 from zipfile import ZipFile
 
 from cookbook.helper.image_processing import get_filetype
-from cookbook.helper.ingredient_parser import parse, get_food, get_unit
+from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Food, Unit, Ingredient
+from cookbook.models import Ingredient, Recipe, Step
 
 
 class Mealie(Integration):
 
     def import_file_name_filter(self, zip_info_object):
-        return re.match(r'^recipes/([A-Za-z\d-])+.json$', zip_info_object.filename)
+        return re.match(r'^recipes/([A-Za-z\d\s\-_()\[\]\u00C0-\u017F])+/([A-Za-z\d\s\-_()\[\]\u00C0-\u017F])+.json$', zip_info_object.filename)
 
     def get_recipe_from_file(self, file):
         recipe_json = json.loads(file.getvalue().decode("utf-8"))
@@ -24,11 +24,12 @@ class Mealie(Integration):
             created_by=self.request.user, internal=True, space=self.request.space)
 
         # TODO parse times (given in PT2H3M )
+        # @vabene check recipe_url_import.iso_duration_to_minutes  I think it does what you are looking for
 
         ingredients_added = False
-        for s in recipe_json['recipeInstructions']:
+        for s in recipe_json['recipe_instructions']:
             step = Step.objects.create(
-                instruction=s['text']
+                instruction=s['text'], space=self.request.space,
             )
             if not ingredients_added:
                 ingredients_added = True
@@ -36,21 +37,35 @@ class Mealie(Integration):
                 if len(recipe_json['description'].strip()) > 500:
                     step.instruction = recipe_json['description'].strip() + '\n\n' + step.instruction
 
-                for ingredient in recipe_json['recipeIngredient']:
-                    amount, unit, ingredient, note = parse(ingredient)
-                    f = get_food(ingredient, self.request.space)
-                    u = get_unit(unit, self.request.space)
-                    step.ingredients.add(Ingredient.objects.create(
-                        food=f, unit=u, amount=amount, note=note
-                    ))
+                ingredient_parser = IngredientParser(self.request, True)
+                for ingredient in recipe_json['recipe_ingredient']:
+                    try:
+                        if ingredient['food']:
+                            f = ingredient_parser.get_food(ingredient['food'])
+                            u = ingredient_parser.get_unit(ingredient['unit'])
+                            amount = ingredient['quantity']
+                            note = ingredient['note']
+                            original_text = None
+                        else:
+                            amount, unit, food, note = ingredient_parser.parse(ingredient['note'])
+                            f = ingredient_parser.get_food(food)
+                            u = ingredient_parser.get_unit(unit)
+                            original_text = ingredient['note']
+                        step.ingredients.add(Ingredient.objects.create(
+                            food=f, unit=u, amount=amount, note=note, original_text=original_text, space=self.request.space,
+                        ))
+                    except Exception:
+                        pass
             recipe.steps.add(step)
 
         for f in self.files:
             if '.zip' in f['name']:
                 import_zip = ZipFile(f['file'])
-                for z in import_zip.filelist:
-                    if re.match(f'^images/{recipe_json["slug"]}.jpg$', z.filename):
-                        self.import_recipe_image(recipe, BytesIO(import_zip.read(z.filename)), filetype=get_filetype(z.filename))
+                try:
+                    self.import_recipe_image(recipe, BytesIO(import_zip.read(f'recipes/{recipe_json["slug"]}/images/min-original.webp')),
+                                             filetype=get_filetype(f'recipes/{recipe_json["slug"]}/images/original'))
+                except Exception:
+                    pass
 
         return recipe
 

cookbook/integration/mealmaster.py

@@ -1,22 +1,17 @@
-import json
 import re
-from io import BytesIO
-from zipfile import ZipFile
 
-from cookbook.helper.ingredient_parser import parse, get_food, get_unit
+from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Food, Unit, Ingredient, Keyword
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class MealMaster(Integration):
 
     def get_recipe_from_file(self, file):
-        print('------------ getting recipe')
         servings = 1
         ingredients = []
         directions = []
         for line in file.replace('\r', '').split('\n'):
-            print('testing line')
             if not line.startswith('MMMMM') and line.strip != '':
                 if 'Title:' in line:
                     title = line.replace('Title:', '').strip()
@@ -44,16 +39,17 @@ class MealMaster(Integration):
             recipe.keywords.add(keyword)
 
         step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n'
+            instruction='\n'.join(directions) + '\n\n', space=self.request.space,
         )
 
+        ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
             if len(ingredient.strip()) > 0:
-                amount, unit, ingredient, note = parse(ingredient)
-                f = get_food(ingredient, self.request.space)
-                u = get_unit(unit, self.request.space)
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
+                f = ingredient_parser.get_food(ingredient)
+                u = ingredient_parser.get_unit(unit)
                 step.ingredients.add(Ingredient.objects.create(
-                    food=f, unit=u, amount=amount, note=note
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                 ))
         recipe.steps.add(step)
 
@@ -67,7 +63,7 @@ class MealMaster(Integration):
         current_recipe = ''
 
         for fl in file.readlines():
-            line = fl.decode("ANSI")
+            line = fl.decode("windows-1250")
             if (line.startswith('MMMMM') or line.startswith('-----')) and 'meal-master' in line.lower():
                 if current_recipe != '':
                     recipe_list.append(current_recipe)

cookbook/integration/melarecipes.py

@@ -0,0 +1,83 @@
+import base64
+import json
+from io import BytesIO
+
+from gettext import gettext as _
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_time
+from cookbook.integration.integration import Integration
+from cookbook.models import Ingredient, Keyword, Recipe, Step
+
+
+class MelaRecipes(Integration):
+
+    def split_recipe_file(self, file):
+        return [json.loads(file.getvalue().decode("utf-8"))]
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        raise NotImplementedError('Method not implemented in storage integration')
+
+    def get_recipe_from_file(self, file):
+        recipe_json = file
+
+        recipe = Recipe.objects.create(
+            name=recipe_json['title'].strip(),
+            created_by=self.request.user, internal=True, space=self.request.space)
+
+        if 'yield' in recipe_json:
+            recipe.servings = parse_servings(recipe_json['yield'])
+
+        if 'cookTime' in recipe_json:
+            recipe.waiting_time = parse_time(recipe_json['cookTime'])
+
+        if 'prepTime' in recipe_json:
+            recipe.working_time = parse_time(recipe_json['prepTime'])
+
+        if 'favorite' in recipe_json and recipe_json['favorite']:
+            recipe.keywords.add(Keyword.objects.get_or_create(space=self.request.space, name=_('Favorite'))[0])
+
+        if 'categories' in recipe_json:
+            try:
+                for x in recipe_json['categories']:
+                    recipe.keywords.add(Keyword.objects.get_or_create(space=self.request.space, name=x)[0])
+            except Exception:
+                pass
+
+        instruction = ''
+        if 'text' in recipe_json:
+            instruction += f'*{recipe_json["text"].strip()}*  \n'
+
+        if 'instructions' in recipe_json:
+            instruction += recipe_json["instructions"].strip() + '  \n'
+
+        if 'notes' in recipe_json:
+            instruction += recipe_json["notes"].strip() + '  \n'
+
+        if 'link' in recipe_json:
+            recipe.source_url = recipe_json['link']
+
+        step = Step.objects.create(
+            instruction=instruction, space=self.request.space,
+        )
+
+        ingredient_parser = IngredientParser(self.request, True)
+        for ingredient in recipe_json['ingredients'].split('\n'):
+            if ingredient.strip() != '':
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
+                f = ingredient_parser.get_food(food)
+                u = ingredient_parser.get_unit(unit)
+                step.ingredients.add(Ingredient.objects.create(
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
+                ))
+        recipe.steps.add(step)
+
+        if recipe_json.get("images", None):
+            try:
+                self.import_recipe_image(recipe, BytesIO(base64.b64decode(recipe_json['images'][0])), filetype='.jpeg')
+            except Exception:
+                pass
+
+        return recipe
+
+    def get_file_from_recipe(self, recipe):
+        raise NotImplementedError('Method not implemented in storage integration')

cookbook/integration/nextcloud_cookbook.py

@@ -4,9 +4,10 @@ from io import BytesIO
 from zipfile import ZipFile
 
 from cookbook.helper.image_processing import get_filetype
-from cookbook.helper.ingredient_parser import parse, get_food, get_unit
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import iso_duration_to_minutes
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Food, Unit, Ingredient
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class NextcloudCookbook(Integration):
@@ -24,13 +25,33 @@ class NextcloudCookbook(Integration):
             created_by=self.request.user, internal=True,
             servings=recipe_json['recipeYield'], space=self.request.space)
 
-        # TODO parse times (given in PT2H3M )
-        # TODO parse keywords
+        try:
+            recipe.working_time = iso_duration_to_minutes(recipe_json['prepTime'])
+            recipe.waiting_time = iso_duration_to_minutes(recipe_json['cookTime'])
+        except Exception:
+            pass
+
+        if 'url' in recipe_json:
+            recipe.source_url = recipe_json['url'].strip()
+
+        if 'recipeCategory' in recipe_json:
+            try:
+                recipe.keywords.add(Keyword.objects.get_or_create(space=self.request.space, name=recipe_json['recipeCategory'])[0])
+            except Exception:
+                pass
+
+        if 'keywords' in recipe_json:
+            try:
+                for x in recipe_json['keywords'].split(','):
+                    if x.strip() != '':
+                        recipe.keywords.add(Keyword.objects.get_or_create(space=self.request.space, name=x)[0])
+            except Exception:
+                pass
 
         ingredients_added = False
         for s in recipe_json['recipeInstructions']:
             step = Step.objects.create(
-                instruction=s
+                instruction=s, space=self.request.space,
             )
             if not ingredients_added:
                 if len(recipe_json['description'].strip()) > 500:
@@ -38,20 +59,30 @@ class NextcloudCookbook(Integration):
 
                 ingredients_added = True
 
+                ingredient_parser = IngredientParser(self.request, True)
                 for ingredient in recipe_json['recipeIngredient']:
-                    amount, unit, ingredient, note = parse(ingredient)
-                    f = get_food(ingredient, self.request.space)
-                    u = get_unit(unit, self.request.space)
+                    amount, unit, food, note = ingredient_parser.parse(ingredient)
+                    f = ingredient_parser.get_food(food)
+                    u = ingredient_parser.get_unit(unit)
                     step.ingredients.add(Ingredient.objects.create(
-                        food=f, unit=u, amount=amount, note=note
+                        food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                     ))
             recipe.steps.add(step)
 
+        if 'nutrition' in recipe_json:
+            try:
+                recipe.nutrition.calories = recipe_json['nutrition']['calories'].replace(' kcal', '').replace(' ', '')
+                recipe.nutrition.proteins = recipe_json['nutrition']['calories'].replace(' g', '').replace(',', '.').replace(' ', '')
+                recipe.nutrition.fats = recipe_json['nutrition']['calories'].replace(' g', '').replace(',', '.').replace(' ', '')
+                recipe.nutrition.carbohydrates = recipe_json['nutrition']['calories'].replace(' g', '').replace(',', '.').replace(' ', '')
+            except Exception:
+                pass
+
         for f in self.files:
             if '.zip' in f['name']:
                 import_zip = ZipFile(f['file'])
                 for z in import_zip.filelist:
-                    if re.match(f'^Recipes/{recipe.name}/full.jpg$', z.filename):
+                    if re.match(f'^(.)+{recipe.name}/full.jpg$', z.filename):
                         self.import_recipe_image(recipe, BytesIO(import_zip.read(z.filename)), filetype=get_filetype(z.filename))
 
         return recipe

cookbook/integration/openeats.py

@@ -1,11 +1,8 @@
 import json
-import re
 
-from django.utils.translation import gettext as _
-
-from cookbook.helper.ingredient_parser import parse, get_food, get_unit
+from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Food, Unit, Ingredient
+from cookbook.models import Ingredient, Recipe, Step
 
 
 class OpenEats(Integration):
@@ -24,13 +21,14 @@ class OpenEats(Integration):
         if file["source"] != '':
             instructions += file["source"]
 
-        step = Step.objects.create(instruction=instructions)
+        step = Step.objects.create(instruction=instructions, space=self.request.space,)
 
+        ingredient_parser = IngredientParser(self.request, True)
         for ingredient in file['ingredients']:
-            f = get_food(ingredient['food'], self.request.space)
-            u = get_unit(ingredient['unit'], self.request.space)
+            f = ingredient_parser.get_food(ingredient['food'])
+            u = ingredient_parser.get_unit(ingredient['unit'])
             step.ingredients.add(Ingredient.objects.create(
-                food=f, unit=u, amount=ingredient['amount']
+                food=f, unit=u, amount=ingredient['amount'], space=self.request.space,
             ))
         recipe.steps.add(step)
 

cookbook/integration/paprika.py

@@ -2,12 +2,13 @@ import base64
 import gzip
 import json
 import re
+from gettext import gettext as _
 from io import BytesIO
 
-from cookbook.helper.ingredient_parser import parse, get_food, get_unit
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient, Keyword
-from gettext import gettext as _
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class Paprika(Integration):
@@ -16,7 +17,7 @@ class Paprika(Integration):
         raise NotImplementedError('Method not implemented in storage integration')
 
     def get_recipe_from_file(self, file):
-        with  gzip.open(file, 'r') as recipe_zip:
+        with gzip.open(file, 'r') as recipe_zip:
             recipe_json = json.loads(recipe_zip.read().decode("utf-8"))
 
             recipe = Recipe.objects.create(
@@ -26,10 +27,9 @@ class Paprika(Integration):
                 recipe.description = '' if len(recipe_json['description'].strip()) > 500 else recipe_json['description'].strip()
 
             try:
-                if re.match(r'([0-9])+\s(.)*', recipe_json['servings']):
-                    s = recipe_json['servings'].split(' ')
-                    recipe.servings = s[0]
-                    recipe.servings_text = s[1]
+                if 'servings' in recipe_json:
+                    recipe.servings = parse_servings(recipe_json['servings'])
+                    recipe.servings_text = parse_servings_text(recipe_json['servings'])
 
                 if len(recipe_json['cook_time'].strip()) > 0:
                     recipe.waiting_time = re.findall(r'\d+', recipe_json['cook_time'])[0]
@@ -55,10 +55,10 @@ class Paprika(Integration):
                 pass
 
             step = Step.objects.create(
-                instruction=instructions
+                instruction=instructions, space=self.request.space,
             )
 
-            if len(recipe_json['description'].strip()) > 500:
+            if 'description' in recipe_json and len(recipe_json['description'].strip()) > 500:
                 step.instruction = recipe_json['description'].strip() + '\n\n' + step.instruction
 
             if 'categories' in recipe_json:
@@ -66,14 +66,15 @@ class Paprika(Integration):
                     keyword, created = Keyword.objects.get_or_create(name=c.strip(), space=self.request.space)
                     recipe.keywords.add(keyword)
 
+            ingredient_parser = IngredientParser(self.request, True)
             try:
                 for ingredient in recipe_json['ingredients'].split('\n'):
                     if len(ingredient.strip()) > 0:
-                        amount, unit, ingredient, note = parse(ingredient)
-                        f = get_food(ingredient, self.request.space)
-                        u = get_unit(unit, self.request.space)
+                        amount, unit, food, note = ingredient_parser.parse(ingredient)
+                        f = ingredient_parser.get_food(food)
+                        u = ingredient_parser.get_unit(unit)
                         step.ingredients.add(Ingredient.objects.create(
-                            food=f, unit=u, amount=amount, note=note
+                            food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                         ))
             except AttributeError:
                 pass

cookbook/integration/pdfexport.py

@@ -0,0 +1,69 @@
+import json
+from io import BytesIO
+from re import match
+from zipfile import ZipFile
+import asyncio
+from pyppeteer import launch
+
+from rest_framework.renderers import JSONRenderer
+
+from cookbook.helper.image_processing import get_filetype
+from cookbook.integration.integration import Integration
+from cookbook.serializer import RecipeExportSerializer
+
+from cookbook.models import ExportLog
+from asgiref.sync import sync_to_async
+
+import django.core.management.commands.runserver as runserver
+import logging
+
+class PDFexport(Integration):
+
+    def get_recipe_from_file(self, file):
+        raise NotImplementedError('Method not implemented in storage integration')
+
+    async def get_files_from_recipes_async(self, recipes, el, cookie):
+        cmd = runserver.Command()
+
+        browser = await launch(
+            handleSIGINT=False,
+            handleSIGTERM=False,
+            handleSIGHUP=False,
+            ignoreHTTPSErrors=True,
+        )
+
+        cookies = {'domain': cmd.default_addr, 'name': 'sessionid', 'value': cookie['sessionid'], }
+        options = {'format': 'letter',
+                   'margin': {
+                       'top': '0.75in',
+                       'bottom': '0.75in',
+                       'left': '0.75in',
+                       'right': '0.75in',
+                   }
+                   }
+
+
+        files = []
+        for recipe in recipes:
+
+            page = await browser.newPage()
+            await page.emulateMedia('print')
+            await page.setCookie(cookies)
+
+            await page.goto('http://'+cmd.default_addr+':'+cmd.default_port+'/view/recipe/'+str(recipe.id), {'waitUntil': 'domcontentloaded'})
+            await page.waitForSelector('#printReady');
+
+            files.append([recipe.name + '.pdf', await page.pdf(options)])
+            await page.close();
+
+            el.exported_recipes += 1
+            el.msg += self.get_recipe_processed_msg(recipe)
+            await sync_to_async(el.save, thread_sensitive=True)()
+
+
+        await browser.close()
+        return files
+
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        return asyncio.run(self.get_files_from_recipes_async(recipes, el, cookie))

cookbook/integration/pepperplate.py

@@ -1,11 +1,6 @@
-import json
-import re
-from io import BytesIO
-from zipfile import ZipFile
-
-from cookbook.helper.ingredient_parser import parse, get_food, get_unit
+from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Food, Unit, Ingredient, Keyword
+from cookbook.models import Ingredient, Recipe, Step
 
 
 class Pepperplate(Integration):
@@ -40,16 +35,17 @@ class Pepperplate(Integration):
         recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space)
 
         step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n'
+            instruction='\n'.join(directions) + '\n\n', space=self.request.space,
         )
 
+        ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
             if len(ingredient.strip()) > 0:
-                amount, unit, ingredient, note = parse(ingredient)
-                f = get_food(ingredient, self.request.space)
-                u = get_unit(unit, self.request.space)
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
+                f = ingredient_parser.get_food(food)
+                u = ingredient_parser.get_unit(unit)
                 step.ingredients.add(Ingredient.objects.create(
-                    food=f, unit=u, amount=amount, note=note
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                 ))
         recipe.steps.add(step)
 

cookbook/integration/plantoeat.py

@@ -0,0 +1,99 @@
+from io import BytesIO
+
+import requests
+
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.integration.integration import Integration
+from cookbook.models import Ingredient, Keyword, Recipe, Step
+
+
+class Plantoeat(Integration):
+
+    def get_recipe_from_file(self, file):
+        ingredient_mode = False
+        direction_mode = False
+
+        image_url = None
+        tags = None
+        ingredients = []
+        directions = []
+        description = ''
+        for line in file.replace('\r', '').split('\n'):
+            if line.strip() != '':
+                if 'Title:' in line:
+                    title = line.replace('Title:', '').replace('"', '').strip()
+                if 'Description:' in line:
+                    description = line.replace('Description:', '').strip()
+                if 'Source:' in line or 'Serves:' in line or 'Prep Time:' in line or 'Cook Time:' in line:
+                    directions.append(line.strip() + '\n')
+                if 'Photo Url:' in line:
+                    image_url = line.replace('Photo Url:', '').strip()
+                if 'Tags:' in line:
+                    tags = line.replace('Tags:', '').strip()
+                if ingredient_mode:
+                    if len(line) > 2 and 'Instructions:' not in line:
+                        ingredients.append(line.strip())
+                if direction_mode:
+                    if len(line) > 2:
+                        directions.append(line.strip() + '\n')
+                if 'Ingredients:' in line:
+                    ingredient_mode = True
+                if 'Directions:' in line:
+                    ingredient_mode = False
+                    direction_mode = True
+
+        recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space)
+
+        step = Step.objects.create(
+            instruction='\n'.join(directions) + '\n\n', space=self.request.space,
+        )
+
+        if tags:
+            tags = tags.replace('^',',')
+            for k in tags.split(','):
+                keyword, created = Keyword.objects.get_or_create(name=k.strip(), space=self.request.space)
+                recipe.keywords.add(keyword)
+
+        ingredient_parser = IngredientParser(self.request, True)
+        for ingredient in ingredients:
+            if len(ingredient.strip()) > 0:
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
+                f = ingredient_parser.get_food(food)
+                u = ingredient_parser.get_unit(unit)
+                step.ingredients.add(Ingredient.objects.create(
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
+                ))
+        recipe.steps.add(step)
+
+        if image_url:
+            try:
+                response = requests.get(image_url)
+                self.import_recipe_image(recipe, BytesIO(response.content))
+            except Exception as e:
+                print('failed to import image ', str(e))
+
+        return recipe
+
+    def split_recipe_file(self, file):
+        recipe_list = []
+        current_recipe = ''
+
+        for fl in file.readlines():
+            try:
+                line = fl.decode("utf-8")
+            except UnicodeDecodeError:
+                line = fl.decode("windows-1250")
+
+            if line.startswith('--------------'):
+                if current_recipe != '':
+                    recipe_list.append(current_recipe)
+                    current_recipe = ''
+                else:
+                    current_recipe = ''
+            else:
+                current_recipe += line + '\n'
+
+        if current_recipe != '':
+            recipe_list.append(current_recipe)
+
+        return recipe_list

cookbook/integration/recettetek.py

@@ -1,16 +1,17 @@
-import re
+import imghdr
 import json
-import base64
-import requests
+import re
 from io import BytesIO
 from zipfile import ZipFile
-import imghdr
-from django.utils.translation import gettext as _
 
+import requests
+import validators
+
+from django.utils.translation import gettext as _
 from cookbook.helper.image_processing import get_filetype
-from cookbook.helper.ingredient_parser import parse, get_food, get_unit
+from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Food, Unit, Ingredient, Keyword
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class RecetteTek(Integration):
@@ -29,10 +30,10 @@ class RecetteTek(Integration):
 
     def get_recipe_from_file(self, file):
 
-        # Create initial recipe with just a title and a decription
+        # Create initial recipe with just a title and a description
         recipe = Recipe.objects.create(name=file['title'], created_by=self.request.user, internal=True, space=self.request.space, )
 
-        # set the description as an empty string for later use for the source URL, incase there is no description text.
+        # set the description as an empty string for later use for the source URL, in case there is no description text.
         recipe.description = ''
 
         try:
@@ -45,25 +46,26 @@ class RecetteTek(Integration):
         if not instructions:
             instructions = ''
 
-        step = Step.objects.create(instruction=instructions)
+        step = Step.objects.create(instruction=instructions, space=self.request.space,)
 
         # Append the original import url to the step (if it exists)
         try:
             if file['url'] != '':
-                step.instruction += '\n\nImported from: ' + file['url']
+                step.instruction += '\n\n' + _('Imported from') + ': ' + file['url']
                 step.save()
         except Exception as e:
             print(recipe.name, ': failed to import source url ', str(e))
 
         try:
             # Process the ingredients. Assumes 1 ingredient per line.
+            ingredient_parser = IngredientParser(self.request, True)
             for ingredient in file['ingredients'].split('\n'):
                 if len(ingredient.strip()) > 0:
-                    amount, unit, ingredient, note = parse(ingredient)
-                    f = get_food(ingredient, self.request.space)
-                    u = get_unit(unit, self.request.space)
+                    amount, unit, food, note = ingredient_parser.parse(food)
+                    f = ingredient_parser.get_food(ingredient)
+                    u = ingredient_parser.get_unit(unit)
                     step.ingredients.add(Ingredient.objects.create(
-                        food=f, unit=u, amount=amount, note=note
+                        food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                     ))
         except Exception as e:
             print(recipe.name, ': failed to parse recipe ingredients ', str(e))
@@ -108,7 +110,7 @@ class RecetteTek(Integration):
                     recipe.keywords.add(k)
             recipe.save()
         except Exception as e:
-            pass
+            print(recipe.name, ': failed to parse keywords ', str(e))
 
         # TODO: Parse Nutritional Information
 
@@ -122,11 +124,13 @@ class RecetteTek(Integration):
                         self.import_recipe_image(recipe, BytesIO(import_zip.read(image_file_name)), filetype=get_filetype(image_file_name))
             else:
                 if file['originalPicture'] != '':
-                    response = requests.get(file['originalPicture'])
-                    if imghdr.what(BytesIO(response.content)) != None:
-                        self.import_recipe_image(recipe, BytesIO(response.content), filetype=get_filetype(file['originalPicture']))
-                    else:
-                        raise Exception("Original image failed to download.")
+                    url = file['originalPicture']
+                    if validators.url(url, public=True):
+                        response = requests.get(url)
+                        if imghdr.what(BytesIO(response.content)) is not None:
+                            self.import_recipe_image(recipe, BytesIO(response.content), filetype=get_filetype(file['originalPicture']))
+                        else:
+                            raise Exception("Original image failed to download.")
         except Exception as e:
             print(recipe.name, ': failed to import image ', str(e))
 

cookbook/integration/recipekeeper.py

@@ -1,14 +1,14 @@
 import re
-from bs4 import BeautifulSoup
 from io import BytesIO
 from zipfile import ZipFile
 
-from django.utils.translation import gettext as _
+from bs4 import BeautifulSoup
 
-from cookbook.helper.ingredient_parser import parse, get_food, get_unit
-from cookbook.helper.recipe_url_import import parse_servings, iso_duration_to_minutes
+from django.utils.translation import gettext as _
+from cookbook.helper.ingredient_parser import IngredientParser
+from cookbook.helper.recipe_url_import import iso_duration_to_minutes, parse_servings
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Food, Unit, Ingredient, Keyword
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class RecipeKeeper(Integration):
@@ -41,16 +41,17 @@ class RecipeKeeper(Integration):
         except AttributeError:
             pass
 
-        step = Step.objects.create(instruction='')
+        step = Step.objects.create(instruction='', space=self.request.space,)
 
+        ingredient_parser = IngredientParser(self.request, True)
         for ingredient in file.find("div", {"itemprop": "recipeIngredients"}).findChildren("p"):
             if ingredient.text == "":
                 continue
-            amount, unit, ingredient, note = parse(ingredient.text.strip())
-            f = get_food(ingredient, self.request.space)
-            u = get_unit(unit, self.request.space)
+            amount, unit, food, note = ingredient_parser.parse(ingredient.text.strip())
+            f = ingredient_parser.get_food(food)
+            u = ingredient_parser.get_unit(unit)
             step.ingredients.add(Ingredient.objects.create(
-                food=f, unit=u, amount=amount, note=note
+                food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
             ))
 
         for s in file.find("div", {"itemprop": "recipeDirections"}).find_all("p"):
@@ -59,9 +60,8 @@ class RecipeKeeper(Integration):
             step.instruction += s.text + ' \n'
 
         if file.find("span", {"itemprop": "recipeSource"}).text != '':
-            step.instruction += "\n\nImported from: " + file.find("span", {"itemprop": "recipeSource"}).text
+            step.instruction += "\n\n" + _("Imported from") + ": " + file.find("span", {"itemprop": "recipeSource"}).text
             step.save()
-            source_url_added = True
 
         recipe.steps.add(step)
 
@@ -72,7 +72,7 @@ class RecipeKeeper(Integration):
                     import_zip = ZipFile(f['file'])
                     self.import_recipe_image(recipe, BytesIO(import_zip.read(file.find("img", class_="recipe-photo").get("src"))), filetype='.jpeg')
         except Exception as e:
-            pass
+            print(recipe.name, ': failed to import image ', str(e))
 
         return recipe
 

cookbook/integration/recipesage.py

@@ -1,13 +1,12 @@
-import base64
 import json
 from io import BytesIO
 
 import requests
-from rest_framework.renderers import JSONRenderer
+import validators
 
-from cookbook.helper.ingredient_parser import parse, get_food, get_unit
+from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Ingredient
+from cookbook.models import Ingredient, Recipe, Step
 
 
 class RecipeSage(Integration):
@@ -33,27 +32,30 @@ class RecipeSage(Integration):
         except Exception as e:
             print('failed to parse yield or time ', str(e))
 
+        ingredient_parser = IngredientParser(self.request, True)
         ingredients_added = False
         for s in file['recipeInstructions']:
             step = Step.objects.create(
-                instruction=s['text']
+                instruction=s['text'], space=self.request.space,
             )
             if not ingredients_added:
                 ingredients_added = True
 
                 for ingredient in file['recipeIngredient']:
-                    amount, unit, ingredient, note = parse(ingredient)
-                    f = get_food(ingredient, self.request.space)
-                    u = get_unit(unit, self.request.space)
+                    amount, unit, food, note = ingredient_parser.parse(ingredient)
+                    f = ingredient_parser.get_food(food)
+                    u = ingredient_parser.get_unit(unit)
                     step.ingredients.add(Ingredient.objects.create(
-                        food=f, unit=u, amount=amount, note=note
+                        food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                     ))
             recipe.steps.add(step)
 
         if len(file['image']) > 0:
             try:
-                response = requests.get(file['image'][0])
-                self.import_recipe_image(recipe, BytesIO(response.content))
+                url = file['image'][0]
+                if validators.url(url, public=True):
+                    response = requests.get(url)
+                    self.import_recipe_image(recipe, BytesIO(response.content))
             except Exception as e:
                 print('failed to import image ', str(e))
 
@@ -78,16 +80,26 @@ class RecipeSage(Integration):
         }
 
         for s in recipe.steps.all():
-            if s.type != Step.TIME:
-                data['recipeInstructions'].append({
-                    '@type': 'HowToStep',
-                    'text': s.instruction
-                })
+            data['recipeInstructions'].append({
+                '@type': 'HowToStep',
+                'text': s.instruction
+            })
 
-                for i in s.ingredients.all():
-                    data['recipeIngredient'].append(f'{float(i.amount)} {i.unit} {i.food}')
+            for i in s.ingredients.all():
+                data['recipeIngredient'].append(f'{float(i.amount)} {i.unit} {i.food}')
 
         return data
 
+    def get_files_from_recipes(self, recipes, el, cookie):
+        json_list = []
+        for r in recipes:
+            json_list.append(self.get_file_from_recipe(r))
+
+            el.exported_recipes += 1
+            el.msg += self.get_recipe_processed_msg(r)
+            el.save()
+
+        return [[self.get_export_file_name('json'), json.dumps(json_list)]]
+
     def split_recipe_file(self, file):
         return json.loads(file.read().decode("utf-8"))

cookbook/integration/rezkonv.py

@@ -1,11 +1,6 @@
-import json
-import re
-from io import BytesIO
-from zipfile import ZipFile
-
-from cookbook.helper.ingredient_parser import parse, get_food, get_unit
+from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Food, Unit, Ingredient, Keyword
+from cookbook.models import Ingredient, Keyword, Recipe, Step
 
 
 class RezKonv(Integration):
@@ -17,42 +12,43 @@ class RezKonv(Integration):
 
         ingredients = []
         directions = []
-        for line in file.replace('\r', '').split('\n'):
+        for line in file.replace('\r', '').replace('\n\n', '\n').split('\n'):
             if 'Titel:' in line:
                 title = line.replace('Titel:', '').strip()
             if 'Kategorien:' in line:
                 tags = line.replace('Kategorien:', '').strip()
-            if ingredient_mode and ('quelle' in line.lower() or 'source' in line.lower()):
+            if ingredient_mode and (
+                    'quelle' in line.lower() or 'source' in line.lower() or (line == '' and len(ingredients) > 0)):
                 ingredient_mode = False
+                direction_mode = True
             if ingredient_mode:
                 if line != '' and '===' not in line and 'Zubereitung' not in line:
                     ingredients.append(line.strip())
             if direction_mode:
                 if line.strip() != '' and line.strip() != '=====':
                     directions.append(line.strip())
-            if 'Zutaten:' in line:
+            if 'Zutaten:' in line or 'Ingredients' in line or 'Menge:' in line:
                 ingredient_mode = True
-            if 'Zubereitung:' in line:
-                ingredient_mode = False
-                direction_mode = True
 
-        recipe = Recipe.objects.create(name=title, created_by=self.request.user, internal=True, space=self.request.space)
+        recipe = Recipe.objects.create(name=title, created_by=self.request.user, internal=True,
+                                       space=self.request.space)
 
         for k in tags.split(','):
             keyword, created = Keyword.objects.get_or_create(name=k.strip(), space=self.request.space)
             recipe.keywords.add(keyword)
 
         step = Step.objects.create(
-            instruction='\n'.join(directions) + '\n\n'
+            instruction='  \n'.join(directions) + '\n\n', space=self.request.space,
         )
 
+        ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
             if len(ingredient.strip()) > 0:
-                amount, unit, ingredient, note = parse(ingredient)
-                f = get_food(ingredient, self.request.space)
-                u = get_unit(unit, self.request.space)
+                amount, unit, food, note = ingredient_parser.parse(ingredient)
+                f = ingredient_parser.get_food(food)
+                u = ingredient_parser.get_unit(unit)
                 step.ingredients.add(Ingredient.objects.create(
-                    food=f, unit=u, amount=amount, note=note
+                    food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
                 ))
         recipe.steps.add(step)
 
@@ -64,9 +60,15 @@ class RezKonv(Integration):
     def split_recipe_file(self, file):
         recipe_list = []
         current_recipe = ''
-
+        encoding_list = ['windows-1250',
+                         'latin-1']  # TODO build algorithm to try trough encodings and fail if none work, use for all importers
+        encoding = 'windows-1250'
         for fl in file.readlines():
-            line = fl.decode("ANSI")
+            try:
+                line = fl.decode(encoding)
+            except UnicodeDecodeError:
+                encoding = 'latin-1'
+                line = fl.decode(encoding)
             if line.startswith('=====') and 'rezkonv' in line.lower():
                 if current_recipe != '':
                     recipe_list.append(current_recipe)

cookbook/integration/saffron.py

@@ -1,11 +1,11 @@
 from django.utils.translation import gettext as _
 
-from cookbook.helper.ingredient_parser import parse, get_food, get_unit
+from cookbook.helper.ingredient_parser import IngredientParser
 from cookbook.integration.integration import Integration
-from cookbook.models import Recipe, Step, Food, Unit, Ingredient
+from cookbook.models import Ingredient, Recipe, Step
 
 
-class Safron(Integration):
+class Saffron(Integration):
 
     def get_recipe_from_file(self, file):
         ingredient_mode = False
@@ -43,18 +43,57 @@ class Safron(Integration):
 
         recipe = Recipe.objects.create(name=title, description=description, created_by=self.request.user, internal=True, space=self.request.space, )
 
-        step = Step.objects.create(instruction='\n'.join(directions))
+        step = Step.objects.create(instruction='\n'.join(directions), space=self.request.space, )
 
+        ingredient_parser = IngredientParser(self.request, True)
         for ingredient in ingredients:
-            amount, unit, ingredient, note = parse(ingredient)
-            f = get_food(ingredient, self.request.space)
-            u = get_unit(unit, self.request.space)
+            amount, unit, food, note = ingredient_parser.parse(ingredient)
+            f = ingredient_parser.get_food(food)
+            u = ingredient_parser.get_unit(unit)
             step.ingredients.add(Ingredient.objects.create(
-                food=f, unit=u, amount=amount, note=note
+                food=f, unit=u, amount=amount, note=note, original_text=ingredient, space=self.request.space,
             ))
         recipe.steps.add(step)
 
         return recipe
 
     def get_file_from_recipe(self, recipe):
-        raise NotImplementedError('Method not implemented in storage integration')
+
+        data = "Title: "+recipe.name if recipe.name else ""+"\n"
+        data += "Description: "+recipe.description if recipe.description else ""+"\n"
+        data += "Source: \n"
+        data += "Original URL: \n"
+        data += "Yield: "+str(recipe.servings)+"\n"
+        data += "Cookbook: \n"
+        data += "Section: \n"
+        data += "Image: \n"
+
+        recipeInstructions = []
+        recipeIngredient = []
+        for s in recipe.steps.all():
+            recipeInstructions.append(s.instruction)
+
+            for i in s.ingredients.all():
+                recipeIngredient.append(f'{float(i.amount)} {i.unit} {i.food}')
+
+        data += "Ingredients: \n"
+        for ingredient in recipeIngredient:
+            data += ingredient+"\n"
+
+        data += "Instructions: \n"
+        for instruction in recipeInstructions:
+            data += instruction+"\n"
+
+        return recipe.name+'.txt', data
+
+    def get_files_from_recipes(self, recipes, el, cookie):
+        files = []
+        for r in recipes:
+            filename, data = self.get_file_from_recipe(r)
+            files.append([filename, data])
+
+            el.exported_recipes += 1
+            el.msg += self.get_recipe_processed_msg(r)
+            el.save()
+
+        return files

cookbook/locale/hy/LC_MESSAGES/django.po

@@ -11,7 +11,7 @@ msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2021-02-09 18:01+0100\n"
-"PO-Revision-Date: 2021-04-12 20:22+0000\n"
+"PO-Revision-Date: 2021-10-13 12:50+0000\n"
 "Last-Translator: Hrachya Kocharyan <hkocharyan@ctemplar.com>\n"
 "Language-Team: Armenian <http://translate.tandoor.dev/projects/tandoor/"
 "recipes-backend/hy/>\n"
@@ -20,7 +20,7 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n != 1);\n"
-"X-Generator: Weblate 4.5.3\n"
+"X-Generator: Weblate 4.8\n"
 
 #: .\cookbook\filters.py:22 .\cookbook\templates\base.html:87
 #: .\cookbook\templates\forms\edit_internal_recipe.html:219
@@ -79,7 +79,7 @@ msgid ""
 "mobile data. If lower than instance limit it is reset when saving."
 msgstr ""
 "0-ն կանջատի ավտոմատ սինքրոնացումը։ Գնումների ցուցակը թարմացվում է "
-"յուրաքանչյուր սահմանված վարկյանը մեկ, մեկ ուրիշի կատարած փոփոխությունները "
+"յուրաքանչյուր սահմանված վարկյանը մեկ, ուրիշի կատարած փոփոխությունները "
 "սինքրոնացնելու համար։ Հարմար է, երբ մեկից ավել մարդ է կատարում գնումները, "
 "բայց կարող է օգտագործել բջջային ինտերնետ։"