From 1235cb8da54f98d5337092b7ca1bf9f85ef5ea97 Mon Sep 17 00:00:00 2001 From: tooboredtocode Date: Fri, 22 Mar 2024 00:31:56 +0100 Subject: [PATCH 01/15] feat: improve docker run speeds --- Dockerfile | 19 +++++++++++++------ boot.sh | 22 +++++++++++++++++----- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3f17d0f11..a39850725 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,8 @@ RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg li #Print all logs without buffering it. ENV PYTHONUNBUFFERED 1 +ENV DOCKER true + #This port will be used by gunicorn. EXPOSE 8080 @@ -23,18 +25,23 @@ RUN \ RUN sed -i '/# Development/,$d' requirements.txt RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-dev jpeg-dev libwebp-dev openssl-dev libffi-dev cargo openldap-dev python3-dev && \ echo -n "INPUT ( libldap.so )" > /usr/lib/libldap_r.so && \ - python -m venv venv && \ - /opt/recipes/venv/bin/python -m pip install --upgrade pip && \ - venv/bin/pip install wheel==0.42.0 && \ - venv/bin/pip install setuptools_rust==1.9.0 && \ - venv/bin/pip install -r requirements.txt --no-cache-dir &&\ + python -m pip install --upgrade pip && \ + pip install wheel==0.42.0 && \ + pip install setuptools_rust==1.9.0 && \ + pip install -r requirements.txt --no-cache-dir &&\ apk --purge del .build-deps #Copy project and execute it. COPY . ./ +# collect the static files +RUN python manage.py collectstatic_js_reverse +RUN python manage.py collectstatic --noinput +# copy the collected static files to a different location, so they can be moved into a potentially mounted volume +RUN mv /opt/recipes/staticfiles /opt/recipes/staticfiles-collect + # collect information from git repositories -RUN /opt/recipes/venv/bin/python version.py +RUN python version.py # delete git repositories to reduce image size RUN find . -type d -name ".git" | xargs rm -rf diff --git a/boot.sh b/boot.sh index ae3dbb51d..1c3c09a76 100644 --- a/boot.sh +++ b/boot.sh @@ -1,5 +1,9 @@ #!/bin/sh -source venv/bin/activate + +# conditionally activate virtualenv, since the docker container does not need it +if [[ "${DOCKER}" != "true" ]]; then + source venv/bin/activate +fi TANDOOR_PORT="${TANDOOR_PORT:-8080}" GUNICORN_WORKERS="${GUNICORN_WORKERS:-3}" @@ -67,12 +71,20 @@ echo "Migrating database" python manage.py migrate -echo "Generating static files" +if [[ "${DOCKER}" == "true" ]]; then + echo "Collecting static files" -python manage.py collectstatic_js_reverse -python manage.py collectstatic --noinput + mkdir -p /opt/recipes/staticfiles + mv /opt/recipes/staticfiles-collect/* /opt/recipes/staticfiles + rm -rf /opt/recipes/staticfiles-collect +else + echo "Generating static files" -echo "Done" + python manage.py collectstatic_js_reverse + python manage.py collectstatic --noinput + + echo "Done" +fi chmod -R 755 /opt/recipes/mediafiles From 5f211e420e2212dd71d1d7120a11e4272aa50643 Mon Sep 17 00:00:00 2001 From: tooboredtocode Date: Fri, 22 Mar 2024 02:50:18 +0100 Subject: [PATCH 02/15] fix: moving into exisiting files --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index a39850725..1de5178b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,7 @@ COPY . ./ RUN python manage.py collectstatic_js_reverse RUN python manage.py collectstatic --noinput # copy the collected static files to a different location, so they can be moved into a potentially mounted volume +RUN rm -rf /opt/recipes/staticfiles/* RUN mv /opt/recipes/staticfiles /opt/recipes/staticfiles-collect # collect information from git repositories From ae37abf8b21d3e9f13773d6857efbaed8db501ae Mon Sep 17 00:00:00 2001 From: tooboredtocode Date: Fri, 22 Mar 2024 11:26:26 +0100 Subject: [PATCH 03/15] fix: fix modified the wrong file --- Dockerfile | 1 - boot.sh | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1de5178b4..a39850725 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,6 @@ COPY . ./ RUN python manage.py collectstatic_js_reverse RUN python manage.py collectstatic --noinput # copy the collected static files to a different location, so they can be moved into a potentially mounted volume -RUN rm -rf /opt/recipes/staticfiles/* RUN mv /opt/recipes/staticfiles /opt/recipes/staticfiles-collect # collect information from git repositories diff --git a/boot.sh b/boot.sh index 1c3c09a76..286d6c983 100644 --- a/boot.sh +++ b/boot.sh @@ -75,6 +75,7 @@ if [[ "${DOCKER}" == "true" ]]; then echo "Collecting static files" mkdir -p /opt/recipes/staticfiles + rm -rf /opt/recipes/staticfiles/* mv /opt/recipes/staticfiles-collect/* /opt/recipes/staticfiles rm -rf /opt/recipes/staticfiles-collect else From 7bfa23b953a2a9ee671f23af7d55fbc997461e10 Mon Sep 17 00:00:00 2001 From: tooboredtocode Date: Fri, 22 Mar 2024 11:32:30 +0100 Subject: [PATCH 04/15] feat: better messages --- boot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/boot.sh b/boot.sh index 286d6c983..7d6ec7442 100644 --- a/boot.sh +++ b/boot.sh @@ -72,14 +72,14 @@ echo "Migrating database" python manage.py migrate if [[ "${DOCKER}" == "true" ]]; then - echo "Collecting static files" + echo "Copying cached static files from docker build" mkdir -p /opt/recipes/staticfiles rm -rf /opt/recipes/staticfiles/* mv /opt/recipes/staticfiles-collect/* /opt/recipes/staticfiles rm -rf /opt/recipes/staticfiles-collect else - echo "Generating static files" + echo "Collecting static files, this may take a while..." python manage.py collectstatic_js_reverse python manage.py collectstatic --noinput From 20b812c2cce1db0101a3fb2a10abca90410e44a3 Mon Sep 17 00:00:00 2001 From: tooboredtocode Date: Fri, 22 Mar 2024 19:04:29 +0100 Subject: [PATCH 05/15] feat: re-add venv --- Dockerfile | 15 ++++++++------- boot.sh | 6 +----- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index a39850725..c5856bcfa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,23 +25,24 @@ RUN \ RUN sed -i '/# Development/,$d' requirements.txt RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-dev jpeg-dev libwebp-dev openssl-dev libffi-dev cargo openldap-dev python3-dev && \ echo -n "INPUT ( libldap.so )" > /usr/lib/libldap_r.so && \ - python -m pip install --upgrade pip && \ - pip install wheel==0.42.0 && \ - pip install setuptools_rust==1.9.0 && \ - pip install -r requirements.txt --no-cache-dir &&\ + python -m venv venv && \ + /opt/recipes/venv/bin/python -m pip install --upgrade pip && \ + venv/bin/pip install wheel==0.42.0 && \ + venv/bin/pip install setuptools_rust==1.9.0 && \ + venv/bin/pip install -r requirements.txt --no-cache-dir &&\ apk --purge del .build-deps #Copy project and execute it. COPY . ./ # collect the static files -RUN python manage.py collectstatic_js_reverse -RUN python manage.py collectstatic --noinput +RUN /opt/recipes/venv/bin/python manage.py collectstatic_js_reverse +RUN /opt/recipes/venv/bin/python manage.py collectstatic --noinput # copy the collected static files to a different location, so they can be moved into a potentially mounted volume RUN mv /opt/recipes/staticfiles /opt/recipes/staticfiles-collect # collect information from git repositories -RUN python version.py +RUN /opt/recipes/venv/bin/python version.py # delete git repositories to reduce image size RUN find . -type d -name ".git" | xargs rm -rf diff --git a/boot.sh b/boot.sh index 7d6ec7442..ab5d7fddd 100644 --- a/boot.sh +++ b/boot.sh @@ -1,9 +1,5 @@ #!/bin/sh - -# conditionally activate virtualenv, since the docker container does not need it -if [[ "${DOCKER}" != "true" ]]; then - source venv/bin/activate -fi +source venv/bin/activate TANDOOR_PORT="${TANDOOR_PORT:-8080}" GUNICORN_WORKERS="${GUNICORN_WORKERS:-3}" From 4e0780d5124f02dd999a16b75b7c477e434e577f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 00:53:06 +0000 Subject: [PATCH 06/15] Bump recipe-scrapers from 14.52.0 to 14.55.0 Bumps [recipe-scrapers](https://github.com/hhursev/recipe-scrapers) from 14.52.0 to 14.55.0. - [Release notes](https://github.com/hhursev/recipe-scrapers/releases) - [Commits](https://github.com/hhursev/recipe-scrapers/compare/14.52.0...14.55.0) --- updated-dependencies: - dependency-name: recipe-scrapers dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3ef2dd2b6..6c6dfdf9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ Jinja2==3.1.3 django-webpack-loader==3.0.1 git+https://github.com/BITSOLVER/django-js-reverse@071e304fd600107bc64bbde6f2491f1fe049ec82 django-allauth==0.61.1 -recipe-scrapers==14.52.0 +recipe-scrapers==14.55.0 django-scopes==2.0.0 django-treebeard==4.7 django-cors-headers==4.3.1 From 10373b6ac58cc93c9c845f91ce30d1cd37c3736e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 00:53:12 +0000 Subject: [PATCH 07/15] Bump pytest-cov from 4.1.0 to 5.0.0 Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 4.1.0 to 5.0.0. - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v4.1.0...v5.0.0) --- updated-dependencies: - dependency-name: pytest-cov dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3ef2dd2b6..f6e17e20a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,7 +48,7 @@ aiohttp==3.9.3 # Development pytest==8.0.0 pytest-django==4.8.0 -pytest-cov===4.1.0 +pytest-cov===5.0.0 pytest-factoryboy==2.6.0 pytest-html==4.1.1 pytest-asyncio==0.23.5 From 768b678c9316a821a7b8f993572507038422903f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 00:53:19 +0000 Subject: [PATCH 08/15] Bump django-crispy-forms from 2.0 to 2.1 Bumps [django-crispy-forms](https://github.com/django-crispy-forms/django-crispy-forms) from 2.0 to 2.1. - [Release notes](https://github.com/django-crispy-forms/django-crispy-forms/releases) - [Changelog](https://github.com/django-crispy-forms/django-crispy-forms/blob/main/CHANGELOG.md) - [Commits](https://github.com/django-crispy-forms/django-crispy-forms/compare/2.0...2.1) --- updated-dependencies: - dependency-name: django-crispy-forms dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3ef2dd2b6..de8f5dcfa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ Django==4.2.11 cryptography===42.0.5 django-annoying==0.10.6 django-cleanup==8.0.0 -django-crispy-forms==2.0 +django-crispy-forms==2.1 crispy-bootstrap4==2022.1 django-tables2==2.7.0 djangorestframework==3.14.0 From 4f0f59a55c9dc6ad73e402f8130e8fa3d8da8f2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 00:53:25 +0000 Subject: [PATCH 09/15] Bump django-debug-toolbar from 4.2.0 to 4.3.0 Bumps [django-debug-toolbar](https://github.com/jazzband/django-debug-toolbar) from 4.2.0 to 4.3.0. - [Release notes](https://github.com/jazzband/django-debug-toolbar/releases) - [Changelog](https://github.com/jazzband/django-debug-toolbar/blob/main/docs/changes.rst) - [Commits](https://github.com/jazzband/django-debug-toolbar/compare/4.2...4.3) --- updated-dependencies: - dependency-name: django-debug-toolbar dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3ef2dd2b6..969972043 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ django-tables2==2.7.0 djangorestframework==3.14.0 drf-writable-nested==0.7.0 django-oauth-toolkit==2.3.0 -django-debug-toolbar==4.2.0 +django-debug-toolbar==4.3.0 bleach==6.0.0 gunicorn==21.2.0 lxml==5.1.0 From 5a0b9e14d2af02e6e3f2674f74f80a60af049971 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 00:53:28 +0000 Subject: [PATCH 10/15] Bump beautifulsoup4 from 4.12.2 to 4.12.3 Bumps [beautifulsoup4](https://www.crummy.com/software/BeautifulSoup/bs4/) from 4.12.2 to 4.12.3. --- updated-dependencies: - dependency-name: beautifulsoup4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3ef2dd2b6..aed155898 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ whitenoise==6.6.0 icalendar==5.0.11 pyyaml==6.0.1 uritemplate==4.1.1 -beautifulsoup4==4.12.2 +beautifulsoup4==4.12.3 microdata==0.8.0 mock==5.1.0 Jinja2==3.1.3 From b38ea866b4df61de332a5facccc928b1ccc25628 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 01:02:35 +0000 Subject: [PATCH 11/15] Bump awalsh128/cache-apt-pkgs-action from 1.4.1 to 1.4.2 Bumps [awalsh128/cache-apt-pkgs-action](https://github.com/awalsh128/cache-apt-pkgs-action) from 1.4.1 to 1.4.2. - [Release notes](https://github.com/awalsh128/cache-apt-pkgs-action/releases) - [Commits](https://github.com/awalsh128/cache-apt-pkgs-action/compare/v1.4.1...v1.4.2) --- updated-dependencies: - dependency-name: awalsh128/cache-apt-pkgs-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 853545cee..6e045b6d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: awalsh128/cache-apt-pkgs-action@v1.4.1 + - uses: awalsh128/cache-apt-pkgs-action@v1.4.2 with: packages: libsasl2-dev python3-dev libldap2-dev libssl-dev version: 1.0 From 4390703c0cbd5331c35fa8fb3673db5c67834cb5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:22:02 +0000 Subject: [PATCH 12/15] Bump pillow from 10.2.0 to 10.3.0 Bumps [pillow](https://github.com/python-pillow/Pillow) from 10.2.0 to 10.3.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/10.2.0...10.3.0) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3ef2dd2b6..59bc9da1f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ bleach==6.0.0 gunicorn==21.2.0 lxml==5.1.0 Markdown==3.5.1 -Pillow==10.2.0 +Pillow==10.3.0 psycopg2-binary==2.9.9 python-dotenv==1.0.0 requests==2.31.0 From 4293ec77c0b4ee46f4cc6cb3346795a87ceb9ccb Mon Sep 17 00:00:00 2001 From: vabene1111 Date: Thu, 4 Apr 2024 20:49:48 +0200 Subject: [PATCH 13/15] fixed changing categories of shopping items --- vue/src/components/ShoppingLineItem.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vue/src/components/ShoppingLineItem.vue b/vue/src/components/ShoppingLineItem.vue index b11f2ce72..f95c97410 100644 --- a/vue/src/components/ShoppingLineItem.vue +++ b/vue/src/components/ShoppingLineItem.vue @@ -54,7 +54,7 @@ text-field="name" value-field="id" v-model="food.supermarket_category" - @change="detail_modal_visible = false; updateFoodCategory(food)" + @input="detail_modal_visible = false; updateFoodCategory(food)" > Date: Fri, 5 Apr 2024 21:18:01 +0200 Subject: [PATCH 14/15] downgraded recipe scrapers to pass tests for now --- cookbook/tests/other/test_automations.py | 1 + requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cookbook/tests/other/test_automations.py b/cookbook/tests/other/test_automations.py index 48416caa7..8b7b5568f 100644 --- a/cookbook/tests/other/test_automations.py +++ b/cookbook/tests/other/test_automations.py @@ -156,6 +156,7 @@ def test_url_import_regex_replace(u1_s1): if 'cookbook' in os.getcwd(): test_file = os.path.join(os.getcwd(), 'other', 'test_data', recipe) + # TODO this catch doesn't really work depending on from where you start the test, must check for duplicate path sections else: test_file = os.path.join(os.getcwd(), 'cookbook', 'tests', 'other', 'test_data', recipe) with open(test_file, 'r', encoding='UTF-8') as d: diff --git a/requirements.txt b/requirements.txt index 49c84c701..b457d2c46 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ Jinja2==3.1.3 django-webpack-loader==3.0.1 git+https://github.com/BITSOLVER/django-js-reverse@071e304fd600107bc64bbde6f2491f1fe049ec82 django-allauth==0.61.1 -recipe-scrapers==14.55.0 +recipe-scrapers==14.53.0 django-scopes==2.0.0 django-treebeard==4.7 django-cors-headers==4.3.1 From f14acc371d7cf7ac7bcd6c1bd102f7d3c6a6b2b8 Mon Sep 17 00:00:00 2001 From: vabene1111 Date: Mon, 8 Apr 2024 07:20:41 +0200 Subject: [PATCH 15/15] Revert "Merge pull request #3055 from tooboredtocode/develop" This reverts commit d7669279ff8d6bf1a77da8366a2d60bf0ddb4051, reversing changes made to 4293ec77c0b4ee46f4cc6cb3346795a87ceb9ccb. --- Dockerfile | 8 -------- boot.sh | 17 ++++------------- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index c5856bcfa..3f17d0f11 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,6 @@ RUN apk add --no-cache postgresql-libs postgresql-client gettext zlib libjpeg li #Print all logs without buffering it. ENV PYTHONUNBUFFERED 1 -ENV DOCKER true - #This port will be used by gunicorn. EXPOSE 8080 @@ -35,12 +33,6 @@ RUN apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev zlib-de #Copy project and execute it. COPY . ./ -# collect the static files -RUN /opt/recipes/venv/bin/python manage.py collectstatic_js_reverse -RUN /opt/recipes/venv/bin/python manage.py collectstatic --noinput -# copy the collected static files to a different location, so they can be moved into a potentially mounted volume -RUN mv /opt/recipes/staticfiles /opt/recipes/staticfiles-collect - # collect information from git repositories RUN /opt/recipes/venv/bin/python version.py # delete git repositories to reduce image size diff --git a/boot.sh b/boot.sh index ab5d7fddd..ae3dbb51d 100644 --- a/boot.sh +++ b/boot.sh @@ -67,21 +67,12 @@ echo "Migrating database" python manage.py migrate -if [[ "${DOCKER}" == "true" ]]; then - echo "Copying cached static files from docker build" +echo "Generating static files" - mkdir -p /opt/recipes/staticfiles - rm -rf /opt/recipes/staticfiles/* - mv /opt/recipes/staticfiles-collect/* /opt/recipes/staticfiles - rm -rf /opt/recipes/staticfiles-collect -else - echo "Collecting static files, this may take a while..." +python manage.py collectstatic_js_reverse +python manage.py collectstatic --noinput - python manage.py collectstatic_js_reverse - python manage.py collectstatic --noinput - - echo "Done" -fi +echo "Done" chmod -R 755 /opt/recipes/mediafiles