From f6f675466955b43df120b665e518a815c682b3f3 Mon Sep 17 00:00:00 2001 From: vabene1111 Date: Tue, 20 Aug 2024 11:45:23 +0200 Subject: [PATCH] removed validators library and improved url validation --- cookbook/helper/HelperFunctions.py | 35 + cookbook/integration/cookbookapp.py | 4 +- cookbook/integration/cookmate.py | 4 +- cookbook/integration/paprika.py | 4 +- cookbook/integration/plantoeat.py | 4 +- cookbook/integration/recettetek.py | 5 +- cookbook/integration/recipesage.py | 4 +- cookbook/provider/dropbox.py | 4 +- cookbook/provider/nextcloud.py | 5 +- .../other/docs/reports/tests/assets/style.css | 319 ++++++++ .../tests/other/docs/reports/tests/pytest.xml | 1 + .../tests/other/docs/reports/tests/tests.html | 770 ++++++++++++++++++ cookbook/tests/other/test_helpers.py | 14 + cookbook/views/api.py | 11 +- requirements.txt | 1 - 15 files changed, 1162 insertions(+), 23 deletions(-) create mode 100644 cookbook/tests/other/docs/reports/tests/assets/style.css create mode 100644 cookbook/tests/other/docs/reports/tests/pytest.xml create mode 100644 cookbook/tests/other/docs/reports/tests/tests.html create mode 100644 cookbook/tests/other/test_helpers.py diff --git a/cookbook/helper/HelperFunctions.py b/cookbook/helper/HelperFunctions.py index 94f46ee8c..585db225c 100644 --- a/cookbook/helper/HelperFunctions.py +++ b/cookbook/helper/HelperFunctions.py @@ -1,4 +1,12 @@ +import socket +from urllib.parse import urlparse + +from django.core.exceptions import ValidationError +from django.core.validators import URLValidator from django.db.models import Func +from ipaddress import ip_address + +from recipes import settings class Round(Func): @@ -11,3 +19,30 @@ def str2bool(v): return v else: return v.lower() in ("yes", "true", "1") + + +""" +validates an url that is supposed to be imported +checks that the protocol used is http(s) and that no local address is accessed +@:param url to test +@:return true if url is valid, false otherwise +""" + + +def validate_import_url(url): + try: + validator = URLValidator(schemes=['http', 'https']) + validator(url) + except ValidationError: + # if schema is not http or https, consider url invalid + return False + + # resolve IP address of url + try: + url_ip_address = ip_address(str(socket.gethostbyname(urlparse(url).hostname))) + except (ValueError, AttributeError, TypeError, Exception) as e: + # if ip cannot be parsed, consider url invalid + return False + + # validate that IP is neither private nor any other special address + return not any([url_ip_address.is_private, url_ip_address.is_reserved, url_ip_address.is_loopback, url_ip_address.is_multicast, url_ip_address.is_link_local, ]) diff --git a/cookbook/integration/cookbookapp.py b/cookbook/integration/cookbookapp.py index a9b5ac132..1f442a58b 100644 --- a/cookbook/integration/cookbookapp.py +++ b/cookbook/integration/cookbookapp.py @@ -2,8 +2,8 @@ import re from io import BytesIO import requests -import validators +from cookbook.helper.HelperFunctions import validate_import_url from cookbook.helper.ingredient_parser import IngredientParser from cookbook.helper.recipe_url_import import (get_from_scraper, get_images_from_soup, iso_duration_to_minutes) @@ -63,7 +63,7 @@ class CookBookApp(Integration): if len(images) > 0: try: url = images[0] - if validators.url(url, public=True): + if validate_import_url(url): response = requests.get(url) self.import_recipe_image(recipe, BytesIO(response.content)) except Exception as e: diff --git a/cookbook/integration/cookmate.py b/cookbook/integration/cookmate.py index a51ca45b9..e03d232a4 100644 --- a/cookbook/integration/cookmate.py +++ b/cookbook/integration/cookmate.py @@ -1,8 +1,8 @@ from io import BytesIO import requests -import validators +from cookbook.helper.HelperFunctions import validate_import_url from cookbook.helper.ingredient_parser import IngredientParser from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time from cookbook.integration.integration import Integration @@ -69,7 +69,7 @@ class Cookmate(Integration): if recipe_xml.find('imageurl') is not None: try: url = recipe_xml.find('imageurl').text.strip() - if validators.url(url, public=True): + if validate_import_url(url): response = requests.get(url) self.import_recipe_image(recipe, BytesIO(response.content)) except Exception as e: diff --git a/cookbook/integration/paprika.py b/cookbook/integration/paprika.py index b830d85fd..20c3d153b 100644 --- a/cookbook/integration/paprika.py +++ b/cookbook/integration/paprika.py @@ -6,8 +6,8 @@ from gettext import gettext as _ from io import BytesIO import requests -import validators +from cookbook.helper.HelperFunctions import validate_import_url from cookbook.helper.ingredient_parser import IngredientParser from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text from cookbook.integration.integration import Integration @@ -87,7 +87,7 @@ class Paprika(Integration): try: if recipe_json.get("image_url", None): url = recipe_json.get("image_url", None) - if validators.url(url, public=True): + if validate_import_url(url): response = requests.get(url) self.import_recipe_image(recipe, BytesIO(response.content)) except Exception: diff --git a/cookbook/integration/plantoeat.py b/cookbook/integration/plantoeat.py index b69b63283..c6117360a 100644 --- a/cookbook/integration/plantoeat.py +++ b/cookbook/integration/plantoeat.py @@ -1,8 +1,8 @@ from io import BytesIO import requests -import validators +from cookbook.helper.HelperFunctions import validate_import_url from cookbook.helper.ingredient_parser import IngredientParser from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time from cookbook.integration.integration import Integration @@ -75,7 +75,7 @@ class Plantoeat(Integration): if image_url: try: - if validators.url(image_url, public=True): + if validate_import_url(image_url): response = requests.get(image_url) self.import_recipe_image(recipe, BytesIO(response.content)) except Exception as e: diff --git a/cookbook/integration/recettetek.py b/cookbook/integration/recettetek.py index 87e145ffc..20b51b837 100644 --- a/cookbook/integration/recettetek.py +++ b/cookbook/integration/recettetek.py @@ -5,9 +5,10 @@ from io import BytesIO from zipfile import ZipFile import requests -import validators from django.utils.translation import gettext as _ + +from cookbook.helper.HelperFunctions import validate_import_url from cookbook.helper.image_processing import get_filetype from cookbook.helper.ingredient_parser import IngredientParser from cookbook.integration.integration import Integration @@ -125,7 +126,7 @@ class RecetteTek(Integration): else: if file['originalPicture'] != '': url = file['originalPicture'] - if validators.url(url, public=True): + if validate_import_url(url): response = requests.get(url) if imghdr.what(BytesIO(response.content)) is not None: self.import_recipe_image(recipe, BytesIO(response.content), filetype=get_filetype(file['originalPicture'])) diff --git a/cookbook/integration/recipesage.py b/cookbook/integration/recipesage.py index ab681c252..5a8c3182f 100644 --- a/cookbook/integration/recipesage.py +++ b/cookbook/integration/recipesage.py @@ -2,8 +2,8 @@ import json from io import BytesIO import requests -import validators +from cookbook.helper.HelperFunctions import validate_import_url from cookbook.helper.ingredient_parser import IngredientParser from cookbook.helper.recipe_url_import import parse_servings, parse_servings_text, parse_time from cookbook.integration.integration import Integration @@ -56,7 +56,7 @@ class RecipeSage(Integration): if len(file['image']) > 0: try: url = file['image'][0] - if validators.url(url, public=True): + if validate_import_url(url): response = requests.get(url) self.import_recipe_image(recipe, BytesIO(response.content)) except Exception as e: diff --git a/cookbook/provider/dropbox.py b/cookbook/provider/dropbox.py index 7d8dcc756..4d8f2f7ee 100644 --- a/cookbook/provider/dropbox.py +++ b/cookbook/provider/dropbox.py @@ -4,8 +4,8 @@ import os from datetime import datetime import requests -import validators +from cookbook.helper.HelperFunctions import validate_import_url from cookbook.models import Recipe, RecipeImport, SyncLog from cookbook.provider.provider import Provider @@ -107,7 +107,7 @@ class Dropbox(Provider): recipe.save() url = recipe.link.replace('www.dropbox.', 'dl.dropboxusercontent.') - if validators.url(url, public=True): + if validate_import_url(url): response = requests.get(url) return io.BytesIO(response.content) diff --git a/cookbook/provider/nextcloud.py b/cookbook/provider/nextcloud.py index 9399e104c..87c430587 100644 --- a/cookbook/provider/nextcloud.py +++ b/cookbook/provider/nextcloud.py @@ -4,8 +4,9 @@ import tempfile from datetime import datetime import requests -import validators import webdav3.client as wc + +from cookbook.helper.HelperFunctions import validate_import_url from cookbook.models import Recipe, RecipeImport, SyncLog from cookbook.provider.provider import Provider from requests.auth import HTTPBasicAuth @@ -93,7 +94,7 @@ class Nextcloud(Provider): "Content-Type": "application/json" } - if validators.url(url, public=True): + if validate_import_url(url): r = requests.get( url, headers=headers, diff --git a/cookbook/tests/other/docs/reports/tests/assets/style.css b/cookbook/tests/other/docs/reports/tests/assets/style.css new file mode 100644 index 000000000..561524c69 --- /dev/null +++ b/cookbook/tests/other/docs/reports/tests/assets/style.css @@ -0,0 +1,319 @@ +body { + font-family: Helvetica, Arial, sans-serif; + font-size: 12px; + /* do not increase min-width as some may use split screens */ + min-width: 800px; + color: #999; +} + +h1 { + font-size: 24px; + color: black; +} + +h2 { + font-size: 16px; + color: black; +} + +p { + color: black; +} + +a { + color: #999; +} + +table { + border-collapse: collapse; +} + +/****************************** + * SUMMARY INFORMATION + ******************************/ +#environment td { + padding: 5px; + border: 1px solid #e6e6e6; + vertical-align: top; +} +#environment tr:nth-child(odd) { + background-color: #f6f6f6; +} +#environment ul { + margin: 0; + padding: 0 20px; +} + +/****************************** + * TEST RESULT COLORS + ******************************/ +span.passed, +.passed .col-result { + color: green; +} + +span.skipped, +span.xfailed, +span.rerun, +.skipped .col-result, +.xfailed .col-result, +.rerun .col-result { + color: orange; +} + +span.error, +span.failed, +span.xpassed, +.error .col-result, +.failed .col-result, +.xpassed .col-result { + color: red; +} + +.col-links__extra { + margin-right: 3px; +} + +/****************************** + * RESULTS TABLE + * + * 1. Table Layout + * 2. Extra + * 3. Sorting items + * + ******************************/ +/*------------------ + * 1. Table Layout + *------------------*/ +#results-table { + border: 1px solid #e6e6e6; + color: #999; + font-size: 12px; + width: 100%; +} +#results-table th, +#results-table td { + padding: 5px; + border: 1px solid #e6e6e6; + text-align: left; +} +#results-table th { + font-weight: bold; +} + +/*------------------ + * 2. Extra + *------------------*/ +.logwrapper { + max-height: 230px; + overflow-y: scroll; + background-color: #e6e6e6; +} +.logwrapper.expanded { + max-height: none; +} +.logwrapper.expanded .logexpander:after { + content: "collapse [-]"; +} +.logwrapper .logexpander { + z-index: 1; + position: sticky; + top: 10px; + width: max-content; + border: 1px solid; + border-radius: 3px; + padding: 5px 7px; + margin: 10px 0 10px calc(100% - 80px); + cursor: pointer; + background-color: #e6e6e6; +} +.logwrapper .logexpander:after { + content: "expand [+]"; +} +.logwrapper .logexpander:hover { + color: #000; + border-color: #000; +} +.logwrapper .log { + min-height: 40px; + position: relative; + top: -50px; + height: calc(100% + 50px); + border: 1px solid #e6e6e6; + color: black; + display: block; + font-family: "Courier New", Courier, monospace; + padding: 5px; + padding-right: 80px; + white-space: pre-wrap; +} + +div.media { + border: 1px solid #e6e6e6; + float: right; + height: 240px; + margin: 0 5px; + overflow: hidden; + width: 320px; +} + +.media-container { + display: grid; + grid-template-columns: 25px auto 25px; + align-items: center; + flex: 1 1; + overflow: hidden; + height: 200px; +} + +.media-container--fullscreen { + grid-template-columns: 0px auto 0px; +} + +.media-container__nav--right, +.media-container__nav--left { + text-align: center; + cursor: pointer; +} + +.media-container__viewport { + cursor: pointer; + text-align: center; + height: inherit; +} +.media-container__viewport img, +.media-container__viewport video { + object-fit: cover; + width: 100%; + max-height: 100%; +} + +.media__name, +.media__counter { + display: flex; + flex-direction: row; + justify-content: space-around; + flex: 0 0 25px; + align-items: center; +} + +.collapsible td:not(.col-links) { + cursor: pointer; +} +.collapsible td:not(.col-links):hover::after { + color: #bbb; + font-style: italic; + cursor: pointer; +} + +.col-result { + width: 130px; +} +.col-result:hover::after { + content: " (hide details)"; +} + +.col-result.collapsed:hover::after { + content: " (show details)"; +} + +#environment-header h2:hover::after { + content: " (hide details)"; + color: #bbb; + font-style: italic; + cursor: pointer; + font-size: 12px; +} + +#environment-header.collapsed h2:hover::after { + content: " (show details)"; + color: #bbb; + font-style: italic; + cursor: pointer; + font-size: 12px; +} + +/*------------------ + * 3. Sorting items + *------------------*/ +.sortable { + cursor: pointer; +} +.sortable.desc:after { + content: " "; + position: relative; + left: 5px; + bottom: -12.5px; + border: 10px solid #4caf50; + border-bottom: 0; + border-left-color: transparent; + border-right-color: transparent; +} +.sortable.asc:after { + content: " "; + position: relative; + left: 5px; + bottom: 12.5px; + border: 10px solid #4caf50; + border-top: 0; + border-left-color: transparent; + border-right-color: transparent; +} + +.hidden, .summary__reload__button.hidden { + display: none; +} + +.summary__data { + flex: 0 0 550px; +} +.summary__reload { + flex: 1 1; + display: flex; + justify-content: center; +} +.summary__reload__button { + flex: 0 0 300px; + display: flex; + color: white; + font-weight: bold; + background-color: #4caf50; + text-align: center; + justify-content: center; + align-items: center; + border-radius: 3px; + cursor: pointer; +} +.summary__reload__button:hover { + background-color: #46a049; +} +.summary__spacer { + flex: 0 0 550px; +} + +.controls { + display: flex; + justify-content: space-between; +} + +.filters, +.collapse { + display: flex; + align-items: center; +} +.filters button, +.collapse button { + color: #999; + border: none; + background: none; + cursor: pointer; + text-decoration: underline; +} +.filters button:hover, +.collapse button:hover { + color: #ccc; +} + +.filter__label { + margin-right: 10px; +} diff --git a/cookbook/tests/other/docs/reports/tests/pytest.xml b/cookbook/tests/other/docs/reports/tests/pytest.xml new file mode 100644 index 000000000..de1721f92 --- /dev/null +++ b/cookbook/tests/other/docs/reports/tests/pytest.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/cookbook/tests/other/docs/reports/tests/tests.html b/cookbook/tests/other/docs/reports/tests/tests.html new file mode 100644 index 000000000..6fbff23e4 --- /dev/null +++ b/cookbook/tests/other/docs/reports/tests/tests.html @@ -0,0 +1,770 @@ + + + + + tests.html + + + +

tests.html

+

Report generated on 20-Aug-2024 at 11:35:26 by pytest-html + v4.1.1

+
+

Environment

+
+
+ + + + + +
+
+

Summary

+
+
+

1 test took 00:00:22.

+

(Un)check the boxes to filter the results.

+
+ +
+
+
+
+ + 0 Failed, + + 1 Passed, + + 0 Skipped, + + 0 Expected failures, + + 0 Unexpected passes, + + 0 Errors, + + 0 Reruns +
+
+  /  +
+
+
+
+
+
+
+
+ + + + + + + + + +
ResultTestDurationLinks
+ + + \ No newline at end of file diff --git a/cookbook/tests/other/test_helpers.py b/cookbook/tests/other/test_helpers.py new file mode 100644 index 000000000..6ffacde8e --- /dev/null +++ b/cookbook/tests/other/test_helpers.py @@ -0,0 +1,14 @@ +from cookbook.helper.HelperFunctions import validate_import_url + + +def test_url_validator(): + # neither local nor public urls without protocol are valid + assert not validate_import_url('localhost:8080') + assert not validate_import_url('www.google.com') + + # public urls with schema and parameters are valid + assert validate_import_url('https://www.google.com') + assert validate_import_url('https://cheatsheetseries.owasp.org/cheatsheets/Server_Side_Request_Forgery_Prevention_Cheat_Sheet.html#case-2-application-can-send-requests-to-any-external-ip-address-or-domain-name') + + assert not validate_import_url('https://localhost') + assert not validate_import_url('http://127.0.0.1') \ No newline at end of file diff --git a/cookbook/views/api.py b/cookbook/views/api.py index a95716a66..21a21d4e6 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -13,7 +13,6 @@ from urllib.parse import unquote from zipfile import ZipFile import requests -import validators from PIL import UnidentifiedImageError from annoying.decorators import ajax_request from annoying.functions import get_object_or_None @@ -53,7 +52,7 @@ from treebeard.exceptions import InvalidMoveToDescendant, InvalidPosition, PathO from cookbook.forms import ImportForm from cookbook.helper import recipe_url_import as helper -from cookbook.helper.HelperFunctions import str2bool +from cookbook.helper.HelperFunctions import str2bool, validate_import_url from cookbook.helper.image_processing import handle_image from cookbook.helper.ingredient_parser import IngredientParser from cookbook.helper.open_data_importer import OpenDataImporter @@ -994,7 +993,7 @@ class RecipeViewSet(viewsets.ModelViewSet): elif 'image_url' in serializer.validated_data: try: url = serializer.validated_data['image_url'] - if validators.url(url, public=True): + if validate_import_url(url): response = requests.get(url, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0"}) image = File(io.BytesIO(response.content)) filetype = mimetypes.guess_extension(response.headers['content-type']) or filetype @@ -1416,7 +1415,7 @@ class RecipeUrlImportView(APIView): elif url and not data: if re.match('^(https?://)?(www\\.youtube\\.com|youtu\\.be)/.+$', url): - if validators.url(url, public=True): + if validate_import_url(url): return Response({'recipe_json': get_from_youtube_scraper(url, request), 'recipe_images': [], }, status=status.HTTP_200_OK) if re.match('^(.)*/view/recipe/[0-9]+/[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', url): recipe_json = requests.get( @@ -1426,7 +1425,7 @@ class RecipeUrlImportView(APIView): serialized_recipe = RecipeExportSerializer(data=recipe_json, context={'request': request}) if serialized_recipe.is_valid(): recipe = serialized_recipe.save() - if validators.url(recipe_json['image'], public=True): + if validate_import_url(recipe_json['image']): recipe.image = File(handle_image(request, File(io.BytesIO(requests.get(recipe_json['image']).content), name='image'), filetype=pathlib.Path(recipe_json['image']).suffix), @@ -1435,7 +1434,7 @@ class RecipeUrlImportView(APIView): return Response({'link': request.build_absolute_uri(reverse('view_recipe', args={recipe.pk}))}, status=status.HTTP_201_CREATED) else: try: - if validators.url(url, public=True): + if validate_import_url(url): html = requests.get( url, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0"} diff --git a/requirements.txt b/requirements.txt index c9cbc4b0c..713738956 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,6 @@ django-hCaptcha==0.2.0 python-ldap==3.4.4 django-auth-ldap==4.6.0 pyppeteer==2.0.0 -validators==0.33.0 pytube==15.0.0 aiohttp==3.10.2 inflection==0.5.1