mirrors/recipes
1
0
Fork 0
mirror of https://github.com/TandoorRecipes/recipes.git synced 2026-01-05 06:08:46 -05:00
Code Issues Packages Projects Releases Wiki Activity

Call scrape_me first when scraping from url

Browse Source
This commit is contained in:
Daniel Schulz
2022-05-10 00:08:37 +02:00
parent 33a7fee1cc
commit 2a7475c435
3 changed files with 50 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cookbook/helper/scrapers/scrapers.py
View File

@@ -1,6 +1,6 @@
from bs4 import BeautifulSoup
from json import JSONDecodeError
from recipe_scrapers import SCRAPERS, get_host_name
from recipe_scrapers import SCRAPERS
from recipe_scrapers._factory import SchemaScraperFactory
from recipe_scrapers._schemaorg import SchemaOrg
@@ -15,13 +15,7 @@ SCRAPERS.update(CUSTOM_SCRAPERS)
def text_scraper(text, url=None):
domain = None
if url:
domain = get_host_name(url)
if domain in SCRAPERS:
scraper_class = SCRAPERS[domain]
else:
scraper_class = SchemaScraperFactory.SchemaScraper
scraper_class = SchemaScraperFactory.SchemaScraper
class TextScraper(scraper_class):
def __init__(