Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: add strict parsers for common attributes (open graph) (#1483)
NTFSvolume71e7c0fa8c7a
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.pyindex bbffa86db..f74645ec6 100644--- a/cyberdrop_dl/crawlers/_chevereto.py+++ b/cyberdrop_dl/crawlers/_chevereto.py@@ -142,8 +142,7 @@ class CheveretoCrawler(Crawler, is_generic=True):await self._unlock_password_protected_album(scrape_item)return await self.album(scrape_item, album_id)- title = open_graph.get_title(soup) or open_graph.get("description", soup) or ""- assert title+ title = open_graph.get_title(soup) or open_graph.description(soup)title = self.create_title(title, album_id)scrape_item.setup_as_album(title, album_id=album_id)self._process_page(scrape_item, soup, results)@@ -198,7 +197,7 @@ class CheveretoCrawler(Crawler, is_generic=True):returnsoup = await self.request_soup(scrape_item.url)- link_str = open_graph.get("video", soup) or open_graph.get("image", soup)+ link_str = open_graph.get_video(soup) or open_graph.get_image(soup)if not link_str or "loading.svg" in link_str:link_str = Selector.MAIN_IMAGE(soup)diff --git a/cyberdrop_dl/crawlers/archivebate.py b/cyberdrop_dl/crawlers/archivebate.pyindex a2884b1ed..054e2d374 100644--- a/cyberdrop_dl/crawlers/archivebate.py+++ b/cyberdrop_dl/crawlers/archivebate.py@@ -67,8 +67,7 @@ class ArchiveBateCrawler(MixDropCrawler):if "This video has been deleted" in soup.text:raise ScrapeError(410)- description = open_graph.get("description", soup)- assert description+ description = open_graph.description(soup)date_str = get_text_between(description, "show on", " - ").strip()user_name = css.select_one_get_text(soup, _SELECTORS.USER_NAME)site_name = css.select_one_get_text(soup, _SELECTORS.SITE_NAME)diff --git a/cyberdrop_dl/utils/open_graph.py b/cyberdrop_dl/utils/open_graph.pyindex fe755c9aa..0510506c7 100644--- a/cyberdrop_dl/utils/open_graph.py+++ b/cyberdrop_dl/utils/open_graph.py@@ -1,17 +1,33 @@+"""The Open Graph protocol: https://ogp.me/"""+from __future__ import annotations-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Finalfrom cyberdrop_dl.exceptions import ScrapeErrorif TYPE_CHECKING:+ from collections.abc import Callable+import bs4-_required_attrs = ("title", "type", "image", "url", "description")+_REQUIRED_ATTRS: Final = ("title", "type", "image", "url", "description")+++def _make_selector(name: str) -> str:+ return f'meta[property^="og:{name}"][content], meta[name^="og:{name}"][content]'+++_ALL = _make_selector("")+++class OpenGraphError(ScrapeError):+ def __init__(self, property: str | int) -> None:+ super().__init__(422, f"Page have no {property} [og properties]")class OpenGraph(dict[str, str | None]):- """Open Graph properties. Each attribute corresponds to an OG property."""+ """Open Graph properties"""title: strtype: str@@ -19,23 +35,27 @@ class OpenGraph(dict[str, str | None]):url: strdescription: str- def __getattr__(self, name) -> str | None:- return self.get(name, None)+ def __getattr__(self, name: str) -> str | None:+ value = self.get(name)+ if name in _REQUIRED_ATTRS and not value:+ raise OpenGraphError(name)+ return valuedef is_valid(self) -> bool:- return all(self.get(attr) for attr in _required_attrs)+ return all(self.get(attr) for attr in _REQUIRED_ATTRS)def parse(soup: bs4.BeautifulSoup) -> OpenGraph:- """Extracts Open Graph properties (og properties) from soup."""+ """Extracts Open Graph (og) properties from soup."""og_props = OpenGraph()- for meta in soup.select('meta[property^="og:"][content], meta[name^="og:"][content]'):- try:- name = _value(meta, "property")- except IndexError:- name = _value(meta, "name")- name = name.replace("og:", "").replace(":", "_")- if value := _value(meta):+ for meta in soup.select(_ALL):+ if value := _get_attr(meta, "content"):+ try:+ name = _get_attr(meta, "property")+ except LookupError:+ name = _get_attr(meta, "name")++ name = name.removeprefix("og:").replace(":", "_")og_props[name] = valueif not og_props.get("title") and (title := soup.select_one("title, h1")):@@ -45,21 +65,30 @@ def parse(soup: bs4.BeautifulSoup) -> OpenGraph:def get(name: str, /, soup: bs4.BeautifulSoup) -> str | None:- if meta := soup.select_one(f'meta[property^="og:{name}"][content], meta[name^="og:{name}"][content]'):- return _value(meta)+ if meta := soup.select_one(_make_selector(name)):+ return _get_attr(meta, "content")-def get_title(soup: bs4.BeautifulSoup) -> str | None:- return get("title", soup)+def _get_attr(meta: bs4.Tag, name: str) -> str:+ value = meta[name]+ assert isinstance(value, str)+ return value.strip()-def title(soup: bs4.BeautifulSoup) -> str:- if title := get_title(soup):- return title- raise ScrapeError(422, "Page have no title [og properties]")+def _make_parsers(name: str) -> tuple[Callable[[bs4.BeautifulSoup], str], Callable[[bs4.BeautifulSoup], str | None]]:+ def get_value(soup: bs4.BeautifulSoup) -> str | None:+ return get(name, soup)+ def _value(soup: bs4.BeautifulSoup) -> str:+ value = get_value(soup)+ if not value:+ raise OpenGraphError(name)+ return value-def _value(meta: bs4.Tag, name: str = "content") -> str:- value = meta[name]- assert isinstance(value, str)- return value.strip()+ return _value, get_value+++video, get_video = _make_parsers("video")+image, get_image = _make_parsers("image")+title, get_title = _make_parsers("title")+description, get_description = _make_parsers("description")