Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: add strict parsers for common attributes (open graph) (#1483)

NTFSvolume71e7c0fa8c7a
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
index bbffa86db..f74645ec6 100644
--- a/cyberdrop_dl/crawlers/_chevereto.py
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -142,8 +142,7 @@ class CheveretoCrawler(Crawler, is_generic=True):
await self._unlock_password_protected_album(scrape_item)
return await self.album(scrape_item, album_id)
- title = open_graph.get_title(soup) or open_graph.get("description", soup) or ""
- assert title
+ title = open_graph.get_title(soup) or open_graph.description(soup)
title = self.create_title(title, album_id)
scrape_item.setup_as_album(title, album_id=album_id)
self._process_page(scrape_item, soup, results)
@@ -198,7 +197,7 @@ class CheveretoCrawler(Crawler, is_generic=True):
return
soup = await self.request_soup(scrape_item.url)
- link_str = open_graph.get("video", soup) or open_graph.get("image", soup)
+ link_str = open_graph.get_video(soup) or open_graph.get_image(soup)
if not link_str or "loading.svg" in link_str:
link_str = Selector.MAIN_IMAGE(soup)
diff --git a/cyberdrop_dl/crawlers/archivebate.py b/cyberdrop_dl/crawlers/archivebate.py
index a2884b1ed..054e2d374 100644
--- a/cyberdrop_dl/crawlers/archivebate.py
+++ b/cyberdrop_dl/crawlers/archivebate.py
@@ -67,8 +67,7 @@ class ArchiveBateCrawler(MixDropCrawler):
if "This video has been deleted" in soup.text:
raise ScrapeError(410)
- description = open_graph.get("description", soup)
- assert description
+ description = open_graph.description(soup)
date_str = get_text_between(description, "show on", " - ").strip()
user_name = css.select_one_get_text(soup, _SELECTORS.USER_NAME)
site_name = css.select_one_get_text(soup, _SELECTORS.SITE_NAME)
diff --git a/cyberdrop_dl/utils/open_graph.py b/cyberdrop_dl/utils/open_graph.py
index fe755c9aa..0510506c7 100644
--- a/cyberdrop_dl/utils/open_graph.py
+++ b/cyberdrop_dl/utils/open_graph.py
@@ -1,17 +1,33 @@
+"""The Open Graph protocol: https://ogp.me/"""
+
from __future__ import annotations
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Final
from cyberdrop_dl.exceptions import ScrapeError
if TYPE_CHECKING:
+ from collections.abc import Callable
+
import bs4
-_required_attrs = ("title", "type", "image", "url", "description")
+_REQUIRED_ATTRS: Final = ("title", "type", "image", "url", "description")
+
+
+def _make_selector(name: str) -> str:
+ return f'meta[property^="og:{name}"][content], meta[name^="og:{name}"][content]'
+
+
+_ALL = _make_selector("")
+
+
+class OpenGraphError(ScrapeError):
+ def __init__(self, property: str | int) -> None:
+ super().__init__(422, f"Page have no {property} [og properties]")
class OpenGraph(dict[str, str | None]):
- """Open Graph properties. Each attribute corresponds to an OG property."""
+ """Open Graph properties"""
title: str
type: str
@@ -19,23 +35,27 @@ class OpenGraph(dict[str, str | None]):
url: str
description: str
- def __getattr__(self, name) -> str | None:
- return self.get(name, None)
+ def __getattr__(self, name: str) -> str | None:
+ value = self.get(name)
+ if name in _REQUIRED_ATTRS and not value:
+ raise OpenGraphError(name)
+ return value
def is_valid(self) -> bool:
- return all(self.get(attr) for attr in _required_attrs)
+ return all(self.get(attr) for attr in _REQUIRED_ATTRS)
def parse(soup: bs4.BeautifulSoup) -> OpenGraph:
- """Extracts Open Graph properties (og properties) from soup."""
+ """Extracts Open Graph (og) properties from soup."""
og_props = OpenGraph()
- for meta in soup.select('meta[property^="og:"][content], meta[name^="og:"][content]'):
- try:
- name = _value(meta, "property")
- except IndexError:
- name = _value(meta, "name")
- name = name.replace("og:", "").replace(":", "_")
- if value := _value(meta):
+ for meta in soup.select(_ALL):
+ if value := _get_attr(meta, "content"):
+ try:
+ name = _get_attr(meta, "property")
+ except LookupError:
+ name = _get_attr(meta, "name")
+
+ name = name.removeprefix("og:").replace(":", "_")
og_props[name] = value
if not og_props.get("title") and (title := soup.select_one("title, h1")):
@@ -45,21 +65,30 @@ def parse(soup: bs4.BeautifulSoup) -> OpenGraph:
def get(name: str, /, soup: bs4.BeautifulSoup) -> str | None:
- if meta := soup.select_one(f'meta[property^="og:{name}"][content], meta[name^="og:{name}"][content]'):
- return _value(meta)
+ if meta := soup.select_one(_make_selector(name)):
+ return _get_attr(meta, "content")
-def get_title(soup: bs4.BeautifulSoup) -> str | None:
- return get("title", soup)
+def _get_attr(meta: bs4.Tag, name: str) -> str:
+ value = meta[name]
+ assert isinstance(value, str)
+ return value.strip()
-def title(soup: bs4.BeautifulSoup) -> str:
- if title := get_title(soup):
- return title
- raise ScrapeError(422, "Page have no title [og properties]")
+def _make_parsers(name: str) -> tuple[Callable[[bs4.BeautifulSoup], str], Callable[[bs4.BeautifulSoup], str | None]]:
+ def get_value(soup: bs4.BeautifulSoup) -> str | None:
+ return get(name, soup)
+ def _value(soup: bs4.BeautifulSoup) -> str:
+ value = get_value(soup)
+ if not value:
+ raise OpenGraphError(name)
+ return value
-def _value(meta: bs4.Tag, name: str = "content") -> str:
- value = meta[name]
- assert isinstance(value, str)
- return value.strip()
+ return _value, get_value
+
+
+video, get_video = _make_parsers("video")
+image, get_image = _make_parsers("image")
+title, get_title = _make_parsers("title")
+description, get_description = _make_parsers("description")