Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: add ld+json utils

NTFSvolume2d7bad54fd5e
diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.py
index 1d3c51fb1..789bdd8c7 100644
--- a/cyberdrop_dl/utils/css.py
+++ b/cyberdrop_dl/utils/css.py
@@ -127,6 +127,18 @@ def iselect(tag: Tag, selector: str, attribute: str | None = None) -> Generator[
yield attr
+def iselect_text(soup: Tag, /, selector: str, contains: tuple[str, ...] | str = ()) -> Generator[str]:
+ if isinstance(contains, str):
+ contains = (contains,)
+
+ for tag in iselect(soup, selector):
+ content = text(tag)
+ for key in contains:
+ if key not in content:
+ continue
+ yield content
+
+
def _parse_srcset(srcset: str) -> str:
# The best src is the last one (usually)
return [src.split(" ")[0] for src in srcset.split(", ")][-1]
diff --git a/cyberdrop_dl/utils/json_ld.py b/cyberdrop_dl/utils/json_ld.py
new file mode 100644
index 000000000..c3d3dc563
--- /dev/null
+++ b/cyberdrop_dl/utils/json_ld.py
@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+from cyberdrop_dl.utils import css, dates, traversal
+
+if TYPE_CHECKING:
+ from collections.abc import Iterable, Mapping
+
+ from bs4.element import Tag
+
+
+def _find(
+ soup: Tag, keys: tuple[str, ...], validate: Mapping[str, Any] | None = None
+) -> tuple[tuple[Any, ...], traversal.DictVisitor]:
+ visitor = traversal.DictVisitor(keys, validate)
+ json_lds = tuple(iselect(soup, tuple(visitor.target_keys)))
+ return json_lds, visitor
+
+
+def ifind(
+ soup: Tag, *keys: str, validate: Mapping[str, Any] | None = None
+) -> Iterable[tuple[traversal.Path, dict[str, Any]]]:
+ json_lds, visitor = _find(soup, keys, validate)
+ return traversal.traverse(json_lds, visitor)
+
+
+def find(soup: Tag, *keys: str, validate: Mapping[str, Any] | None = None) -> tuple[traversal.Path, dict[str, Any]]:
+ json_lds, visitor = _find(soup, keys, validate)
+ try:
+ return next(traversal.traverse(json_lds, visitor))
+ except StopIteration:
+ contains = tuple(sorted(visitor.target_keys))
+ details = f" with keys {contains}" if contains else ""
+ raise css.SelectorError(f"ld-json{details} not found") from None
+
+
+def upload_date(soup: Tag, *keys: str, validate: Mapping[str, Any] | None = None) -> float:
+ _, obj = find(soup, "uploadDate", *keys, validate=validate)
+ return dates.parse_iso(obj["uploadDate"]).timestamp()
+
+
+def iselect(soup: Tag, /, contains: tuple[str, ...] | str = ()) -> Iterable[Any]:
+ return map(json.loads, css.iselect_text(soup, "script[type='application/ld+json']", contains))
diff --git a/cyberdrop_dl/utils/traversal.py b/cyberdrop_dl/utils/traversal.py
new file mode 100644
index 000000000..adc5ecdb1
--- /dev/null
+++ b/cyberdrop_dl/utils/traversal.py
@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import contextlib
+from collections.abc import Callable, Generator, Mapping
+from typing import Any, TypeGuard, overload
+
+from cyberdrop_dl.exceptions import ScrapeError
+from cyberdrop_dl.signature import simple_repr
+
+type Path = tuple[str | int, ...]
+type Validator = Callable[[object], bool]
+
+
+@overload
+def traverse[T](
+ value: object, visitor: Callable[[Path, Any], TypeGuard[T]], path: Path = ()
+) -> Generator[tuple[Path, T]]: ...
+
+
+@overload
+def traverse(value: object, visitor: Callable[[Path, Any], bool], path: Path = ()) -> Generator[tuple[Path, Any]]: ...
+
+
+def traverse(value: object, visitor: Callable[[Path, Any], Any], path: Path = ()) -> Generator[tuple[Path, Any]]:
+ if visitor(path, value):
+ yield path, value
+ return
+
+ match value:
+ case dict():
+ for key, item in value.items():
+ yield from traverse(item, visitor, (*path, key))
+
+ case list() | tuple():
+ for index, item in enumerate(value):
+ yield from traverse(item, visitor, (*path, index))
+
+ case _:
+ pass
+
+
+def catch[T](name: str | None = None) -> Generator[None]:
+ try:
+ yield
+ except StopIteration:
+ raise ScrapeError(422, f"Unable to find {name}" if name else None) from None
+
+
+def is_class_info(obj: object) -> TypeGuard[type | tuple[type, ...]]:
+ return isinstance(obj, type) or (isinstance(obj, tuple) and all(isinstance(val, type) for val in obj))
+
+
+def cast_validator(obj: object) -> Validator:
+ if obj is ...:
+ return lambda _: True
+ if is_class_info(obj):
+ return lambda x: isinstance(x, obj)
+
+ return lambda x: x == obj
+
+
+class DictVisitor:
+ def __init__(self, keys: tuple[str, ...], validate: Mapping[str, Any] | None = None) -> None:
+ validate = validate or {}
+ self.target_keys: set[str] = set(validate).union(keys)
+ self.validators: dict[str, Validator] = {key: cast_validator(value) for key, value in validate.items()}
+
+ __repr__ = simple_repr("target_keys", "validators")
+
+ def __call__(self, path: Path, obj: object) -> TypeGuard[dict[str, Any]]: # noqa: ARG002
+ if not (isinstance(obj, dict) and self.target_keys.issubset(obj)):
+ return False
+
+ return all(validator(obj[name]) for name, validator in self.validators.items())
+
+
+def find_objs(
+ value: object, *keys: str, validate: Mapping[str, Any] | None = None
+) -> Generator[tuple[Path, dict[str, Any]]]:
+ return traverse(value, DictVisitor(keys, validate))
+
+
+def find_obj(value: object, *keys: str, validate: Mapping[str, Any] | None = None) -> tuple[Path, dict[str, Any]]:
+ visitor = DictVisitor(keys, validate)
+ with catch(f"object with keys = {sorted(visitor.target_keys)}"):
+ return next(iter(traverse(value, visitor)))