Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: add ld+json utils
NTFSvolume2d7bad54fd5e
diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.pyindex 1d3c51fb1..789bdd8c7 100644--- a/cyberdrop_dl/utils/css.py+++ b/cyberdrop_dl/utils/css.py@@ -127,6 +127,18 @@ def iselect(tag: Tag, selector: str, attribute: str | None = None) -> Generator[yield attr+def iselect_text(soup: Tag, /, selector: str, contains: tuple[str, ...] | str = ()) -> Generator[str]:+ if isinstance(contains, str):+ contains = (contains,)++ for tag in iselect(soup, selector):+ content = text(tag)+ for key in contains:+ if key not in content:+ continue+ yield content++def _parse_srcset(srcset: str) -> str:# The best src is the last one (usually)return [src.split(" ")[0] for src in srcset.split(", ")][-1]diff --git a/cyberdrop_dl/utils/json_ld.py b/cyberdrop_dl/utils/json_ld.pynew file mode 100644index 000000000..c3d3dc563--- /dev/null+++ b/cyberdrop_dl/utils/json_ld.py@@ -0,0 +1,45 @@+from __future__ import annotations++import json+from typing import TYPE_CHECKING, Any++from cyberdrop_dl.utils import css, dates, traversal++if TYPE_CHECKING:+ from collections.abc import Iterable, Mapping++ from bs4.element import Tag+++def _find(+ soup: Tag, keys: tuple[str, ...], validate: Mapping[str, Any] | None = None+) -> tuple[tuple[Any, ...], traversal.DictVisitor]:+ visitor = traversal.DictVisitor(keys, validate)+ json_lds = tuple(iselect(soup, tuple(visitor.target_keys)))+ return json_lds, visitor+++def ifind(+ soup: Tag, *keys: str, validate: Mapping[str, Any] | None = None+) -> Iterable[tuple[traversal.Path, dict[str, Any]]]:+ json_lds, visitor = _find(soup, keys, validate)+ return traversal.traverse(json_lds, visitor)+++def find(soup: Tag, *keys: str, validate: Mapping[str, Any] | None = None) -> tuple[traversal.Path, dict[str, Any]]:+ json_lds, visitor = _find(soup, keys, validate)+ try:+ return next(traversal.traverse(json_lds, visitor))+ except StopIteration:+ contains = tuple(sorted(visitor.target_keys))+ details = f" with keys {contains}" if contains else ""+ raise css.SelectorError(f"ld-json{details} not found") from None+++def upload_date(soup: Tag, *keys: str, validate: Mapping[str, Any] | None = None) -> float:+ _, obj = find(soup, "uploadDate", *keys, validate=validate)+ return dates.parse_iso(obj["uploadDate"]).timestamp()+++def iselect(soup: Tag, /, contains: tuple[str, ...] | str = ()) -> Iterable[Any]:+ return map(json.loads, css.iselect_text(soup, "script[type='application/ld+json']", contains))diff --git a/cyberdrop_dl/utils/traversal.py b/cyberdrop_dl/utils/traversal.pynew file mode 100644index 000000000..adc5ecdb1--- /dev/null+++ b/cyberdrop_dl/utils/traversal.py@@ -0,0 +1,87 @@+from __future__ import annotations++import contextlib+from collections.abc import Callable, Generator, Mapping+from typing import Any, TypeGuard, overload++from cyberdrop_dl.exceptions import ScrapeError+from cyberdrop_dl.signature import simple_repr++type Path = tuple[str | int, ...]+type Validator = Callable[[object], bool]+++@overload+def traverse[T](+ value: object, visitor: Callable[[Path, Any], TypeGuard[T]], path: Path = ()+) -> Generator[tuple[Path, T]]: ...+++@overload+def traverse(value: object, visitor: Callable[[Path, Any], bool], path: Path = ()) -> Generator[tuple[Path, Any]]: ...+++def traverse(value: object, visitor: Callable[[Path, Any], Any], path: Path = ()) -> Generator[tuple[Path, Any]]:+ if visitor(path, value):+ yield path, value+ return++ match value:+ case dict():+ for key, item in value.items():+ yield from traverse(item, visitor, (*path, key))++ case list() | tuple():+ for index, item in enumerate(value):+ yield from traverse(item, visitor, (*path, index))++ case _:+ pass+++def catch[T](name: str | None = None) -> Generator[None]:+ try:+ yield+ except StopIteration:+ raise ScrapeError(422, f"Unable to find {name}" if name else None) from None+++def is_class_info(obj: object) -> TypeGuard[type | tuple[type, ...]]:+ return isinstance(obj, type) or (isinstance(obj, tuple) and all(isinstance(val, type) for val in obj))+++def cast_validator(obj: object) -> Validator:+ if obj is ...:+ return lambda _: True+ if is_class_info(obj):+ return lambda x: isinstance(x, obj)++ return lambda x: x == obj+++class DictVisitor:+ def __init__(self, keys: tuple[str, ...], validate: Mapping[str, Any] | None = None) -> None:+ validate = validate or {}+ self.target_keys: set[str] = set(validate).union(keys)+ self.validators: dict[str, Validator] = {key: cast_validator(value) for key, value in validate.items()}++ __repr__ = simple_repr("target_keys", "validators")++ def __call__(self, path: Path, obj: object) -> TypeGuard[dict[str, Any]]: # noqa: ARG002+ if not (isinstance(obj, dict) and self.target_keys.issubset(obj)):+ return False++ return all(validator(obj[name]) for name, validator in self.validators.items())+++def find_objs(+ value: object, *keys: str, validate: Mapping[str, Any] | None = None+) -> Generator[tuple[Path, dict[str, Any]]]:+ return traverse(value, DictVisitor(keys, validate))+++def find_obj(value: object, *keys: str, validate: Mapping[str, Any] | None = None) -> tuple[Path, dict[str, Any]]:+ visitor = DictVisitor(keys, validate)+ with catch(f"object with keys = {sorted(visitor.target_keys)}"):+ return next(iter(traverse(value, visitor)))