Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
test: add tests for traversal (#2187)
NTFSvolumee55dd5b433be
diff --git a/cyberdrop_dl/utils/traversal.py b/cyberdrop_dl/utils/traversal.pyindex c836753b6..5b6d52d03 100644--- a/cyberdrop_dl/utils/traversal.py+++ b/cyberdrop_dl/utils/traversal.py@@ -1,7 +1,7 @@from __future__ import annotationsimport contextlib-from collections.abc import Callable, Generator, Mapping+from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sequencefrom typing import Any, TypeGuard, overloadfrom cyberdrop_dl.exceptions import ScrapeError@@ -9,6 +9,37 @@ from cyberdrop_dl.signature import dataclass_reprtype Path = tuple[str | int, ...]type Validator = Callable[[object], bool]+type Visitor = Callable[[Path, Any], Any]+++@dataclass_repr("_traverse_gen", "_results", "_name")+class TraverseResults[T, U]:+ def __init__(self, traverse_gen: Iterable[tuple[T, U]], *, name: str | None = None) -> None:+ self._traverse_gen: Iterable[tuple[T, U]] = traverse_gen+ self._results: int = 0+ self._name: str | None = name++ def __iter__(self) -> Iterator[tuple[T, U]]:+ for pair in self._traverse_gen:+ self._results += 1+ yield pair++ def paths(self) -> Iterator[T]:+ for path, _ in self:+ yield path++ def values(self) -> Iterator[U]:+ for _, value in self:+ yield value++ def first(self, name: str | None = None) -> U:+ if self._results > 0:+ raise RuntimeError("We already yielded the first result")+ name = name or self._name or "a required traverse attribute"+ try:+ return next(self.values())+ except StopIteration as e:+ raise ScrapeError(422, f"Unable to find {name}") from e@overload@@ -39,6 +70,17 @@ def traverse(value: object, visitor: Callable[[Path, Any], Any], path: Path = ()pass+def _traverse_many(+ value: object, visitor: Visitor, next_visitors: Sequence[Visitor], *, path: Path = ()+) -> Generator[tuple[Path, Any]]:+ is_last_visitor = len(next_visitors) == 0+ for current_path, inner_value in traverse(value, visitor, path):+ if is_last_visitor:+ yield current_path, inner_value+ else:+ yield from _traverse_many(inner_value, next_visitors[0], next_visitors[1:], path=current_path)++@contextlib.contextmanagerdef catch[T](name: str | None = None) -> Generator[None]:try:@@ -51,12 +93,23 @@ def is_class_info(obj: object) -> TypeGuard[type | tuple[type, ...]]:return isinstance(obj, type) or (isinstance(obj, tuple) and all(isinstance(val, type) for val in obj))+def value_visitor[T, R](fn: Callable[[T], R]) -> Callable[[Any, T], R]:++ def visit(_: Any, value: T) -> R:+ return fn(value)++ return visit++def cast_validator(obj: object) -> Validator:if obj is ...:return lambda _: Trueif is_class_info(obj):return lambda x: isinstance(x, obj)+ if callable(obj):+ return lambda x: bool(obj(x))+return lambda x: x == obj@@ -84,3 +137,21 @@ def find_obj(value: object, *keys: str, validate: Mapping[str, Any] | None = Nonvisitor = DictVisitor(keys, validate)with catch(f"object with keys = {sorted(visitor.target_keys)}"):return next(iter(traverse(value, visitor)))+++def traverse_many(+ value: object, visitor: Visitor, *next_visitors: Visitor, path: Path = (), name: str | None = None+) -> TraverseResults[Path, Any]:+ return TraverseResults(_traverse_many(value, visitor, next_visitors, path=path), name=name)+++def traverse_v2(+ value: object, cond: object, *next_conds: object, path: Path = (), name: str | None = None+) -> TraverseResults[Path, Any]:+ return traverse_many(+ value,+ value_visitor(cast_validator(cond)),+ *(value_visitor(cast_validator(cond)) for cond in next_conds),+ path=path,+ name=name,+ )diff --git a/tests/test_traverse.py b/tests/test_traverse.pynew file mode 100644index 000000000..7108e7bc4--- /dev/null+++ b/tests/test_traverse.py@@ -0,0 +1,193 @@+from typing import Any++from cyberdrop_dl.utils.traversal import traverse+++def test_traverse() -> None:+ data = {+ "a": 1,+ "b": "2",+ "c": 3,+ "d": ["", 2],+ }+ result = list(traverse(data, lambda _, value: isinstance(value, int)))++ assert result == [+ (("a",), 1),+ (("c",), 3),+ (("d", 1), 2),+ ]+++def test_nested_dicts() -> None:+ data = {+ "a": {+ "b": 1,+ "c": "2",+ "d": 3,+ },+ "e": 4,+ }++ result = list(traverse(data, lambda _, value: isinstance(value, int)))++ assert result == [+ (("a", "b"), 1),+ (("a", "d"), 3),+ (("e",), 4),+ ]+++def test_mixed_nested_structures() -> None:+ data = [+ 1,+ (+ 2,+ [3, "text", {}, 4, {}, 5],+ ),+ 6,+ {+ "users": [+ {+ "id": 1,+ "name": "john",+ }+ ]+ },+ ]++ result = list(traverse(data, lambda _, value: isinstance(value, int)))++ assert result == [+ ((0,), 1),+ ((1, 0), 2),+ ((1, 1, 0), 3),+ ((1, 1, 3), 4),+ ((1, 1, 5), 5),+ ((2,), 6),+ ((3, "users", 0, "id"), 1),+ ]++ result = list(traverse(data, lambda _, value: value and type(value) is dict))+ assert result == [+ (+ (3,),+ {+ "users": [+ {"id": 1, "name": "john"},+ ]+ },+ ),+ ]+ result = list(traverse(data, lambda _, value: type(value) is dict and "name" in value))+ assert result == [+ (+ (3, "users", 0),+ {"id": 1, "name": "john"},+ )+ ]+++def test_matching_parent_stops_recursion() -> None:+ data = {+ "a": {+ "b": 1,+ },+ "c": 2,+ }++ result = list(traverse(data, lambda _, value: isinstance(value, dict)))+ assert result == [((), data)]+++def test_no_matches() -> None:+ data = {"a": [1, 2], "b": 3}++ result = list(traverse(data, lambda _, value: isinstance(value, tuple)))++ assert result == []+++def test_root_value_matches() -> None:+ value = 42++ result = list(traverse(value, lambda _, item: item == 42))+ assert result == [((), 42)]+++def test_root_path_is_preserved() -> None:+ data = {"a": 1}++ result = list(+ traverse(+ data,+ lambda _, value: isinstance(value, int),+ path=("root", 10),+ )+ )++ assert result == [(("root", 10, "a"), 1)]+++def test_empty_containers() -> None:+ data = {"a": [], "b": {}, "c": ()}+ result = list(traverse(data, lambda *_: False))+ assert result == []+++def test_empty_container_can_match() -> None:+ data = {"a": [], "b": {}}+ result = list(traverse(data, lambda _, value: isinstance(value, (list, dict))))+ assert result == [((), data)]+ result = list(traverse(data, lambda _, value: isinstance(value, list)))+ assert result == [(("a",), [])]+++def test_preserves_dictionary_insertion_order():+ data = {"z": 1, "a": 2, "m": 3}++ result = list(traverse(data, lambda _, value: isinstance(value, int)))++ assert result == [+ (("z",), 1),+ (("a",), 2),+ (("m",), 3),+ ]+++def test_visits_values_not_dictionary_keys() -> None:+ data = {1: "one", 2: "two"}+ result = list(traverse(data, lambda _, value: isinstance(value, int)))+ assert result == []+++def test_visitor_receives_path_and_value() -> None:+ calls = []+ data = {"a": [10]}++ def visitor(path: Any, value: Any):+ calls.append((path, value))+ return False++ list(traverse(data, visitor))++ assert calls == [+ ((), {"a": [10]}),+ (("a",), [10]),+ (("a", 0), 10),+ ]+++def test_does_not_mutate_traversed_objects() -> None:+ from copy import deepcopy++ inner_data = {"a": "target", "b": -1, 3: [123, 321]}+ data = {1: "one", "my_key": {"another_key": [42, inner_data]}}+ sample = deepcopy(data)+ result = list(traverse(data, lambda _, value: type(value) is dict and value.get("a") == "target"))+ assert result+ path, value = result[0]+ assert path == ("my_key", "another_key", 1)+ assert value == inner_data+ assert value is inner_data+ assert data == sample