Skip to content

Bulk asynchronous downloader for multiple file hosts

test: add tests for traversal (#2187)

NTFSvolumee55dd5b433be
diff --git a/cyberdrop_dl/utils/traversal.py b/cyberdrop_dl/utils/traversal.py
index c836753b6..5b6d52d03 100644
--- a/cyberdrop_dl/utils/traversal.py
+++ b/cyberdrop_dl/utils/traversal.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import contextlib
-from collections.abc import Callable, Generator, Mapping
+from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sequence
from typing import Any, TypeGuard, overload
from cyberdrop_dl.exceptions import ScrapeError
@@ -9,6 +9,37 @@ from cyberdrop_dl.signature import dataclass_repr
type Path = tuple[str | int, ...]
type Validator = Callable[[object], bool]
+type Visitor = Callable[[Path, Any], Any]
+
+
+@dataclass_repr("_traverse_gen", "_results", "_name")
+class TraverseResults[T, U]:
+ def __init__(self, traverse_gen: Iterable[tuple[T, U]], *, name: str | None = None) -> None:
+ self._traverse_gen: Iterable[tuple[T, U]] = traverse_gen
+ self._results: int = 0
+ self._name: str | None = name
+
+ def __iter__(self) -> Iterator[tuple[T, U]]:
+ for pair in self._traverse_gen:
+ self._results += 1
+ yield pair
+
+ def paths(self) -> Iterator[T]:
+ for path, _ in self:
+ yield path
+
+ def values(self) -> Iterator[U]:
+ for _, value in self:
+ yield value
+
+ def first(self, name: str | None = None) -> U:
+ if self._results > 0:
+ raise RuntimeError("We already yielded the first result")
+ name = name or self._name or "a required traverse attribute"
+ try:
+ return next(self.values())
+ except StopIteration as e:
+ raise ScrapeError(422, f"Unable to find {name}") from e
@overload
@@ -39,6 +70,17 @@ def traverse(value: object, visitor: Callable[[Path, Any], Any], path: Path = ()
pass
+def _traverse_many(
+ value: object, visitor: Visitor, next_visitors: Sequence[Visitor], *, path: Path = ()
+) -> Generator[tuple[Path, Any]]:
+ is_last_visitor = len(next_visitors) == 0
+ for current_path, inner_value in traverse(value, visitor, path):
+ if is_last_visitor:
+ yield current_path, inner_value
+ else:
+ yield from _traverse_many(inner_value, next_visitors[0], next_visitors[1:], path=current_path)
+
+
@contextlib.contextmanager
def catch[T](name: str | None = None) -> Generator[None]:
try:
@@ -51,12 +93,23 @@ def is_class_info(obj: object) -> TypeGuard[type | tuple[type, ...]]:
return isinstance(obj, type) or (isinstance(obj, tuple) and all(isinstance(val, type) for val in obj))
+def value_visitor[T, R](fn: Callable[[T], R]) -> Callable[[Any, T], R]:
+
+ def visit(_: Any, value: T) -> R:
+ return fn(value)
+
+ return visit
+
+
def cast_validator(obj: object) -> Validator:
if obj is ...:
return lambda _: True
if is_class_info(obj):
return lambda x: isinstance(x, obj)
+ if callable(obj):
+ return lambda x: bool(obj(x))
+
return lambda x: x == obj
@@ -84,3 +137,21 @@ def find_obj(value: object, *keys: str, validate: Mapping[str, Any] | None = Non
visitor = DictVisitor(keys, validate)
with catch(f"object with keys = {sorted(visitor.target_keys)}"):
return next(iter(traverse(value, visitor)))
+
+
+def traverse_many(
+ value: object, visitor: Visitor, *next_visitors: Visitor, path: Path = (), name: str | None = None
+) -> TraverseResults[Path, Any]:
+ return TraverseResults(_traverse_many(value, visitor, next_visitors, path=path), name=name)
+
+
+def traverse_v2(
+ value: object, cond: object, *next_conds: object, path: Path = (), name: str | None = None
+) -> TraverseResults[Path, Any]:
+ return traverse_many(
+ value,
+ value_visitor(cast_validator(cond)),
+ *(value_visitor(cast_validator(cond)) for cond in next_conds),
+ path=path,
+ name=name,
+ )
diff --git a/tests/test_traverse.py b/tests/test_traverse.py
new file mode 100644
index 000000000..7108e7bc4
--- /dev/null
+++ b/tests/test_traverse.py
@@ -0,0 +1,193 @@
+from typing import Any
+
+from cyberdrop_dl.utils.traversal import traverse
+
+
+def test_traverse() -> None:
+ data = {
+ "a": 1,
+ "b": "2",
+ "c": 3,
+ "d": ["", 2],
+ }
+ result = list(traverse(data, lambda _, value: isinstance(value, int)))
+
+ assert result == [
+ (("a",), 1),
+ (("c",), 3),
+ (("d", 1), 2),
+ ]
+
+
+def test_nested_dicts() -> None:
+ data = {
+ "a": {
+ "b": 1,
+ "c": "2",
+ "d": 3,
+ },
+ "e": 4,
+ }
+
+ result = list(traverse(data, lambda _, value: isinstance(value, int)))
+
+ assert result == [
+ (("a", "b"), 1),
+ (("a", "d"), 3),
+ (("e",), 4),
+ ]
+
+
+def test_mixed_nested_structures() -> None:
+ data = [
+ 1,
+ (
+ 2,
+ [3, "text", {}, 4, {}, 5],
+ ),
+ 6,
+ {
+ "users": [
+ {
+ "id": 1,
+ "name": "john",
+ }
+ ]
+ },
+ ]
+
+ result = list(traverse(data, lambda _, value: isinstance(value, int)))
+
+ assert result == [
+ ((0,), 1),
+ ((1, 0), 2),
+ ((1, 1, 0), 3),
+ ((1, 1, 3), 4),
+ ((1, 1, 5), 5),
+ ((2,), 6),
+ ((3, "users", 0, "id"), 1),
+ ]
+
+ result = list(traverse(data, lambda _, value: value and type(value) is dict))
+ assert result == [
+ (
+ (3,),
+ {
+ "users": [
+ {"id": 1, "name": "john"},
+ ]
+ },
+ ),
+ ]
+ result = list(traverse(data, lambda _, value: type(value) is dict and "name" in value))
+ assert result == [
+ (
+ (3, "users", 0),
+ {"id": 1, "name": "john"},
+ )
+ ]
+
+
+def test_matching_parent_stops_recursion() -> None:
+ data = {
+ "a": {
+ "b": 1,
+ },
+ "c": 2,
+ }
+
+ result = list(traverse(data, lambda _, value: isinstance(value, dict)))
+ assert result == [((), data)]
+
+
+def test_no_matches() -> None:
+ data = {"a": [1, 2], "b": 3}
+
+ result = list(traverse(data, lambda _, value: isinstance(value, tuple)))
+
+ assert result == []
+
+
+def test_root_value_matches() -> None:
+ value = 42
+
+ result = list(traverse(value, lambda _, item: item == 42))
+ assert result == [((), 42)]
+
+
+def test_root_path_is_preserved() -> None:
+ data = {"a": 1}
+
+ result = list(
+ traverse(
+ data,
+ lambda _, value: isinstance(value, int),
+ path=("root", 10),
+ )
+ )
+
+ assert result == [(("root", 10, "a"), 1)]
+
+
+def test_empty_containers() -> None:
+ data = {"a": [], "b": {}, "c": ()}
+ result = list(traverse(data, lambda *_: False))
+ assert result == []
+
+
+def test_empty_container_can_match() -> None:
+ data = {"a": [], "b": {}}
+ result = list(traverse(data, lambda _, value: isinstance(value, (list, dict))))
+ assert result == [((), data)]
+ result = list(traverse(data, lambda _, value: isinstance(value, list)))
+ assert result == [(("a",), [])]
+
+
+def test_preserves_dictionary_insertion_order():
+ data = {"z": 1, "a": 2, "m": 3}
+
+ result = list(traverse(data, lambda _, value: isinstance(value, int)))
+
+ assert result == [
+ (("z",), 1),
+ (("a",), 2),
+ (("m",), 3),
+ ]
+
+
+def test_visits_values_not_dictionary_keys() -> None:
+ data = {1: "one", 2: "two"}
+ result = list(traverse(data, lambda _, value: isinstance(value, int)))
+ assert result == []
+
+
+def test_visitor_receives_path_and_value() -> None:
+ calls = []
+ data = {"a": [10]}
+
+ def visitor(path: Any, value: Any):
+ calls.append((path, value))
+ return False
+
+ list(traverse(data, visitor))
+
+ assert calls == [
+ ((), {"a": [10]}),
+ (("a",), [10]),
+ (("a", 0), 10),
+ ]
+
+
+def test_does_not_mutate_traversed_objects() -> None:
+ from copy import deepcopy
+
+ inner_data = {"a": "target", "b": -1, 3: [123, 321]}
+ data = {1: "one", "my_key": {"another_key": [42, inner_data]}}
+ sample = deepcopy(data)
+ result = list(traverse(data, lambda _, value: type(value) is dict and value.get("a") == "target"))
+ assert result
+ path, value = result[0]
+ assert path == ("my_key", "another_key", 1)
+ assert value == inner_data
+ assert value is inner_data
+ assert data == sample