Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: allow custom converters and aliases for dataclasses

NTFSvolumec403ec579db5
diff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.py
index dcc6201eb..98df8e37e 100644
--- a/cyberdrop_dl/utils/__init__.py
+++ b/cyberdrop_dl/utils/__init__.py
@@ -131,12 +131,13 @@ def basic_auth(username: str, password: str) -> str:
return f"Basic {token}"
-def unique[T](itr: Iterable[T], /) -> Generator[T]:
- seen: set[T] = set()
- for ele in itr:
- if ele not in seen:
- seen.add(ele)
- yield ele
+def unique[T](itr: Iterable[T], /, key: Callable[[T], Any] | None = None) -> Generator[T]:
+ seen: set[Any] = set()
+ for elem in itr:
+ lookup = elem if key is None else key(elem)
+ if lookup not in seen:
+ seen.add(lookup)
+ yield elem
def fast_cache[T, R](fn: Callable[[T], R]) -> Callable[[T], R]:
diff --git a/cyberdrop_dl/utils/dataclass.py b/cyberdrop_dl/utils/dataclass.py
index 9578c8558..4678895ac 100644
--- a/cyberdrop_dl/utils/dataclass.py
+++ b/cyberdrop_dl/utils/dataclass.py
@@ -7,7 +7,7 @@ from cyberdrop_dl.constants import MISSING
from cyberdrop_dl.utils import fast_cache
if TYPE_CHECKING:
- from collections.abc import Iterator, Mapping
+ from collections.abc import Callable, Generator, Iterator, Mapping, MutableMapping
class _DataClass(Protocol):
@@ -26,14 +26,43 @@ def filter_data[DataClassT: _DataClass](cls: type[DataClassT], data: Mapping[str
return {name: value for name in _fields_names(cls) if (value := data.get(name, MISSING)) is not MISSING}
-def deserialize[DataClassT: _DataClass](
- cls: type[DataClassT], data: Mapping[str, Any], /, **overrides: Any
-) -> DataClassT:
- """Parses 'data' to build an instance of 'cls' without type validation"""
- data = filter_data(cls, data)
- if overrides:
- data.update(overrides)
- return cls(**data)
[email protected](slots=True, frozen=True, eq=False)
+class Deserializer:
+ aliases: Mapping[str, str] | None = None
+ converters: Mapping[str, Callable[[Any], Any]] | None = None
+
+ def __call__[T: _DataClass](self, cls: type[T], data: Mapping[str, Any], **overrides: Any) -> T:
+ params = filter_data(cls, data)
+ if overrides:
+ params.update(overrides)
+
+ for name, value in self._extract_aliases(data):
+ params.setdefault(name, value)
+
+ self._apply_converters(params)
+ return cls(**params)
+
+ def _extract_aliases(self, data: Mapping[str, Any]) -> Generator[tuple[str, Any]]:
+ if self.aliases:
+ for name, alias in self.aliases.items():
+ try:
+ value = data[alias]
+ except KeyError:
+ continue
+ yield name, value
+
+ def _apply_converters(self, params: MutableMapping[str, Any]) -> None:
+ if self.converters:
+ for name, coerce in self.converters.items():
+ try:
+ value = params[name]
+ except KeyError:
+ continue
+
+ params[name] = coerce(value)
+
+
+deserialize = Deserializer()
class DictDataclass(_DataClass, Protocol):
diff --git a/tests/test_dataclass.py b/tests/test_dataclass.py
new file mode 100644
index 000000000..6cd0bff5f
--- /dev/null
+++ b/tests/test_dataclass.py
@@ -0,0 +1,60 @@
+import dataclasses
+
+import pytest
+
+from cyberdrop_dl.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils import dataclass
+from cyberdrop_dl.utils._url import parse_http_url
+
+
+class A:
+ b: int
+ c: str
+
+
+def test_default_deserialize() -> None:
+ result = dataclass.deserialize(A, {"a": 1, "b": 2, "c": "string"})
+ assert type(result) is A
+ assert result.b == 2
+ assert result.c == "string"
+
+
+def test_deserialize_w_overrides() -> None:
+ result = dataclass.deserialize(A, {"a": 1, "b": 2}, c="string")
+ assert result.b == 2
+ assert result.c == "string"
+ result = dataclass.deserialize(A, {"a": 1, "b": 2, "c": "string"}, c="other string")
+ assert result.c == "other string"
+
+
+def test_deserialize_w_aliases() -> None:
+ data = {"a": 1, "b": 2, "custom_key": "string"}
+
+ with pytest.raises(TypeError):
+ dataclass.deserialize(A, data)
+ deserialize = dataclass.Deserializer({"c": "custom_key"})
+ result = deserialize(A, data)
+ assert result.b == 2
+ assert result.c == "string"
+
+
+def test_deserialize_w_converter() -> None:
+
+ @dataclasses.dataclass
+ class B:
+ a: int
+ b: int
+ c: AbsoluteHttpURL
+
+ data = {"a": 1, "b": "2", "c": "https://example.com"}
+
+ deserialize = dataclass.Deserializer(converters={"b": int, "c": parse_http_url})
+ result = deserialize(B, data)
+ assert type(result) is B
+ assert result.a == 1
+ assert result.b == 2
+ assert result.c == AbsoluteHttpURL("https://example.com")
+
+ with pytest.raises(ValueError, match="Relative URL with no known origin"):
+ deserialize(B, {"a": 1, "b": "2", "c": "not and url"})