Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: allow custom converters and aliases for dataclasses
NTFSvolumec403ec579db5
diff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.pyindex dcc6201eb..98df8e37e 100644--- a/cyberdrop_dl/utils/__init__.py+++ b/cyberdrop_dl/utils/__init__.py@@ -131,12 +131,13 @@ def basic_auth(username: str, password: str) -> str:return f"Basic {token}"-def unique[T](itr: Iterable[T], /) -> Generator[T]:- seen: set[T] = set()- for ele in itr:- if ele not in seen:- seen.add(ele)- yield ele+def unique[T](itr: Iterable[T], /, key: Callable[[T], Any] | None = None) -> Generator[T]:+ seen: set[Any] = set()+ for elem in itr:+ lookup = elem if key is None else key(elem)+ if lookup not in seen:+ seen.add(lookup)+ yield elemdef fast_cache[T, R](fn: Callable[[T], R]) -> Callable[[T], R]:diff --git a/cyberdrop_dl/utils/dataclass.py b/cyberdrop_dl/utils/dataclass.pyindex 9578c8558..4678895ac 100644--- a/cyberdrop_dl/utils/dataclass.py+++ b/cyberdrop_dl/utils/dataclass.py@@ -7,7 +7,7 @@ from cyberdrop_dl.constants import MISSINGfrom cyberdrop_dl.utils import fast_cacheif TYPE_CHECKING:- from collections.abc import Iterator, Mapping+ from collections.abc import Callable, Generator, Iterator, Mapping, MutableMappingclass _DataClass(Protocol):@@ -26,14 +26,43 @@ def filter_data[DataClassT: _DataClass](cls: type[DataClassT], data: Mapping[strreturn {name: value for name in _fields_names(cls) if (value := data.get(name, MISSING)) is not MISSING}-def deserialize[DataClassT: _DataClass](- cls: type[DataClassT], data: Mapping[str, Any], /, **overrides: Any-) -> DataClassT:- """Parses 'data' to build an instance of 'cls' without type validation"""- data = filter_data(cls, data)- if overrides:- data.update(overrides)- return cls(**data)[email protected](slots=True, frozen=True, eq=False)+class Deserializer:+ aliases: Mapping[str, str] | None = None+ converters: Mapping[str, Callable[[Any], Any]] | None = None++ def __call__[T: _DataClass](self, cls: type[T], data: Mapping[str, Any], **overrides: Any) -> T:+ params = filter_data(cls, data)+ if overrides:+ params.update(overrides)++ for name, value in self._extract_aliases(data):+ params.setdefault(name, value)++ self._apply_converters(params)+ return cls(**params)++ def _extract_aliases(self, data: Mapping[str, Any]) -> Generator[tuple[str, Any]]:+ if self.aliases:+ for name, alias in self.aliases.items():+ try:+ value = data[alias]+ except KeyError:+ continue+ yield name, value++ def _apply_converters(self, params: MutableMapping[str, Any]) -> None:+ if self.converters:+ for name, coerce in self.converters.items():+ try:+ value = params[name]+ except KeyError:+ continue++ params[name] = coerce(value)+++deserialize = Deserializer()class DictDataclass(_DataClass, Protocol):diff --git a/tests/test_dataclass.py b/tests/test_dataclass.pynew file mode 100644index 000000000..6cd0bff5f--- /dev/null+++ b/tests/test_dataclass.py@@ -0,0 +1,60 @@+import dataclasses++import pytest++from cyberdrop_dl.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils import dataclass+from cyberdrop_dl.utils._url import parse_http_url+++class A:+ b: int+ c: str+++def test_default_deserialize() -> None:+ result = dataclass.deserialize(A, {"a": 1, "b": 2, "c": "string"})+ assert type(result) is A+ assert result.b == 2+ assert result.c == "string"+++def test_deserialize_w_overrides() -> None:+ result = dataclass.deserialize(A, {"a": 1, "b": 2}, c="string")+ assert result.b == 2+ assert result.c == "string"+ result = dataclass.deserialize(A, {"a": 1, "b": 2, "c": "string"}, c="other string")+ assert result.c == "other string"+++def test_deserialize_w_aliases() -> None:+ data = {"a": 1, "b": 2, "custom_key": "string"}++ with pytest.raises(TypeError):+ dataclass.deserialize(A, data)+ deserialize = dataclass.Deserializer({"c": "custom_key"})+ result = deserialize(A, data)+ assert result.b == 2+ assert result.c == "string"+++def test_deserialize_w_converter() -> None:++ @dataclasses.dataclass+ class B:+ a: int+ b: int+ c: AbsoluteHttpURL++ data = {"a": 1, "b": "2", "c": "https://example.com"}++ deserialize = dataclass.Deserializer(converters={"b": int, "c": parse_http_url})+ result = deserialize(B, data)+ assert type(result) is B+ assert result.a == 1+ assert result.b == 2+ assert result.c == AbsoluteHttpURL("https://example.com")++ with pytest.raises(ValueError, match="Relative URL with no known origin"):+ deserialize(B, {"a": 1, "b": "2", "c": "not and url"})