Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: make json utils blocking (#77)
NTFSvolumef6fcabc5bd3e
diff --git a/cyberdrop_dl/managers/logs.py b/cyberdrop_dl/managers/logs.pyindex bf6308588..2f5138391 100644--- a/cyberdrop_dl/managers/logs.py+++ b/cyberdrop_dl/managers/logs.py@@ -71,7 +71,7 @@ class LogManager:async def write_jsonl(self, data: Iterable[dict[str, Any]]) -> None:async with self._file_locks[self.files.jsonl_file]:- await json.dump_jsonl(data, self.files.jsonl_file)+ await asyncio.to_thread(json.dump_jsonl, data, self.files.jsonl_file)async def _write_to_csv(self, file: Path, **row: object) -> None:async with self._file_locks[file]:diff --git a/cyberdrop_dl/utils/json.py b/cyberdrop_dl/utils/json.pyindex a218160a4..d9c6913ed 100644--- a/cyberdrop_dl/utils/json.py+++ b/cyberdrop_dl/utils/json.py@@ -1,6 +1,5 @@from __future__ import annotations-import asyncioimport base64import dataclassesimport datetime@@ -32,7 +31,7 @@ else:_py_make_scanner = json.scanner.py_make_scanner_encoders: dict[tuple[bool, int | None], LenientJSONEncoder] = {}-_REPLACE_QUOTES_PAIRS = [+_REPLACE_QUOTES_PAIRS = (("{'", '{"'),("'}", '"}'),("['", '["'),@@ -43,58 +42,48 @@ _REPLACE_QUOTES_PAIRS = [("' :", '" :'),("',", '",'),(": '", ': "'),-]+)class _DataclassInstance(Protocol):__dataclass_fields__: ClassVar[dict[str, dataclasses.Field[Any]]]+def default(obj: object, /) -> Any:+ if isinstance(obj, datetime.date):+ return obj.isoformat()+ if isinstance(obj, enum.Enum):+ return obj.value+ if isinstance(obj, set):+ return sorted(obj)+ if callable(serialize := getattr(obj, "__json__", None)):+ return serialize()+ if _is_namedtuple_instance(obj):+ return obj._asdict()+ if _is_dataclass_instance(obj):+ return dataclasses.asdict(obj)+ if _is_pydantic_instance(obj):+ return obj.model_dump()+ return str(obj)++class LenientJSONEncoder(json.JSONEncoder):def __init__(self, *, sort_keys: bool = False, indent: int | None = None) -> None:- """Custom encoder that can handle:-- - dataclasses- - namedtuples- - enums- - date & datetime- - sets- - subclasses of dict-- Any incompatible object will be serialized as str, except types."""- super().__init__(check_circular=False, ensure_ascii=False, sort_keys=sort_keys, indent=indent)-- def default(self, o: object) -> Any:- if isinstance(o, datetime.date):- return o.isoformat()- if isinstance(o, enum.Enum):- return self.default(o.value)- if isinstance(o, set):- return sorted(o)- if _is_namedtuple_instance(o):- return o._asdict()- if _is_dataclass_instance(o):- return dataclasses.asdict(o)- if isinstance(o, dict): # Handle subclasses of dict- return dict(o)- if _is_pydantic_instance(o):- return o.model_dump()- if isinstance(o, type): # Do not serialize classes, only instances- return super().default(o)- return str(o)+ super().__init__(check_circular=False, ensure_ascii=False, sort_keys=sort_keys, indent=indent, default=default)class JSDecoder(json.JSONDecoder):- def __init__(self):- """Custom decoder that tries to transforms javascript objects into valid json+ """Custom decoder that tries to transforms javascript objects into valid json- It can only handle simple js objects but that's enough most of the time"""+ It can only handle simple js objects"""++ def __init__(self) -> None:super().__init__()self.parse_string = _parse_js_stringself.scan_once = _py_make_scanner(self)-def _verbose_decode_error_msg(func: Callable[_P, _R]) -> Callable[_P, _R]:+def _verbose(func: Callable[_P, _R]) -> Callable[_P, _R]:@functools.wraps(func)def wrapper(*args, **kwargs) -> _R:try:@@ -107,11 +96,12 @@ def _verbose_decode_error_msg(func: Callable[_P, _R]) -> Callable[_P, _R]:return wrapper-def _get_encoder(*, sort_keys: bool = False, indent: int | None = None):+def _get_encoder(*, sort_keys: bool = False, indent: int | None = None) -> LenientJSONEncoder:key = sort_keys, indent- if key not in _encoders:- _encoders[key] = LenientJSONEncoder(sort_keys=sort_keys, indent=indent)- return _encoders[key]+ encoder = _encoders.get(key)+ if encoder is None:+ encoder = _encoders[key] = LenientJSONEncoder(sort_keys=sort_keys, indent=indent)+ return encoderdef _parse_js_string(*args, **kwargs) -> tuple[Any, int]:@@ -146,27 +136,24 @@ def _is_pydantic_instance(obj: object, /) -> TypeGuard[BaseModel]:return hasattr(obj, "model_dump") and not isinstance(obj, type)-def dumps(obj: object, /, *, sort_keys: bool = False, indent: int | None = None) -> Any:+def dumps(obj: object, /, *, sort_keys: bool = False, indent: int | None = None, **_) -> Any:encoder = _get_encoder(sort_keys=sort_keys, indent=indent)return encoder.encode(obj)-async def dump_jsonl(data: Iterable[dict[str, Any]], /, file: Path, *, append: bool = True) -> None:- def dump():- with file.open(mode="a" if append else "w", encoding="utf8") as f:- for item in data:- f.writelines(_DEFAULT_ENCODER.iterencode(item))- f.write("\n")-- await asyncio.to_thread(dump)+def dump_jsonl(data: Iterable[dict[str, Any]], /, file: Path) -> None:+ with file.open(mode="a", encoding="utf8") as f:+ for item in data:+ f.writelines(_DEFAULT_ENCODER.iterencode(item))+ f.write("\n")-loads = _verbose_decode_error_msg(json.loads)+loads = _verbose(json.loads)_JS_DECODER = JSDecoder()_DEFAULT_ENCODER = _get_encoder()-@_verbose_decode_error_msg+@_verbosedef load_js_obj(string: str, /) -> Any:"""Parses a string representation of a JavaScript object into a Python object.@@ -183,14 +170,14 @@ def load_js_obj(string: str, /) -> Any:return _JS_DECODER.decode(string)[email protected](frozen=True, slots=True)[email protected](slots=True, frozen=True)class JSONWebToken:# https://www.rfc-editor.org/rfc/rfc7519.htmlalg: str- headers: dict[str, str] = dataclasses.field(repr=False)- payload: dict[str, Any] = dataclasses.field(repr=False)+ headers: dict[str, str]+ payload: dict[str, Any]signature: str- _encoded_token: str+ encoded: str@classmethoddef decode(cls, jwt: str, /) -> Self:@@ -199,8 +186,8 @@ class JSONWebToken:return cls(headers["alg"], headers, cls._decode(b64_payload), b64_signature, jwt)@classmethod- def _decode(cls, value: str, /) -> dict[str, Any]:- return loads(base64.urlsafe_b64decode(f"{value}==="))+ def _decode(cls, payload: str, /) -> dict[str, Any]:+ return loads(base64.urlsafe_b64decode(f"{payload}==="))def is_expired(self, threshold: int = 0) -> bool:"""Checks if the token has expired or is about to expire.@@ -213,7 +200,7 @@ class JSONWebToken:return Falsedef __str__(self) -> str:- return self._encoded_token+ return self.encodeddef is_jwt(string: str) -> bool: