Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: make json utils blocking (#77)

NTFSvolumef6fcabc5bd3e
diff --git a/cyberdrop_dl/managers/logs.py b/cyberdrop_dl/managers/logs.py
index bf6308588..2f5138391 100644
--- a/cyberdrop_dl/managers/logs.py
+++ b/cyberdrop_dl/managers/logs.py
@@ -71,7 +71,7 @@ class LogManager:
async def write_jsonl(self, data: Iterable[dict[str, Any]]) -> None:
async with self._file_locks[self.files.jsonl_file]:
- await json.dump_jsonl(data, self.files.jsonl_file)
+ await asyncio.to_thread(json.dump_jsonl, data, self.files.jsonl_file)
async def _write_to_csv(self, file: Path, **row: object) -> None:
async with self._file_locks[file]:
diff --git a/cyberdrop_dl/utils/json.py b/cyberdrop_dl/utils/json.py
index a218160a4..d9c6913ed 100644
--- a/cyberdrop_dl/utils/json.py
+++ b/cyberdrop_dl/utils/json.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import asyncio
import base64
import dataclasses
import datetime
@@ -32,7 +31,7 @@ else:
_py_make_scanner = json.scanner.py_make_scanner
_encoders: dict[tuple[bool, int | None], LenientJSONEncoder] = {}
-_REPLACE_QUOTES_PAIRS = [
+_REPLACE_QUOTES_PAIRS = (
("{'", '{"'),
("'}", '"}'),
("['", '["'),
@@ -43,58 +42,48 @@ _REPLACE_QUOTES_PAIRS = [
("' :", '" :'),
("',", '",'),
(": '", ': "'),
-]
+)
class _DataclassInstance(Protocol):
__dataclass_fields__: ClassVar[dict[str, dataclasses.Field[Any]]]
+def default(obj: object, /) -> Any:
+ if isinstance(obj, datetime.date):
+ return obj.isoformat()
+ if isinstance(obj, enum.Enum):
+ return obj.value
+ if isinstance(obj, set):
+ return sorted(obj)
+ if callable(serialize := getattr(obj, "__json__", None)):
+ return serialize()
+ if _is_namedtuple_instance(obj):
+ return obj._asdict()
+ if _is_dataclass_instance(obj):
+ return dataclasses.asdict(obj)
+ if _is_pydantic_instance(obj):
+ return obj.model_dump()
+ return str(obj)
+
+
class LenientJSONEncoder(json.JSONEncoder):
def __init__(self, *, sort_keys: bool = False, indent: int | None = None) -> None:
- """Custom encoder that can handle:
-
- - dataclasses
- - namedtuples
- - enums
- - date & datetime
- - sets
- - subclasses of dict
-
- Any incompatible object will be serialized as str, except types."""
- super().__init__(check_circular=False, ensure_ascii=False, sort_keys=sort_keys, indent=indent)
-
- def default(self, o: object) -> Any:
- if isinstance(o, datetime.date):
- return o.isoformat()
- if isinstance(o, enum.Enum):
- return self.default(o.value)
- if isinstance(o, set):
- return sorted(o)
- if _is_namedtuple_instance(o):
- return o._asdict()
- if _is_dataclass_instance(o):
- return dataclasses.asdict(o)
- if isinstance(o, dict): # Handle subclasses of dict
- return dict(o)
- if _is_pydantic_instance(o):
- return o.model_dump()
- if isinstance(o, type): # Do not serialize classes, only instances
- return super().default(o)
- return str(o)
+ super().__init__(check_circular=False, ensure_ascii=False, sort_keys=sort_keys, indent=indent, default=default)
class JSDecoder(json.JSONDecoder):
- def __init__(self):
- """Custom decoder that tries to transforms javascript objects into valid json
+ """Custom decoder that tries to transforms javascript objects into valid json
- It can only handle simple js objects but that's enough most of the time"""
+ It can only handle simple js objects"""
+
+ def __init__(self) -> None:
super().__init__()
self.parse_string = _parse_js_string
self.scan_once = _py_make_scanner(self)
-def _verbose_decode_error_msg(func: Callable[_P, _R]) -> Callable[_P, _R]:
+def _verbose(func: Callable[_P, _R]) -> Callable[_P, _R]:
@functools.wraps(func)
def wrapper(*args, **kwargs) -> _R:
try:
@@ -107,11 +96,12 @@ def _verbose_decode_error_msg(func: Callable[_P, _R]) -> Callable[_P, _R]:
return wrapper
-def _get_encoder(*, sort_keys: bool = False, indent: int | None = None):
+def _get_encoder(*, sort_keys: bool = False, indent: int | None = None) -> LenientJSONEncoder:
key = sort_keys, indent
- if key not in _encoders:
- _encoders[key] = LenientJSONEncoder(sort_keys=sort_keys, indent=indent)
- return _encoders[key]
+ encoder = _encoders.get(key)
+ if encoder is None:
+ encoder = _encoders[key] = LenientJSONEncoder(sort_keys=sort_keys, indent=indent)
+ return encoder
def _parse_js_string(*args, **kwargs) -> tuple[Any, int]:
@@ -146,27 +136,24 @@ def _is_pydantic_instance(obj: object, /) -> TypeGuard[BaseModel]:
return hasattr(obj, "model_dump") and not isinstance(obj, type)
-def dumps(obj: object, /, *, sort_keys: bool = False, indent: int | None = None) -> Any:
+def dumps(obj: object, /, *, sort_keys: bool = False, indent: int | None = None, **_) -> Any:
encoder = _get_encoder(sort_keys=sort_keys, indent=indent)
return encoder.encode(obj)
-async def dump_jsonl(data: Iterable[dict[str, Any]], /, file: Path, *, append: bool = True) -> None:
- def dump():
- with file.open(mode="a" if append else "w", encoding="utf8") as f:
- for item in data:
- f.writelines(_DEFAULT_ENCODER.iterencode(item))
- f.write("\n")
-
- await asyncio.to_thread(dump)
+def dump_jsonl(data: Iterable[dict[str, Any]], /, file: Path) -> None:
+ with file.open(mode="a", encoding="utf8") as f:
+ for item in data:
+ f.writelines(_DEFAULT_ENCODER.iterencode(item))
+ f.write("\n")
-loads = _verbose_decode_error_msg(json.loads)
+loads = _verbose(json.loads)
_JS_DECODER = JSDecoder()
_DEFAULT_ENCODER = _get_encoder()
-@_verbose_decode_error_msg
+@_verbose
def load_js_obj(string: str, /) -> Any:
"""Parses a string representation of a JavaScript object into a Python object.
@@ -183,14 +170,14 @@ def load_js_obj(string: str, /) -> Any:
return _JS_DECODER.decode(string)
[email protected](frozen=True, slots=True)
[email protected](slots=True, frozen=True)
class JSONWebToken:
# https://www.rfc-editor.org/rfc/rfc7519.html
alg: str
- headers: dict[str, str] = dataclasses.field(repr=False)
- payload: dict[str, Any] = dataclasses.field(repr=False)
+ headers: dict[str, str]
+ payload: dict[str, Any]
signature: str
- _encoded_token: str
+ encoded: str
@classmethod
def decode(cls, jwt: str, /) -> Self:
@@ -199,8 +186,8 @@ class JSONWebToken:
return cls(headers["alg"], headers, cls._decode(b64_payload), b64_signature, jwt)
@classmethod
- def _decode(cls, value: str, /) -> dict[str, Any]:
- return loads(base64.urlsafe_b64decode(f"{value}==="))
+ def _decode(cls, payload: str, /) -> dict[str, Any]:
+ return loads(base64.urlsafe_b64decode(f"{payload}==="))
def is_expired(self, threshold: int = 0) -> bool:
"""Checks if the token has expired or is about to expire.
@@ -213,7 +200,7 @@ class JSONWebToken:
return False
def __str__(self) -> str:
- return self._encoded_token
+ return self.encoded
def is_jwt(string: str) -> bool: