Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
from __future__ import annotations
import base64
import contextlib
import dataclasses
import datetime as dt
import enum
import functools
import json
import json.decoder
import json.scanner
import re
import time
from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, Protocol, Self, TypeGuard
from cyberdrop_dl.utils import b64_pad
if TYPE_CHECKING:
from collections.abc import Callable, Generator, Iterable
from pathlib import Path
from pydantic import BaseModel
_scanstring: Callable[..., tuple[str, int]] = json.decoder.scanstring # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]
_py_make_scanner: Callable[..., tuple[Any, int]] = json.scanner.py_make_scanner # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]
_encoders: dict[tuple[bool, int | None], LenientJSONEncoder] = {}
_REPLACE_QUOTES_PAIRS = (
("{'", '{"'),
("'}", '"}'),
("['", '["'),
("']", '"]'),
(",'", ',"'),
("':", '":'),
(", '", ', "'),
("' :", '" :'),
("',", '",'),
(": '", ': "'),
)
class _DataclassInstance(Protocol):
__dataclass_fields__: ClassVar[dict[str, dataclasses.Field[Any]]]
def default(obj: object, /) -> Any: # noqa: PLR0911
if isinstance(obj, dt.date):
return obj.isoformat()
if isinstance(obj, enum.Enum):
return obj.value
if isinstance(obj, set):
return sorted(obj)
if callable(serialize := getattr(obj, "__json__", None)):
return serialize()
if _is_namedtuple_instance(obj):
return obj._asdict()
if _is_dataclass_instance(obj):
return dataclasses.asdict(obj)
if _is_pydantic_instance(obj):
return obj.model_dump()
return str(obj)
class LenientJSONEncoder(json.JSONEncoder):
def __init__(self, *, sort_keys: bool = False, indent: int | None = None) -> None:
super().__init__(check_circular=False, ensure_ascii=False, sort_keys=sort_keys, indent=indent, default=default)
class JSDecoder(json.JSONDecoder):
"""Custom decoder that tries to transforms javascript objects into valid json
It can only handle simple js objects"""
def __init__(self) -> None:
super().__init__()
self.parse_string = _parse_js_string
self.scan_once = _py_make_scanner(self)
JSONDecodeError = json.JSONDecodeError
@contextlib.contextmanager
def _verbose_context() -> Generator[None]:
try:
yield
except json.JSONDecodeError as e:
sub_string = e.doc[e.pos - 10 : e.pos + 10]
if not sub_string:
raise
msg = f"{e.msg} at around '{sub_string}', char: '{e.doc[e.pos]}'"
raise json.JSONDecodeError(msg, e.doc, e.pos) from None
def _verbose[**P, R](func: Callable[P, R]) -> Callable[P, R]:
@functools.wraps(func)
def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
with _verbose_context():
return func(*args, **kwargs)
return wrapper
def _get_encoder(*, sort_keys: bool = False, indent: int | None = None) -> LenientJSONEncoder:
key = sort_keys, indent
encoder = _encoders.get(key)
if encoder is None:
encoder = _encoders[key] = LenientJSONEncoder(sort_keys=sort_keys, indent=indent)
return encoder
def _parse_js_string(*args: Any, **kwargs: Any) -> tuple[Any, int]:
string, end = _scanstring(*args, **kwargs)
for quote in ("'", '"'):
if len(string) > 2 and string.startswith(quote) and string.endswith(quote):
string = string[1:-1]
return _literal_value(string), end
def _literal_value(string: str) -> Any:
if string.isdigit():
return int(string)
if string == "undefined":
return None
if string in {"true", "!0"}:
return True
if string in {"false", "!1"}:
return False
return string
def _is_namedtuple_instance(obj: object, /) -> TypeGuard[NamedTuple]:
return isinstance(obj, tuple) and hasattr(obj, "_asdict") and hasattr(obj, "_fields")
def _is_dataclass_instance(obj: object, /) -> TypeGuard[_DataclassInstance]:
return dataclasses.is_dataclass(obj) and not isinstance(obj, type)
def _is_pydantic_instance(obj: object, /) -> TypeGuard[BaseModel]:
return hasattr(obj, "model_dump") and not isinstance(obj, type)
def dumps(obj: object, /, *, sort_keys: bool = False, indent: int | None = None, **_: object) -> Any:
encoder = _get_encoder(sort_keys=sort_keys, indent=indent)
return encoder.encode(obj)
def dump_jsonl(data: Iterable[dict[str, Any]], /, file: Path) -> None:
with file.open(mode="a", encoding="utf8") as f:
for item in data:
f.writelines(_DEFAULT_ENCODER.iterencode(item))
f.write("\n")
_JS_DECODER = JSDecoder()
_DEFAULT_ENCODER = _get_encoder()
loads = _verbose(json.loads)
@_verbose
def load_js_obj(string: str, /) -> Any:
"""Parses a string representation of a JavaScript object into a Python object.
It can handle JavaScript object strings that may not be valid JSON"""
string = string.replace("\t", "").replace("\n", "").strip()
# Remove tailing comma
string = string[:-1].strip().removesuffix(",") + string[-1]
# Make it valid json by replacing single quotes with double quotes
# we can't just replace every single ' with " because it will brake with english words like: it's
for old, new in _REPLACE_QUOTES_PAIRS:
string = string.replace(old, new)
string = re.sub(r"\s\b(?!http)(\w+)\s?:", r' "\1" : ', string) # wrap keys without quotes with double quotes
return _JS_DECODER.decode(string)
@dataclasses.dataclass(slots=True, frozen=True)
class JSONWebToken:
# https://www.rfc-editor.org/rfc/rfc7519.html
alg: str
headers: dict[str, str]
payload: dict[str, Any]
signature: str
encoded: str
@classmethod
def decode(cls, jwt: str, /) -> Self:
if not looks_like_jwt(jwt):
raise ValueError("Not a valid JSON Web Token", jwt)
b64_headers, b64_payload, b64_signature = jwt.split(".")
headers = cls.decode_payload(b64_headers)
return cls(headers["alg"], headers, cls.decode_payload(b64_payload), b64_signature, jwt)
@classmethod
def decode_payload(cls, payload: str, /) -> dict[str, Any]:
return loads(base64.urlsafe_b64decode(b64_pad(payload)))
def is_expired(self, threshold: int = 0) -> bool:
"""Checks if the token has expired or is about to expire.
threshold is the time in seconds before the token's expiration to consider it as expired.
"""
expires: int | None = self.payload.get("exp")
if expires:
return (expires - time.time()) < threshold
return False
def __str__(self) -> str:
return self.encoded
def looks_like_jwt(string: str) -> bool:
return string.startswith("eyJ") and string.count(".") == 2
def jwt_decode(jwt: str) -> dict[str, Any]:
return JSONWebToken.decode(jwt).payload