Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
"""Pydantic models"""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, ClassVar, Final, Self, TypedDict, final, get_args, get_origin, override
from cyclopts import Parameter
from pydantic import AnyUrl, BaseModel, Secret, SerializationInfo, TypeAdapter, model_serializer, model_validator
from pydantic.fields import FieldInfo # noqa: TC002
from cyberdrop_dl import env
from cyberdrop_dl.constants import DEFAULT_PARAMETER
from cyberdrop_dl.utils import fast_cache, operators
if TYPE_CHECKING:
from collections.abc import Generator, Iterable
logger = logging.getLogger(__name__)
class DeferredModel(
BaseModel,
validate_by_name=True,
validate_by_alias=True,
defer_build=True,
allow_inf_nan=False,
url_preserve_empty_path=True,
val_temporal_unit="milliseconds",
validate_default=env.DEBUG_MODE,
validation_error_cause=env.DEBUG_MODE,
): ...
_warned: set[tuple[type, str]] = set()
# Merges re-validate from a full dump, marking every field as set; this context lets model_post_init tell.
MERGE_CONTEXT: Final[dict[str, bool]] = {"merging": True}
@DEFAULT_PARAMETER
class ConfigModel(DeferredModel, extra="forbid"):
@override
def model_post_init(self, context: Any, /) -> None:
super().model_post_init(context)
if context is MERGE_CONTEXT:
return
deprecated = self.model_fields_set.intersection(_deprecated_fields(self))
if not deprecated:
return
import time
for field in deprecated:
warn_id = type(self), field
if warn_id not in _warned:
_warned.add(warn_id)
logger.warning("'%s' config option is deprecated and will be removed in a future version", field)
time.sleep(2)
def _deprecated_fields(model: BaseModel) -> list[str]:
return [name for name, field in type(model).model_fields.items() if field.deprecated]
class ConfigGroup(ConfigModel):
def __init_subclass__(cls, *, group: str | None = None, name: str | None = "*") -> None:
_ = Parameter(group=group or cls.__name__, name=name)(cls)
return super().__init_subclass__()
class _AppriseURLDict(TypedDict):
url: str
tags: set[str]
@Parameter(name="*")
class AppriseURL(ConfigModel):
url: Secret[AnyUrl]
tags: set[str] = set()
_OS_SCHEMES: ClassVar[tuple[str, ...]] = "windows", "macosx", "dbus", "qt", "glib", "kde"
_VALID_TAGS: ClassVar[set[str]] = {"no_logs", "attach_logs", "simplified"}
@override
def model_post_init(self, context: Any, /) -> None:
super().model_post_init(context)
if not self.tags.intersection(self._VALID_TAGS):
self.tags |= {"no_logs"}
if self.is_os_url:
self.tags = (self.tags - self._VALID_TAGS) | {"simplified"}
def __str__(self) -> str:
return self.format(dump_secret=True)
@property
def scheme(self) -> str:
return self.url.get_secret_value().scheme
@property
def is_os_url(self) -> bool:
return any(scheme in self.scheme for scheme in self._OS_SCHEMES)
@property
def attach_logs(self) -> bool:
return "attach_logs" in self.tags
@model_serializer()
def serialize(self, info: SerializationInfo) -> str:
return self.format(dump_secret=info.mode != "json")
def format(self, *, dump_secret: bool) -> str:
url = str(self.url.get_secret_value() if dump_secret else self.url)
if not self.tags:
return url
return f"{','.join(sorted(self.tags))}={url}"
@model_validator(mode="before")
@classmethod
def _validate(cls, obj: object) -> _AppriseURLDict:
match obj:
case str():
return cls._parse_url(obj)
case dict():
tags = obj.get("tags") or set()
url = str(obj.get("url", ""))
if not tags:
return cls._parse_url(url)
return {"url": url, "tags": tags}
case _:
return {"url": str(obj), "tags": set()}
@staticmethod
def _parse_url(obj: str) -> _AppriseURLDict:
match obj.split("://", 1)[0].split("=", 1):
case [tags_, _scheme]:
tags = set(tags_.split(","))
url = obj.split("=", 1)[-1]
case _:
tags: set[str] = set()
url: str = obj
return {"url": url, "tags": tags}
def merge_dicts(
dict1: dict[str, Any],
dict2: dict[str, Any],
additive_keys: Iterable[tuple[str, ...]] = (),
) -> dict[str, Any]:
for keys in additive_keys:
get = operators.nested_itemgetter(*keys)
try:
current_value, new_value = get(dict1), get(dict2)
except KeyError:
continue
else:
new_value = merge_additive_args(current_value, new_value)
operators.nested_itemsetter(*keys)(dict2, new_value)
return _merge_dicts(dict1, dict2)
def _merge_dicts(dict1: dict[str, Any], dict2: dict[str, Any]) -> dict[str, Any]:
for key, val in dict1.items():
if isinstance(val, dict):
if key in dict2 and isinstance(dict2[key], dict):
_merge_dicts(val, dict2[key])
elif key in dict2:
dict1[key] = dict2[key]
for key, val in dict2.items():
if key not in dict1:
dict1[key] = val
return dict1
def merge_models[M: BaseModel](
default: M,
new: M,
additive_keys: Iterable[tuple[str, ...]] = (),
) -> M:
current_data = default.model_dump()
new_data = new.model_dump(exclude_unset=True)
updated_dict = merge_dicts(current_data, new_data, additive_keys)
merged = default.model_validate(updated_dict, context=MERGE_CONTEXT)
_restore_fields_set(merged, default, new)
return merged
def _restore_fields_set(merged: BaseModel, default: BaseModel, new: BaseModel) -> None:
# Reads __dict__ instead of getattr so deprecated fields don't trigger their DeprecationWarning.
merged.__pydantic_fields_set__.intersection_update(default.model_fields_set | new.model_fields_set)
for name, value in merged.__dict__.items():
if all(isinstance(val, BaseModel) for val in (value, default.__dict__[name], new.__dict__[name])):
_restore_fields_set(value, default.__dict__[name], new.__dict__[name])
@fast_cache
def type_adapter[T](cls: type[T]) -> TypeAdapter[T]:
"""Get a type adapter for this class.
Type adapters are cached. Multiple calls return the same adapter"""
return TypeAdapter(cls)
def merge_additive_args[T: list[str] | tuple[str, ...] | set[str]](current_values: Iterable[str], overrides: T) -> T:
if isinstance(overrides, set):
if "+" in overrides:
new_values = set(current_values).union(overrides)
elif "-" in overrides:
new_values = set(current_values) - set(overrides)
else:
return overrides
else:
match overrides:
case ["+", *_]:
new_values = set(current_values).union(overrides)
case ["-", *_]:
new_values = set(current_values) - set(overrides)
case _:
return overrides
return type(overrides)(sorted(new_values - {"+", "-"}))
class FieldMetadata:
IGNORE: Final = "<IGNORE>"
def __init__(self, **data: Any) -> None:
self.data: dict[str, Any] = data
@override
def __repr__(self) -> str:
return f"{type(self).__name__}(data={self.data!r})"
@classmethod
def get(cls, field: FieldInfo) -> Self | None:
if field.metadata:
for data in field.metadata:
if type(data) is cls:
return data
@classmethod
def _check(cls, field: FieldInfo) -> bool:
return cls.get(field) is not None
@classmethod
def resolve(cls, model: BaseModel) -> Generator[tuple[str, ...]]:
for name in cls._resolve(model):
if cls.IGNORE not in name:
yield tuple(name.split("."))
@classmethod
def _resolve(cls, model: BaseModel) -> Generator[str]:
import warnings
for name, field in type(model).model_fields.items():
if cls._check(field):
yield name
continue
with warnings.catch_warnings(action="ignore"):
value = getattr(model, name)
if isinstance(value, BaseModel):
for inner_name in cls._resolve(value):
yield f"{name}.{inner_name}"
@final
class AdditiveArg(FieldMetadata):
@override
@classmethod
def _check(cls, field: FieldInfo) -> bool:
from cyclopts.annotations import resolve
if get_origin(field.annotation) in {set, list, tuple}:
arg = resolve(get_args(field.annotation)[0])
all_args = get_args(arg) or [arg]
if all(map(_is_str, all_args)):
return True
return super()._check(field)
def _is_str(type_: object) -> bool:
from cyclopts.annotations import resolve
type_ = resolve(type_)
if type_ is str:
return True
try:
return issubclass(type_, str)
except Exception: # noqa: BLE001
try:
return isinstance(type_, str)
except Exception: # noqa: BLE001
return False