Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: move pydantic models and types to its own module (#1039)
NTFSvolume1f76ca7b9547
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex 5cfc8b17d..32e57bc16 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -4,7 +4,6 @@ repos:hooks:- id: check-added-large-files- id: check-case-conflict- - id: check-executables-have-shebangs- id: check-illegal-windows-names- id: check-merge-conflict- id: check-shebang-scripts-are-executablediff --git a/cyberdrop_dl/config/__init__.py b/cyberdrop_dl/config/__init__.pynew file mode 100755index 000000000..fbb36673c--- /dev/null+++ b/cyberdrop_dl/config/__init__.py@@ -0,0 +1,166 @@+from __future__ import annotations++import shutil+from pathlib import Path+from time import sleep+from typing import TYPE_CHECKING++from cyberdrop_dl import constants, env+from cyberdrop_dl.utils.apprise import get_apprise_urls++from .auth_model import AuthSettings+from .config_model import ConfigSettings+from .global_model import GlobalSettings++if TYPE_CHECKING:+ from cyberdrop_dl.utils.apprise import AppriseURL+ from cyberdrop_dl.utils.args import ParsedArgs++__all__ = [+ "AuthSettings",+ "ConfigSettings",+ "GlobalSettings",+]++deep_scrape: bool = False++current_config: Config+cli: ParsedArgs+appdata: AppData++# re-export current config values for easy access+auth: AuthSettings+settings: ConfigSettings+global_settings: GlobalSettings+++def startup() -> None:+ global appdata, cli+ from cyberdrop_dl.utils.args import parse_args++ cli = parse_args()++ if env.RUNNING_IN_IDE and Path.cwd().name == "cyberdrop_dl":+ """This is for testing purposes only"""+ constants.DEFAULT_APP_STORAGE = Path("../AppData")+ constants.DEFAULT_DOWNLOAD_STORAGE = Path("../Downloads")++ appdata_path = cli.cli_only_args.appdata_folder or constants.DEFAULT_APP_STORAGE+ appdata = AppData(appdata_path.resolve())+ appdata.mkdirs()+ # cache.startup(appdata.cache_file)+ load_config(get_default_config())+ settings.logs._delete_old_logs_and_folders(constants.STARTUP_TIME)+++class AppData(Path):+ def __init__(self, app_data_path: Path) -> None:+ self.configs_dir = app_data_path / "Configs"+ self.cache_dir = app_data_path / "Cache"+ self.cookies_dir = app_data_path / "Cookies"+ self.cache_file = self.cache_dir / "cache.yaml"+ self.default_auth_config_file = self.configs_dir / "authentication.yaml"+ self.global_config_file = self.configs_dir / "global_settings.yaml"+ self.cache_db = self.cache_dir / "request_cache.db"+ self.history_db = self.cache_dir / "cyberdrop.db"++ def mkdirs(self) -> None:+ for dir in (self.configs_dir, self.cache_dir, self.cookies_dir):+ dir.mkdir(parents=True, exist_ok=True)+++class Config:+ """Helper class to group a single config, not necessarily the current config"""++ def __init__(self, name: str) -> None:+ self.apprise_urls: list[AppriseURL] = []+ self.folder = appdata.configs_dir / name+ self.apprise_file = self.folder / "apprise.txt"+ self.config_file = self.folder / "settings.yaml"+ auth_override = self.folder / "authentication.yaml"+ if auth_override.is_file():+ self.auth_config_file = auth_override+ else:+ self.auth_config_file = appdata.default_auth_config_file+ self.auth: AuthSettings+ self.settings: ConfigSettings+ self.global_settings: GlobalSettings++ @staticmethod+ def build(name: str, auth: AuthSettings, settings: ConfigSettings, global_settings: GlobalSettings) -> Config:+ self = Config(name)+ self.auth = auth+ self.settings = settings+ self.global_settings = global_settings+ self.apprise_urls = get_apprise_urls(file=self.apprise_file)+ return self++ @staticmethod+ def new_empty_config(name: str) -> Config:+ assert name not in get_all_configs()+ self = Config(name)+ self._load()+ return self++ def _load(self) -> None:+ """Read each config module from their respective files++ If a files does not exists, uses the default config and creates it"""+ self.auth = AuthSettings.load_file(self.auth_config_file, "socialmediagirls_username:")+ self.settings = ConfigSettings.load_file(self.config_file, "download_error_urls_filename:")+ self.global_settings = GlobalSettings.load_file(appdata.global_config_file, "Dupe_Cleanup_Options:")+ self.apprise_urls = get_apprise_urls(file=self.apprise_file)++ def _resolve_all_paths(self) -> None:+ self.auth.resolve_paths(self.folder.name)+ self.settings.resolve_paths(self.folder.name)+ self.global_settings.resolve_paths(self.folder.name)++ def _all_settings(self) -> tuple[ConfigSettings, AuthSettings, GlobalSettings]:+ return self.settings, self.auth, self.global_settings++ def write_updated_config(self) -> None:+ """Writes config to disk."""+ self.auth.save_to_file(self.auth_config_file)+ self.settings.save_to_file(self.config_file)+ self.global_settings.save_to_file(appdata.global_config_file)+++def get_default_config() -> str:+ ...+ # return cache.get(cache.DEFAULT_CONFIG_KEY) or "Default"+++def get_all_configs() -> list:+ return sorted(config.name for config in appdata.configs_dir.iterdir() if config.is_dir())+++def set_default_config(config_name: str) -> None:+ ...+ # cache.save(cache.DEFAULT_CONFIG_KEY, config_name)+++def delete_config(config_name: str) -> None:+ all_configs = get_all_configs()+ assert config_name in all_configs+ assert len(all_configs) > 1+ assert config_name != current_config.folder.name+ all_configs.remove(config_name)++ # if cache.get(cache.DEFAULT_CONFIG_KEY) == config_name:+ # set_default_config(all_configs[0])++ config_path = appdata.configs_dir / config_name+ shutil.rmtree(config_path)+++def load_config(config_name: str) -> None:+ global current_config, auth, global_settings, settings+ assert config_name+ current_config = Config(config_name)+ current_config._load()+ current_config._resolve_all_paths()+ settings, auth, global_settings = current_config._all_settings()+ settings.logs._set_output_filenames(constants.STARTUP_TIME)++ sleep(1)diff --git a/cyberdrop_dl/config/_common.py b/cyberdrop_dl/config/_common.pynew file mode 100755index 000000000..b5cf96a7e--- /dev/null+++ b/cyberdrop_dl/config/_common.py@@ -0,0 +1,44 @@+from pathlib import Path+from typing import Any, Self++from pydantic import Field as P_Field+from pydantic.fields import _Unset++from cyberdrop_dl.exceptions import InvalidYamlError+from cyberdrop_dl.models import PathAliasModel, get_model_fields+from cyberdrop_dl.utils import yaml+++def Field(default: Any, validation_alias: str = _Unset, **kwargs) -> Any: # noqa: N802+ return P_Field(default=default, validation_alias=validation_alias, **kwargs)+++class ConfigModel(PathAliasModel):+ @classmethod+ def load_file(cls, file: Path, update_if_has_string: str) -> Self:+ default = cls()+ if not file.is_file():+ config = default+ needs_update = True++ else:+ all_fields = get_model_fields(default, exclude_unset=False)+ config = cls.model_validate(yaml.load(file))+ set_fields = get_model_fields(config)+ needs_update = all_fields != set_fields or _is_in_file(update_if_has_string, file)++ if needs_update:+ yaml.save(file, config)+ return config++ def save_to_file(self, file: Path) -> None:+ yaml.save(file, self)+++def _is_in_file(search_value: str, file: Path) -> bool:+ try:+ return search_value.casefold() in file.read_text().casefold()+ except FileNotFoundError:+ return False+ except Exception as e:+ raise InvalidYamlError(file, e) from ediff --git a/cyberdrop_dl/config/auth_model.py b/cyberdrop_dl/config/auth_model.pynew file mode 100755index 000000000..dc0f8853d--- /dev/null+++ b/cyberdrop_dl/config/auth_model.py@@ -0,0 +1,62 @@+from pydantic import BaseModel++from cyberdrop_dl.models import AliasModel++from ._common import ConfigModel, Field+++class CoomerAuth(BaseModel):+ session: str = ""+++class XXXBunkerAuth(BaseModel):+ PHPSESSID: str = ""+++class ImgurAuth(AliasModel):+ client_id: str = Field("", "imgur_client_id")+++class MegaNzAuth(AliasModel):+ email: str = ""+ password: str = ""+++class JDownloaderAuth(AliasModel):+ username: str = Field("", "jdownloader_username")+ password: str = Field("", "jdownloader_password")+ device: str = Field("", "jdownloader_device")+++class KemonoAuth(AliasModel):+ session: str = ""+++class RedditAuth(AliasModel):+ personal_use_script: str = Field("", "reddit_personal_use_script")+ secret: str = Field("", "reddit_secret")+++class GoFileAuth(AliasModel):+ api_key: str = Field("", "gofile_api_key")+++class PixeldrainAuth(AliasModel):+ api_key: str = Field("", "pixeldrain_api_key")+++class RealDebridAuth(AliasModel):+ api_key: str = Field("", "realdebrid_api_key")+++class AuthSettings(ConfigModel):+ coomer: CoomerAuth = Field(CoomerAuth(), "Coomer")+ gofile: GoFileAuth = Field(GoFileAuth(), "GoFile")+ imgur: ImgurAuth = Field(ImgurAuth(), "Imgur")+ jdownloader: JDownloaderAuth = Field(JDownloaderAuth(), "JDownloader")+ kemono: KemonoAuth = Field(KemonoAuth(), "Kemono")+ meganz: MegaNzAuth = Field(MegaNzAuth(), "MegaNz")+ pixeldrain: PixeldrainAuth = Field(PixeldrainAuth(), "PixelDrain")+ realdebrid: RealDebridAuth = Field(RealDebridAuth(), "RealDebrid")+ reddit: RedditAuth = Field(RedditAuth(), "Reddit")+ xxxbunker: XXXBunkerAuth = Field(XXXBunkerAuth(), "XXXBunker")diff --git a/cyberdrop_dl/config_definitions/config_settings.py b/cyberdrop_dl/config/config_model.pyold mode 100644new mode 100755similarity index 60%rename from cyberdrop_dl/config_definitions/config_settings.pyrename to cyberdrop_dl/config/config_model.pyindex cf90a20a0..5c02e4e53--- a/cyberdrop_dl/config_definitions/config_settings.py+++ b/cyberdrop_dl/config/config_model.py@@ -1,17 +1,18 @@+import itertoolsimport re-from datetime import timedelta+from datetime import datetime, timedeltafrom logging import DEBUGfrom pathlib import Path-from pydantic import BaseModel, ByteSize, Field, NonNegativeInt, PositiveInt, field_serializer, field_validator+from pydantic import BaseModel, ByteSize, NonNegativeInt, PositiveInt, field_serializer, field_validator+from cyberdrop_dl import constantsfrom cyberdrop_dl.constants import BROWSERS, DEFAULT_APP_STORAGE, DEFAULT_DOWNLOAD_STORAGEfrom cyberdrop_dl.data_structures.hash import Hashingfrom cyberdrop_dl.data_structures.supported_domains import SUPPORTED_SITES_DOMAINS-from cyberdrop_dl.types import (- AliasModel,+from cyberdrop_dl.models import HttpAppriseURL+from cyberdrop_dl.models.types import (ByteSizeSerilized,- HttpAppriseURL,ListNonEmptyStr,ListNonNegativeInt,LogPath,@@ -20,7 +21,10 @@ from cyberdrop_dl.types import (NonEmptyStrOrNone,PathOrNone,)-from cyberdrop_dl.utils.validators import parse_duration_as_timedelta, parse_falsy_as+from cyberdrop_dl.models.validators import falsy_as, to_timedelta+from cyberdrop_dl.utils.utilities import purge_dir_tree++from ._common import ConfigModel, Field, PathAliasModelALL_SUPPORTED_SITES = ["<<ALL_SUPPORTED_SITES>>"]@@ -42,42 +46,66 @@ class DownloadOptions(BaseModel):maximum_thread_depth: NonNegativeInt = 0-class Files(AliasModel):- download_folder: Path = Field(validation_alias="d", default=DEFAULT_DOWNLOAD_STORAGE)- dump_json: bool = Field(default=False, validation_alias="j")- input_file: Path = Field(validation_alias="i", default=DEFAULT_APP_STORAGE / "Configs" / "{config}" / "URLs.txt")+class Files(PathAliasModel):+ download_folder: Path = Field(DEFAULT_DOWNLOAD_STORAGE, "d")+ dump_json: bool = Field(False, "j")+ input_file: Path = Field(DEFAULT_APP_STORAGE / "Configs{config}/URLs.txt", "i")save_pages_html: bool = False-class Logs(AliasModel):- download_error_urls: LogPath = Field(- default=Path("Download_Error_URLs.csv"), validation_alias="download_error_urls_filename"- )- last_forum_post: LogPath = Field(- default=Path("Last_Scraped_Forum_Posts.csv"), validation_alias="last_forum_post_filename"- )- log_folder: Path = DEFAULT_APP_STORAGE / "Configs" / "{config}" / "Logs"- log_line_width: PositiveInt = Field(default=240, ge=50)+class Logs(PathAliasModel):+ download_error_urls: LogPath = Field(Path("Download_Error_URLs.csv"), "download_error_urls_filename")+ last_forum_post: LogPath = Field(Path("Last_Scraped_Forum_Posts.csv"), "last_forum_post_filename")+ log_folder: Path = DEFAULT_APP_STORAGE / "Configs/{config}/Logs"+ log_line_width: PositiveInt = Field(240, ge=50)logs_expire_after: timedelta | None = None- main_log: MainLogPath = Field(default=Path("downloader.log"), validation_alias="main_log_filename")+ main_log: MainLogPath = Field(Path("downloader.log"), "main_log_filename")rotate_logs: bool = False- scrape_error_urls: LogPath = Field(- default=Path("Scrape_Error_URLs.csv"), validation_alias="scrape_error_urls_filename"- )- unsupported_urls: LogPath = Field(- default=Path("Unsupported_URLs.csv"), validation_alias="unsupported_urls_filename"- )- webhook: HttpAppriseURL | None = Field(default=None, validation_alias="webhook_url")+ scrape_error_urls: LogPath = Field(Path("Scrape_Error_URLs.csv"), "scrape_error_urls_filename")+ unsupported_urls: LogPath = Field(Path("Unsupported_URLs.csv"), "unsupported_urls_filename")+ webhook: HttpAppriseURL | None = Field(None, "webhook_url")++ @property+ def jsonl_file(self) -> Path:+ return self.main_log.with_suffix(".results.jsonl")++ @property+ def cdl_responses_dir(self) -> Path:+ return self.main_log.parent / "cdl_responses"@field_validator("webhook", mode="before")@classmethoddef handle_falsy(cls, value: str) -> str | None:- return parse_falsy_as(value, None)+ return falsy_as(value, None)@field_validator("logs_expire_after", mode="before")@staticmethoddef parse_logs_duration(input_date: timedelta | str | int | None) -> timedelta | str | None:- return parse_falsy_as(input_date, None, parse_duration_as_timedelta)+ return falsy_as(input_date, None, to_timedelta)++ def _set_output_filenames(self, now: datetime) -> None:+ self.log_folder.mkdir(exist_ok=True, parents=True)+ current_time_file_iso: str = now.strftime(constants.LOGS_DATETIME_FORMAT)+ current_time_folder_iso: str = now.strftime(constants.LOGS_DATE_FORMAT)+ for attr, log_file in vars(self).items():+ if not isinstance(log_file, Path) or log_file.suffix not in (".csv", ".log"):+ continue++ if self.rotate_logs:+ new_name = f"{log_file.stem}_{current_time_file_iso}{log_file.suffix}"+ log_file: Path = log_file.parent / current_time_folder_iso / new_name+ setattr(self, attr, self.log_folder / log_file)++ log_file.parent.mkdir(exist_ok=True, parents=True)++ def _delete_old_logs_and_folders(self, now: datetime | None = None) -> None:+ if now and self.logs_expire_after:+ for file in itertools.chain(self.log_folder.rglob("*.log"), self.log_folder.rglob("*.csv")):+ file_date = file.stat().st_ctime+ t_delta = now - datetime.fromtimestamp(file_date)+ if t_delta > self.logs_expire_after:+ file.unlink(missing_ok=True)+ purge_dir_tree(self.log_folder)class FileSizeLimits(BaseModel):@@ -106,7 +134,7 @@ class MediaDurationLimits(BaseModel):"""if input_date is None:return timedelta(seconds=0)- return parse_duration_as_timedelta(input_date)+ return to_timedelta(input_date)class IgnoreOptions(BaseModel):@@ -167,7 +195,7 @@ class BrowserCookies(BaseModel):@field_validator("browsers", mode="before")@classmethoddef parse_browsers(cls, values: list) -> list:- values = parse_falsy_as(values, [])+ values = falsy_as(values, [])if isinstance(values, list):return sorted(str(value).lower() for value in values)return values@@ -175,7 +203,7 @@ class BrowserCookies(BaseModel):@field_validator("sites", mode="before")@classmethoddef handle_list(cls, values: list) -> list:- values = parse_falsy_as(values, [])+ values = falsy_as(values, [])if values == ALL_SUPPORTED_SITES:return SUPPORTED_SITES_DOMAINSif isinstance(values, list):@@ -197,16 +225,14 @@ class DupeCleanup(BaseModel):send_deleted_to_trash: bool = True-class ConfigSettings(AliasModel):- browser_cookies: BrowserCookies = Field(validation_alias="Browser_Cookies", default=BrowserCookies())- download_options: DownloadOptions = Field(validation_alias="Download_Options", default=DownloadOptions())- dupe_cleanup_options: DupeCleanup = Field(validation_alias="Dupe_Cleanup_Options", default=DupeCleanup())- file_size_limits: FileSizeLimits = Field(validation_alias="File_Size_Limits", default=FileSizeLimits())- media_duration_limits: MediaDurationLimits = Field(- validation_alias="Media_Duration_Limits", default=MediaDurationLimits()- )- files: Files = Field(validation_alias="Files", default=Files())- ignore_options: IgnoreOptions = Field(validation_alias="Ignore_Options", default=IgnoreOptions())- logs: Logs = Field(validation_alias="Logs", default=Logs())- runtime_options: RuntimeOptions = Field(validation_alias="Runtime_Options", default=RuntimeOptions())- sorting: Sorting = Field(validation_alias="Sorting", default=Sorting())+class ConfigSettings(ConfigModel):+ browser_cookies: BrowserCookies = Field(BrowserCookies(), "Browser_Cookies")+ download_options: DownloadOptions = Field(DownloadOptions(), "Download_Options")+ dupe_cleanup_options: DupeCleanup = Field(DupeCleanup(), "Dupe_Cleanup_Options")+ file_size_limits: FileSizeLimits = Field(FileSizeLimits(), "File_Size_Limits")+ media_duration_limits: MediaDurationLimits = Field(MediaDurationLimits(), "Media_Duration_Limits")+ files: Files = Field(Files(), "Files")+ ignore_options: IgnoreOptions = Field(IgnoreOptions(), "Ignore_Options")+ logs: Logs = Field(Logs(), "Logs")+ runtime_options: RuntimeOptions = Field(RuntimeOptions(), "Runtime_Options")+ sorting: Sorting = Field(Sorting(), "Sorting")diff --git a/cyberdrop_dl/config_definitions/global_settings.py b/cyberdrop_dl/config/global_model.pyold mode 100644new mode 100755similarity index 70%rename from cyberdrop_dl/config_definitions/global_settings.pyrename to cyberdrop_dl/config/global_model.pyindex 425ac5dd9..902826727--- a/cyberdrop_dl/config_definitions/global_settings.py+++ b/cyberdrop_dl/config/global_model.py@@ -1,15 +1,15 @@import randomfrom datetime import timedelta-from pydantic import BaseModel, ByteSize, Field, NonNegativeFloat, PositiveInt, field_serializer, field_validator+from pydantic import BaseModel, ByteSize, NonNegativeFloat, PositiveInt, field_serializer, field_validatorfrom yarl import URL-from cyberdrop_dl.types import AliasModel, ByteSizeSerilized, HttpURL, NonEmptyStr-from cyberdrop_dl.utils.converters import convert_to_byte_size-from cyberdrop_dl.utils.validators import parse_duration_as_timedelta, parse_falsy_as+from cyberdrop_dl.config._common import ConfigModel, Field+from cyberdrop_dl.models.types import ByteSizeSerilized, HttpURL, NonEmptyStr+from cyberdrop_dl.models.validators import falsy_as, to_bytesize, to_timedelta-MIN_REQUIRED_FREE_SPACE = convert_to_byte_size("512MB")-DEFAULT_REQUIRED_FREE_SPACE = convert_to_byte_size("5GB")+MIN_REQUIRED_FREE_SPACE = to_bytesize("512MB")+DEFAULT_REQUIRED_FREE_SPACE = to_bytesize("5GB")class General(BaseModel):@@ -25,12 +25,12 @@ class General(BaseModel):@field_serializer("flaresolverr", "proxy")def serialize(self, value: URL | str) -> str | None:- return parse_falsy_as(value, None, str)+ return falsy_as(value, None, str)@field_validator("flaresolverr", "proxy", mode="before")@classmethoddef convert_to_str(cls, value: URL | str) -> str | None:- return parse_falsy_as(value, None, str)+ return falsy_as(value, None, str)@field_validator("required_free_space", mode="after")@classmethod@@ -54,7 +54,7 @@ class RateLimiting(BaseModel):@field_validator("file_host_cache_expire_after", "forum_cache_expire_after", mode="before")@staticmethoddef parse_cache_duration(input_date: timedelta | str | int) -> timedelta:- return parse_duration_as_timedelta(input_date)+ return to_timedelta(input_date)@propertydef total_delay(self) -> NonNegativeFloat:@@ -73,7 +73,7 @@ class UIOptions(BaseModel):vi_mode: bool = False-class GlobalSettings(AliasModel):- general: General = Field(validation_alias="General", default=General())- rate_limiting_options: RateLimiting = Field(validation_alias="Rate_Limiting_Options", default=RateLimiting())- ui_options: UIOptions = Field(validation_alias="UI_Options", default=UIOptions())+class GlobalSettings(ConfigModel):+ general: General = Field(General(), "General")+ rate_limiting_options: RateLimiting = Field(RateLimiting(), "Rate_Limiting_Options")+ ui_options: UIOptions = Field(UIOptions(), "UI_Options")diff --git a/cyberdrop_dl/config_definitions/__init__.py b/cyberdrop_dl/config_definitions/__init__.pydeleted file mode 100644index 7e9fb09cb..000000000--- a/cyberdrop_dl/config_definitions/__init__.py+++ /dev/null@@ -1,9 +0,0 @@-from .authentication_settings import AuthSettings-from .config_settings import ConfigSettings-from .global_settings import GlobalSettings--__all__ = [- "AuthSettings",- "ConfigSettings",- "GlobalSettings",-]diff --git a/cyberdrop_dl/config_definitions/authentication_settings.py b/cyberdrop_dl/config_definitions/authentication_settings.pydeleted file mode 100644index e7a6b1fad..000000000--- a/cyberdrop_dl/config_definitions/authentication_settings.py+++ /dev/null@@ -1,60 +0,0 @@-from pydantic import BaseModel, Field--from cyberdrop_dl.types import AliasModel---class CoomerAuth(BaseModel):- session: str = ""---class XXXBunkerAuth(BaseModel):- PHPSESSID: str = ""---class ImgurAuth(AliasModel):- client_id: str = Field("", validation_alias="imgur_client_id")---class MegaNzAuth(AliasModel):- email: str = ""- password: str = ""---class JDownloaderAuth(AliasModel):- username: str = Field("", validation_alias="jdownloader_username")- password: str = Field("", validation_alias="jdownloader_password")- device: str = Field("", validation_alias="jdownloader_device")---class KemonoAuth(AliasModel):- session: str = ""---class RedditAuth(AliasModel):- personal_use_script: str = Field("", validation_alias="reddit_personal_use_script")- secret: str = Field("", validation_alias="reddit_secret")---class GoFileAuth(AliasModel):- api_key: str = Field("", validation_alias="gofile_api_key")---class PixeldrainAuth(AliasModel):- api_key: str = Field("", validation_alias="pixeldrain_api_key")---class RealDebridAuth(AliasModel):- api_key: str = Field("", validation_alias="realdebrid_api_key")---class AuthSettings(AliasModel):- coomer: CoomerAuth = Field(validation_alias="Coomer", default=CoomerAuth())- gofile: GoFileAuth = Field(validation_alias="GoFile", default=GoFileAuth()) # type: ignore- imgur: ImgurAuth = Field(validation_alias="Imgur", default=ImgurAuth()) # type: ignore- jdownloader: JDownloaderAuth = Field(validation_alias="JDownloader", default=JDownloaderAuth()) # type: ignore- kemono: KemonoAuth = Field(validation_alias="Kemono", default=KemonoAuth()) # type: ignore- meganz: MegaNzAuth = Field(validation_alias="MegaNz", default=MegaNzAuth()) # type: ignore- pixeldrain: PixeldrainAuth = Field(validation_alias="PixelDrain", default=PixeldrainAuth()) # type: ignore- realdebrid: RealDebridAuth = Field(validation_alias="RealDebrid", default=RealDebridAuth()) # type: ignore- reddit: RedditAuth = Field(validation_alias="Reddit", default=RedditAuth()) # type: ignore- xxxbunker: XXXBunkerAuth = Field(validation_alias="XXXBunker", default=XXXBunkerAuth()) # type: ignorediff --git a/cyberdrop_dl/crawlers/_kemono_base.py b/cyberdrop_dl/crawlers/_kemono_base.pyindex 7ea725281..ef701834b 100644--- a/cyberdrop_dl/crawlers/_kemono_base.py+++ b/cyberdrop_dl/crawlers/_kemono_base.py@@ -13,11 +13,11 @@ from typing_extensions import TypedDict # Import from typing is not compatiblefrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import NoExtensionError, ScrapeError-from cyberdrop_dl.types import AbsoluteHttpURL, AliasModel+from cyberdrop_dl.models import AliasModel+from cyberdrop_dl.models.validators import falsy_as_nonefrom cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.dates import to_timestampfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_parts-from cyberdrop_dl.utils.validators import parse_falsy_as_noneif TYPE_CHECKING:from collections.abc import AsyncGenerator, Callable, Coroutine, Generator@@ -27,6 +27,7 @@ if TYPE_CHECKING:from yarl import URLfrom cyberdrop_dl.data_structures.url_objects import ScrapeItem+ from cyberdrop_dl.types import AbsoluteHttpURLMAX_OFFSET_PER_CALL = 50LINK_REGEX = re.compile(r"(?:http(?!.*\.\.)[^ ]*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]|</|'))")@@ -112,7 +113,7 @@ class File(TypedDict):server: NotRequired[str] # Sometimes present in attachments-FileOrNone = Annotated[File | None, BeforeValidator(parse_falsy_as_none)]+FileOrNone = Annotated[File | None, BeforeValidator(falsy_as_none)]class Post(AliasModel):diff --git a/cyberdrop_dl/crawlers/box_dot_com.py b/cyberdrop_dl/crawlers/box_dot_com.pyindex 124bf75f1..9250c1b21 100644--- a/cyberdrop_dl/crawlers/box_dot_com.py+++ b/cyberdrop_dl/crawlers/box_dot_com.py@@ -10,7 +10,8 @@ from pydantic import Fieldfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.exceptions import ScrapeError-from cyberdrop_dl.types import AbsoluteHttpURL, AliasModel, SupportedDomains, SupportedPaths+from cyberdrop_dl.models import AliasModel+from cyberdrop_dl.types import AbsoluteHttpURL, SupportedDomains, SupportedPathsfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.pyindex 663bfc8a1..046211c64 100644--- a/cyberdrop_dl/crawlers/xhamster.py+++ b/cyberdrop_dl/crawlers/xhamster.py@@ -7,7 +7,8 @@ from typing import TYPE_CHECKING, Annotated, Any, ClassVar, NamedTuplefrom pydantic import AliasPath, Field, PlainValidatorfrom cyberdrop_dl.crawlers.crawler import Crawler-from cyberdrop_dl.types import AbsoluteHttpURL, AliasModel, SupportedPaths+from cyberdrop_dl.models import AliasModel+from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPathsfrom cyberdrop_dl.utils.logger import log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, parse_urldiff --git a/cyberdrop_dl/data_structures/hash.py b/cyberdrop_dl/data_structures/hash.pyindex de76fcd8b..47cf42aed 100644--- a/cyberdrop_dl/data_structures/hash.py+++ b/cyberdrop_dl/data_structures/hash.py@@ -7,7 +7,7 @@ class Hashing(StrEnum):POST_DOWNLOAD = auto()@classmethod- def _missing_(cls, value):+ def _missing_(cls, value: str):try:return cls[str(value.upper())]except KeyError as e:diff --git a/cyberdrop_dl/managers/config_manager.py b/cyberdrop_dl/managers/config_manager.pyindex aa0b75f2d..1750f0dc1 100644--- a/cyberdrop_dl/managers/config_manager.py+++ b/cyberdrop_dl/managers/config_manager.py@@ -6,7 +6,7 @@ from dataclasses import fieldfrom time import sleepfrom typing import TYPE_CHECKING-from cyberdrop_dl.config_definitions import AuthSettings, ConfigSettings, GlobalSettings+from cyberdrop_dl.config import AuthSettings, ConfigSettings, GlobalSettingsfrom cyberdrop_dl.exceptions import InvalidYamlErrorfrom cyberdrop_dl.managers.log_manager import LogManagerfrom cyberdrop_dl.utils import yamldiff --git a/cyberdrop_dl/models/__init__.py b/cyberdrop_dl/models/__init__.pynew file mode 100755index 000000000..825a75079--- /dev/null+++ b/cyberdrop_dl/models/__init__.py@@ -0,0 +1,15 @@+from pydantic import BaseModel++from .base_models import AliasModel, AppriseURLModel, FrozenModel, HttpAppriseURL, PathAliasModel+++def get_model_fields(model: BaseModel, *, exclude_unset: bool = True) -> set[str]:+ fields = set()+ default_dict: dict = model.model_dump(exclude_unset=exclude_unset)+ for submodel_name, submodel in default_dict.items():+ for field_name in submodel:+ fields.add(f"{submodel_name}.{field_name}")+ return fields+++__all__ = ["AliasModel", "AppriseURLModel", "FrozenModel", "HttpAppriseURL", "PathAliasModel", "get_model_fields"]diff --git a/cyberdrop_dl/models/base_models.py b/cyberdrop_dl/models/base_models.pynew file mode 100755index 000000000..d50fca9f0--- /dev/null+++ b/cyberdrop_dl/models/base_models.py@@ -0,0 +1,53 @@+"""Pydantic models"""++from collections.abc import Mapping+from pathlib import Path++import yarl+from pydantic import AnyUrl, BaseModel, ConfigDict, Secret, SerializationInfo, model_serializer, model_validator++from cyberdrop_dl.models.types import HttpURL+from cyberdrop_dl.models.validators import to_apprise_url_dict+++class AliasModel(BaseModel):+ model_config = ConfigDict(populate_by_name=True)+++class FrozenModel(BaseModel):+ model_config = ConfigDict(frozen=True)+++class AppriseURLModel(FrozenModel):+ url: Secret[AnyUrl]+ tags: set[str] = set()++ @model_serializer()+ def serialize(self, info: SerializationInfo) -> str:+ dump_secret = info.mode != "json"+ url = self.url.get_secret_value() if dump_secret else self.url+ tags = self.tags - set("no_logs")+ tags = sorted(tags)+ return f"{','.join(tags)}{'=' if tags else ''}{url}"++ @model_validator(mode="before")+ @staticmethod+ def parse_input(value: yarl.URL | dict | str) -> Mapping:+ return to_apprise_url_dict(value)+++class HttpAppriseURL(AppriseURLModel):+ url: Secret[HttpURL]+++class PathAliasModel(AliasModel):+ @staticmethod+ def replace_config_and_resolve(path: Path, current_config: str) -> Path:+ return Path(str(path).replace("{config}", current_config)).resolve()++ def resolve_paths(self, config_name: str) -> None:+ for name, value in vars(self).items():+ if isinstance(value, Path):+ setattr(self, name, self.replace_config_and_resolve(value, config_name))+ elif isinstance(value, PathAliasModel):+ value.resolve_paths(config_name)diff --git a/cyberdrop_dl/models/types.py b/cyberdrop_dl/models/types.pynew file mode 100755index 000000000..f16b0d292--- /dev/null+++ b/cyberdrop_dl/models/types.py@@ -0,0 +1,43 @@+from __future__ import annotations++from pathlib import Path+from typing import Annotated++import yarl+from pydantic import (+ AfterValidator,+ BeforeValidator,+ ByteSize,+ NonNegativeInt,+ PlainSerializer,+ PlainValidator,+ StringConstraints,+)++from cyberdrop_dl.models.validators import (+ bytesize_to_str,+ change_path_suffix,+ falsy_as_list,+ falsy_as_none,+ to_yarl_url_w_pydantyc_validation,+)++# ~~~~~ Strings ~~~~~~~+StrSerializer = PlainSerializer(str, return_type=str, when_used="json-unless-none")+NonEmptyStr = Annotated[str, StringConstraints(min_length=1, strip_whitespace=True)]+NonEmptyStrOrNone = Annotated[NonEmptyStr | None, BeforeValidator(falsy_as_none)]+ListNonEmptyStr = Annotated[list[NonEmptyStr], BeforeValidator(falsy_as_list)]++# ~~~~~ Paths ~~~~~~~+PathOrNone = Annotated[Path | None, BeforeValidator(falsy_as_none)]+LogPath = Annotated[Path, AfterValidator(change_path_suffix(".csv"))]+MainLogPath = Annotated[LogPath, AfterValidator(change_path_suffix(".log"))]++# URL with pydantic.HttpUrl validation (must be absolute, must be http/https, detailed validation error).+# In type hints it's a yarl.URL. After validation the result is parsed with `parse_url` so this is also a yarl.URL at runtime+# Only use for config validation. To parse URLs internally while scraping, call `parse_url` directly+HttpURL = Annotated[yarl.URL, PlainValidator(to_yarl_url_w_pydantyc_validation), StrSerializer]++# ~~~~~ Others ~~~~~~~+ByteSizeSerilized = Annotated[ByteSize, PlainSerializer(bytesize_to_str, return_type=str)]+ListNonNegativeInt = Annotated[list[NonNegativeInt], BeforeValidator(falsy_as_list)]diff --git a/cyberdrop_dl/models/validators.py b/cyberdrop_dl/models/validators.pynew file mode 100755index 000000000..529aa3d1e--- /dev/null+++ b/cyberdrop_dl/models/validators.py@@ -0,0 +1,175 @@+from __future__ import annotations++import re+from datetime import timedelta+from functools import singledispatch+from typing import TYPE_CHECKING, Any, SupportsIndex, SupportsInt, TypeAlias, TypedDict++from pydantic import AnyUrl, ByteSize, HttpUrl, TypeAdapter++if TYPE_CHECKING:+ from collections.abc import Callable+ from pathlib import Path++ import yarl+++_DATE_PATTERN_REGEX = r"(\d+)\s*(second|seconds|minute|minutes|hour|hours|day|days|week|weeks|month|months|year|years)"+_DATE_PATTERN = re.compile(_DATE_PATTERN_REGEX, re.IGNORECASE)+_BYTE_SIZE_ADAPTER = TypeAdapter(ByteSize)+_ConvertibleToInt: TypeAlias = str | SupportsInt | SupportsIndex+++def bytesize_to_str(value: _ConvertibleToInt) -> str:+ if not isinstance(value, ByteSize):+ value = ByteSize(value)+ return value.human_readable(decimal=True)+++def to_yarl_url(value: AnyUrl | str, *args, **kwargs) -> yarl.URL:+ from cyberdrop_dl.utils.utilities import parse_url++ try:+ return parse_url(str(value), *args, **kwargs)+ except Exception as e:+ raise ValueError(str(e)) from e+++def to_yarl_url_w_pydantyc_validation(value: str) -> yarl.URL:+ url = HttpUrl(value)+ return to_yarl_url(url)+++def to_bytesize(value: ByteSize | str | int) -> ByteSize:+ return _BYTE_SIZE_ADAPTER.validate_python(value)+++def change_path_suffix(suffix: str) -> Callable[[Path], Path]:+ def with_suffix(value: Path) -> Path:+ return value.with_suffix(suffix)++ return with_suffix+++def str_to_timedelta(input_date: str) -> timedelta:+ time_str = input_date.casefold()+ matches: list[str] = re.findall(_DATE_PATTERN, time_str)+ seen_units = set()+ time_dict = {"days": 0}++ for value, unit in matches:+ value = int(value)+ unit = unit.lower()+ normalized_unit = unit.rstrip("s")+ plural_unit = normalized_unit + "s"+ if normalized_unit in seen_units:+ msg = f"Duplicate time unit detected: '{unit}' conflicts with another entry"+ raise ValueError(msg)+ seen_units.add(normalized_unit)++ if "day" in unit:+ time_dict["days"] += value+ elif "month" in unit:+ time_dict["days"] += value * 30+ elif "year" in unit:+ time_dict["days"] += value * 365+ else:+ time_dict[plural_unit] = value++ if not matches:+ msg = f"Unable to convert '{input_date}' to timedelta object"+ raise ValueError(msg)+ return timedelta(**time_dict)+++def to_timedelta(input_date: timedelta | str | int) -> timedelta:+ """Parses `datetime.timedelta`, `str` or `int` into a timedelta format.++ For `str`, the expected format is `<value> <unit>`, ex: `5 days`, `10 minutes`, `1 year`++ Valid units:+ `year(s)`, `week(s)`, `day(s)`, `hour(s)`, `minute(s)`, `second(s)`, `millisecond(s)`, `microsecond(s)`++ For `int`, `input_date` is assumed as `days`+ """+ return falsy_as(input_date, timedelta(0), _parse_as_timedelta)+++@singledispatch+def _parse_as_timedelta(input_date: timedelta) -> timedelta | str:+ return input_date+++@_parse_as_timedelta.register+def _(input_date: int) -> timedelta:+ return timedelta(days=input_date)+++@_parse_as_timedelta.register+def _(input_date: str, raise_error: bool = False) -> timedelta | str:+ try:+ return str_to_timedelta(input_date)+ except ValueError:+ if raise_error:+ raise+ return input_date+++def falsy_as(value: Any, falsy_value: Any, func: Callable | None = None, *args, **kwargs) -> Any:+ """If `value` is falsy, returns `falsy_value`++ If `value` is NOT falsy AND `func` is provided, returns `func(value, *args, **kwargs)`++ Returns `value` otherwise+ """+ if isinstance(value, str) and value.casefold() in ("none", "null"):+ value = None+ if not value:+ return falsy_value+ if not func:+ return value+ return func(value, *args, **kwargs)+++def falsy_as_list(value: Any) -> Any:+ return falsy_as(value, [])+++def falsy_as_none(value: Any) -> Any:+ return falsy_as(value, None)+++def falsy_as_dict(value: Any) -> Any:+ return falsy_as(value, {})+++class AppriseURLDict(TypedDict):+ url: str+ tags: set[str]+++@singledispatch+def to_apprise_url_dict(value: yarl.URL) -> AppriseURLDict:+ return {"url": str(value), "tags": set()}+++@to_apprise_url_dict.register+def _(value: dict) -> AppriseURLDict:+ tags = value.get("tags") or set()+ url = str(value.get("url", ""))+ if not tags:+ return to_apprise_url_dict(url)++ return {"url": url, "tags": tags}+++@to_apprise_url_dict.register+def _(value: str) -> AppriseURLDict:+ tags = set()+ url = value+ parts = url.split("://", 1)[0].split("=", 1)+ if len(parts) == 2:+ tags = set(parts[0].split(","))+ url: str = url.split("=", 1)[-1]++ return {"url": url, "tags": tags}diff --git a/cyberdrop_dl/types.py b/cyberdrop_dl/types.pyindex de17d62cc..6fb047449 100644--- a/cyberdrop_dl/types.py+++ b/cyberdrop_dl/types.py@@ -7,64 +7,35 @@from __future__ import annotations-import functools-import inspectfrom collections.abc import Callable, Mapping, Sequence-from functools import partial-from pathlib import Path-from typing import TYPE_CHECKING, Annotated, Any, Literal, NewType, ParamSpec, TypeAlias, TypeVar, overload+from typing import TYPE_CHECKING, Any, Literal, NewType, ParamSpec, TypeAlias, TypeVar, overloadimport yarl-from pydantic import (- AfterValidator,- AnyUrl,- BaseModel,- BeforeValidator,- ByteSize,- ConfigDict,- NonNegativeInt,- PlainSerializer,- PlainValidator,- Secret,- SerializationInfo,- StringConstraints,- model_serializer,- model_validator,-)--from cyberdrop_dl.utils.converters import change_path_suffix, convert_byte_size_to_str-from cyberdrop_dl.utils.validators import (- parse_apprise_url,- parse_falsy_as_none,- parse_list,- pydantyc_yarl_url,-)P = ParamSpec("P")R = TypeVar("R")T = TypeVar("T")-def copy_signature(target: Callable[P, R]) -> Callable[[Callable[..., T]], Callable[P, T]]:- """Decorator to make a function mimic the signature of another function,- but preserve the return type of the decorated function."""+if TYPE_CHECKING:+ import functools+ import inspect- def decorator(func: Callable[..., T]) -> Callable[P, T]:- @functools.wraps(func)- def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:- return func(*args, **kwargs)+ from propcache.api import under_cached_property as cached_property+ from yarl._query import Query, QueryVariable- wrapper.__signature__ = inspect.signature(target).replace( # type: ignore- return_annotation=inspect.signature(func).return_annotation- )- return wrapper+ def copy_signature(target: Callable[P, R]) -> Callable[[Callable[..., T]], Callable[P, T]]:+ def decorator(func: Callable[..., T]) -> Callable[P, T]:+ @functools.wraps(func)+ def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:+ return func(*args, **kwargs)- return decorator+ wrapper.__signature__ = inspect.signature(target).replace( # type: ignore+ return_annotation=inspect.signature(func).return_annotation+ )+ return wrapper--if TYPE_CHECKING:- from propcache.api import under_cached_property as cached_property- from yarl._query import Query, QueryVariable+ return decoratorclass AbsoluteHttpURL(yarl.URL):@copy_signature(yarl.URL.__new__)@@ -149,57 +120,6 @@ else:AnyURL = TypeVar("AnyURL", yarl.URL, AbsoluteHttpURL)-# ~~~~~ Strings ~~~~~~~-StrSerializer = PlainSerializer(str, return_type=str, when_used="json-unless-none")-NonEmptyStr = Annotated[str, StringConstraints(min_length=1, strip_whitespace=True)]-NonEmptyStrOrNone = Annotated[NonEmptyStr | None, BeforeValidator(parse_falsy_as_none)]-ListNonEmptyStr = Annotated[list[NonEmptyStr], BeforeValidator(parse_list)]--# ~~~~~ Paths ~~~~~~~-PathOrNone = Annotated[Path | None, BeforeValidator(parse_falsy_as_none)]-LogPath = Annotated[Path, AfterValidator(partial(change_path_suffix, suffix=".csv"))]-MainLogPath = Annotated[LogPath, AfterValidator(partial(change_path_suffix, suffix=".log"))]--# URL with pydantic.HttpUrl validation (must be absolute, must be http/https, detailed validation error).-# In type hints it's a yarl.URL. After validation the result is parsed with `parse_url` so this is also a yarl.URL at runtime-# Only use for config validation. To parse URLs internally while scraping, call `parse_url` directly-HttpURL = Annotated[yarl.URL, PlainValidator(pydantyc_yarl_url), StrSerializer]--# ~~~~~ Others ~~~~~~~-ByteSizeSerilized = Annotated[ByteSize, PlainSerializer(convert_byte_size_to_str, return_type=str)]-ListNonNegativeInt = Annotated[list[NonNegativeInt], BeforeValidator(parse_list)]---class AliasModel(BaseModel):- model_config = ConfigDict(populate_by_name=True)---class FrozenModel(BaseModel):- model_config = ConfigDict(frozen=True)---class AppriseURLModel(FrozenModel):- url: Secret[AnyUrl]- tags: set[str] = set()-- @model_serializer()- def serialize(self, info: SerializationInfo) -> str:- dump_secret = info.mode != "json"- url = self.url.get_secret_value() if dump_secret else self.url- tags = self.tags - set("no_logs")- tags = sorted(tags)- return f"{','.join(tags)}{'=' if tags else ''}{url}"-- @model_validator(mode="before")- @staticmethod- def parse_input(value: yarl.URL | dict | str) -> dict:- return parse_apprise_url(value)---class HttpAppriseURL(AppriseURLModel):- url: Secret[HttpURL]--Array: TypeAlias = list[T] | tuple[T, ...]CMD: TypeAlias = Array[str]U32Int: TypeAlias = int@@ -207,7 +127,6 @@ U32IntArray: TypeAlias = Array[U32Int]U32IntSequence: TypeAlias = Sequence[U32Int]AnyDict: TypeAlias = dict[str, Any]-AbsolutePath = NewType("AbsolutePath", Path)HashValue = NewType("HashValue", str)TimeStamp = NewType("TimeStamp", int)diff --git a/cyberdrop_dl/utils/apprise.py b/cyberdrop_dl/utils/apprise.pyindex 812ec0c68..4e2c86f49 100644--- a/cyberdrop_dl/utils/apprise.py+++ b/cyberdrop_dl/utils/apprise.py@@ -16,7 +16,7 @@ from pydantic import ValidationErrorfrom rich.text import Textfrom cyberdrop_dl import constants-from cyberdrop_dl.types import AppriseURLModel+from cyberdrop_dl.models import AppriseURLModelfrom cyberdrop_dl.utils.logger import log, log_debug, log_spacerfrom cyberdrop_dl.utils.yaml import handle_validation_errordiff --git a/cyberdrop_dl/utils/args.py b/cyberdrop_dl/utils/args.pyindex 141a93388..3bcddc63a 100644--- a/cyberdrop_dl/utils/args.py+++ b/cyberdrop_dl/utils/args.py@@ -8,13 +8,14 @@ from datetime import datefrom enum import StrEnum, autofrom pathlib import Pathfrom shutil import get_terminal_size-from typing import TYPE_CHECKING, NoReturn, Self+from typing import TYPE_CHECKING, Any, NoReturn, Selffrom pydantic import BaseModel, Field, ValidationError, computed_field, field_validator, model_validatorfrom cyberdrop_dl import __version__, env-from cyberdrop_dl.config_definitions import ConfigSettings, GlobalSettings-from cyberdrop_dl.types import AliasModel, AnyDict, HttpURL+from cyberdrop_dl.config import ConfigSettings, GlobalSettings+from cyberdrop_dl.models import AliasModel+from cyberdrop_dl.models.types import HttpURLfrom cyberdrop_dl.utils.yaml import handle_validation_errorif TYPE_CHECKING:@@ -262,10 +263,10 @@ def make_parser() -> tuple[ArgumentParser, dict[str, list[ArgGroup]]]:return parser, group_lists-def get_parsed_args_dict(args: Sequence[str] | None = None) -> dict[str, AnyDict]:+def get_parsed_args_dict(args: Sequence[str] | None = None) -> dict[str, dict[str, Any]]:parser, group_lists = make_parser()namespace = parser.parse_intermixed_args(args)- parsed_args: dict[str, AnyDict] = {}+ parsed_args: dict[str, dict[str, Any]] = {}for name, groups in group_lists.items():parsed_args[name] = {}for group in groups:diff --git a/cyberdrop_dl/utils/converters.py b/cyberdrop_dl/utils/converters.pydeleted file mode 100644index 54ad8f67a..000000000--- a/cyberdrop_dl/utils/converters.py+++ /dev/null@@ -1,76 +0,0 @@-"""-Functions to use with `AfterValidator`, `field_validator(mode="after")` or `model_validator(mode="after")`-"""--from __future__ import annotations--import re-from datetime import timedelta-from typing import TYPE_CHECKING--from pydantic import AnyUrl, ByteSize, TypeAdapter--from cyberdrop_dl.exceptions import InvalidURLError-from cyberdrop_dl.utils.utilities import parse_url--if TYPE_CHECKING:- from pathlib import Path-- import yarl--DATE_PATTERN_REGEX = r"(\d+)\s*(second|seconds|minute|minutes|hour|hours|day|days|week|weeks|month|months|year|years)"-DATE_PATTERN = re.compile(DATE_PATTERN_REGEX, re.IGNORECASE)--byte_size_adapter = TypeAdapter(ByteSize)---def convert_byte_size_to_str(value: ByteSize) -> str:- if not isinstance(value, ByteSize):- value = ByteSize(value)- return value.human_readable(decimal=True)---def convert_to_yarl(value: AnyUrl | str, *args, **kwargs) -> yarl.URL:- try:- return parse_url(str(value), *args, **kwargs)- except (InvalidURLError, TypeError) as e:- raise ValueError(str(e)) from e---def change_path_suffix(value: Path, suffix: str) -> Path:- return value.with_suffix(suffix)---def convert_to_byte_size(value: ByteSize | str | int) -> ByteSize:- return byte_size_adapter.validate_python(value)---def convert_str_to_timedelta(input_date: str) -> timedelta:- time_str = input_date.casefold()- matches: list[str] = re.findall(DATE_PATTERN, time_str)- seen_units = set()- time_dict = {"days": 0}-- for value, unit in matches:- value = int(value)- unit = unit.lower()- normalized_unit = unit.rstrip("s")- plural_unit = normalized_unit + "s"- if normalized_unit in seen_units:- msg = f"Duplicate time unit detected: '{unit}' conflicts with another entry"- raise ValueError(msg)- seen_units.add(normalized_unit)-- if "day" in unit:- time_dict["days"] += value- elif "month" in unit:- time_dict["days"] += value * 30- elif "year" in unit:- time_dict["days"] += value * 365- else:- time_dict[plural_unit] = value-- if not matches:- msg = f"Unable to convert '{input_date}' to timedelta object"- raise ValueError(msg)- return timedelta(**time_dict)diff --git a/cyberdrop_dl/utils/validators.py b/cyberdrop_dl/utils/validators.pydeleted file mode 100644index 681997f80..000000000--- a/cyberdrop_dl/utils/validators.py+++ /dev/null@@ -1,107 +0,0 @@-"""-Functions to use with `BeforeValidator`, `field_validator(mode="before")` or `model_validator(mode="before")`-"""--from __future__ import annotations--from datetime import timedelta-from functools import singledispatch-from typing import TYPE_CHECKING, Any--from pydantic import HttpUrl--from cyberdrop_dl.utils.converters import convert_str_to_timedelta, convert_to_yarl--if TYPE_CHECKING:- from collections.abc import Callable-- from yarl import URL---def pydantyc_yarl_url(value: str) -> URL:- url = HttpUrl(value)- return convert_to_yarl(url)---def parse_falsy_as(value: Any, falsy_value: Any, func: Callable | None = None, *args, **kwargs) -> Any:- """If `value` is falsy, returns `falsy_value`-- If `value` is NOT falsy AND `func` is provided, returns `func(value, *args, **kwargs)`-- Returns `value` otherwise- """- if isinstance(value, str) and value.casefold() in ("none", "null"):- value = None- if not value:- return falsy_value- if not func:- return value- return func(value, *args, **kwargs)---def parse_duration_as_timedelta(input_date: timedelta | str | int) -> timedelta:- """Parses `datetime.timedelta`, `str` or `int` into a timedelta format.-- For `str`, the expected format is `<value> <unit>`, ex: `5 days`, `10 minutes`, `1 year`-- Valid units:- `year(s)`, `week(s)`, `day(s)`, `hour(s)`, `minute(s)`, `second(s)`, `millisecond(s)`, `microsecond(s)`-- For `int`, `input_date` is assumed as `days`- """- return parse_falsy_as(input_date, timedelta(0), _parse_as_timedelta)---@singledispatch-def _parse_as_timedelta(input_date: timedelta) -> timedelta | str:- return input_date---@_parse_as_timedelta.register-def _(input_date: int) -> timedelta:- return timedelta(days=input_date)---@_parse_as_timedelta.register-def _(input_date: str, raise_error: bool = False) -> timedelta | str:- try:- return convert_str_to_timedelta(input_date)- except ValueError:- if raise_error:- raise- return input_date---def parse_list(value: list) -> Any:- return parse_falsy_as(value, [])---def parse_falsy_as_none(value: Any) -> Any:- return parse_falsy_as(value, None)---@singledispatch-def parse_apprise_url(value: URL) -> dict:- return {"url": str(value), "tags": set()}---@parse_apprise_url.register-def _(value: dict) -> dict:- tags = value.get("tags") or set()- url = str(value.get("url", ""))- if not tags:- return parse_apprise_url(url)-- return {"url": url, "tags": tags}---@parse_apprise_url.register-def _(value: str) -> dict:- tags = set()- url = value- parts = url.split("://", 1)[0].split("=", 1)- if len(parts) == 2:- tags = set(parts[0].split(","))- url: str = url.split("=", 1)[-1]-- return {"url": url, "tags": tags}