Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: move pydantic models and types to its own module (#1039)

NTFSvolume1f76ca7b9547
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 5cfc8b17d..32e57bc16 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -4,7 +4,6 @@ repos:
hooks:
- id: check-added-large-files
- id: check-case-conflict
- - id: check-executables-have-shebangs
- id: check-illegal-windows-names
- id: check-merge-conflict
- id: check-shebang-scripts-are-executable
diff --git a/cyberdrop_dl/config/__init__.py b/cyberdrop_dl/config/__init__.py
new file mode 100755
index 000000000..fbb36673c
--- /dev/null
+++ b/cyberdrop_dl/config/__init__.py
@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from time import sleep
+from typing import TYPE_CHECKING
+
+from cyberdrop_dl import constants, env
+from cyberdrop_dl.utils.apprise import get_apprise_urls
+
+from .auth_model import AuthSettings
+from .config_model import ConfigSettings
+from .global_model import GlobalSettings
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.utils.apprise import AppriseURL
+ from cyberdrop_dl.utils.args import ParsedArgs
+
+__all__ = [
+ "AuthSettings",
+ "ConfigSettings",
+ "GlobalSettings",
+]
+
+deep_scrape: bool = False
+
+current_config: Config
+cli: ParsedArgs
+appdata: AppData
+
+# re-export current config values for easy access
+auth: AuthSettings
+settings: ConfigSettings
+global_settings: GlobalSettings
+
+
+def startup() -> None:
+ global appdata, cli
+ from cyberdrop_dl.utils.args import parse_args
+
+ cli = parse_args()
+
+ if env.RUNNING_IN_IDE and Path.cwd().name == "cyberdrop_dl":
+ """This is for testing purposes only"""
+ constants.DEFAULT_APP_STORAGE = Path("../AppData")
+ constants.DEFAULT_DOWNLOAD_STORAGE = Path("../Downloads")
+
+ appdata_path = cli.cli_only_args.appdata_folder or constants.DEFAULT_APP_STORAGE
+ appdata = AppData(appdata_path.resolve())
+ appdata.mkdirs()
+ # cache.startup(appdata.cache_file)
+ load_config(get_default_config())
+ settings.logs._delete_old_logs_and_folders(constants.STARTUP_TIME)
+
+
+class AppData(Path):
+ def __init__(self, app_data_path: Path) -> None:
+ self.configs_dir = app_data_path / "Configs"
+ self.cache_dir = app_data_path / "Cache"
+ self.cookies_dir = app_data_path / "Cookies"
+ self.cache_file = self.cache_dir / "cache.yaml"
+ self.default_auth_config_file = self.configs_dir / "authentication.yaml"
+ self.global_config_file = self.configs_dir / "global_settings.yaml"
+ self.cache_db = self.cache_dir / "request_cache.db"
+ self.history_db = self.cache_dir / "cyberdrop.db"
+
+ def mkdirs(self) -> None:
+ for dir in (self.configs_dir, self.cache_dir, self.cookies_dir):
+ dir.mkdir(parents=True, exist_ok=True)
+
+
+class Config:
+ """Helper class to group a single config, not necessarily the current config"""
+
+ def __init__(self, name: str) -> None:
+ self.apprise_urls: list[AppriseURL] = []
+ self.folder = appdata.configs_dir / name
+ self.apprise_file = self.folder / "apprise.txt"
+ self.config_file = self.folder / "settings.yaml"
+ auth_override = self.folder / "authentication.yaml"
+ if auth_override.is_file():
+ self.auth_config_file = auth_override
+ else:
+ self.auth_config_file = appdata.default_auth_config_file
+ self.auth: AuthSettings
+ self.settings: ConfigSettings
+ self.global_settings: GlobalSettings
+
+ @staticmethod
+ def build(name: str, auth: AuthSettings, settings: ConfigSettings, global_settings: GlobalSettings) -> Config:
+ self = Config(name)
+ self.auth = auth
+ self.settings = settings
+ self.global_settings = global_settings
+ self.apprise_urls = get_apprise_urls(file=self.apprise_file)
+ return self
+
+ @staticmethod
+ def new_empty_config(name: str) -> Config:
+ assert name not in get_all_configs()
+ self = Config(name)
+ self._load()
+ return self
+
+ def _load(self) -> None:
+ """Read each config module from their respective files
+
+ If a files does not exists, uses the default config and creates it"""
+ self.auth = AuthSettings.load_file(self.auth_config_file, "socialmediagirls_username:")
+ self.settings = ConfigSettings.load_file(self.config_file, "download_error_urls_filename:")
+ self.global_settings = GlobalSettings.load_file(appdata.global_config_file, "Dupe_Cleanup_Options:")
+ self.apprise_urls = get_apprise_urls(file=self.apprise_file)
+
+ def _resolve_all_paths(self) -> None:
+ self.auth.resolve_paths(self.folder.name)
+ self.settings.resolve_paths(self.folder.name)
+ self.global_settings.resolve_paths(self.folder.name)
+
+ def _all_settings(self) -> tuple[ConfigSettings, AuthSettings, GlobalSettings]:
+ return self.settings, self.auth, self.global_settings
+
+ def write_updated_config(self) -> None:
+ """Writes config to disk."""
+ self.auth.save_to_file(self.auth_config_file)
+ self.settings.save_to_file(self.config_file)
+ self.global_settings.save_to_file(appdata.global_config_file)
+
+
+def get_default_config() -> str:
+ ...
+ # return cache.get(cache.DEFAULT_CONFIG_KEY) or "Default"
+
+
+def get_all_configs() -> list:
+ return sorted(config.name for config in appdata.configs_dir.iterdir() if config.is_dir())
+
+
+def set_default_config(config_name: str) -> None:
+ ...
+ # cache.save(cache.DEFAULT_CONFIG_KEY, config_name)
+
+
+def delete_config(config_name: str) -> None:
+ all_configs = get_all_configs()
+ assert config_name in all_configs
+ assert len(all_configs) > 1
+ assert config_name != current_config.folder.name
+ all_configs.remove(config_name)
+
+ # if cache.get(cache.DEFAULT_CONFIG_KEY) == config_name:
+ # set_default_config(all_configs[0])
+
+ config_path = appdata.configs_dir / config_name
+ shutil.rmtree(config_path)
+
+
+def load_config(config_name: str) -> None:
+ global current_config, auth, global_settings, settings
+ assert config_name
+ current_config = Config(config_name)
+ current_config._load()
+ current_config._resolve_all_paths()
+ settings, auth, global_settings = current_config._all_settings()
+ settings.logs._set_output_filenames(constants.STARTUP_TIME)
+
+ sleep(1)
diff --git a/cyberdrop_dl/config/_common.py b/cyberdrop_dl/config/_common.py
new file mode 100755
index 000000000..b5cf96a7e
--- /dev/null
+++ b/cyberdrop_dl/config/_common.py
@@ -0,0 +1,44 @@
+from pathlib import Path
+from typing import Any, Self
+
+from pydantic import Field as P_Field
+from pydantic.fields import _Unset
+
+from cyberdrop_dl.exceptions import InvalidYamlError
+from cyberdrop_dl.models import PathAliasModel, get_model_fields
+from cyberdrop_dl.utils import yaml
+
+
+def Field(default: Any, validation_alias: str = _Unset, **kwargs) -> Any: # noqa: N802
+ return P_Field(default=default, validation_alias=validation_alias, **kwargs)
+
+
+class ConfigModel(PathAliasModel):
+ @classmethod
+ def load_file(cls, file: Path, update_if_has_string: str) -> Self:
+ default = cls()
+ if not file.is_file():
+ config = default
+ needs_update = True
+
+ else:
+ all_fields = get_model_fields(default, exclude_unset=False)
+ config = cls.model_validate(yaml.load(file))
+ set_fields = get_model_fields(config)
+ needs_update = all_fields != set_fields or _is_in_file(update_if_has_string, file)
+
+ if needs_update:
+ yaml.save(file, config)
+ return config
+
+ def save_to_file(self, file: Path) -> None:
+ yaml.save(file, self)
+
+
+def _is_in_file(search_value: str, file: Path) -> bool:
+ try:
+ return search_value.casefold() in file.read_text().casefold()
+ except FileNotFoundError:
+ return False
+ except Exception as e:
+ raise InvalidYamlError(file, e) from e
diff --git a/cyberdrop_dl/config/auth_model.py b/cyberdrop_dl/config/auth_model.py
new file mode 100755
index 000000000..dc0f8853d
--- /dev/null
+++ b/cyberdrop_dl/config/auth_model.py
@@ -0,0 +1,62 @@
+from pydantic import BaseModel
+
+from cyberdrop_dl.models import AliasModel
+
+from ._common import ConfigModel, Field
+
+
+class CoomerAuth(BaseModel):
+ session: str = ""
+
+
+class XXXBunkerAuth(BaseModel):
+ PHPSESSID: str = ""
+
+
+class ImgurAuth(AliasModel):
+ client_id: str = Field("", "imgur_client_id")
+
+
+class MegaNzAuth(AliasModel):
+ email: str = ""
+ password: str = ""
+
+
+class JDownloaderAuth(AliasModel):
+ username: str = Field("", "jdownloader_username")
+ password: str = Field("", "jdownloader_password")
+ device: str = Field("", "jdownloader_device")
+
+
+class KemonoAuth(AliasModel):
+ session: str = ""
+
+
+class RedditAuth(AliasModel):
+ personal_use_script: str = Field("", "reddit_personal_use_script")
+ secret: str = Field("", "reddit_secret")
+
+
+class GoFileAuth(AliasModel):
+ api_key: str = Field("", "gofile_api_key")
+
+
+class PixeldrainAuth(AliasModel):
+ api_key: str = Field("", "pixeldrain_api_key")
+
+
+class RealDebridAuth(AliasModel):
+ api_key: str = Field("", "realdebrid_api_key")
+
+
+class AuthSettings(ConfigModel):
+ coomer: CoomerAuth = Field(CoomerAuth(), "Coomer")
+ gofile: GoFileAuth = Field(GoFileAuth(), "GoFile")
+ imgur: ImgurAuth = Field(ImgurAuth(), "Imgur")
+ jdownloader: JDownloaderAuth = Field(JDownloaderAuth(), "JDownloader")
+ kemono: KemonoAuth = Field(KemonoAuth(), "Kemono")
+ meganz: MegaNzAuth = Field(MegaNzAuth(), "MegaNz")
+ pixeldrain: PixeldrainAuth = Field(PixeldrainAuth(), "PixelDrain")
+ realdebrid: RealDebridAuth = Field(RealDebridAuth(), "RealDebrid")
+ reddit: RedditAuth = Field(RedditAuth(), "Reddit")
+ xxxbunker: XXXBunkerAuth = Field(XXXBunkerAuth(), "XXXBunker")
diff --git a/cyberdrop_dl/config_definitions/config_settings.py b/cyberdrop_dl/config/config_model.py
old mode 100644
new mode 100755
similarity index 60%
rename from cyberdrop_dl/config_definitions/config_settings.py
rename to cyberdrop_dl/config/config_model.py
index cf90a20a0..5c02e4e53
--- a/cyberdrop_dl/config_definitions/config_settings.py
+++ b/cyberdrop_dl/config/config_model.py
@@ -1,17 +1,18 @@
+import itertools
import re
-from datetime import timedelta
+from datetime import datetime, timedelta
from logging import DEBUG
from pathlib import Path
-from pydantic import BaseModel, ByteSize, Field, NonNegativeInt, PositiveInt, field_serializer, field_validator
+from pydantic import BaseModel, ByteSize, NonNegativeInt, PositiveInt, field_serializer, field_validator
+from cyberdrop_dl import constants
from cyberdrop_dl.constants import BROWSERS, DEFAULT_APP_STORAGE, DEFAULT_DOWNLOAD_STORAGE
from cyberdrop_dl.data_structures.hash import Hashing
from cyberdrop_dl.data_structures.supported_domains import SUPPORTED_SITES_DOMAINS
-from cyberdrop_dl.types import (
- AliasModel,
+from cyberdrop_dl.models import HttpAppriseURL
+from cyberdrop_dl.models.types import (
ByteSizeSerilized,
- HttpAppriseURL,
ListNonEmptyStr,
ListNonNegativeInt,
LogPath,
@@ -20,7 +21,10 @@ from cyberdrop_dl.types import (
NonEmptyStrOrNone,
PathOrNone,
)
-from cyberdrop_dl.utils.validators import parse_duration_as_timedelta, parse_falsy_as
+from cyberdrop_dl.models.validators import falsy_as, to_timedelta
+from cyberdrop_dl.utils.utilities import purge_dir_tree
+
+from ._common import ConfigModel, Field, PathAliasModel
ALL_SUPPORTED_SITES = ["<<ALL_SUPPORTED_SITES>>"]
@@ -42,42 +46,66 @@ class DownloadOptions(BaseModel):
maximum_thread_depth: NonNegativeInt = 0
-class Files(AliasModel):
- download_folder: Path = Field(validation_alias="d", default=DEFAULT_DOWNLOAD_STORAGE)
- dump_json: bool = Field(default=False, validation_alias="j")
- input_file: Path = Field(validation_alias="i", default=DEFAULT_APP_STORAGE / "Configs" / "{config}" / "URLs.txt")
+class Files(PathAliasModel):
+ download_folder: Path = Field(DEFAULT_DOWNLOAD_STORAGE, "d")
+ dump_json: bool = Field(False, "j")
+ input_file: Path = Field(DEFAULT_APP_STORAGE / "Configs{config}/URLs.txt", "i")
save_pages_html: bool = False
-class Logs(AliasModel):
- download_error_urls: LogPath = Field(
- default=Path("Download_Error_URLs.csv"), validation_alias="download_error_urls_filename"
- )
- last_forum_post: LogPath = Field(
- default=Path("Last_Scraped_Forum_Posts.csv"), validation_alias="last_forum_post_filename"
- )
- log_folder: Path = DEFAULT_APP_STORAGE / "Configs" / "{config}" / "Logs"
- log_line_width: PositiveInt = Field(default=240, ge=50)
+class Logs(PathAliasModel):
+ download_error_urls: LogPath = Field(Path("Download_Error_URLs.csv"), "download_error_urls_filename")
+ last_forum_post: LogPath = Field(Path("Last_Scraped_Forum_Posts.csv"), "last_forum_post_filename")
+ log_folder: Path = DEFAULT_APP_STORAGE / "Configs/{config}/Logs"
+ log_line_width: PositiveInt = Field(240, ge=50)
logs_expire_after: timedelta | None = None
- main_log: MainLogPath = Field(default=Path("downloader.log"), validation_alias="main_log_filename")
+ main_log: MainLogPath = Field(Path("downloader.log"), "main_log_filename")
rotate_logs: bool = False
- scrape_error_urls: LogPath = Field(
- default=Path("Scrape_Error_URLs.csv"), validation_alias="scrape_error_urls_filename"
- )
- unsupported_urls: LogPath = Field(
- default=Path("Unsupported_URLs.csv"), validation_alias="unsupported_urls_filename"
- )
- webhook: HttpAppriseURL | None = Field(default=None, validation_alias="webhook_url")
+ scrape_error_urls: LogPath = Field(Path("Scrape_Error_URLs.csv"), "scrape_error_urls_filename")
+ unsupported_urls: LogPath = Field(Path("Unsupported_URLs.csv"), "unsupported_urls_filename")
+ webhook: HttpAppriseURL | None = Field(None, "webhook_url")
+
+ @property
+ def jsonl_file(self) -> Path:
+ return self.main_log.with_suffix(".results.jsonl")
+
+ @property
+ def cdl_responses_dir(self) -> Path:
+ return self.main_log.parent / "cdl_responses"
@field_validator("webhook", mode="before")
@classmethod
def handle_falsy(cls, value: str) -> str | None:
- return parse_falsy_as(value, None)
+ return falsy_as(value, None)
@field_validator("logs_expire_after", mode="before")
@staticmethod
def parse_logs_duration(input_date: timedelta | str | int | None) -> timedelta | str | None:
- return parse_falsy_as(input_date, None, parse_duration_as_timedelta)
+ return falsy_as(input_date, None, to_timedelta)
+
+ def _set_output_filenames(self, now: datetime) -> None:
+ self.log_folder.mkdir(exist_ok=True, parents=True)
+ current_time_file_iso: str = now.strftime(constants.LOGS_DATETIME_FORMAT)
+ current_time_folder_iso: str = now.strftime(constants.LOGS_DATE_FORMAT)
+ for attr, log_file in vars(self).items():
+ if not isinstance(log_file, Path) or log_file.suffix not in (".csv", ".log"):
+ continue
+
+ if self.rotate_logs:
+ new_name = f"{log_file.stem}_{current_time_file_iso}{log_file.suffix}"
+ log_file: Path = log_file.parent / current_time_folder_iso / new_name
+ setattr(self, attr, self.log_folder / log_file)
+
+ log_file.parent.mkdir(exist_ok=True, parents=True)
+
+ def _delete_old_logs_and_folders(self, now: datetime | None = None) -> None:
+ if now and self.logs_expire_after:
+ for file in itertools.chain(self.log_folder.rglob("*.log"), self.log_folder.rglob("*.csv")):
+ file_date = file.stat().st_ctime
+ t_delta = now - datetime.fromtimestamp(file_date)
+ if t_delta > self.logs_expire_after:
+ file.unlink(missing_ok=True)
+ purge_dir_tree(self.log_folder)
class FileSizeLimits(BaseModel):
@@ -106,7 +134,7 @@ class MediaDurationLimits(BaseModel):
"""
if input_date is None:
return timedelta(seconds=0)
- return parse_duration_as_timedelta(input_date)
+ return to_timedelta(input_date)
class IgnoreOptions(BaseModel):
@@ -167,7 +195,7 @@ class BrowserCookies(BaseModel):
@field_validator("browsers", mode="before")
@classmethod
def parse_browsers(cls, values: list) -> list:
- values = parse_falsy_as(values, [])
+ values = falsy_as(values, [])
if isinstance(values, list):
return sorted(str(value).lower() for value in values)
return values
@@ -175,7 +203,7 @@ class BrowserCookies(BaseModel):
@field_validator("sites", mode="before")
@classmethod
def handle_list(cls, values: list) -> list:
- values = parse_falsy_as(values, [])
+ values = falsy_as(values, [])
if values == ALL_SUPPORTED_SITES:
return SUPPORTED_SITES_DOMAINS
if isinstance(values, list):
@@ -197,16 +225,14 @@ class DupeCleanup(BaseModel):
send_deleted_to_trash: bool = True
-class ConfigSettings(AliasModel):
- browser_cookies: BrowserCookies = Field(validation_alias="Browser_Cookies", default=BrowserCookies())
- download_options: DownloadOptions = Field(validation_alias="Download_Options", default=DownloadOptions())
- dupe_cleanup_options: DupeCleanup = Field(validation_alias="Dupe_Cleanup_Options", default=DupeCleanup())
- file_size_limits: FileSizeLimits = Field(validation_alias="File_Size_Limits", default=FileSizeLimits())
- media_duration_limits: MediaDurationLimits = Field(
- validation_alias="Media_Duration_Limits", default=MediaDurationLimits()
- )
- files: Files = Field(validation_alias="Files", default=Files())
- ignore_options: IgnoreOptions = Field(validation_alias="Ignore_Options", default=IgnoreOptions())
- logs: Logs = Field(validation_alias="Logs", default=Logs())
- runtime_options: RuntimeOptions = Field(validation_alias="Runtime_Options", default=RuntimeOptions())
- sorting: Sorting = Field(validation_alias="Sorting", default=Sorting())
+class ConfigSettings(ConfigModel):
+ browser_cookies: BrowserCookies = Field(BrowserCookies(), "Browser_Cookies")
+ download_options: DownloadOptions = Field(DownloadOptions(), "Download_Options")
+ dupe_cleanup_options: DupeCleanup = Field(DupeCleanup(), "Dupe_Cleanup_Options")
+ file_size_limits: FileSizeLimits = Field(FileSizeLimits(), "File_Size_Limits")
+ media_duration_limits: MediaDurationLimits = Field(MediaDurationLimits(), "Media_Duration_Limits")
+ files: Files = Field(Files(), "Files")
+ ignore_options: IgnoreOptions = Field(IgnoreOptions(), "Ignore_Options")
+ logs: Logs = Field(Logs(), "Logs")
+ runtime_options: RuntimeOptions = Field(RuntimeOptions(), "Runtime_Options")
+ sorting: Sorting = Field(Sorting(), "Sorting")
diff --git a/cyberdrop_dl/config_definitions/global_settings.py b/cyberdrop_dl/config/global_model.py
old mode 100644
new mode 100755
similarity index 70%
rename from cyberdrop_dl/config_definitions/global_settings.py
rename to cyberdrop_dl/config/global_model.py
index 425ac5dd9..902826727
--- a/cyberdrop_dl/config_definitions/global_settings.py
+++ b/cyberdrop_dl/config/global_model.py
@@ -1,15 +1,15 @@
import random
from datetime import timedelta
-from pydantic import BaseModel, ByteSize, Field, NonNegativeFloat, PositiveInt, field_serializer, field_validator
+from pydantic import BaseModel, ByteSize, NonNegativeFloat, PositiveInt, field_serializer, field_validator
from yarl import URL
-from cyberdrop_dl.types import AliasModel, ByteSizeSerilized, HttpURL, NonEmptyStr
-from cyberdrop_dl.utils.converters import convert_to_byte_size
-from cyberdrop_dl.utils.validators import parse_duration_as_timedelta, parse_falsy_as
+from cyberdrop_dl.config._common import ConfigModel, Field
+from cyberdrop_dl.models.types import ByteSizeSerilized, HttpURL, NonEmptyStr
+from cyberdrop_dl.models.validators import falsy_as, to_bytesize, to_timedelta
-MIN_REQUIRED_FREE_SPACE = convert_to_byte_size("512MB")
-DEFAULT_REQUIRED_FREE_SPACE = convert_to_byte_size("5GB")
+MIN_REQUIRED_FREE_SPACE = to_bytesize("512MB")
+DEFAULT_REQUIRED_FREE_SPACE = to_bytesize("5GB")
class General(BaseModel):
@@ -25,12 +25,12 @@ class General(BaseModel):
@field_serializer("flaresolverr", "proxy")
def serialize(self, value: URL | str) -> str | None:
- return parse_falsy_as(value, None, str)
+ return falsy_as(value, None, str)
@field_validator("flaresolverr", "proxy", mode="before")
@classmethod
def convert_to_str(cls, value: URL | str) -> str | None:
- return parse_falsy_as(value, None, str)
+ return falsy_as(value, None, str)
@field_validator("required_free_space", mode="after")
@classmethod
@@ -54,7 +54,7 @@ class RateLimiting(BaseModel):
@field_validator("file_host_cache_expire_after", "forum_cache_expire_after", mode="before")
@staticmethod
def parse_cache_duration(input_date: timedelta | str | int) -> timedelta:
- return parse_duration_as_timedelta(input_date)
+ return to_timedelta(input_date)
@property
def total_delay(self) -> NonNegativeFloat:
@@ -73,7 +73,7 @@ class UIOptions(BaseModel):
vi_mode: bool = False
-class GlobalSettings(AliasModel):
- general: General = Field(validation_alias="General", default=General())
- rate_limiting_options: RateLimiting = Field(validation_alias="Rate_Limiting_Options", default=RateLimiting())
- ui_options: UIOptions = Field(validation_alias="UI_Options", default=UIOptions())
+class GlobalSettings(ConfigModel):
+ general: General = Field(General(), "General")
+ rate_limiting_options: RateLimiting = Field(RateLimiting(), "Rate_Limiting_Options")
+ ui_options: UIOptions = Field(UIOptions(), "UI_Options")
diff --git a/cyberdrop_dl/config_definitions/__init__.py b/cyberdrop_dl/config_definitions/__init__.py
deleted file mode 100644
index 7e9fb09cb..000000000
--- a/cyberdrop_dl/config_definitions/__init__.py
+++ /dev/null
@@ -1,9 +0,0 @@
-from .authentication_settings import AuthSettings
-from .config_settings import ConfigSettings
-from .global_settings import GlobalSettings
-
-__all__ = [
- "AuthSettings",
- "ConfigSettings",
- "GlobalSettings",
-]
diff --git a/cyberdrop_dl/config_definitions/authentication_settings.py b/cyberdrop_dl/config_definitions/authentication_settings.py
deleted file mode 100644
index e7a6b1fad..000000000
--- a/cyberdrop_dl/config_definitions/authentication_settings.py
+++ /dev/null
@@ -1,60 +0,0 @@
-from pydantic import BaseModel, Field
-
-from cyberdrop_dl.types import AliasModel
-
-
-class CoomerAuth(BaseModel):
- session: str = ""
-
-
-class XXXBunkerAuth(BaseModel):
- PHPSESSID: str = ""
-
-
-class ImgurAuth(AliasModel):
- client_id: str = Field("", validation_alias="imgur_client_id")
-
-
-class MegaNzAuth(AliasModel):
- email: str = ""
- password: str = ""
-
-
-class JDownloaderAuth(AliasModel):
- username: str = Field("", validation_alias="jdownloader_username")
- password: str = Field("", validation_alias="jdownloader_password")
- device: str = Field("", validation_alias="jdownloader_device")
-
-
-class KemonoAuth(AliasModel):
- session: str = ""
-
-
-class RedditAuth(AliasModel):
- personal_use_script: str = Field("", validation_alias="reddit_personal_use_script")
- secret: str = Field("", validation_alias="reddit_secret")
-
-
-class GoFileAuth(AliasModel):
- api_key: str = Field("", validation_alias="gofile_api_key")
-
-
-class PixeldrainAuth(AliasModel):
- api_key: str = Field("", validation_alias="pixeldrain_api_key")
-
-
-class RealDebridAuth(AliasModel):
- api_key: str = Field("", validation_alias="realdebrid_api_key")
-
-
-class AuthSettings(AliasModel):
- coomer: CoomerAuth = Field(validation_alias="Coomer", default=CoomerAuth())
- gofile: GoFileAuth = Field(validation_alias="GoFile", default=GoFileAuth()) # type: ignore
- imgur: ImgurAuth = Field(validation_alias="Imgur", default=ImgurAuth()) # type: ignore
- jdownloader: JDownloaderAuth = Field(validation_alias="JDownloader", default=JDownloaderAuth()) # type: ignore
- kemono: KemonoAuth = Field(validation_alias="Kemono", default=KemonoAuth()) # type: ignore
- meganz: MegaNzAuth = Field(validation_alias="MegaNz", default=MegaNzAuth()) # type: ignore
- pixeldrain: PixeldrainAuth = Field(validation_alias="PixelDrain", default=PixeldrainAuth()) # type: ignore
- realdebrid: RealDebridAuth = Field(validation_alias="RealDebrid", default=RealDebridAuth()) # type: ignore
- reddit: RedditAuth = Field(validation_alias="Reddit", default=RedditAuth()) # type: ignore
- xxxbunker: XXXBunkerAuth = Field(validation_alias="XXXBunker", default=XXXBunkerAuth()) # type: ignore
diff --git a/cyberdrop_dl/crawlers/_kemono_base.py b/cyberdrop_dl/crawlers/_kemono_base.py
index 7ea725281..ef701834b 100644
--- a/cyberdrop_dl/crawlers/_kemono_base.py
+++ b/cyberdrop_dl/crawlers/_kemono_base.py
@@ -13,11 +13,11 @@ from typing_extensions import TypedDict # Import from typing is not compatible
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import NoExtensionError, ScrapeError
-from cyberdrop_dl.types import AbsoluteHttpURL, AliasModel
+from cyberdrop_dl.models import AliasModel
+from cyberdrop_dl.models.validators import falsy_as_none
from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.dates import to_timestamp
from cyberdrop_dl.utils.utilities import error_handling_wrapper, remove_parts
-from cyberdrop_dl.utils.validators import parse_falsy_as_none
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Callable, Coroutine, Generator
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
from yarl import URL
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+ from cyberdrop_dl.types import AbsoluteHttpURL
MAX_OFFSET_PER_CALL = 50
LINK_REGEX = re.compile(r"(?:http(?!.*\.\.)[^ ]*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]|</|'))")
@@ -112,7 +113,7 @@ class File(TypedDict):
server: NotRequired[str] # Sometimes present in attachments
-FileOrNone = Annotated[File | None, BeforeValidator(parse_falsy_as_none)]
+FileOrNone = Annotated[File | None, BeforeValidator(falsy_as_none)]
class Post(AliasModel):
diff --git a/cyberdrop_dl/crawlers/box_dot_com.py b/cyberdrop_dl/crawlers/box_dot_com.py
index 124bf75f1..9250c1b21 100644
--- a/cyberdrop_dl/crawlers/box_dot_com.py
+++ b/cyberdrop_dl/crawlers/box_dot_com.py
@@ -10,7 +10,8 @@ from pydantic import Field
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.exceptions import ScrapeError
-from cyberdrop_dl.types import AbsoluteHttpURL, AliasModel, SupportedDomains, SupportedPaths
+from cyberdrop_dl.models import AliasModel
+from cyberdrop_dl.types import AbsoluteHttpURL, SupportedDomains, SupportedPaths
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.py
index 663bfc8a1..046211c64 100644
--- a/cyberdrop_dl/crawlers/xhamster.py
+++ b/cyberdrop_dl/crawlers/xhamster.py
@@ -7,7 +7,8 @@ from typing import TYPE_CHECKING, Annotated, Any, ClassVar, NamedTuple
from pydantic import AliasPath, Field, PlainValidator
from cyberdrop_dl.crawlers.crawler import Crawler
-from cyberdrop_dl.types import AbsoluteHttpURL, AliasModel, SupportedPaths
+from cyberdrop_dl.models import AliasModel
+from cyberdrop_dl.types import AbsoluteHttpURL, SupportedPaths
from cyberdrop_dl.utils.logger import log_debug
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, parse_url
diff --git a/cyberdrop_dl/data_structures/hash.py b/cyberdrop_dl/data_structures/hash.py
index de76fcd8b..47cf42aed 100644
--- a/cyberdrop_dl/data_structures/hash.py
+++ b/cyberdrop_dl/data_structures/hash.py
@@ -7,7 +7,7 @@ class Hashing(StrEnum):
POST_DOWNLOAD = auto()
@classmethod
- def _missing_(cls, value):
+ def _missing_(cls, value: str):
try:
return cls[str(value.upper())]
except KeyError as e:
diff --git a/cyberdrop_dl/managers/config_manager.py b/cyberdrop_dl/managers/config_manager.py
index aa0b75f2d..1750f0dc1 100644
--- a/cyberdrop_dl/managers/config_manager.py
+++ b/cyberdrop_dl/managers/config_manager.py
@@ -6,7 +6,7 @@ from dataclasses import field
from time import sleep
from typing import TYPE_CHECKING
-from cyberdrop_dl.config_definitions import AuthSettings, ConfigSettings, GlobalSettings
+from cyberdrop_dl.config import AuthSettings, ConfigSettings, GlobalSettings
from cyberdrop_dl.exceptions import InvalidYamlError
from cyberdrop_dl.managers.log_manager import LogManager
from cyberdrop_dl.utils import yaml
diff --git a/cyberdrop_dl/models/__init__.py b/cyberdrop_dl/models/__init__.py
new file mode 100755
index 000000000..825a75079
--- /dev/null
+++ b/cyberdrop_dl/models/__init__.py
@@ -0,0 +1,15 @@
+from pydantic import BaseModel
+
+from .base_models import AliasModel, AppriseURLModel, FrozenModel, HttpAppriseURL, PathAliasModel
+
+
+def get_model_fields(model: BaseModel, *, exclude_unset: bool = True) -> set[str]:
+ fields = set()
+ default_dict: dict = model.model_dump(exclude_unset=exclude_unset)
+ for submodel_name, submodel in default_dict.items():
+ for field_name in submodel:
+ fields.add(f"{submodel_name}.{field_name}")
+ return fields
+
+
+__all__ = ["AliasModel", "AppriseURLModel", "FrozenModel", "HttpAppriseURL", "PathAliasModel", "get_model_fields"]
diff --git a/cyberdrop_dl/models/base_models.py b/cyberdrop_dl/models/base_models.py
new file mode 100755
index 000000000..d50fca9f0
--- /dev/null
+++ b/cyberdrop_dl/models/base_models.py
@@ -0,0 +1,53 @@
+"""Pydantic models"""
+
+from collections.abc import Mapping
+from pathlib import Path
+
+import yarl
+from pydantic import AnyUrl, BaseModel, ConfigDict, Secret, SerializationInfo, model_serializer, model_validator
+
+from cyberdrop_dl.models.types import HttpURL
+from cyberdrop_dl.models.validators import to_apprise_url_dict
+
+
+class AliasModel(BaseModel):
+ model_config = ConfigDict(populate_by_name=True)
+
+
+class FrozenModel(BaseModel):
+ model_config = ConfigDict(frozen=True)
+
+
+class AppriseURLModel(FrozenModel):
+ url: Secret[AnyUrl]
+ tags: set[str] = set()
+
+ @model_serializer()
+ def serialize(self, info: SerializationInfo) -> str:
+ dump_secret = info.mode != "json"
+ url = self.url.get_secret_value() if dump_secret else self.url
+ tags = self.tags - set("no_logs")
+ tags = sorted(tags)
+ return f"{','.join(tags)}{'=' if tags else ''}{url}"
+
+ @model_validator(mode="before")
+ @staticmethod
+ def parse_input(value: yarl.URL | dict | str) -> Mapping:
+ return to_apprise_url_dict(value)
+
+
+class HttpAppriseURL(AppriseURLModel):
+ url: Secret[HttpURL]
+
+
+class PathAliasModel(AliasModel):
+ @staticmethod
+ def replace_config_and_resolve(path: Path, current_config: str) -> Path:
+ return Path(str(path).replace("{config}", current_config)).resolve()
+
+ def resolve_paths(self, config_name: str) -> None:
+ for name, value in vars(self).items():
+ if isinstance(value, Path):
+ setattr(self, name, self.replace_config_and_resolve(value, config_name))
+ elif isinstance(value, PathAliasModel):
+ value.resolve_paths(config_name)
diff --git a/cyberdrop_dl/models/types.py b/cyberdrop_dl/models/types.py
new file mode 100755
index 000000000..f16b0d292
--- /dev/null
+++ b/cyberdrop_dl/models/types.py
@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import yarl
+from pydantic import (
+ AfterValidator,
+ BeforeValidator,
+ ByteSize,
+ NonNegativeInt,
+ PlainSerializer,
+ PlainValidator,
+ StringConstraints,
+)
+
+from cyberdrop_dl.models.validators import (
+ bytesize_to_str,
+ change_path_suffix,
+ falsy_as_list,
+ falsy_as_none,
+ to_yarl_url_w_pydantyc_validation,
+)
+
+# ~~~~~ Strings ~~~~~~~
+StrSerializer = PlainSerializer(str, return_type=str, when_used="json-unless-none")
+NonEmptyStr = Annotated[str, StringConstraints(min_length=1, strip_whitespace=True)]
+NonEmptyStrOrNone = Annotated[NonEmptyStr | None, BeforeValidator(falsy_as_none)]
+ListNonEmptyStr = Annotated[list[NonEmptyStr], BeforeValidator(falsy_as_list)]
+
+# ~~~~~ Paths ~~~~~~~
+PathOrNone = Annotated[Path | None, BeforeValidator(falsy_as_none)]
+LogPath = Annotated[Path, AfterValidator(change_path_suffix(".csv"))]
+MainLogPath = Annotated[LogPath, AfterValidator(change_path_suffix(".log"))]
+
+# URL with pydantic.HttpUrl validation (must be absolute, must be http/https, detailed validation error).
+# In type hints it's a yarl.URL. After validation the result is parsed with `parse_url` so this is also a yarl.URL at runtime
+# Only use for config validation. To parse URLs internally while scraping, call `parse_url` directly
+HttpURL = Annotated[yarl.URL, PlainValidator(to_yarl_url_w_pydantyc_validation), StrSerializer]
+
+# ~~~~~ Others ~~~~~~~
+ByteSizeSerilized = Annotated[ByteSize, PlainSerializer(bytesize_to_str, return_type=str)]
+ListNonNegativeInt = Annotated[list[NonNegativeInt], BeforeValidator(falsy_as_list)]
diff --git a/cyberdrop_dl/models/validators.py b/cyberdrop_dl/models/validators.py
new file mode 100755
index 000000000..529aa3d1e
--- /dev/null
+++ b/cyberdrop_dl/models/validators.py
@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import re
+from datetime import timedelta
+from functools import singledispatch
+from typing import TYPE_CHECKING, Any, SupportsIndex, SupportsInt, TypeAlias, TypedDict
+
+from pydantic import AnyUrl, ByteSize, HttpUrl, TypeAdapter
+
+if TYPE_CHECKING:
+ from collections.abc import Callable
+ from pathlib import Path
+
+ import yarl
+
+
+_DATE_PATTERN_REGEX = r"(\d+)\s*(second|seconds|minute|minutes|hour|hours|day|days|week|weeks|month|months|year|years)"
+_DATE_PATTERN = re.compile(_DATE_PATTERN_REGEX, re.IGNORECASE)
+_BYTE_SIZE_ADAPTER = TypeAdapter(ByteSize)
+_ConvertibleToInt: TypeAlias = str | SupportsInt | SupportsIndex
+
+
+def bytesize_to_str(value: _ConvertibleToInt) -> str:
+ if not isinstance(value, ByteSize):
+ value = ByteSize(value)
+ return value.human_readable(decimal=True)
+
+
+def to_yarl_url(value: AnyUrl | str, *args, **kwargs) -> yarl.URL:
+ from cyberdrop_dl.utils.utilities import parse_url
+
+ try:
+ return parse_url(str(value), *args, **kwargs)
+ except Exception as e:
+ raise ValueError(str(e)) from e
+
+
+def to_yarl_url_w_pydantyc_validation(value: str) -> yarl.URL:
+ url = HttpUrl(value)
+ return to_yarl_url(url)
+
+
+def to_bytesize(value: ByteSize | str | int) -> ByteSize:
+ return _BYTE_SIZE_ADAPTER.validate_python(value)
+
+
+def change_path_suffix(suffix: str) -> Callable[[Path], Path]:
+ def with_suffix(value: Path) -> Path:
+ return value.with_suffix(suffix)
+
+ return with_suffix
+
+
+def str_to_timedelta(input_date: str) -> timedelta:
+ time_str = input_date.casefold()
+ matches: list[str] = re.findall(_DATE_PATTERN, time_str)
+ seen_units = set()
+ time_dict = {"days": 0}
+
+ for value, unit in matches:
+ value = int(value)
+ unit = unit.lower()
+ normalized_unit = unit.rstrip("s")
+ plural_unit = normalized_unit + "s"
+ if normalized_unit in seen_units:
+ msg = f"Duplicate time unit detected: '{unit}' conflicts with another entry"
+ raise ValueError(msg)
+ seen_units.add(normalized_unit)
+
+ if "day" in unit:
+ time_dict["days"] += value
+ elif "month" in unit:
+ time_dict["days"] += value * 30
+ elif "year" in unit:
+ time_dict["days"] += value * 365
+ else:
+ time_dict[plural_unit] = value
+
+ if not matches:
+ msg = f"Unable to convert '{input_date}' to timedelta object"
+ raise ValueError(msg)
+ return timedelta(**time_dict)
+
+
+def to_timedelta(input_date: timedelta | str | int) -> timedelta:
+ """Parses `datetime.timedelta`, `str` or `int` into a timedelta format.
+
+ For `str`, the expected format is `<value> <unit>`, ex: `5 days`, `10 minutes`, `1 year`
+
+ Valid units:
+ `year(s)`, `week(s)`, `day(s)`, `hour(s)`, `minute(s)`, `second(s)`, `millisecond(s)`, `microsecond(s)`
+
+ For `int`, `input_date` is assumed as `days`
+ """
+ return falsy_as(input_date, timedelta(0), _parse_as_timedelta)
+
+
+@singledispatch
+def _parse_as_timedelta(input_date: timedelta) -> timedelta | str:
+ return input_date
+
+
+@_parse_as_timedelta.register
+def _(input_date: int) -> timedelta:
+ return timedelta(days=input_date)
+
+
+@_parse_as_timedelta.register
+def _(input_date: str, raise_error: bool = False) -> timedelta | str:
+ try:
+ return str_to_timedelta(input_date)
+ except ValueError:
+ if raise_error:
+ raise
+ return input_date
+
+
+def falsy_as(value: Any, falsy_value: Any, func: Callable | None = None, *args, **kwargs) -> Any:
+ """If `value` is falsy, returns `falsy_value`
+
+ If `value` is NOT falsy AND `func` is provided, returns `func(value, *args, **kwargs)`
+
+ Returns `value` otherwise
+ """
+ if isinstance(value, str) and value.casefold() in ("none", "null"):
+ value = None
+ if not value:
+ return falsy_value
+ if not func:
+ return value
+ return func(value, *args, **kwargs)
+
+
+def falsy_as_list(value: Any) -> Any:
+ return falsy_as(value, [])
+
+
+def falsy_as_none(value: Any) -> Any:
+ return falsy_as(value, None)
+
+
+def falsy_as_dict(value: Any) -> Any:
+ return falsy_as(value, {})
+
+
+class AppriseURLDict(TypedDict):
+ url: str
+ tags: set[str]
+
+
+@singledispatch
+def to_apprise_url_dict(value: yarl.URL) -> AppriseURLDict:
+ return {"url": str(value), "tags": set()}
+
+
+@to_apprise_url_dict.register
+def _(value: dict) -> AppriseURLDict:
+ tags = value.get("tags") or set()
+ url = str(value.get("url", ""))
+ if not tags:
+ return to_apprise_url_dict(url)
+
+ return {"url": url, "tags": tags}
+
+
+@to_apprise_url_dict.register
+def _(value: str) -> AppriseURLDict:
+ tags = set()
+ url = value
+ parts = url.split("://", 1)[0].split("=", 1)
+ if len(parts) == 2:
+ tags = set(parts[0].split(","))
+ url: str = url.split("=", 1)[-1]
+
+ return {"url": url, "tags": tags}
diff --git a/cyberdrop_dl/types.py b/cyberdrop_dl/types.py
index de17d62cc..6fb047449 100644
--- a/cyberdrop_dl/types.py
+++ b/cyberdrop_dl/types.py
@@ -7,64 +7,35 @@
from __future__ import annotations
-import functools
-import inspect
from collections.abc import Callable, Mapping, Sequence
-from functools import partial
-from pathlib import Path
-from typing import TYPE_CHECKING, Annotated, Any, Literal, NewType, ParamSpec, TypeAlias, TypeVar, overload
+from typing import TYPE_CHECKING, Any, Literal, NewType, ParamSpec, TypeAlias, TypeVar, overload
import yarl
-from pydantic import (
- AfterValidator,
- AnyUrl,
- BaseModel,
- BeforeValidator,
- ByteSize,
- ConfigDict,
- NonNegativeInt,
- PlainSerializer,
- PlainValidator,
- Secret,
- SerializationInfo,
- StringConstraints,
- model_serializer,
- model_validator,
-)
-
-from cyberdrop_dl.utils.converters import change_path_suffix, convert_byte_size_to_str
-from cyberdrop_dl.utils.validators import (
- parse_apprise_url,
- parse_falsy_as_none,
- parse_list,
- pydantyc_yarl_url,
-)
P = ParamSpec("P")
R = TypeVar("R")
T = TypeVar("T")
-def copy_signature(target: Callable[P, R]) -> Callable[[Callable[..., T]], Callable[P, T]]:
- """Decorator to make a function mimic the signature of another function,
- but preserve the return type of the decorated function."""
+if TYPE_CHECKING:
+ import functools
+ import inspect
- def decorator(func: Callable[..., T]) -> Callable[P, T]:
- @functools.wraps(func)
- def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
- return func(*args, **kwargs)
+ from propcache.api import under_cached_property as cached_property
+ from yarl._query import Query, QueryVariable
- wrapper.__signature__ = inspect.signature(target).replace( # type: ignore
- return_annotation=inspect.signature(func).return_annotation
- )
- return wrapper
+ def copy_signature(target: Callable[P, R]) -> Callable[[Callable[..., T]], Callable[P, T]]:
+ def decorator(func: Callable[..., T]) -> Callable[P, T]:
+ @functools.wraps(func)
+ def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+ return func(*args, **kwargs)
- return decorator
+ wrapper.__signature__ = inspect.signature(target).replace( # type: ignore
+ return_annotation=inspect.signature(func).return_annotation
+ )
+ return wrapper
-
-if TYPE_CHECKING:
- from propcache.api import under_cached_property as cached_property
- from yarl._query import Query, QueryVariable
+ return decorator
class AbsoluteHttpURL(yarl.URL):
@copy_signature(yarl.URL.__new__)
@@ -149,57 +120,6 @@ else:
AnyURL = TypeVar("AnyURL", yarl.URL, AbsoluteHttpURL)
-# ~~~~~ Strings ~~~~~~~
-StrSerializer = PlainSerializer(str, return_type=str, when_used="json-unless-none")
-NonEmptyStr = Annotated[str, StringConstraints(min_length=1, strip_whitespace=True)]
-NonEmptyStrOrNone = Annotated[NonEmptyStr | None, BeforeValidator(parse_falsy_as_none)]
-ListNonEmptyStr = Annotated[list[NonEmptyStr], BeforeValidator(parse_list)]
-
-# ~~~~~ Paths ~~~~~~~
-PathOrNone = Annotated[Path | None, BeforeValidator(parse_falsy_as_none)]
-LogPath = Annotated[Path, AfterValidator(partial(change_path_suffix, suffix=".csv"))]
-MainLogPath = Annotated[LogPath, AfterValidator(partial(change_path_suffix, suffix=".log"))]
-
-# URL with pydantic.HttpUrl validation (must be absolute, must be http/https, detailed validation error).
-# In type hints it's a yarl.URL. After validation the result is parsed with `parse_url` so this is also a yarl.URL at runtime
-# Only use for config validation. To parse URLs internally while scraping, call `parse_url` directly
-HttpURL = Annotated[yarl.URL, PlainValidator(pydantyc_yarl_url), StrSerializer]
-
-# ~~~~~ Others ~~~~~~~
-ByteSizeSerilized = Annotated[ByteSize, PlainSerializer(convert_byte_size_to_str, return_type=str)]
-ListNonNegativeInt = Annotated[list[NonNegativeInt], BeforeValidator(parse_list)]
-
-
-class AliasModel(BaseModel):
- model_config = ConfigDict(populate_by_name=True)
-
-
-class FrozenModel(BaseModel):
- model_config = ConfigDict(frozen=True)
-
-
-class AppriseURLModel(FrozenModel):
- url: Secret[AnyUrl]
- tags: set[str] = set()
-
- @model_serializer()
- def serialize(self, info: SerializationInfo) -> str:
- dump_secret = info.mode != "json"
- url = self.url.get_secret_value() if dump_secret else self.url
- tags = self.tags - set("no_logs")
- tags = sorted(tags)
- return f"{','.join(tags)}{'=' if tags else ''}{url}"
-
- @model_validator(mode="before")
- @staticmethod
- def parse_input(value: yarl.URL | dict | str) -> dict:
- return parse_apprise_url(value)
-
-
-class HttpAppriseURL(AppriseURLModel):
- url: Secret[HttpURL]
-
-
Array: TypeAlias = list[T] | tuple[T, ...]
CMD: TypeAlias = Array[str]
U32Int: TypeAlias = int
@@ -207,7 +127,6 @@ U32IntArray: TypeAlias = Array[U32Int]
U32IntSequence: TypeAlias = Sequence[U32Int]
AnyDict: TypeAlias = dict[str, Any]
-AbsolutePath = NewType("AbsolutePath", Path)
HashValue = NewType("HashValue", str)
TimeStamp = NewType("TimeStamp", int)
diff --git a/cyberdrop_dl/utils/apprise.py b/cyberdrop_dl/utils/apprise.py
index 812ec0c68..4e2c86f49 100644
--- a/cyberdrop_dl/utils/apprise.py
+++ b/cyberdrop_dl/utils/apprise.py
@@ -16,7 +16,7 @@ from pydantic import ValidationError
from rich.text import Text
from cyberdrop_dl import constants
-from cyberdrop_dl.types import AppriseURLModel
+from cyberdrop_dl.models import AppriseURLModel
from cyberdrop_dl.utils.logger import log, log_debug, log_spacer
from cyberdrop_dl.utils.yaml import handle_validation_error
diff --git a/cyberdrop_dl/utils/args.py b/cyberdrop_dl/utils/args.py
index 141a93388..3bcddc63a 100644
--- a/cyberdrop_dl/utils/args.py
+++ b/cyberdrop_dl/utils/args.py
@@ -8,13 +8,14 @@ from datetime import date
from enum import StrEnum, auto
from pathlib import Path
from shutil import get_terminal_size
-from typing import TYPE_CHECKING, NoReturn, Self
+from typing import TYPE_CHECKING, Any, NoReturn, Self
from pydantic import BaseModel, Field, ValidationError, computed_field, field_validator, model_validator
from cyberdrop_dl import __version__, env
-from cyberdrop_dl.config_definitions import ConfigSettings, GlobalSettings
-from cyberdrop_dl.types import AliasModel, AnyDict, HttpURL
+from cyberdrop_dl.config import ConfigSettings, GlobalSettings
+from cyberdrop_dl.models import AliasModel
+from cyberdrop_dl.models.types import HttpURL
from cyberdrop_dl.utils.yaml import handle_validation_error
if TYPE_CHECKING:
@@ -262,10 +263,10 @@ def make_parser() -> tuple[ArgumentParser, dict[str, list[ArgGroup]]]:
return parser, group_lists
-def get_parsed_args_dict(args: Sequence[str] | None = None) -> dict[str, AnyDict]:
+def get_parsed_args_dict(args: Sequence[str] | None = None) -> dict[str, dict[str, Any]]:
parser, group_lists = make_parser()
namespace = parser.parse_intermixed_args(args)
- parsed_args: dict[str, AnyDict] = {}
+ parsed_args: dict[str, dict[str, Any]] = {}
for name, groups in group_lists.items():
parsed_args[name] = {}
for group in groups:
diff --git a/cyberdrop_dl/utils/converters.py b/cyberdrop_dl/utils/converters.py
deleted file mode 100644
index 54ad8f67a..000000000
--- a/cyberdrop_dl/utils/converters.py
+++ /dev/null
@@ -1,76 +0,0 @@
-"""
-Functions to use with `AfterValidator`, `field_validator(mode="after")` or `model_validator(mode="after")`
-"""
-
-from __future__ import annotations
-
-import re
-from datetime import timedelta
-from typing import TYPE_CHECKING
-
-from pydantic import AnyUrl, ByteSize, TypeAdapter
-
-from cyberdrop_dl.exceptions import InvalidURLError
-from cyberdrop_dl.utils.utilities import parse_url
-
-if TYPE_CHECKING:
- from pathlib import Path
-
- import yarl
-
-DATE_PATTERN_REGEX = r"(\d+)\s*(second|seconds|minute|minutes|hour|hours|day|days|week|weeks|month|months|year|years)"
-DATE_PATTERN = re.compile(DATE_PATTERN_REGEX, re.IGNORECASE)
-
-byte_size_adapter = TypeAdapter(ByteSize)
-
-
-def convert_byte_size_to_str(value: ByteSize) -> str:
- if not isinstance(value, ByteSize):
- value = ByteSize(value)
- return value.human_readable(decimal=True)
-
-
-def convert_to_yarl(value: AnyUrl | str, *args, **kwargs) -> yarl.URL:
- try:
- return parse_url(str(value), *args, **kwargs)
- except (InvalidURLError, TypeError) as e:
- raise ValueError(str(e)) from e
-
-
-def change_path_suffix(value: Path, suffix: str) -> Path:
- return value.with_suffix(suffix)
-
-
-def convert_to_byte_size(value: ByteSize | str | int) -> ByteSize:
- return byte_size_adapter.validate_python(value)
-
-
-def convert_str_to_timedelta(input_date: str) -> timedelta:
- time_str = input_date.casefold()
- matches: list[str] = re.findall(DATE_PATTERN, time_str)
- seen_units = set()
- time_dict = {"days": 0}
-
- for value, unit in matches:
- value = int(value)
- unit = unit.lower()
- normalized_unit = unit.rstrip("s")
- plural_unit = normalized_unit + "s"
- if normalized_unit in seen_units:
- msg = f"Duplicate time unit detected: '{unit}' conflicts with another entry"
- raise ValueError(msg)
- seen_units.add(normalized_unit)
-
- if "day" in unit:
- time_dict["days"] += value
- elif "month" in unit:
- time_dict["days"] += value * 30
- elif "year" in unit:
- time_dict["days"] += value * 365
- else:
- time_dict[plural_unit] = value
-
- if not matches:
- msg = f"Unable to convert '{input_date}' to timedelta object"
- raise ValueError(msg)
- return timedelta(**time_dict)
diff --git a/cyberdrop_dl/utils/validators.py b/cyberdrop_dl/utils/validators.py
deleted file mode 100644
index 681997f80..000000000
--- a/cyberdrop_dl/utils/validators.py
+++ /dev/null
@@ -1,107 +0,0 @@
-"""
-Functions to use with `BeforeValidator`, `field_validator(mode="before")` or `model_validator(mode="before")`
-"""
-
-from __future__ import annotations
-
-from datetime import timedelta
-from functools import singledispatch
-from typing import TYPE_CHECKING, Any
-
-from pydantic import HttpUrl
-
-from cyberdrop_dl.utils.converters import convert_str_to_timedelta, convert_to_yarl
-
-if TYPE_CHECKING:
- from collections.abc import Callable
-
- from yarl import URL
-
-
-def pydantyc_yarl_url(value: str) -> URL:
- url = HttpUrl(value)
- return convert_to_yarl(url)
-
-
-def parse_falsy_as(value: Any, falsy_value: Any, func: Callable | None = None, *args, **kwargs) -> Any:
- """If `value` is falsy, returns `falsy_value`
-
- If `value` is NOT falsy AND `func` is provided, returns `func(value, *args, **kwargs)`
-
- Returns `value` otherwise
- """
- if isinstance(value, str) and value.casefold() in ("none", "null"):
- value = None
- if not value:
- return falsy_value
- if not func:
- return value
- return func(value, *args, **kwargs)
-
-
-def parse_duration_as_timedelta(input_date: timedelta | str | int) -> timedelta:
- """Parses `datetime.timedelta`, `str` or `int` into a timedelta format.
-
- For `str`, the expected format is `<value> <unit>`, ex: `5 days`, `10 minutes`, `1 year`
-
- Valid units:
- `year(s)`, `week(s)`, `day(s)`, `hour(s)`, `minute(s)`, `second(s)`, `millisecond(s)`, `microsecond(s)`
-
- For `int`, `input_date` is assumed as `days`
- """
- return parse_falsy_as(input_date, timedelta(0), _parse_as_timedelta)
-
-
-@singledispatch
-def _parse_as_timedelta(input_date: timedelta) -> timedelta | str:
- return input_date
-
-
-@_parse_as_timedelta.register
-def _(input_date: int) -> timedelta:
- return timedelta(days=input_date)
-
-
-@_parse_as_timedelta.register
-def _(input_date: str, raise_error: bool = False) -> timedelta | str:
- try:
- return convert_str_to_timedelta(input_date)
- except ValueError:
- if raise_error:
- raise
- return input_date
-
-
-def parse_list(value: list) -> Any:
- return parse_falsy_as(value, [])
-
-
-def parse_falsy_as_none(value: Any) -> Any:
- return parse_falsy_as(value, None)
-
-
-@singledispatch
-def parse_apprise_url(value: URL) -> dict:
- return {"url": str(value), "tags": set()}
-
-
-@parse_apprise_url.register
-def _(value: dict) -> dict:
- tags = value.get("tags") or set()
- url = str(value.get("url", ""))
- if not tags:
- return parse_apprise_url(url)
-
- return {"url": url, "tags": tags}
-
-
-@parse_apprise_url.register
-def _(value: str) -> dict:
- tags = set()
- url = value
- parts = url.split("://", 1)[0].split("=", 1)
- if len(parts) == 2:
- tags = set(parts[0].split(","))
- url: str = url.split("=", 1)[-1]
-
- return {"url": url, "tags": tags}