Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: validate formats strings at startup (#1097)
NTFSvolume9664dbc59870
diff --git a/cyberdrop_dl/config/config_model.py b/cyberdrop_dl/config/config_model.pyindex 32d67695b..c50b1f696 100755--- a/cyberdrop_dl/config/config_model.py+++ b/cyberdrop_dl/config/config_model.py@@ -22,11 +22,13 @@ from cyberdrop_dl.models.types import (PathOrNone,)from cyberdrop_dl.models.validators import falsy_as, to_timedelta+from cyberdrop_dl.utils.strings import validate_format_stringfrom cyberdrop_dl.utils.utilities import purge_dir_treefrom ._common import ConfigModel, Field, PathAliasModelALL_SUPPORTED_SITES = ["<<ALL_SUPPORTED_SITES>>"]+SORTING_COMMON_KEYS = {"sort_dir", "base_dir", "parent_dir", "filename", "ext"}class DownloadOptions(BaseModel):@@ -48,13 +50,9 @@ class DownloadOptions(BaseModel):@field_validator("separate_posts_format", mode="after")@classmethoddef valid_format(cls, value: str) -> str:- valid_keys = ("default", "title", "id", "number", "date")- try:- value.format(**dict.fromkeys(valid_keys, "TEST"))- return value- except KeyError as e:- msg = f"'{e.args[0]}' is not a valid key for this option. Valid keys: {valid_keys}"- raise ValueError(msg) from None+ valid_keys = {"default", "title", "id", "number", "date"}+ validate_format_string(value, valid_keys)+ return valueclass Files(PathAliasModel):@@ -198,6 +196,40 @@ class Sorting(BaseModel):sorted_other: NonEmptyStrOrNone = "{sort_dir}/{base_dir}/Other/{filename}{ext}"sorted_video: NonEmptyStrOrNone = "{sort_dir}/{base_dir}/Videos/{filename}{ext}"+ @field_validator("sorted_audio", mode="after")+ @classmethod+ def valid_sorted_audio(cls, value: str | None) -> str | None:+ if value is not None:+ valid_keys = SORTING_COMMON_KEYS.union("bitrate", "duration", "length", "sample_rate")+ validate_format_string(value, valid_keys)+ return value++ @field_validator("sorted_image", mode="after")+ @classmethod+ def valid_sorted_image(cls, value: str | None) -> str | None:+ if value is not None:+ valid_keys = SORTING_COMMON_KEYS.union("height", "width", "resolution")+ validate_format_string(value, valid_keys)+ return value++ @field_validator("sorted_other", mode="after")+ @classmethod+ def valid_sorted_other(cls, value: str | None) -> str | None:+ if value is not None:+ valid_keys = SORTING_COMMON_KEYS.union("bitrate", "duration", "length", "sample_rate")+ validate_format_string(value, valid_keys)+ return value++ @field_validator("sorted_video", mode="after")+ @classmethod+ def valid_sorted_video(cls, value: str | None) -> str | None:+ if value is not None:+ valid_keys = SORTING_COMMON_KEYS.union(+ "codec", "duration", "fps", "length", "height", "width", "resolution"+ )+ validate_format_string(value, valid_keys)+ return value+class BrowserCookies(BaseModel):auto_import: bool = Falsediff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex c5113a3e6..b71ba7727 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -24,6 +24,7 @@ from cyberdrop_dl.utils.database.tables.history_table import get_db_pathfrom cyberdrop_dl.utils.dates import TimeStamp, parse_human_date, to_timestampfrom cyberdrop_dl.utils.logger import log, log_debugfrom cyberdrop_dl.utils.m3u8 import M3U8, M3U8Media, RenditionGroup+from cyberdrop_dl.utils.strings import safe_formatfrom cyberdrop_dl.utils.utilities import (error_handling_wrapper,get_download_path,@@ -385,18 +386,9 @@ class Crawler(ABC):title_format = self.DEFAULT_POST_TITLE_FORMATif isinstance(date, int):date = datetime.datetime.fromtimestamp(date)- if isinstance(date, datetime.datetime | datetime.date):- date_str = date.isoformat()- else:- date_str: str | None = date-- def default_if_none(value: str | None, default: str) -> str:- return default if value is None else value- id = default_if_none(id, "Unknown")- title = default_if_none(title, "Untitled")- date_str = default_if_none(date_str, "NO_DATE")- return title_format.format(id=id, number=id, date=date_str, title=title)+ post_title, _ = safe_format(title_format, id=id, number=id, date=date, title=title)+ return post_titledef parse_url(self, link_str: str, relative_to: URL | None = None, *, trim: bool = True) -> AbsoluteHttpURL:"""Wrapper arround `utils.parse_url` to use `self.PRIMARY_URL` as base"""diff --git a/cyberdrop_dl/utils/strings.py b/cyberdrop_dl/utils/strings.pynew file mode 100644index 000000000..1fee7bab9--- /dev/null+++ b/cyberdrop_dl/utils/strings.py@@ -0,0 +1,90 @@+import string+from typing import Any++_FORMATER = string.Formatter()+++class UnknownPlaceholder:+ """An object that always represents itself the same regardless of how it's formatted in an f-string++ Use this to prevent errors if the user supplied custom formatting for a value that the crawler was not able to scrape"""++ def __init__(self, value: str) -> None:+ self.value = value++ def __str__(self) -> str:+ return self.value++ def __repr__(self) -> str:+ return self.value++ def __format__(self, format_spec: str) -> str:+ return self.value++ @staticmethod+ def make(name: str) -> "UnknownPlaceholder":+ return UnknownPlaceholder(f"UNKNOWN_{name.upper()}")+++_UNTITLED = UnknownPlaceholder("Untitled")+++def safe_format(format_string: str, **fields: Any) -> tuple[str, set[str]]:+ """+ Formats a string replacing any unknown keys and `None` values in the format string with an instance of `UnknownPlaceholder`++ Args:+ format_string (str): The string to be formatted.+ **kwargs: Keyword arguments for formatting.++ Returns:+ str: The formatted string with unknown keys replaced by "UNKNOWN_{KEY_NAME}".+ set[str]: colletion of unknown field_names in the format string+ """++ new_kwargs = dict(fields)+ unknown_field_names = get_unknown_field_names(format_string, set(new_kwargs.keys()))++ for field_name, value in new_kwargs.items():+ if value is None:+ if field_name.lower() == "title":+ new_kwargs[field_name] = _UNTITLED+ else:+ new_kwargs[field_name] = UnknownPlaceholder.make(field_name)++ for field_name in unknown_field_names:+ new_kwargs[field_name] = UnknownPlaceholder.make(field_name)++ return _FORMATER.vformat(format_string, (), new_kwargs), unknown_field_names+++# To use with pydantic+def validate_format_string(format_string: str, valid_keys: set[str]) -> None:+ msg = "Invalid format string. "+ for _, field_name, _, _ in _FORMATER.parse(format_string):+ if field_name is not None:+ if field_name.isdigit() or field_name == "":+ msg += "Format strings with positional arguments are not valid config options"+ raise ValueError(msg)+ if not field_name.isidentifier():+ msg += "Operations within a format string are not supported"+ raise ValueError(msg)++ if unknown_field_names := get_unknown_field_names(format_string, valid_keys):+ msg += f"{sorted(unknown_field_names)} are not valid keys for this option. Valid keys: {sorted(valid_keys)}"+ raise ValueError(msg)+++def get_field_names(format_string: str) -> set[str]:+ field_names = set()++ # literal_text, field_name, format_spec, conversion+ for _, field_name, _, _ in _FORMATER.parse(format_string):+ if field_name and not field_name.isdigit(): # Ignore positional args and empty fields+ field_names.add(field_name)+ return field_names+++def get_unknown_field_names(format_string: str, valid_keys: set[str]) -> set[str]:+ field_names = get_field_names(format_string)+ return field_names - valid_keys