Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: validate formats strings at startup (#1097)

NTFSvolume9664dbc59870
diff --git a/cyberdrop_dl/config/config_model.py b/cyberdrop_dl/config/config_model.py
index 32d67695b..c50b1f696 100755
--- a/cyberdrop_dl/config/config_model.py
+++ b/cyberdrop_dl/config/config_model.py
@@ -22,11 +22,13 @@ from cyberdrop_dl.models.types import (
PathOrNone,
)
from cyberdrop_dl.models.validators import falsy_as, to_timedelta
+from cyberdrop_dl.utils.strings import validate_format_string
from cyberdrop_dl.utils.utilities import purge_dir_tree
from ._common import ConfigModel, Field, PathAliasModel
ALL_SUPPORTED_SITES = ["<<ALL_SUPPORTED_SITES>>"]
+SORTING_COMMON_KEYS = {"sort_dir", "base_dir", "parent_dir", "filename", "ext"}
class DownloadOptions(BaseModel):
@@ -48,13 +50,9 @@ class DownloadOptions(BaseModel):
@field_validator("separate_posts_format", mode="after")
@classmethod
def valid_format(cls, value: str) -> str:
- valid_keys = ("default", "title", "id", "number", "date")
- try:
- value.format(**dict.fromkeys(valid_keys, "TEST"))
- return value
- except KeyError as e:
- msg = f"'{e.args[0]}' is not a valid key for this option. Valid keys: {valid_keys}"
- raise ValueError(msg) from None
+ valid_keys = {"default", "title", "id", "number", "date"}
+ validate_format_string(value, valid_keys)
+ return value
class Files(PathAliasModel):
@@ -198,6 +196,40 @@ class Sorting(BaseModel):
sorted_other: NonEmptyStrOrNone = "{sort_dir}/{base_dir}/Other/{filename}{ext}"
sorted_video: NonEmptyStrOrNone = "{sort_dir}/{base_dir}/Videos/{filename}{ext}"
+ @field_validator("sorted_audio", mode="after")
+ @classmethod
+ def valid_sorted_audio(cls, value: str | None) -> str | None:
+ if value is not None:
+ valid_keys = SORTING_COMMON_KEYS.union("bitrate", "duration", "length", "sample_rate")
+ validate_format_string(value, valid_keys)
+ return value
+
+ @field_validator("sorted_image", mode="after")
+ @classmethod
+ def valid_sorted_image(cls, value: str | None) -> str | None:
+ if value is not None:
+ valid_keys = SORTING_COMMON_KEYS.union("height", "width", "resolution")
+ validate_format_string(value, valid_keys)
+ return value
+
+ @field_validator("sorted_other", mode="after")
+ @classmethod
+ def valid_sorted_other(cls, value: str | None) -> str | None:
+ if value is not None:
+ valid_keys = SORTING_COMMON_KEYS.union("bitrate", "duration", "length", "sample_rate")
+ validate_format_string(value, valid_keys)
+ return value
+
+ @field_validator("sorted_video", mode="after")
+ @classmethod
+ def valid_sorted_video(cls, value: str | None) -> str | None:
+ if value is not None:
+ valid_keys = SORTING_COMMON_KEYS.union(
+ "codec", "duration", "fps", "length", "height", "width", "resolution"
+ )
+ validate_format_string(value, valid_keys)
+ return value
+
class BrowserCookies(BaseModel):
auto_import: bool = False
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index c5113a3e6..b71ba7727 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -24,6 +24,7 @@ from cyberdrop_dl.utils.database.tables.history_table import get_db_path
from cyberdrop_dl.utils.dates import TimeStamp, parse_human_date, to_timestamp
from cyberdrop_dl.utils.logger import log, log_debug
from cyberdrop_dl.utils.m3u8 import M3U8, M3U8Media, RenditionGroup
+from cyberdrop_dl.utils.strings import safe_format
from cyberdrop_dl.utils.utilities import (
error_handling_wrapper,
get_download_path,
@@ -385,18 +386,9 @@ class Crawler(ABC):
title_format = self.DEFAULT_POST_TITLE_FORMAT
if isinstance(date, int):
date = datetime.datetime.fromtimestamp(date)
- if isinstance(date, datetime.datetime | datetime.date):
- date_str = date.isoformat()
- else:
- date_str: str | None = date
-
- def default_if_none(value: str | None, default: str) -> str:
- return default if value is None else value
- id = default_if_none(id, "Unknown")
- title = default_if_none(title, "Untitled")
- date_str = default_if_none(date_str, "NO_DATE")
- return title_format.format(id=id, number=id, date=date_str, title=title)
+ post_title, _ = safe_format(title_format, id=id, number=id, date=date, title=title)
+ return post_title
def parse_url(self, link_str: str, relative_to: URL | None = None, *, trim: bool = True) -> AbsoluteHttpURL:
"""Wrapper arround `utils.parse_url` to use `self.PRIMARY_URL` as base"""
diff --git a/cyberdrop_dl/utils/strings.py b/cyberdrop_dl/utils/strings.py
new file mode 100644
index 000000000..1fee7bab9
--- /dev/null
+++ b/cyberdrop_dl/utils/strings.py
@@ -0,0 +1,90 @@
+import string
+from typing import Any
+
+_FORMATER = string.Formatter()
+
+
+class UnknownPlaceholder:
+ """An object that always represents itself the same regardless of how it's formatted in an f-string
+
+ Use this to prevent errors if the user supplied custom formatting for a value that the crawler was not able to scrape"""
+
+ def __init__(self, value: str) -> None:
+ self.value = value
+
+ def __str__(self) -> str:
+ return self.value
+
+ def __repr__(self) -> str:
+ return self.value
+
+ def __format__(self, format_spec: str) -> str:
+ return self.value
+
+ @staticmethod
+ def make(name: str) -> "UnknownPlaceholder":
+ return UnknownPlaceholder(f"UNKNOWN_{name.upper()}")
+
+
+_UNTITLED = UnknownPlaceholder("Untitled")
+
+
+def safe_format(format_string: str, **fields: Any) -> tuple[str, set[str]]:
+ """
+ Formats a string replacing any unknown keys and `None` values in the format string with an instance of `UnknownPlaceholder`
+
+ Args:
+ format_string (str): The string to be formatted.
+ **kwargs: Keyword arguments for formatting.
+
+ Returns:
+ str: The formatted string with unknown keys replaced by "UNKNOWN_{KEY_NAME}".
+ set[str]: colletion of unknown field_names in the format string
+ """
+
+ new_kwargs = dict(fields)
+ unknown_field_names = get_unknown_field_names(format_string, set(new_kwargs.keys()))
+
+ for field_name, value in new_kwargs.items():
+ if value is None:
+ if field_name.lower() == "title":
+ new_kwargs[field_name] = _UNTITLED
+ else:
+ new_kwargs[field_name] = UnknownPlaceholder.make(field_name)
+
+ for field_name in unknown_field_names:
+ new_kwargs[field_name] = UnknownPlaceholder.make(field_name)
+
+ return _FORMATER.vformat(format_string, (), new_kwargs), unknown_field_names
+
+
+# To use with pydantic
+def validate_format_string(format_string: str, valid_keys: set[str]) -> None:
+ msg = "Invalid format string. "
+ for _, field_name, _, _ in _FORMATER.parse(format_string):
+ if field_name is not None:
+ if field_name.isdigit() or field_name == "":
+ msg += "Format strings with positional arguments are not valid config options"
+ raise ValueError(msg)
+ if not field_name.isidentifier():
+ msg += "Operations within a format string are not supported"
+ raise ValueError(msg)
+
+ if unknown_field_names := get_unknown_field_names(format_string, valid_keys):
+ msg += f"{sorted(unknown_field_names)} are not valid keys for this option. Valid keys: {sorted(valid_keys)}"
+ raise ValueError(msg)
+
+
+def get_field_names(format_string: str) -> set[str]:
+ field_names = set()
+
+ # literal_text, field_name, format_spec, conversion
+ for _, field_name, _, _ in _FORMATER.parse(format_string):
+ if field_name and not field_name.isdigit(): # Ignore positional args and empty fields
+ field_names.add(field_name)
+ return field_names
+
+
+def get_unknown_field_names(format_string: str, valid_keys: set[str]) -> set[str]:
+ field_names = get_field_names(format_string)
+ return field_names - valid_keys