Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add `--restrict-path` option (#2101)
NTFSvolumecbeb9fa9cae2
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6654f27db..be5d63a6c 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Added+- New `--restrict-path` option- New `--input-folder` option that scans all `.txt` within a folder for URLs (non recursive). URLs within the same file will create a group. The name of the group is the filename;Each group will be downloaded to their own subfolder within the downloads folder. See: <https://script-ware.gitbook.io/cyberdrop-dl/reference/config/sorting#group-urls>- New `--input` option that accepts either a file or a folderdiff --git a/cyberdrop_dl/config/__init__.py b/cyberdrop_dl/config/__init__.pyindex 161a19698..3aa1f08ee 100644--- a/cyberdrop_dl/config/__init__.py+++ b/cyberdrop_dl/config/__init__.py@@ -14,7 +14,7 @@ from cyberdrop_dl.constants import DEFAULT_PARAMETERfrom cyberdrop_dl.exceptions import CDLConfigRuntimeErrorsGroup, InvalidYamlErrorfrom cyberdrop_dl.models import AdditiveArg, ConfigModel, merge_dicts, merge_modelsfrom cyberdrop_dl.models.types import ByteSizeSerilized # noqa: TC001-from cyberdrop_dl.models.validators import to_bytesize+from cyberdrop_dl.models.validators import remove_duplicates, to_bytesizefrom cyberdrop_dl.utils import cleanupfrom .auth import Authentication, Notifications@@ -109,6 +109,11 @@ class Config(ConfigModel, title="cyberdrop-dl config"):network: Network = Field(default_factory=Network)notifications: Notifications = Field(default_factory=Notifications)+ restrict_path: Annotated[+ tuple[Literal["unix", "windows", "no_emoji", "ascii"], ...],+ AfterValidator(remove_duplicates),+ Parameter(alias="restrict-filenames"),+ ] = ()sort: Sort = Field(default_factory=Sort)subfolders: SubFolders = Field(default_factory=SubFolders)ui: UIOptions = Field(default_factory=UIOptions)diff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yamlindex 2db12c4ac..46c997325 100644--- a/cyberdrop_dl/config/default.yaml+++ b/cyberdrop_dl/config/default.yaml@@ -170,6 +170,7 @@ network:notifications:apprise: []webhook: null+restrict_path: []sort:enabled: falseformats:diff --git a/cyberdrop_dl/config/schema.json b/cyberdrop_dl/config/schema.jsonindex 6f7fd8e5c..aa5f975d5 100644--- a/cyberdrop_dl/config/schema.json+++ b/cyberdrop_dl/config/schema.json@@ -1392,6 +1392,20 @@"notifications": {"$ref": "#/$defs/Notifications"},+ "restrict_path": {+ "default": [],+ "items": {+ "enum": [+ "unix",+ "windows",+ "no_emoji",+ "ascii"+ ],+ "type": "string"+ },+ "title": "Restrict Path",+ "type": "array"+ },"sort": {"$ref": "#/$defs/Sort"},diff --git a/cyberdrop_dl/filepath.py b/cyberdrop_dl/filepath.pyindex 2c8134693..d2494969c 100644--- a/cyberdrop_dl/filepath.py+++ b/cyberdrop_dl/filepath.py@@ -5,38 +5,93 @@ import platformimport reimport unicodedatafrom contextvars import ContextVar+from enum import StrEnumfrom pathlib import Path+from typing import TYPE_CHECKING, Literal, Selffrom cyberdrop_dl.constants import FileExtfrom cyberdrop_dl.exceptions import FileNameError, InvalidExtensionError, NoExtensionError, PathTraversalError+from cyberdrop_dl.signature import simple_repr-_ALLOWED_FILEPATH_PUNCTUATION = " .-_!#$%'()+,;=@[]^{}~"-_SANITIZE_FILENAME_PATTERN = r'[<>:"/\\|?*\']'+if TYPE_CHECKING:+ from collections.abc import Callable++type PathOptions = Literal["unix", "windows", "no_emoji", "ascii"]++_ALLOWED_UNICODE_SYMBOLS_AND_PUNCTUATION = " .-_!#$%'()+,;=@[]^{}~"_RAR_MULTIPART_PATTERN = r"^part\d+"+_MAX_FILE_LEN: ContextVar[int] = ContextVar("_MAX_FILE_LEN", default=95)+_MAX_FOLDER_LEN: ContextVar[int] = ContextVar("_MAX_FOLDER_LEN", default=60)+_PATH_SANITIZER: ContextVar[PathSanitizer] = ContextVar("_PATH_SANITIZER")+++class RestrictPath(StrEnum):+ ASCII = "^0-9A-Za-z_."+ WINDOWS = r'<>:"/\\|?*\''+ UNIX = "/"+++class UnicodeCategory(StrEnum):+ CONTROL = "C"+ LETTER = "L"+ MARK = "M"+ NUMBER = "N"+ PUNCTUATION = "P"+ SIMBOL = "S"+ SEPARATOR = "Z"+++class PathSanitizer:+ def __init__(self, banned_chars: str | None = None, *post_process: Callable[[str], str]) -> None:+ self.banned_chars: str | None = banned_chars+ self.post_process: tuple[Callable[[str], str], ...] = post_process++ __repr__ = simple_repr("banned_chars", "post_process")++ def __call__(self, name: str, repl: str = "") -> str:+ if self.banned_chars:+ name = re.sub(f"[{self.banned_chars}]", repl, name).strip()+ for fn in self.post_process:+ name = fn(name)-MAX_FILE_LEN: ContextVar[int] = ContextVar("_MAX_FILE_LEN", default=95)-MAX_FOLDER_LEN: ContextVar[int] = ContextVar("_MAX_FOLDER_LEN", default=60)+ return name++ def __or__(self, other: Self) -> Self:+ return type(self)(self.banned_chars, *self.post_process, other)++ @classmethod+ def v9_default(cls) -> Self:+ if platform.system() in {"Windows", "Darwin"}:+ return cls(RestrictPath.WINDOWS, remove_emojis_and_symbols)+ return cls(RestrictPath.WINDOWS)+++def _is_allowed_unicode(char: str) -> bool:+ return char in _ALLOWED_UNICODE_SYMBOLS_AND_PUNCTUATION or unicodedata.category(char)[0] in {+ UnicodeCategory.LETTER,+ UnicodeCategory.NUMBER,+ UnicodeCategory.MARK,+ }def remove_emojis_and_symbols(filename: str) -> str:- """Allow all Unicode letters/numbers/marks, plus safe filename punctuation, but not symbols or emoji."""- return "".join(- char- for char in filename- if (char in _ALLOWED_FILEPATH_PUNCTUATION or unicodedata.category(char)[0] in {"L", "N", "M"})- ).strip()+ """Allow all Unicode letters/numbers/marks, plus safe filename punctuation, but not symbols (emojis)."""+ return "".join(filter(_is_allowed_unicode, filename)).strip()def sanitize_filename(name: str, sub: str = "") -> str:- clean_name = re.sub(_SANITIZE_FILENAME_PATTERN, sub, name).strip()- if platform.system() in {"Windows", "Darwin"}:- clean_name = remove_emojis_and_symbols(clean_name)- path = Path(clean_name)+ try:+ clean = _PATH_SANITIZER.get()+ except LookupError:+ clean = PathSanitizer.v9_default()+ _PATH_SANITIZER.set(clean)++ path = Path(clean(name, sub))return path.stem.strip() + path.suffixdef sanitize_folder(title: str, max_len: int | None = None) -> str:- max_len = max_len or MAX_FOLDER_LEN.get()+ max_len = max_len or _MAX_FOLDER_LEN.get()title = title.replace("\n", "").replace("\t", "").strip()title = sanitize_filename(re.sub(r" +", " ", title), "-")title = re.sub(r"\.{2,}", ".", title).rstrip(".").strip()@@ -94,7 +149,7 @@ def compose_filename(name: str, suffix: str, *extras: str, max_len: int | None =assert suffix.startswith(".")name = sanitize_filename(remove_os_sep(name)).removesuffix(suffix)- max_len = (max_len or MAX_FILE_LEN.get()) - len(suffix)+ max_len = (max_len or _MAX_FILE_LEN.get()) - len(suffix)if extras:extra_info = sanitize_filename("".join(f"[{info}]" for info in extras))if (new_max_len := max_len - len(extra_info) - 1) > 0:@@ -142,3 +197,26 @@ def check_dangerous_filename(filename: str) -> None:path = Path(filename)if "\\" in filename or "/" in filename or path.name != filename or path.suffix.lower() in FileExt.DANGEROUS:raise FileNameError("Dangerous File Extension", message=filename)+++def setup(+ max_file_len: int,+ max_folder_len: int,+ restrict_path: tuple[PathOptions, ...],+) -> None:+ _MAX_FILE_LEN.set(max_file_len)+ _MAX_FOLDER_LEN.set(max_folder_len)++ if not restrict_path:+ return++ sanitizer = PathSanitizer()+ for name in restrict_path:+ other = (+ PathSanitizer(None, remove_emojis_and_symbols)+ if name == "no_emoji"+ else PathSanitizer(RestrictPath[name.upper()])+ )+ sanitizer = sanitizer | other++ _PATH_SANITIZER.set(sanitizer)diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.pyindex 4fbb5d986..d23d492a8 100644--- a/cyberdrop_dl/scrape_mapper.py+++ b/cyberdrop_dl/scrape_mapper.py@@ -169,10 +169,12 @@ class ScrapeMapper:assert not self.task_mngr.scrape.done.is_set()self.logs.delete_old_logs()config = self.manager.config- _ = filepath.MAX_FILE_LEN.set(config.max_file_name_length)- _ = filepath.MAX_FOLDER_LEN.set(config.max_folder_name_length)+_ = CONCURRENT_SEGMENTS.set(config.downloads.concurrent_segments)_ = ALLOW_NO_EXT.set(config.filters.allow_files_with_no_extension)++ filepath.setup(config.max_file_name_length, config.max_folder_name_length, config.restrict_path)+if config.ui.portrait:env.FORCE_PORTRAIT_MODE = Truediff --git a/docs/reference/config/general.md b/docs/reference/config/general.mdindex b5fbdf6fd..9a56b51e2 100644--- a/docs/reference/config/general.md+++ b/docs/reference/config/general.md@@ -79,6 +79,39 @@ Maximum number of characters a folder should have. CDL will truncate folders lonmax_folder_name_length: 60```+# `restrict_path`++| Type | Default |+| ------------------------------------------------ | ------- |+| list of `windows`, `unix`, `ascii` or `no_emoji` | [] |++Sanitize file/folder names according to the options in the list. An empty list (the default) will use the default config according to the OS.+Due to compatibility reasons, the default on Windows and macOS is [`windows`, `no_emoji`]. The default on any other system is [`windows`]++```yaml+restrict_path: []+```++## `windows`++remove all characters invalid on Windows: `\\`, `\`, `|` ,`/`, `<`, `>`, `:`, `?`, `*`++## `unix`++remove all characters invalid on Unix: `/`++## `ascii`++remove all characters except ascii letters, underscores, and dots++## `no_emoji`++remove all unicode characters except letters, numbers and marks (categories: `C`, `N` and `M`). A few hardcoded exceptions are still allowed:++- literal space `U+0020`+- Simbols (category: `S`): `^`, `~`, `$`, `+`, `=`+- Punctuation (category: `P`): `.`, `-`, `_`, `!`, `#`, `%`, `'`, `(`, `)`, `,`, `;`, `@`, `[`, `]`, `{`, `}`+# `min_free_space`| Type | Default | Restrictions |diff --git a/tests/test_config.py b/tests/test_config.pyindex bd71e3612..073d4fe59 100644--- a/tests/test_config.py+++ b/tests/test_config.py@@ -399,4 +399,5 @@ def test_additive_args_list() -> None:("filters", "skip_hosts"),("hashing", "algorithms"),("jdownloader", "whitelist"),+ ("restrict_path",),)