Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add `--restrict-path` option (#2101)

NTFSvolumecbeb9fa9cae2
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6654f27db..be5d63a6c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
+- New `--restrict-path` option
- New `--input-folder` option that scans all `.txt` within a folder for URLs (non recursive). URLs within the same file will create a group. The name of the group is the filename;
Each group will be downloaded to their own subfolder within the downloads folder. See: <https://script-ware.gitbook.io/cyberdrop-dl/reference/config/sorting#group-urls>
- New `--input` option that accepts either a file or a folder
diff --git a/cyberdrop_dl/config/__init__.py b/cyberdrop_dl/config/__init__.py
index 161a19698..3aa1f08ee 100644
--- a/cyberdrop_dl/config/__init__.py
+++ b/cyberdrop_dl/config/__init__.py
@@ -14,7 +14,7 @@ from cyberdrop_dl.constants import DEFAULT_PARAMETER
from cyberdrop_dl.exceptions import CDLConfigRuntimeErrorsGroup, InvalidYamlError
from cyberdrop_dl.models import AdditiveArg, ConfigModel, merge_dicts, merge_models
from cyberdrop_dl.models.types import ByteSizeSerilized # noqa: TC001
-from cyberdrop_dl.models.validators import to_bytesize
+from cyberdrop_dl.models.validators import remove_duplicates, to_bytesize
from cyberdrop_dl.utils import cleanup
from .auth import Authentication, Notifications
@@ -109,6 +109,11 @@ class Config(ConfigModel, title="cyberdrop-dl config"):
network: Network = Field(default_factory=Network)
notifications: Notifications = Field(default_factory=Notifications)
+ restrict_path: Annotated[
+ tuple[Literal["unix", "windows", "no_emoji", "ascii"], ...],
+ AfterValidator(remove_duplicates),
+ Parameter(alias="restrict-filenames"),
+ ] = ()
sort: Sort = Field(default_factory=Sort)
subfolders: SubFolders = Field(default_factory=SubFolders)
ui: UIOptions = Field(default_factory=UIOptions)
diff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yaml
index 2db12c4ac..46c997325 100644
--- a/cyberdrop_dl/config/default.yaml
+++ b/cyberdrop_dl/config/default.yaml
@@ -170,6 +170,7 @@ network:
notifications:
apprise: []
webhook: null
+restrict_path: []
sort:
enabled: false
formats:
diff --git a/cyberdrop_dl/config/schema.json b/cyberdrop_dl/config/schema.json
index 6f7fd8e5c..aa5f975d5 100644
--- a/cyberdrop_dl/config/schema.json
+++ b/cyberdrop_dl/config/schema.json
@@ -1392,6 +1392,20 @@
"notifications": {
"$ref": "#/$defs/Notifications"
},
+ "restrict_path": {
+ "default": [],
+ "items": {
+ "enum": [
+ "unix",
+ "windows",
+ "no_emoji",
+ "ascii"
+ ],
+ "type": "string"
+ },
+ "title": "Restrict Path",
+ "type": "array"
+ },
"sort": {
"$ref": "#/$defs/Sort"
},
diff --git a/cyberdrop_dl/filepath.py b/cyberdrop_dl/filepath.py
index 2c8134693..d2494969c 100644
--- a/cyberdrop_dl/filepath.py
+++ b/cyberdrop_dl/filepath.py
@@ -5,38 +5,93 @@ import platform
import re
import unicodedata
from contextvars import ContextVar
+from enum import StrEnum
from pathlib import Path
+from typing import TYPE_CHECKING, Literal, Self
from cyberdrop_dl.constants import FileExt
from cyberdrop_dl.exceptions import FileNameError, InvalidExtensionError, NoExtensionError, PathTraversalError
+from cyberdrop_dl.signature import simple_repr
-_ALLOWED_FILEPATH_PUNCTUATION = " .-_!#$%'()+,;=@[]^{}~"
-_SANITIZE_FILENAME_PATTERN = r'[<>:"/\\|?*\']'
+if TYPE_CHECKING:
+ from collections.abc import Callable
+
+type PathOptions = Literal["unix", "windows", "no_emoji", "ascii"]
+
+_ALLOWED_UNICODE_SYMBOLS_AND_PUNCTUATION = " .-_!#$%'()+,;=@[]^{}~"
_RAR_MULTIPART_PATTERN = r"^part\d+"
+_MAX_FILE_LEN: ContextVar[int] = ContextVar("_MAX_FILE_LEN", default=95)
+_MAX_FOLDER_LEN: ContextVar[int] = ContextVar("_MAX_FOLDER_LEN", default=60)
+_PATH_SANITIZER: ContextVar[PathSanitizer] = ContextVar("_PATH_SANITIZER")
+
+
+class RestrictPath(StrEnum):
+ ASCII = "^0-9A-Za-z_."
+ WINDOWS = r'<>:"/\\|?*\''
+ UNIX = "/"
+
+
+class UnicodeCategory(StrEnum):
+ CONTROL = "C"
+ LETTER = "L"
+ MARK = "M"
+ NUMBER = "N"
+ PUNCTUATION = "P"
+ SIMBOL = "S"
+ SEPARATOR = "Z"
+
+
+class PathSanitizer:
+ def __init__(self, banned_chars: str | None = None, *post_process: Callable[[str], str]) -> None:
+ self.banned_chars: str | None = banned_chars
+ self.post_process: tuple[Callable[[str], str], ...] = post_process
+
+ __repr__ = simple_repr("banned_chars", "post_process")
+
+ def __call__(self, name: str, repl: str = "") -> str:
+ if self.banned_chars:
+ name = re.sub(f"[{self.banned_chars}]", repl, name).strip()
+ for fn in self.post_process:
+ name = fn(name)
-MAX_FILE_LEN: ContextVar[int] = ContextVar("_MAX_FILE_LEN", default=95)
-MAX_FOLDER_LEN: ContextVar[int] = ContextVar("_MAX_FOLDER_LEN", default=60)
+ return name
+
+ def __or__(self, other: Self) -> Self:
+ return type(self)(self.banned_chars, *self.post_process, other)
+
+ @classmethod
+ def v9_default(cls) -> Self:
+ if platform.system() in {"Windows", "Darwin"}:
+ return cls(RestrictPath.WINDOWS, remove_emojis_and_symbols)
+ return cls(RestrictPath.WINDOWS)
+
+
+def _is_allowed_unicode(char: str) -> bool:
+ return char in _ALLOWED_UNICODE_SYMBOLS_AND_PUNCTUATION or unicodedata.category(char)[0] in {
+ UnicodeCategory.LETTER,
+ UnicodeCategory.NUMBER,
+ UnicodeCategory.MARK,
+ }
def remove_emojis_and_symbols(filename: str) -> str:
- """Allow all Unicode letters/numbers/marks, plus safe filename punctuation, but not symbols or emoji."""
- return "".join(
- char
- for char in filename
- if (char in _ALLOWED_FILEPATH_PUNCTUATION or unicodedata.category(char)[0] in {"L", "N", "M"})
- ).strip()
+ """Allow all Unicode letters/numbers/marks, plus safe filename punctuation, but not symbols (emojis)."""
+ return "".join(filter(_is_allowed_unicode, filename)).strip()
def sanitize_filename(name: str, sub: str = "") -> str:
- clean_name = re.sub(_SANITIZE_FILENAME_PATTERN, sub, name).strip()
- if platform.system() in {"Windows", "Darwin"}:
- clean_name = remove_emojis_and_symbols(clean_name)
- path = Path(clean_name)
+ try:
+ clean = _PATH_SANITIZER.get()
+ except LookupError:
+ clean = PathSanitizer.v9_default()
+ _PATH_SANITIZER.set(clean)
+
+ path = Path(clean(name, sub))
return path.stem.strip() + path.suffix
def sanitize_folder(title: str, max_len: int | None = None) -> str:
- max_len = max_len or MAX_FOLDER_LEN.get()
+ max_len = max_len or _MAX_FOLDER_LEN.get()
title = title.replace("\n", "").replace("\t", "").strip()
title = sanitize_filename(re.sub(r" +", " ", title), "-")
title = re.sub(r"\.{2,}", ".", title).rstrip(".").strip()
@@ -94,7 +149,7 @@ def compose_filename(name: str, suffix: str, *extras: str, max_len: int | None =
assert suffix.startswith(".")
name = sanitize_filename(remove_os_sep(name)).removesuffix(suffix)
- max_len = (max_len or MAX_FILE_LEN.get()) - len(suffix)
+ max_len = (max_len or _MAX_FILE_LEN.get()) - len(suffix)
if extras:
extra_info = sanitize_filename("".join(f"[{info}]" for info in extras))
if (new_max_len := max_len - len(extra_info) - 1) > 0:
@@ -142,3 +197,26 @@ def check_dangerous_filename(filename: str) -> None:
path = Path(filename)
if "\\" in filename or "/" in filename or path.name != filename or path.suffix.lower() in FileExt.DANGEROUS:
raise FileNameError("Dangerous File Extension", message=filename)
+
+
+def setup(
+ max_file_len: int,
+ max_folder_len: int,
+ restrict_path: tuple[PathOptions, ...],
+) -> None:
+ _MAX_FILE_LEN.set(max_file_len)
+ _MAX_FOLDER_LEN.set(max_folder_len)
+
+ if not restrict_path:
+ return
+
+ sanitizer = PathSanitizer()
+ for name in restrict_path:
+ other = (
+ PathSanitizer(None, remove_emojis_and_symbols)
+ if name == "no_emoji"
+ else PathSanitizer(RestrictPath[name.upper()])
+ )
+ sanitizer = sanitizer | other
+
+ _PATH_SANITIZER.set(sanitizer)
diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.py
index 4fbb5d986..d23d492a8 100644
--- a/cyberdrop_dl/scrape_mapper.py
+++ b/cyberdrop_dl/scrape_mapper.py
@@ -169,10 +169,12 @@ class ScrapeMapper:
assert not self.task_mngr.scrape.done.is_set()
self.logs.delete_old_logs()
config = self.manager.config
- _ = filepath.MAX_FILE_LEN.set(config.max_file_name_length)
- _ = filepath.MAX_FOLDER_LEN.set(config.max_folder_name_length)
+
_ = CONCURRENT_SEGMENTS.set(config.downloads.concurrent_segments)
_ = ALLOW_NO_EXT.set(config.filters.allow_files_with_no_extension)
+
+ filepath.setup(config.max_file_name_length, config.max_folder_name_length, config.restrict_path)
+
if config.ui.portrait:
env.FORCE_PORTRAIT_MODE = True
diff --git a/docs/reference/config/general.md b/docs/reference/config/general.md
index b5fbdf6fd..9a56b51e2 100644
--- a/docs/reference/config/general.md
+++ b/docs/reference/config/general.md
@@ -79,6 +79,39 @@ Maximum number of characters a folder should have. CDL will truncate folders lon
max_folder_name_length: 60
```
+# `restrict_path`
+
+| Type | Default |
+| ------------------------------------------------ | ------- |
+| list of `windows`, `unix`, `ascii` or `no_emoji` | [] |
+
+Sanitize file/folder names according to the options in the list. An empty list (the default) will use the default config according to the OS.
+Due to compatibility reasons, the default on Windows and macOS is [`windows`, `no_emoji`]. The default on any other system is [`windows`]
+
+```yaml
+restrict_path: []
+```
+
+## `windows`
+
+remove all characters invalid on Windows: `\\`, `\`, `|` ,`/`, `<`, `>`, `:`, `?`, `*`
+
+## `unix`
+
+remove all characters invalid on Unix: `/`
+
+## `ascii`
+
+remove all characters except ascii letters, underscores, and dots
+
+## `no_emoji`
+
+remove all unicode characters except letters, numbers and marks (categories: `C`, `N` and `M`). A few hardcoded exceptions are still allowed:
+
+- literal space `U+0020`
+- Simbols (category: `S`): `^`, `~`, `$`, `+`, `=`
+- Punctuation (category: `P`): `.`, `-`, `_`, `!`, `#`, `%`, `'`, `(`, `)`, `,`, `;`, `@`, `[`, `]`, `{`, `}`
+
# `min_free_space`
| Type | Default | Restrictions |
diff --git a/tests/test_config.py b/tests/test_config.py
index bd71e3612..073d4fe59 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -399,4 +399,5 @@ def test_additive_args_list() -> None:
("filters", "skip_hosts"),
("hashing", "algorithms"),
("jdownloader", "whitelist"),
+ ("restrict_path",),
)