Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
from __future__ import annotations
import mimetypes
import platform
import re
import unicodedata
from contextvars import ContextVar
from enum import StrEnum
from pathlib import Path
from typing import TYPE_CHECKING, Literal, NamedTuple, Self
from cyberdrop_dl.constants import FileExt
from cyberdrop_dl.exceptions import FileNameError, InvalidExtensionError, NoExtensionError, PathTraversalError
from cyberdrop_dl.signature import dataclass_repr
if TYPE_CHECKING:
from collections.abc import Callable
type PathOptions = Literal["unix", "windows", "no_emoji", "ascii"]
_ALLOWED_UNICODE_SYMBOLS_AND_PUNCTUATION = " .-_!#$%'()+,;=@[]^{}~"
_RAR_MULTIPART_PATTERN = r"^part\d+"
_MAX_FILE_LEN: ContextVar[int] = ContextVar("_MAX_FILE_LEN", default=95)
_MAX_FOLDER_LEN: ContextVar[int] = ContextVar("_MAX_FOLDER_LEN", default=60)
_PATH_SANITIZER: ContextVar[PathSanitizer] = ContextVar("_PATH_SANITIZER")
class RestrictPath(StrEnum):
ASCII = "^0-9A-Za-z_."
WINDOWS = r'<>:"/\\|?*\''
UNIX = "/"
class UnicodeCategory(StrEnum):
CONTROL = "C"
LETTER = "L"
MARK = "M"
NUMBER = "N"
PUNCTUATION = "P"
SIMBOL = "S"
SEPARATOR = "Z"
class Filename(NamedTuple):
name: str
suffix: str
def as_path(self) -> Path:
return Path(self.name)
@property
def stem(self) -> str:
return self.name.removesuffix(self.suffix)
@dataclass_repr("banned_chars", "post_process")
class PathSanitizer:
def __init__(self, banned_chars: str | None = None, *post_process: Callable[[str], str]) -> None:
self.banned_chars: str | None = banned_chars
self.post_process: tuple[Callable[[str], str], ...] = post_process
def __call__(self, name: str, repl: str = "") -> str:
if self.banned_chars:
name = re.sub(f"[{self.banned_chars}]", repl, name).strip()
for fn in self.post_process:
name = fn(name)
return name
def __or__(self, other: Self) -> Self:
return type(self)(self.banned_chars, *self.post_process, other)
@classmethod
def v9_default(cls) -> Self:
if platform.system() in {"Windows", "Darwin"}:
return cls(RestrictPath.WINDOWS, remove_emojis_and_symbols)
return cls(RestrictPath.WINDOWS)
def _is_allowed_unicode(char: str) -> bool:
return char in _ALLOWED_UNICODE_SYMBOLS_AND_PUNCTUATION or unicodedata.category(char)[0] in {
UnicodeCategory.LETTER,
UnicodeCategory.NUMBER,
UnicodeCategory.MARK,
}
def remove_emojis_and_symbols(filename: str) -> str:
"""Allow all Unicode letters/numbers/marks, plus safe filename punctuation, but not symbols (emojis)."""
return "".join(filter(_is_allowed_unicode, filename)).strip()
def sanitize_filename(name: str, sub: str = "") -> str:
try:
clean = _PATH_SANITIZER.get()
except LookupError:
clean = PathSanitizer.v9_default()
_PATH_SANITIZER.set(clean)
path = Path(clean(name, sub))
return path.stem.strip().lstrip(".") + path.suffix
def sanitize_folder(title: str, max_len: int | None = None) -> str:
max_len = max_len or _MAX_FOLDER_LEN.get()
title = title.replace("\n", "").replace("\t", "").strip()
title = sanitize_filename(re.sub(r" +", " ", title), "-")
title = re.sub(r"\.{2,}", ".", title).rstrip(".").strip()
if all(char in title for char in ("(", ")")):
new_title, domain_part = title.rsplit("(", 1)
new_title = _truncate_text(new_title, max_len)
return f"{new_title} ({domain_part.strip()}"
return _truncate_text(title, max_len)
def _truncate_text(text: str, max_bytes: int) -> str:
str_bytes = text.encode("utf-8")[:max_bytes]
return str_bytes.decode("utf-8", "ignore").strip()
def _guess_ext(mime: str) -> str | None:
mime = {"image/jpg": "image/jpeg"}.get(mime, mime)
return mimetypes.guess_extension(mime)
def get_filename_and_ext(
filename: str,
/,
mime_type: str | None = None,
*,
xenforo: bool = False,
max_len: int | None = None,
) -> Filename:
filename_as_path = Path(remove_os_sep(filename))
if not filename_as_path.suffix:
if mime_type and (ext := _guess_ext(mime_type)):
filename_as_path = filename_as_path.with_suffix(ext)
else:
raise NoExtensionError(filename)
if xenforo and "-" in filename and filename_as_path.suffix.lstrip(".").isdigit():
name, _, ext = filename_as_path.name.rpartition("-")
ext = "." + ext.rsplit(".")[0]
filename = f"{name}{ext}"
if ext.lower() not in FileExt.MEDIA:
raise InvalidExtensionError(filename)
filename_as_path = Path(filename)
if len(filename_as_path.suffix) > 5:
raise InvalidExtensionError(str(filename_as_path))
filename_as_path = Path(compose_filename(filename_as_path.stem, filename_as_path.suffix, max_len=max_len))
return Filename(filename_as_path.name, filename_as_path.suffix)
def remove_os_sep(filename: str) -> str:
return Path(filename).as_posix().replace("/", "-")
def compose_filename(name: str, suffix: str, *extras: str, max_len: int | None = None) -> str:
assert suffix.startswith(".")
name = sanitize_filename(remove_os_sep(name)).removesuffix(suffix)
max_len = (max_len or _MAX_FILE_LEN.get()) - len(suffix)
if extras:
extra_info = sanitize_filename("".join(f"[{info}]" for info in extras))
if (new_max_len := max_len - len(extra_info) - 1) > 0:
truncated_stem = f"{_truncate_text(name, new_max_len)} {extra_info}"
else:
truncated_stem = _truncate_text(f"{name} {extra_info}", max_len)
else:
truncated_stem = _truncate_text(name, max_len)
return f"{truncated_stem}{suffix.lower()}"
def remove_file_id(filename: str, ext: str) -> str:
"""Removes the additional string some websites adds to the end of every filename."""
filename = filename.rsplit(ext, 1)[0].rsplit("-", 1)[0]
tail_no_dot = filename.rsplit("-", 1)[-1]
ext_no_dot = ext.rsplit(".", 1)[-1]
tail = f".{tail_no_dot}"
if re.match(_RAR_MULTIPART_PATTERN, tail_no_dot) and ext == ".rar" and "-" in filename:
filename, part = filename.rsplit("-", 1)
filename = f"{filename}.{part}"
elif ext_no_dot.isdigit() and tail in FileExt.SEVEN_Z and "-" in filename:
filename, _7z_ext = filename.rsplit("-", 1)
filename = f"{filename}.{_7z_ext}"
if not filename.endswith(ext):
filename += ext
return filename
def check_path_traversal(download_folder: Path, folder: Path) -> None:
parts = folder.parts
if "." in parts or ".." in parts:
raise PathTraversalError(folder)
if not folder.resolve().is_relative_to(download_folder):
raise PathTraversalError(folder)
def check_dangerous_filename(filename: str) -> None:
if filename.startswith("."):
raise FileNameError("Dot file", message=f"Dot files are restricted: {filename}")
path = Path(filename)
if "\\" in filename or "/" in filename or path.name != filename or path.suffix.lower() in FileExt.DANGEROUS:
raise FileNameError("Dangerous File Extension", message=filename)
def setup(
max_file_len: int,
max_folder_len: int,
restrict_path: tuple[PathOptions, ...],
) -> None:
_MAX_FILE_LEN.set(max_file_len)
_MAX_FOLDER_LEN.set(max_folder_len)
if not restrict_path:
return
sanitizer = PathSanitizer()
for name in restrict_path:
other = (
PathSanitizer(None, remove_emojis_and_symbols)
if name == "no_emoji"
else PathSanitizer(RestrictPath[name.upper()])
)
sanitizer = sanitizer | other
_PATH_SANITIZER.set(sanitizer)