Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: allow guessing ext by mimetype (#1636)
NTFSvolumeadd9bce3fe3f
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 1faf49a4f..52737a2a3 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -550,17 +550,17 @@ class Crawler(ABC):# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~def get_filename_and_ext(- self, filename: str, forum: bool = False, assume_ext: str | None = None+ self, filename: str, forum: bool = False, assume_ext: str | None = ".mp4", *, mime_type: str | None = None) -> tuple[str, str]:"""Wrapper around `utils.get_filename_and_ext`.Calls it as is.If that fails, appends `assume_ext` and tries again, but only if the user had exclude_files_with_no_extension = `False`"""try:- return get_filename_and_ext(filename, forum)+ return get_filename_and_ext(filename, forum, mime_type)except NoExtensionError:- if assume_ext and self.allow_no_extension:- return get_filename_and_ext(filename + assume_ext, forum)+ if self.allow_no_extension and assume_ext:+ return get_filename_and_ext(filename + assume_ext, forum, mime_type)raisedef check_album_results(self, url: URL, album_results: dict[str, Any]) -> bool:diff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.pyindex edb0ff881..e06db6166 100644--- a/cyberdrop_dl/crawlers/gofile.py+++ b/cyberdrop_dl/crawlers/gofile.py@@ -208,10 +208,9 @@ class GoFileCrawler(Crawler):if file.get("isFrozen"):self.log(f"{link} is marked as frozen, download may fail", 30)- name = file["name"]- filename, ext = self.get_filename_and_ext(name, assume_ext=".mp4")+ filename, ext = self.get_filename_and_ext(file["name"], mime_type=file.get("mimetype"))scrape_item.possible_datetime = file["createTime"]- await self.handle_file(link, scrape_item, name, ext, custom_filename=filename, metadata=file)+ await self.handle_file(link, scrape_item, file["name"], ext, custom_filename=filename, metadata=file)@error_handling_wrapperasync def _get_credentials(self, _) -> None:diff --git a/cyberdrop_dl/crawlers/google_photos.py b/cyberdrop_dl/crawlers/google_photos.pyindex fcb86d77b..9b1aa4d4c 100644--- a/cyberdrop_dl/crawlers/google_photos.py+++ b/cyberdrop_dl/crawlers/google_photos.py@@ -1,7 +1,6 @@from __future__ import annotationsimport json-import mimetypesfrom dataclasses import dataclassfrom typing import TYPE_CHECKING, Any, ClassVar@@ -88,10 +87,8 @@ class GooglePhotosCrawler(Crawler):scrape_item.possible_datetime = image.dateasync with self.request(link) as resp:- ext = mimetypes.guess_extension(resp.content_type)+ filename, ext = self.get_filename_and_ext(image.id, mime_type=resp.content_type)- assert ext- filename, ext = self.get_filename_and_ext(image.id + ext)custom_filename = f"{str(idx).zfill(3)} - {filename}"await self.handle_file(scrape_item.url,diff --git a/cyberdrop_dl/crawlers/pixeldrain.py b/cyberdrop_dl/crawlers/pixeldrain.pyindex a9695273a..68befb4a6 100644--- a/cyberdrop_dl/crawlers/pixeldrain.py+++ b/cyberdrop_dl/crawlers/pixeldrain.py@@ -2,7 +2,6 @@from __future__ import annotations-import mimetypesfrom typing import TYPE_CHECKING, ClassVar, Literalfrom pydantic import BaseModel@@ -10,7 +9,6 @@ from pydantic import BaseModelfrom cyberdrop_dl import envfrom cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedDomains, SupportedPaths, auto_task_idfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.exceptions import NoExtensionErrorfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:@@ -254,15 +252,7 @@ class PixelDrainCrawler(Crawler):if "text/plain" in file.mime_type:return await self._text(scrape_item, file)- try:- filename, ext = self.get_filename_and_ext(file.name)- except NoExtensionError:- ext = mimetypes.guess_extension(file.mime_type)- if not ext:- raise-- filename, ext = self.get_filename_and_ext(f"{file.name}{ext}")-+ filename, ext = self.get_filename_and_ext(file.name, mime_type=file.mime_type)scrape_item.possible_datetime = self.parse_iso_date(file.date_upload)await self.handle_file(link, scrape_item, file.name, ext, debrid_link=debrid_link, custom_filename=filename)diff --git a/cyberdrop_dl/exceptions.py b/cyberdrop_dl/exceptions.pyindex a118b76f6..06fb0fb22 100644--- a/cyberdrop_dl/exceptions.py+++ b/cyberdrop_dl/exceptions.py@@ -84,16 +84,16 @@ class InvalidContentTypeError(CDLBaseError):class NoExtensionError(CDLBaseError):- def __init__(self, *, message: str | None = None, origin: ScrapeItem | MediaItem | URL | None = None) -> None:+ def __init__(self, filename: str | None = None, *, origin: ScrapeItem | MediaItem | URL | None = None) -> None:"""This error will be thrown when no extension is given for a file."""ui_failure = "No File Extension"- super().__init__(ui_failure, message=message, origin=origin)+ super().__init__(ui_failure, message=filename, origin=origin)class InvalidExtensionError(NoExtensionError):- def __init__(self, *, message: str | None = None, origin: ScrapeItem | MediaItem | URL | None = None) -> None:+ def __init__(self, filename: str | None = None, *, origin: ScrapeItem | MediaItem | URL | None = None) -> None:"""This error will be thrown when no extension is given for a file."""- super().__init__(message=message, origin=origin)+ super().__init__(filename=filename, origin=origin)self.ui_failure = "Invalid File Extension"diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex 385135370..575b9dae1 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -5,6 +5,7 @@ import dataclassesimport functoolsimport inspectimport itertools+import mimetypesimport osimport platformimport re@@ -204,25 +205,32 @@ def truncate_str(text: str, max_length: int = 0) -> str:return text.strip()-def get_filename_and_ext(filename: str, forum: bool = False) -> tuple[str, str]:+def get_filename_and_ext(filename: str, forum: bool = False, mime_type: str | None = None) -> tuple[str, str]:"""Returns the filename and extension of a given file, throws `NoExtensionError` if there is no extension."""clean_filename = Path(filename).as_posix().replace("/", "-") # remove OS separatorsfilename_as_path = Path(clean_filename)- if not filename_as_path.suffix:- raise NoExtensionError- ext_no_dot = filename_as_path.suffix.split(".")[1]+ ext = filename_as_path.suffix or (mimetypes.guess_extension(mime_type) if mime_type else None)+ if not ext:+ raise NoExtensionError(filename)++ filename_as_path = filename_as_path.with_suffix(ext)++ ext_no_dot = filename_as_path.suffix.split(".", 1)[-1]if ext_no_dot.isdigit() and forum and "-" in filename:name, ext = filename_as_path.name.rsplit("-", 1)ext = ext.rsplit(".")[0]ext_w_dot = f".{ext}".lower()+ full_name = f"{name}.{ext}"if ext_w_dot not in constants.MEDIA_EXTENSIONS:- raise InvalidExtensionError- filename_as_path = Path(f"{name}.{ext}")+ raise InvalidExtensionError(full_name)++ filename_as_path = Path(full_name)+if len(filename_as_path.suffix) > 5:- raise InvalidExtensionError+ raise InvalidExtensionError(str(filename_as_path))filename_as_path = filename_as_path.with_suffix(filename_as_path.suffix.lower())- filename_as_str = truncate_str(filename_as_path.stem.removesuffix(".")) + filename_as_path.suffix+ filename_as_str = truncate_str(filename_as_path.stem) + filename_as_path.suffixfilename_as_path = Path(sanitize_filename(filename_as_str))filename_as_path = Path(filename_as_path.stem.strip() + filename_as_path.suffix)return filename_as_path.name, filename_as_path.suffixdiff --git a/tests/test_utils.py b/tests/test_utils.pynew file mode 100644index 000000000..6aaf707fb--- /dev/null+++ b/tests/test_utils.py@@ -0,0 +1,93 @@+from __future__ import annotations++import platform+from pathlib import Path++import pytest++from cyberdrop_dl.exceptions import InvalidExtensionError, NoExtensionError+from cyberdrop_dl.utils.utilities import get_filename_and_ext+++class TestGetFilenameAndExt:+ _file = Path("Cyberdrop-DL.v8.4.0.zip")++ def _ext(self, *args, **kargs) -> str:+ return get_filename_and_ext(*args, **kargs)[1]++ def _name(self, *args, **kargs) -> str:+ return get_filename_and_ext(*args, **kargs)[0]++ def test_ext_should_always_be_lowercase(self) -> None:+ exts = [self._ext(self._file.with_suffix(ext).name) for ext in (".zip", ".Zip", ".zIP", ".ziP")]+ assert exts == [".zip"] * len(exts)++ def test_ext_should_not_include_multiple_suffixes(self) -> None:+ file = self._file.name + ".rar"+ assert self._ext(file) == ".rar"++ @pytest.mark.parametrize(+ "name, mimetype, expected_ext",+ [+ ("archive", "application/zip", ".zip"),+ ("Katalina Kyle, Savanah Storm - What If She Hears Us!", "video/mp4", ".mp4"),+ ],+ )+ def test_mime_type_fallback(self, name: str, mimetype: str, expected_ext: str) -> None:+ filename, ext = get_filename_and_ext(name, mime_type=mimetype)+ assert ext == expected_ext+ assert filename == name + expected_ext++ @pytest.mark.skipif(platform.system() in ("Windows", "Darwin"), reason="Emojis are stripped on Windows and MacOS")+ @pytest.mark.parametrize(+ "name, expected_name, expected_ext",+ [+ ("Vídeo de verificación [uvfdtpm4c2a]/video.MP4", "Vídeo de verificación [uvfdtpm4c2a]-video.mp4", ".mp4"),+ (+ "Katalina Kyle, Savanah Storm - “What If She Hears Us! 🍕”.mp4",+ "Katalina Kyle, Savanah Storm - “What If She Hears Us! 🍕”.mp4",+ ".mp4",+ ),+ ],+ )+ def test_complex_name(self, name: str, expected_name: str, expected_ext: str) -> None:+ name, ext = get_filename_and_ext(name)+ assert ext == expected_ext+ assert name == expected_name++ @pytest.mark.parametrize(+ "mime_type",+ [+ "application/zip",+ "application/pdf",+ "image/png",+ "video/mp4",+ "text/plain",+ "application/json",+ "image/jpeg",+ ],+ )+ def test_mime_type_should_be_ignore_if_the_file_has_a_valid_ext(self, mime_type: str) -> None:+ filename, ext = get_filename_and_ext(self._file.name, mime_type=mime_type)+ assert ext == self._file.suffix+ assert filename == self._file.name++ @pytest.mark.parametrize(+ "name, expected_name, expected_ext",+ [+ ("img_3763-webp.6091490", "img_3763.webp", ".webp"),+ ("1-gif.5021643", "1.gif", ".gif"),+ ],+ )+ def test_forum_filename(self, name: str, expected_name: str, expected_ext: str) -> None:+ name, ext = get_filename_and_ext(name, forum=True)+ assert ext == expected_ext+ assert name == expected_name++ def test_long_extension_should_raise_invalid_ext_error(self) -> None:+ with pytest.raises(InvalidExtensionError):+ get_filename_and_ext("video.abcdef")++ def test_no_extension_should_raise_no_ext_error(self) -> None:+ with pytest.raises(NoExtensionError):+ get_filename_and_ext("README")