Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: allow guessing ext by mimetype (#1636)

NTFSvolumeadd9bce3fe3f
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 1faf49a4f..52737a2a3 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -550,17 +550,17 @@ class Crawler(ABC):
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def get_filename_and_ext(
- self, filename: str, forum: bool = False, assume_ext: str | None = None
+ self, filename: str, forum: bool = False, assume_ext: str | None = ".mp4", *, mime_type: str | None = None
) -> tuple[str, str]:
"""Wrapper around `utils.get_filename_and_ext`.
Calls it as is.
If that fails, appends `assume_ext` and tries again, but only if the user had exclude_files_with_no_extension = `False`
"""
try:
- return get_filename_and_ext(filename, forum)
+ return get_filename_and_ext(filename, forum, mime_type)
except NoExtensionError:
- if assume_ext and self.allow_no_extension:
- return get_filename_and_ext(filename + assume_ext, forum)
+ if self.allow_no_extension and assume_ext:
+ return get_filename_and_ext(filename + assume_ext, forum, mime_type)
raise
def check_album_results(self, url: URL, album_results: dict[str, Any]) -> bool:
diff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.py
index edb0ff881..e06db6166 100644
--- a/cyberdrop_dl/crawlers/gofile.py
+++ b/cyberdrop_dl/crawlers/gofile.py
@@ -208,10 +208,9 @@ class GoFileCrawler(Crawler):
if file.get("isFrozen"):
self.log(f"{link} is marked as frozen, download may fail", 30)
- name = file["name"]
- filename, ext = self.get_filename_and_ext(name, assume_ext=".mp4")
+ filename, ext = self.get_filename_and_ext(file["name"], mime_type=file.get("mimetype"))
scrape_item.possible_datetime = file["createTime"]
- await self.handle_file(link, scrape_item, name, ext, custom_filename=filename, metadata=file)
+ await self.handle_file(link, scrape_item, file["name"], ext, custom_filename=filename, metadata=file)
@error_handling_wrapper
async def _get_credentials(self, _) -> None:
diff --git a/cyberdrop_dl/crawlers/google_photos.py b/cyberdrop_dl/crawlers/google_photos.py
index fcb86d77b..9b1aa4d4c 100644
--- a/cyberdrop_dl/crawlers/google_photos.py
+++ b/cyberdrop_dl/crawlers/google_photos.py
@@ -1,7 +1,6 @@
from __future__ import annotations
import json
-import mimetypes
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, ClassVar
@@ -88,10 +87,8 @@ class GooglePhotosCrawler(Crawler):
scrape_item.possible_datetime = image.date
async with self.request(link) as resp:
- ext = mimetypes.guess_extension(resp.content_type)
+ filename, ext = self.get_filename_and_ext(image.id, mime_type=resp.content_type)
- assert ext
- filename, ext = self.get_filename_and_ext(image.id + ext)
custom_filename = f"{str(idx).zfill(3)} - {filename}"
await self.handle_file(
scrape_item.url,
diff --git a/cyberdrop_dl/crawlers/pixeldrain.py b/cyberdrop_dl/crawlers/pixeldrain.py
index a9695273a..68befb4a6 100644
--- a/cyberdrop_dl/crawlers/pixeldrain.py
+++ b/cyberdrop_dl/crawlers/pixeldrain.py
@@ -2,7 +2,6 @@
from __future__ import annotations
-import mimetypes
from typing import TYPE_CHECKING, ClassVar, Literal
from pydantic import BaseModel
@@ -10,7 +9,6 @@ from pydantic import BaseModel
from cyberdrop_dl import env
from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedDomains, SupportedPaths, auto_task_id
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.exceptions import NoExtensionError
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -254,15 +252,7 @@ class PixelDrainCrawler(Crawler):
if "text/plain" in file.mime_type:
return await self._text(scrape_item, file)
- try:
- filename, ext = self.get_filename_and_ext(file.name)
- except NoExtensionError:
- ext = mimetypes.guess_extension(file.mime_type)
- if not ext:
- raise
-
- filename, ext = self.get_filename_and_ext(f"{file.name}{ext}")
-
+ filename, ext = self.get_filename_and_ext(file.name, mime_type=file.mime_type)
scrape_item.possible_datetime = self.parse_iso_date(file.date_upload)
await self.handle_file(link, scrape_item, file.name, ext, debrid_link=debrid_link, custom_filename=filename)
diff --git a/cyberdrop_dl/exceptions.py b/cyberdrop_dl/exceptions.py
index a118b76f6..06fb0fb22 100644
--- a/cyberdrop_dl/exceptions.py
+++ b/cyberdrop_dl/exceptions.py
@@ -84,16 +84,16 @@ class InvalidContentTypeError(CDLBaseError):
class NoExtensionError(CDLBaseError):
- def __init__(self, *, message: str | None = None, origin: ScrapeItem | MediaItem | URL | None = None) -> None:
+ def __init__(self, filename: str | None = None, *, origin: ScrapeItem | MediaItem | URL | None = None) -> None:
"""This error will be thrown when no extension is given for a file."""
ui_failure = "No File Extension"
- super().__init__(ui_failure, message=message, origin=origin)
+ super().__init__(ui_failure, message=filename, origin=origin)
class InvalidExtensionError(NoExtensionError):
- def __init__(self, *, message: str | None = None, origin: ScrapeItem | MediaItem | URL | None = None) -> None:
+ def __init__(self, filename: str | None = None, *, origin: ScrapeItem | MediaItem | URL | None = None) -> None:
"""This error will be thrown when no extension is given for a file."""
- super().__init__(message=message, origin=origin)
+ super().__init__(filename=filename, origin=origin)
self.ui_failure = "Invalid File Extension"
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index 385135370..575b9dae1 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -5,6 +5,7 @@ import dataclasses
import functools
import inspect
import itertools
+import mimetypes
import os
import platform
import re
@@ -204,25 +205,32 @@ def truncate_str(text: str, max_length: int = 0) -> str:
return text.strip()
-def get_filename_and_ext(filename: str, forum: bool = False) -> tuple[str, str]:
+def get_filename_and_ext(filename: str, forum: bool = False, mime_type: str | None = None) -> tuple[str, str]:
"""Returns the filename and extension of a given file, throws `NoExtensionError` if there is no extension."""
clean_filename = Path(filename).as_posix().replace("/", "-") # remove OS separators
filename_as_path = Path(clean_filename)
- if not filename_as_path.suffix:
- raise NoExtensionError
- ext_no_dot = filename_as_path.suffix.split(".")[1]
+ ext = filename_as_path.suffix or (mimetypes.guess_extension(mime_type) if mime_type else None)
+ if not ext:
+ raise NoExtensionError(filename)
+
+ filename_as_path = filename_as_path.with_suffix(ext)
+
+ ext_no_dot = filename_as_path.suffix.split(".", 1)[-1]
if ext_no_dot.isdigit() and forum and "-" in filename:
name, ext = filename_as_path.name.rsplit("-", 1)
ext = ext.rsplit(".")[0]
ext_w_dot = f".{ext}".lower()
+ full_name = f"{name}.{ext}"
if ext_w_dot not in constants.MEDIA_EXTENSIONS:
- raise InvalidExtensionError
- filename_as_path = Path(f"{name}.{ext}")
+ raise InvalidExtensionError(full_name)
+
+ filename_as_path = Path(full_name)
+
if len(filename_as_path.suffix) > 5:
- raise InvalidExtensionError
+ raise InvalidExtensionError(str(filename_as_path))
filename_as_path = filename_as_path.with_suffix(filename_as_path.suffix.lower())
- filename_as_str = truncate_str(filename_as_path.stem.removesuffix(".")) + filename_as_path.suffix
+ filename_as_str = truncate_str(filename_as_path.stem) + filename_as_path.suffix
filename_as_path = Path(sanitize_filename(filename_as_str))
filename_as_path = Path(filename_as_path.stem.strip() + filename_as_path.suffix)
return filename_as_path.name, filename_as_path.suffix
diff --git a/tests/test_utils.py b/tests/test_utils.py
new file mode 100644
index 000000000..6aaf707fb
--- /dev/null
+++ b/tests/test_utils.py
@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import platform
+from pathlib import Path
+
+import pytest
+
+from cyberdrop_dl.exceptions import InvalidExtensionError, NoExtensionError
+from cyberdrop_dl.utils.utilities import get_filename_and_ext
+
+
+class TestGetFilenameAndExt:
+ _file = Path("Cyberdrop-DL.v8.4.0.zip")
+
+ def _ext(self, *args, **kargs) -> str:
+ return get_filename_and_ext(*args, **kargs)[1]
+
+ def _name(self, *args, **kargs) -> str:
+ return get_filename_and_ext(*args, **kargs)[0]
+
+ def test_ext_should_always_be_lowercase(self) -> None:
+ exts = [self._ext(self._file.with_suffix(ext).name) for ext in (".zip", ".Zip", ".zIP", ".ziP")]
+ assert exts == [".zip"] * len(exts)
+
+ def test_ext_should_not_include_multiple_suffixes(self) -> None:
+ file = self._file.name + ".rar"
+ assert self._ext(file) == ".rar"
+
+ @pytest.mark.parametrize(
+ "name, mimetype, expected_ext",
+ [
+ ("archive", "application/zip", ".zip"),
+ ("Katalina Kyle, Savanah Storm - What If She Hears Us!", "video/mp4", ".mp4"),
+ ],
+ )
+ def test_mime_type_fallback(self, name: str, mimetype: str, expected_ext: str) -> None:
+ filename, ext = get_filename_and_ext(name, mime_type=mimetype)
+ assert ext == expected_ext
+ assert filename == name + expected_ext
+
+ @pytest.mark.skipif(platform.system() in ("Windows", "Darwin"), reason="Emojis are stripped on Windows and MacOS")
+ @pytest.mark.parametrize(
+ "name, expected_name, expected_ext",
+ [
+ ("Vídeo de verificación [uvfdtpm4c2a]/video.MP4", "Vídeo de verificación [uvfdtpm4c2a]-video.mp4", ".mp4"),
+ (
+ "Katalina Kyle, Savanah Storm - “What If She Hears Us! 🍕”.mp4",
+ "Katalina Kyle, Savanah Storm - “What If She Hears Us! 🍕”.mp4",
+ ".mp4",
+ ),
+ ],
+ )
+ def test_complex_name(self, name: str, expected_name: str, expected_ext: str) -> None:
+ name, ext = get_filename_and_ext(name)
+ assert ext == expected_ext
+ assert name == expected_name
+
+ @pytest.mark.parametrize(
+ "mime_type",
+ [
+ "application/zip",
+ "application/pdf",
+ "image/png",
+ "video/mp4",
+ "text/plain",
+ "application/json",
+ "image/jpeg",
+ ],
+ )
+ def test_mime_type_should_be_ignore_if_the_file_has_a_valid_ext(self, mime_type: str) -> None:
+ filename, ext = get_filename_and_ext(self._file.name, mime_type=mime_type)
+ assert ext == self._file.suffix
+ assert filename == self._file.name
+
+ @pytest.mark.parametrize(
+ "name, expected_name, expected_ext",
+ [
+ ("img_3763-webp.6091490", "img_3763.webp", ".webp"),
+ ("1-gif.5021643", "1.gif", ".gif"),
+ ],
+ )
+ def test_forum_filename(self, name: str, expected_name: str, expected_ext: str) -> None:
+ name, ext = get_filename_and_ext(name, forum=True)
+ assert ext == expected_ext
+ assert name == expected_name
+
+ def test_long_extension_should_raise_invalid_ext_error(self) -> None:
+ with pytest.raises(InvalidExtensionError):
+ get_filename_and_ext("video.abcdef")
+
+ def test_no_extension_should_raise_no_ext_error(self) -> None:
+ with pytest.raises(NoExtensionError):
+ get_filename_and_ext("README")