Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: generalize etag check (#1788)

NTFSvolume9d413398c1e2
diff --git a/cyberdrop_dl/clients/client.py b/cyberdrop_dl/clients/client.py
index 4851f232f..b7c3f6116 100644
--- a/cyberdrop_dl/clients/client.py
+++ b/cyberdrop_dl/clients/client.py
@@ -16,7 +16,7 @@ from aiolimiter import AsyncLimiter
from multidict import CIMultiDict
from cyberdrop_dl import cookies, ddos_guard, signature
-from cyberdrop_dl.clients import etag, flaresolverr, tcp
+from cyberdrop_dl.clients import flaresolverr, tcp
from cyberdrop_dl.clients.download_client import DownloadClient
from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.cookies import make_simple_cookie
@@ -199,16 +199,11 @@ class HTTPClient:
async for cookie in cookies.read_netscape_files(cookie_files):
self.cookies.update_cookies(cookie)
- async def check_http_status(
- self, response: ClientResponse | CurlResponse | AbstractResponse[Any], download: bool = False
- ) -> None:
+ async def check_http_status(self, response: ClientResponse | CurlResponse | AbstractResponse[Any]) -> None:
"""Checks the HTTP status code and raises an exception if it's not acceptable."""
if not isinstance(response, AbstractResponse):
response = AbstractResponse.create(response)
- if download:
- etag.check(response.headers)
-
if HTTPStatus.OK <= response.status < HTTPStatus.BAD_REQUEST:
# Check DDosGuard even on successful pages
await ddos_guard.check_resp(response)
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index 649c3ed99..fe7bdf7c9 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -9,6 +9,7 @@ from http import HTTPStatus
from typing import TYPE_CHECKING, Any
from cyberdrop_dl import aio, constants, ffmpeg, storage
+from cyberdrop_dl.clients import etag
from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.constants import FileExt
from cyberdrop_dl.exceptions import DownloadError, InvalidContentTypeError, SlowDownloadError
@@ -77,7 +78,8 @@ class DownloadClient:
if resp.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE:
await aio.unlink(media_item.partial_file)
- await self.client.check_http_status(resp, download=True)
+ etag.check(resp.headers)
+ await self.client.check_http_status(resp)
if not media_item.is_segment:
_ = get_content_type(media_item.ext, resp.headers)
diff --git a/cyberdrop_dl/clients/etag.py b/cyberdrop_dl/clients/etag.py
index 90924a436..9c659aa58 100644
--- a/cyberdrop_dl/clients/etag.py
+++ b/cyberdrop_dl/clients/etag.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import dataclasses
from http import HTTPStatus
from typing import TYPE_CHECKING
@@ -8,19 +9,39 @@ from cyberdrop_dl.exceptions import DownloadError
if TYPE_CHECKING:
from collections.abc import Mapping
-_DOWNLOAD_ERROR_ETAGS = {
- "d835884373f4d6c8f24742ceabe74946": "Imgur image has been removed",
- "65b7753c-528a": "SC Scrape Image",
- "5c4fb843-ece": "PixHost Removed Image",
- "637be5da-11d2b": "eFukt Video removed",
- "63a05f27-11d2b": "eFukt Video removed",
- "5a56b09d-1485eb": "eFukt Video removed",
- "19fdf2cd6-383c-5a4cd5b6710ed": "ImageVenue image not Found",
- "383c-5a4cd5b6710ed": "ImageVenue image not Found",
-}
+_ETAGS: dict[str, ETag] = {}
+
+
[email protected](slots=True, frozen=True)
+class ETag:
+ domain: str
+ value: str
+ msg: str | None = None
+ error: int | str = HTTPStatus.NOT_FOUND
+
+
+for domain, params in {
+ "Imgur": [("d835884373f4d6c8f24742ceabe74946", "Image has been removed")],
+ "SimpCity": [("65b7753c-528a", "SC Scrape Image")],
+ "PixHost": [("5c4fb843-ece", "PixHost Removed Image")],
+ "eFukt": [
+ ("637be5da-11d2b", "Video removed"),
+ ("63a05f27-11d2b", "Video removed"),
+ ("5a56b09d-1485eb", "Video removed"),
+ ],
+ "ImageVenue": [
+ ("19fdf2cd6-383c-5a4cd5b6710ed", "Image not found"),
+ ("383c-5a4cd5b6710ed", "Image not found"),
+ ],
+}.items():
+ for param in params:
+ new_etag = ETag(domain, *param)
+ if old_tag := _ETAGS.get(new_etag.value):
+ raise RuntimeError(f"Duplicated etag: {old_tag = }, {new_etag = }")
+ _ETAGS[new_etag.value] = new_etag
def check(headers: Mapping[str, str]) -> None:
- e_tag = headers.get("ETag", "").strip('"')
- if message := _DOWNLOAD_ERROR_ETAGS.get(e_tag):
- raise DownloadError(HTTPStatus.NOT_FOUND, message)
+ e_tag_value = headers.get("ETag", "").strip('"')
+ if e_tag := _ETAGS.get(e_tag_value):
+ raise DownloadError(e_tag.error, e_tag.msg)
diff --git a/pyproject.toml b/pyproject.toml
index 50aaa640e..deb57a04c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -52,7 +52,7 @@ license-files = ["LICENSE"]
maintainers = [{name = "NTFSvolume", email = "[email protected]"}]
readme = "README.md"
requires-python = ">=3.11"
-version = "9.7.0"
+version = "9.7.1.dev0"
[project.optional-dependencies]
apprise = [
diff --git a/tests/test_etags.py b/tests/test_etags.py
new file mode 100644
index 000000000..4d9ee4b56
--- /dev/null
+++ b/tests/test_etags.py
@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from http import HTTPStatus
+
+import pytest
+from multidict import CIMultiDict
+
+from cyberdrop_dl.clients.etag import check
+from cyberdrop_dl.exceptions import DownloadError
+
+
+def test_missing_etag_header() -> None:
+ check({})
+ check({"Content-Type": "text/plain"})
+
+
+def test_unknown_etag() -> None:
+ check({"ETag": '"unknown-value"'})
+
+
+def test_known_etag() -> None:
+ value = "d835884373f4d6c8f24742ceabe74946"
+ with pytest.raises(DownloadError) as exc_info:
+ check({"ETag": f"{value}"})
+ assert exc_info.value.status == HTTPStatus.NOT_FOUND
+ assert exc_info.value.message == "Image has been removed"
+
+
+def test_etag_with_quotes() -> None:
+ value = "65b7753c-528a"
+ with pytest.raises(DownloadError):
+ check({"ETag": f'"{value}"'})
+
+
+def test_case_insensitive_header_lookup() -> None:
+ value = "5c4fb843-ece"
+ headers = {"etag": f'"{value}"'}
+ check(headers)
+
+ with pytest.raises(DownloadError):
+ check(CIMultiDict(headers))
diff --git a/uv.lock b/uv.lock
index 481ba9491..ee27b2eb1 100644
--- a/uv.lock
+++ b/uv.lock
@@ -725,7 +725,7 @@ wheels = [
[[package]]
name = "cyberdrop-dl-patched"
-version = "9.7.0"
+version = "9.7.1.dev0"
source = { editable = "." }
dependencies = [
{ name = "aiodns", marker = "sys_platform != 'android' and sys_platform != 'win32'" },