Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: generalize etag check (#1788)
NTFSvolume9d413398c1e2
diff --git a/cyberdrop_dl/clients/client.py b/cyberdrop_dl/clients/client.pyindex 4851f232f..b7c3f6116 100644--- a/cyberdrop_dl/clients/client.py+++ b/cyberdrop_dl/clients/client.py@@ -16,7 +16,7 @@ from aiolimiter import AsyncLimiterfrom multidict import CIMultiDictfrom cyberdrop_dl import cookies, ddos_guard, signature-from cyberdrop_dl.clients import etag, flaresolverr, tcp+from cyberdrop_dl.clients import flaresolverr, tcpfrom cyberdrop_dl.clients.download_client import DownloadClientfrom cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.cookies import make_simple_cookie@@ -199,16 +199,11 @@ class HTTPClient:async for cookie in cookies.read_netscape_files(cookie_files):self.cookies.update_cookies(cookie)- async def check_http_status(- self, response: ClientResponse | CurlResponse | AbstractResponse[Any], download: bool = False- ) -> None:+ async def check_http_status(self, response: ClientResponse | CurlResponse | AbstractResponse[Any]) -> None:"""Checks the HTTP status code and raises an exception if it's not acceptable."""if not isinstance(response, AbstractResponse):response = AbstractResponse.create(response)- if download:- etag.check(response.headers)-if HTTPStatus.OK <= response.status < HTTPStatus.BAD_REQUEST:# Check DDosGuard even on successful pagesawait ddos_guard.check_resp(response)diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex 649c3ed99..fe7bdf7c9 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -9,6 +9,7 @@ from http import HTTPStatusfrom typing import TYPE_CHECKING, Anyfrom cyberdrop_dl import aio, constants, ffmpeg, storage+from cyberdrop_dl.clients import etagfrom cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.constants import FileExtfrom cyberdrop_dl.exceptions import DownloadError, InvalidContentTypeError, SlowDownloadError@@ -77,7 +78,8 @@ class DownloadClient:if resp.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE:await aio.unlink(media_item.partial_file)- await self.client.check_http_status(resp, download=True)+ etag.check(resp.headers)+ await self.client.check_http_status(resp)if not media_item.is_segment:_ = get_content_type(media_item.ext, resp.headers)diff --git a/cyberdrop_dl/clients/etag.py b/cyberdrop_dl/clients/etag.pyindex 90924a436..9c659aa58 100644--- a/cyberdrop_dl/clients/etag.py+++ b/cyberdrop_dl/clients/etag.py@@ -1,5 +1,6 @@from __future__ import annotations+import dataclassesfrom http import HTTPStatusfrom typing import TYPE_CHECKING@@ -8,19 +9,39 @@ from cyberdrop_dl.exceptions import DownloadErrorif TYPE_CHECKING:from collections.abc import Mapping-_DOWNLOAD_ERROR_ETAGS = {- "d835884373f4d6c8f24742ceabe74946": "Imgur image has been removed",- "65b7753c-528a": "SC Scrape Image",- "5c4fb843-ece": "PixHost Removed Image",- "637be5da-11d2b": "eFukt Video removed",- "63a05f27-11d2b": "eFukt Video removed",- "5a56b09d-1485eb": "eFukt Video removed",- "19fdf2cd6-383c-5a4cd5b6710ed": "ImageVenue image not Found",- "383c-5a4cd5b6710ed": "ImageVenue image not Found",-}+_ETAGS: dict[str, ETag] = {}++[email protected](slots=True, frozen=True)+class ETag:+ domain: str+ value: str+ msg: str | None = None+ error: int | str = HTTPStatus.NOT_FOUND+++for domain, params in {+ "Imgur": [("d835884373f4d6c8f24742ceabe74946", "Image has been removed")],+ "SimpCity": [("65b7753c-528a", "SC Scrape Image")],+ "PixHost": [("5c4fb843-ece", "PixHost Removed Image")],+ "eFukt": [+ ("637be5da-11d2b", "Video removed"),+ ("63a05f27-11d2b", "Video removed"),+ ("5a56b09d-1485eb", "Video removed"),+ ],+ "ImageVenue": [+ ("19fdf2cd6-383c-5a4cd5b6710ed", "Image not found"),+ ("383c-5a4cd5b6710ed", "Image not found"),+ ],+}.items():+ for param in params:+ new_etag = ETag(domain, *param)+ if old_tag := _ETAGS.get(new_etag.value):+ raise RuntimeError(f"Duplicated etag: {old_tag = }, {new_etag = }")+ _ETAGS[new_etag.value] = new_etagdef check(headers: Mapping[str, str]) -> None:- e_tag = headers.get("ETag", "").strip('"')- if message := _DOWNLOAD_ERROR_ETAGS.get(e_tag):- raise DownloadError(HTTPStatus.NOT_FOUND, message)+ e_tag_value = headers.get("ETag", "").strip('"')+ if e_tag := _ETAGS.get(e_tag_value):+ raise DownloadError(e_tag.error, e_tag.msg)diff --git a/pyproject.toml b/pyproject.tomlindex 50aaa640e..deb57a04c 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -52,7 +52,7 @@ license-files = ["LICENSE"]maintainers = [{name = "NTFSvolume", email = "[email protected]"}]readme = "README.md"requires-python = ">=3.11"-version = "9.7.0"+version = "9.7.1.dev0"[project.optional-dependencies]apprise = [diff --git a/tests/test_etags.py b/tests/test_etags.pynew file mode 100644index 000000000..4d9ee4b56--- /dev/null+++ b/tests/test_etags.py@@ -0,0 +1,41 @@+from __future__ import annotations++from http import HTTPStatus++import pytest+from multidict import CIMultiDict++from cyberdrop_dl.clients.etag import check+from cyberdrop_dl.exceptions import DownloadError+++def test_missing_etag_header() -> None:+ check({})+ check({"Content-Type": "text/plain"})+++def test_unknown_etag() -> None:+ check({"ETag": '"unknown-value"'})+++def test_known_etag() -> None:+ value = "d835884373f4d6c8f24742ceabe74946"+ with pytest.raises(DownloadError) as exc_info:+ check({"ETag": f"{value}"})+ assert exc_info.value.status == HTTPStatus.NOT_FOUND+ assert exc_info.value.message == "Image has been removed"+++def test_etag_with_quotes() -> None:+ value = "65b7753c-528a"+ with pytest.raises(DownloadError):+ check({"ETag": f'"{value}"'})+++def test_case_insensitive_header_lookup() -> None:+ value = "5c4fb843-ece"+ headers = {"etag": f'"{value}"'}+ check(headers)++ with pytest.raises(DownloadError):+ check(CIMultiDict(headers))diff --git a/uv.lock b/uv.lockindex 481ba9491..ee27b2eb1 100644--- a/uv.lock+++ b/uv.lock@@ -725,7 +725,7 @@ wheels = [[[package]]name = "cyberdrop-dl-patched"-version = "9.7.0"+version = "9.7.1.dev0"source = { editable = "." }dependencies = [{ name = "aiodns", marker = "sys_platform != 'android' and sys_platform != 'win32'" },