Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: move custom response checks to crawlers (#1301)
NTFSvolume2da5d01b9ca9
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex fbc83d9e6..075747053 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -150,6 +150,35 @@ class Crawler(ABC):def __post_init__(self) -> None: ... # noqa: B027+ @final+ def _register_response_checks(self) -> None:+ if self._json_response_check is Crawler._json_response_check:+ return++ for host in (self.DOMAIN, self.PRIMARY_URL.host):+ self.client.client_manager._json_response_checks[host] = self._json_response_check++ @classmethod+ def _json_response_check(cls, json_resp: Any) -> None:+ """Custom check for JSON responses.++ This method is called automatically by the `client_manager` when a JSON response is received from `cls.DOMAIN`+ and it was **NOT** successful (`4xx` or `5xx` HTTP code).++ Override this method in subclasses to raise a custom `ScrapeError` instead of the default HTTP error++ Example:+ ```python+ if isinstance(json, dict) and json.get("status") == "error":+ raise ScrapeError(422, f"API error: {json['message']}")+ ```++ IMPORTANT:+ Cases were the response **IS** successful (200, OK) but the JSON indicates an error+ should be handled by the crawler itself+ """+ raise NotImplementedError+@final@staticmethoddef _assert_fields_overrides(subclass: type[Crawler], *fields: str):@@ -228,6 +257,7 @@ class Crawler(ABC):if self._DOWNLOAD_SLOTS:self.manager.client_manager.download_slots[self.DOMAIN] = self._DOWNLOAD_SLOTSself.downloader = self._init_downloader()+ self._register_response_checks()await self.async_startup()self.ready = Truediff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.pyindex 9f96f13c8..f3ef950b5 100644--- a/cyberdrop_dl/crawlers/gofile.py+++ b/cyberdrop_dl/crawlers/gofile.py@@ -5,7 +5,7 @@ import httpimport refrom datetime import UTC, datetime, timedeltafrom hashlib import sha256-from typing import TYPE_CHECKING, ClassVar, Literal, NotRequired, TypedDict, cast+from typing import TYPE_CHECKING, Any, ClassVar, Literal, NotRequired, TypedDict, castfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import FILE_HOST_ALBUM, AbsoluteHttpURL, ScrapeItem@@ -93,6 +93,13 @@ class GoFileCrawler(Crawler):self.headers: dict[str, str] = {}self._website_token_date = datetime.now(UTC) - timedelta(days=7)+ @classmethod+ def _json_response_check(cls, json_resp: Any) -> None:+ if not isinstance(json_resp, dict):+ return+ if "notFound" in json_resp["status"]:+ raise ScrapeError(404)+async def async_startup(self) -> None:await self.get_account_token(API_ENTRYPOINT)diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.pyindex df16a5998..bd7b060c0 100644--- a/cyberdrop_dl/crawlers/imgur.py+++ b/cyberdrop_dl/crawlers/imgur.py@@ -31,6 +31,12 @@ class ImgurCrawler(Crawler):self.imgur_client_remaining = 12500self.headers = {"Authorization": f"Client-ID {self.imgur_client_id}"}+ @classmethod+ def _json_response_check(cls, json_resp: Any) -> None:+ if not isinstance(json_resp, dict) or "data" not in json_resp:+ return+ raise ScrapeError(json_resp["status"], json_resp["data"]["error"])+async def fetch(self, scrape_item: ScrapeItem) -> None:if scrape_item.url.host == "i.imgur.com":return await self.handle_direct_link(scrape_item)diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.pyindex 2f5facd6d..e0b072e1b 100644--- a/cyberdrop_dl/crawlers/mediafire.py+++ b/cyberdrop_dl/crawlers/mediafire.py@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, NamedTuplefrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, auto_task_idfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.exceptions import MediaFireError, ScrapeError+from cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, is_blob_or_svg@@ -45,6 +45,16 @@ class MediaFireCrawler(Crawler):DOMAIN: ClassVar[str] = "mediafire"SKIP_PRE_CHECK = True+ @classmethod+ def _json_response_check(cls, json_resp: Any) -> None:+ if not isinstance(json_resp, dict) or "response" not in json_resp:+ return+ resp: dict[str, Any] = json_resp["response"]+ if resp["result"] != "Success":+ code: int = resp["error"]+ ui_failure = f"MediaFire Error ({code})"+ raise ScrapeError(ui_failure, resp["message"])+async def fetch(self, scrape_item: ScrapeItem) -> None:if (scrape_item.url.path == "/"@@ -64,11 +74,7 @@ class MediaFireCrawler(Crawler):async def _api_request(self, path: str, **params: Any) -> dict[str, Any]:params["response_format"] = "json"api_url = (_API_URL / path).with_query(params)- resp: dict[str, Any] = (await self.request_json(api_url))["response"]- # TODO: register with client manager- if resp["result"] != "Success":- raise MediaFireError(resp["error"], resp["message"])- return resp+ return (await self.request_json(api_url))["response"]async def _iter_folder_content(self, folder_key: str, content_type: str) -> AsyncGenerator[list[dict[str, Any]]]:async def get_content(chunk: int) -> dict[str, Any]:diff --git a/cyberdrop_dl/exceptions.py b/cyberdrop_dl/exceptions.pyindex 4365b2c04..47ca8cd6e 100644--- a/cyberdrop_dl/exceptions.py+++ b/cyberdrop_dl/exceptions.py@@ -163,15 +163,6 @@ class DurationError(CDLBaseError):super().__init__(ui_failure, origin=origin)-class MediaFireError(CDLBaseError):- def __init__(- self, code: int, message: str | None = None, origin: ScrapeItem | MediaItem | URL | None = None- ) -> None:- """This error will be thrown when a scrape fails."""- ui_failure = f"MediaFire Error ({code})"- super().__init__(ui_failure, message=message, status=code, origin=origin)--class RealDebridError(CDLBaseError):"""Base RealDebrid API error."""diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.pyindex 9002096b4..332c5a4ba 100644--- a/cyberdrop_dl/managers/client_manager.py+++ b/cyberdrop_dl/managers/client_manager.py@@ -39,7 +39,7 @@ _VALID_EXTENSIONS = ()if TYPE_CHECKING:- from collections.abc import AsyncGenerator, Generator, Iterable, Mapping+ from collections.abc import AsyncGenerator, Callable, Generator, Iterable, Mappingfrom http.cookies import BaseCookiefrom aiohttp_client_cache.response import CachedResponse@@ -171,6 +171,7 @@ class ClientManager:self._session: CachedSessionself._download_session: aiohttp.ClientSessionself._curl_session: AsyncSession[CurlResponse]+ self._json_response_checks: dict[str, Callable[[Any], None]] = {}def _startup(self) -> None:self._session = self.new_scrape_session()@@ -367,9 +368,8 @@ class ClientManager:return self.rate_limits[domain]return self.rate_limits["other"]- @classmethodasync def check_http_status(- cls,+ self,response: ClientResponse | CachedResponse | CurlResponse | AbstractResponse,download: bool = False,) -> BeautifulSoup | None:@@ -390,48 +390,39 @@ class ClientManager:async def check_ddos_guard() -> BeautifulSoup | None:if "html" not in response.content_type:return-- # TODO: use the response text instead of the raw content to prevent double encoding detection-try:soup = BeautifulSoup(await response.text(), "html.parser")except UnicodeDecodeError:returnelse:- if cls.check_ddos_guard(soup) or cls.check_cloudflare(soup):+ if self.check_ddos_guard(soup) or self.check_cloudflare(soup):raise DDOSGuardErrorreturn soup- async def check_json_status() -> None:- if "json" not in response.content_type:- return-- # TODO: Define these checks inside their actual crawlers- # and make them register them on instantation- if not any(domain in response.url.host for domain in ("gofile", "imgur")):- return-- json_resp: dict[str, Any] | None = await response.json()- if not json_resp:- return-- json_status: str | int | None = json_resp.get("status")- if json_status and isinstance(json_status, str) and "notFound" in json_status:- raise ScrapeError(404)-- if (data := json_resp.get("data")) and isinstance(data, dict) and "error" in data:- raise ScrapeError(json_status or response.status, data["error"])-check_etag()if HTTPStatus.OK <= response.status < HTTPStatus.BAD_REQUEST:# Check DDosGuard even on successful pages# await check_ddos_guard()return- await check_json_status()+ await self._check_json(response)await check_ddos_guard()raise DownloadError(status=response.status, message=message)+ async def _check_json(self, response: AbstractResponse) -> None:+ if "json" not in response.content_type:+ return++ if check := self._json_response_checks.get(response.url.host):+ check(await response.json())+ return++ for domain, check in self._json_response_checks.items():+ if domain in response.url.host:+ self._json_response_checks[response.url.host] = check+ check(await response.json())+ return+@staticmethoddef check_content_length(headers: Mapping[str, Any]) -> None:content_length, content_type = headers.get("Content-Length"), headers.get("Content-Type")