Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: move custom response checks to crawlers (#1301)

NTFSvolume2da5d01b9ca9
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index fbc83d9e6..075747053 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -150,6 +150,35 @@ class Crawler(ABC):
def __post_init__(self) -> None: ... # noqa: B027
+ @final
+ def _register_response_checks(self) -> None:
+ if self._json_response_check is Crawler._json_response_check:
+ return
+
+ for host in (self.DOMAIN, self.PRIMARY_URL.host):
+ self.client.client_manager._json_response_checks[host] = self._json_response_check
+
+ @classmethod
+ def _json_response_check(cls, json_resp: Any) -> None:
+ """Custom check for JSON responses.
+
+ This method is called automatically by the `client_manager` when a JSON response is received from `cls.DOMAIN`
+ and it was **NOT** successful (`4xx` or `5xx` HTTP code).
+
+ Override this method in subclasses to raise a custom `ScrapeError` instead of the default HTTP error
+
+ Example:
+ ```python
+ if isinstance(json, dict) and json.get("status") == "error":
+ raise ScrapeError(422, f"API error: {json['message']}")
+ ```
+
+ IMPORTANT:
+ Cases were the response **IS** successful (200, OK) but the JSON indicates an error
+ should be handled by the crawler itself
+ """
+ raise NotImplementedError
+
@final
@staticmethod
def _assert_fields_overrides(subclass: type[Crawler], *fields: str):
@@ -228,6 +257,7 @@ class Crawler(ABC):
if self._DOWNLOAD_SLOTS:
self.manager.client_manager.download_slots[self.DOMAIN] = self._DOWNLOAD_SLOTS
self.downloader = self._init_downloader()
+ self._register_response_checks()
await self.async_startup()
self.ready = True
diff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.py
index 9f96f13c8..f3ef950b5 100644
--- a/cyberdrop_dl/crawlers/gofile.py
+++ b/cyberdrop_dl/crawlers/gofile.py
@@ -5,7 +5,7 @@ import http
import re
from datetime import UTC, datetime, timedelta
from hashlib import sha256
-from typing import TYPE_CHECKING, ClassVar, Literal, NotRequired, TypedDict, cast
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, NotRequired, TypedDict, cast
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import FILE_HOST_ALBUM, AbsoluteHttpURL, ScrapeItem
@@ -93,6 +93,13 @@ class GoFileCrawler(Crawler):
self.headers: dict[str, str] = {}
self._website_token_date = datetime.now(UTC) - timedelta(days=7)
+ @classmethod
+ def _json_response_check(cls, json_resp: Any) -> None:
+ if not isinstance(json_resp, dict):
+ return
+ if "notFound" in json_resp["status"]:
+ raise ScrapeError(404)
+
async def async_startup(self) -> None:
await self.get_account_token(API_ENTRYPOINT)
diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.py
index df16a5998..bd7b060c0 100644
--- a/cyberdrop_dl/crawlers/imgur.py
+++ b/cyberdrop_dl/crawlers/imgur.py
@@ -31,6 +31,12 @@ class ImgurCrawler(Crawler):
self.imgur_client_remaining = 12500
self.headers = {"Authorization": f"Client-ID {self.imgur_client_id}"}
+ @classmethod
+ def _json_response_check(cls, json_resp: Any) -> None:
+ if not isinstance(json_resp, dict) or "data" not in json_resp:
+ return
+ raise ScrapeError(json_resp["status"], json_resp["data"]["error"])
+
async def fetch(self, scrape_item: ScrapeItem) -> None:
if scrape_item.url.host == "i.imgur.com":
return await self.handle_direct_link(scrape_item)
diff --git a/cyberdrop_dl/crawlers/mediafire.py b/cyberdrop_dl/crawlers/mediafire.py
index 2f5facd6d..e0b072e1b 100644
--- a/cyberdrop_dl/crawlers/mediafire.py
+++ b/cyberdrop_dl/crawlers/mediafire.py
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, auto_task_id
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.exceptions import MediaFireError, ScrapeError
+from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.utilities import error_handling_wrapper, is_blob_or_svg
@@ -45,6 +45,16 @@ class MediaFireCrawler(Crawler):
DOMAIN: ClassVar[str] = "mediafire"
SKIP_PRE_CHECK = True
+ @classmethod
+ def _json_response_check(cls, json_resp: Any) -> None:
+ if not isinstance(json_resp, dict) or "response" not in json_resp:
+ return
+ resp: dict[str, Any] = json_resp["response"]
+ if resp["result"] != "Success":
+ code: int = resp["error"]
+ ui_failure = f"MediaFire Error ({code})"
+ raise ScrapeError(ui_failure, resp["message"])
+
async def fetch(self, scrape_item: ScrapeItem) -> None:
if (
scrape_item.url.path == "/"
@@ -64,11 +74,7 @@ class MediaFireCrawler(Crawler):
async def _api_request(self, path: str, **params: Any) -> dict[str, Any]:
params["response_format"] = "json"
api_url = (_API_URL / path).with_query(params)
- resp: dict[str, Any] = (await self.request_json(api_url))["response"]
- # TODO: register with client manager
- if resp["result"] != "Success":
- raise MediaFireError(resp["error"], resp["message"])
- return resp
+ return (await self.request_json(api_url))["response"]
async def _iter_folder_content(self, folder_key: str, content_type: str) -> AsyncGenerator[list[dict[str, Any]]]:
async def get_content(chunk: int) -> dict[str, Any]:
diff --git a/cyberdrop_dl/exceptions.py b/cyberdrop_dl/exceptions.py
index 4365b2c04..47ca8cd6e 100644
--- a/cyberdrop_dl/exceptions.py
+++ b/cyberdrop_dl/exceptions.py
@@ -163,15 +163,6 @@ class DurationError(CDLBaseError):
super().__init__(ui_failure, origin=origin)
-class MediaFireError(CDLBaseError):
- def __init__(
- self, code: int, message: str | None = None, origin: ScrapeItem | MediaItem | URL | None = None
- ) -> None:
- """This error will be thrown when a scrape fails."""
- ui_failure = f"MediaFire Error ({code})"
- super().__init__(ui_failure, message=message, status=code, origin=origin)
-
-
class RealDebridError(CDLBaseError):
"""Base RealDebrid API error."""
diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.py
index 9002096b4..332c5a4ba 100644
--- a/cyberdrop_dl/managers/client_manager.py
+++ b/cyberdrop_dl/managers/client_manager.py
@@ -39,7 +39,7 @@ _VALID_EXTENSIONS = (
)
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Generator, Iterable, Mapping
+ from collections.abc import AsyncGenerator, Callable, Generator, Iterable, Mapping
from http.cookies import BaseCookie
from aiohttp_client_cache.response import CachedResponse
@@ -171,6 +171,7 @@ class ClientManager:
self._session: CachedSession
self._download_session: aiohttp.ClientSession
self._curl_session: AsyncSession[CurlResponse]
+ self._json_response_checks: dict[str, Callable[[Any], None]] = {}
def _startup(self) -> None:
self._session = self.new_scrape_session()
@@ -367,9 +368,8 @@ class ClientManager:
return self.rate_limits[domain]
return self.rate_limits["other"]
- @classmethod
async def check_http_status(
- cls,
+ self,
response: ClientResponse | CachedResponse | CurlResponse | AbstractResponse,
download: bool = False,
) -> BeautifulSoup | None:
@@ -390,48 +390,39 @@ class ClientManager:
async def check_ddos_guard() -> BeautifulSoup | None:
if "html" not in response.content_type:
return
-
- # TODO: use the response text instead of the raw content to prevent double encoding detection
-
try:
soup = BeautifulSoup(await response.text(), "html.parser")
except UnicodeDecodeError:
return
else:
- if cls.check_ddos_guard(soup) or cls.check_cloudflare(soup):
+ if self.check_ddos_guard(soup) or self.check_cloudflare(soup):
raise DDOSGuardError
return soup
- async def check_json_status() -> None:
- if "json" not in response.content_type:
- return
-
- # TODO: Define these checks inside their actual crawlers
- # and make them register them on instantation
- if not any(domain in response.url.host for domain in ("gofile", "imgur")):
- return
-
- json_resp: dict[str, Any] | None = await response.json()
- if not json_resp:
- return
-
- json_status: str | int | None = json_resp.get("status")
- if json_status and isinstance(json_status, str) and "notFound" in json_status:
- raise ScrapeError(404)
-
- if (data := json_resp.get("data")) and isinstance(data, dict) and "error" in data:
- raise ScrapeError(json_status or response.status, data["error"])
-
check_etag()
if HTTPStatus.OK <= response.status < HTTPStatus.BAD_REQUEST:
# Check DDosGuard even on successful pages
# await check_ddos_guard()
return
- await check_json_status()
+ await self._check_json(response)
await check_ddos_guard()
raise DownloadError(status=response.status, message=message)
+ async def _check_json(self, response: AbstractResponse) -> None:
+ if "json" not in response.content_type:
+ return
+
+ if check := self._json_response_checks.get(response.url.host):
+ check(await response.json())
+ return
+
+ for domain, check in self._json_response_checks.items():
+ if domain in response.url.host:
+ self._json_response_checks[response.url.host] = check
+ check(await response.json())
+ return
+
@staticmethod
def check_content_length(headers: Mapping[str, Any]) -> None:
content_length, content_type = headers.get("Content-Length"), headers.get("Content-Type")