Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: don't disable flaresolverr when cloudflare blocks a request (#2182)
Michael Menanno18b3753374d9
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.pyindex 9abf3f461..8310570fa 100644--- a/cyberdrop_dl/clients/flaresolverr.py+++ b/cyberdrop_dl/clients/flaresolverr.py@@ -15,7 +15,7 @@ from multidict import CIMultiDict, CIMultiDictProxyfrom cyberdrop_dl import ddos_guardfrom cyberdrop_dl.clients import HttpMethod, get_logger-from cyberdrop_dl.exceptions import DDOSGuardError, FlaresolverrError+from cyberdrop_dl.exceptions import CloudflareBlockedError, DDOSGuardError, FlaresolverrErrorfrom cyberdrop_dl.progress.scraping import show_msgfrom cyberdrop_dl.signature import simple_reprfrom cyberdrop_dl.url_objects import AbsoluteHttpURL@@ -29,6 +29,7 @@ logger = get_logger(__name__)USER_AGENT: ContextVar[str] = ContextVar("USER_AGENT")MAX_TIMEOUT = 60_000+_CF_BLOCKED_MSG = "Cloudflare has blocked this request"class Command(StrEnum):@@ -206,6 +207,8 @@ class Client:except aiohttp.ClientError as e:self.disable()self.raise_conn_error(e)+ except CloudflareBlockedError:+ raiseexcept Exception:self.disable()raise@@ -259,6 +262,8 @@ class Client:req = build_request(url, self.config, params, self.session)resp = await self._request(req.command, req.payload, **req.aiohttp_params)if not resp.ok:+ if _CF_BLOCKED_MSG in resp.message:+ raise CloudflareBlockedError(resp.message)raise FlaresolverrError(f"Failed to resolve URL with Flaresolverr. {resp.message}")if not resp.solution:diff --git a/cyberdrop_dl/exceptions.py b/cyberdrop_dl/exceptions.pyindex 1cd16ed5e..195270744 100644--- a/cyberdrop_dl/exceptions.py+++ b/cyberdrop_dl/exceptions.py@@ -155,6 +155,13 @@ class DDOSGuardError(CDLBaseError):super().__init__(ui_failure, message=msg, origin=origin)+class CloudflareBlockedError(DDOSGuardError):+ def __init__(self, message: str | None = None, *, origin: HasParents | yarl.URL | None = None) -> None:+ """This error will be thrown when Cloudflare refuses a request outright (ex: IP banned)."""+ super().__init__(message or "Cloudflare blocked the request", origin=origin)+ self.ui_failure = "Cloudflare Blocked"++class DownloadError(CDLBaseError):def __init__(self,diff --git a/tests/test_flaresolverr.py b/tests/test_flaresolverr.pyindex 18af9eea2..c36c11ae7 100644--- a/tests/test_flaresolverr.py+++ b/tests/test_flaresolverr.py@@ -11,9 +11,11 @@ from cyberdrop_dl.clients.flaresolverr import (Config,Request,RequestCommand,+ Response,_cmd_from_http_method,_default_session_name,)+from cyberdrop_dl.exceptions import CloudflareBlockedError, DDOSGuardError, FlaresolverrErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURLENV_NAME = "CDL_FLARESOLVERR"@@ -174,3 +176,36 @@ def test_request_build(assert req.command is expected.commandassert req.payload == expected.payloadassert req.aiohttp_params == expected.aiohttp_params+++_CF_BLOCKED = (+ "Error: Error solving the challenge. Cloudflare has blocked this request. "+ "Probably your IP is banned for this site, check in your web browser."+)+++ ("message", "expected_error", "disabled"),+ [+ (_CF_BLOCKED, CloudflareBlockedError, False),+ ("Error: Error solving the challenge. Timeout after 60.0 seconds.", FlaresolverrError, True),+ ],+)+async def test_request_error_response(+ monkeypatch: pytest.MonkeyPatch, message: str, expected_error: type[Exception], *, disabled: bool+) -> None:+ async def fake_request(*_, **__) -> Response:+ return Response.parse(1, {"status": "error", "message": message})++ async with aiohttp.ClientSession() as session:+ client = Client(session, _config(proxy=False, use_session=False))+ monkeypatch.setattr(client, "_request", fake_request)+ with pytest.raises(expected_error):+ await client.request(AbsoluteHttpURL("https://example.com"))++ assert client.is_down is disabled+++def test_cloudflare_blocked_is_ddos_guard_error() -> None:+ assert issubclass(CloudflareBlockedError, DDOSGuardError)+ assert CloudflareBlockedError().ui_failure == "Cloudflare Blocked"