Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: don't disable flaresolverr when cloudflare blocks a request (#2182)

Michael Menanno18b3753374d9
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.py
index 9abf3f461..8310570fa 100644
--- a/cyberdrop_dl/clients/flaresolverr.py
+++ b/cyberdrop_dl/clients/flaresolverr.py
@@ -15,7 +15,7 @@ from multidict import CIMultiDict, CIMultiDictProxy
from cyberdrop_dl import ddos_guard
from cyberdrop_dl.clients import HttpMethod, get_logger
-from cyberdrop_dl.exceptions import DDOSGuardError, FlaresolverrError
+from cyberdrop_dl.exceptions import CloudflareBlockedError, DDOSGuardError, FlaresolverrError
from cyberdrop_dl.progress.scraping import show_msg
from cyberdrop_dl.signature import simple_repr
from cyberdrop_dl.url_objects import AbsoluteHttpURL
@@ -29,6 +29,7 @@ logger = get_logger(__name__)
USER_AGENT: ContextVar[str] = ContextVar("USER_AGENT")
MAX_TIMEOUT = 60_000
+_CF_BLOCKED_MSG = "Cloudflare has blocked this request"
class Command(StrEnum):
@@ -206,6 +207,8 @@ class Client:
except aiohttp.ClientError as e:
self.disable()
self.raise_conn_error(e)
+ except CloudflareBlockedError:
+ raise
except Exception:
self.disable()
raise
@@ -259,6 +262,8 @@ class Client:
req = build_request(url, self.config, params, self.session)
resp = await self._request(req.command, req.payload, **req.aiohttp_params)
if not resp.ok:
+ if _CF_BLOCKED_MSG in resp.message:
+ raise CloudflareBlockedError(resp.message)
raise FlaresolverrError(f"Failed to resolve URL with Flaresolverr. {resp.message}")
if not resp.solution:
diff --git a/cyberdrop_dl/exceptions.py b/cyberdrop_dl/exceptions.py
index 1cd16ed5e..195270744 100644
--- a/cyberdrop_dl/exceptions.py
+++ b/cyberdrop_dl/exceptions.py
@@ -155,6 +155,13 @@ class DDOSGuardError(CDLBaseError):
super().__init__(ui_failure, message=msg, origin=origin)
+class CloudflareBlockedError(DDOSGuardError):
+ def __init__(self, message: str | None = None, *, origin: HasParents | yarl.URL | None = None) -> None:
+ """This error will be thrown when Cloudflare refuses a request outright (ex: IP banned)."""
+ super().__init__(message or "Cloudflare blocked the request", origin=origin)
+ self.ui_failure = "Cloudflare Blocked"
+
+
class DownloadError(CDLBaseError):
def __init__(
self,
diff --git a/tests/test_flaresolverr.py b/tests/test_flaresolverr.py
index 18af9eea2..c36c11ae7 100644
--- a/tests/test_flaresolverr.py
+++ b/tests/test_flaresolverr.py
@@ -11,9 +11,11 @@ from cyberdrop_dl.clients.flaresolverr import (
Config,
Request,
RequestCommand,
+ Response,
_cmd_from_http_method,
_default_session_name,
)
+from cyberdrop_dl.exceptions import CloudflareBlockedError, DDOSGuardError, FlaresolverrError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
ENV_NAME = "CDL_FLARESOLVERR"
@@ -174,3 +176,36 @@ def test_request_build(
assert req.command is expected.command
assert req.payload == expected.payload
assert req.aiohttp_params == expected.aiohttp_params
+
+
+_CF_BLOCKED = (
+ "Error: Error solving the challenge. Cloudflare has blocked this request. "
+ "Probably your IP is banned for this site, check in your web browser."
+)
+
+
+ ("message", "expected_error", "disabled"),
+ [
+ (_CF_BLOCKED, CloudflareBlockedError, False),
+ ("Error: Error solving the challenge. Timeout after 60.0 seconds.", FlaresolverrError, True),
+ ],
+)
+async def test_request_error_response(
+ monkeypatch: pytest.MonkeyPatch, message: str, expected_error: type[Exception], *, disabled: bool
+) -> None:
+ async def fake_request(*_, **__) -> Response:
+ return Response.parse(1, {"status": "error", "message": message})
+
+ async with aiohttp.ClientSession() as session:
+ client = Client(session, _config(proxy=False, use_session=False))
+ monkeypatch.setattr(client, "_request", fake_request)
+ with pytest.raises(expected_error):
+ await client.request(AbsoluteHttpURL("https://example.com"))
+
+ assert client.is_down is disabled
+
+
+def test_cloudflare_blocked_is_ddos_guard_error() -> None:
+ assert issubclass(CloudflareBlockedError, DDOSGuardError)
+ assert CloudflareBlockedError().ui_failure == "Cloudflare Blocked"