Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: parse CF token from Flaresolverr (#2141)

NTFSvolume3c74193dbc36
diff --git a/.zed/debug.json b/.zed/debug.json
index 887c04e1b..ffb102b1a 100644
--- a/.zed/debug.json
+++ b/.zed/debug.json
@@ -18,6 +18,7 @@
"2",
"--flaresolverr",
"http://localhost:8191",
+ "--no-flaresolverr-use-session",
"--input-file",
"URLs.txt",
"--log-file",
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.py
index 6210c9946..f370d6a4e 100644
--- a/cyberdrop_dl/clients/flaresolverr.py
+++ b/cyberdrop_dl/clients/flaresolverr.py
@@ -57,6 +57,7 @@ class Solution:
url: AbsoluteHttpURL
user_agent: str
status: int
+ turnstile_token: str | None = None
id: str = dataclasses.field(init=False, default="")
@staticmethod
@@ -68,6 +69,7 @@ class Solution:
content=solution["response"],
url=AbsoluteHttpURL(solution["url"]),
headers=CIMultiDictProxy(CIMultiDict(solution["headers"])),
+ turnstile_token=solution.get("turnstile_token"),
)
diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.py
index ea2f35c95..de3ca3ebb 100644
--- a/cyberdrop_dl/clients/http.py
+++ b/cyberdrop_dl/clients/http.py
@@ -15,7 +15,7 @@ from aiohttp import hdrs
from cyberdrop_dl import aio, cookies, ddos_guard
from cyberdrop_dl.clients import curl_cffi, flaresolverr, get_logger, tcp, wreq
from cyberdrop_dl.clients.request import Request, RequestParams
-from cyberdrop_dl.clients.response import AbstractResponse
+from cyberdrop_dl.clients.response import AbstractResponse, FlareSolverrResponse
from cyberdrop_dl.cookies import make_simple_cookie
from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError
from cyberdrop_dl.signature import simple_repr
@@ -374,7 +374,7 @@ class HTTPClient:
self,
url: AbsoluteHttpURL,
**params: Unpack[flaresolverr.RequestParams],
- ) -> AbstractResponse[Any]:
+ ) -> FlareSolverrResponse:
flare = self.flaresolverr
if not flare:
raise ScrapeError(
@@ -388,7 +388,7 @@ class HTTPClient:
url: AbsoluteHttpURL,
/,
**params: Unpack[flaresolverr.RequestParams],
- ) -> AbstractResponse[Any]:
+ ) -> FlareSolverrResponse:
"""Make a request with FlareSolverr.
Returns an AbstractResponse confirmed to not be a DDOS Guard page, even if flaresolverr fails to detect/solve a challenge"""
@@ -401,7 +401,7 @@ class HTTPClient:
flaresolverr.verify_solution(self.config.network.user_agent, solution)
self._use_flaresolverr_ua.add(url.host)
self._flaresolverr_ua = flaresolverr.USER_AGENT.get()
- return AbstractResponse.create(solution)
+ return FlareSolverrResponse.create(solution)
async def _check_json(response: AbstractResponse[Any]) -> None:
@@ -462,7 +462,7 @@ class HTTPMixin(HTTPController, Protocol):
self,
url: AbsoluteHttpURL,
**kwargs: Unpack[flaresolverr.RequestParams],
- ) -> AbstractResponse[Any]:
+ ) -> FlareSolverrResponse:
async with self.rate_limit_ctx():
return await self.client.flaresolverr_request(url, **kwargs)
diff --git a/cyberdrop_dl/clients/response.py b/cyberdrop_dl/clients/response.py
index 95ce6739e..5f8149d7d 100644
--- a/cyberdrop_dl/clients/response.py
+++ b/cyberdrop_dl/clients/response.py
@@ -150,11 +150,11 @@ class AbstractResponse(ABC, Generic[_ResponseT]):
async def aclose(self) -> None: ...
@classmethod
- def create(cls, resp: _ResponseT, /) -> _AIOHTTPResponse | _FlareSolverrResponse | _CurlResponse | _WreqResponse:
+ def create(cls, resp: _ResponseT, /) -> _AIOHTTPResponse | FlareSolverrResponse | _CurlResponse | _WreqResponse:
try:
cls_ = {
ClientResponse: _AIOHTTPResponse,
- FlaresolverrSolution: _FlareSolverrResponse,
+ FlaresolverrSolution: FlareSolverrResponse,
CurlResponse: _CurlResponse,
wreq.Response: _WreqResponse,
}[type(resp)]
@@ -376,9 +376,13 @@ class _WreqResponse(AbstractResponse[wreq.Response]):
)
-class _FlareSolverrResponse(AbstractResponse[FlaresolverrSolution]):
+class FlareSolverrResponse(AbstractResponse[FlaresolverrSolution]):
__slots__ = ()
+ @property
+ def solution(self) -> FlaresolverrSolution:
+ return self._resp
+
def __post_init__(self) -> None:
super().__post_init__()
self.id: str = self._resp.id
diff --git a/tests/test_flaresolverr_response.py b/tests/test_flaresolverr_response.py
index 152047341..e89fb2bc6 100644
--- a/tests/test_flaresolverr_response.py
+++ b/tests/test_flaresolverr_response.py
@@ -4,7 +4,7 @@ import time
from typing import TYPE_CHECKING, Any
from cyberdrop_dl.clients.flaresolverr import Solution, _parse_cookies
-from cyberdrop_dl.clients.response import _FlareSolverrResponse, _infer_content_type_from_body
+from cyberdrop_dl.clients.response import FlareSolverrResponse, _infer_content_type_from_body
if TYPE_CHECKING:
import pytest
@@ -264,7 +264,7 @@ def test_solution_from_dict_json_resp() -> None:
def test_flaresolverr_response_infers_html_from_empty_headers(logs: pytest.LogCaptureFixture) -> None:
"""When FlareSolverr returns empty headers, content-type should be inferred from the body."""
solution = Solution.from_dict(FLARESOLVERR_RESPONSE_EMPTY_HEADERS["solution"])
- response = _FlareSolverrResponse.create(solution)
+ response = FlareSolverrResponse.create(solution)
assert response.content_type == "text/html"
assert response.status == 200
assert response.location is None
@@ -273,14 +273,14 @@ def test_flaresolverr_response_infers_html_from_empty_headers(logs: pytest.LogCa
async def test_flaresolverr_response_reads_text() -> None:
solution = Solution.from_dict(FLARESOLVERR_RESPONSE_EMPTY_HEADERS["solution"])
- response = _FlareSolverrResponse.create(solution)
+ response = FlareSolverrResponse.create(solution)
text = await response.text()
assert "<html>" in text
async def test_flaresolverr_response_from_json_resp() -> None:
solution = Solution.from_dict(FLARESOLVER_RESP_JSON["solution"])
- response = _FlareSolverrResponse.create(solution)
+ response = FlareSolverrResponse.create(solution)
assert not response._text
assert response.content_type == "application/json"
assert response._get_content() == solution.content
@@ -296,7 +296,7 @@ def test_flaresolverr_response_with_explicit_content_type() -> None:
"response": '{"data": True}',
}
solution = Solution.from_dict(solution_data)
- response = _FlareSolverrResponse.create(solution)
+ response = FlareSolverrResponse.create(solution)
assert response.content_type == "application/json"
@@ -308,7 +308,7 @@ def test_flaresolverr_response_empty_body_and_empty_headers(logs: pytest.LogCapt
}
solution = Solution.from_dict(solution_data)
solution.id = "1"
- response = _FlareSolverrResponse.create(solution)
+ response = FlareSolverrResponse.create(solution)
assert response.content_type == "text/html"
assert len(logs.records) == 1
assert (
@@ -318,7 +318,7 @@ def test_flaresolverr_response_empty_body_and_empty_headers(logs: pytest.LogCapt
async def test_flaresolverr_response_from_json_resp_wrapped_in_html() -> None:
solution = Solution.from_dict(FLARESOLVER_RESP_JSON_WRAPPED_IN_HTML["solution"])
- resp = _FlareSolverrResponse.create(solution)
+ resp = FlareSolverrResponse.create(solution)
assert resp._text
assert resp.content_type == "text/html"
assert type(solution.content) is str