Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: parse CF token from Flaresolverr (#2141)
NTFSvolume3c74193dbc36
diff --git a/.zed/debug.json b/.zed/debug.jsonindex 887c04e1b..ffb102b1a 100644--- a/.zed/debug.json+++ b/.zed/debug.json@@ -18,6 +18,7 @@"2","--flaresolverr","http://localhost:8191",+ "--no-flaresolverr-use-session","--input-file","URLs.txt","--log-file",diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.pyindex 6210c9946..f370d6a4e 100644--- a/cyberdrop_dl/clients/flaresolverr.py+++ b/cyberdrop_dl/clients/flaresolverr.py@@ -57,6 +57,7 @@ class Solution:url: AbsoluteHttpURLuser_agent: strstatus: int+ turnstile_token: str | None = Noneid: str = dataclasses.field(init=False, default="")@staticmethod@@ -68,6 +69,7 @@ class Solution:content=solution["response"],url=AbsoluteHttpURL(solution["url"]),headers=CIMultiDictProxy(CIMultiDict(solution["headers"])),+ turnstile_token=solution.get("turnstile_token"),)diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.pyindex ea2f35c95..de3ca3ebb 100644--- a/cyberdrop_dl/clients/http.py+++ b/cyberdrop_dl/clients/http.py@@ -15,7 +15,7 @@ from aiohttp import hdrsfrom cyberdrop_dl import aio, cookies, ddos_guardfrom cyberdrop_dl.clients import curl_cffi, flaresolverr, get_logger, tcp, wreqfrom cyberdrop_dl.clients.request import Request, RequestParams-from cyberdrop_dl.clients.response import AbstractResponse+from cyberdrop_dl.clients.response import AbstractResponse, FlareSolverrResponsefrom cyberdrop_dl.cookies import make_simple_cookiefrom cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeErrorfrom cyberdrop_dl.signature import simple_repr@@ -374,7 +374,7 @@ class HTTPClient:self,url: AbsoluteHttpURL,**params: Unpack[flaresolverr.RequestParams],- ) -> AbstractResponse[Any]:+ ) -> FlareSolverrResponse:flare = self.flaresolverrif not flare:raise ScrapeError(@@ -388,7 +388,7 @@ class HTTPClient:url: AbsoluteHttpURL,/,**params: Unpack[flaresolverr.RequestParams],- ) -> AbstractResponse[Any]:+ ) -> FlareSolverrResponse:"""Make a request with FlareSolverr.Returns an AbstractResponse confirmed to not be a DDOS Guard page, even if flaresolverr fails to detect/solve a challenge"""@@ -401,7 +401,7 @@ class HTTPClient:flaresolverr.verify_solution(self.config.network.user_agent, solution)self._use_flaresolverr_ua.add(url.host)self._flaresolverr_ua = flaresolverr.USER_AGENT.get()- return AbstractResponse.create(solution)+ return FlareSolverrResponse.create(solution)async def _check_json(response: AbstractResponse[Any]) -> None:@@ -462,7 +462,7 @@ class HTTPMixin(HTTPController, Protocol):self,url: AbsoluteHttpURL,**kwargs: Unpack[flaresolverr.RequestParams],- ) -> AbstractResponse[Any]:+ ) -> FlareSolverrResponse:async with self.rate_limit_ctx():return await self.client.flaresolverr_request(url, **kwargs)diff --git a/cyberdrop_dl/clients/response.py b/cyberdrop_dl/clients/response.pyindex 95ce6739e..5f8149d7d 100644--- a/cyberdrop_dl/clients/response.py+++ b/cyberdrop_dl/clients/response.py@@ -150,11 +150,11 @@ class AbstractResponse(ABC, Generic[_ResponseT]):async def aclose(self) -> None: ...@classmethod- def create(cls, resp: _ResponseT, /) -> _AIOHTTPResponse | _FlareSolverrResponse | _CurlResponse | _WreqResponse:+ def create(cls, resp: _ResponseT, /) -> _AIOHTTPResponse | FlareSolverrResponse | _CurlResponse | _WreqResponse:try:cls_ = {ClientResponse: _AIOHTTPResponse,- FlaresolverrSolution: _FlareSolverrResponse,+ FlaresolverrSolution: FlareSolverrResponse,CurlResponse: _CurlResponse,wreq.Response: _WreqResponse,}[type(resp)]@@ -376,9 +376,13 @@ class _WreqResponse(AbstractResponse[wreq.Response]):)-class _FlareSolverrResponse(AbstractResponse[FlaresolverrSolution]):+class FlareSolverrResponse(AbstractResponse[FlaresolverrSolution]):__slots__ = ()+ @property+ def solution(self) -> FlaresolverrSolution:+ return self._resp+def __post_init__(self) -> None:super().__post_init__()self.id: str = self._resp.iddiff --git a/tests/test_flaresolverr_response.py b/tests/test_flaresolverr_response.pyindex 152047341..e89fb2bc6 100644--- a/tests/test_flaresolverr_response.py+++ b/tests/test_flaresolverr_response.py@@ -4,7 +4,7 @@ import timefrom typing import TYPE_CHECKING, Anyfrom cyberdrop_dl.clients.flaresolverr import Solution, _parse_cookies-from cyberdrop_dl.clients.response import _FlareSolverrResponse, _infer_content_type_from_body+from cyberdrop_dl.clients.response import FlareSolverrResponse, _infer_content_type_from_bodyif TYPE_CHECKING:import pytest@@ -264,7 +264,7 @@ def test_solution_from_dict_json_resp() -> None:def test_flaresolverr_response_infers_html_from_empty_headers(logs: pytest.LogCaptureFixture) -> None:"""When FlareSolverr returns empty headers, content-type should be inferred from the body."""solution = Solution.from_dict(FLARESOLVERR_RESPONSE_EMPTY_HEADERS["solution"])- response = _FlareSolverrResponse.create(solution)+ response = FlareSolverrResponse.create(solution)assert response.content_type == "text/html"assert response.status == 200assert response.location is None@@ -273,14 +273,14 @@ def test_flaresolverr_response_infers_html_from_empty_headers(logs: pytest.LogCaasync def test_flaresolverr_response_reads_text() -> None:solution = Solution.from_dict(FLARESOLVERR_RESPONSE_EMPTY_HEADERS["solution"])- response = _FlareSolverrResponse.create(solution)+ response = FlareSolverrResponse.create(solution)text = await response.text()assert "<html>" in textasync def test_flaresolverr_response_from_json_resp() -> None:solution = Solution.from_dict(FLARESOLVER_RESP_JSON["solution"])- response = _FlareSolverrResponse.create(solution)+ response = FlareSolverrResponse.create(solution)assert not response._textassert response.content_type == "application/json"assert response._get_content() == solution.content@@ -296,7 +296,7 @@ def test_flaresolverr_response_with_explicit_content_type() -> None:"response": '{"data": True}',}solution = Solution.from_dict(solution_data)- response = _FlareSolverrResponse.create(solution)+ response = FlareSolverrResponse.create(solution)assert response.content_type == "application/json"@@ -308,7 +308,7 @@ def test_flaresolverr_response_empty_body_and_empty_headers(logs: pytest.LogCapt}solution = Solution.from_dict(solution_data)solution.id = "1"- response = _FlareSolverrResponse.create(solution)+ response = FlareSolverrResponse.create(solution)assert response.content_type == "text/html"assert len(logs.records) == 1assert (@@ -318,7 +318,7 @@ def test_flaresolverr_response_empty_body_and_empty_headers(logs: pytest.LogCaptasync def test_flaresolverr_response_from_json_resp_wrapped_in_html() -> None:solution = Solution.from_dict(FLARESOLVER_RESP_JSON_WRAPPED_IN_HTML["solution"])- resp = _FlareSolverrResponse.create(solution)+ resp = FlareSolverrResponse.create(solution)assert resp._textassert resp.content_type == "text/html"assert type(solution.content) is str