Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: handle malformed JSON wrapped in HTML (Flaresolverr) (#2094)
NTFSvolumea2c78081bf37
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 3a0763354..a7cf39386 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Added- New `--logs.http-traffic` option+- New `--flaresolverr-use-session` option- Support direct local connection to JDownloader using their deprecated API- `wreq` as optional HTTP backend for browser impersonation (via the `wreq` extra)@@ -38,6 +39,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0(sorting, hashing, send notifications, etc..). Press `Ctrl + C` again to force quit- Reduced the maximum size the log file can have to be sent as attachment in notifications, from 25 MB to 20 MB.+### Fixed++- Handle malformed JSON responses wrapped in HTML (Flaresolverr)+## [10.5.0] - 2026-08-15### Addeddiff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.pyindex 69d7d9f01..5625af257 100644--- a/cyberdrop_dl/clients/flaresolverr.py+++ b/cyberdrop_dl/clients/flaresolverr.py@@ -1,7 +1,6 @@from __future__ import annotationsimport asyncio-import contextlibimport dataclassesimport itertoolsimport time@@ -20,7 +19,7 @@ from cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import truncated_previewif TYPE_CHECKING:- from collections.abc import Callable, Generator, Iterable, Mapping+ from collections.abc import Callable, Iterable, Mappinglogger = get_logger(__name__)@@ -43,6 +42,7 @@ class Solution:url: AbsoluteHttpURLuser_agent: strstatus: int+ id: str = dataclasses.field(init=False, default="")@staticmethoddef from_dict(solution: Mapping[str, Any]) -> Solution:@@ -104,6 +104,7 @@ class Client:url: AbsoluteHttpURL_aiohttp_session: aiohttp.ClientSession = dataclasses.field(repr=False)+ use_session: bool_session_id: str = dataclasses.field(init=False, default="")_session_lock: asyncio.Lock = dataclasses.field(init=False, default_factory=asyncio.Lock)@@ -126,21 +127,19 @@ class Client:except Exception as e: # noqa: BLE001logger.error(f"Unable to destroy flaresolver session ({e}!r)")- @contextlib.contextmanager- def _disable_on_error(self) -> Generator[None]:- try:- yield- except Exception:- if not self._down:- self._down = True- logger.warning("Flaresolverr has been disabled")- raise+ def disable(self) -> None:+ if not self._down:+ self._down = True+ logger.warning("Flaresolverr has been disabled")async def _ensure_session(self) -> None:msg = "Unable to create Flaresolverr session"if self._down:raise FlaresolverrError(msg)+ if not self.use_session:+ return+if self._session_id:return@@ -149,25 +148,32 @@ class Client:returntry:- with self._disable_on_error():- await self._create_session()- except aiohttp.ClientConnectionError as e:+ await self._create_session()+ except aiohttp.ClientError as e:+ self.disable()raise FlaresolverrError(f"Could not connect to Flaresolverr at {self.url} ({e!r})") from None+ except FlaresolverrError:+ self.disable()+ raiseexcept Exception as e:+ self.disable()raise FlaresolverrError(msg) from easync def request(self, url: AbsoluteHttpURL, data: dict[str, Any] | None = None) -> Solution:await self._ensure_session()try:- resp = await self._request(- Command.POST_REQUEST if data else Command.GET_REQUEST,- url=str(url),- data=data,- session=self._session_id,- )-- except (TypeError, KeyError) as e:- raise FlaresolverrError("Invalid response from Flaresolverr") from e+ return await self.raw_request(url, data)+ except aiohttp.ClientError as e:+ self.disable()+ raise FlaresolverrError(f"Could not connect to Flaresolverr at {self.url} ({e!r})") from None++ async def raw_request(self, url: AbsoluteHttpURL, data: dict[str, Any] | None = None) -> Solution:+ resp = await self._request(+ Command.POST_REQUEST if data else Command.GET_REQUEST,+ url=str(url),+ data=data,+ session=self._session_id,+ )if not resp.ok:raise FlaresolverrError(f"Failed to resolve URL with Flaresolverr. {resp.message}")@@ -184,6 +190,8 @@ class Client:# timeout in milliseconds (60s)params = {"cmd": str(command), "maxTimeout": 60_000} | params+ if not self.use_session:+ params.pop("session", None)if data:assert command is Command.POST_REQUEST@@ -203,6 +211,10 @@ class Client:try:resp = Response.from_dict(resp_json)resp.id = str(request_id)+ if resp.solution:+ resp.solution.id = resp.id+ except (TypeError, KeyError) as e:+ raise FlaresolverrError("Invalid response from Flaresolverr") from efinally:logger.traffic("Finished FlareSolverr request [id=%s]\n%s", request_id, _LazyResponseLog(resp_json)diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.pyindex 9ef0a5c34..87389f1e9 100644--- a/cyberdrop_dl/clients/http.py+++ b/cyberdrop_dl/clients/http.py@@ -139,7 +139,11 @@ class HTTPClient:@propertydef flaresolverr(self) -> flaresolverr.Client | None:if self._flaresolverr is None and (url := self.config.network.flaresolverr):- self._flaresolverr = flaresolverr.Client(url, self._session)+ self._flaresolverr = flaresolverr.Client(+ url,+ self._session,+ use_session=self.config.network.flaresolverr_use_session,+ )if self._flaresolverr and self._flaresolverr.is_down:return Nonereturn self._flaresolverrdiff --git a/cyberdrop_dl/clients/response.py b/cyberdrop_dl/clients/response.pyindex af4c82bb9..9d6452100 100644--- a/cyberdrop_dl/clients/response.py+++ b/cyberdrop_dl/clients/response.py@@ -1,10 +1,10 @@from __future__ import annotationsimport asyncio+import copyimport dataclassesimport datetimeimport json-import loggingfrom abc import ABC, abstractmethodfrom enum import StrEnumfrom types import MappingProxyType@@ -17,7 +17,7 @@ from multidict import CIMultiDict, CIMultiDictProxyfrom propcache import under_cached_propertyfrom typing_extensions import TypeVar-from cyberdrop_dl.clients import wreq+from cyberdrop_dl.clients import get_logger, wreqfrom cyberdrop_dl.clients.flaresolverr import Solution as FlaresolverrSolutionfrom cyberdrop_dl.exceptions import InvalidContentTypeError, ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL@@ -35,7 +35,7 @@ else:class CurlResponse: ...-logger = logging.getLogger(__name__)+logger = get_logger(__name__)_ResponseT = TypeVar("_ResponseT",@@ -90,6 +90,8 @@ class AbstractResponse(ABC, Generic[_ResponseT]):default_factory=lambda: datetime.datetime.now(datetime.UTC).replace(microsecond=0),)+ def __post_init__(self) -> None: ...+def __repr__(self) -> str:return f"<{type(self).__name__} [{self.status}] ({self.url})>"@@ -210,25 +212,27 @@ class AbstractResponse(ABC, Generic[_ResponseT]):raise ScrapeError(204, "Received empty HTML response")- @finalasync def json(self,encoding: str | None = None,content_type: tuple[str, ...] | str | Literal[False] | None = ("text/plain", "json"),) -> Any:+ self._check_json(content_type)+ return json.loads(await self.text(encoding))++ def _check_json(+ self,+ content_type: tuple[str, ...] | str | Literal[False] | None = ("text/plain", "json"),+ ) -> None:if self.status == 204:raise ScrapeError(204)- if content_type:- if isinstance(content_type, str):- content_type = (content_type,)-- self.__check_content_type(*content_type, expecting="JSON")+ if not content_type:+ return+ if isinstance(content_type, str):+ content_type = (content_type,)- return await self._json(encoding)-- async def _json(self, encoding: str | None = None) -> Any:- return json.loads(await self.text(encoding))+ self.__check_content_type(*content_type, expecting="JSON")@finaldef create_report(self, exc: Exception | None = None, **extras: Any) -> str:@@ -256,6 +260,10 @@ class AbstractResponse(ABC, Generic[_ResponseT]):class _FlareSolverrResponse(AbstractResponse[FlaresolverrSolution]):__slots__ = ()+ def __post_init__(self) -> None:+ super().__post_init__()+ self.id: str = self._resp.id+@overrideasync def _read(self) -> bytes:return self._text.encode()@@ -271,13 +279,39 @@ class _FlareSolverrResponse(AbstractResponse[FlaresolverrSolution]):@overrideasync def aclose(self) -> None: ...- @override- async def _json(self, encoding: str | None = None) -> Any:- if self._text:- return json.loads(self._text)+ async def json(+ self,+ encoding: str | None = None, # noqa: ARG002+ content_type: tuple[str, ...] | str | Literal[False] | None = ("text/plain", "json"),+ ) -> Any:+ if not self._text:+ # Resp content is already parsed JSON+ assert "json" in self.content_type+ return self._resp.content++ try:+ return self._load_json(self._text)+ finally:+ self._check_json(content_type)- assert "json" in self.content_type- return self._resp.content+ def _load_json(self, text: str) -> Any:+ try:+ return json.loads(text)+ except ValueError:+ if "html" not in self.content_type:+ raise+ text = BeautifulSoup(text, "html.parser").text+ data = json.loads(text)+ self.content_type = "application/json"+ self._text = text+ self._resp.content = copy.deepcopy(data)+ logger.warning(+ "Detected wrapped JSON in Flaresolverr response [id=%s], overriding content type to '%s'",+ self.id,+ self.content_type,+ )+ logger.traffic("Content from Flaresolverr request [id=%s]\n%s", self.id, {"content": self._resp.content})+ return datadef _get_content(self) -> Any:return super()._get_content() or self._resp.contentdiff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yamlindex 255e01cbd..2db12c4ac 100644--- a/cyberdrop_dl/config/default.yaml+++ b/cyberdrop_dl/config/default.yaml@@ -156,6 +156,7 @@ network:connection_timeout: 15.0dump_responses: falseflaresolverr: null+ flaresolverr_use_session: trueimpersonate: nullproxy: nullrate_limit: 25.0diff --git a/cyberdrop_dl/config/schema.json b/cyberdrop_dl/config/schema.jsonindex cbaa25571..6f7fd8e5c 100644--- a/cyberdrop_dl/config/schema.json+++ b/cyberdrop_dl/config/schema.json@@ -797,6 +797,11 @@"$ref": "#/$defs/FalsyAsNone_HttpURL_","default": null},+ "flaresolverr_use_session": {+ "default": true,+ "title": "Flaresolverr Use Session",+ "type": "boolean"+ },"proxy": {"$ref": "#/$defs/FalsyAsNone_HttpURL_","default": nulldiff --git a/cyberdrop_dl/config/settings.py b/cyberdrop_dl/config/settings.pyindex 2350803eb..dea4fd957 100644--- a/cyberdrop_dl/config/settings.py+++ b/cyberdrop_dl/config/settings.py@@ -343,6 +343,9 @@ class Network(ConfigGroup):flaresolverr: FalsyAsNone[HttpURL] = None"HTTP URL of an existing flaresolverr instance"+ flaresolverr_use_session: bool = True+ "Create a custom session before making any request with flaresolverr"+proxy: Annotated[FalsyAsNone[HttpURL], Parameter(alias=("http-proxy"))] = None"HTTP/HTTPS proxy"diff --git a/docs/reference/config/network.md b/docs/reference/config/network.mdindex 966ec96be..9c0350c12 100644--- a/docs/reference/config/network.md+++ b/docs/reference/config/network.md@@ -70,6 +70,17 @@ This wiki does not cover flaresolverr setup process. If you need help, refer toSee: [How to extract cookies (DDoSGuard or login errors) #839](https://github.com/Cyberdrop-DL/cyberdrop-dl/discussions/839) for alternatives using cookies{% endhint %}+# `flaresolverr_use_session`++| Type | Default |+| ------ | ------- |+| `bool` | `true` |++Create a custom flaresolverr session that keeps cookies. This reduces the likelihood of CF challenges and speeds up requests since Flaresolverr won't+have to launch a new browser instance on every new one.++Set this to `false` if you are using a Flaresolverr fork that does not support the `sessions.create` command+# `proxy`| Type | Default |@@ -193,11 +204,13 @@ If you use flaresolverr, this value **MUST** match with flaresolverr's user agenThese crawlers will ignore custom user-agents and will always use `cyberdrop-dl/<version>`<!-- START_CUSTOM_UA_CRAWLERS -->+- Archive.org- E621- MegaNz- RealDebrid- Transfer.it+<!-- END_CUSTOM_UA_CRAWLERS -->{% endhint %}diff --git a/tests/test_flaresolverr.py b/tests/test_flaresolverr.pyindex c938c6f4c..94f216494 100644--- a/tests/test_flaresolverr.py+++ b/tests/test_flaresolverr.py@@ -16,7 +16,11 @@ pytestmark = pytest.mark.skipif(not FLARESOLVER_URL, reason=f"{ENV_NAME} environ@pytest.fixtureasync def flaresolverr() -> AsyncGenerator[Client]:async with aiohttp.ClientSession() as session:- yield Client(AbsoluteHttpURL(FLARESOLVER_URL) / "v1", session)+ yield Client(+ AbsoluteHttpURL(FLARESOLVER_URL) / "v1",+ session,+ use_session=True,+ )def test_flaresolver(flaresolverr: Client) -> None:diff --git a/tests/test_flaresolverr_response.py b/tests/test_flaresolverr_response.pyindex 0ef14b581..c18957832 100644--- a/tests/test_flaresolverr_response.py+++ b/tests/test_flaresolverr_response.py@@ -1,6 +1,7 @@from __future__ import annotationsimport time+from typing import Anyfrom cyberdrop_dl.clients.flaresolverr import Solution, _parse_cookiesfrom cyberdrop_dl.clients.response import _FlareSolverrResponse, _infer_content_type_from_body@@ -9,7 +10,7 @@ from cyberdrop_dl.clients.response import _FlareSolverrResponse, _infer_content_# Fixtures: example FlareSolverr JSON responses# ----------------------------------------------------------------------------FLARESOLVERR_RESPONSE_EMPTY_HEADERS = {+FLARESOLVERR_RESPONSE_EMPTY_HEADERS: dict[str, Any] = {"status": "ok","message": "Challenge solved!","solution": {@@ -36,7 +37,34 @@ FLARESOLVERR_RESPONSE_EMPTY_HEADERS = {"version": "3.4.6",}-FLARESOLVER_RESP_JSON = {+FLARESOLVER_RESP_JSON_WRAPPED_IN_HTML: dict[str, Any] = {+ "status": "ok",+ "message": "Challenge solved!",+ "solution": {+ "url": "https://www.tikwm.com/api/user/posts?unique_id=user_embongngo&count=50&cursor=0",+ "status": 200,+ "cookies": [+ {+ "domain": ".tikwm.com",+ "expiry": 1819086364,+ "httpOnly": True,+ "name": "cf_clearance",+ "path": "/",+ "sameSite": "None",+ "secure": True,+ "value": "uOF3vEc7QtZfPUB28qZfCy.GKmByEgAyDpoyO9EYfSU-1787550364-1.2.1.1-HzZDz",+ }+ ],+ "userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",+ "headers": {},+ "response": '<html><head><meta name="color-scheme" content="light dark"><meta charset="utf-8"></head><body><pre>{"code":0,"msg":"success","processed_time":0.4496,"data":{}}</pre><div class="json-formatter-container"></div></body></html>',+ },+ "startTimestamp": 1787550355291,+ "endTimestamp": 1787550367316,+ "version": "3.4.6",+}++FLARESOLVER_RESP_JSON: dict[str, Any] = {"status": "ok","message": "Challenge not detected!","solution": {@@ -277,3 +305,25 @@ async def test_flaresolverr_response_empty_body_and_empty_headers() -> None:solution = Solution.from_dict(solution_data)response = _FlareSolverrResponse.create(solution)assert response.content_type == ""+++async def test_flaresolverr_response_from_json_resp_wrapped_in_html() -> None:+ solution = Solution.from_dict(FLARESOLVER_RESP_JSON_WRAPPED_IN_HTML["solution"])+ resp = _FlareSolverrResponse.create(solution)+ assert resp._text+ assert resp.content_type == "text/html"+ assert type(solution.content) is str+ data = await resp.json()+ assert type(data) is dict+ assert resp.content_type == "application/json"+ assert data == await resp.json()+ assert data is not await resp.json()+ assert type(solution.content) is dict+ assert data == {+ "code": 0,+ "msg": "success",+ "processed_time": 0.4496,+ "data": {},+ }+ data["extra_key"] = 0+ assert data != await resp.json()