Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: handle malformed JSON wrapped in HTML (Flaresolverr) (#2094)

NTFSvolumea2c78081bf37
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3a0763354..a7cf39386 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- New `--logs.http-traffic` option
+- New `--flaresolverr-use-session` option
- Support direct local connection to JDownloader using their deprecated API
- `wreq` as optional HTTP backend for browser impersonation (via the `wreq` extra)
@@ -38,6 +39,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
(sorting, hashing, send notifications, etc..). Press `Ctrl + C` again to force quit
- Reduced the maximum size the log file can have to be sent as attachment in notifications, from 25 MB to 20 MB.
+### Fixed
+
+- Handle malformed JSON responses wrapped in HTML (Flaresolverr)
+
## [10.5.0] - 2026-08-15
### Added
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.py
index 69d7d9f01..5625af257 100644
--- a/cyberdrop_dl/clients/flaresolverr.py
+++ b/cyberdrop_dl/clients/flaresolverr.py
@@ -1,7 +1,6 @@
from __future__ import annotations
import asyncio
-import contextlib
import dataclasses
import itertools
import time
@@ -20,7 +19,7 @@ from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import truncated_preview
if TYPE_CHECKING:
- from collections.abc import Callable, Generator, Iterable, Mapping
+ from collections.abc import Callable, Iterable, Mapping
logger = get_logger(__name__)
@@ -43,6 +42,7 @@ class Solution:
url: AbsoluteHttpURL
user_agent: str
status: int
+ id: str = dataclasses.field(init=False, default="")
@staticmethod
def from_dict(solution: Mapping[str, Any]) -> Solution:
@@ -104,6 +104,7 @@ class Client:
url: AbsoluteHttpURL
_aiohttp_session: aiohttp.ClientSession = dataclasses.field(repr=False)
+ use_session: bool
_session_id: str = dataclasses.field(init=False, default="")
_session_lock: asyncio.Lock = dataclasses.field(init=False, default_factory=asyncio.Lock)
@@ -126,21 +127,19 @@ class Client:
except Exception as e: # noqa: BLE001
logger.error(f"Unable to destroy flaresolver session ({e}!r)")
- @contextlib.contextmanager
- def _disable_on_error(self) -> Generator[None]:
- try:
- yield
- except Exception:
- if not self._down:
- self._down = True
- logger.warning("Flaresolverr has been disabled")
- raise
+ def disable(self) -> None:
+ if not self._down:
+ self._down = True
+ logger.warning("Flaresolverr has been disabled")
async def _ensure_session(self) -> None:
msg = "Unable to create Flaresolverr session"
if self._down:
raise FlaresolverrError(msg)
+ if not self.use_session:
+ return
+
if self._session_id:
return
@@ -149,25 +148,32 @@ class Client:
return
try:
- with self._disable_on_error():
- await self._create_session()
- except aiohttp.ClientConnectionError as e:
+ await self._create_session()
+ except aiohttp.ClientError as e:
+ self.disable()
raise FlaresolverrError(f"Could not connect to Flaresolverr at {self.url} ({e!r})") from None
+ except FlaresolverrError:
+ self.disable()
+ raise
except Exception as e:
+ self.disable()
raise FlaresolverrError(msg) from e
async def request(self, url: AbsoluteHttpURL, data: dict[str, Any] | None = None) -> Solution:
await self._ensure_session()
try:
- resp = await self._request(
- Command.POST_REQUEST if data else Command.GET_REQUEST,
- url=str(url),
- data=data,
- session=self._session_id,
- )
-
- except (TypeError, KeyError) as e:
- raise FlaresolverrError("Invalid response from Flaresolverr") from e
+ return await self.raw_request(url, data)
+ except aiohttp.ClientError as e:
+ self.disable()
+ raise FlaresolverrError(f"Could not connect to Flaresolverr at {self.url} ({e!r})") from None
+
+ async def raw_request(self, url: AbsoluteHttpURL, data: dict[str, Any] | None = None) -> Solution:
+ resp = await self._request(
+ Command.POST_REQUEST if data else Command.GET_REQUEST,
+ url=str(url),
+ data=data,
+ session=self._session_id,
+ )
if not resp.ok:
raise FlaresolverrError(f"Failed to resolve URL with Flaresolverr. {resp.message}")
@@ -184,6 +190,8 @@ class Client:
# timeout in milliseconds (60s)
params = {"cmd": str(command), "maxTimeout": 60_000} | params
+ if not self.use_session:
+ params.pop("session", None)
if data:
assert command is Command.POST_REQUEST
@@ -203,6 +211,10 @@ class Client:
try:
resp = Response.from_dict(resp_json)
resp.id = str(request_id)
+ if resp.solution:
+ resp.solution.id = resp.id
+ except (TypeError, KeyError) as e:
+ raise FlaresolverrError("Invalid response from Flaresolverr") from e
finally:
logger.traffic(
"Finished FlareSolverr request [id=%s]\n%s", request_id, _LazyResponseLog(resp_json)
diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.py
index 9ef0a5c34..87389f1e9 100644
--- a/cyberdrop_dl/clients/http.py
+++ b/cyberdrop_dl/clients/http.py
@@ -139,7 +139,11 @@ class HTTPClient:
@property
def flaresolverr(self) -> flaresolverr.Client | None:
if self._flaresolverr is None and (url := self.config.network.flaresolverr):
- self._flaresolverr = flaresolverr.Client(url, self._session)
+ self._flaresolverr = flaresolverr.Client(
+ url,
+ self._session,
+ use_session=self.config.network.flaresolverr_use_session,
+ )
if self._flaresolverr and self._flaresolverr.is_down:
return None
return self._flaresolverr
diff --git a/cyberdrop_dl/clients/response.py b/cyberdrop_dl/clients/response.py
index af4c82bb9..9d6452100 100644
--- a/cyberdrop_dl/clients/response.py
+++ b/cyberdrop_dl/clients/response.py
@@ -1,10 +1,10 @@
from __future__ import annotations
import asyncio
+import copy
import dataclasses
import datetime
import json
-import logging
from abc import ABC, abstractmethod
from enum import StrEnum
from types import MappingProxyType
@@ -17,7 +17,7 @@ from multidict import CIMultiDict, CIMultiDictProxy
from propcache import under_cached_property
from typing_extensions import TypeVar
-from cyberdrop_dl.clients import wreq
+from cyberdrop_dl.clients import get_logger, wreq
from cyberdrop_dl.clients.flaresolverr import Solution as FlaresolverrSolution
from cyberdrop_dl.exceptions import InvalidContentTypeError, ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
@@ -35,7 +35,7 @@ else:
class CurlResponse: ...
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
_ResponseT = TypeVar(
"_ResponseT",
@@ -90,6 +90,8 @@ class AbstractResponse(ABC, Generic[_ResponseT]):
default_factory=lambda: datetime.datetime.now(datetime.UTC).replace(microsecond=0),
)
+ def __post_init__(self) -> None: ...
+
def __repr__(self) -> str:
return f"<{type(self).__name__} [{self.status}] ({self.url})>"
@@ -210,25 +212,27 @@ class AbstractResponse(ABC, Generic[_ResponseT]):
raise ScrapeError(204, "Received empty HTML response")
- @final
async def json(
self,
encoding: str | None = None,
content_type: tuple[str, ...] | str | Literal[False] | None = ("text/plain", "json"),
) -> Any:
+ self._check_json(content_type)
+ return json.loads(await self.text(encoding))
+
+ def _check_json(
+ self,
+ content_type: tuple[str, ...] | str | Literal[False] | None = ("text/plain", "json"),
+ ) -> None:
if self.status == 204:
raise ScrapeError(204)
- if content_type:
- if isinstance(content_type, str):
- content_type = (content_type,)
-
- self.__check_content_type(*content_type, expecting="JSON")
+ if not content_type:
+ return
+ if isinstance(content_type, str):
+ content_type = (content_type,)
- return await self._json(encoding)
-
- async def _json(self, encoding: str | None = None) -> Any:
- return json.loads(await self.text(encoding))
+ self.__check_content_type(*content_type, expecting="JSON")
@final
def create_report(self, exc: Exception | None = None, **extras: Any) -> str:
@@ -256,6 +260,10 @@ class AbstractResponse(ABC, Generic[_ResponseT]):
class _FlareSolverrResponse(AbstractResponse[FlaresolverrSolution]):
__slots__ = ()
+ def __post_init__(self) -> None:
+ super().__post_init__()
+ self.id: str = self._resp.id
+
@override
async def _read(self) -> bytes:
return self._text.encode()
@@ -271,13 +279,39 @@ class _FlareSolverrResponse(AbstractResponse[FlaresolverrSolution]):
@override
async def aclose(self) -> None: ...
- @override
- async def _json(self, encoding: str | None = None) -> Any:
- if self._text:
- return json.loads(self._text)
+ async def json(
+ self,
+ encoding: str | None = None, # noqa: ARG002
+ content_type: tuple[str, ...] | str | Literal[False] | None = ("text/plain", "json"),
+ ) -> Any:
+ if not self._text:
+ # Resp content is already parsed JSON
+ assert "json" in self.content_type
+ return self._resp.content
+
+ try:
+ return self._load_json(self._text)
+ finally:
+ self._check_json(content_type)
- assert "json" in self.content_type
- return self._resp.content
+ def _load_json(self, text: str) -> Any:
+ try:
+ return json.loads(text)
+ except ValueError:
+ if "html" not in self.content_type:
+ raise
+ text = BeautifulSoup(text, "html.parser").text
+ data = json.loads(text)
+ self.content_type = "application/json"
+ self._text = text
+ self._resp.content = copy.deepcopy(data)
+ logger.warning(
+ "Detected wrapped JSON in Flaresolverr response [id=%s], overriding content type to '%s'",
+ self.id,
+ self.content_type,
+ )
+ logger.traffic("Content from Flaresolverr request [id=%s]\n%s", self.id, {"content": self._resp.content})
+ return data
def _get_content(self) -> Any:
return super()._get_content() or self._resp.content
diff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yaml
index 255e01cbd..2db12c4ac 100644
--- a/cyberdrop_dl/config/default.yaml
+++ b/cyberdrop_dl/config/default.yaml
@@ -156,6 +156,7 @@ network:
connection_timeout: 15.0
dump_responses: false
flaresolverr: null
+ flaresolverr_use_session: true
impersonate: null
proxy: null
rate_limit: 25.0
diff --git a/cyberdrop_dl/config/schema.json b/cyberdrop_dl/config/schema.json
index cbaa25571..6f7fd8e5c 100644
--- a/cyberdrop_dl/config/schema.json
+++ b/cyberdrop_dl/config/schema.json
@@ -797,6 +797,11 @@
"$ref": "#/$defs/FalsyAsNone_HttpURL_",
"default": null
},
+ "flaresolverr_use_session": {
+ "default": true,
+ "title": "Flaresolverr Use Session",
+ "type": "boolean"
+ },
"proxy": {
"$ref": "#/$defs/FalsyAsNone_HttpURL_",
"default": null
diff --git a/cyberdrop_dl/config/settings.py b/cyberdrop_dl/config/settings.py
index 2350803eb..dea4fd957 100644
--- a/cyberdrop_dl/config/settings.py
+++ b/cyberdrop_dl/config/settings.py
@@ -343,6 +343,9 @@ class Network(ConfigGroup):
flaresolverr: FalsyAsNone[HttpURL] = None
"HTTP URL of an existing flaresolverr instance"
+ flaresolverr_use_session: bool = True
+ "Create a custom session before making any request with flaresolverr"
+
proxy: Annotated[FalsyAsNone[HttpURL], Parameter(alias=("http-proxy"))] = None
"HTTP/HTTPS proxy"
diff --git a/docs/reference/config/network.md b/docs/reference/config/network.md
index 966ec96be..9c0350c12 100644
--- a/docs/reference/config/network.md
+++ b/docs/reference/config/network.md
@@ -70,6 +70,17 @@ This wiki does not cover flaresolverr setup process. If you need help, refer to
See: [How to extract cookies (DDoSGuard or login errors) #839](https://github.com/Cyberdrop-DL/cyberdrop-dl/discussions/839) for alternatives using cookies
{% endhint %}
+# `flaresolverr_use_session`
+
+| Type | Default |
+| ------ | ------- |
+| `bool` | `true` |
+
+Create a custom flaresolverr session that keeps cookies. This reduces the likelihood of CF challenges and speeds up requests since Flaresolverr won't
+have to launch a new browser instance on every new one.
+
+Set this to `false` if you are using a Flaresolverr fork that does not support the `sessions.create` command
+
# `proxy`
| Type | Default |
@@ -193,11 +204,13 @@ If you use flaresolverr, this value **MUST** match with flaresolverr's user agen
These crawlers will ignore custom user-agents and will always use `cyberdrop-dl/<version>`
<!-- START_CUSTOM_UA_CRAWLERS -->
+
- Archive.org
- E621
- MegaNz
- RealDebrid
- Transfer.it
+
<!-- END_CUSTOM_UA_CRAWLERS -->
{% endhint %}
diff --git a/tests/test_flaresolverr.py b/tests/test_flaresolverr.py
index c938c6f4c..94f216494 100644
--- a/tests/test_flaresolverr.py
+++ b/tests/test_flaresolverr.py
@@ -16,7 +16,11 @@ pytestmark = pytest.mark.skipif(not FLARESOLVER_URL, reason=f"{ENV_NAME} environ
@pytest.fixture
async def flaresolverr() -> AsyncGenerator[Client]:
async with aiohttp.ClientSession() as session:
- yield Client(AbsoluteHttpURL(FLARESOLVER_URL) / "v1", session)
+ yield Client(
+ AbsoluteHttpURL(FLARESOLVER_URL) / "v1",
+ session,
+ use_session=True,
+ )
def test_flaresolver(flaresolverr: Client) -> None:
diff --git a/tests/test_flaresolverr_response.py b/tests/test_flaresolverr_response.py
index 0ef14b581..c18957832 100644
--- a/tests/test_flaresolverr_response.py
+++ b/tests/test_flaresolverr_response.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import time
+from typing import Any
from cyberdrop_dl.clients.flaresolverr import Solution, _parse_cookies
from cyberdrop_dl.clients.response import _FlareSolverrResponse, _infer_content_type_from_body
@@ -9,7 +10,7 @@ from cyberdrop_dl.clients.response import _FlareSolverrResponse, _infer_content_
# Fixtures: example FlareSolverr JSON responses
# ---------------------------------------------------------------------------
-FLARESOLVERR_RESPONSE_EMPTY_HEADERS = {
+FLARESOLVERR_RESPONSE_EMPTY_HEADERS: dict[str, Any] = {
"status": "ok",
"message": "Challenge solved!",
"solution": {
@@ -36,7 +37,34 @@ FLARESOLVERR_RESPONSE_EMPTY_HEADERS = {
"version": "3.4.6",
}
-FLARESOLVER_RESP_JSON = {
+FLARESOLVER_RESP_JSON_WRAPPED_IN_HTML: dict[str, Any] = {
+ "status": "ok",
+ "message": "Challenge solved!",
+ "solution": {
+ "url": "https://www.tikwm.com/api/user/posts?unique_id=user_embongngo&count=50&cursor=0",
+ "status": 200,
+ "cookies": [
+ {
+ "domain": ".tikwm.com",
+ "expiry": 1819086364,
+ "httpOnly": True,
+ "name": "cf_clearance",
+ "path": "/",
+ "sameSite": "None",
+ "secure": True,
+ "value": "uOF3vEc7QtZfPUB28qZfCy.GKmByEgAyDpoyO9EYfSU-1787550364-1.2.1.1-HzZDz",
+ }
+ ],
+ "userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
+ "headers": {},
+ "response": '<html><head><meta name="color-scheme" content="light dark"><meta charset="utf-8"></head><body><pre>{"code":0,"msg":"success","processed_time":0.4496,"data":{}}</pre><div class="json-formatter-container"></div></body></html>',
+ },
+ "startTimestamp": 1787550355291,
+ "endTimestamp": 1787550367316,
+ "version": "3.4.6",
+}
+
+FLARESOLVER_RESP_JSON: dict[str, Any] = {
"status": "ok",
"message": "Challenge not detected!",
"solution": {
@@ -277,3 +305,25 @@ async def test_flaresolverr_response_empty_body_and_empty_headers() -> None:
solution = Solution.from_dict(solution_data)
response = _FlareSolverrResponse.create(solution)
assert response.content_type == ""
+
+
+async def test_flaresolverr_response_from_json_resp_wrapped_in_html() -> None:
+ solution = Solution.from_dict(FLARESOLVER_RESP_JSON_WRAPPED_IN_HTML["solution"])
+ resp = _FlareSolverrResponse.create(solution)
+ assert resp._text
+ assert resp.content_type == "text/html"
+ assert type(solution.content) is str
+ data = await resp.json()
+ assert type(data) is dict
+ assert resp.content_type == "application/json"
+ assert data == await resp.json()
+ assert data is not await resp.json()
+ assert type(solution.content) is dict
+ assert data == {
+ "code": 0,
+ "msg": "success",
+ "processed_time": 0.4496,
+ "data": {},
+ }
+ data["extra_key"] = 0
+ assert data != await resp.json()