Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: improve flaresolverr logs (#1787)

NTFSvolumed83009720b15
diff --git a/cyberdrop_dl/clients/client.py b/cyberdrop_dl/clients/client.py
index cb60ccd71..4851f232f 100644
--- a/cyberdrop_dl/clients/client.py
+++ b/cyberdrop_dl/clients/client.py
@@ -247,7 +247,7 @@ class HTTPClient:
else:
_ = headers.setdefault("User-Agent", self.manager.config.global_settings.general.user_agent)
- async with self.__request(url, method, request_params, impersonate=bool(impersonate)) as resp:
+ async with self._request(url, method, request_params, impersonate=bool(impersonate)) as resp:
exc = None
try:
yield await self._check_response(resp, url)
@@ -272,7 +272,7 @@ class HTTPClient:
self.cookies.update_cookies(simple_cookie, url)
@contextlib.asynccontextmanager
- async def __request(
+ async def _request(
self,
url: AbsoluteHttpURL,
method: HttpMethod,
@@ -288,27 +288,32 @@ class HTTPClient:
url,
_LazyRequestLog(request_params),
)
- resp = None
- try:
- if impersonate:
- async with contextlib.aclosing(
- await self.curl_session.request(method, str(url), stream=True, **request_params)
- ) as curl_resp:
- resp = AbstractResponse.create(curl_resp)
- yield resp
- self.__sync_session_cookies(url)
- return
+ async with self.__request(url, method, request_params, impersonate=impersonate) as resp:
+ logger.debug("Finished %s request [id=%s]\n%s", method, request_id, _LazyResponseLog(resp))
+ yield resp
- async with (
- self._session.request(method, url, **request_params) as aio_resp,
- ):
- resp = AbstractResponse.create(aio_resp)
- yield resp
+ @contextlib.asynccontextmanager
+ async def __request(
+ self,
+ url: AbsoluteHttpURL,
+ method: HttpMethod,
+ request_params: Mapping[str, Any],
+ *,
+ impersonate: bool,
+ ) -> AsyncGenerator[AbstractResponse[Any]]:
- finally:
- if resp is not None:
- logger.debug("Finished %s request [id=%s]\n%s", method, request_id, _LazyResponseLog(resp))
+ if impersonate:
+ async with contextlib.aclosing(
+ await self.curl_session.request(method, str(url), stream=True, **request_params)
+ ) as curl_resp:
+ yield AbstractResponse.create(curl_resp)
+ self.__sync_session_cookies(url)
+
+ return
+
+ async with self._session.request(method, url, **request_params) as aio_resp:
+ yield AbstractResponse.create(aio_resp)
async def _check_response(self, abs_resp: AbstractResponse[Any], url: AbsoluteHttpURL, data: Any | None = None):
"""Checks the HTTP response status and retries DDOS Guard errors with FlareSolverr.
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.py
index 1b2489aae..bc5741ca1 100644
--- a/cyberdrop_dl/clients/flaresolverr.py
+++ b/cyberdrop_dl/clients/flaresolverr.py
@@ -16,6 +16,7 @@ from cyberdrop_dl import ddos_guard
from cyberdrop_dl.exceptions import DDOSGuardError
from cyberdrop_dl.progress.scraping import show_msg
from cyberdrop_dl.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils import truncated_preview
if TYPE_CHECKING:
from collections.abc import Callable, Iterable, Mapping
@@ -73,6 +74,25 @@ class Response:
)
+class _LazyResponseLog:
+ def __init__(self, response: dict[str, Any]) -> None:
+ self.resp = response
+
+ def __json__(self) -> dict[str, Any]:
+ try:
+ html = self.resp["solution"]["response"]
+ except LookupError:
+ pass
+ else:
+ if type(html) is str:
+ self.resp["solution"]["response"] = truncated_preview(html)
+
+ return self.resp
+
+ def __str__(self) -> str:
+ return str(self.__json__())
+
+
@dataclasses.dataclass(slots=True)
class Client:
"""Class that handles communication with flaresolverr."""
@@ -93,7 +113,7 @@ class Client:
try:
await self._destroy_session()
except Exception as e:
- logger.error(f"Unable to destroy flaresolver session ({e})")
+ logger.error(f"Unable to destroy flaresolver session ({e}!r)")
async def _ensure_session(self) -> None:
msg = "Unable to create Flaresolverr session"
@@ -143,7 +163,7 @@ class Client:
timeout.update(timeout=aiohttp.ClientTimeout(total=5 * 60, connect=60)) # 5 minutes to create session
# timeout in milliseconds (60s)
- params = {"cmd": command, "maxTimeout": 60_000} | params
+ params = {"cmd": str(command), "maxTimeout": 60_000} | params
if data:
assert command is Command.POST_REQUEST
@@ -157,9 +177,11 @@ class Client:
else f"Waiting for flaresolverr [{request_id}]"
)
with show_msg(msg):
- logger.debug(f"Making FlareSolverr request #{request_id} with {params = }")
+ logger.debug("Making FlareSolverr request [id=%s]\n%s", request_id, params)
async with self._aiohttp_session.post(self.url, json=params, **timeout) as response:
- return Response.from_dict(await response.json())
+ resp = await response.json()
+ logger.debug("Finished FlareSolverr request [id=%s]\n%s", request_id, _LazyResponseLog(resp))
+ return Response.from_dict(resp)
async def _create_session(self) -> None:
session_id = "cyberdrop-dl"
diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.py
index c27e0fc46..6526e2c9b 100644
--- a/cyberdrop_dl/crawlers/mega_nz.py
+++ b/cyberdrop_dl/crawlers/mega_nz.py
@@ -18,6 +18,7 @@ from cyberdrop_dl.constants import CDL_USER_AGENT
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, auto_task_id
from cyberdrop_dl.downloader.mega_nz import MegaDownloader
from cyberdrop_dl.exceptions import LoginError, ScrapeError
+from cyberdrop_dl.progress.scraping import show_msg
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import error_handling_wrapper
@@ -151,7 +152,8 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):
# TODO: Add a way to cache this login
# TODO: Show some logging message / UI about login
try:
- await self.core.login(self.user, self.password)
+ with show_msg("Login into mega.nz"):
+ await self.core.login(self.user, self.password)
self._logged_in = True
except Exception as e:
self.disabled = True