Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: improve flaresolverr logs (#1787)
NTFSvolumed83009720b15
diff --git a/cyberdrop_dl/clients/client.py b/cyberdrop_dl/clients/client.pyindex cb60ccd71..4851f232f 100644--- a/cyberdrop_dl/clients/client.py+++ b/cyberdrop_dl/clients/client.py@@ -247,7 +247,7 @@ class HTTPClient:else:_ = headers.setdefault("User-Agent", self.manager.config.global_settings.general.user_agent)- async with self.__request(url, method, request_params, impersonate=bool(impersonate)) as resp:+ async with self._request(url, method, request_params, impersonate=bool(impersonate)) as resp:exc = Nonetry:yield await self._check_response(resp, url)@@ -272,7 +272,7 @@ class HTTPClient:self.cookies.update_cookies(simple_cookie, url)@contextlib.asynccontextmanager- async def __request(+ async def _request(self,url: AbsoluteHttpURL,method: HttpMethod,@@ -288,27 +288,32 @@ class HTTPClient:url,_LazyRequestLog(request_params),)- resp = None- try:- if impersonate:- async with contextlib.aclosing(- await self.curl_session.request(method, str(url), stream=True, **request_params)- ) as curl_resp:- resp = AbstractResponse.create(curl_resp)- yield resp- self.__sync_session_cookies(url)- return+ async with self.__request(url, method, request_params, impersonate=impersonate) as resp:+ logger.debug("Finished %s request [id=%s]\n%s", method, request_id, _LazyResponseLog(resp))+ yield resp- async with (- self._session.request(method, url, **request_params) as aio_resp,- ):- resp = AbstractResponse.create(aio_resp)- yield resp+ @contextlib.asynccontextmanager+ async def __request(+ self,+ url: AbsoluteHttpURL,+ method: HttpMethod,+ request_params: Mapping[str, Any],+ *,+ impersonate: bool,+ ) -> AsyncGenerator[AbstractResponse[Any]]:- finally:- if resp is not None:- logger.debug("Finished %s request [id=%s]\n%s", method, request_id, _LazyResponseLog(resp))+ if impersonate:+ async with contextlib.aclosing(+ await self.curl_session.request(method, str(url), stream=True, **request_params)+ ) as curl_resp:+ yield AbstractResponse.create(curl_resp)+ self.__sync_session_cookies(url)++ return++ async with self._session.request(method, url, **request_params) as aio_resp:+ yield AbstractResponse.create(aio_resp)async def _check_response(self, abs_resp: AbstractResponse[Any], url: AbsoluteHttpURL, data: Any | None = None):"""Checks the HTTP response status and retries DDOS Guard errors with FlareSolverr.diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.pyindex 1b2489aae..bc5741ca1 100644--- a/cyberdrop_dl/clients/flaresolverr.py+++ b/cyberdrop_dl/clients/flaresolverr.py@@ -16,6 +16,7 @@ from cyberdrop_dl import ddos_guardfrom cyberdrop_dl.exceptions import DDOSGuardErrorfrom cyberdrop_dl.progress.scraping import show_msgfrom cyberdrop_dl.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils import truncated_previewif TYPE_CHECKING:from collections.abc import Callable, Iterable, Mapping@@ -73,6 +74,25 @@ class Response:)+class _LazyResponseLog:+ def __init__(self, response: dict[str, Any]) -> None:+ self.resp = response++ def __json__(self) -> dict[str, Any]:+ try:+ html = self.resp["solution"]["response"]+ except LookupError:+ pass+ else:+ if type(html) is str:+ self.resp["solution"]["response"] = truncated_preview(html)++ return self.resp++ def __str__(self) -> str:+ return str(self.__json__())++@dataclasses.dataclass(slots=True)class Client:"""Class that handles communication with flaresolverr."""@@ -93,7 +113,7 @@ class Client:try:await self._destroy_session()except Exception as e:- logger.error(f"Unable to destroy flaresolver session ({e})")+ logger.error(f"Unable to destroy flaresolver session ({e}!r)")async def _ensure_session(self) -> None:msg = "Unable to create Flaresolverr session"@@ -143,7 +163,7 @@ class Client:timeout.update(timeout=aiohttp.ClientTimeout(total=5 * 60, connect=60)) # 5 minutes to create session# timeout in milliseconds (60s)- params = {"cmd": command, "maxTimeout": 60_000} | params+ params = {"cmd": str(command), "maxTimeout": 60_000} | paramsif data:assert command is Command.POST_REQUEST@@ -157,9 +177,11 @@ class Client:else f"Waiting for flaresolverr [{request_id}]")with show_msg(msg):- logger.debug(f"Making FlareSolverr request #{request_id} with {params = }")+ logger.debug("Making FlareSolverr request [id=%s]\n%s", request_id, params)async with self._aiohttp_session.post(self.url, json=params, **timeout) as response:- return Response.from_dict(await response.json())+ resp = await response.json()+ logger.debug("Finished FlareSolverr request [id=%s]\n%s", request_id, _LazyResponseLog(resp))+ return Response.from_dict(resp)async def _create_session(self) -> None:session_id = "cyberdrop-dl"diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.pyindex c27e0fc46..6526e2c9b 100644--- a/cyberdrop_dl/crawlers/mega_nz.py+++ b/cyberdrop_dl/crawlers/mega_nz.py@@ -18,6 +18,7 @@ from cyberdrop_dl.constants import CDL_USER_AGENTfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, auto_task_idfrom cyberdrop_dl.downloader.mega_nz import MegaDownloaderfrom cyberdrop_dl.exceptions import LoginError, ScrapeError+from cyberdrop_dl.progress.scraping import show_msgfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import error_handling_wrapper@@ -151,7 +152,8 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):# TODO: Add a way to cache this login# TODO: Show some logging message / UI about logintry:- await self.core.login(self.user, self.password)+ with show_msg("Login into mega.nz"):+ await self.core.login(self.user, self.password)self._logged_in = Trueexcept Exception as e:self.disabled = True