Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: nicely handle Flaresolverr setup errors (#1950)
NTFSvolumece0bbb15962a
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.pyindex d707fd46e..8d9a956fe 100644--- a/cyberdrop_dl/clients/flaresolverr.py+++ b/cyberdrop_dl/clients/flaresolverr.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import contextlibimport dataclassesimport itertoolsimport logging@@ -13,13 +14,13 @@ import aiohttpfrom multidict import CIMultiDict, CIMultiDictProxyfrom cyberdrop_dl import ddos_guard-from cyberdrop_dl.exceptions import DDOSGuardError+from cyberdrop_dl.exceptions import DDOSGuardError, FlaresolverrErrorfrom cyberdrop_dl.progress.scraping import show_msgfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import truncated_previewif TYPE_CHECKING:- from collections.abc import Callable, Iterable, Mapping+ from collections.abc import Callable, Generator, Iterable, Mappinglogger = logging.getLogger(__name__)@@ -96,7 +97,7 @@ class _LazyResponseLog:@dataclasses.dataclass(slots=True)class Client:- """Class that handles communication with flaresolverr."""+ """Class that handles communication with Flaresolverr."""url: AbsoluteHttpURL_aiohttp_session: aiohttp.ClientSession@@ -107,6 +108,10 @@ class Client:_request_id: Callable[[], int] = dataclasses.field(init=False, default_factory=lambda: itertools.count(1).__next__)_down: bool = dataclasses.field(init=False, default=False)+ @property+ def is_down(self) -> bool:+ return self._down+def __post_init__(self) -> None:self.url = self.url.origin() / "v1"@@ -116,10 +121,20 @@ class Client:except Exception as e: # noqa: BLE001logger.error(f"Unable to destroy flaresolver session ({e}!r)")+ @contextlib.contextmanager+ def _disable_on_error(self) -> Generator[None]:+ try:+ yield+ except Exception:+ if not self._down:+ self._down = True+ logger.warning("Flaresolverr has been disabled")+ raise+async def _ensure_session(self) -> None:msg = "Unable to create Flaresolverr session"if self._down:- raise RuntimeError(msg)+ raise FlaresolverrError(msg)if self._session_id:return@@ -129,16 +144,15 @@ class Client:returntry:- await self._create_session()+ with self._disable_on_error():+ await self._create_session()+ except aiohttp.ClientConnectionError as e:+ raise FlaresolverrError(f"Could not connect to Flaresolverr at {self.url} ({e!r})") from Noneexcept Exception as e:- self._down = True- logger.exception(msg)- raise RuntimeError(msg) from e+ raise FlaresolverrError(msg) from easync def request(self, url: AbsoluteHttpURL, data: dict[str, Any] | None = None) -> Solution:-await self._ensure_session()- invalid_response_error = DDOSGuardError("Invalid response from flaresolverr")try:resp = await self._request(Command.POST_REQUEST if data else Command.GET_REQUEST,@@ -148,13 +162,13 @@ class Client:)except (TypeError, KeyError) as e:- raise invalid_response_error from e+ raise FlaresolverrError("Invalid response from Flaresolverr") from eif not resp.ok:- raise DDOSGuardError(f"Failed to resolve URL with flaresolverr. {resp.message}")+ raise FlaresolverrError(f"Failed to resolve URL with Flaresolverr. {resp.message}")if not resp.solution:- raise invalid_response_error+ raise FlaresolverrError("Flaresolverr response did not include a solution")return resp.solution@@ -173,9 +187,9 @@ class Client:async with self._request_lock:request_id = self._request_id()msg = (- "Destroying flaresolverr session"+ "Destroying Flaresolverr session"if command is Command.DESTROY_SESSION- else f"Waiting for flaresolverr [{request_id}]"+ else f"Waiting for Flaresolverr [{request_id}]")with show_msg(msg):logger.debug("Making FlareSolverr request [id=%s]\n%s", request_id, params)@@ -199,7 +213,7 @@ class Client:resp = await self._request(Command.CREATE_SESSION, session=session_id, **params)if not resp.ok:- raise RuntimeError(f"FlareSolverr said: {resp.message}")+ raise FlaresolverrError(f"FlareSolverr said: {resp.message}")self._session_id = session_idasync def _destroy_session(self) -> None:@@ -225,7 +239,7 @@ def _parse_cookies(cookies: Iterable[Mapping[str, Any]]) -> SimpleCookie:def verify_solution(cdl_user_agent: str, solution: Solution) -> None:mismatch_ua_msg = (- "Config user_agent and flaresolverr user_agent do not match:"+ "Config user_agent and Flaresolverr user_agent do not match:"f"\n Cyberdrop-DL: '{cdl_user_agent}'"f"\n Flaresolverr: '{solution.user_agent}'")@@ -233,9 +247,10 @@ def verify_solution(cdl_user_agent: str, solution: Solution) -> None:if type(solution.content) is str:try:ddos_guard.check_html(solution.content)- except DDOSGuardError:+ except DDOSGuardError as e:if solution.user_agent != cdl_user_agent:- raise DDOSGuardError(mismatch_ua_msg) from None+ e.add_note(mismatch_ua_msg)+ raiseif solution.user_agent != cdl_user_agent:logger.warning(f"{mismatch_ua_msg}\n Response was successful but cookies will not be valid")diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.pyindex 6cb1f0a7e..b89325eb9 100644--- a/cyberdrop_dl/clients/http.py+++ b/cyberdrop_dl/clients/http.py@@ -125,6 +125,8 @@ class HTTPClient:def flaresolverr(self) -> flaresolverr.Client | None:if self._flaresolverr is None and (url := self.config.global_settings.general.flaresolverr):self._flaresolverr = flaresolverr.Client(url, self._session)+ if self._flaresolverr and self._flaresolverr.is_down:+ return Nonereturn self._flaresolverrdef __sync_session_cookies(self, url: AbsoluteHttpURL) -> None:diff --git a/cyberdrop_dl/exceptions.py b/cyberdrop_dl/exceptions.pyindex 097307810..2fd464425 100644--- a/cyberdrop_dl/exceptions.py+++ b/cyberdrop_dl/exceptions.py@@ -8,15 +8,18 @@ from typing import TYPE_CHECKINGfrom yarl import URLif TYPE_CHECKING:+ from collections.abc import Sequence+from yaml import YAMLErrorfrom cyberdrop_dl.url_objects import MediaItem, ScrapeItem-def _format_error(ui_failure: str, message: str) -> str:- if ui_failure == message:- return message- return f"{ui_failure} - {message}"+def _format_error(ui_failure: str, message: str, notes: Sequence[str] | None = None) -> str:+ msg = message if ui_failure == message else f"{ui_failure} - {message}"+ if notes:+ msg = msg + "\n" + "\n".join(f"[NOTE]: {note}" for note in notes)+ return msg# See: https://developers.cloudflare.com/support/troubleshooting/cloudflare-errors/troubleshooting-cloudflare-5xx-errors/@@ -46,10 +49,8 @@ HTTP_ERROR_CODES = {}-class TooManyCrawlerErrors(Exception): # noqa: N818- """This exception will be raised after a crawler had too many errors processing URLs"""-- # This exception does not inherit from `CDLBaseError`` cause it not intended to ever by shown to the user+def _notes(e: BaseException) -> list[str] | tuple[()]:+ return getattr(e, "__notes__", ())class CDLBaseError(Exception):@@ -72,7 +73,13 @@ class CDLBaseError(Exception):super().__init__(self.status)def __str__(self) -> str:- return _format_error(self.ui_failure, self.message)+ return _format_error(self.ui_failure, self.message, _notes(self))+++class FlaresolverrError(CDLBaseError):+ def __init__(self, message: str | None = None) -> None:+ ui_failure = "Flaresolverr Error"+ super().__init__(ui_failure, message=message)class InvalidContentTypeError(CDLBaseError):@@ -284,7 +291,7 @@ class ErrorLogMessage:e_status = getattr(e, "status", None)e_message = getattr(e, "message", None)ui_failure = create_error_msg(e_status) if e_status else "Unknown"- log_msg = _format_error(ui_failure, e_message or str(e))+ log_msg = _format_error(ui_failure, e_message or str(e), _notes(e))return ErrorLogMessage(ui_failure, log_msg)diff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.pyindex 409d8ac2a..af88cccfe 100644--- a/cyberdrop_dl/utils/__init__.py+++ b/cyberdrop_dl/utils/__init__.py@@ -18,14 +18,7 @@ from mega.errors import MegaNzErrorfrom pydantic import ValidationErrorfrom typing_extensions import TypeIs-from cyberdrop_dl.exceptions import (- CDLBaseError,- ErrorLogMessage,- InvalidURLError,- TooManyCrawlerErrors,- create_error_msg,- get_origin,-)+from cyberdrop_dl.exceptions import CDLBaseError, ErrorLogMessage, InvalidURLError, create_error_msg, get_originfrom cyberdrop_dl.utils._dataclasses import DictDataclass as DictDataclassfrom cyberdrop_dl.utils._dataclasses import deserialize as deserializefrom cyberdrop_dl.utils._dataclasses import filter_data as filter_data@@ -90,8 +83,6 @@ def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | yartry:with group_exceptions():yield- except TooManyCrawlerErrors:- returnexcept ExceptionGroup as e:error_log_msg = ErrorLogMessage(_exc_group_msg(e), str(e))exc_info = e.with_traceback(None)@@ -156,7 +147,7 @@ def _log_error( # noqa: PLR0913link_to_show: yarl.URL | str,item: ScrapeItem | MediaItem | yarl.URL,error_log_msg: ErrorLogMessage,- exc_info: Exception | None,+ exc_info: BaseException | None,origin: ScrapeItem | MediaItem | yarl.URL | Path | None,) -> None:origin = origin or get_origin(item)