Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: nicely handle Flaresolverr setup errors (#1950)

NTFSvolumece0bbb15962a
diff --git a/cyberdrop_dl/clients/flaresolverr.py b/cyberdrop_dl/clients/flaresolverr.py
index d707fd46e..8d9a956fe 100644
--- a/cyberdrop_dl/clients/flaresolverr.py
+++ b/cyberdrop_dl/clients/flaresolverr.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import contextlib
import dataclasses
import itertools
import logging
@@ -13,13 +14,13 @@ import aiohttp
from multidict import CIMultiDict, CIMultiDictProxy
from cyberdrop_dl import ddos_guard
-from cyberdrop_dl.exceptions import DDOSGuardError
+from cyberdrop_dl.exceptions import DDOSGuardError, FlaresolverrError
from cyberdrop_dl.progress.scraping import show_msg
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import truncated_preview
if TYPE_CHECKING:
- from collections.abc import Callable, Iterable, Mapping
+ from collections.abc import Callable, Generator, Iterable, Mapping
logger = logging.getLogger(__name__)
@@ -96,7 +97,7 @@ class _LazyResponseLog:
@dataclasses.dataclass(slots=True)
class Client:
- """Class that handles communication with flaresolverr."""
+ """Class that handles communication with Flaresolverr."""
url: AbsoluteHttpURL
_aiohttp_session: aiohttp.ClientSession
@@ -107,6 +108,10 @@ class Client:
_request_id: Callable[[], int] = dataclasses.field(init=False, default_factory=lambda: itertools.count(1).__next__)
_down: bool = dataclasses.field(init=False, default=False)
+ @property
+ def is_down(self) -> bool:
+ return self._down
+
def __post_init__(self) -> None:
self.url = self.url.origin() / "v1"
@@ -116,10 +121,20 @@ class Client:
except Exception as e: # noqa: BLE001
logger.error(f"Unable to destroy flaresolver session ({e}!r)")
+ @contextlib.contextmanager
+ def _disable_on_error(self) -> Generator[None]:
+ try:
+ yield
+ except Exception:
+ if not self._down:
+ self._down = True
+ logger.warning("Flaresolverr has been disabled")
+ raise
+
async def _ensure_session(self) -> None:
msg = "Unable to create Flaresolverr session"
if self._down:
- raise RuntimeError(msg)
+ raise FlaresolverrError(msg)
if self._session_id:
return
@@ -129,16 +144,15 @@ class Client:
return
try:
- await self._create_session()
+ with self._disable_on_error():
+ await self._create_session()
+ except aiohttp.ClientConnectionError as e:
+ raise FlaresolverrError(f"Could not connect to Flaresolverr at {self.url} ({e!r})") from None
except Exception as e:
- self._down = True
- logger.exception(msg)
- raise RuntimeError(msg) from e
+ raise FlaresolverrError(msg) from e
async def request(self, url: AbsoluteHttpURL, data: dict[str, Any] | None = None) -> Solution:
-
await self._ensure_session()
- invalid_response_error = DDOSGuardError("Invalid response from flaresolverr")
try:
resp = await self._request(
Command.POST_REQUEST if data else Command.GET_REQUEST,
@@ -148,13 +162,13 @@ class Client:
)
except (TypeError, KeyError) as e:
- raise invalid_response_error from e
+ raise FlaresolverrError("Invalid response from Flaresolverr") from e
if not resp.ok:
- raise DDOSGuardError(f"Failed to resolve URL with flaresolverr. {resp.message}")
+ raise FlaresolverrError(f"Failed to resolve URL with Flaresolverr. {resp.message}")
if not resp.solution:
- raise invalid_response_error
+ raise FlaresolverrError("Flaresolverr response did not include a solution")
return resp.solution
@@ -173,9 +187,9 @@ class Client:
async with self._request_lock:
request_id = self._request_id()
msg = (
- "Destroying flaresolverr session"
+ "Destroying Flaresolverr session"
if command is Command.DESTROY_SESSION
- else f"Waiting for flaresolverr [{request_id}]"
+ else f"Waiting for Flaresolverr [{request_id}]"
)
with show_msg(msg):
logger.debug("Making FlareSolverr request [id=%s]\n%s", request_id, params)
@@ -199,7 +213,7 @@ class Client:
resp = await self._request(Command.CREATE_SESSION, session=session_id, **params)
if not resp.ok:
- raise RuntimeError(f"FlareSolverr said: {resp.message}")
+ raise FlaresolverrError(f"FlareSolverr said: {resp.message}")
self._session_id = session_id
async def _destroy_session(self) -> None:
@@ -225,7 +239,7 @@ def _parse_cookies(cookies: Iterable[Mapping[str, Any]]) -> SimpleCookie:
def verify_solution(cdl_user_agent: str, solution: Solution) -> None:
mismatch_ua_msg = (
- "Config user_agent and flaresolverr user_agent do not match:"
+ "Config user_agent and Flaresolverr user_agent do not match:"
f"\n Cyberdrop-DL: '{cdl_user_agent}'"
f"\n Flaresolverr: '{solution.user_agent}'"
)
@@ -233,9 +247,10 @@ def verify_solution(cdl_user_agent: str, solution: Solution) -> None:
if type(solution.content) is str:
try:
ddos_guard.check_html(solution.content)
- except DDOSGuardError:
+ except DDOSGuardError as e:
if solution.user_agent != cdl_user_agent:
- raise DDOSGuardError(mismatch_ua_msg) from None
+ e.add_note(mismatch_ua_msg)
+ raise
if solution.user_agent != cdl_user_agent:
logger.warning(f"{mismatch_ua_msg}\n Response was successful but cookies will not be valid")
diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.py
index 6cb1f0a7e..b89325eb9 100644
--- a/cyberdrop_dl/clients/http.py
+++ b/cyberdrop_dl/clients/http.py
@@ -125,6 +125,8 @@ class HTTPClient:
def flaresolverr(self) -> flaresolverr.Client | None:
if self._flaresolverr is None and (url := self.config.global_settings.general.flaresolverr):
self._flaresolverr = flaresolverr.Client(url, self._session)
+ if self._flaresolverr and self._flaresolverr.is_down:
+ return None
return self._flaresolverr
def __sync_session_cookies(self, url: AbsoluteHttpURL) -> None:
diff --git a/cyberdrop_dl/exceptions.py b/cyberdrop_dl/exceptions.py
index 097307810..2fd464425 100644
--- a/cyberdrop_dl/exceptions.py
+++ b/cyberdrop_dl/exceptions.py
@@ -8,15 +8,18 @@ from typing import TYPE_CHECKING
from yarl import URL
if TYPE_CHECKING:
+ from collections.abc import Sequence
+
from yaml import YAMLError
from cyberdrop_dl.url_objects import MediaItem, ScrapeItem
-def _format_error(ui_failure: str, message: str) -> str:
- if ui_failure == message:
- return message
- return f"{ui_failure} - {message}"
+def _format_error(ui_failure: str, message: str, notes: Sequence[str] | None = None) -> str:
+ msg = message if ui_failure == message else f"{ui_failure} - {message}"
+ if notes:
+ msg = msg + "\n" + "\n".join(f"[NOTE]: {note}" for note in notes)
+ return msg
# See: https://developers.cloudflare.com/support/troubleshooting/cloudflare-errors/troubleshooting-cloudflare-5xx-errors/
@@ -46,10 +49,8 @@ HTTP_ERROR_CODES = {
}
-class TooManyCrawlerErrors(Exception): # noqa: N818
- """This exception will be raised after a crawler had too many errors processing URLs"""
-
- # This exception does not inherit from `CDLBaseError`` cause it not intended to ever by shown to the user
+def _notes(e: BaseException) -> list[str] | tuple[()]:
+ return getattr(e, "__notes__", ())
class CDLBaseError(Exception):
@@ -72,7 +73,13 @@ class CDLBaseError(Exception):
super().__init__(self.status)
def __str__(self) -> str:
- return _format_error(self.ui_failure, self.message)
+ return _format_error(self.ui_failure, self.message, _notes(self))
+
+
+class FlaresolverrError(CDLBaseError):
+ def __init__(self, message: str | None = None) -> None:
+ ui_failure = "Flaresolverr Error"
+ super().__init__(ui_failure, message=message)
class InvalidContentTypeError(CDLBaseError):
@@ -284,7 +291,7 @@ class ErrorLogMessage:
e_status = getattr(e, "status", None)
e_message = getattr(e, "message", None)
ui_failure = create_error_msg(e_status) if e_status else "Unknown"
- log_msg = _format_error(ui_failure, e_message or str(e))
+ log_msg = _format_error(ui_failure, e_message or str(e), _notes(e))
return ErrorLogMessage(ui_failure, log_msg)
diff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.py
index 409d8ac2a..af88cccfe 100644
--- a/cyberdrop_dl/utils/__init__.py
+++ b/cyberdrop_dl/utils/__init__.py
@@ -18,14 +18,7 @@ from mega.errors import MegaNzError
from pydantic import ValidationError
from typing_extensions import TypeIs
-from cyberdrop_dl.exceptions import (
- CDLBaseError,
- ErrorLogMessage,
- InvalidURLError,
- TooManyCrawlerErrors,
- create_error_msg,
- get_origin,
-)
+from cyberdrop_dl.exceptions import CDLBaseError, ErrorLogMessage, InvalidURLError, create_error_msg, get_origin
from cyberdrop_dl.utils._dataclasses import DictDataclass as DictDataclass
from cyberdrop_dl.utils._dataclasses import deserialize as deserialize
from cyberdrop_dl.utils._dataclasses import filter_data as filter_data
@@ -90,8 +83,6 @@ def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | yar
try:
with group_exceptions():
yield
- except TooManyCrawlerErrors:
- return
except ExceptionGroup as e:
error_log_msg = ErrorLogMessage(_exc_group_msg(e), str(e))
exc_info = e.with_traceback(None)
@@ -156,7 +147,7 @@ def _log_error( # noqa: PLR0913
link_to_show: yarl.URL | str,
item: ScrapeItem | MediaItem | yarl.URL,
error_log_msg: ErrorLogMessage,
- exc_info: Exception | None,
+ exc_info: BaseException | None,
origin: ScrapeItem | MediaItem | yarl.URL | Path | None,
) -> None:
origin = origin or get_origin(item)