Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: disable crawlers after too many errors (#1124)

NTFSvolume8b31e8287c50
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index 5c94c6fd2..92ee3166d 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -46,20 +46,21 @@ def limiter(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutin
async def wrapper(*args, **kwargs) -> R:
self: DownloadClient = args[0]
domain: str = args[1]
- domain_limiter = await self.client_manager.get_rate_limiter(domain)
- await asyncio.sleep(await self.client_manager.get_downloader_spacer(domain))
- await self._global_limiter.acquire()
- await domain_limiter.acquire()
-
- async with aiohttp.ClientSession(
- headers=self._headers,
- raise_for_status=False,
- cookie_jar=self.client_manager.cookies,
- timeout=self._timeouts,
- trace_configs=self.trace_configs,
- ) as client:
- kwargs["client_session"] = client
- return await func(*args, **kwargs)
+ with self.client_manager.request_context(domain):
+ domain_limiter = await self.client_manager.get_rate_limiter(domain)
+ await asyncio.sleep(await self.client_manager.get_downloader_spacer(domain))
+ await self._global_limiter.acquire()
+ await domain_limiter.acquire()
+
+ async with aiohttp.ClientSession(
+ headers=self._headers,
+ raise_for_status=False,
+ cookie_jar=self.client_manager.cookies,
+ timeout=self._timeouts,
+ trace_configs=self.trace_configs,
+ ) as client:
+ kwargs["client_session"] = client
+ return await func(*args, **kwargs)
return wrapper
diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.py
index 547242da6..1c732447a 100644
--- a/cyberdrop_dl/clients/scraper_client.py
+++ b/cyberdrop_dl/clients/scraper_client.py
@@ -34,48 +34,44 @@ if TYPE_CHECKING:
from cyberdrop_dl.managers.client_manager import ClientManager
-curl_import_error = None
+_curl_import_error = None
try:
from curl_cffi.requests import AsyncSession
except ImportError as e:
- curl_import_error = e
+ _curl_import_error = e
-P = ParamSpec("P")
-R = TypeVar("R")
-T = TypeVar("T")
+_P = ParamSpec("_P")
+_R = TypeVar("_R")
+_T = TypeVar("_T")
-def limiter(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutine[None, None, R]]:
+def limiter(func: Callable[_P, Coroutine[None, None, _R]]) -> Callable[_P, Coroutine[None, None, _R]]:
"""Wrapper to handle limits for scrape session."""
@wraps(func)
- async def wrapper(*args, **kwargs) -> R:
+ async def wrapper(*args, **kwargs) -> _R:
self: ScraperClient = args[0]
domain: str = args[1]
- domain_limiter = await self.client_manager.get_rate_limiter(domain)
- async with self.client_manager.session_limit, self._global_limiter, domain_limiter:
- await self.client_manager.manager.states.RUNNING.wait()
-
- if "cffi" in func.__name__:
- if curl_import_error is not None:
- system = "Android" if env.RUNNING_IN_TERMUX else "the system"
- msg = f"curl_cffi is required to scrape URLs from {domain}, but a dependency it's not available on {system}.\n"
- msg += f"See: https://github.com/lexiforest/curl_cffi/issues/74#issuecomment-1849365636\n{curl_import_error!r}"
- raise ScrapeError("Missing Dependency", msg)
- return await func(*args, **kwargs)
+ with self.client_manager.request_context(domain):
+ domain_limiter = await self.client_manager.get_rate_limiter(domain)
+ async with self.client_manager.session_limit, self._global_limiter, domain_limiter:
+ await self.client_manager.manager.states.RUNNING.wait()
+
+ if "cffi" in func.__name__:
+ _check_curl_cffi_is_available(domain)
- return await func(*args, **kwargs)
+ return await func(*args, **kwargs)
return wrapper
-def copy_signature(target: Callable[P, R]) -> Callable[[Callable[..., T]], Callable[P, T]]:
+def copy_signature(target: Callable[_P, _R]) -> Callable[[Callable[..., _T]], Callable[_P, _T]]:
"""Decorator to make a function mimic the signature of another function,
but preserve the return type of the decorated function."""
- def decorator(func: Callable[..., T]) -> Callable[P, T]:
+ def decorator(func: Callable[..., _T]) -> Callable[_P, _T]:
@functools.wraps(func)
- def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T:
return func(*args, **kwargs)
wrapper.__signature__ = inspect.signature(target).replace( # type: ignore
@@ -125,7 +121,7 @@ class ScraperClient:
add_request_log_hooks(self._trace_configs)
self._session = self.new_session()
self.reddit_session = self.new_session()
- if curl_import_error is not None:
+ if _curl_import_error is not None:
return
proxy = str(self.client_manager.proxy) if self.client_manager.proxy else None
@@ -150,7 +146,7 @@ class ScraperClient:
) -> None:
await self._session.close()
await self.reddit_session.close()
- if curl_import_error is not None:
+ if _curl_import_error is not None:
return
try:
await self._curl_session.close()
@@ -479,3 +475,11 @@ def add_request_log_hooks(trace_configs: list[aiohttp.TraceConfig]) -> None:
trace_config.on_request_start.append(on_request_start)
trace_config.on_request_end.append(on_request_end)
trace_configs.append(trace_config)
+
+
+def _check_curl_cffi_is_available(domain: str) -> None:
+ if _curl_import_error is not None:
+ system = "Android" if env.RUNNING_IN_TERMUX else "the system"
+ msg = f"curl_cffi is required to scrape URLs from {domain}, but a dependency it's not available on {system}.\n"
+ msg += f"See: https://github.com/lexiforest/curl_cffi/issues/74#issuecomment-1849365636\n{_curl_import_error!r}"
+ raise ScrapeError("Missing Dependency", msg)
diff --git a/cyberdrop_dl/exceptions.py b/cyberdrop_dl/exceptions.py
index 4328b6890..7ec65077a 100644
--- a/cyberdrop_dl/exceptions.py
+++ b/cyberdrop_dl/exceptions.py
@@ -29,6 +29,12 @@ CLOUDFLARE_ERRORS = {
}
+class TooManyCrawlerErrors(Exception): # noqa: N818
+ """This exception will be raised after a crawler had too many errors proccesing URLs"""
+
+ # This exception does not inherit from `CDLBaseError`` cause it not intented to ever by shown to the user
+
+
class CDLBaseError(Exception):
"""Base exception for cyberdrop-dl errors."""
diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.py
index 4c674bec2..8aae43479 100644
--- a/cyberdrop_dl/managers/client_manager.py
+++ b/cyberdrop_dl/managers/client_manager.py
@@ -4,6 +4,7 @@ import asyncio
import contextlib
import ssl
import time
+from collections import defaultdict
from dataclasses import dataclass
from http import HTTPStatus
from http.cookiejar import MozillaCookieJar
@@ -20,14 +21,14 @@ from yarl import URL
from cyberdrop_dl.clients.download_client import DownloadClient
from cyberdrop_dl.clients.scraper_client import ScraperClient
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError
+from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError, TooManyCrawlerErrors
from cyberdrop_dl.managers.download_speed_manager import DownloadSpeedLimiter
from cyberdrop_dl.ui.prompts.user_prompts import get_cookies_from_browsers
from cyberdrop_dl.utils.logger import log, log_spacer
from cyberdrop_dl.utils.utilities import get_soup_no_error
if TYPE_CHECKING:
- from collections.abc import Mapping
+ from collections.abc import Generator, Mapping
from aiohttp_client_cache.response import CachedResponse
from curl_cffi.requests.models import Response as CurlResponse
@@ -44,6 +45,9 @@ DOWNLOAD_ERROR_ETAGS = {
"5a56b09d-1485eb": "eFukt Video removed",
}
+_crawler_errors: dict[str, int] = defaultdict(int)
+MAX_CRAWLER_ERRORS = 10
+
class DDosGuard:
TITLES = ("Just a moment...", "DDoS-Guard")
@@ -132,6 +136,32 @@ class ClientManager:
self.downloader_session = DownloadClient(manager, self)
self.flaresolverr = Flaresolverr(self)
+ def check_domain_errors(self, domain: str) -> None:
+ if _crawler_errors[domain] >= MAX_CRAWLER_ERRORS:
+ if crawler := self.manager.scrape_mapper.disable_crawler(domain):
+ msg = (
+ f"{crawler.__name__} has been disabled after too many errors. "
+ f"URLs from the following domains will be ignored: {crawler.SCRAPE_MAPPER_KEYS}"
+ )
+ log(msg, 40)
+ raise TooManyCrawlerErrors
+
+ @contextlib.contextmanager
+ def request_context(self, domain: str) -> Generator[None]:
+ self.check_domain_errors(domain)
+ try:
+ yield
+ except DDOSGuardError:
+ _crawler_errors[domain] += 1
+ self.check_domain_errors(domain)
+ raise
+ else:
+ # we could potencially reset the counter here
+ # _crawler_errors[domain] = 0
+ pass
+ finally:
+ pass
+
def load_cookie_files(self) -> None:
if self.manager.config_manager.settings_data.browser_cookies.auto_import:
assert self.manager.config_manager.settings_data.browser_cookies.browser
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index ac5c61c25..c08c3f1c6 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -347,6 +347,24 @@ class ScrapeMapper:
return False
+ def disable_crawler(self, domain: str) -> Crawler | None:
+ """Disables a crawler at runtime, after the scrape mapper is already running.
+
+ It does not remove the crawler from the crawlers map, it just sets it as `disabled"`
+
+ This has the effect to silently ignore any URL that maps to that crawler, without any "unsupported" or "errors" log messages
+
+ `domain` must match _exactly_, AKA: it must be the value of `crawler.DOMAIN`
+
+ Returns the crawler instance that was disabled (if Any)
+
+ """
+
+ crawler = next((crawler for crawler in self.existing_crawlers.values() if crawler.DOMAIN == domain), None)
+ if crawler and not crawler.disabled:
+ crawler.disabled = True
+ return crawler
+
def regex_links(line: str) -> list[AbsoluteHttpURL]:
"""Regex grab the links from the URLs.txt file.
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index f86376a91..ef2fb4bf9 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -32,6 +32,7 @@ from cyberdrop_dl.exceptions import (
InvalidExtensionError,
InvalidURLError,
NoExtensionError,
+ TooManyCrawlerErrors,
get_origin,
)
from cyberdrop_dl.utils import css
@@ -96,6 +97,8 @@ def error_handling_wrapper(
link_to_show: URL | str = ""
try:
return await func(*args, **kwargs)
+ except TooManyCrawlerErrors:
+ return
except CDLBaseError as e:
error_log_msg = ErrorLogMessage(e.ui_failure, str(e))
origin = e.origin