Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: disable crawlers after too many errors (#1124)
NTFSvolume8b31e8287c50
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex 5c94c6fd2..92ee3166d 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -46,20 +46,21 @@ def limiter(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutinasync def wrapper(*args, **kwargs) -> R:self: DownloadClient = args[0]domain: str = args[1]- domain_limiter = await self.client_manager.get_rate_limiter(domain)- await asyncio.sleep(await self.client_manager.get_downloader_spacer(domain))- await self._global_limiter.acquire()- await domain_limiter.acquire()-- async with aiohttp.ClientSession(- headers=self._headers,- raise_for_status=False,- cookie_jar=self.client_manager.cookies,- timeout=self._timeouts,- trace_configs=self.trace_configs,- ) as client:- kwargs["client_session"] = client- return await func(*args, **kwargs)+ with self.client_manager.request_context(domain):+ domain_limiter = await self.client_manager.get_rate_limiter(domain)+ await asyncio.sleep(await self.client_manager.get_downloader_spacer(domain))+ await self._global_limiter.acquire()+ await domain_limiter.acquire()++ async with aiohttp.ClientSession(+ headers=self._headers,+ raise_for_status=False,+ cookie_jar=self.client_manager.cookies,+ timeout=self._timeouts,+ trace_configs=self.trace_configs,+ ) as client:+ kwargs["client_session"] = client+ return await func(*args, **kwargs)return wrapperdiff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.pyindex 547242da6..1c732447a 100644--- a/cyberdrop_dl/clients/scraper_client.py+++ b/cyberdrop_dl/clients/scraper_client.py@@ -34,48 +34,44 @@ if TYPE_CHECKING:from cyberdrop_dl.managers.client_manager import ClientManager-curl_import_error = None+_curl_import_error = Nonetry:from curl_cffi.requests import AsyncSessionexcept ImportError as e:- curl_import_error = e+ _curl_import_error = e-P = ParamSpec("P")-R = TypeVar("R")-T = TypeVar("T")+_P = ParamSpec("_P")+_R = TypeVar("_R")+_T = TypeVar("_T")-def limiter(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Coroutine[None, None, R]]:+def limiter(func: Callable[_P, Coroutine[None, None, _R]]) -> Callable[_P, Coroutine[None, None, _R]]:"""Wrapper to handle limits for scrape session."""@wraps(func)- async def wrapper(*args, **kwargs) -> R:+ async def wrapper(*args, **kwargs) -> _R:self: ScraperClient = args[0]domain: str = args[1]- domain_limiter = await self.client_manager.get_rate_limiter(domain)- async with self.client_manager.session_limit, self._global_limiter, domain_limiter:- await self.client_manager.manager.states.RUNNING.wait()-- if "cffi" in func.__name__:- if curl_import_error is not None:- system = "Android" if env.RUNNING_IN_TERMUX else "the system"- msg = f"curl_cffi is required to scrape URLs from {domain}, but a dependency it's not available on {system}.\n"- msg += f"See: https://github.com/lexiforest/curl_cffi/issues/74#issuecomment-1849365636\n{curl_import_error!r}"- raise ScrapeError("Missing Dependency", msg)- return await func(*args, **kwargs)+ with self.client_manager.request_context(domain):+ domain_limiter = await self.client_manager.get_rate_limiter(domain)+ async with self.client_manager.session_limit, self._global_limiter, domain_limiter:+ await self.client_manager.manager.states.RUNNING.wait()++ if "cffi" in func.__name__:+ _check_curl_cffi_is_available(domain)- return await func(*args, **kwargs)+ return await func(*args, **kwargs)return wrapper-def copy_signature(target: Callable[P, R]) -> Callable[[Callable[..., T]], Callable[P, T]]:+def copy_signature(target: Callable[_P, _R]) -> Callable[[Callable[..., _T]], Callable[_P, _T]]:"""Decorator to make a function mimic the signature of another function,but preserve the return type of the decorated function."""- def decorator(func: Callable[..., T]) -> Callable[P, T]:+ def decorator(func: Callable[..., _T]) -> Callable[_P, _T]:@functools.wraps(func)- def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:+ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T:return func(*args, **kwargs)wrapper.__signature__ = inspect.signature(target).replace( # type: ignore@@ -125,7 +121,7 @@ class ScraperClient:add_request_log_hooks(self._trace_configs)self._session = self.new_session()self.reddit_session = self.new_session()- if curl_import_error is not None:+ if _curl_import_error is not None:returnproxy = str(self.client_manager.proxy) if self.client_manager.proxy else None@@ -150,7 +146,7 @@ class ScraperClient:) -> None:await self._session.close()await self.reddit_session.close()- if curl_import_error is not None:+ if _curl_import_error is not None:returntry:await self._curl_session.close()@@ -479,3 +475,11 @@ def add_request_log_hooks(trace_configs: list[aiohttp.TraceConfig]) -> None:trace_config.on_request_start.append(on_request_start)trace_config.on_request_end.append(on_request_end)trace_configs.append(trace_config)+++def _check_curl_cffi_is_available(domain: str) -> None:+ if _curl_import_error is not None:+ system = "Android" if env.RUNNING_IN_TERMUX else "the system"+ msg = f"curl_cffi is required to scrape URLs from {domain}, but a dependency it's not available on {system}.\n"+ msg += f"See: https://github.com/lexiforest/curl_cffi/issues/74#issuecomment-1849365636\n{_curl_import_error!r}"+ raise ScrapeError("Missing Dependency", msg)diff --git a/cyberdrop_dl/exceptions.py b/cyberdrop_dl/exceptions.pyindex 4328b6890..7ec65077a 100644--- a/cyberdrop_dl/exceptions.py+++ b/cyberdrop_dl/exceptions.py@@ -29,6 +29,12 @@ CLOUDFLARE_ERRORS = {}+class TooManyCrawlerErrors(Exception): # noqa: N818+ """This exception will be raised after a crawler had too many errors proccesing URLs"""++ # This exception does not inherit from `CDLBaseError`` cause it not intented to ever by shown to the user++class CDLBaseError(Exception):"""Base exception for cyberdrop-dl errors."""diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.pyindex 4c674bec2..8aae43479 100644--- a/cyberdrop_dl/managers/client_manager.py+++ b/cyberdrop_dl/managers/client_manager.py@@ -4,6 +4,7 @@ import asyncioimport contextlibimport sslimport time+from collections import defaultdictfrom dataclasses import dataclassfrom http import HTTPStatusfrom http.cookiejar import MozillaCookieJar@@ -20,14 +21,14 @@ from yarl import URLfrom cyberdrop_dl.clients.download_client import DownloadClientfrom cyberdrop_dl.clients.scraper_client import ScraperClientfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError+from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError, TooManyCrawlerErrorsfrom cyberdrop_dl.managers.download_speed_manager import DownloadSpeedLimiterfrom cyberdrop_dl.ui.prompts.user_prompts import get_cookies_from_browsersfrom cyberdrop_dl.utils.logger import log, log_spacerfrom cyberdrop_dl.utils.utilities import get_soup_no_errorif TYPE_CHECKING:- from collections.abc import Mapping+ from collections.abc import Generator, Mappingfrom aiohttp_client_cache.response import CachedResponsefrom curl_cffi.requests.models import Response as CurlResponse@@ -44,6 +45,9 @@ DOWNLOAD_ERROR_ETAGS = {"5a56b09d-1485eb": "eFukt Video removed",}+_crawler_errors: dict[str, int] = defaultdict(int)+MAX_CRAWLER_ERRORS = 10+class DDosGuard:TITLES = ("Just a moment...", "DDoS-Guard")@@ -132,6 +136,32 @@ class ClientManager:self.downloader_session = DownloadClient(manager, self)self.flaresolverr = Flaresolverr(self)+ def check_domain_errors(self, domain: str) -> None:+ if _crawler_errors[domain] >= MAX_CRAWLER_ERRORS:+ if crawler := self.manager.scrape_mapper.disable_crawler(domain):+ msg = (+ f"{crawler.__name__} has been disabled after too many errors. "+ f"URLs from the following domains will be ignored: {crawler.SCRAPE_MAPPER_KEYS}"+ )+ log(msg, 40)+ raise TooManyCrawlerErrors++ @contextlib.contextmanager+ def request_context(self, domain: str) -> Generator[None]:+ self.check_domain_errors(domain)+ try:+ yield+ except DDOSGuardError:+ _crawler_errors[domain] += 1+ self.check_domain_errors(domain)+ raise+ else:+ # we could potencially reset the counter here+ # _crawler_errors[domain] = 0+ pass+ finally:+ pass+def load_cookie_files(self) -> None:if self.manager.config_manager.settings_data.browser_cookies.auto_import:assert self.manager.config_manager.settings_data.browser_cookies.browserdiff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex ac5c61c25..c08c3f1c6 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -347,6 +347,24 @@ class ScrapeMapper:return False+ def disable_crawler(self, domain: str) -> Crawler | None:+ """Disables a crawler at runtime, after the scrape mapper is already running.++ It does not remove the crawler from the crawlers map, it just sets it as `disabled"`++ This has the effect to silently ignore any URL that maps to that crawler, without any "unsupported" or "errors" log messages++ `domain` must match _exactly_, AKA: it must be the value of `crawler.DOMAIN`++ Returns the crawler instance that was disabled (if Any)++ """++ crawler = next((crawler for crawler in self.existing_crawlers.values() if crawler.DOMAIN == domain), None)+ if crawler and not crawler.disabled:+ crawler.disabled = True+ return crawler+def regex_links(line: str) -> list[AbsoluteHttpURL]:"""Regex grab the links from the URLs.txt file.diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex f86376a91..ef2fb4bf9 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -32,6 +32,7 @@ from cyberdrop_dl.exceptions import (InvalidExtensionError,InvalidURLError,NoExtensionError,+ TooManyCrawlerErrors,get_origin,)from cyberdrop_dl.utils import css@@ -96,6 +97,8 @@ def error_handling_wrapper(link_to_show: URL | str = ""try:return await func(*args, **kwargs)+ except TooManyCrawlerErrors:+ returnexcept CDLBaseError as e:error_log_msg = ErrorLogMessage(e.ui_failure, str(e))origin = e.origin