Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
tests: add test for @error_handling_wrapper usage (#1920)
NTFSvolumeae5efdb037b1
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex a410b0f8b..1c60b3b0d 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -25,16 +25,7 @@ from cyberdrop_dl.downloader.http import Downloaderfrom cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.mediaprops import ISO639Subtitle, Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem-from cyberdrop_dl.utils import (- css,- dates,- error_handling_context,- error_handling_wrapper,- is_absolute_http_url,- is_blob_or_svg,- m3u8,- parse_url,-)+from cyberdrop_dl.utils import css, dates, error_handling_context, is_absolute_http_url, is_blob_or_svg, m3u8, parse_urlfrom cyberdrop_dl.utils.filepath import compose_filename, get_filename_and_ext, remove_file_idfrom cyberdrop_dl.utils.strings import safe_format@@ -816,14 +807,14 @@ class Crawler(HTTPMixin, HLSMixin, ABC):breakpage_url = self.parse_url(page_url_str, relative_to=relative_to, trim=trim)- @error_handling_wrapperasync def direct_file(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None, assume_ext: str | None = None) -> None:"""Download a direct link file. Filename will be the url slug"""- link = url or scrape_item.url- filename, ext = self.get_filename_and_ext(link.name or link.parent.name, assume_ext=assume_ext)- await self.handle_file(link, scrape_item, filename, ext)+ url = url or scrape_item.url+ with self.catch_errors(url):+ filename, ext = self.get_filename_and_ext(url.name or url.parent.name, assume_ext=assume_ext)+ await self.handle_file(url, scrape_item, filename, ext)@final@contextlib.asynccontextmanager@@ -847,13 +838,13 @@ class Crawler(HTTPMixin, HLSMixin, ABC):return resp.url@final- @error_handling_wrapperasync def follow_redirect(self, scrape_item: ScrapeItem) -> None:- redirect = await self._get_redirect_url(scrape_item.url)- if scrape_item.url == redirect:- raise ScrapeError(422, "Infinite redirect")- scrape_item.url = redirect- self.create_task(self.run(scrape_item))+ with self.catch_errors(scrape_item):+ redirect = await self._get_redirect_url(scrape_item.url)+ if scrape_item.url == redirect:+ raise ScrapeError(422, "Infinite redirect")+ scrape_item.url = redirect+ self.create_task(self.run(scrape_item))async def request_m3u8_playlist(self,diff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.pyindex d35f6e404..5989df338 100644--- a/cyberdrop_dl/utils/__init__.py+++ b/cyberdrop_dl/utils/__init__.py@@ -47,11 +47,23 @@ if TYPE_CHECKING:_Origin = TypeVar("_Origin", bound=ScrapeItem | MediaItem | yarl.URL)_P = ParamSpec("_P")+_T = TypeVar("_T")_R = TypeVar("_R")logger = logging.getLogger(__name__)+_ERROR_WRAPPER_ATTR = "__cdl_error_wrapped__"+++def is_error_wrapped(method: object) -> bool:+ return getattr(method, _ERROR_WRAPPER_ATTR, False)+++def mark_as_safe(fn: _T) -> _T:+ setattr(fn, _ERROR_WRAPPER_ATTR, True)+ return fn+@contextlib.contextmanagerdef group_exceptions(message: str | None = None) -> Generator[None]:@@ -188,7 +200,7 @@ def error_handling_wrapper(with error_handling_context(self, item):return await func(self, item, *args, **kwargs)- return async_wrapper+ return mark_as_safe(async_wrapper)@functools.wraps(func)def wrapper(self: _HasManagerT, item: _Origin, *args: _P.args, **kwargs: _P.kwargs) -> _R:@@ -197,7 +209,7 @@ def error_handling_wrapper(assert not inspect.isawaitable(result)return result- return wrapper+ return mark_as_safe(wrapper)def delete_empty_files_and_folders(path: Path) -> None:diff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.pyindex b4a28a6bf..4e103b6da 100644--- a/tests/crawlers/test_crawlers.py+++ b/tests/crawlers/test_crawlers.py@@ -3,7 +3,7 @@ from __future__ import annotationsimport dataclassesimport importlib.utilimport re-from collections.abc import Sequence+from collections.abc import Callable, Generator, Sequencefrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, NotRequiredfrom unittest import mock@@ -221,3 +221,43 @@ def test_invalid_crawler_modules_should_raise_import_error() -> None:with pytest.raises(ImportError, match="Could not import crawlers from module"):Registry._import_module("cyberdrop_dl.crawler.fake_crawler_12345")+++def test_public_methods_have_error_handling_wrapper() -> None:+ import inspect++ from cyberdrop_dl.crawlers.crawler import Crawler, Registry+ from cyberdrop_dl.utils import is_error_wrapped++ def returns_none(func: Callable[..., Any]) -> bool:+ return_ = inspect.signature(func).return_annotation+ return return_ == "None" or return_ is type(None)++ def public_methods(cls: type):+ return (+ (name, method)+ for name, method in inspect.getmembers(cls, predicate=inspect.isfunction)+ if not name.startswith("_")+ )++ base_methods = {name for name, _ in public_methods(Crawler)}++ def unsafe_public_methods(cls: type) -> Generator[str]:+ return (+ name+ for name, method in public_methods(cls)+ if name not in base_methods and not is_error_wrapped(method) and returns_none(method)+ )++ Registry.import_all()++ errors: list[Exception] = []+ for crawler in sorted(Registry.concrete | Registry.generic, key=lambda x: x.__name__):+ unwrapped_methods = sorted(unsafe_public_methods(crawler))+ if unwrapped_methods:+ errors.append(ValueError(crawler.__name__, unwrapped_methods))++ if errors:+ exc = BaseExceptionGroup("Some crawler has unsafe public methods that could crash CDL", errors)+ exc.add_note("Wrap them with @error_handling_wrapper or make them private")+ raise exc