Skip to content

Bulk asynchronous downloader for multiple file hosts

tests: add test for @error_handling_wrapper usage (#1920)

NTFSvolumeae5efdb037b1
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index a410b0f8b..1c60b3b0d 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -25,16 +25,7 @@ from cyberdrop_dl.downloader.http import Downloader
from cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeError
from cyberdrop_dl.mediaprops import ISO639Subtitle, Resolution
from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem
-from cyberdrop_dl.utils import (
- css,
- dates,
- error_handling_context,
- error_handling_wrapper,
- is_absolute_http_url,
- is_blob_or_svg,
- m3u8,
- parse_url,
-)
+from cyberdrop_dl.utils import css, dates, error_handling_context, is_absolute_http_url, is_blob_or_svg, m3u8, parse_url
from cyberdrop_dl.utils.filepath import compose_filename, get_filename_and_ext, remove_file_id
from cyberdrop_dl.utils.strings import safe_format
@@ -816,14 +807,14 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
break
page_url = self.parse_url(page_url_str, relative_to=relative_to, trim=trim)
- @error_handling_wrapper
async def direct_file(
self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None, assume_ext: str | None = None
) -> None:
"""Download a direct link file. Filename will be the url slug"""
- link = url or scrape_item.url
- filename, ext = self.get_filename_and_ext(link.name or link.parent.name, assume_ext=assume_ext)
- await self.handle_file(link, scrape_item, filename, ext)
+ url = url or scrape_item.url
+ with self.catch_errors(url):
+ filename, ext = self.get_filename_and_ext(url.name or url.parent.name, assume_ext=assume_ext)
+ await self.handle_file(url, scrape_item, filename, ext)
@final
@contextlib.asynccontextmanager
@@ -847,13 +838,13 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
return resp.url
@final
- @error_handling_wrapper
async def follow_redirect(self, scrape_item: ScrapeItem) -> None:
- redirect = await self._get_redirect_url(scrape_item.url)
- if scrape_item.url == redirect:
- raise ScrapeError(422, "Infinite redirect")
- scrape_item.url = redirect
- self.create_task(self.run(scrape_item))
+ with self.catch_errors(scrape_item):
+ redirect = await self._get_redirect_url(scrape_item.url)
+ if scrape_item.url == redirect:
+ raise ScrapeError(422, "Infinite redirect")
+ scrape_item.url = redirect
+ self.create_task(self.run(scrape_item))
async def request_m3u8_playlist(
self,
diff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.py
index d35f6e404..5989df338 100644
--- a/cyberdrop_dl/utils/__init__.py
+++ b/cyberdrop_dl/utils/__init__.py
@@ -47,11 +47,23 @@ if TYPE_CHECKING:
_Origin = TypeVar("_Origin", bound=ScrapeItem | MediaItem | yarl.URL)
_P = ParamSpec("_P")
+_T = TypeVar("_T")
_R = TypeVar("_R")
logger = logging.getLogger(__name__)
+_ERROR_WRAPPER_ATTR = "__cdl_error_wrapped__"
+
+
+def is_error_wrapped(method: object) -> bool:
+ return getattr(method, _ERROR_WRAPPER_ATTR, False)
+
+
+def mark_as_safe(fn: _T) -> _T:
+ setattr(fn, _ERROR_WRAPPER_ATTR, True)
+ return fn
+
@contextlib.contextmanager
def group_exceptions(message: str | None = None) -> Generator[None]:
@@ -188,7 +200,7 @@ def error_handling_wrapper(
with error_handling_context(self, item):
return await func(self, item, *args, **kwargs)
- return async_wrapper
+ return mark_as_safe(async_wrapper)
@functools.wraps(func)
def wrapper(self: _HasManagerT, item: _Origin, *args: _P.args, **kwargs: _P.kwargs) -> _R:
@@ -197,7 +209,7 @@ def error_handling_wrapper(
assert not inspect.isawaitable(result)
return result
- return wrapper
+ return mark_as_safe(wrapper)
def delete_empty_files_and_folders(path: Path) -> None:
diff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.py
index b4a28a6bf..4e103b6da 100644
--- a/tests/crawlers/test_crawlers.py
+++ b/tests/crawlers/test_crawlers.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import dataclasses
import importlib.util
import re
-from collections.abc import Sequence
+from collections.abc import Callable, Generator, Sequence
from pathlib import Path
from typing import TYPE_CHECKING, Any, NotRequired
from unittest import mock
@@ -221,3 +221,43 @@ def test_invalid_crawler_modules_should_raise_import_error() -> None:
with pytest.raises(ImportError, match="Could not import crawlers from module"):
Registry._import_module("cyberdrop_dl.crawler.fake_crawler_12345")
+
+
+def test_public_methods_have_error_handling_wrapper() -> None:
+ import inspect
+
+ from cyberdrop_dl.crawlers.crawler import Crawler, Registry
+ from cyberdrop_dl.utils import is_error_wrapped
+
+ def returns_none(func: Callable[..., Any]) -> bool:
+ return_ = inspect.signature(func).return_annotation
+ return return_ == "None" or return_ is type(None)
+
+ def public_methods(cls: type):
+ return (
+ (name, method)
+ for name, method in inspect.getmembers(cls, predicate=inspect.isfunction)
+ if not name.startswith("_")
+ )
+
+ base_methods = {name for name, _ in public_methods(Crawler)}
+
+ def unsafe_public_methods(cls: type) -> Generator[str]:
+ return (
+ name
+ for name, method in public_methods(cls)
+ if name not in base_methods and not is_error_wrapped(method) and returns_none(method)
+ )
+
+ Registry.import_all()
+
+ errors: list[Exception] = []
+ for crawler in sorted(Registry.concrete | Registry.generic, key=lambda x: x.__name__):
+ unwrapped_methods = sorted(unsafe_public_methods(crawler))
+ if unwrapped_methods:
+ errors.append(ValueError(crawler.__name__, unwrapped_methods))
+
+ if errors:
+ exc = BaseExceptionGroup("Some crawler has unsafe public methods that could crash CDL", errors)
+ exc.add_note("Wrap them with @error_handling_wrapper or make them private")
+ raise exc