Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: move error utils to their own module

NTFSvolume198470fb1332
diff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.py
index 6112c9713..799b28b50 100644
--- a/cyberdrop_dl/utils/__init__.py
+++ b/cyberdrop_dl/utils/__init__.py
@@ -1,206 +1,28 @@
from __future__ import annotations
import base64
-import contextlib
-import functools
-import inspect
import itertools
import logging
import platform
import sys
-from http import HTTPStatus
-from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, Protocol, TypeVar, cast, overload
+from typing import TYPE_CHECKING, Any, TypeVar
-import yarl
-from aiohttp import ClientConnectorError, TooManyRedirects
-from mega.errors import MegaNzError
-from pydantic import ValidationError
-
-from cyberdrop_dl.exceptions import CDLBaseError, ErrorLogMessage, create_error_msg, get_origin
from cyberdrop_dl.utils._dataclasses import DictDataclass, deserialize, filter_data, type_adapter # noqa: F401
+from cyberdrop_dl.utils._errors import error_handling_context, error_handling_wrapper # noqa: F401
from cyberdrop_dl.utils._path_traverse import has_partial_files, partial_files
from cyberdrop_dl.utils._url import is_absolute_http_url, remove_trailing_slash # noqa: F401
-from cyberdrop_dl.utils._url import parse_http_url as parse_url
+from cyberdrop_dl.utils._url import parse_http_url as parse_url # noqa: F401
if TYPE_CHECKING:
- from collections.abc import Callable, Coroutine, Generator, Iterable
+ from collections.abc import Generator, Iterable
from pathlib import Path
- from cyberdrop_dl.downloader.http import Downloader
from cyberdrop_dl.manager import Manager
- from cyberdrop_dl.url_objects import MediaItem, ScrapeItem
-
- class _HasManager(Protocol):
- manager: Manager
- _HasManagerT = TypeVar("_HasManagerT", bound=_HasManager)
- _Origin = TypeVar("_Origin", bound=ScrapeItem | MediaItem | yarl.URL)
-_P = ParamSpec("_P")
_T = TypeVar("_T")
-_R = TypeVar("_R")
-
-
logger = logging.getLogger(__name__)
-_ERROR_WRAPPER_ATTR = "__cdl_error_wrapped__"
-_ = parse_url
-
-
-def is_error_wrapped(method: object) -> bool:
- return getattr(method, _ERROR_WRAPPER_ATTR, False)
-
-
-def _mark_as_safe(fn: _T) -> _T:
- setattr(fn, _ERROR_WRAPPER_ATTR, True)
- return fn
-
-
-def group_exceptions(message: str | None = None) -> Generator[None]:
- try:
- yield
- except ExceptionGroup as e:
- raise ExceptionGroup(message or _exc_group_msg(e), e.exceptions) from None
-
-
-def _exc_group_msg(e: ExceptionGroup) -> str:
- if e.message and "unhandled errors in a TaskGroup" not in e.message:
- return e.message
-
- first = e.exceptions[0]
- return getattr(first, "ui_failure", None) or str(first)
-
-
-def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | yarl.URL) -> Generator[None]: # noqa: C901, PLR0912
- link: yarl.URL = item if isinstance(item, yarl.URL) else item.url
- error_log_msg = origin = exc_info = None
- link_to_show: yarl.URL | str = ""
- is_segment: bool = getattr(item, "is_segment", False)
- try:
- with group_exceptions():
- yield
- except ExceptionGroup as e:
- error_log_msg = ErrorLogMessage(_exc_group_msg(e), str(e))
- exc_info = e.with_traceback(None)
- except CDLBaseError as e:
- error_log_msg = ErrorLogMessage(e.ui_failure, str(e))
- origin = e.origin
- link_to_show = getattr(e, "url", None) or link_to_show
- exc_info = e.__cause__
- except NotImplementedError as e:
- error_log_msg = ErrorLogMessage("NotImplemented")
- exc_info = e
- except TooManyRedirects as e:
- ui_failure = "Too Many Redirects"
- info = {
- "url": str(e.request_info.real_url),
- "history": tuple(str(r.real_url) for r in e.history),
- }
- error_log_msg = ErrorLogMessage(ui_failure, f"{ui_failure}\n{info}")
- except MegaNzError as e:
- if code := getattr(e, "code", None):
- if http_code := {
- -4: HTTPStatus.TOO_MANY_REQUESTS,
- -8: HTTPStatus.GONE,
- -9: HTTPStatus.GONE,
- -16: HTTPStatus.FORBIDDEN,
- -17: 509,
- -24: 509,
- -401: 509,
- }.get(code):
- ui_failure = create_error_msg(http_code)
- else:
- ui_failure = f"MegaNZ Error [{code}]"
- else:
- ui_failure = "MegaNZ Error"
-
- error_log_msg = ErrorLogMessage(ui_failure, f"{ui_failure} {e!s}")
-
- except TimeoutError as e:
- error_log_msg = ErrorLogMessage("Timeout", repr(e))
- except ClientConnectorError as e:
- ui_failure = "Client Connector Error"
- suffix = "" if (link.host or "").startswith(e.host) else f" from {link}"
- log_msg = f"{e}{suffix}. If you're using a VPN, try turning it off"
- error_log_msg = ErrorLogMessage(ui_failure, log_msg)
- except ValidationError as e:
- exc_info = e
- ui_failure = create_error_msg(422)
- log_msg = str(e).partition("For further information")[0].strip()
- error_log_msg = ErrorLogMessage(ui_failure, log_msg)
- except Exception as e: # noqa: BLE001
- exc_info = e
- error_log_msg = ErrorLogMessage.from_unknown_exc(e)
-
- if error_log_msg is None or is_segment:
- return
-
- _log_error(self, link_to_show or link, item, error_log_msg, exc_info, origin)
-
-
-def _log_error( # noqa: PLR0913
- self: _HasManager,
- link_to_show: yarl.URL | str,
- item: ScrapeItem | MediaItem | yarl.URL,
- error_log_msg: ErrorLogMessage,
- exc_info: BaseException | None,
- origin: ScrapeItem | MediaItem | yarl.URL | Path | None,
-) -> None:
- origin = origin or get_origin(item)
- is_downloader = bool(getattr(self, "log_prefix", False))
- if is_downloader:
- self, item = cast("Downloader", self), cast("MediaItem", item)
- logger.error(
- f"{self.log_prefix} Failed: {item.url} ({error_log_msg.main_log_msg}) \n -> Referer: {item.referer}",
- exc_info=exc_info,
- )
- self.manager.logs.write_download_error(item, error_log_msg.csv_log_msg)
- self.manager.scrape_mapper.tui.files.stats.failed += 1
- self.manager.scrape_mapper.tui.download_errors.add(error_log_msg.ui_failure)
- return
-
- logger.error(f"Scrape Failed: {link_to_show} ({error_log_msg.main_log_msg})", exc_info=exc_info)
- self.manager.logs.write_scrape_error(link_to_show, error_log_msg.csv_log_msg, origin) # pyright: ignore[reportArgumentType]
- self.manager.scrape_mapper.tui.scrape_errors.add(error_log_msg.ui_failure)
-
-
-@overload
-def error_handling_wrapper(
- func: Callable[Concatenate[_HasManagerT, _Origin, _P], _R],
-) -> Callable[Concatenate[_HasManagerT, _Origin, _P], _R]: ...
-
-
-@overload
-def error_handling_wrapper(
- func: Callable[Concatenate[_HasManagerT, _Origin, _P], Coroutine[None, None, _R]],
-) -> Callable[Concatenate[_HasManagerT, _Origin, _P], Coroutine[None, None, _R]]: ...
-
-
-def error_handling_wrapper(
- func: Callable[Concatenate[_HasManagerT, _Origin, _P], _R | Coroutine[None, None, _R]],
-) -> Callable[Concatenate[_HasManagerT, _Origin, _P], _R | Coroutine[None, None, _R]]:
- """Wrapper handles errors for url scraping."""
-
- if inspect.iscoroutinefunction(func):
-
- @functools.wraps(func)
- async def async_wrapper(self: _HasManagerT, item: _Origin, *args: _P.args, **kwargs: _P.kwargs) -> _R:
- with error_handling_context(self, item):
- return await func(self, item, *args, **kwargs)
-
- return _mark_as_safe(async_wrapper)
-
- @functools.wraps(func)
- def wrapper(self: _HasManagerT, item: _Origin, *args: _P.args, **kwargs: _P.kwargs) -> _R:
- with error_handling_context(self, item):
- result = func(self, item, *args, **kwargs)
- assert not inspect.isawaitable(result)
- return result
-
- return _mark_as_safe(wrapper)
-
def delete_empty_files_and_folders(path: Path) -> None:
"""walks and removes in place"""
diff --git a/cyberdrop_dl/utils/_errors.py b/cyberdrop_dl/utils/_errors.py
new file mode 100644
index 000000000..5de420b4d
--- /dev/null
+++ b/cyberdrop_dl/utils/_errors.py
@@ -0,0 +1,193 @@
+from __future__ import annotations
+
+import contextlib
+import functools
+import inspect
+import logging
+from http import HTTPStatus
+from typing import TYPE_CHECKING, Concatenate, ParamSpec, Protocol, TypeVar, cast, overload
+
+import yarl
+from aiohttp import ClientConnectorError, TooManyRedirects
+from mega.errors import MegaNzError
+from pydantic import ValidationError
+
+from cyberdrop_dl.exceptions import CDLBaseError, ErrorLogMessage, create_error_msg, get_origin
+
+if TYPE_CHECKING:
+ from collections.abc import Callable, Coroutine, Generator
+ from pathlib import Path
+
+ from cyberdrop_dl.downloader.http import Downloader
+ from cyberdrop_dl.manager import Manager
+ from cyberdrop_dl.url_objects import MediaItem, ScrapeItem
+
+ class _HasManager(Protocol):
+ manager: Manager
+
+ _HasManagerT = TypeVar("_HasManagerT", bound=_HasManager)
+ _Origin = TypeVar("_Origin", bound=ScrapeItem | MediaItem | yarl.URL)
+
+_P = ParamSpec("_P")
+_T = TypeVar("_T")
+_R = TypeVar("_R")
+
+
+logger = logging.getLogger(__name__)
+
+_ERROR_WRAPPER_ATTR = "__cdl_error_wrapped__"
+
+
+def is_error_wrapped(method: object) -> bool:
+ return getattr(method, _ERROR_WRAPPER_ATTR, False)
+
+
+def _mark_as_safe(fn: _T) -> _T:
+ setattr(fn, _ERROR_WRAPPER_ATTR, True)
+ return fn
+
+
+def group_exceptions(message: str | None = None) -> Generator[None]:
+ try:
+ yield
+ except ExceptionGroup as e:
+ raise ExceptionGroup(message or _exc_group_msg(e), e.exceptions) from None
+
+
+def _exc_group_msg(e: ExceptionGroup) -> str:
+ if e.message and "unhandled errors in a TaskGroup" not in e.message:
+ return e.message
+
+ first = e.exceptions[0]
+ return getattr(first, "ui_failure", None) or str(first)
+
+
+def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | yarl.URL) -> Generator[None]: # noqa: C901, PLR0912
+ link: yarl.URL = item if isinstance(item, yarl.URL) else item.url
+ error_log_msg = origin = exc_info = None
+ link_to_show: yarl.URL | str = ""
+ is_segment: bool = getattr(item, "is_segment", False)
+ try:
+ with group_exceptions():
+ yield
+ except ExceptionGroup as e:
+ error_log_msg = ErrorLogMessage(_exc_group_msg(e), str(e))
+ exc_info = e.with_traceback(None)
+ except CDLBaseError as e:
+ error_log_msg = ErrorLogMessage(e.ui_failure, str(e))
+ origin = e.origin
+ link_to_show = getattr(e, "url", None) or link_to_show
+ exc_info = e.__cause__
+ except NotImplementedError as e:
+ error_log_msg = ErrorLogMessage("NotImplemented")
+ exc_info = e
+ except TooManyRedirects as e:
+ ui_failure = "Too Many Redirects"
+ info = {
+ "url": str(e.request_info.real_url),
+ "history": tuple(str(r.real_url) for r in e.history),
+ }
+ error_log_msg = ErrorLogMessage(ui_failure, f"{ui_failure}\n{info}")
+ except MegaNzError as e:
+ if code := getattr(e, "code", None):
+ if http_code := {
+ -4: HTTPStatus.TOO_MANY_REQUESTS,
+ -8: HTTPStatus.GONE,
+ -9: HTTPStatus.GONE,
+ -16: HTTPStatus.FORBIDDEN,
+ -17: 509,
+ -24: 509,
+ -401: 509,
+ }.get(code):
+ ui_failure = create_error_msg(http_code)
+ else:
+ ui_failure = f"MegaNZ Error [{code}]"
+ else:
+ ui_failure = "MegaNZ Error"
+
+ error_log_msg = ErrorLogMessage(ui_failure, f"{ui_failure} {e!s}")
+
+ except TimeoutError as e:
+ error_log_msg = ErrorLogMessage("Timeout", repr(e))
+ except ClientConnectorError as e:
+ ui_failure = "Client Connector Error"
+ suffix = "" if (link.host or "").startswith(e.host) else f" from {link}"
+ log_msg = f"{e}{suffix}. If you're using a VPN, try turning it off"
+ error_log_msg = ErrorLogMessage(ui_failure, log_msg)
+ except ValidationError as e:
+ exc_info = e
+ ui_failure = create_error_msg(422)
+ log_msg = str(e).partition("For further information")[0].strip()
+ error_log_msg = ErrorLogMessage(ui_failure, log_msg)
+ except Exception as e: # noqa: BLE001
+ exc_info = e
+ error_log_msg = ErrorLogMessage.from_unknown_exc(e)
+
+ if error_log_msg is None or is_segment:
+ return
+
+ _log_error(self, link_to_show or link, item, error_log_msg, exc_info, origin)
+
+
+def _log_error( # noqa: PLR0913
+ self: _HasManager,
+ link_to_show: yarl.URL | str,
+ item: ScrapeItem | MediaItem | yarl.URL,
+ error_log_msg: ErrorLogMessage,
+ exc_info: BaseException | None,
+ origin: ScrapeItem | MediaItem | yarl.URL | Path | None,
+) -> None:
+ origin = origin or get_origin(item)
+ is_downloader = bool(getattr(self, "log_prefix", False))
+ if is_downloader:
+ self, item = cast("Downloader", self), cast("MediaItem", item)
+ logger.error(
+ f"{self.log_prefix} Failed: {item.url} ({error_log_msg.main_log_msg}) \n -> Referer: {item.referer}",
+ exc_info=exc_info,
+ )
+ self.manager.logs.write_download_error(item, error_log_msg.csv_log_msg)
+ self.manager.scrape_mapper.tui.files.stats.failed += 1
+ self.manager.scrape_mapper.tui.download_errors.add(error_log_msg.ui_failure)
+ return
+
+ logger.error(f"Scrape Failed: {link_to_show} ({error_log_msg.main_log_msg})", exc_info=exc_info)
+ self.manager.logs.write_scrape_error(link_to_show, error_log_msg.csv_log_msg, origin) # pyright: ignore[reportArgumentType]
+ self.manager.scrape_mapper.tui.scrape_errors.add(error_log_msg.ui_failure)
+
+
+@overload
+def error_handling_wrapper(
+ func: Callable[Concatenate[_HasManagerT, _Origin, _P], _R],
+) -> Callable[Concatenate[_HasManagerT, _Origin, _P], _R]: ...
+
+
+@overload
+def error_handling_wrapper(
+ func: Callable[Concatenate[_HasManagerT, _Origin, _P], Coroutine[None, None, _R]],
+) -> Callable[Concatenate[_HasManagerT, _Origin, _P], Coroutine[None, None, _R]]: ...
+
+
+def error_handling_wrapper(
+ func: Callable[Concatenate[_HasManagerT, _Origin, _P], _R | Coroutine[None, None, _R]],
+) -> Callable[Concatenate[_HasManagerT, _Origin, _P], _R | Coroutine[None, None, _R]]:
+ """Wrapper handles errors for url scraping."""
+
+ if inspect.iscoroutinefunction(func):
+
+ @functools.wraps(func)
+ async def async_wrapper(self: _HasManagerT, item: _Origin, *args: _P.args, **kwargs: _P.kwargs) -> _R:
+ with error_handling_context(self, item):
+ return await func(self, item, *args, **kwargs)
+
+ return _mark_as_safe(async_wrapper)
+
+ @functools.wraps(func)
+ def wrapper(self: _HasManagerT, item: _Origin, *args: _P.args, **kwargs: _P.kwargs) -> _R:
+ with error_handling_context(self, item):
+ result = func(self, item, *args, **kwargs)
+ assert not inspect.isawaitable(result)
+ return result
+
+ return _mark_as_safe(wrapper)
diff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.py
index 46cc54da9..4874c9b89 100644
--- a/tests/crawlers/test_crawlers.py
+++ b/tests/crawlers/test_crawlers.py
@@ -233,7 +233,7 @@ def test_public_methods_have_error_handling_wrapper() -> None:
import inspect
from cyberdrop_dl.crawlers.crawler import Crawler, Registry
- from cyberdrop_dl.utils import is_error_wrapped
+ from cyberdrop_dl.utils._errors import is_error_wrapped
def returns_none(func: Callable[..., Any]) -> bool:
return_ = inspect.signature(func).return_annotation