Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: strip redundant frames from tracebacks (#2143)
NTFSvolume091192a3ea9c
diff --git a/cyberdrop_dl/utils/errors.py b/cyberdrop_dl/utils/errors.pyindex 50338e9d6..e907ddb86 100644--- a/cyberdrop_dl/utils/errors.py+++ b/cyberdrop_dl/utils/errors.py@@ -12,11 +12,12 @@ import mega.errorsimport yarlfrom pydantic import ValidationError-from cyberdrop_dl.exceptions import CDLAppError, CDLBaseError, create_error_msg, get_origin+from cyberdrop_dl.exceptions import CDLAppError, CDLBaseError, ScrapeError, create_error_msg, get_originif TYPE_CHECKING:from collections.abc import Callable, Coroutine, Generatorfrom pathlib import Path+ from types import TracebackTypefrom cyberdrop_dl.downloader.http import Downloaderfrom cyberdrop_dl.manager import Manager@@ -29,6 +30,7 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)_ERROR_WRAPPER_ATTR = "__cdl_error_wrapped__"+__tracebackhide__ = Truedef is_error_wrapped(method: object) -> bool:@@ -40,53 +42,94 @@ def _mark_as_safe[T](fn: T) -> T:return fn-def _clean_curl_error(e: object) -> str:- return str(e).partition(". See https://curl.se/")[0]+def _strip_tb_frames(tb: TracebackType | None) -> TracebackType | None:+ "Similar to pytest, hide anything with __tracebackhide__"+ if tb is None:+ return None+ for ns in (tb.tb_frame.f_locals, tb.tb_frame.f_globals):+ if ns.get("__tracebackhide__"):+ return _strip_tb_frames(tb.tb_next)-def _curl_context() -> Generator[None]:- try:- from curl_cffi.requests import exceptions as curl_exceptions- except ImportError:- yield+ tb.tb_next = _strip_tb_frames(tb.tb_next)+ return tb+++def _clean_exception(exc: BaseException | None) -> None:+ if exc is None:return- try:- yield- except curl_exceptions.Timeout as e:- log_msg = _clean_curl_error(repr(e))- raise CDLAppError("Timeout", log_msg) from None- except curl_exceptions.DNSError as e:- log_msg = _clean_curl_error(repr(e))- raise CDLAppError("Client Connector Error", log_msg) from None- except curl_exceptions.RequestException as e:- log_msg = _clean_curl_error(e)- raise CDLAppError(f"Curl Error ({e.code})", log_msg) from None++ exc.__traceback__ = _strip_tb_frames(exc.__traceback__)++ _clean_exception(exc.__cause__)+ _clean_exception(exc.__context__)@contextlib.contextmanager-def _wreq_context() -> Generator[None]:- try:- from wreq import exceptions- except ImportError:- yield- return+def _tb_ctx() -> Generator[None]:try:yield+ except Exception as e:+ _clean_exception(e)+ raise- except exceptions.TimeoutError as e:- raise CDLAppError("Timeout", repr(e)) from None- except (- exceptions.ConnectionResetError,- exceptions.TlsError,- exceptions.ProxyConnectionError,- exceptions.ConnectionError,- exceptions.RedirectError,- exceptions.UpgradeError,- ) as e:- raise CDLAppError(type(e).__name__, repr(e)) from None- except (exceptions.BodyError, exceptions.BuilderError, exceptions.DecodingError, exceptions.RequestError) as e:- raise CDLAppError(type(e).__name__, repr(e)) from e++def _clean_curl_error(e: object) -> str:+ return str(e).partition(". See https://curl.se/")[0]+++try:+ from curl_cffi.requests import exceptions as curl_exceptions+except ImportError:+ _curl_context: Callable[[], contextlib.AbstractContextManager[None]] = contextlib.nullcontext++else:++ @contextlib.contextmanager+ def _curl_context() -> Generator[None]:+ try:+ yield+ except curl_exceptions.Timeout as e:+ log_msg = _clean_curl_error(repr(e))+ raise CDLAppError("Timeout", log_msg) from None+ except curl_exceptions.DNSError as e:+ log_msg = _clean_curl_error(repr(e))+ raise CDLAppError("Client Connector Error", log_msg) from None+ except curl_exceptions.RequestException as e:+ log_msg = _clean_curl_error(e)+ raise CDLAppError(f"Curl Error ({e.code})", log_msg) from None+++try:+ from wreq import exceptions as wreq_exceptions+except ImportError:+ _wreq_context: Callable[[], contextlib.AbstractContextManager[None]] = contextlib.nullcontext++else:++ @contextlib.contextmanager+ def _wreq_context() -> Generator[None]:+ try:+ yield++ except wreq_exceptions.TimeoutError as e:+ raise CDLAppError("Timeout", repr(e)) from None+ except (+ wreq_exceptions.ConnectionResetError,+ wreq_exceptions.TlsError,+ wreq_exceptions.ProxyConnectionError,+ wreq_exceptions.ConnectionError,+ wreq_exceptions.RedirectError,+ wreq_exceptions.UpgradeError,+ ) as e:+ raise CDLAppError(type(e).__name__, repr(e)) from None+ except (+ wreq_exceptions.BodyError,+ wreq_exceptions.BuilderError,+ wreq_exceptions.DecodingError,+ wreq_exceptions.RequestError,+ ) as e:+ raise CDLAppError(type(e).__name__, repr(e)) from e@contextlib.contextmanager@@ -160,6 +203,8 @@ def _builtin_context() -> Generator[None]:yieldexcept NotImplementedError as e:raise CDLAppError("NotImplemented") from e+ except LookupError as e:+ raise ScrapeError(422, repr(e)) from eexcept TimeoutError as e:raise CDLAppError("Timeout", repr(e)) from None@@ -176,6 +221,7 @@ def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | yarreal_url: yarl.URL | str = ""try:with (+ _tb_ctx(),_builtin_context(),_pydantic_context(),_aiohttp_context(),diff --git a/pyproject.toml b/pyproject.tomlindex 9b84e7c69..f57a50eee 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,6 +1,6 @@[project]name = "cyberdrop-dl-patched"-version = "10.9.0"+version = "10.9.1dev0"authors = [{name = "Jacob B", email = "[email protected]"}]classifiers = ["Development Status :: 5 - Production/Stable",diff --git a/uv.lock b/uv.lockindex c0e4246b5..daf4afe72 100644--- a/uv.lock+++ b/uv.lock@@ -683,7 +683,7 @@ wheels = [[[package]]name = "cyberdrop-dl-patched"-version = "10.9.0"+version = "10.9.1.dev0"source = { editable = "." }dependencies = [{ name = "aiodns", marker = "sys_platform != 'android' and sys_platform != 'win32'" },