Skip to content

Bulk asynchronous downloader for multiple file hosts

10.4 KB
Raw Blame
from __future__ import annotations

import contextlib
import functools
import inspect
import logging
from http import HTTPStatus
from typing import TYPE_CHECKING, Concatenate, Protocol, cast, final, overload

import aiohttp.client_exceptions
import mega.errors
import yarl
from pydantic import ValidationError

from cyberdrop_dl.exceptions import CDLAppError, CDLBaseError, ScrapeError, create_error_msg, get_origin

if TYPE_CHECKING:
    from collections.abc import AsyncGenerator, Callable, Coroutine, Generator
    from pathlib import Path

    from cyberdrop_dl.downloader.http import Downloader
    from cyberdrop_dl.manager import Manager
    from cyberdrop_dl.url_objects import MediaItem, ScrapeItem

    class _HasManager(Protocol):
        manager: Manager


logger = logging.getLogger(__name__)

_ERROR_WRAPPER_ATTR = "__cdl_error_wrapped__"
__tracebackhide__ = True


def is_error_wrapped(method: object) -> bool:
    return getattr(method, _ERROR_WRAPPER_ATTR, False)


def _mark_as_safe[T](fn: T) -> T:
    setattr(fn, _ERROR_WRAPPER_ATTR, True)
    return fn


def _clean_curl_error(e: object) -> str:
    return str(e).partition(". See https://curl.se/")[0]


try:
    from curl_cffi.requests import exceptions as curl_exceptions
except ImportError:
    _curl_context: Callable[[], contextlib.AbstractContextManager[None]] = contextlib.nullcontext

else:

    @contextlib.contextmanager
    def _curl_context() -> Generator[None]:
        try:
            yield
        except curl_exceptions.Timeout as e:
            log_msg = _clean_curl_error(repr(e))
            raise CDLAppError("Timeout", log_msg) from None
        except curl_exceptions.DNSError as e:
            log_msg = _clean_curl_error(repr(e))
            raise CDLAppError("Client Connector Error", log_msg) from None
        except curl_exceptions.RequestException as e:
            log_msg = _clean_curl_error(e)
            raise CDLAppError(f"Curl Error ({e.code})", log_msg) from None


try:
    from wreq import exceptions as wreq_exceptions
except ImportError:
    _wreq_context: Callable[[], contextlib.AbstractContextManager[None]] = contextlib.nullcontext

else:

    @contextlib.contextmanager
    def _wreq_context() -> Generator[None]:
        try:
            yield

        except wreq_exceptions.TimeoutError as e:
            raise CDLAppError("Timeout", repr(e)) from None
        except (
            wreq_exceptions.ConnectionResetError,
            wreq_exceptions.TlsError,
            wreq_exceptions.ProxyConnectionError,
            wreq_exceptions.ConnectionError,
            wreq_exceptions.RedirectError,
            wreq_exceptions.UpgradeError,
        ) as e:
            raise CDLAppError(type(e).__name__, repr(e)) from None
        except (
            wreq_exceptions.BodyError,
            wreq_exceptions.BuilderError,
            wreq_exceptions.DecodingError,
            wreq_exceptions.RequestError,
        ) as e:
            raise CDLAppError(type(e).__name__, repr(e)) from e


@contextlib.contextmanager
def _aiohttp_context() -> Generator[None]:
    try:
        yield
    except aiohttp.client_exceptions.TooManyRedirects as e:
        ui_failure = "Too Many Redirects"
        info = {
            "url": str(e.request_info.real_url),
            "history": tuple(str(r.real_url) for r in e.history),
        }
        raise CDLAppError(ui_failure, f"{ui_failure}\n{info}") from None
    except aiohttp.client_exceptions.ClientConnectorError as e:
        raise CDLAppError("Client Connector Error", str(e)) from None


@contextlib.contextmanager
def _exc_group_context() -> Generator[None]:
    try:
        yield
    except ExceptionGroup as e:
        if e.message and "unhandled errors in a TaskGroup" not in e.message:
            ui_error = e.message
        else:
            first = e.exceptions[0]
            ui_error = getattr(first, "ui_failure", None) or str(first)
        raise CDLAppError(ui_error, str(e)) from e.with_traceback(None)


_MEGA_HTTP_CODES = Final = {
    -4: HTTPStatus.TOO_MANY_REQUESTS,
    -16: HTTPStatus.FORBIDDEN,
    **dict.fromkeys((-8, -9), HTTPStatus.GONE),
    **dict.fromkeys((-17, -24, -401), 509),
}


@contextlib.contextmanager
def _mega_nz_context() -> Generator[None]:
    try:
        yield
    except mega.errors.RequestError as e:
        if e.code and (http_code := _MEGA_HTTP_CODES.get(e.code)):
            ui_failure = create_error_msg(http_code)
        else:
            ui_failure = f"MegaNZ Error [{e.code}]"

        raise CDLAppError(ui_failure, f"{ui_failure} {e.message}") from None

    except mega.errors.MegaNzError as e:
        raise CDLAppError("MegaNZ Error", str(e)) from None


@contextlib.contextmanager
def _pydantic_context() -> Generator[None]:
    try:
        yield
    except ValidationError as e:
        ui_failure = create_error_msg(422)
        log_msg = str(e).partition("For further information")[0].strip()
        raise CDLAppError(ui_failure, log_msg) from e


@contextlib.contextmanager
def _builtin_context() -> Generator[None]:
    try:
        yield
    except NotImplementedError as e:
        raise CDLAppError("NotImplemented") from e
    except LookupError as e:
        raise ScrapeError(422, repr(e)) from e
    except TimeoutError as e:
        raise CDLAppError("Timeout", repr(e)) from None


@contextlib.contextmanager
def _combined_error_context() -> Generator[None]:
    with (
        _builtin_context(),
        _pydantic_context(),
        _aiohttp_context(),
        _curl_context(),
        _wreq_context(),
        _mega_nz_context(),
        _exc_group_context(),
    ):
        yield


@contextlib.contextmanager
def _error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | yarl.URL) -> Generator[None]:
    if getattr(item, "is_segment", False):
        with contextlib.suppress(Exception):
            yield
        return

    url: yarl.URL = item if type(item) is yarl.URL else item.url  # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]
    app_error = origin = exc = None
    real_url: yarl.URL | str = ""
    try:
        yield

    except CDLBaseError as e:
        app_error = CDLAppError(e.ui_failure, str(e))
        origin = e.origin
        real_url = getattr(e, "url", None) or real_url
        exc = e.__cause__
    except CDLAppError as e:
        app_error = e
        exc = e.__cause__
    except Exception as e:  # noqa: BLE001
        exc = e
        app_error = CDLAppError.from_unknown_exc(e)

    if app_error is None:
        return

    origin = origin or get_origin(item)
    if getattr(self, "log_prefix", False):
        self, item = cast("Downloader", self), cast("MediaItem", item)
        _log_dl_error(self, item, app_error, exc, origin)
    else:
        _log_scrape_error(self, real_url or url, app_error, exc, origin)


@contextlib.contextmanager
def _blocking_error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | yarl.URL) -> Generator[None]:
    with _error_handling_context(self, item), _combined_error_context():
        yield


@contextlib.asynccontextmanager
async def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | yarl.URL) -> AsyncGenerator[None]:
    # The same as _blocking_error_handling_context (for now)
    with _error_handling_context(self, item), _combined_error_context():
        yield


def _log_scrape_error(
    self: _HasManager,
    url: yarl.URL | str,
    app_error: CDLAppError,
    exc: BaseException | None,
    origin: yarl.URL | Path | None,
) -> None:
    logger.error(f"Scrape Failed: {url} ({app_error.msg})", exc_info=exc)
    self.manager.scrape_mapper.logs.write_scrape_error(url, app_error.csv_msg, origin)
    self.manager.scrape_mapper.tui.scrape_errors.add(app_error.ui_error)


def _log_dl_error(
    self: Downloader,
    item: MediaItem,
    app_error: CDLAppError,
    exc: BaseException | None,
    origin: yarl.URL | Path | None,
) -> None:
    logger.error(
        f"{self.log_prefix} Failed: {item.url} ({app_error.msg}) \n -> Referer: {item.referer}",
        exc_info=exc,
    )
    self.manager.scrape_mapper.logs.write_download_error(
        item.url,
        item.referer,
        app_error.csv_msg,
        origin,
        parent=item.parents[-1] if item.parents else None,
    )
    self.manager.scrape_mapper.tui.files.stats.failed += 1
    self.manager.scrape_mapper.tui.download_errors.add(app_error.ui_error)


@overload
def error_handling_wrapper[HasManagerT: _HasManager, Origin: ScrapeItem | MediaItem | yarl.URL, **P, R](
    func: Callable[Concatenate[HasManagerT, Origin, P], R],
) -> Callable[Concatenate[HasManagerT, Origin, P], R]: ...


@overload
def error_handling_wrapper[HasManagerT: _HasManager, Origin: ScrapeItem | MediaItem | yarl.URL, **P, R](
    func: Callable[Concatenate[HasManagerT, Origin, P], Coroutine[None, None, R]],
) -> Callable[Concatenate[HasManagerT, Origin, P], Coroutine[None, None, R]]: ...


def error_handling_wrapper[HasManagerT: _HasManager, Origin: ScrapeItem | MediaItem | yarl.URL, **P, R](
    func: Callable[Concatenate[HasManagerT, Origin, P], R | Coroutine[None, None, R]],
) -> Callable[Concatenate[HasManagerT, Origin, P], R | Coroutine[None, None, R]]:
    """Wrapper handles errors for url scraping."""

    if inspect.iscoroutinefunction(func):

        @functools.wraps(func)
        async def async_wrapper(self: HasManagerT, item: Origin, *args: P.args, **kwargs: P.kwargs) -> R:
            async with error_handling_context(self, item):
                return await func(self, item, *args, **kwargs)

        return _mark_as_safe(async_wrapper)

    @functools.wraps(func)
    def wrapper(self: HasManagerT, item: Origin, *args: P.args, **kwargs: P.kwargs) -> R:
        with _blocking_error_handling_context(self, item):
            result = func(self, item, *args, **kwargs)
            assert not inspect.isawaitable(result)
            return result

    return _mark_as_safe(wrapper)


@final
class OnExceptionCallback:
    def __init__(self, cb: Callable[[BaseException], None]) -> None:
        self.cb = cb

    def __enter__(self) -> None:
        return None

    def __exit__(
        self,
        exc_type: type[BaseException] | None,
        exc_val: BaseException | None,
        exc_tb: object,
    ) -> None:
        if exc_val is not None:
            self.cb(exc_val)

    async def __aenter__(self) -> None:
        return None

    async def __aexit__(
        self,
        exc_type: type[BaseException] | None,
        exc_val: BaseException | None,
        exc_tb: object,
    ) -> None:
        if exc_val is not None:
            self.cb(exc_val)