Skip to content

Bulk asynchronous downloader for multiple file hosts

21.2 KB
Raw Blame
from __future__ import annotations

import asyncio
import contextlib
import dataclasses
import http.cookies
import time
from contextvars import ContextVar
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, ClassVar, Protocol, Self, Unpack, final, override

import aiohttp
from aiohttp import hdrs

from cyberdrop_dl import aio, cookies, ddos_guard, env
from cyberdrop_dl.clients import curl_cffi, flaresolverr, get_logger, tcp, wreq
from cyberdrop_dl.clients.request import Request, RequestParams
from cyberdrop_dl.clients.response import AbstractResponse, FlareSolverrResponse
from cyberdrop_dl.cookies import make_simple_cookie
from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError
from cyberdrop_dl.signature import dataclass_repr
from cyberdrop_dl.utils import enter_context, truncated_preview
from cyberdrop_dl.utils.dataclass import ConfigDataclass, struct

if TYPE_CHECKING:
    from collections.abc import AsyncGenerator, Awaitable, Callable
    from pathlib import Path

    from bs4 import BeautifulSoup
    from curl_cffi.requests import AsyncSession
    from curl_cffi.requests.models import Response as CurlResponse

    from cyberdrop_dl.clients.wreq import WreqClient  # pyright: ignore[reportPrivateLocalImportUsage]
    from cyberdrop_dl.config import Config
    from cyberdrop_dl.url_objects import AbsoluteHttpURL

    from . import HttpMethod


type RequestContext = contextlib.AbstractAsyncContextManager[AbstractResponse[Any]]
type RateLimit = tuple[float, float]
type JSONCheck = Callable[[Any, AbstractResponse[Any]], None]

JSON_CHECK: ContextVar[JSONCheck | None] = ContextVar("JSON_CHECK", default=None)

logger = get_logger(__name__)


class _LazyResponseLog:
    def __init__(self, response: AbstractResponse[Any], *, content_only: bool = False) -> None:
        self.resp: AbstractResponse[Any] = response
        self.content_only: bool = content_only

    def __json__(self) -> dict[str, Any]:
        resp = self.resp.__json__()
        resp["response_headers"] = {k: truncated_preview(v) for k, v in self.resp.headers.items()}
        if type(resp["content"]) is str:
            resp["content"] = truncated_preview(resp["content"], env.MAX_LOG_MSG_LENGTH if self.content_only else 100)

        if self.content_only:
            return {"content": resp["content"]}

        del resp["created_at"]
        return resp

    def __str__(self) -> str:
        return str(self.__json__())

    def __repr__(self) -> str:
        return f"<{type(self).__name__}(resp={self.resp!r})>"


class RequestDoneCallback(Protocol):
    def __call__(
        self, url: AbsoluteHttpURL, response: AbstractResponse[Any], exc: Exception | None = None, /
    ) -> None: ...


@dataclasses.dataclass(slots=True, frozen=True)
class HTTPLimiter:
    global_: aio.RateLimiter
    downloads: asyncio.Semaphore
    per_domain: dict[str, aio.RateLimiter] = dataclasses.field(default_factory=dict)

    def __setitem__(self, domain: str, rate: RateLimit) -> None:
        self.per_domain[domain] = aio.RateLimiter.w_no_burst(*rate)


@final
@dataclass_repr(
    "config", "_ssl_context", "_cookies", "_flaresolverr", "limiter", "request_done_callback", "_use_flaresolverr_ua"
)
class HTTPClient:
    request_done_callback: RequestDoneCallback | None = None

    def __init__(self, config: Config) -> None:
        self.config = config
        self.limiter = HTTPLimiter(
            aio.RateLimiter.w_no_burst(config.network.rate_limit),
            asyncio.Semaphore(config.downloads.concurrency),
        )

        self._ssl_context = None
        self._cookies: aiohttp.CookieJar | None = None
        self._flaresolverr: flaresolverr.Client | None = None
        self._curl_session: AsyncSession[CurlResponse] | None = None
        self._use_flaresolverr_ua: set[str] = set()
        self._flaresolverr_ua: str = ""
        self._wreq_session: WreqClient | None = None
        self._session: aiohttp.ClientSession
        self._download_session: aiohttp.ClientSession

    @property
    def ssl_context(self):
        if self._ssl_context is None:
            self._ssl_context = self.config.network.tls.verify and tcp.create_ssl_context(
                tcp.resolve_tls_version(self.config.network.tls.min_version),
                self.config.network.tls.ca_certs,
            )
        return self._ssl_context

    @property
    def curl_session(self) -> AsyncSession[CurlResponse]:
        if self._curl_session is None:
            self._curl_session = self._create_curl_session()
        return self._curl_session

    @property
    def wreq_session(self) -> WreqClient:
        if self._wreq_session is None:
            self._wreq_session = wreq.create_client(self.config)
            jar = self._wreq_session.cookie_jar
            assert jar is not None
            for (domain, path), cookie in self.cookies.cookies.items():
                for morsel in cookie.values():
                    jar.add(str(morsel), f"https://{domain}{path}")

        return self._wreq_session

    @property
    def cookies(self) -> aiohttp.CookieJar:
        # lazy cause it is loop bound for some reason
        if self._cookies is None:
            self._cookies = aiohttp.CookieJar(quote_cookie=False)
        return self._cookies

    @property
    def flaresolverr(self) -> flaresolverr.Client | None:
        if self._flaresolverr is None and (url := self.config.network.flaresolverr):
            net = self.config.network
            self._flaresolverr = flaresolverr.Client(
                self._session,
                flaresolverr.Config(
                    url=url.origin() / "v1",
                    use_session=net.flaresolverr_use_session,
                    concurrency=net.flaresolverr_concurrency,
                    wait=net.flaresolverr_wait,
                    proxy=net.proxy,
                ),
            )
        return self._flaresolverr

    def __sync_session_cookies(self, url: AbsoluteHttpURL) -> None:
        """
        Apply to the cookies from the `curl` session into the `aiohttp` session, filtering them by the URL

        This is mostly just to get the `cf_cleareance` cookie value into the `aiohttp` session

        The reverse (sync `aiohttp` -> `curl`) is not needed at the moment, so it is skipped
        """
        now = time.time()
        for cookie in self.curl_session.cookies.jar:
            simple_cookie = make_simple_cookie(cookie, now)
            self.cookies.update_cookies(simple_cookie, url)

    def __sync_wreq_cookies(self, url: AbsoluteHttpURL) -> None:
        now = time.time()
        jar = self.wreq_session.cookie_jar
        assert jar is not None
        for cookie in jar.get_all():
            try:
                simple_cookie = wreq.make_simple_cookie(cookie, now)
            except (http.cookies.CookieError, ValueError):
                continue
            self.cookies.update_cookies(simple_cookie, url)

    async def __aenter__(self) -> Self:
        self._session = self.create_aiohttp_session()
        self._download_session = self.create_aiohttp_session()
        return self

    async def __aexit__(self, *_: object) -> None:
        async with asyncio.TaskGroup() as tg:
            tg.create_task(self._download_session.close())
            if self._curl_session is not None:
                tg.create_task(self._curl_session.close())

            if self._wreq_session is not None:
                self._wreq_session.close()

            if self._flaresolverr is not None:
                # close before closing aiohttp session
                await self._flaresolverr.aclose()
            await self._session.close()

    def _create_curl_session(self) -> AsyncSession[CurlResponse]:
        session = curl_cffi.create_session(self.config)
        session.cookies = {cookie.key: cookie.value for cookie in self.cookies}
        for (domain, path), cookie in self.cookies.cookies.items():
            for morsel in cookie.values():
                session.cookies.set(
                    morsel.key,
                    morsel.value,
                    domain=morsel["domain"] or domain,
                    path=morsel["path"] or path or "/",
                )
        return session

    def create_aiohttp_session(self) -> aiohttp.ClientSession:
        return aiohttp.ClientSession(
            headers={"User-Agent": self.config.network.user_agent},
            raise_for_status=False,
            cookie_jar=self.cookies,
            timeout=aiohttp.ClientTimeout(
                total=None,
                sock_connect=self.config.network.connection_timeout,
                sock_read=self.config.network.read_timeout,
            ),
            proxy=self.config.network.proxy,
            connector=tcp.create_connector(self.ssl_context),
            requote_redirect_url=False,
        )

    async def load_cookie_files(self, cookie_files: list[Path]) -> None:
        if not cookie_files:
            return

        async for cookie in cookies.read_netscape_files(cookie_files):
            self.cookies.update_cookies(cookie)

    @contextlib.asynccontextmanager
    async def request(
        self,
        url: AbsoluteHttpURL,
        /,
        method: HttpMethod = "GET",
        **kwargs: Unpack[RequestParams],
    ) -> AsyncGenerator[AbstractResponse[Any]]:
        """Make an HTTP request and retry w flaresolverr if required"""
        async with self.raw_request(url, method, **kwargs) as resp:
            try:
                await check_http_status(resp)
            except DDOSGuardError:
                await resp.aclose()
                flare = self.flaresolverr
                if not flare or flare.is_down:
                    raise
            else:
                yield resp
                return

        # TODO: implement per host weak locks to only make one flaresolverr request
        # Use the cookies from that request for all future ones
        resp = await self._flaresolverr_request(url, method=method, data=kwargs.get("data"))
        custom_headers = tuple(
            sorted(map(str, set(kwargs.get("headers", ())) - {hdrs.USER_AGENT, hdrs.REFERER, hdrs.ORIGIN}))
        )
        has_json = kwargs.get("json") is not None
        if not (has_json or custom_headers):
            yield resp
            return

        logger.info(
            "Making %s request to %s again with Flaresolverr cookies. Reasons: %s",
            method,
            url,
            f"{has_json = }, {custom_headers = }",
        )
        async with self.raw_request(url, method, **kwargs) as resp:
            await check_http_status(resp)
            yield resp

    def raw_request(
        self,
        url: AbsoluteHttpURL,
        /,
        method: HttpMethod = "GET",
        **kwargs: Unpack[RequestParams],
    ) -> RequestContext:
        request = Request.from_params(url, method, kwargs)
        if self.config.network.impersonate and request.impersonate is not False:
            request.impersonate = self.config.network.impersonate

        if url.host in self._use_flaresolverr_ua:
            # We already made a (successful) flaresolverr request to this host
            # Use the same UA as flaresolverr to make sure cookies are valid
            request.impersonate = False
            request.headers[hdrs.USER_AGENT] = self._flaresolverr_ua

        elif request.impersonate:
            request.headers.pop(hdrs.USER_AGENT, None)
        else:
            request.headers.setdefault(hdrs.USER_AGENT, self.config.network.user_agent)

        return self._request(request)

    @contextlib.asynccontextmanager
    async def _request(self, request: Request) -> AsyncGenerator[AbstractResponse[Any]]:
        logger.traffic("Starting %s request [id=%s]\n%s", request.method, request.id, request)
        exc = None
        async with self.__request(request) as resp:
            resp.id = request.id
            logger.traffic("Finished %s request [id=%s]\n%s", request.method, request.id, _LazyResponseLog(resp))
            try:
                yield resp
            except Exception as e:
                exc = e
                raise
            finally:
                if resp.has_content_not_logged:
                    logger.traffic(
                        "Content from %s request [id=%s]\n%s",
                        request.method,
                        request.id,
                        _LazyResponseLog(resp, content_only=True),
                    )
                if self.request_done_callback:
                    self.request_done_callback(request.url, resp, exc)
                del exc
                del resp

    @contextlib.asynccontextmanager
    async def __request(self, request: Request) -> AsyncGenerator[AbstractResponse[Any]]:
        if request.impersonate:
            if wreq.IS_INSTALLED:
                resp = await self.wreq_session.request(
                    wreq.cast_method(request.method),
                    str(request.url),
                    headers=dict(request.headers),
                    json=request.json,
                    body=request.data,
                    emulation=wreq.cast_impersonate(request.impersonate),  # pyright: ignore[reportArgumentType]
                    **request.params,
                )
                async with resp:
                    resp = AbstractResponse.create(resp)
                    self.__sync_wreq_cookies(resp.url)
                    yield resp
                    return

            async with contextlib.aclosing(
                await self.curl_session.request(
                    request.method,
                    str(request.url),
                    stream=True,
                    headers=request.headers,
                    json=request.json,
                    data=request.data,
                    impersonate=curl_cffi.cast_impersonation(request.impersonate),
                    **request.params,
                )
            ) as curl_resp:
                self.__sync_session_cookies(request.url)
                yield AbstractResponse.create(curl_resp)

            return

        async with self._session.request(
            request.method,
            request.url,
            headers=request.headers,
            json=request.json,
            data=request.data,
            **request.params,
        ) as aio_resp:
            yield AbstractResponse.create(aio_resp)

    async def flaresolverr_request(
        self,
        url: AbsoluteHttpURL,
        **params: Unpack[flaresolverr.RequestParams],
    ) -> FlareSolverrResponse:
        flare = self.flaresolverr
        if not flare:
            raise ScrapeError(
                "Flaresolverr Required", "This request needs a real running browser to execute javascript"
            )
        flare.check_can_connect()
        return await self._flaresolverr_request(url, **params)

    async def _flaresolverr_request(
        self,
        url: AbsoluteHttpURL,
        /,
        **params: Unpack[flaresolverr.RequestParams],
    ) -> FlareSolverrResponse:
        """Make a request with FlareSolverr.

        Returns an AbstractResponse confirmed to not be a DDOS Guard page, even if flaresolverr fails to detect/solve a challenge"""

        flare = self.flaresolverr
        assert flare
        assert not flare.is_down
        solution = await flare.request(url, **params)
        self.cookies.update_cookies(solution.cookies)
        await flaresolverr.verify_solution(self.config.network.user_agent, solution)
        self._use_flaresolverr_ua.add(url.host)
        self._flaresolverr_ua = flaresolverr.USER_AGENT.get()
        return FlareSolverrResponse.create(solution)


async def _check_json(response: AbstractResponse[Any]) -> None:
    if "json" not in response.content_type:
        return

    try:
        data = await response.json()
    except Exception:
        logger.exception("Unable to decode JSON response from %s", response.url)
        return

    if check := JSON_CHECK.get():
        check(data, response)
        return


class HTTPController(Protocol):
    __http_config__: ClassVar[HTTPConfig]
    __http_ctx__: HTTPContext
    client: HTTPClient


class HTTPMixin(HTTPController, Protocol):
    @contextlib.asynccontextmanager
    async def rate_limit_ctx(
        self,
    ) -> AsyncGenerator[None]:
        ctx = self.__http_ctx__
        if ctx.throttle is not None:
            await ctx.throttle()

        limiter = self.client.limiter.per_domain.get(ctx.domain, contextlib.nullcontext())
        with enter_context(JSON_CHECK, ctx.json_check):
            async with limiter, self.client.limiter.global_:
                yield

    @contextlib.asynccontextmanager
    async def request(
        self,
        url: AbsoluteHttpURL,
        /,
        method: HttpMethod = "GET",
        **kwargs: Unpack[RequestParams],
    ) -> AsyncGenerator[AbstractResponse[Any]]:

        ctx = self.__http_ctx__
        kwargs.setdefault("impersonate", ctx.impersonate)
        kwargs["headers"] = ctx.headers | kwargs.setdefault("headers", {})

        async with (
            self.rate_limit_ctx(),
            self.client.request(url, method, **kwargs) as resp,
        ):
            yield resp

    async def flaresolverr_request(
        self,
        url: AbsoluteHttpURL,
        **kwargs: Unpack[flaresolverr.RequestParams],
    ) -> FlareSolverrResponse:
        async with self.rate_limit_ctx():
            return await self.client.flaresolverr_request(url, **kwargs)

    async def request_json(
        self,
        url: AbsoluteHttpURL,
        /,
        method: HttpMethod = "GET",
        **kwargs: Unpack[RequestParams],
    ) -> Any:
        async with self.request(url, method, **kwargs) as resp:
            return await resp.json()

    async def request_soup(
        self,
        url: AbsoluteHttpURL,
        /,
        method: HttpMethod = "GET",
        **kwargs: Unpack[RequestParams],
    ) -> BeautifulSoup:
        async with self.request(url, method, **kwargs) as resp:
            return await resp.soup()

    async def request_text(
        self,
        url: AbsoluteHttpURL,
        /,
        method: HttpMethod = "GET",
        **kwargs: Unpack[RequestParams],
    ) -> str:
        async with self.request(url, method, **kwargs) as resp:
            return await resp.text()

    async def request_location(
        self,
        url: AbsoluteHttpURL,
        /,
        method: HttpMethod = "HEAD",
        **kwargs: Unpack[RequestParams],
    ) -> AbsoluteHttpURL | None:
        async with self.request(url, method, **kwargs) as resp:
            return resp.location

    async def request_redirect(
        self,
        url: AbsoluteHttpURL,
        /,
        method: HttpMethod = "GET",
        **kwargs: Unpack[RequestParams],
    ) -> AbsoluteHttpURL:
        "Like request_location, but it uses a GET request instead of HEAD to follow all redirects upto the final destination"
        async with self.request(url, method, **kwargs) as resp:
            return resp.url


@final
@struct
class HTTPConfig(ConfigDataclass):
    __attr_name__: ClassVar[str] = "__http_config__"
    headers: dict[str, str] | None = None
    impersonate: str | bool | None = None
    rate_limit: RateLimit | None = None
    json_check: JSONCheck | None = None
    probe_method: HttpMethod | None = None

    @classmethod
    def default_headers(
        cls,
        user_agent: str | None = None,
        referer: str | None = None,
        host: str | None = None,
        content_type: str | None = None,
        accept: str | None = None,
        **kwargs: str,
    ) -> HTTPConfig:
        headers = {
            hdrs.USER_AGENT: user_agent,
            hdrs.REFERER: referer,
            hdrs.HOST: host,
            hdrs.CONTENT_TYPE: content_type,
            hdrs.ACCEPT: accept,
        } | kwargs
        return HTTPConfig(headers={k: v for k, v in headers.items() if v is not None})

    @override
    def __or__(self, other: Self) -> Self:  # pyright: ignore[reportIncompatibleMethodOverride]
        changes = other._changes()

        if self.headers and other.headers:
            changes["headers"] = self.headers | other.headers

        elif (headers := (self.headers or other.headers)) is not None:
            changes["headers"] = headers.copy()

        return dataclasses.replace(self, **changes)


@final
@dataclasses.dataclass(slots=True, frozen=True)
class HTTPContext:
    domain: str
    rate_limit: RateLimit
    json_check: JSONCheck | None = None
    impersonate: str | bool | None = None
    throttle: Callable[[], Awaitable[Any]] | None = None
    probe_method: HttpMethod = "HEAD"
    headers: dict[str, str] = dataclasses.field(default_factory=dict)

    @classmethod
    def build(
        cls,
        domain: str,
        config: HTTPConfig,
        throttle: Callable[[], Awaitable[Any]] | None = None,
    ) -> HTTPContext:
        assert config.rate_limit is not None

        return cls(
            domain=domain,
            rate_limit=config.rate_limit,
            headers=config.headers or {},
            impersonate=config.impersonate,
            json_check=config.json_check,
            throttle=throttle,
            probe_method=config.probe_method or "HEAD",
        )


async def check_http_status(response: AbstractResponse[Any]) -> None:
    if HTTPStatus.OK <= response.status < HTTPStatus.BAD_REQUEST:
        # Check DDosGuard even on successful pages
        await ddos_guard.check_resp(response)
        return

    await _check_json(response)
    await ddos_guard.check_resp(response)
    raise DownloadError(response.status)