Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: simplify http client
NTFSvolume2ae7d14451df
diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.pyindex 6c39dfd63..6404a8e63 100644--- a/cyberdrop_dl/clients/http.py+++ b/cyberdrop_dl/clients/http.py@@ -8,7 +8,7 @@ import timeimport warningsfrom contextvars import ContextVarfrom http import HTTPStatus-from typing import TYPE_CHECKING, Any, ClassVar, Literal, Protocol, Self, Unpack, final, override+from typing import TYPE_CHECKING, Any, ClassVar, Protocol, Self, Unpack, final, overrideimport aiohttpfrom aiohttp import hdrs@@ -22,11 +22,11 @@ from cyberdrop_dl.clients.request import Request, RequestParamsfrom cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.cookies import make_simple_cookiefrom cyberdrop_dl.exceptions import DDOSGuardError, DownloadError+from cyberdrop_dl.signature import simple_reprfrom cyberdrop_dl.utils import enter_context, truncated_previewfrom cyberdrop_dl.utils.dataclass import ConfigDataclass, frozenif TYPE_CHECKING:- import sslfrom collections.abc import AsyncGenerator, Awaitable, Callablefrom pathlib import Path@@ -75,38 +75,23 @@ class RequestDoneCallback(Protocol):@final[email protected](slots=True)class HTTPClient:- config: Configrequest_done_callback: RequestDoneCallback | None = None- impersonate: (- Literal[- "chrome",- "edge",- "safari",- "safari_ios",- "chrome_android",- "firefox",- ]- | None- ) = dataclasses.field(init=False)-- rate_limits: dict[str, aio.RateLimiter] = dataclasses.field(init=False, default_factory=dict)- global_rate_limiter: aio.RateLimiter = dataclasses.field(init=False)- global_download_limiter: asyncio.Semaphore = dataclasses.field(init=False)-- _ssl_context: ssl.SSLContext | Literal[False] = dataclasses.field(init=False)- _cookies: aiohttp.CookieJar | None = dataclasses.field(init=False, default=None)- _flaresolverr: flaresolverr.Client | None = dataclasses.field(init=False, default=None)- _curl_session: AsyncSession[CurlResponse] | None = dataclasses.field(init=False, default=None)- _session: aiohttp.ClientSession = dataclasses.field(init=False, repr=False)- _download_session: aiohttp.ClientSession = dataclasses.field(init=False, repr=False)-- def __post_init__(self) -> None:- self.impersonate = self.config.network.impersonate- self._ssl_context = tcp.create_ssl_context(self.config.network.ssl_context)- self.global_rate_limiter = aio.RateLimiter.w_no_burst(self.config.network.rate_limit)- self.global_download_limiter = asyncio.Semaphore(self.config.downloads.concurrency)++ def __init__(self, config: Config) -> None:+ self.config = config+ self.rate_limits: dict[str, aio.RateLimiter] = {}+ self.global_rate_limiter = aio.RateLimiter.w_no_burst(config.network.rate_limit)+ self.global_download_limiter = asyncio.Semaphore(config.downloads.concurrency)++ self._ssl_context = tcp.create_ssl_context(config.network.ssl_context)+ self._cookies: aiohttp.CookieJar | None = None+ self._flaresolverr: flaresolverr.Client | None = None+ self._curl_session: AsyncSession[CurlResponse] | None = None+ self._session: aiohttp.ClientSession+ self._download_session: aiohttp.ClientSession++ __repr__ = simple_repr("config", "_ssl_context", "_cookies", "_flaresolverr", "request_done_callback")@propertydef curl_session(self) -> AsyncSession[CurlResponse]:@@ -214,8 +199,8 @@ class HTTPClient:**kwargs: Unpack[RequestParams],) -> RequestContext:request = Request.from_params(url, method, kwargs)- if self.impersonate:- request.impersonate = self.impersonate+ if self.config.network.impersonate:+ request.impersonate = self.config.network.impersonateif request.impersonate:request.headers.pop(hdrs.USER_AGENT, None)