Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: add HTTPLimiter class
NTFSvolumecf91fdd8679f
diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.pyindex 6404a8e63..ddb24bd00 100644--- a/cyberdrop_dl/clients/http.py+++ b/cyberdrop_dl/clients/http.py@@ -37,13 +37,13 @@ if TYPE_CHECKING:from cyberdrop_dl.config import Configfrom cyberdrop_dl.url_objects import AbsoluteHttpURL++type RequestContext = contextlib.AbstractAsyncContextManager[AbstractResponse[Any]]+type RateLimit = tuple[float, float]type JSONCheck = Callable[[Any, AbstractResponse[Any]], None]JSON_CHECK: ContextVar[JSONCheck | None] = ContextVar("JSON_CHECK", default=None)--RequestContext = contextlib._AsyncGeneratorContextManager[AbstractResponse[Any]] # pyright: ignore[reportPrivateUsage]-logger = logging.getLogger(__name__)@@ -74,15 +74,26 @@ class RequestDoneCallback(Protocol):) -> None: ...[email protected](slots=True, frozen=True)+class HTTPLimiter:+ global_: aio.RateLimiter+ downloads: asyncio.Semaphore+ per_domain: dict[str, aio.RateLimiter] = dataclasses.field(default_factory=dict)++ def __setitem__(self, domain: str, rate: RateLimit) -> None:+ self.per_domain[domain] = aio.RateLimiter.w_no_burst(*rate)++@finalclass HTTPClient:request_done_callback: RequestDoneCallback | None = Nonedef __init__(self, config: Config) -> None:self.config = config- self.rate_limits: dict[str, aio.RateLimiter] = {}- self.global_rate_limiter = aio.RateLimiter.w_no_burst(config.network.rate_limit)- self.global_download_limiter = asyncio.Semaphore(config.downloads.concurrency)+ self.limiter = HTTPLimiter(+ aio.RateLimiter.w_no_burst(config.network.rate_limit),+ asyncio.Semaphore(config.downloads.concurrency),+ )self._ssl_context = tcp.create_ssl_context(config.network.ssl_context)self._cookies: aiohttp.CookieJar | None = None@@ -91,7 +102,7 @@ class HTTPClient:self._session: aiohttp.ClientSessionself._download_session: aiohttp.ClientSession- __repr__ = simple_repr("config", "_ssl_context", "_cookies", "_flaresolverr", "request_done_callback")+ __repr__ = simple_repr("config", "_ssl_context", "_cookies", "_flaresolverr", "limiter", "request_done_callback")@propertydef curl_session(self) -> AsyncSession[CurlResponse]:@@ -282,7 +293,7 @@ class HTTPClient:@contextlib.asynccontextmanagerasync def rate_limit_ctx(self, domain: str, json_check: JSONCheck | None = None) -> AsyncGenerator[None]:with enter_context(JSON_CHECK, json_check):- async with self.rate_limits[domain], self.global_rate_limiter:+ async with self.limiter.per_domain[domain], self.limiter.global_:yield@@ -412,9 +423,6 @@ def _create_curl_session(config: Config) -> AsyncSession[CurlResponse]:)-type RateLimit = tuple[float, float]--@final@frozenclass HTTPConfig(ConfigDataclass):diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex b3090a702..266d3fe38 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -254,7 +254,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):if self._ready:return- self.client.rate_limits[self.DOMAIN] = aio.RateLimiter.w_no_burst(*self.__http_ctx__.rate_limit)+ self.client.limiter[self.DOMAIN] = self.__http_ctx__.rate_limittry:await self.__async_post_init__()except Exception:diff --git a/cyberdrop_dl/downloader/http.py b/cyberdrop_dl/downloader/http.pyindex 36b74b24a..b1a0ca166 100644--- a/cyberdrop_dl/downloader/http.py+++ b/cyberdrop_dl/downloader/http.py@@ -247,7 +247,7 @@ class Downloader:async with (server_lock,self._semaphore,- self.manager.http_client.global_download_limiter,+ self.manager.http_client.limiter.downloads,):yielddiff --git a/tests/test_client.py b/tests/test_client.pyindex 4e649aa82..1c9528ba7 100644--- a/tests/test_client.py+++ b/tests/test_client.py@@ -18,7 +18,7 @@ def test_initial_state(client: HTTPClient) -> None:assert client._cookies is Noneassert client._flaresolverr is Noneassert isinstance(client._ssl_context, ssl.SSLContext)- assert client.rate_limits == {}+ assert client.limiter.per_domain == {}async def test_context_manager(client: HTTPClient) -> None: