Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: add HTTPLimiter class

NTFSvolumecf91fdd8679f
diff --git a/cyberdrop_dl/clients/http.py b/cyberdrop_dl/clients/http.py
index 6404a8e63..ddb24bd00 100644
--- a/cyberdrop_dl/clients/http.py
+++ b/cyberdrop_dl/clients/http.py
@@ -37,13 +37,13 @@ if TYPE_CHECKING:
from cyberdrop_dl.config import Config
from cyberdrop_dl.url_objects import AbsoluteHttpURL
+
+type RequestContext = contextlib.AbstractAsyncContextManager[AbstractResponse[Any]]
+type RateLimit = tuple[float, float]
type JSONCheck = Callable[[Any, AbstractResponse[Any]], None]
JSON_CHECK: ContextVar[JSONCheck | None] = ContextVar("JSON_CHECK", default=None)
-
-RequestContext = contextlib._AsyncGeneratorContextManager[AbstractResponse[Any]] # pyright: ignore[reportPrivateUsage]
-
logger = logging.getLogger(__name__)
@@ -74,15 +74,26 @@ class RequestDoneCallback(Protocol):
) -> None: ...
[email protected](slots=True, frozen=True)
+class HTTPLimiter:
+ global_: aio.RateLimiter
+ downloads: asyncio.Semaphore
+ per_domain: dict[str, aio.RateLimiter] = dataclasses.field(default_factory=dict)
+
+ def __setitem__(self, domain: str, rate: RateLimit) -> None:
+ self.per_domain[domain] = aio.RateLimiter.w_no_burst(*rate)
+
+
@final
class HTTPClient:
request_done_callback: RequestDoneCallback | None = None
def __init__(self, config: Config) -> None:
self.config = config
- self.rate_limits: dict[str, aio.RateLimiter] = {}
- self.global_rate_limiter = aio.RateLimiter.w_no_burst(config.network.rate_limit)
- self.global_download_limiter = asyncio.Semaphore(config.downloads.concurrency)
+ self.limiter = HTTPLimiter(
+ aio.RateLimiter.w_no_burst(config.network.rate_limit),
+ asyncio.Semaphore(config.downloads.concurrency),
+ )
self._ssl_context = tcp.create_ssl_context(config.network.ssl_context)
self._cookies: aiohttp.CookieJar | None = None
@@ -91,7 +102,7 @@ class HTTPClient:
self._session: aiohttp.ClientSession
self._download_session: aiohttp.ClientSession
- __repr__ = simple_repr("config", "_ssl_context", "_cookies", "_flaresolverr", "request_done_callback")
+ __repr__ = simple_repr("config", "_ssl_context", "_cookies", "_flaresolverr", "limiter", "request_done_callback")
@property
def curl_session(self) -> AsyncSession[CurlResponse]:
@@ -282,7 +293,7 @@ class HTTPClient:
@contextlib.asynccontextmanager
async def rate_limit_ctx(self, domain: str, json_check: JSONCheck | None = None) -> AsyncGenerator[None]:
with enter_context(JSON_CHECK, json_check):
- async with self.rate_limits[domain], self.global_rate_limiter:
+ async with self.limiter.per_domain[domain], self.limiter.global_:
yield
@@ -412,9 +423,6 @@ def _create_curl_session(config: Config) -> AsyncSession[CurlResponse]:
)
-type RateLimit = tuple[float, float]
-
-
@final
@frozen
class HTTPConfig(ConfigDataclass):
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index b3090a702..266d3fe38 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -254,7 +254,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
if self._ready:
return
- self.client.rate_limits[self.DOMAIN] = aio.RateLimiter.w_no_burst(*self.__http_ctx__.rate_limit)
+ self.client.limiter[self.DOMAIN] = self.__http_ctx__.rate_limit
try:
await self.__async_post_init__()
except Exception:
diff --git a/cyberdrop_dl/downloader/http.py b/cyberdrop_dl/downloader/http.py
index 36b74b24a..b1a0ca166 100644
--- a/cyberdrop_dl/downloader/http.py
+++ b/cyberdrop_dl/downloader/http.py
@@ -247,7 +247,7 @@ class Downloader:
async with (
server_lock,
self._semaphore,
- self.manager.http_client.global_download_limiter,
+ self.manager.http_client.limiter.downloads,
):
yield
diff --git a/tests/test_client.py b/tests/test_client.py
index 4e649aa82..1c9528ba7 100644
--- a/tests/test_client.py
+++ b/tests/test_client.py
@@ -18,7 +18,7 @@ def test_initial_state(client: HTTPClient) -> None:
assert client._cookies is None
assert client._flaresolverr is None
assert isinstance(client._ssl_context, ssl.SSLContext)
- assert client.rate_limits == {}
+ assert client.limiter.per_domain == {}
async def test_context_manager(client: HTTPClient) -> None: