Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add per server limiters (#1384)

NTFSvolume4bccc6d962de
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index c867495b6..e625b7aa9 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -4,6 +4,7 @@ import asyncio
import contextlib
import itertools
import time
+import weakref
from http import HTTPStatus
from typing import TYPE_CHECKING
@@ -17,12 +18,11 @@ from cyberdrop_dl.utils.logger import log, log_debug
from cyberdrop_dl.utils.utilities import get_size_or_none
if TYPE_CHECKING:
- from collections.abc import Callable, Coroutine, Generator
+ from collections.abc import Callable, Coroutine, Generator, Mapping
from pathlib import Path
from typing import Any
import aiohttp
- from multidict import CIMultiDictProxy
from cyberdrop_dl.data_structures.url_objects import MediaItem
from cyberdrop_dl.managers.client_manager import ClientManager
@@ -44,9 +44,23 @@ class DownloadClient:
self.manager = manager
self.client_manager = client_manager
self.download_speed_threshold = self.manager.config_manager.settings_data.runtime_options.slow_download_speed
+ self._null_context = contextlib.nullcontext()
+ self._server_locks: weakref.WeakValueDictionary[str, asyncio.Lock] = weakref.WeakValueDictionary()
+ self._use_server_locks: set[str] = set()
+
+ def server_limiter(self, domain: str, server: str) -> asyncio.Lock | contextlib.nullcontext[None]:
+ if domain not in self._use_server_locks:
+ return self._null_context
+
+ if server not in self._server_locks:
+ lock = asyncio.Lock()
+ self._server_locks[server] = lock
+
+ log_debug(f"Using lock for server '{server}'", 20)
+ return self._server_locks[server]
@contextlib.asynccontextmanager
- async def _limiter(self, domain: str):
+ async def _track_errors(self, domain: str):
with self.client_manager.request_context(domain):
await self.client_manager.manager.states.RUNNING.wait()
yield
@@ -250,7 +264,7 @@ class DownloadClient:
await self.process_completed(media_item, domain)
return False
- async with self._limiter(domain):
+ async with self._track_errors(domain):
downloaded = await self._download(domain, media_item)
if downloaded:
@@ -444,7 +458,7 @@ class DownloadClient:
return proceed
-def get_content_type(ext: str, headers: CIMultiDictProxy) -> str | None:
+def get_content_type(ext: str, headers: Mapping[str, str]) -> str | None:
content_type: str = headers.get("Content-Type", "")
content_length = headers.get("Content-Length")
if not content_type and not content_length:
@@ -466,7 +480,7 @@ def get_content_type(ext: str, headers: CIMultiDictProxy) -> str | None:
return content_type
-def get_last_modified(headers: CIMultiDictProxy) -> int | None:
+def get_last_modified(headers: Mapping[str, str]) -> int | None:
if date_str := headers.get("Last-Modified"):
return parse_http_date(date_str)
diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.py
index fb4050dc1..77924ba54 100644
--- a/cyberdrop_dl/crawlers/bunkrr.py
+++ b/cyberdrop_dl/crawlers/bunkrr.py
@@ -129,6 +129,7 @@ class BunkrrCrawler(Crawler):
DOMAIN: ClassVar[str] = "bunkrr"
_RATE_LIMIT: ClassVar[tuple[float, float]] = 5, 1
_DOWNLOAD_SLOTS: ClassVar[int | None] = 3
+ _USE_DOWNLOAD_SERVERS_LOCKS: ClassVar[bool] = True
def __post_init__(self) -> None:
self.switch_host_locks: dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 58a2d0431..2daf4b9cd 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -104,6 +104,7 @@ class Crawler(ABC):
_RATE_LIMIT: ClassVar[RateLimit] = 25, 1
_DOWNLOAD_SLOTS: ClassVar[int | None] = None
+ _USE_DOWNLOAD_SERVERS_LOCKS: ClassVar[bool] = False
@copy_signature(ScraperClient._request)
@contextlib.asynccontextmanager
@@ -256,6 +257,8 @@ class Crawler(ABC):
self.manager.client_manager.rate_limits[self.DOMAIN] = self.RATE_LIMIT
if self._DOWNLOAD_SLOTS:
self.manager.client_manager.download_slots[self.DOMAIN] = self._DOWNLOAD_SLOTS
+ if self._USE_DOWNLOAD_SERVERS_LOCKS:
+ self.manager.client_manager.download_client._use_server_locks.add(self.DOMAIN)
self.downloader = self._init_downloader()
self._register_response_checks()
await self.async_startup()
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index d67cbff94..99989bc74 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -163,7 +163,8 @@ class Downloader:
await self.client.mark_incomplete(media_item, self.domain)
if not media_item.is_segment:
self.update_queued_files()
- async with self._semaphore:
+ server = (media_item.debrid_link or media_item.url).host
+ async with self.client.server_limiter(media_item.domain, server), self._semaphore:
await self.manager.states.RUNNING.wait()
self.waiting_items -= 1
self.processed_items.add(media_item.db_path)