Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add per server limiters (#1384)
NTFSvolume4bccc6d962de
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex c867495b6..e625b7aa9 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -4,6 +4,7 @@ import asyncioimport contextlibimport itertoolsimport time+import weakreffrom http import HTTPStatusfrom typing import TYPE_CHECKING@@ -17,12 +18,11 @@ from cyberdrop_dl.utils.logger import log, log_debugfrom cyberdrop_dl.utils.utilities import get_size_or_noneif TYPE_CHECKING:- from collections.abc import Callable, Coroutine, Generator+ from collections.abc import Callable, Coroutine, Generator, Mappingfrom pathlib import Pathfrom typing import Anyimport aiohttp- from multidict import CIMultiDictProxyfrom cyberdrop_dl.data_structures.url_objects import MediaItemfrom cyberdrop_dl.managers.client_manager import ClientManager@@ -44,9 +44,23 @@ class DownloadClient:self.manager = managerself.client_manager = client_managerself.download_speed_threshold = self.manager.config_manager.settings_data.runtime_options.slow_download_speed+ self._null_context = contextlib.nullcontext()+ self._server_locks: weakref.WeakValueDictionary[str, asyncio.Lock] = weakref.WeakValueDictionary()+ self._use_server_locks: set[str] = set()++ def server_limiter(self, domain: str, server: str) -> asyncio.Lock | contextlib.nullcontext[None]:+ if domain not in self._use_server_locks:+ return self._null_context++ if server not in self._server_locks:+ lock = asyncio.Lock()+ self._server_locks[server] = lock++ log_debug(f"Using lock for server '{server}'", 20)+ return self._server_locks[server]@contextlib.asynccontextmanager- async def _limiter(self, domain: str):+ async def _track_errors(self, domain: str):with self.client_manager.request_context(domain):await self.client_manager.manager.states.RUNNING.wait()yield@@ -250,7 +264,7 @@ class DownloadClient:await self.process_completed(media_item, domain)return False- async with self._limiter(domain):+ async with self._track_errors(domain):downloaded = await self._download(domain, media_item)if downloaded:@@ -444,7 +458,7 @@ class DownloadClient:return proceed-def get_content_type(ext: str, headers: CIMultiDictProxy) -> str | None:+def get_content_type(ext: str, headers: Mapping[str, str]) -> str | None:content_type: str = headers.get("Content-Type", "")content_length = headers.get("Content-Length")if not content_type and not content_length:@@ -466,7 +480,7 @@ def get_content_type(ext: str, headers: CIMultiDictProxy) -> str | None:return content_type-def get_last_modified(headers: CIMultiDictProxy) -> int | None:+def get_last_modified(headers: Mapping[str, str]) -> int | None:if date_str := headers.get("Last-Modified"):return parse_http_date(date_str)diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.pyindex fb4050dc1..77924ba54 100644--- a/cyberdrop_dl/crawlers/bunkrr.py+++ b/cyberdrop_dl/crawlers/bunkrr.py@@ -129,6 +129,7 @@ class BunkrrCrawler(Crawler):DOMAIN: ClassVar[str] = "bunkrr"_RATE_LIMIT: ClassVar[tuple[float, float]] = 5, 1_DOWNLOAD_SLOTS: ClassVar[int | None] = 3+ _USE_DOWNLOAD_SERVERS_LOCKS: ClassVar[bool] = Truedef __post_init__(self) -> None:self.switch_host_locks: dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 58a2d0431..2daf4b9cd 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -104,6 +104,7 @@ class Crawler(ABC):_RATE_LIMIT: ClassVar[RateLimit] = 25, 1_DOWNLOAD_SLOTS: ClassVar[int | None] = None+ _USE_DOWNLOAD_SERVERS_LOCKS: ClassVar[bool] = False@copy_signature(ScraperClient._request)@contextlib.asynccontextmanager@@ -256,6 +257,8 @@ class Crawler(ABC):self.manager.client_manager.rate_limits[self.DOMAIN] = self.RATE_LIMITif self._DOWNLOAD_SLOTS:self.manager.client_manager.download_slots[self.DOMAIN] = self._DOWNLOAD_SLOTS+ if self._USE_DOWNLOAD_SERVERS_LOCKS:+ self.manager.client_manager.download_client._use_server_locks.add(self.DOMAIN)self.downloader = self._init_downloader()self._register_response_checks()await self.async_startup()diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex d67cbff94..99989bc74 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -163,7 +163,8 @@ class Downloader:await self.client.mark_incomplete(media_item, self.domain)if not media_item.is_segment:self.update_queued_files()- async with self._semaphore:+ server = (media_item.debrid_link or media_item.url).host+ async with self.client.server_limiter(media_item.domain, server), self._semaphore:await self.manager.states.RUNNING.wait()self.waiting_items -= 1self.processed_items.add(media_item.db_path)