Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: support dl slots over servers
NTFSvolume84eea4fed191
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 97c8da24b..5ea25ac15 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -185,7 +185,7 @@ class URLConfig(ConfigDataclass):class DownloadConfig(ConfigDataclass):__attr_name__: ClassVar[str] = "__dl_config__"slots: int | None = None- server_lock: bool | None = None+ server_lock: int | bool | None = Noneignore_content_type: bool | None = Noneimpersonate: str | bool | None = Nonedelay: float = 0.0diff --git a/cyberdrop_dl/downloader/http.py b/cyberdrop_dl/downloader/http.pyindex a7702cf71..7745df1a5 100644--- a/cyberdrop_dl/downloader/http.py+++ b/cyberdrop_dl/downloader/http.py@@ -117,15 +117,17 @@ class Downloader:@finaldef __init__(- self, manager: Manager, slots: int | None = None, *, use_server_lock: bool = False, domain: str+ self, manager: Manager, slots: int | None = None, *, use_server_lock: bool | int = False, domain: str) -> None:self.manager: Manager = manager- self.use_server_lock: bool = use_server_lock+ self.server_slots: int | bool = use_server_lockself.capacity: Capacity = Capacity(domain)self.config: Config = self.manager.configself._processed_items: set[str] = set()self._current_attempt_filesize: dict[str, int] = {}self._server_locks: aio.WeakAsyncLocks[str] = aio.WeakAsyncLocks()++ self._server_sems: dict[str, asyncio.BoundedSemaphore] = {}self._slots: int | None = slotsself._semaphore: asyncio.Semaphoreself._set_capacity_limit(slots)@@ -133,7 +135,7 @@ class Downloader:def __post_init__(self) -> None: ...- __repr__ = simple_repr("slots", "use_server_lock", "log_prefix", "capacity", "_semaphore")+ __repr__ = simple_repr("slots", "server_slots", "log_prefix", "capacity", "_semaphore")@propertydef tui(self) -> ScrapingUI:@@ -254,7 +256,16 @@ class Downloader:@contextlib.asynccontextmanagerasync def lock(self, url: AbsoluteHttpURL) -> AsyncGenerator[None]:- server_lock = self._server_locks[url.host] if self.use_server_lock else _NULL_CONTEXT+ if self.server_slots is True:+ server_lock = self._server_locks[url.host]+ elif self.server_slots:+ try:+ server_lock = self._server_sems[url.host]+ except LookupError:+ server_lock = self._server_sems.setdefault(url.host, asyncio.BoundedSemaphore(self.server_slots))+ else:+ server_lock = _NULL_CONTEXT+async with (server_lock,self._semaphore,