Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: make capacity domain specific

NTFSvolume7ffd86153cee
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ccdef4daf..dc93ed0b4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -27,7 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
- Do not crash if an input file has invalid URLs
-- Scraping being limited to 1 concurrent URL when a site's scrape queue exceeds its limit even after the queue shrinks back bellow it
+- Scraping being limited to 1 concurrent URL when a site's scrape queue exceeds its limit even after the queue shrinks back below it
- `--hashing.post-download` never hashing any files
- Thread URLs found within a another thread always being skipping even if ` --max-thread-depth` is greater than 0
diff --git a/cyberdrop_dl/crawlers/bunny_cdn.py b/cyberdrop_dl/crawlers/bunny_cdn.py
index afedb91ba..782211183 100644
--- a/cyberdrop_dl/crawlers/bunny_cdn.py
+++ b/cyberdrop_dl/crawlers/bunny_cdn.py
@@ -38,7 +38,7 @@ class BunnyCDNCrawler(Crawler):
downloader: BunnyCDNDownloader
def __post_init__(self) -> None:
- self.downloader = BunnyCDNDownloader(self.manager) # pyright: ignore[reportIncompatibleVariableOverride]
+ self.downloader = BunnyCDNDownloader(self.manager, domain=self.FOLDER_DOMAIN) # pyright: ignore[reportIncompatibleVariableOverride]
@classmethod
@override
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 284135fc6..a2874cc4c 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -256,6 +256,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
manager,
use_server_lock=self.__dl_config__.server_lock,
slots=self.__dl_config__.slots,
+ domain=self.FOLDER_DOMAIN,
)
self.__http_ctx__: HTTPContext = HTTPContext.build(self.DOMAIN, self.__http_config__, self.__throttle)
@@ -353,7 +354,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
async def __throttle(self) -> None:
if _CHECK_DL_CAPACITY.get():
- await self.downloader.capacity.wait(self.FOLDER_DOMAIN)
+ await self.downloader.capacity.wait()
def __json_resp_check__(self, json_resp: Any, resp: AbstractResponse[Any], /) -> None:
"""Custom check for JSON responses.
diff --git a/cyberdrop_dl/crawlers/http_direct.py b/cyberdrop_dl/crawlers/http_direct.py
index 638c22a63..8538a0d6f 100644
--- a/cyberdrop_dl/crawlers/http_direct.py
+++ b/cyberdrop_dl/crawlers/http_direct.py
@@ -17,7 +17,7 @@ class DirectHttpFileCrawler(Crawler, is_generic=True):
DOMAIN: ClassVar[str] = "no_crawler"
async def __async_post_init__(self) -> None:
- self.downloader = Downloader(self.manager)
+ self.downloader = Downloader(self.manager, domain=self.DOMAIN)
self.downloader.log_prefix = "Download attempt (unsupported domain)"
async def fetch(self, scrape_item: ScrapeItem) -> None:
diff --git a/cyberdrop_dl/downloader/http.py b/cyberdrop_dl/downloader/http.py
index a6c39e086..4e3306134 100644
--- a/cyberdrop_dl/downloader/http.py
+++ b/cyberdrop_dl/downloader/http.py
@@ -70,41 +70,43 @@ async def _exclusive_lock(media_item: MediaItem) -> AsyncGenerator[None]:
@dataclasses.dataclass(slots=True)
class Capacity:
- limit: int | None = None
+ domain: str
waiting: int = 0
+ limit: int = 50
condition: asyncio.Condition = dataclasses.field(default_factory=asyncio.Condition)
- _should_warn: bool = True
+ _was_overloaded: bool = False
- @property
- def overloaded(self) -> bool:
- if self.limit is None:
- return False
- return self.waiting > max((self.limit * 1.2), self.limit + 10)
+ def __post_init__(self) -> None:
+ assert self.limit >= 1
- def has_capacity(self) -> bool:
- return self.limit is None or self.waiting <= self.limit
+ def is_overloaded(self) -> bool:
+ return self.waiting > max(self.limit * 1.2, self.limit + 10)
- async def _wait(self) -> None:
- if self.has_capacity():
- return
+ def has_capacity(self) -> bool:
+ return self.waiting < self.limit
+ async def wait(self) -> None:
async with self.condition:
+ if not self._was_overloaded and self.is_overloaded():
+ self._was_overloaded = True
+ logger.warning(
+ "[%s] Too many downloads queued (%s+). All scraping has been temporarily paused",
+ self.domain,
+ self.limit,
+ )
+
+ if not self._was_overloaded:
+ return
+
await self.condition.wait_for(self.has_capacity)
- async def wait(self, domain: str) -> None:
- overloaded = self.overloaded
- if overloaded and self._should_warn:
- logger.warning(
- "[%s] Too many downloads queued (%s+). All scraping has been temporarily paused",
- domain,
- self.limit,
- )
- self._should_warn = False
+ if self._was_overloaded:
+ self._was_overloaded = False
+ logger.debug("[%s] Resuming scraping", self.domain)
- await self._wait()
- if overloaded and not self.overloaded:
- logger.debug("[%s] Resuming scraping", domain)
- self._should_warn = True
+ async def notify(self) -> None:
+ async with self.condition:
+ self.condition.notify_all()
class Downloader:
@@ -114,10 +116,12 @@ class Downloader:
log_prefix: str = "Download"
@final
- def __init__(self, manager: Manager, slots: int | None = None, *, use_server_lock: bool = False) -> None:
+ def __init__(
+ self, manager: Manager, slots: int | None = None, *, use_server_lock: bool = False, domain: str
+ ) -> None:
self.manager: Manager = manager
self.use_server_lock: bool = use_server_lock
- self.capacity: Capacity = Capacity()
+ self.capacity: Capacity = Capacity(domain)
self.config: Config = self.manager.config
self._processed_items: set[str] = set()
self._current_attempt_filesize: dict[str, int] = {}
@@ -204,8 +208,7 @@ class Downloader:
async with self.lock(media_item.real_url):
self._processed_items.add(media_item.db_path)
self.capacity.waiting -= 1
- async with self.capacity.condition:
- self.capacity.condition.notify_all()
+ await self.capacity.notify()
yield
async def __download_file(self, media_item: MediaItem) -> bool | None: # noqa: C901