Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: make capacity domain specific
NTFSvolume7ffd86153cee
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ccdef4daf..dc93ed0b4 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -27,7 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed- Do not crash if an input file has invalid URLs-- Scraping being limited to 1 concurrent URL when a site's scrape queue exceeds its limit even after the queue shrinks back bellow it+- Scraping being limited to 1 concurrent URL when a site's scrape queue exceeds its limit even after the queue shrinks back below it- `--hashing.post-download` never hashing any files- Thread URLs found within a another thread always being skipping even if ` --max-thread-depth` is greater than 0diff --git a/cyberdrop_dl/crawlers/bunny_cdn.py b/cyberdrop_dl/crawlers/bunny_cdn.pyindex afedb91ba..782211183 100644--- a/cyberdrop_dl/crawlers/bunny_cdn.py+++ b/cyberdrop_dl/crawlers/bunny_cdn.py@@ -38,7 +38,7 @@ class BunnyCDNCrawler(Crawler):downloader: BunnyCDNDownloaderdef __post_init__(self) -> None:- self.downloader = BunnyCDNDownloader(self.manager) # pyright: ignore[reportIncompatibleVariableOverride]+ self.downloader = BunnyCDNDownloader(self.manager, domain=self.FOLDER_DOMAIN) # pyright: ignore[reportIncompatibleVariableOverride]@classmethod@overridediff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 284135fc6..a2874cc4c 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -256,6 +256,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):manager,use_server_lock=self.__dl_config__.server_lock,slots=self.__dl_config__.slots,+ domain=self.FOLDER_DOMAIN,)self.__http_ctx__: HTTPContext = HTTPContext.build(self.DOMAIN, self.__http_config__, self.__throttle)@@ -353,7 +354,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):async def __throttle(self) -> None:if _CHECK_DL_CAPACITY.get():- await self.downloader.capacity.wait(self.FOLDER_DOMAIN)+ await self.downloader.capacity.wait()def __json_resp_check__(self, json_resp: Any, resp: AbstractResponse[Any], /) -> None:"""Custom check for JSON responses.diff --git a/cyberdrop_dl/crawlers/http_direct.py b/cyberdrop_dl/crawlers/http_direct.pyindex 638c22a63..8538a0d6f 100644--- a/cyberdrop_dl/crawlers/http_direct.py+++ b/cyberdrop_dl/crawlers/http_direct.py@@ -17,7 +17,7 @@ class DirectHttpFileCrawler(Crawler, is_generic=True):DOMAIN: ClassVar[str] = "no_crawler"async def __async_post_init__(self) -> None:- self.downloader = Downloader(self.manager)+ self.downloader = Downloader(self.manager, domain=self.DOMAIN)self.downloader.log_prefix = "Download attempt (unsupported domain)"async def fetch(self, scrape_item: ScrapeItem) -> None:diff --git a/cyberdrop_dl/downloader/http.py b/cyberdrop_dl/downloader/http.pyindex a6c39e086..4e3306134 100644--- a/cyberdrop_dl/downloader/http.py+++ b/cyberdrop_dl/downloader/http.py@@ -70,41 +70,43 @@ async def _exclusive_lock(media_item: MediaItem) -> AsyncGenerator[None]:@dataclasses.dataclass(slots=True)class Capacity:- limit: int | None = None+ domain: strwaiting: int = 0+ limit: int = 50condition: asyncio.Condition = dataclasses.field(default_factory=asyncio.Condition)- _should_warn: bool = True+ _was_overloaded: bool = False- @property- def overloaded(self) -> bool:- if self.limit is None:- return False- return self.waiting > max((self.limit * 1.2), self.limit + 10)+ def __post_init__(self) -> None:+ assert self.limit >= 1- def has_capacity(self) -> bool:- return self.limit is None or self.waiting <= self.limit+ def is_overloaded(self) -> bool:+ return self.waiting > max(self.limit * 1.2, self.limit + 10)- async def _wait(self) -> None:- if self.has_capacity():- return+ def has_capacity(self) -> bool:+ return self.waiting < self.limit+ async def wait(self) -> None:async with self.condition:+ if not self._was_overloaded and self.is_overloaded():+ self._was_overloaded = True+ logger.warning(+ "[%s] Too many downloads queued (%s+). All scraping has been temporarily paused",+ self.domain,+ self.limit,+ )++ if not self._was_overloaded:+ return+await self.condition.wait_for(self.has_capacity)- async def wait(self, domain: str) -> None:- overloaded = self.overloaded- if overloaded and self._should_warn:- logger.warning(- "[%s] Too many downloads queued (%s+). All scraping has been temporarily paused",- domain,- self.limit,- )- self._should_warn = False+ if self._was_overloaded:+ self._was_overloaded = False+ logger.debug("[%s] Resuming scraping", self.domain)- await self._wait()- if overloaded and not self.overloaded:- logger.debug("[%s] Resuming scraping", domain)- self._should_warn = True+ async def notify(self) -> None:+ async with self.condition:+ self.condition.notify_all()class Downloader:@@ -114,10 +116,12 @@ class Downloader:log_prefix: str = "Download"@final- def __init__(self, manager: Manager, slots: int | None = None, *, use_server_lock: bool = False) -> None:+ def __init__(+ self, manager: Manager, slots: int | None = None, *, use_server_lock: bool = False, domain: str+ ) -> None:self.manager: Manager = managerself.use_server_lock: bool = use_server_lock- self.capacity: Capacity = Capacity()+ self.capacity: Capacity = Capacity(domain)self.config: Config = self.manager.configself._processed_items: set[str] = set()self._current_attempt_filesize: dict[str, int] = {}@@ -204,8 +208,7 @@ class Downloader:async with self.lock(media_item.real_url):self._processed_items.add(media_item.db_path)self.capacity.waiting -= 1- async with self.capacity.condition:- self.capacity.condition.notify_all()+ await self.capacity.notify()yieldasync def __download_file(self, media_item: MediaItem) -> bool | None: # noqa: C901