Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: improve dl speed accuracy
NTFSvolume423ba39cbe8e
diff --git a/cyberdrop_dl/aio.py b/cyberdrop_dl/aio.pyindex 3f1a8a21a..73a132450 100644--- a/cyberdrop_dl/aio.py+++ b/cyberdrop_dl/aio.py@@ -461,3 +461,54 @@ def periodic_sleep(period: int, /) -> Callable[[], Awaitable[None]]:await asyncio.sleep(0)return sleep++[email protected](frozen=True, slots=True, eq=False)+class BackgroundTask:+ "Run a callable very <period>"++ fn: Callable[[], Awaitable[Any]]+ period: float+ name: str | None = None+ cancel_msg: str = "Background task took too long"+ cancelled: asyncio.Event = dataclasses.field(init=False, default_factory=asyncio.Event)+ done: asyncio.Event = dataclasses.field(init=False, default_factory=asyncio.Event)+ _task: asyncio.Task[None] = dataclasses.field(init=False, repr=False)++ async def __aenter__(self) -> Self:+ object.__setattr__(self, "_task", asyncio.create_task(self._run_forever(), name=self.name))+ return self++ async def __aexit__(+ self,+ et: type[BaseException] | None,+ exc: BaseException | None,+ tb: TracebackType | None,+ ) -> None:+ self.cancelled.set()+ try:+ async with asyncio.timeout(self.period + 0.1):+ await self.done.wait()+ except TimeoutError:+ await self._force_cancel()++ async def _force_cancel(self) -> None:+ try:+ _ = self._task.cancel(self.cancel_msg)+ await self._task+ except asyncio.CancelledError:+ pass++ async def _run_forever(self) -> None:+ try:+ while True:+ await self.fn()+ try:+ async with asyncio.timeout(self.period):+ await self.cancelled.wait()+ except TimeoutError:+ continue+ else:+ return+ finally:+ self.done.set()diff --git a/cyberdrop_dl/clients/downloads.py b/cyberdrop_dl/clients/downloads.pyindex acb10b4f4..d3b37d1b3 100644--- a/cyberdrop_dl/clients/downloads.py+++ b/cyberdrop_dl/clients/downloads.py@@ -140,13 +140,21 @@ class DownloadClient:url=media_item.url,)+ def _track_speed(self, hook: ProgressHook) -> aio.BackgroundTask:+ "force update task speed at least every 0.1 seconds"++ async def update_speed() -> None:+ hook.advance(0)++ return aio.BackgroundTask(update_speed, period=0.1)+async def _append_content(self, media_item: MediaItem, hook: ProgressHook, resp: AbstractResponse[Any]) -> None:check_free_space = storage.create_free_space_checker(media_item)check_download_speed = make_speed_checker(media_item, hook, self.download_speed_threshold)await check_free_space(media_item.size)await self._pre_download_check(media_item)- async with aio.open(media_item.partial_file, mode="ab") as f:+ async with self._track_speed(hook), aio.open(media_item.partial_file, mode="ab") as f:async for chunk in resp.iter_chunked(self.chunk_size):n_bytes = len(chunk)await self.speed_limiter.acquire(n_bytes)diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.pyindex 5ca216b02..d6a68cbd8 100644--- a/cyberdrop_dl/downloader/mega_nz.py+++ b/cyberdrop_dl/downloader/mega_nz.py@@ -35,7 +35,7 @@ class MegaDownloadClient(DownloadClient): # pyright: ignore[reportGeneralTypeIschunk_decryptor = MegaChunker(crypto.key, crypto.iv, crypto.meta_mac)aiohttp_resp = resp.aiohttp_resp- async with aio.open(media_item.partial_file, mode="ab") as f:+ async with self._track_speed(hook), aio.open(media_item.partial_file, mode="ab") as f:for _, chunk_size in get_chunks(file_size):raw_chunk = await aiohttp_resp.content.readexactly(chunk_size)chunk = chunk_decryptor.read(raw_chunk)diff --git a/cyberdrop_dl/progress/scraping/downloads.py b/cyberdrop_dl/progress/scraping/downloads.pyindex c318270bb..1db0d99da 100644--- a/cyberdrop_dl/progress/scraping/downloads.py+++ b/cyberdrop_dl/progress/scraping/downloads.py@@ -144,7 +144,10 @@ class DownloadsPanel(OverFlowPanel):),max_rows=max_rows,)+self._hls_progress: Final[DictProgress] = DictProgress("")+ for prog in (self._progress, self._hls_progress):+ prog.speed_estimate_period = 4self._total_bytes = 0@override