Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: improve dl speed accuracy

NTFSvolume423ba39cbe8e
diff --git a/cyberdrop_dl/aio.py b/cyberdrop_dl/aio.py
index 3f1a8a21a..73a132450 100644
--- a/cyberdrop_dl/aio.py
+++ b/cyberdrop_dl/aio.py
@@ -461,3 +461,54 @@ def periodic_sleep(period: int, /) -> Callable[[], Awaitable[None]]:
await asyncio.sleep(0)
return sleep
+
+
[email protected](frozen=True, slots=True, eq=False)
+class BackgroundTask:
+ "Run a callable very <period>"
+
+ fn: Callable[[], Awaitable[Any]]
+ period: float
+ name: str | None = None
+ cancel_msg: str = "Background task took too long"
+ cancelled: asyncio.Event = dataclasses.field(init=False, default_factory=asyncio.Event)
+ done: asyncio.Event = dataclasses.field(init=False, default_factory=asyncio.Event)
+ _task: asyncio.Task[None] = dataclasses.field(init=False, repr=False)
+
+ async def __aenter__(self) -> Self:
+ object.__setattr__(self, "_task", asyncio.create_task(self._run_forever(), name=self.name))
+ return self
+
+ async def __aexit__(
+ self,
+ et: type[BaseException] | None,
+ exc: BaseException | None,
+ tb: TracebackType | None,
+ ) -> None:
+ self.cancelled.set()
+ try:
+ async with asyncio.timeout(self.period + 0.1):
+ await self.done.wait()
+ except TimeoutError:
+ await self._force_cancel()
+
+ async def _force_cancel(self) -> None:
+ try:
+ _ = self._task.cancel(self.cancel_msg)
+ await self._task
+ except asyncio.CancelledError:
+ pass
+
+ async def _run_forever(self) -> None:
+ try:
+ while True:
+ await self.fn()
+ try:
+ async with asyncio.timeout(self.period):
+ await self.cancelled.wait()
+ except TimeoutError:
+ continue
+ else:
+ return
+ finally:
+ self.done.set()
diff --git a/cyberdrop_dl/clients/downloads.py b/cyberdrop_dl/clients/downloads.py
index acb10b4f4..d3b37d1b3 100644
--- a/cyberdrop_dl/clients/downloads.py
+++ b/cyberdrop_dl/clients/downloads.py
@@ -140,13 +140,21 @@ class DownloadClient:
url=media_item.url,
)
+ def _track_speed(self, hook: ProgressHook) -> aio.BackgroundTask:
+ "force update task speed at least every 0.1 seconds"
+
+ async def update_speed() -> None:
+ hook.advance(0)
+
+ return aio.BackgroundTask(update_speed, period=0.1)
+
async def _append_content(self, media_item: MediaItem, hook: ProgressHook, resp: AbstractResponse[Any]) -> None:
check_free_space = storage.create_free_space_checker(media_item)
check_download_speed = make_speed_checker(media_item, hook, self.download_speed_threshold)
await check_free_space(media_item.size)
await self._pre_download_check(media_item)
- async with aio.open(media_item.partial_file, mode="ab") as f:
+ async with self._track_speed(hook), aio.open(media_item.partial_file, mode="ab") as f:
async for chunk in resp.iter_chunked(self.chunk_size):
n_bytes = len(chunk)
await self.speed_limiter.acquire(n_bytes)
diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.py
index 5ca216b02..d6a68cbd8 100644
--- a/cyberdrop_dl/downloader/mega_nz.py
+++ b/cyberdrop_dl/downloader/mega_nz.py
@@ -35,7 +35,7 @@ class MegaDownloadClient(DownloadClient): # pyright: ignore[reportGeneralTypeIs
chunk_decryptor = MegaChunker(crypto.key, crypto.iv, crypto.meta_mac)
aiohttp_resp = resp.aiohttp_resp
- async with aio.open(media_item.partial_file, mode="ab") as f:
+ async with self._track_speed(hook), aio.open(media_item.partial_file, mode="ab") as f:
for _, chunk_size in get_chunks(file_size):
raw_chunk = await aiohttp_resp.content.readexactly(chunk_size)
chunk = chunk_decryptor.read(raw_chunk)
diff --git a/cyberdrop_dl/progress/scraping/downloads.py b/cyberdrop_dl/progress/scraping/downloads.py
index c318270bb..1db0d99da 100644
--- a/cyberdrop_dl/progress/scraping/downloads.py
+++ b/cyberdrop_dl/progress/scraping/downloads.py
@@ -144,7 +144,10 @@ class DownloadsPanel(OverFlowPanel):
),
max_rows=max_rows,
)
+
self._hls_progress: Final[DictProgress] = DictProgress("")
+ for prog in (self._progress, self._hls_progress):
+ prog.speed_estimate_period = 4
self._total_bytes = 0
@override