Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: task cancel checkpoints
NTFSvolume913569735e45
diff --git a/cyberdrop_dl/aio.py b/cyberdrop_dl/aio.pyindex 35da46546..f0404b680 100644--- a/cyberdrop_dl/aio.py+++ b/cyberdrop_dl/aio.py@@ -62,8 +62,6 @@ class EagerTaskGroup(asyncio.TaskGroup):return await cororun = lazy()- run.__name__ = coro.__name__- run.__repr__ = coro.__repr__else:run = corodiff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.pyindex c83ab5727..59bf642c4 100644--- a/cyberdrop_dl/scrape_mapper.py+++ b/cyberdrop_dl/scrape_mapper.py@@ -4,10 +4,11 @@ import asyncioimport contextlibimport dataclassesimport logging+import signalimport sysimport timefrom pathlib import Path-from typing import TYPE_CHECKING, Literal, Self, override+from typing import TYPE_CHECKING, Any, Literal, Self, overridefrom cyberdrop_dl import aio, env, filepath, storagefrom cyberdrop_dl.constants import BLOCKED_DOMAINS@@ -30,7 +31,7 @@ from cyberdrop_dl.utils import remove_trailing_slashfrom cyberdrop_dl.utils._url import matches_any_hostif TYPE_CHECKING:- from collections.abc import AsyncGenerator, Generator, Iterator+ from collections.abc import AsyncGenerator, Callable, Generator, Iteratorfrom cyberdrop_dl.clients.jd.client import JDownloaderfrom cyberdrop_dl.config import Config@@ -235,7 +236,7 @@ class ScrapeMapper:@contextlib.contextmanagerdef __cancel_context(self) -> Generator[None]:cancelled: bool = False- with self.tui():+ with self.tui(), self._patch_kb_interrupt_sig():try:yieldexcept asyncio.CancelledError:@@ -244,11 +245,34 @@ class ScrapeMapper:raisecancelled = True- self.tui.status.shutdown()if cancelled:logger.warning("Scraping aborted ('Ctrl + C' pressed)")+ @contextlib.contextmanager+ def _patch_kb_interrupt_sig(self) -> Generator[None]:+ asyncio_runner_sig_handler = signal.getsignal(signal.SIGINT)+ new_signal: Callable[..., Any] | None = None++ if callable(asyncio_runner_sig_handler):++ def override(*args, **kwargs):+ self.tui.status.shutdown()+ return asyncio_runner_sig_handler(*args, **kwargs)++ try:+ signal.signal(signal.SIGINT, override)+ except ValueError:+ pass+ else:+ new_signal = override++ try:+ yield+ finally:+ if new_signal is not None and signal.getsignal(signal.SIGINT) is new_signal:+ signal.signal(signal.SIGINT, asyncio_runner_sig_handler)+async def __async_init__(self) -> None:if self._ready:return@@ -278,20 +302,23 @@ class ScrapeMapper:return ScrapeStats("")stats, get_items = _parse_source(src, self.manager)- async with contextlib.aclosing(get_items) as items:- self.task_mngr.downloads.create_task(self._wait_until_scrape_is_done(stats))- max_children = _build_max_children_map(self.manager.config)- async for item in items:- item.max_children = max_children- item.download_folder = self.manager.config.download_folder- if self._should_scrape(item):- stats.update(item)- self.task_mngr.scrape.create_task(self._send_to_crawler(item))+ async def dispatch() -> None:+ async with contextlib.aclosing(get_items) as items:+ self.task_mngr.downloads.create_task(self._wait_until_scrape_is_done(stats))+ max_children = _build_max_children_map(self.manager.config)++ async for item in items:+ item.max_children = max_children+ item.download_folder = self.manager.config.download_folder+ if self._should_scrape(item):+ stats.update(item)+ self.task_mngr.scrape.create_task(self._send_to_crawler(item))- if not stats.count:- logger.warning("No valid links found")+ if not stats.count:+ logger.warning("No valid links found")+ self.task_mngr.scrape.create_task(dispatch())return statsasync def send_to_crawler(self, scrape_item: ScrapeItem) -> None: