Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: unify all aio.gathers
NTFSvolumeaade585021fc
diff --git a/cyberdrop_dl/aio.py b/cyberdrop_dl/aio.pyindex b6ee02209..6d8ff1640 100644--- a/cyberdrop_dl/aio.py+++ b/cyberdrop_dl/aio.py@@ -28,6 +28,7 @@ if TYPE_CHECKING:Coroutine,Iterable,Iterator,+ Mapping,Sequence,)from contextvars import Context@@ -237,49 +238,100 @@ class AsyncIteratorWrapper[T]:return cast("T", value)-async def gather[T](*coros: Awaitable[T]) -> list[T]:- """Like asyncio.gather but an exception on any coro cancels all pending coros+@overload+async def gather[T1](coro1: Awaitable[T1], /, *, fail_fast: bool = True) -> tuple[T1]: ...- AKA: all or nothing"""- async def wrap(coro: Awaitable[T]) -> T:- return await coro+@overload+async def gather[T1, T2](coro1: Awaitable[T1], coro2: Awaitable[T2], /, *, fail_fast: bool = True) -> tuple[T1, T2]: ...- async with asyncio.TaskGroup() as tg:- tasks = [tg.create_task(wrap(coro)) for coro in coros]- return [t.result() for t in tasks]+@overload+async def gather[T1, T2, T3](+ coro1: Awaitable[T1], coro2: Awaitable[T2], coro3: Awaitable[T3], /, *, fail_fast: bool = True+) -> tuple[T1, T2, T3]: ...@overload-async def safe_gather[T1](coro: Awaitable[T1], /) -> tuple[T1]: ...-@overload-async def safe_gather[T1, T2](coro_1: Awaitable[T1], coro_2: Awaitable[T2], /) -> tuple[T1, T2]: ...+async def gather[T1, T2, T3, T4](+ coro1: Awaitable[T1], coro2: Awaitable[T2], coro3: Awaitable[T3], coro4: Awaitable[T4], /, *, fail_fast: bool = True+) -> tuple[T1, T2, T3, T4]: ...++@overload-async def safe_gather[T1, T2, T3](- coro_1: Awaitable[T1],- coro_2: Awaitable[T2],- coro_3: Awaitable[T3],+async def gather[T1, T2, T3, T4, T5](+ coro1: Awaitable[T1],+ coro2: Awaitable[T2],+ coro3: Awaitable[T3],+ coro4: Awaitable[T4],+ coro5: Awaitable[T5],/,-) -> tuple[T1, T2, T3]: ...+ *,+ fail_fast: bool = True,+) -> tuple[T1, T2, T3, T4, T5]: ...-async def safe_gather[T1, T2, T3](- coro_1: Awaitable[T1],- coro_2: Awaitable[T2] | None = None,- coro_3: Awaitable[T3] | None = None,+@overload+async def gather[T1, T2, T3, T4, T5, T6](+ coro1: Awaitable[T1],+ coro2: Awaitable[T2],+ coro3: Awaitable[T3],+ coro4: Awaitable[T4],+ coro5: Awaitable[T5],+ coro6: Awaitable[T6],/,-) -> Sequence[T1 | T2 | T3]:- """Like `asyncio.gather(*coros, return_exceptions=True)`, but all exceptions are re-raised as an ExceptionGroup+ *,+ fail_fast: bool = True,+) -> tuple[T1, T2, T3, T4, T5, T6]: ...+++@overload+async def gather[T](*coros: Awaitable[T], fail_fast: bool = True) -> list[T]: ...+- In the same order as they were scheduled. This makes errors deterministic"""+async def gather[T](*coros: Awaitable[T], fail_fast: bool = True) -> Sequence[T]: # pyright: ignore[reportInconsistentOverload]+ """Like asyncio.gather(*coros, return_exceptions=False).++ if `fail_fast` is `True`, an exception on any coro immediately cancels all pending coros and is re-raised as an ExceptionGroup++ if `fail_fast` is `False`, it waits for all coros to complete. It there was any exception, they are grouped and re-raised as an ExceptionGroup+ in the same order as they were scheduled. This makes errors deterministic.+ """++ if fail_fast:+ results, _ = await _tg_gather(coros, return_exceptions=False)+ return results- coros = filter(None, (coro_1, coro_2, coro_3))results = await asyncio.gather(*coros, return_exceptions=True) # noqa: TID251errors = tuple(r for r in results if isinstance(r, BaseException))if errors:raise BaseExceptionGroup("", errors)- return cast("list[T1 | T2 | T3]", results)+ return cast("list[T]", results)+++def _values_sorted_by_key[T](results: Mapping[int, T]) -> list[T]:+ return [result for _, result in sorted(results.items())]+++async def _tg_gather[T](+ coros: Iterable[Awaitable[T]], *, return_exceptions: bool = False+) -> tuple[list[T], list[Exception]]:+ results: dict[int, T] = {}+ errors: dict[int, Exception] = {}++ async def wrap(idx: int, coro: Awaitable[T]) -> None:+ try:+ results[idx] = await coro+ except Exception as e:+ if not return_exceptions:+ raise+ errors[idx] = e++ async with asyncio.TaskGroup() as tg:+ for idx, coro in enumerate(coros):+ tg.create_task(wrap(idx, coro))++ return _values_sorted_by_key(results), _values_sorted_by_key(errors)async def map[T, R](@@ -305,12 +357,12 @@ async def afilter[T]() -> filter[T]:## TODO use queue for lazy iteration with queue.shutdown when dropping pythohn 3.12- async def fn(value: T) -> T | MISSING: # pyright: ignore[reportInvalidTypeForm]+ async def fn(value: T) -> T:if await predicate(value):return valuereturn MISSING- results = await map(fn, params, task_limit=task_limit) # pyright: ignore[reportUnknownArgumentType]+ results = await map(fn, params, task_limit=task_limit)return filter(lambda x: x is not MISSING, results)@@ -372,7 +424,7 @@ async def map_tuples[*Ts, R](else:tg.create_task(run(idx, coro_factory(*params)))- return [result for _, result in sorted(results.items())]+ return _values_sorted_by_key(results)def run[T](coro: Coroutine[Any, Any, T]) -> T:diff --git a/cyberdrop_dl/crawlers/_hls.py b/cyberdrop_dl/crawlers/_hls.pyindex 7acafca1a..4b8efcd6a 100644--- a/cyberdrop_dl/crawlers/_hls.py+++ b/cyberdrop_dl/crawlers/_hls.py@@ -74,23 +74,13 @@ class HLSMixin(ABC):**kwargs: Unpack[RequestParams],) -> tuple[m3u8.Rendition, m3u8.RenditionDetails]:- async def resolve(- url: AbsoluteHttpURL | None, media_type: Literal["video", "audio", "subtitle"]- ) -> m3u8.M3U8 | None:+ async def resolve(media_type: Literal["video", "audio", "subtitle"]) -> m3u8.M3U8 | None:+ url: AbsoluteHttpURL | None = getattr(rendition.urls, media_type)if not url:return Nonereturn await self._request_m3u8(url, method, media_type, **kwargs)- video, audio, subs = await aio.safe_gather(- *(- resolve(url, name)- for name, url in zip(- ("video", "audio", "subtitle"),- rendition.urls,- strict=True,- )- )- )+ video, audio, subs = await aio.gather(resolve("video"), resolve("audio"), resolve("subtitle"))assert videoreturn m3u8.Rendition(video, audio, subs), renditiondiff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.pyindex 69f0b3f67..d0baef9dc 100644--- a/cyberdrop_dl/crawlers/cyberdrop.py+++ b/cyberdrop_dl/crawlers/cyberdrop.py@@ -73,7 +73,7 @@ class CyberdropCrawler(Crawler):if await self.check_complete_from_referer(scrape_item.url):return- info, auth = await aio.safe_gather(self.api.file_info(file_id), self.api.file_auth(file_id))+ info, auth = await aio.gather(self.api.file_info(file_id), self.api.file_auth(file_id), fail_fast=False)name: str = info["name"]filename, ext = self.get_filename_and_ext(name)diff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.pyindex 031964b13..663bbdfb1 100644--- a/cyberdrop_dl/crawlers/gofile.py+++ b/cyberdrop_dl/crawlers/gofile.py@@ -27,6 +27,7 @@ class GoFileCrawler(Crawler):"/download/<content_id>/<filename>","/download/web/<content_id>/<filename>",),+ "Direct link (Premium)": ("/download/direct/<content_id>/<filename>",),"**NOTE**": ("Use `password` as a query param to download password protected folders","ex: https://gofile.io/d/ABC654?password=1234",diff --git a/cyberdrop_dl/crawlers/hohoj_tv.py b/cyberdrop_dl/crawlers/hohoj_tv.pyindex 02256c7b2..b45bfca56 100644--- a/cyberdrop_dl/crawlers/hohoj_tv.py+++ b/cyberdrop_dl/crawlers/hohoj_tv.py@@ -71,7 +71,7 @@ class HohojTVCrawler(Crawler):url = (self.PRIMARY_URL / "lang_en" / "video").with_query(id=video_id)embed_url = (self.PRIMARY_URL / "embed").with_query(id=video_id)- soup, embed = await aio.safe_gather(self.request_soup(url), self.request_text(embed_url))+ soup, embed = await aio.gather(self.request_soup(url), self.request_text(embed_url), fail_fast=False)player = css.select(soup, ".player-col")return Video(diff --git a/cyberdrop_dl/crawlers/mixdrop.py b/cyberdrop_dl/crawlers/mixdrop.pyindex 545df1c3c..3c736286c 100644--- a/cyberdrop_dl/crawlers/mixdrop.py+++ b/cyberdrop_dl/crawlers/mixdrop.py@@ -59,7 +59,7 @@ class MixDropCrawler(Crawler):video_url = self.PRIMARY_URL / "f" / file_idembed_url = self.PRIMARY_URL / "e" / file_id- soup, embed_html = await aio.safe_gather(self.request_soup(video_url), self.request_text(embed_url))+ soup, embed_html = await aio.gather(self.request_soup(video_url), self.request_text(embed_url), fail_fast=False)title = css.select_text(soup, "div.tbl-c.title b")md_props = dict(_extract_properties(embed_html))return title, self.parse_url(md_props["wurl"])diff --git a/cyberdrop_dl/crawlers/realdebrid.py b/cyberdrop_dl/crawlers/realdebrid.pyindex 68e457979..855d99b80 100644--- a/cyberdrop_dl/crawlers/realdebrid.py+++ b/cyberdrop_dl/crawlers/realdebrid.py@@ -145,9 +145,10 @@ class RealDebridAPI(API):return bool(self._folder_regex.search(str(url)))async def connect(self) -> None:- responses: tuple[list[str], list[str]] = await aio.safe_gather(+ responses: tuple[list[str], list[str]] = await aio.gather(self._request("hosts/regex"),self._request("hosts/regexFolder"),+ fail_fast=False,)file_regex = (pattern[1:-1] for pattern in responses[0])