Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: unify all aio.gathers

NTFSvolumeaade585021fc
diff --git a/cyberdrop_dl/aio.py b/cyberdrop_dl/aio.py
index b6ee02209..6d8ff1640 100644
--- a/cyberdrop_dl/aio.py
+++ b/cyberdrop_dl/aio.py
@@ -28,6 +28,7 @@ if TYPE_CHECKING:
Coroutine,
Iterable,
Iterator,
+ Mapping,
Sequence,
)
from contextvars import Context
@@ -237,49 +238,100 @@ class AsyncIteratorWrapper[T]:
return cast("T", value)
-async def gather[T](*coros: Awaitable[T]) -> list[T]:
- """Like asyncio.gather but an exception on any coro cancels all pending coros
+@overload
+async def gather[T1](coro1: Awaitable[T1], /, *, fail_fast: bool = True) -> tuple[T1]: ...
- AKA: all or nothing"""
- async def wrap(coro: Awaitable[T]) -> T:
- return await coro
+@overload
+async def gather[T1, T2](coro1: Awaitable[T1], coro2: Awaitable[T2], /, *, fail_fast: bool = True) -> tuple[T1, T2]: ...
- async with asyncio.TaskGroup() as tg:
- tasks = [tg.create_task(wrap(coro)) for coro in coros]
- return [t.result() for t in tasks]
+@overload
+async def gather[T1, T2, T3](
+ coro1: Awaitable[T1], coro2: Awaitable[T2], coro3: Awaitable[T3], /, *, fail_fast: bool = True
+) -> tuple[T1, T2, T3]: ...
@overload
-async def safe_gather[T1](coro: Awaitable[T1], /) -> tuple[T1]: ...
-@overload
-async def safe_gather[T1, T2](coro_1: Awaitable[T1], coro_2: Awaitable[T2], /) -> tuple[T1, T2]: ...
+async def gather[T1, T2, T3, T4](
+ coro1: Awaitable[T1], coro2: Awaitable[T2], coro3: Awaitable[T3], coro4: Awaitable[T4], /, *, fail_fast: bool = True
+) -> tuple[T1, T2, T3, T4]: ...
+
+
@overload
-async def safe_gather[T1, T2, T3](
- coro_1: Awaitable[T1],
- coro_2: Awaitable[T2],
- coro_3: Awaitable[T3],
+async def gather[T1, T2, T3, T4, T5](
+ coro1: Awaitable[T1],
+ coro2: Awaitable[T2],
+ coro3: Awaitable[T3],
+ coro4: Awaitable[T4],
+ coro5: Awaitable[T5],
/,
-) -> tuple[T1, T2, T3]: ...
+ *,
+ fail_fast: bool = True,
+) -> tuple[T1, T2, T3, T4, T5]: ...
-async def safe_gather[T1, T2, T3](
- coro_1: Awaitable[T1],
- coro_2: Awaitable[T2] | None = None,
- coro_3: Awaitable[T3] | None = None,
+@overload
+async def gather[T1, T2, T3, T4, T5, T6](
+ coro1: Awaitable[T1],
+ coro2: Awaitable[T2],
+ coro3: Awaitable[T3],
+ coro4: Awaitable[T4],
+ coro5: Awaitable[T5],
+ coro6: Awaitable[T6],
/,
-) -> Sequence[T1 | T2 | T3]:
- """Like `asyncio.gather(*coros, return_exceptions=True)`, but all exceptions are re-raised as an ExceptionGroup
+ *,
+ fail_fast: bool = True,
+) -> tuple[T1, T2, T3, T4, T5, T6]: ...
+
+
+@overload
+async def gather[T](*coros: Awaitable[T], fail_fast: bool = True) -> list[T]: ...
+
- In the same order as they were scheduled. This makes errors deterministic"""
+async def gather[T](*coros: Awaitable[T], fail_fast: bool = True) -> Sequence[T]: # pyright: ignore[reportInconsistentOverload]
+ """Like asyncio.gather(*coros, return_exceptions=False).
+
+ if `fail_fast` is `True`, an exception on any coro immediately cancels all pending coros and is re-raised as an ExceptionGroup
+
+ if `fail_fast` is `False`, it waits for all coros to complete. It there was any exception, they are grouped and re-raised as an ExceptionGroup
+ in the same order as they were scheduled. This makes errors deterministic.
+ """
+
+ if fail_fast:
+ results, _ = await _tg_gather(coros, return_exceptions=False)
+ return results
- coros = filter(None, (coro_1, coro_2, coro_3))
results = await asyncio.gather(*coros, return_exceptions=True) # noqa: TID251
errors = tuple(r for r in results if isinstance(r, BaseException))
if errors:
raise BaseExceptionGroup("", errors)
- return cast("list[T1 | T2 | T3]", results)
+ return cast("list[T]", results)
+
+
+def _values_sorted_by_key[T](results: Mapping[int, T]) -> list[T]:
+ return [result for _, result in sorted(results.items())]
+
+
+async def _tg_gather[T](
+ coros: Iterable[Awaitable[T]], *, return_exceptions: bool = False
+) -> tuple[list[T], list[Exception]]:
+ results: dict[int, T] = {}
+ errors: dict[int, Exception] = {}
+
+ async def wrap(idx: int, coro: Awaitable[T]) -> None:
+ try:
+ results[idx] = await coro
+ except Exception as e:
+ if not return_exceptions:
+ raise
+ errors[idx] = e
+
+ async with asyncio.TaskGroup() as tg:
+ for idx, coro in enumerate(coros):
+ tg.create_task(wrap(idx, coro))
+
+ return _values_sorted_by_key(results), _values_sorted_by_key(errors)
async def map[T, R](
@@ -305,12 +357,12 @@ async def afilter[T](
) -> filter[T]:
## TODO use queue for lazy iteration with queue.shutdown when dropping pythohn 3.12
- async def fn(value: T) -> T | MISSING: # pyright: ignore[reportInvalidTypeForm]
+ async def fn(value: T) -> T:
if await predicate(value):
return value
return MISSING
- results = await map(fn, params, task_limit=task_limit) # pyright: ignore[reportUnknownArgumentType]
+ results = await map(fn, params, task_limit=task_limit)
return filter(lambda x: x is not MISSING, results)
@@ -372,7 +424,7 @@ async def map_tuples[*Ts, R](
else:
tg.create_task(run(idx, coro_factory(*params)))
- return [result for _, result in sorted(results.items())]
+ return _values_sorted_by_key(results)
def run[T](coro: Coroutine[Any, Any, T]) -> T:
diff --git a/cyberdrop_dl/crawlers/_hls.py b/cyberdrop_dl/crawlers/_hls.py
index 7acafca1a..4b8efcd6a 100644
--- a/cyberdrop_dl/crawlers/_hls.py
+++ b/cyberdrop_dl/crawlers/_hls.py
@@ -74,23 +74,13 @@ class HLSMixin(ABC):
**kwargs: Unpack[RequestParams],
) -> tuple[m3u8.Rendition, m3u8.RenditionDetails]:
- async def resolve(
- url: AbsoluteHttpURL | None, media_type: Literal["video", "audio", "subtitle"]
- ) -> m3u8.M3U8 | None:
+ async def resolve(media_type: Literal["video", "audio", "subtitle"]) -> m3u8.M3U8 | None:
+ url: AbsoluteHttpURL | None = getattr(rendition.urls, media_type)
if not url:
return None
return await self._request_m3u8(url, method, media_type, **kwargs)
- video, audio, subs = await aio.safe_gather(
- *(
- resolve(url, name)
- for name, url in zip(
- ("video", "audio", "subtitle"),
- rendition.urls,
- strict=True,
- )
- )
- )
+ video, audio, subs = await aio.gather(resolve("video"), resolve("audio"), resolve("subtitle"))
assert video
return m3u8.Rendition(video, audio, subs), rendition
diff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.py
index 69f0b3f67..d0baef9dc 100644
--- a/cyberdrop_dl/crawlers/cyberdrop.py
+++ b/cyberdrop_dl/crawlers/cyberdrop.py
@@ -73,7 +73,7 @@ class CyberdropCrawler(Crawler):
if await self.check_complete_from_referer(scrape_item.url):
return
- info, auth = await aio.safe_gather(self.api.file_info(file_id), self.api.file_auth(file_id))
+ info, auth = await aio.gather(self.api.file_info(file_id), self.api.file_auth(file_id), fail_fast=False)
name: str = info["name"]
filename, ext = self.get_filename_and_ext(name)
diff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.py
index 031964b13..663bbdfb1 100644
--- a/cyberdrop_dl/crawlers/gofile.py
+++ b/cyberdrop_dl/crawlers/gofile.py
@@ -27,6 +27,7 @@ class GoFileCrawler(Crawler):
"/download/<content_id>/<filename>",
"/download/web/<content_id>/<filename>",
),
+ "Direct link (Premium)": ("/download/direct/<content_id>/<filename>",),
"**NOTE**": (
"Use `password` as a query param to download password protected folders",
"ex: https://gofile.io/d/ABC654?password=1234",
diff --git a/cyberdrop_dl/crawlers/hohoj_tv.py b/cyberdrop_dl/crawlers/hohoj_tv.py
index 02256c7b2..b45bfca56 100644
--- a/cyberdrop_dl/crawlers/hohoj_tv.py
+++ b/cyberdrop_dl/crawlers/hohoj_tv.py
@@ -71,7 +71,7 @@ class HohojTVCrawler(Crawler):
url = (self.PRIMARY_URL / "lang_en" / "video").with_query(id=video_id)
embed_url = (self.PRIMARY_URL / "embed").with_query(id=video_id)
- soup, embed = await aio.safe_gather(self.request_soup(url), self.request_text(embed_url))
+ soup, embed = await aio.gather(self.request_soup(url), self.request_text(embed_url), fail_fast=False)
player = css.select(soup, ".player-col")
return Video(
diff --git a/cyberdrop_dl/crawlers/mixdrop.py b/cyberdrop_dl/crawlers/mixdrop.py
index 545df1c3c..3c736286c 100644
--- a/cyberdrop_dl/crawlers/mixdrop.py
+++ b/cyberdrop_dl/crawlers/mixdrop.py
@@ -59,7 +59,7 @@ class MixDropCrawler(Crawler):
video_url = self.PRIMARY_URL / "f" / file_id
embed_url = self.PRIMARY_URL / "e" / file_id
- soup, embed_html = await aio.safe_gather(self.request_soup(video_url), self.request_text(embed_url))
+ soup, embed_html = await aio.gather(self.request_soup(video_url), self.request_text(embed_url), fail_fast=False)
title = css.select_text(soup, "div.tbl-c.title b")
md_props = dict(_extract_properties(embed_html))
return title, self.parse_url(md_props["wurl"])
diff --git a/cyberdrop_dl/crawlers/realdebrid.py b/cyberdrop_dl/crawlers/realdebrid.py
index 68e457979..855d99b80 100644
--- a/cyberdrop_dl/crawlers/realdebrid.py
+++ b/cyberdrop_dl/crawlers/realdebrid.py
@@ -145,9 +145,10 @@ class RealDebridAPI(API):
return bool(self._folder_regex.search(str(url)))
async def connect(self) -> None:
- responses: tuple[list[str], list[str]] = await aio.safe_gather(
+ responses: tuple[list[str], list[str]] = await aio.gather(
self._request("hosts/regex"),
self._request("hosts/regexFolder"),
+ fail_fast=False,
)
file_regex = (pattern[1:-1] for pattern in responses[0])