Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: make overflow take queue into account
NTFSvolumefd4a24c8a22a
diff --git a/cyberdrop_dl/progress/overflow.py b/cyberdrop_dl/progress/overflow.pyindex 33a704527..5be484910 100644--- a/cyberdrop_dl/progress/overflow.py+++ b/cyberdrop_dl/progress/overflow.py@@ -15,6 +15,11 @@ if TYPE_CHECKING:from rich.progress import ProgressColumn, Task, TaskID_COLOR: str = "plum3"+_COLOR2: str = "yellow"+++def _plural(ammount: int, unit: str) -> str:+ return f"{unit}{'s' if ammount > 1 else ''}"@final@@ -22,9 +27,10 @@ _COLOR: str = "plum3"class OverFlow:unit: strcount: int = 0+ queued: int = 0def __bool__(self) -> bool:- return self.count > 0+ return self.count > 0 or self.queued > 0def __rich__(self) -> str:if self:@@ -32,7 +38,17 @@ class OverFlow:return ""def __str__(self) -> str:- return f"[{_COLOR}]... and {self.count:,} other {self.unit}{'s' if self.count > 1 else ''}"+ overflow = f"[{_COLOR}]... and {self.count:,} other {_plural(self.count, self.unit)}"+ if self.count > 0 and self.queued > 0:+ return overflow + f"[{_COLOR2}] ({self.queued:,} queued)"++ if self.queued > 0:+ return f"[{_COLOR2}] ... and {self.queued:,} {_plural(self.queued, self.unit)} queued"++ if self.count > 0:+ return overflow++ return ""class OverflowPanel:@@ -42,8 +58,8 @@ class OverflowPanel:self.max_rows: int = max_rowsself._progress: Final[DictProgress] = DictProgress(*columns, expand=expand)self._overflow: Final[OverFlow] = OverFlow(self.unit)- self._invisible_queue: Final[deque[TaskID]] = deque()- self._visible_tasks: int = 0+ self._invisible_rows: Final[deque[TaskID]] = deque()+ self._visible_rows: int = 0self._panel: Final[Panel] = Panel(Group(self._progress, self._overflow),title=type(self).__name__.removesuffix("Panel"),@@ -53,12 +69,12 @@ class OverflowPanel:@finaldef __rich__(self) -> Panel:- self._overflow.count = len(self._progress) - self._visible_tasks+ self._overflow.count = len(self._progress) - self._visible_rowsreturn self._panel@finaldef _add_task(self, description: object, total: float | None = None, /, *, completed: int = 0) -> Task:- visible = self._visible_tasks < self.max_rows+ visible = self._visible_rows < self.max_rowstask_id = self._progress.add_task(f"[{_COLOR}]{escape(str(description))}",total=total,@@ -66,9 +82,9 @@ class OverflowPanel:completed=completed,)if visible:- self._visible_tasks += 1+ self._visible_rows += 1else:- self._invisible_queue.append(task_id)+ self._invisible_rows.append(task_id)return self._progress[task_id]@@ -78,7 +94,7 @@ class OverflowPanel:self._progress.remove_task(task.id)if was_visible:- self._visible_tasks -= 1+ self._visible_rows -= 1try:self._push_one_invisible()except IndexError:@@ -87,11 +103,11 @@ class OverflowPanel:@finaldef _push_one_invisible(self) -> None:while True:- invisible_task_id = self._invisible_queue.popleft()+ task_id = self._invisible_rows.popleft()try:- self._progress.update(invisible_task_id, visible=True)+ self._progress.update(task_id, visible=True)except KeyError:continueelse:- self._visible_tasks += 1+ self._visible_rows += 1breakdiff --git a/cyberdrop_dl/progress/scraping/downloads.py b/cyberdrop_dl/progress/scraping/downloads.pyindex 8581e3ad7..58ff4a5a5 100644--- a/cyberdrop_dl/progress/scraping/downloads.py+++ b/cyberdrop_dl/progress/scraping/downloads.py@@ -3,8 +3,10 @@ from __future__ import annotationsimport asyncioimport contextlibimport dataclasses+import itertoolsimport randomfrom contextvars import ContextVar+from pathlib import Pathfrom typing import TYPE_CHECKING, ClassVar, Final, finalfrom rich.jupyter import JupyterMixin@@ -38,7 +40,7 @@ _HLS_TASK_FIELD_NAME: Final = "HLS"@dataclasses.dataclass(slots=True)class AutoWidth(JupyterMixin):- """Auto expands (if possible) or truncates (if not enought space) a renderable to a desired width ratio of the screen"""+ """Expand (if possible) or truncate a renderable to a desired width ratio of the screen"""renderable: RenderableTyperatio: float@@ -72,7 +74,7 @@ class AutoTransferSpeedColumn(TransferSpeedColumn):class AutoDownloadColumn(DownloadColumn):- """Shows `<completed>/<total> MBs` for files and `<downloaded_bytes> MBs (<completed>/<total> segments)` for HLS downloads"""+ """Shows `<x>/<y> MBs` for files and `<downloaded_bytes> MBs (<x>/<y> segments)` for HLS downloads"""@overridedef render(self, task: Task) -> Text:@@ -115,7 +117,7 @@ class DownloadsPanel(OverflowPanel):@propertydef bytes_downloaded(self) -> int:- return self._total_amount+ return self._total_bytesdef __init__(self, max_rows: int = 6) -> None:super().__init__(@@ -135,7 +137,7 @@ class DownloadsPanel(OverflowPanel):max_rows=max_rows,)self._hls_progress: Final[DictProgress] = DictProgress("")- self._total_amount = 0+ self._total_bytes = 0@contextlib.contextmanagerdef download_hls(self, filename: str, /, segments: float | None = None) -> Generator[None]:@@ -162,7 +164,7 @@ class DownloadsPanel(OverflowPanel):task = self._add_task(filename, total)def advance(amount: int = 1) -> None:- self._total_amount += amount+ self._total_bytes += amountself._progress.advance(task.id, amount)def on_exit() -> None:@@ -178,7 +180,7 @@ class DownloadsPanel(OverflowPanel):hls_task: Task = self._progress[segments_task_id].fields[_HLS_TASK_FIELD_NAME]def advance(amount: int) -> None:- self._total_amount += amount+ self._total_bytes += amountself._hls_progress.advance(hls_task.id, amount)def on_exit() -> None:@@ -190,8 +192,6 @@ class DownloadsPanel(OverflowPanel):return ProgressHook(advance, get_speed, on_exit)async def simulate(self) -> None:- import itertools- from pathlib import Pathasync def download(hook: ProgressHook, size: int) -> None:total = 0@@ -249,7 +249,7 @@ class DownloadsPanel(OverflowPanel):await asyncio.sleep(2)-if __name__ == "__main__": # pragma: no cover+if __name__ == "__main__":panel = DownloadsPanel()with create_test_live(panel):asyncio.run(panel.simulate())diff --git a/cyberdrop_dl/progress/scraping/errors.py b/cyberdrop_dl/progress/scraping/errors.pyindex 4a1a159de..b2ac009b5 100644--- a/cyberdrop_dl/progress/scraping/errors.py+++ b/cyberdrop_dl/progress/scraping/errors.py@@ -66,10 +66,6 @@ class _ErrorsPanel:def __repr__(self) -> str:return f"{type(self).__name__}(error_count={self._total!r}, errors={tuple(self._errors_map)!r})"- def __init_subclass__(cls, title: str | None = None) -> None:- if title:- cls.title = title-def __init__(self) -> None:self._progress: DictProgress = DictProgress("[progress.description]{task.description}",@@ -132,10 +128,13 @@ class _ErrorsPanel:await asyncio.sleep(random.random() * 5)-class DownloadErrorsPanel(_ErrorsPanel, title="Download"): ...+class DownloadErrorsPanel(_ErrorsPanel):+ title: ClassVar[str] = "Download"++class ScrapeErrorsPanel(_ErrorsPanel):+ title: ClassVar[str] = "Scrape"-class ScrapeErrorsPanel(_ErrorsPanel, title="Scrape"):def __init__(self) -> None:super().__init__()self._unsupported: int = 0diff --git a/cyberdrop_dl/progress/scraping/panel.py b/cyberdrop_dl/progress/scraping/panel.pyindex d51dc0c67..9a31a3318 100644--- a/cyberdrop_dl/progress/scraping/panel.py+++ b/cyberdrop_dl/progress/scraping/panel.py@@ -57,7 +57,7 @@ class StatusMessage:async with asyncio.TaskGroup() as tg:for idx in range(1, 10):- tg.create_task(show(f"test msg {idx}"))+ _ = tg.create_task(show(f"test msg {idx}"))@finaldiff --git a/cyberdrop_dl/progress/sorting.py b/cyberdrop_dl/progress/sorting.pyindex 321180f3a..100e57078 100644--- a/cyberdrop_dl/progress/sorting.py+++ b/cyberdrop_dl/progress/sorting.py@@ -48,17 +48,17 @@ class SortingUI(LiveUI):self._total: int = 0self._tasks_map: dict[str, TaskID] = dict(self._init_tasks())- def file_row(name: str, file: Path) -> Text:+ def file(name: str, file: Path) -> Text:return Text.assemble((f"{name}: ", "green"), hyperlink(file))self._panel: Panel = Panel(Group(- file_row("Source", source),- file_row("Destination", dest),+ file("Source", source),+ file("Destination", dest),"",self._progress,"",- Spinner("dots", "Sorting files....", style="green"),+ Spinner("dots3", "Sorting files....", style="green"),),title="Sorting Downloads ",border_style="green",