Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: make overflow take queue into account

NTFSvolumefd4a24c8a22a
diff --git a/cyberdrop_dl/progress/overflow.py b/cyberdrop_dl/progress/overflow.py
index 33a704527..5be484910 100644
--- a/cyberdrop_dl/progress/overflow.py
+++ b/cyberdrop_dl/progress/overflow.py
@@ -15,6 +15,11 @@ if TYPE_CHECKING:
from rich.progress import ProgressColumn, Task, TaskID
_COLOR: str = "plum3"
+_COLOR2: str = "yellow"
+
+
+def _plural(ammount: int, unit: str) -> str:
+ return f"{unit}{'s' if ammount > 1 else ''}"
@final
@@ -22,9 +27,10 @@ _COLOR: str = "plum3"
class OverFlow:
unit: str
count: int = 0
+ queued: int = 0
def __bool__(self) -> bool:
- return self.count > 0
+ return self.count > 0 or self.queued > 0
def __rich__(self) -> str:
if self:
@@ -32,7 +38,17 @@ class OverFlow:
return ""
def __str__(self) -> str:
- return f"[{_COLOR}]... and {self.count:,} other {self.unit}{'s' if self.count > 1 else ''}"
+ overflow = f"[{_COLOR}]... and {self.count:,} other {_plural(self.count, self.unit)}"
+ if self.count > 0 and self.queued > 0:
+ return overflow + f"[{_COLOR2}] ({self.queued:,} queued)"
+
+ if self.queued > 0:
+ return f"[{_COLOR2}] ... and {self.queued:,} {_plural(self.queued, self.unit)} queued"
+
+ if self.count > 0:
+ return overflow
+
+ return ""
class OverflowPanel:
@@ -42,8 +58,8 @@ class OverflowPanel:
self.max_rows: int = max_rows
self._progress: Final[DictProgress] = DictProgress(*columns, expand=expand)
self._overflow: Final[OverFlow] = OverFlow(self.unit)
- self._invisible_queue: Final[deque[TaskID]] = deque()
- self._visible_tasks: int = 0
+ self._invisible_rows: Final[deque[TaskID]] = deque()
+ self._visible_rows: int = 0
self._panel: Final[Panel] = Panel(
Group(self._progress, self._overflow),
title=type(self).__name__.removesuffix("Panel"),
@@ -53,12 +69,12 @@ class OverflowPanel:
@final
def __rich__(self) -> Panel:
- self._overflow.count = len(self._progress) - self._visible_tasks
+ self._overflow.count = len(self._progress) - self._visible_rows
return self._panel
@final
def _add_task(self, description: object, total: float | None = None, /, *, completed: int = 0) -> Task:
- visible = self._visible_tasks < self.max_rows
+ visible = self._visible_rows < self.max_rows
task_id = self._progress.add_task(
f"[{_COLOR}]{escape(str(description))}",
total=total,
@@ -66,9 +82,9 @@ class OverflowPanel:
completed=completed,
)
if visible:
- self._visible_tasks += 1
+ self._visible_rows += 1
else:
- self._invisible_queue.append(task_id)
+ self._invisible_rows.append(task_id)
return self._progress[task_id]
@@ -78,7 +94,7 @@ class OverflowPanel:
self._progress.remove_task(task.id)
if was_visible:
- self._visible_tasks -= 1
+ self._visible_rows -= 1
try:
self._push_one_invisible()
except IndexError:
@@ -87,11 +103,11 @@ class OverflowPanel:
@final
def _push_one_invisible(self) -> None:
while True:
- invisible_task_id = self._invisible_queue.popleft()
+ task_id = self._invisible_rows.popleft()
try:
- self._progress.update(invisible_task_id, visible=True)
+ self._progress.update(task_id, visible=True)
except KeyError:
continue
else:
- self._visible_tasks += 1
+ self._visible_rows += 1
break
diff --git a/cyberdrop_dl/progress/scraping/downloads.py b/cyberdrop_dl/progress/scraping/downloads.py
index 8581e3ad7..58ff4a5a5 100644
--- a/cyberdrop_dl/progress/scraping/downloads.py
+++ b/cyberdrop_dl/progress/scraping/downloads.py
@@ -3,8 +3,10 @@ from __future__ import annotations
import asyncio
import contextlib
import dataclasses
+import itertools
import random
from contextvars import ContextVar
+from pathlib import Path
from typing import TYPE_CHECKING, ClassVar, Final, final
from rich.jupyter import JupyterMixin
@@ -38,7 +40,7 @@ _HLS_TASK_FIELD_NAME: Final = "HLS"
@dataclasses.dataclass(slots=True)
class AutoWidth(JupyterMixin):
- """Auto expands (if possible) or truncates (if not enought space) a renderable to a desired width ratio of the screen"""
+ """Expand (if possible) or truncate a renderable to a desired width ratio of the screen"""
renderable: RenderableType
ratio: float
@@ -72,7 +74,7 @@ class AutoTransferSpeedColumn(TransferSpeedColumn):
class AutoDownloadColumn(DownloadColumn):
- """Shows `<completed>/<total> MBs` for files and `<downloaded_bytes> MBs (<completed>/<total> segments)` for HLS downloads"""
+ """Shows `<x>/<y> MBs` for files and `<downloaded_bytes> MBs (<x>/<y> segments)` for HLS downloads"""
@override
def render(self, task: Task) -> Text:
@@ -115,7 +117,7 @@ class DownloadsPanel(OverflowPanel):
@property
def bytes_downloaded(self) -> int:
- return self._total_amount
+ return self._total_bytes
def __init__(self, max_rows: int = 6) -> None:
super().__init__(
@@ -135,7 +137,7 @@ class DownloadsPanel(OverflowPanel):
max_rows=max_rows,
)
self._hls_progress: Final[DictProgress] = DictProgress("")
- self._total_amount = 0
+ self._total_bytes = 0
@contextlib.contextmanager
def download_hls(self, filename: str, /, segments: float | None = None) -> Generator[None]:
@@ -162,7 +164,7 @@ class DownloadsPanel(OverflowPanel):
task = self._add_task(filename, total)
def advance(amount: int = 1) -> None:
- self._total_amount += amount
+ self._total_bytes += amount
self._progress.advance(task.id, amount)
def on_exit() -> None:
@@ -178,7 +180,7 @@ class DownloadsPanel(OverflowPanel):
hls_task: Task = self._progress[segments_task_id].fields[_HLS_TASK_FIELD_NAME]
def advance(amount: int) -> None:
- self._total_amount += amount
+ self._total_bytes += amount
self._hls_progress.advance(hls_task.id, amount)
def on_exit() -> None:
@@ -190,8 +192,6 @@ class DownloadsPanel(OverflowPanel):
return ProgressHook(advance, get_speed, on_exit)
async def simulate(self) -> None:
- import itertools
- from pathlib import Path
async def download(hook: ProgressHook, size: int) -> None:
total = 0
@@ -249,7 +249,7 @@ class DownloadsPanel(OverflowPanel):
await asyncio.sleep(2)
-if __name__ == "__main__": # pragma: no cover
+if __name__ == "__main__":
panel = DownloadsPanel()
with create_test_live(panel):
asyncio.run(panel.simulate())
diff --git a/cyberdrop_dl/progress/scraping/errors.py b/cyberdrop_dl/progress/scraping/errors.py
index 4a1a159de..b2ac009b5 100644
--- a/cyberdrop_dl/progress/scraping/errors.py
+++ b/cyberdrop_dl/progress/scraping/errors.py
@@ -66,10 +66,6 @@ class _ErrorsPanel:
def __repr__(self) -> str:
return f"{type(self).__name__}(error_count={self._total!r}, errors={tuple(self._errors_map)!r})"
- def __init_subclass__(cls, title: str | None = None) -> None:
- if title:
- cls.title = title
-
def __init__(self) -> None:
self._progress: DictProgress = DictProgress(
"[progress.description]{task.description}",
@@ -132,10 +128,13 @@ class _ErrorsPanel:
await asyncio.sleep(random.random() * 5)
-class DownloadErrorsPanel(_ErrorsPanel, title="Download"): ...
+class DownloadErrorsPanel(_ErrorsPanel):
+ title: ClassVar[str] = "Download"
+
+class ScrapeErrorsPanel(_ErrorsPanel):
+ title: ClassVar[str] = "Scrape"
-class ScrapeErrorsPanel(_ErrorsPanel, title="Scrape"):
def __init__(self) -> None:
super().__init__()
self._unsupported: int = 0
diff --git a/cyberdrop_dl/progress/scraping/panel.py b/cyberdrop_dl/progress/scraping/panel.py
index d51dc0c67..9a31a3318 100644
--- a/cyberdrop_dl/progress/scraping/panel.py
+++ b/cyberdrop_dl/progress/scraping/panel.py
@@ -57,7 +57,7 @@ class StatusMessage:
async with asyncio.TaskGroup() as tg:
for idx in range(1, 10):
- tg.create_task(show(f"test msg {idx}"))
+ _ = tg.create_task(show(f"test msg {idx}"))
@final
diff --git a/cyberdrop_dl/progress/sorting.py b/cyberdrop_dl/progress/sorting.py
index 321180f3a..100e57078 100644
--- a/cyberdrop_dl/progress/sorting.py
+++ b/cyberdrop_dl/progress/sorting.py
@@ -48,17 +48,17 @@ class SortingUI(LiveUI):
self._total: int = 0
self._tasks_map: dict[str, TaskID] = dict(self._init_tasks())
- def file_row(name: str, file: Path) -> Text:
+ def file(name: str, file: Path) -> Text:
return Text.assemble((f"{name}: ", "green"), hyperlink(file))
self._panel: Panel = Panel(
Group(
- file_row("Source", source),
- file_row("Destination", dest),
+ file("Source", source),
+ file("Destination", dest),
"",
self._progress,
"",
- Spinner("dots", "Sorting files....", style="green"),
+ Spinner("dots3", "Sorting files....", style="green"),
),
title="Sorting Downloads ",
border_style="green",