Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: move stats logic to its own module (#1775)

NTFSvolumeba2e02dae4ac
diff --git a/cyberdrop_dl/hasher.py b/cyberdrop_dl/hasher.py
index eb50e815a..8daad2cc2 100644
--- a/cyberdrop_dl/hasher.py
+++ b/cyberdrop_dl/hasher.py
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Final, Literal
import xxhash
-from cyberdrop_dl import aio
+from cyberdrop_dl import aio, stats
from cyberdrop_dl.constants import Hashing, TempExt
from cyberdrop_dl.progress.hashing import HashingStats, HashingUI
@@ -47,9 +47,9 @@ def _compute_hash(file: Path, algorithm: Literal["xxh128", "md5", "sha256"]) ->
async def hash_directory_scanner(manager: Manager, path: Path) -> None:
async with manager.database:
- stats = await hash_directory(manager, path)
+ hash_stats = await hash_directory(manager, path)
- manager.print_hashing_stats(stats)
+ stats.print(hash_stats)
async def hash_directory(manager: Manager, path: Path) -> HashingStats:
diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
index c155cbeba..11bd42843 100644
--- a/cyberdrop_dl/managers/manager.py
+++ b/cyberdrop_dl/managers/manager.py
@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Any, Self
from pydantic.types import ByteSize
-from cyberdrop_dl import __version__, env, ffmpeg, yaml
+from cyberdrop_dl import __version__, env, ffmpeg, stats, yaml
from cyberdrop_dl.cli import CLIargs
from cyberdrop_dl.config import Config
from cyberdrop_dl.database import Database
@@ -26,13 +26,9 @@ from cyberdrop_dl.sorter import Sorter
from cyberdrop_dl.utils import get_system_information
if TYPE_CHECKING:
- from collections.abc import Generator, Sequence
+ from collections.abc import Generator
from os import PathLike
- from cyberdrop_dl.progress.dedupe import DedupeStats
- from cyberdrop_dl.progress.hashing import HashingStats
- from cyberdrop_dl.progress.scraping.errors import UIError
- from cyberdrop_dl.progress.sorting import SortStats
from cyberdrop_dl.scrape_mapper import ScrapeMapper, ScrapeStats
from cyberdrop_dl.url_objects import MediaItem
@@ -169,94 +165,40 @@ class Manager:
return stream.getvalue()
- def __print_stats(self, stats: ScrapeStats) -> None:
+ def __print_stats(self, scrape_stats: ScrapeStats) -> None:
- elapsed = timedelta(seconds=int(time.monotonic() - stats.start_time))
+ elapsed = timedelta(seconds=int(time.monotonic() - scrape_stats.start_time))
total_data_written = ByteSize(self.scrape_mapper.tui.downloads.bytes_downloaded).human_readable(decimal=True)
logger.info("Run Stats:", extra={"color": "cyan"})
logger.info(f" Config file: {self.config.source}")
- logger.info(f" URLs source: {stats.source}")
- logger.info(f" URLs: {stats.count:,}")
- logger.info(f" URL groups: {len(stats.unique_groups):,}")
+ logger.info(f" URLs source: {scrape_stats.source}")
+ logger.info(f" URLs: {scrape_stats.count:,}")
+ logger.info(f" URL groups: {len(scrape_stats.unique_groups):,}")
logger.info(f" Logs folder: {self.config.settings.logs.log_folder}")
logger.info(f" Total runtime: {elapsed}")
logger.info(f" Total downloaded data: {total_data_written}")
- if stats.domain_stats:
+ if scrape_stats.domain_stats:
log_spacer()
logger.info("URLs by domain (includes children):", extra={"color": "cyan"})
def lines():
- for domain, count in sorted(stats.domain_stats.items()):
+ for domain, count in sorted(scrape_stats.domain_stats.items()):
yield f" - {domain}: {count:,}"
logger.info("\n".join(lines()))
- log_spacer()
- logger.info("Download Stats:", extra={"color": "cyan"})
- logger.info(f" Downloaded: {self.scrape_mapper.tui.files.stats.completed:,} files")
- logger.info(f" Skipped (by config): {self.scrape_mapper.tui.files.stats.skipped:,} files")
- logger.info(
- f" Skipped (previously downloaded): {self.scrape_mapper.tui.files.stats.previously_completed:,} files"
- )
- logger.info(f" Failed: {self.scrape_mapper.tui.files.stats.failed:,} files")
-
- log_spacer()
- logger.info("Unsupported URLs Stats:", extra={"color": "cyan"})
- logger.info(f" Sent to Jdownloader: {self.scrape_mapper.tui.scrape_errors.sent_to_jdownloader:,}")
- logger.info(f" Skipped: {self.scrape_mapper.tui.scrape_errors.skipped:,}")
-
- self.print_hashing_stats(self.hasher.stats)
- self.print_dedupe_stats(self.deduper.stats)
- self.print_sort_stats(self.sorter.stats)
- _log_errors(
+ stats.print(self.scrape_mapper.tui.files.stats)
+ stats.print(self.scrape_mapper.tui.scrape_errors)
+ stats.print(self.hasher.stats)
+ stats.print(self.deduper.stats)
+ stats.print(self.sorter.stats)
+ stats.print_errors(
tuple(self.scrape_mapper.tui.scrape_errors),
tuple(self.scrape_mapper.tui.download_errors),
)
- def print_sort_stats(self, stats: SortStats):
- log_spacer()
- logger.info("Sort Stats:", extra={"color": "cyan"})
- logger.info(f" Audios: {stats.audios:,}")
- logger.info(f" Images: {stats.images:,}")
- logger.info(f" Videos: {stats.videos:,}")
- logger.info(f" Other files: {stats.others:,}")
-
- def print_hashing_stats(self, stats: HashingStats) -> None:
- log_spacer()
- logger.info("Checksum Stats:", extra={"color": "cyan"})
- logger.info(f" Newly hashed: {stats.new_hashed:,} files")
- logger.info(f" Previously hashed: {stats.prev_hashed:,} files")
-
- def print_dedupe_stats(self, stats: DedupeStats) -> None:
- log_spacer()
- logger.info("Dedupe Stats:", extra={"color": "cyan"})
- logger.info(f" Deleted (duplicates of previous downloads): {stats.deleted:,} files")
- logger.info(f" Errors: {stats.total - stats.deleted:,} files")
-
-
-def _log_errors(scrape_errors: Sequence[UIError], download_errors: Sequence[UIError]) -> None:
- error_codes = (error.code for error in (*scrape_errors, *download_errors) if error.code is not None)
-
- try:
- padding = len(str(max(error_codes)))
- except ValueError:
- padding = 0
-
- for title, errors in (
- ("Scrape Errors:", scrape_errors),
- ("Download Errors:", download_errors),
- ):
- log_spacer()
- logger.info(title, extra={"color": "cyan"})
- if not errors:
- logger.info(f" {'None':>{padding}}", extra={"color": "green"})
- continue
-
- for error in errors:
- logger.info(f" {error.format(padding)}", extra={"color": "red"})
-
@contextlib.contextmanager
def _cache_context(cache_file: Path, cache: dict[str, Any]) -> Generator[None]:
diff --git a/cyberdrop_dl/stats.py b/cyberdrop_dl/stats.py
new file mode 100644
index 000000000..5962a0eb4
--- /dev/null
+++ b/cyberdrop_dl/stats.py
@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+import functools
+import logging
+from typing import TYPE_CHECKING
+
+from cyberdrop_dl.logs import log_spacer
+from cyberdrop_dl.progress.dedupe import DedupeStats
+from cyberdrop_dl.progress.hashing import HashingStats
+from cyberdrop_dl.progress.scraping.errors import ScrapeErrorsPanel
+from cyberdrop_dl.progress.scraping.files import FileStats
+from cyberdrop_dl.progress.sorting import SortStats
+
+if TYPE_CHECKING:
+ from collections.abc import Sequence
+
+ from cyberdrop_dl.progress.scraping.errors import UIError
+
+
+logger = logging.getLogger(__name__)
+
+
+def print(stats: object) -> None:
+ raise NotImplementedError(f"Unable to print stats for {stats!r}")
+
+
+def _(stats: SortStats) -> None:
+ log_spacer()
+ logger.info("Sort Stats:", extra={"color": "cyan"})
+ logger.info(f" Audios: {stats.audios:,}")
+ logger.info(f" Images: {stats.images:,}")
+ logger.info(f" Videos: {stats.videos:,}")
+ logger.info(f" Other files: {stats.others:,}")
+
+
+def _(stats: HashingStats) -> None:
+ log_spacer()
+ logger.info("Checksum Stats:", extra={"color": "cyan"})
+ logger.info(f" Newly hashed: {stats.new_hashed:,} files")
+ logger.info(f" Previously hashed: {stats.prev_hashed:,} files")
+
+
+def _(stats: DedupeStats) -> None:
+ log_spacer()
+ logger.info("Dedupe Stats:", extra={"color": "cyan"})
+ logger.info(f" Deleted (duplicates of previous downloads): {stats.deleted:,} files")
+ logger.info(f" Errors: {stats.total - stats.deleted:,} files")
+
+
+def _(stats: FileStats) -> None:
+ log_spacer()
+ logger.info("Download Stats:", extra={"color": "cyan"})
+ logger.info(f" Downloaded: {stats.completed:,} files")
+ logger.info(f" Skipped (by config): {stats.skipped:,} files")
+ logger.info(f" Skipped (previously downloaded): {stats.previously_completed:,} files")
+ logger.info(f" Failed: {stats.failed:,} files")
+
+
+def _(stats: ScrapeErrorsPanel) -> None:
+ log_spacer()
+ logger.info("Unsupported URLs Stats:", extra={"color": "cyan"})
+ logger.info(f" Sent to Jdownloader: {stats.sent_to_jdownloader:,}")
+ logger.info(f" Skipped: {stats.skipped:,}")
+
+
+def print_errors(scrape_errors: Sequence[UIError], download_errors: Sequence[UIError]) -> None:
+ error_codes = (error.code for error in (*scrape_errors, *download_errors) if error.code is not None)
+
+ try:
+ padding = len(str(max(error_codes)))
+ except ValueError:
+ padding = 0
+
+ for title, errors in (
+ ("Scrape Errors:", scrape_errors),
+ ("Download Errors:", download_errors),
+ ):
+ log_spacer()
+ logger.info(title, extra={"color": "cyan"})
+ if not errors:
+ logger.info(f" {'None':>{padding}}", extra={"color": "green"})
+ continue
+
+ for error in errors:
+ logger.info(f" {error.format(padding)}", extra={"color": "red"})
diff --git a/tests/test_progress.py b/tests/test_progress.py
index c6e1f9b56..2043cbf1b 100644
--- a/tests/test_progress.py
+++ b/tests/test_progress.py
@@ -1,6 +1,5 @@
import pytest
-from cyberdrop_dl.managers.manager import _log_errors
from cyberdrop_dl.progress.scraping.errors import UIError
@@ -34,64 +33,3 @@ def test_ui_error_parsing(name: str, expected_msg: str, expected_code: int | Non
def test_ui_errors_formatting(msg: str, code: int | None, padding: int, expected: str):
error = UIError(msg, 0, code)
assert error.format(padding) == expected + ": 0"
-
-
- "scrape_errors, download_errors, expected_msgs",
- [
- (
- ["Client Connector SSL Error", "502 Bad Gateway"],
- ["1234 Bad Gateway"],
- [
- "------------------------------",
- "Scrape Errors:",
- " Client Connector SSL Error: 0",
- " 502 Bad Gateway: 0",
- "------------------------------",
- "Download Errors:",
- " 1234 Bad Gateway: 0",
- ],
- ),
- (
- ["Error1", "Error2"],
- ["Error3", "Error4"],
- [
- "------------------------------",
- "Scrape Errors:",
- " Error1: 0",
- " Error2: 0",
- "------------------------------",
- "Download Errors:",
- " Error3: 0",
- " Error4: 0",
- ],
- ),
- (
- ["Error1", "Error2"],
- ["Error3", "2 Error4", "Error5"],
- [
- "------------------------------",
- "Scrape Errors:",
- " Error1: 0",
- " Error2: 0",
- "------------------------------",
- "Download Errors:",
- " Error3: 0",
- " 2 Error4: 0",
- " Error5: 0",
- ],
- ),
- ],
-)
-def test_stats_formating(
- logs: pytest.LogCaptureFixture,
- scrape_errors: tuple[str, ...],
- download_errors: tuple[str, ...],
- expected_msgs: list[str],
-) -> None:
-
- _log_errors(
- tuple(UIError.parse(msg, count=0) for msg in scrape_errors),
- tuple(UIError.parse(msg, count=0) for msg in download_errors),
- )
- assert logs.messages == expected_msgs
diff --git a/tests/test_stats.py b/tests/test_stats.py
new file mode 100644
index 000000000..31f3facb9
--- /dev/null
+++ b/tests/test_stats.py
@@ -0,0 +1,90 @@
+import pytest
+
+from cyberdrop_dl import stats
+from cyberdrop_dl.progress.dedupe import DedupeStats
+from cyberdrop_dl.progress.hashing import HashingStats
+from cyberdrop_dl.progress.scraping.errors import ScrapeErrorsPanel, UIError
+from cyberdrop_dl.progress.scraping.files import FileStats
+from cyberdrop_dl.progress.sorting import SortStats
+
+
+ "scrape_errors, download_errors, expected_msgs",
+ [
+ (
+ ["Client Connector SSL Error", "502 Bad Gateway"],
+ ["1234 Bad Gateway"],
+ [
+ "------------------------------",
+ "Scrape Errors:",
+ " Client Connector SSL Error: 0",
+ " 502 Bad Gateway: 0",
+ "------------------------------",
+ "Download Errors:",
+ " 1234 Bad Gateway: 0",
+ ],
+ ),
+ (
+ ["Error1", "Error2"],
+ ["Error3", "Error4"],
+ [
+ "------------------------------",
+ "Scrape Errors:",
+ " Error1: 0",
+ " Error2: 0",
+ "------------------------------",
+ "Download Errors:",
+ " Error3: 0",
+ " Error4: 0",
+ ],
+ ),
+ (
+ ["Error1", "Error2"],
+ ["Error3", "2 Error4", "Error5"],
+ [
+ "------------------------------",
+ "Scrape Errors:",
+ " Error1: 0",
+ " Error2: 0",
+ "------------------------------",
+ "Download Errors:",
+ " Error3: 0",
+ " 2 Error4: 0",
+ " Error5: 0",
+ ],
+ ),
+ ],
+)
+def test_stats_formating(
+ logs: pytest.LogCaptureFixture,
+ scrape_errors: tuple[str, ...],
+ download_errors: tuple[str, ...],
+ expected_msgs: list[str],
+) -> None:
+
+ stats.print_errors(
+ tuple(UIError.parse(msg, count=0) for msg in scrape_errors),
+ tuple(UIError.parse(msg, count=0) for msg in download_errors),
+ )
+ assert logs.messages == expected_msgs
+
+
+def test_stats_of_unknown_obj_raise_not_implemented_error() -> None:
+ with pytest.raises(NotImplementedError):
+ stats.print(40)
+
+
+ "stats_cls, msg",
+ [
+ (DedupeStats, "Dedupe Stats"),
+ (HashingStats, "Checksum Stats"),
+ (ScrapeErrorsPanel, "Unsupported URLs Stats"),
+ (FileStats, "Download Stats"),
+ (SortStats, "Sort Stats"),
+ ],
+)
+def test_stats_prints_the_expected_stats(stats_cls: type, msg: str, logs: pytest.LogCaptureFixture) -> None:
+ stats.print(stats_cls())
+ assert logs.messages[0].startswith("-" * 10)
+ assert logs.messages[1].startswith(msg)