Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: move stats logic to its own module (#1775)
NTFSvolumeba2e02dae4ac
diff --git a/cyberdrop_dl/hasher.py b/cyberdrop_dl/hasher.pyindex eb50e815a..8daad2cc2 100644--- a/cyberdrop_dl/hasher.py+++ b/cyberdrop_dl/hasher.py@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Final, Literalimport xxhash-from cyberdrop_dl import aio+from cyberdrop_dl import aio, statsfrom cyberdrop_dl.constants import Hashing, TempExtfrom cyberdrop_dl.progress.hashing import HashingStats, HashingUI@@ -47,9 +47,9 @@ def _compute_hash(file: Path, algorithm: Literal["xxh128", "md5", "sha256"]) ->async def hash_directory_scanner(manager: Manager, path: Path) -> None:async with manager.database:- stats = await hash_directory(manager, path)+ hash_stats = await hash_directory(manager, path)- manager.print_hashing_stats(stats)+ stats.print(hash_stats)async def hash_directory(manager: Manager, path: Path) -> HashingStats:diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex c155cbeba..11bd42843 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Any, Selffrom pydantic.types import ByteSize-from cyberdrop_dl import __version__, env, ffmpeg, yaml+from cyberdrop_dl import __version__, env, ffmpeg, stats, yamlfrom cyberdrop_dl.cli import CLIargsfrom cyberdrop_dl.config import Configfrom cyberdrop_dl.database import Database@@ -26,13 +26,9 @@ from cyberdrop_dl.sorter import Sorterfrom cyberdrop_dl.utils import get_system_informationif TYPE_CHECKING:- from collections.abc import Generator, Sequence+ from collections.abc import Generatorfrom os import PathLike- from cyberdrop_dl.progress.dedupe import DedupeStats- from cyberdrop_dl.progress.hashing import HashingStats- from cyberdrop_dl.progress.scraping.errors import UIError- from cyberdrop_dl.progress.sorting import SortStatsfrom cyberdrop_dl.scrape_mapper import ScrapeMapper, ScrapeStatsfrom cyberdrop_dl.url_objects import MediaItem@@ -169,94 +165,40 @@ class Manager:return stream.getvalue()- def __print_stats(self, stats: ScrapeStats) -> None:+ def __print_stats(self, scrape_stats: ScrapeStats) -> None:- elapsed = timedelta(seconds=int(time.monotonic() - stats.start_time))+ elapsed = timedelta(seconds=int(time.monotonic() - scrape_stats.start_time))total_data_written = ByteSize(self.scrape_mapper.tui.downloads.bytes_downloaded).human_readable(decimal=True)logger.info("Run Stats:", extra={"color": "cyan"})logger.info(f" Config file: {self.config.source}")- logger.info(f" URLs source: {stats.source}")- logger.info(f" URLs: {stats.count:,}")- logger.info(f" URL groups: {len(stats.unique_groups):,}")+ logger.info(f" URLs source: {scrape_stats.source}")+ logger.info(f" URLs: {scrape_stats.count:,}")+ logger.info(f" URL groups: {len(scrape_stats.unique_groups):,}")logger.info(f" Logs folder: {self.config.settings.logs.log_folder}")logger.info(f" Total runtime: {elapsed}")logger.info(f" Total downloaded data: {total_data_written}")- if stats.domain_stats:+ if scrape_stats.domain_stats:log_spacer()logger.info("URLs by domain (includes children):", extra={"color": "cyan"})def lines():- for domain, count in sorted(stats.domain_stats.items()):+ for domain, count in sorted(scrape_stats.domain_stats.items()):yield f" - {domain}: {count:,}"logger.info("\n".join(lines()))- log_spacer()- logger.info("Download Stats:", extra={"color": "cyan"})- logger.info(f" Downloaded: {self.scrape_mapper.tui.files.stats.completed:,} files")- logger.info(f" Skipped (by config): {self.scrape_mapper.tui.files.stats.skipped:,} files")- logger.info(- f" Skipped (previously downloaded): {self.scrape_mapper.tui.files.stats.previously_completed:,} files"- )- logger.info(f" Failed: {self.scrape_mapper.tui.files.stats.failed:,} files")-- log_spacer()- logger.info("Unsupported URLs Stats:", extra={"color": "cyan"})- logger.info(f" Sent to Jdownloader: {self.scrape_mapper.tui.scrape_errors.sent_to_jdownloader:,}")- logger.info(f" Skipped: {self.scrape_mapper.tui.scrape_errors.skipped:,}")-- self.print_hashing_stats(self.hasher.stats)- self.print_dedupe_stats(self.deduper.stats)- self.print_sort_stats(self.sorter.stats)- _log_errors(+ stats.print(self.scrape_mapper.tui.files.stats)+ stats.print(self.scrape_mapper.tui.scrape_errors)+ stats.print(self.hasher.stats)+ stats.print(self.deduper.stats)+ stats.print(self.sorter.stats)+ stats.print_errors(tuple(self.scrape_mapper.tui.scrape_errors),tuple(self.scrape_mapper.tui.download_errors),)- def print_sort_stats(self, stats: SortStats):- log_spacer()- logger.info("Sort Stats:", extra={"color": "cyan"})- logger.info(f" Audios: {stats.audios:,}")- logger.info(f" Images: {stats.images:,}")- logger.info(f" Videos: {stats.videos:,}")- logger.info(f" Other files: {stats.others:,}")-- def print_hashing_stats(self, stats: HashingStats) -> None:- log_spacer()- logger.info("Checksum Stats:", extra={"color": "cyan"})- logger.info(f" Newly hashed: {stats.new_hashed:,} files")- logger.info(f" Previously hashed: {stats.prev_hashed:,} files")-- def print_dedupe_stats(self, stats: DedupeStats) -> None:- log_spacer()- logger.info("Dedupe Stats:", extra={"color": "cyan"})- logger.info(f" Deleted (duplicates of previous downloads): {stats.deleted:,} files")- logger.info(f" Errors: {stats.total - stats.deleted:,} files")---def _log_errors(scrape_errors: Sequence[UIError], download_errors: Sequence[UIError]) -> None:- error_codes = (error.code for error in (*scrape_errors, *download_errors) if error.code is not None)-- try:- padding = len(str(max(error_codes)))- except ValueError:- padding = 0-- for title, errors in (- ("Scrape Errors:", scrape_errors),- ("Download Errors:", download_errors),- ):- log_spacer()- logger.info(title, extra={"color": "cyan"})- if not errors:- logger.info(f" {'None':>{padding}}", extra={"color": "green"})- continue-- for error in errors:- logger.info(f" {error.format(padding)}", extra={"color": "red"})-@contextlib.contextmanagerdef _cache_context(cache_file: Path, cache: dict[str, Any]) -> Generator[None]:diff --git a/cyberdrop_dl/stats.py b/cyberdrop_dl/stats.pynew file mode 100644index 000000000..5962a0eb4--- /dev/null+++ b/cyberdrop_dl/stats.py@@ -0,0 +1,91 @@+from __future__ import annotations++import functools+import logging+from typing import TYPE_CHECKING++from cyberdrop_dl.logs import log_spacer+from cyberdrop_dl.progress.dedupe import DedupeStats+from cyberdrop_dl.progress.hashing import HashingStats+from cyberdrop_dl.progress.scraping.errors import ScrapeErrorsPanel+from cyberdrop_dl.progress.scraping.files import FileStats+from cyberdrop_dl.progress.sorting import SortStats++if TYPE_CHECKING:+ from collections.abc import Sequence++ from cyberdrop_dl.progress.scraping.errors import UIError+++logger = logging.getLogger(__name__)+++def print(stats: object) -> None:+ raise NotImplementedError(f"Unable to print stats for {stats!r}")+++def _(stats: SortStats) -> None:+ log_spacer()+ logger.info("Sort Stats:", extra={"color": "cyan"})+ logger.info(f" Audios: {stats.audios:,}")+ logger.info(f" Images: {stats.images:,}")+ logger.info(f" Videos: {stats.videos:,}")+ logger.info(f" Other files: {stats.others:,}")+++def _(stats: HashingStats) -> None:+ log_spacer()+ logger.info("Checksum Stats:", extra={"color": "cyan"})+ logger.info(f" Newly hashed: {stats.new_hashed:,} files")+ logger.info(f" Previously hashed: {stats.prev_hashed:,} files")+++def _(stats: DedupeStats) -> None:+ log_spacer()+ logger.info("Dedupe Stats:", extra={"color": "cyan"})+ logger.info(f" Deleted (duplicates of previous downloads): {stats.deleted:,} files")+ logger.info(f" Errors: {stats.total - stats.deleted:,} files")+++def _(stats: FileStats) -> None:+ log_spacer()+ logger.info("Download Stats:", extra={"color": "cyan"})+ logger.info(f" Downloaded: {stats.completed:,} files")+ logger.info(f" Skipped (by config): {stats.skipped:,} files")+ logger.info(f" Skipped (previously downloaded): {stats.previously_completed:,} files")+ logger.info(f" Failed: {stats.failed:,} files")+++def _(stats: ScrapeErrorsPanel) -> None:+ log_spacer()+ logger.info("Unsupported URLs Stats:", extra={"color": "cyan"})+ logger.info(f" Sent to Jdownloader: {stats.sent_to_jdownloader:,}")+ logger.info(f" Skipped: {stats.skipped:,}")+++def print_errors(scrape_errors: Sequence[UIError], download_errors: Sequence[UIError]) -> None:+ error_codes = (error.code for error in (*scrape_errors, *download_errors) if error.code is not None)++ try:+ padding = len(str(max(error_codes)))+ except ValueError:+ padding = 0++ for title, errors in (+ ("Scrape Errors:", scrape_errors),+ ("Download Errors:", download_errors),+ ):+ log_spacer()+ logger.info(title, extra={"color": "cyan"})+ if not errors:+ logger.info(f" {'None':>{padding}}", extra={"color": "green"})+ continue++ for error in errors:+ logger.info(f" {error.format(padding)}", extra={"color": "red"})diff --git a/tests/test_progress.py b/tests/test_progress.pyindex c6e1f9b56..2043cbf1b 100644--- a/tests/test_progress.py+++ b/tests/test_progress.py@@ -1,6 +1,5 @@import pytest-from cyberdrop_dl.managers.manager import _log_errorsfrom cyberdrop_dl.progress.scraping.errors import UIError@@ -34,64 +33,3 @@ def test_ui_error_parsing(name: str, expected_msg: str, expected_code: int | Nondef test_ui_errors_formatting(msg: str, code: int | None, padding: int, expected: str):error = UIError(msg, 0, code)assert error.format(padding) == expected + ": 0"--- "scrape_errors, download_errors, expected_msgs",- [- (- ["Client Connector SSL Error", "502 Bad Gateway"],- ["1234 Bad Gateway"],- [- "------------------------------",- "Scrape Errors:",- " Client Connector SSL Error: 0",- " 502 Bad Gateway: 0",- "------------------------------",- "Download Errors:",- " 1234 Bad Gateway: 0",- ],- ),- (- ["Error1", "Error2"],- ["Error3", "Error4"],- [- "------------------------------",- "Scrape Errors:",- " Error1: 0",- " Error2: 0",- "------------------------------",- "Download Errors:",- " Error3: 0",- " Error4: 0",- ],- ),- (- ["Error1", "Error2"],- ["Error3", "2 Error4", "Error5"],- [- "------------------------------",- "Scrape Errors:",- " Error1: 0",- " Error2: 0",- "------------------------------",- "Download Errors:",- " Error3: 0",- " 2 Error4: 0",- " Error5: 0",- ],- ),- ],-)-def test_stats_formating(- logs: pytest.LogCaptureFixture,- scrape_errors: tuple[str, ...],- download_errors: tuple[str, ...],- expected_msgs: list[str],-) -> None:-- _log_errors(- tuple(UIError.parse(msg, count=0) for msg in scrape_errors),- tuple(UIError.parse(msg, count=0) for msg in download_errors),- )- assert logs.messages == expected_msgsdiff --git a/tests/test_stats.py b/tests/test_stats.pynew file mode 100644index 000000000..31f3facb9--- /dev/null+++ b/tests/test_stats.py@@ -0,0 +1,90 @@+import pytest++from cyberdrop_dl import stats+from cyberdrop_dl.progress.dedupe import DedupeStats+from cyberdrop_dl.progress.hashing import HashingStats+from cyberdrop_dl.progress.scraping.errors import ScrapeErrorsPanel, UIError+from cyberdrop_dl.progress.scraping.files import FileStats+from cyberdrop_dl.progress.sorting import SortStats+++ "scrape_errors, download_errors, expected_msgs",+ [+ (+ ["Client Connector SSL Error", "502 Bad Gateway"],+ ["1234 Bad Gateway"],+ [+ "------------------------------",+ "Scrape Errors:",+ " Client Connector SSL Error: 0",+ " 502 Bad Gateway: 0",+ "------------------------------",+ "Download Errors:",+ " 1234 Bad Gateway: 0",+ ],+ ),+ (+ ["Error1", "Error2"],+ ["Error3", "Error4"],+ [+ "------------------------------",+ "Scrape Errors:",+ " Error1: 0",+ " Error2: 0",+ "------------------------------",+ "Download Errors:",+ " Error3: 0",+ " Error4: 0",+ ],+ ),+ (+ ["Error1", "Error2"],+ ["Error3", "2 Error4", "Error5"],+ [+ "------------------------------",+ "Scrape Errors:",+ " Error1: 0",+ " Error2: 0",+ "------------------------------",+ "Download Errors:",+ " Error3: 0",+ " 2 Error4: 0",+ " Error5: 0",+ ],+ ),+ ],+)+def test_stats_formating(+ logs: pytest.LogCaptureFixture,+ scrape_errors: tuple[str, ...],+ download_errors: tuple[str, ...],+ expected_msgs: list[str],+) -> None:++ stats.print_errors(+ tuple(UIError.parse(msg, count=0) for msg in scrape_errors),+ tuple(UIError.parse(msg, count=0) for msg in download_errors),+ )+ assert logs.messages == expected_msgs+++def test_stats_of_unknown_obj_raise_not_implemented_error() -> None:+ with pytest.raises(NotImplementedError):+ stats.print(40)+++ "stats_cls, msg",+ [+ (DedupeStats, "Dedupe Stats"),+ (HashingStats, "Checksum Stats"),+ (ScrapeErrorsPanel, "Unsupported URLs Stats"),+ (FileStats, "Download Stats"),+ (SortStats, "Sort Stats"),+ ],+)+def test_stats_prints_the_expected_stats(stats_cls: type, msg: str, logs: pytest.LogCaptureFixture) -> None:+ stats.print(stats_cls())+ assert logs.messages[0].startswith("-" * 10)+ assert logs.messages[1].startswith(msg)