Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: use new hashing UI (#147)

NTFSvolume6b33182ab065
diff --git a/cyberdrop_dl/hasher.py b/cyberdrop_dl/hasher.py
index 844d9b9a0..c8ca560b2 100644
--- a/cyberdrop_dl/hasher.py
+++ b/cyberdrop_dl/hasher.py
@@ -13,6 +13,8 @@ from send2trash import send2trash
from cyberdrop_dl import aio
from cyberdrop_dl.constants import Hashing, TempExt
+from cyberdrop_dl.progress.dedupe import DedupeUI
+from cyberdrop_dl.progress.hashing import HashingStats, HashingUI
if TYPE_CHECKING:
from yarl import URL
@@ -47,9 +49,9 @@ def _compute_hash(file: Path, algorithm: Literal["xxh128", "md5", "sha256"]) ->
async def hash_directory_scanner(manager: Manager, path: Path) -> None:
manager.async_db_hash_startup()
async with manager.database:
- await manager.hasher.hash_directory(path)
- manager.progress_manager.print_dedupe_stats()
- manager.progress_manager.hash_progress.reset()
+ stats = await manager.hasher.hash_directory(path)
+
+ manager.progress_manager.print_hashing_stats(stats)
@dataclasses.dataclass(slots=True)
@@ -62,6 +64,17 @@ class Hasher:
_sem: asyncio.BoundedSemaphore = dataclasses.field(init=False, default_factory=lambda: asyncio.BoundedSemaphore(20))
_cwd: Path = dataclasses.field(init=False, default_factory=Path.cwd)
_hashed_items: set[tuple[str, ...]] = dataclasses.field(default_factory=set)
+ _tui: HashingUI = dataclasses.field(init=False)
+ _dedupe_tui: DedupeUI = dataclasses.field(init=False)
+
+ def __post_init__(self) -> None:
+ base_dir = self.manager.config.settings.files.download_folder
+ self._tui = HashingUI(base_dir)
+ self._dedupe_tui = DedupeUI(base_dir)
+
+ @property
+ def stats(self):
+ return self._tui.stats, self._dedupe_tui.stats
@property
def config(self) -> DupeCleanup:
@@ -79,17 +92,22 @@ class Hasher:
def _deleted_file_suffix(self) -> Literal["Sent to trash", "Permanently deleted"]:
return "Sent to trash" if self._to_trash else "Permanently deleted"
- async def hash_directory(self, path: Path) -> None:
+ async def hash_directory(self, path: Path) -> HashingStats:
path = Path(path)
- with (
- self.manager.live_manager.get_hash_live(stop=True),
- self.manager.progress_manager.hash_progress.currently_hashing_dir(path),
- ):
+ tui = HashingUI(path)
+ old_tui = self._tui
+ with tui():
if not await aio.is_dir(path):
raise NotADirectoryError(None, path)
- async for file in aio.rglob(path, "*"):
- _ = await self.update_db_and_retrive_hash(file)
+ try:
+ self._tui = tui
+ async for file in aio.rglob(path, "*"):
+ _ = await self.update_db_and_retrive_hash(file)
+ finally:
+ self._tui = old_tui
+
+ return tui.stats
async def hash_item(self, media_item: MediaItem) -> None:
if media_item.is_segment:
@@ -127,14 +145,15 @@ class Hasher:
if not await aio.get_size(file):
return
- logger.info("Computing hashes of '%s'", file)
- hash = await self._update_db_and_retrive_hash(file, original_filename, referer, hash_type="xxh128")
- if self.config.add_md5_hash:
- await self._update_db_and_retrive_hash(file, original_filename, referer, hash_type="md5")
- if self.config.add_sha256_hash:
- await self._update_db_and_retrive_hash(file, original_filename, referer, hash_type="sha256")
+ with self._tui.new_file(file):
+ logger.info("Computing hashes of '%s'", file)
+ hash = await self._update_db_and_retrive_hash(file, original_filename, referer, hash_type="xxh128")
+ if self.config.add_md5_hash:
+ await self._update_db_and_retrive_hash(file, original_filename, referer, hash_type="md5")
+ if self.config.add_sha256_hash:
+ await self._update_db_and_retrive_hash(file, original_filename, referer, hash_type="sha256")
- return hash
+ return hash
async def _update_db_and_retrive_hash(
self,
@@ -144,7 +163,7 @@ class Hasher:
hash_type: Literal["xxh128", "md5", "sha256"],
) -> str | None:
"""Generates hash of a file."""
- self.manager.progress_manager.hash_progress.update_currently_hashing(file)
+
hash = await self.manager.database.hash.get_file_hash_exists(file, hash_type)
try:
if not hash:
@@ -156,9 +175,9 @@ class Hasher:
original_filename,
referer,
)
- self.manager.progress_manager.hash_progress.add_new_completed_hash(hash_type)
+ self._tui.add_completed(hash_type)
else:
- self.manager.progress_manager.hash_progress.add_prev_hash()
+ self._tui.stats.prev_hashed += 1
await self.manager.database.hash.insert_or_update_hash_db(
hash,
hash_type,
@@ -191,9 +210,9 @@ class Hasher:
return
if self.manager.config.settings.runtime_options.ignore_history:
return
- with self.manager.live_manager.get_hash_live(stop=True):
+ with self._tui():
file_hashes_dict = await self.get_file_hashes_dict()
- with self.manager.live_manager.get_remove_file_via_hash_live(stop=True):
+ with self._dedupe_tui():
await self.final_dupe_cleanup(file_hashes_dict)
async def final_dupe_cleanup(self, final_dict: dict[str, dict[int, set[Path]]]) -> None:
@@ -215,24 +234,25 @@ class Hasher:
async def _delete_and_log(self, file: Path, xxh128_value: str) -> None:
hash_string = f"xxh128:{xxh128_value}"
- try:
- deleted = await _delete_file(file, self._to_trash)
- except OSError as e:
- logger.exception(f"Unable to remove '{file}' ({hash_string}): {e}")
+ with self._dedupe_tui.new_file(file):
+ try:
+ deleted = await _delete_file(file, self._to_trash)
+ except OSError as e:
+ logger.exception(f"Unable to remove '{file}' ({hash_string}): {e}")
- else:
- if not deleted:
- return
+ else:
+ if not deleted:
+ return
- msg = (
- f"Removed new download '{file}' [{self._deleted_file_suffix}]. "
- f"File hash matches with a previous download ({hash_string})"
- )
- logger.info(msg)
- self.manager.progress_manager.hash_progress.add_removed_file()
+ msg = (
+ f"Removed new download '{file}' [{self._deleted_file_suffix}]. "
+ f"File hash matches with a previous download ({hash_string})"
+ )
+ logger.info(msg)
+ self._dedupe_tui.stats.deleted += 1
- finally:
- self._sem.release()
+ finally:
+ self._sem.release()
async def get_file_hashes_dict(self) -> dict[str, dict[int, set[Path]]]:
diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.py
index 152865f03..afe492228 100644
--- a/cyberdrop_dl/managers/progress_manager.py
+++ b/cyberdrop_dl/managers/progress_manager.py
@@ -27,6 +27,8 @@ if TYPE_CHECKING:
from rich.panel import Panel
from cyberdrop_dl.managers.manager import Manager
+ from cyberdrop_dl.progress.dedupe import DedupeStats
+ from cyberdrop_dl.progress.hashing import HashingStats
from cyberdrop_dl.scrape_mapper import ScrapeStats
from cyberdrop_dl.ui.progress.statistic_progress import UiFailureTotal
@@ -154,7 +156,9 @@ class ProgressManager:
logger.info(f" Sent to Jdownloader: {self.scrape_stats_progress.sent_to_jdownloader:,}")
logger.info(f" Skipped: {self.scrape_stats_progress.unsupported_urls_skipped:,}")
- self.print_dedupe_stats()
+ hash_stats, dedupe_stats = self.manager.hasher.stats
+ self.print_hashing_stats(hash_stats)
+ self.print_dedupe_stats(dedupe_stats)
log_spacer()
logger.info("Sort Stats:", extra={"color": "cyan"})
@@ -163,14 +167,23 @@ class ProgressManager:
logger.info(f" Videos: {self.sort_progress.video_count:,}")
logger.info(f" Other Files: {self.sort_progress.other_count:,}")
+ self.print_errors()
+
+ def print_errors(self):
+
_log_errors(self.scrape_stats_progress.return_totals(), self.download_stats_progress.return_totals())
- def print_dedupe_stats(self) -> None:
+ def print_hashing_stats(self, stats: HashingStats) -> None:
+ log_spacer()
+ logger.info("Checksum Stats:", extra={"color": "cyan"})
+ logger.info(f" Newly hashed: {stats.new_hashed:,} files")
+ logger.info(f" Previously hashed: {stats.prev_hashed:,} files")
+
+ def print_dedupe_stats(self, stats: DedupeStats) -> None:
log_spacer()
logger.info("Dupe Stats:", extra={"color": "cyan"})
- logger.info(f" Newly hashed: {self.hash_progress.hashed_files:,} files")
- logger.info(f" Previously hashed: {self.hash_progress.prev_hashed_files:,} files")
- logger.info(f" Deleted (duplicates of previous downloads): {self.hash_progress.removed_files:,} files")
+ logger.info(f" Deleted (duplicates of previous downloads): {stats.deleted:,} files")
+ logger.info(f" Errors: {stats.total - stats.deleted:,} files")
def _log_errors(scrape_errors: Sequence[UiFailureTotal], download_errors: Sequence[UiFailureTotal]) -> None:
diff --git a/cyberdrop_dl/progress/dedupe.py b/cyberdrop_dl/progress/dedupe.py
index 558f7cd49..cfdb0b11d 100644
--- a/cyberdrop_dl/progress/dedupe.py
+++ b/cyberdrop_dl/progress/dedupe.py
@@ -19,12 +19,8 @@ if TYPE_CHECKING:
@dataclasses.dataclass(slots=True)
class DedupeStats:
- keeped: int = 0
deleted: int = 0
-
- @property
- def total(self):
- return self.keeped + self.deleted
+ total: int = 0
@final
@@ -76,6 +72,7 @@ class DedupeUI(LiveUI):
"[blue]" + escape(str(file.relative_to(self._base_dir))),
total=None,
)
+ self._stats.total += 1
return ProgressHook(lambda _: None, lambda: 0, lambda: self._files.remove_task(task_id))
@@ -89,7 +86,7 @@ if __name__ == "__main__":
panel.stats.deleted += 1
with panel.new_file(folder / "subfolder/file2.txt"), panel.new_file(folder / "subfolder/file3.txt"):
time.sleep(1)
- panel.stats.keeped += 5
+ panel.stats.total += 5
time.sleep(1)
panel.stats.deleted += 15
time.sleep(3)
diff --git a/cyberdrop_dl/progress/hashing.py b/cyberdrop_dl/progress/hashing.py
index bfe5bd994..caa3fdeb7 100644
--- a/cyberdrop_dl/progress/hashing.py
+++ b/cyberdrop_dl/progress/hashing.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import dataclasses
import time
from pathlib import Path
-from typing import TYPE_CHECKING, final
+from typing import TYPE_CHECKING, Literal, final
from rich.align import Align
from rich.console import Group
@@ -92,6 +92,9 @@ class HashingUI(LiveUI):
return ProgressHook(lambda _: None, lambda: 0, lambda: self._files.remove_task(task_id))
+ def add_completed(self, hash_type: Literal["xxh128", "md5", "sha256"]):
+ setattr(self._stats, hash_type, getattr(self._stats, hash_type) + 1)
+
if __name__ == "__main__":
panel = HashingUI(folder := Path("/folder1/cdl_downloads"))