Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: use new hashing UI (#147)
NTFSvolume6b33182ab065
diff --git a/cyberdrop_dl/hasher.py b/cyberdrop_dl/hasher.pyindex 844d9b9a0..c8ca560b2 100644--- a/cyberdrop_dl/hasher.py+++ b/cyberdrop_dl/hasher.py@@ -13,6 +13,8 @@ from send2trash import send2trashfrom cyberdrop_dl import aiofrom cyberdrop_dl.constants import Hashing, TempExt+from cyberdrop_dl.progress.dedupe import DedupeUI+from cyberdrop_dl.progress.hashing import HashingStats, HashingUIif TYPE_CHECKING:from yarl import URL@@ -47,9 +49,9 @@ def _compute_hash(file: Path, algorithm: Literal["xxh128", "md5", "sha256"]) ->async def hash_directory_scanner(manager: Manager, path: Path) -> None:manager.async_db_hash_startup()async with manager.database:- await manager.hasher.hash_directory(path)- manager.progress_manager.print_dedupe_stats()- manager.progress_manager.hash_progress.reset()+ stats = await manager.hasher.hash_directory(path)++ manager.progress_manager.print_hashing_stats(stats)@dataclasses.dataclass(slots=True)@@ -62,6 +64,17 @@ class Hasher:_sem: asyncio.BoundedSemaphore = dataclasses.field(init=False, default_factory=lambda: asyncio.BoundedSemaphore(20))_cwd: Path = dataclasses.field(init=False, default_factory=Path.cwd)_hashed_items: set[tuple[str, ...]] = dataclasses.field(default_factory=set)+ _tui: HashingUI = dataclasses.field(init=False)+ _dedupe_tui: DedupeUI = dataclasses.field(init=False)++ def __post_init__(self) -> None:+ base_dir = self.manager.config.settings.files.download_folder+ self._tui = HashingUI(base_dir)+ self._dedupe_tui = DedupeUI(base_dir)++ @property+ def stats(self):+ return self._tui.stats, self._dedupe_tui.stats@propertydef config(self) -> DupeCleanup:@@ -79,17 +92,22 @@ class Hasher:def _deleted_file_suffix(self) -> Literal["Sent to trash", "Permanently deleted"]:return "Sent to trash" if self._to_trash else "Permanently deleted"- async def hash_directory(self, path: Path) -> None:+ async def hash_directory(self, path: Path) -> HashingStats:path = Path(path)- with (- self.manager.live_manager.get_hash_live(stop=True),- self.manager.progress_manager.hash_progress.currently_hashing_dir(path),- ):+ tui = HashingUI(path)+ old_tui = self._tui+ with tui():if not await aio.is_dir(path):raise NotADirectoryError(None, path)- async for file in aio.rglob(path, "*"):- _ = await self.update_db_and_retrive_hash(file)+ try:+ self._tui = tui+ async for file in aio.rglob(path, "*"):+ _ = await self.update_db_and_retrive_hash(file)+ finally:+ self._tui = old_tui++ return tui.statsasync def hash_item(self, media_item: MediaItem) -> None:if media_item.is_segment:@@ -127,14 +145,15 @@ class Hasher:if not await aio.get_size(file):return- logger.info("Computing hashes of '%s'", file)- hash = await self._update_db_and_retrive_hash(file, original_filename, referer, hash_type="xxh128")- if self.config.add_md5_hash:- await self._update_db_and_retrive_hash(file, original_filename, referer, hash_type="md5")- if self.config.add_sha256_hash:- await self._update_db_and_retrive_hash(file, original_filename, referer, hash_type="sha256")+ with self._tui.new_file(file):+ logger.info("Computing hashes of '%s'", file)+ hash = await self._update_db_and_retrive_hash(file, original_filename, referer, hash_type="xxh128")+ if self.config.add_md5_hash:+ await self._update_db_and_retrive_hash(file, original_filename, referer, hash_type="md5")+ if self.config.add_sha256_hash:+ await self._update_db_and_retrive_hash(file, original_filename, referer, hash_type="sha256")- return hash+ return hashasync def _update_db_and_retrive_hash(self,@@ -144,7 +163,7 @@ class Hasher:hash_type: Literal["xxh128", "md5", "sha256"],) -> str | None:"""Generates hash of a file."""- self.manager.progress_manager.hash_progress.update_currently_hashing(file)+hash = await self.manager.database.hash.get_file_hash_exists(file, hash_type)try:if not hash:@@ -156,9 +175,9 @@ class Hasher:original_filename,referer,)- self.manager.progress_manager.hash_progress.add_new_completed_hash(hash_type)+ self._tui.add_completed(hash_type)else:- self.manager.progress_manager.hash_progress.add_prev_hash()+ self._tui.stats.prev_hashed += 1await self.manager.database.hash.insert_or_update_hash_db(hash,hash_type,@@ -191,9 +210,9 @@ class Hasher:returnif self.manager.config.settings.runtime_options.ignore_history:return- with self.manager.live_manager.get_hash_live(stop=True):+ with self._tui():file_hashes_dict = await self.get_file_hashes_dict()- with self.manager.live_manager.get_remove_file_via_hash_live(stop=True):+ with self._dedupe_tui():await self.final_dupe_cleanup(file_hashes_dict)async def final_dupe_cleanup(self, final_dict: dict[str, dict[int, set[Path]]]) -> None:@@ -215,24 +234,25 @@ class Hasher:async def _delete_and_log(self, file: Path, xxh128_value: str) -> None:hash_string = f"xxh128:{xxh128_value}"- try:- deleted = await _delete_file(file, self._to_trash)- except OSError as e:- logger.exception(f"Unable to remove '{file}' ({hash_string}): {e}")+ with self._dedupe_tui.new_file(file):+ try:+ deleted = await _delete_file(file, self._to_trash)+ except OSError as e:+ logger.exception(f"Unable to remove '{file}' ({hash_string}): {e}")- else:- if not deleted:- return+ else:+ if not deleted:+ return- msg = (- f"Removed new download '{file}' [{self._deleted_file_suffix}]. "- f"File hash matches with a previous download ({hash_string})"- )- logger.info(msg)- self.manager.progress_manager.hash_progress.add_removed_file()+ msg = (+ f"Removed new download '{file}' [{self._deleted_file_suffix}]. "+ f"File hash matches with a previous download ({hash_string})"+ )+ logger.info(msg)+ self._dedupe_tui.stats.deleted += 1- finally:- self._sem.release()+ finally:+ self._sem.release()async def get_file_hashes_dict(self) -> dict[str, dict[int, set[Path]]]:diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.pyindex 152865f03..afe492228 100644--- a/cyberdrop_dl/managers/progress_manager.py+++ b/cyberdrop_dl/managers/progress_manager.py@@ -27,6 +27,8 @@ if TYPE_CHECKING:from rich.panel import Panelfrom cyberdrop_dl.managers.manager import Manager+ from cyberdrop_dl.progress.dedupe import DedupeStats+ from cyberdrop_dl.progress.hashing import HashingStatsfrom cyberdrop_dl.scrape_mapper import ScrapeStatsfrom cyberdrop_dl.ui.progress.statistic_progress import UiFailureTotal@@ -154,7 +156,9 @@ class ProgressManager:logger.info(f" Sent to Jdownloader: {self.scrape_stats_progress.sent_to_jdownloader:,}")logger.info(f" Skipped: {self.scrape_stats_progress.unsupported_urls_skipped:,}")- self.print_dedupe_stats()+ hash_stats, dedupe_stats = self.manager.hasher.stats+ self.print_hashing_stats(hash_stats)+ self.print_dedupe_stats(dedupe_stats)log_spacer()logger.info("Sort Stats:", extra={"color": "cyan"})@@ -163,14 +167,23 @@ class ProgressManager:logger.info(f" Videos: {self.sort_progress.video_count:,}")logger.info(f" Other Files: {self.sort_progress.other_count:,}")+ self.print_errors()++ def print_errors(self):+_log_errors(self.scrape_stats_progress.return_totals(), self.download_stats_progress.return_totals())- def print_dedupe_stats(self) -> None:+ def print_hashing_stats(self, stats: HashingStats) -> None:+ log_spacer()+ logger.info("Checksum Stats:", extra={"color": "cyan"})+ logger.info(f" Newly hashed: {stats.new_hashed:,} files")+ logger.info(f" Previously hashed: {stats.prev_hashed:,} files")++ def print_dedupe_stats(self, stats: DedupeStats) -> None:log_spacer()logger.info("Dupe Stats:", extra={"color": "cyan"})- logger.info(f" Newly hashed: {self.hash_progress.hashed_files:,} files")- logger.info(f" Previously hashed: {self.hash_progress.prev_hashed_files:,} files")- logger.info(f" Deleted (duplicates of previous downloads): {self.hash_progress.removed_files:,} files")+ logger.info(f" Deleted (duplicates of previous downloads): {stats.deleted:,} files")+ logger.info(f" Errors: {stats.total - stats.deleted:,} files")def _log_errors(scrape_errors: Sequence[UiFailureTotal], download_errors: Sequence[UiFailureTotal]) -> None:diff --git a/cyberdrop_dl/progress/dedupe.py b/cyberdrop_dl/progress/dedupe.pyindex 558f7cd49..cfdb0b11d 100644--- a/cyberdrop_dl/progress/dedupe.py+++ b/cyberdrop_dl/progress/dedupe.py@@ -19,12 +19,8 @@ if TYPE_CHECKING:@dataclasses.dataclass(slots=True)class DedupeStats:- keeped: int = 0deleted: int = 0-- @property- def total(self):- return self.keeped + self.deleted+ total: int = 0@final@@ -76,6 +72,7 @@ class DedupeUI(LiveUI):"[blue]" + escape(str(file.relative_to(self._base_dir))),total=None,)+ self._stats.total += 1return ProgressHook(lambda _: None, lambda: 0, lambda: self._files.remove_task(task_id))@@ -89,7 +86,7 @@ if __name__ == "__main__":panel.stats.deleted += 1with panel.new_file(folder / "subfolder/file2.txt"), panel.new_file(folder / "subfolder/file3.txt"):time.sleep(1)- panel.stats.keeped += 5+ panel.stats.total += 5time.sleep(1)panel.stats.deleted += 15time.sleep(3)diff --git a/cyberdrop_dl/progress/hashing.py b/cyberdrop_dl/progress/hashing.pyindex bfe5bd994..caa3fdeb7 100644--- a/cyberdrop_dl/progress/hashing.py+++ b/cyberdrop_dl/progress/hashing.py@@ -3,7 +3,7 @@ from __future__ import annotationsimport dataclassesimport timefrom pathlib import Path-from typing import TYPE_CHECKING, final+from typing import TYPE_CHECKING, Literal, finalfrom rich.align import Alignfrom rich.console import Group@@ -92,6 +92,9 @@ class HashingUI(LiveUI):return ProgressHook(lambda _: None, lambda: 0, lambda: self._files.remove_task(task_id))+ def add_completed(self, hash_type: Literal["xxh128", "md5", "sha256"]):+ setattr(self._stats, hash_type, getattr(self._stats, hash_type) + 1)+if __name__ == "__main__":panel = HashingUI(folder := Path("/folder1/cdl_downloads"))