Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: incorrect hash being logged when deleting files (#1364)
NTFSvolume63d1df70fcae
diff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.pyindex c4cda80eb..011b1b30a 100644--- a/cyberdrop_dl/clients/hash_client.py+++ b/cyberdrop_dl/clients/hash_client.py@@ -3,9 +3,8 @@ from __future__ import annotationsimport asynciofrom collections import defaultdictfrom pathlib import Path-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Literal-import aiofiles.osfrom send2trash import send2trashfrom cyberdrop_dl.data_structures.hash import Hashing@@ -16,6 +15,7 @@ from cyberdrop_dl.utils.utilities import get_size_or_noneif TYPE_CHECKING:from yarl import URL+ from cyberdrop_dl.config.config_model import DupeCleanupfrom cyberdrop_dl.data_structures.url_objects import MediaItemfrom cyberdrop_dl.managers.manager import Manager@@ -42,6 +42,19 @@ class HashClient:self.sha256 = "sha256"self.hashed_media_items: set[MediaItem] = set()self.hashes_dict: defaultdict[str, defaultdict[int, set[Path]]] = defaultdict(lambda: defaultdict(set))+ self._sem = asyncio.BoundedSemaphore(20)++ @property+ def _to_trash(self) -> bool:+ return self.dupe_cleanup_options.send_deleted_to_trash++ @property+ def _deleted_file_suffix(self) -> Literal["Sent to trash", "Permanently deleted"]:+ return "Sent to trash" if self._to_trash else "Permanently deleted"++ @property+ def dupe_cleanup_options(self) -> DupeCleanup:+ return self.manager.config.dupe_cleanup_optionsasync def startup(self) -> None:pass@@ -153,29 +166,40 @@ class HashClient:async def final_dupe_cleanup(self, final_dict: dict[str, dict]) -> None:"""cleanup files based on dedupe setting"""- to_trash = self.manager.config_manager.settings_data.dupe_cleanup_options.send_deleted_to_trash- suffix = "Sent to trash " if to_trash else "Permanently deleted"- async def delete_and_log(file: Path):- try:- deleted = await delete_file(file, to_trash)- if deleted:- log(f"Removed new download '{file}' with hash {hash} [{suffix}]", 10)- self.manager.progress_manager.hash_progress.add_removed_file()+ get_matches = self.manager.db_manager.hash_table.get_files_with_hash_matches+ async with asyncio.TaskGroup() as tg:- except OSError as e:- log(f"Unable to remove '{file}' with hash {hash}: {e}", 40)+ async def delete_dupes(hash_value: str, size: int) -> None:+ db_matches = await get_matches(hash_value, size, "xxh128")+ for row in db_matches[1:]:+ file = Path(row["folder"], row["download_filename"])+ await self._sem.acquire()+ tg.create_task(self._delete_and_log(file, hash_value))- tasks = []- get_matches = self.manager.db_manager.hash_table.get_files_with_hash_matches- for hash, size_dict in final_dict.items():- for size in size_dict:- db_matches = await get_matches(hash, size, self.xxhash)- for match in db_matches[1:]:- file = Path(*match[:2])- tasks.append(delete_and_log(file))+ for hash_value, size_dict in final_dict.items():+ for size in size_dict:+ tg.create_task(delete_dupes(hash_value, size))++ async def _delete_and_log(self, file: Path, xxh128_value: str) -> None:+ hash_string = f"xxh128:{xxh128_value}"+ try:+ deleted = await _delete_file(file, self._to_trash)+ except OSError as e:+ log(f"Unable to remove '{file}' ({hash_string}): {e}", 40)+ else:+ if not deleted:+ return++ msg = (+ f"Removed new download '{file}' [{self._deleted_file_suffix}]. "+ f"File hash matches with a previous download ({hash_string})"+ )+ log(msg, 10)+ self.manager.progress_manager.hash_progress.add_removed_file()- await asyncio.gather(*tasks)+ finally:+ self._sem.release()async def get_file_hashes_dict(self) -> dict:"""Get a dictionary of files based on matching file hashes and file size."""@@ -197,7 +221,7 @@ class HashClient:return self.hashes_dict-async def delete_file(path: Path, to_trash: bool = True) -> bool:+async def _delete_file(path: Path, to_trash: bool = True) -> bool:"""Deletes a file and return `True` on success, `False` is the file was not found.Any other exception is propagated"""@@ -205,10 +229,11 @@ async def delete_file(path: Path, to_trash: bool = True) -> bool:if to_trash:coro = asyncio.to_thread(send2trash, path)else:- coro = aiofiles.os.unlink(Path(path))+ coro = asyncio.to_thread(path.unlink)try:await coro+ return Trueexcept FileNotFoundError:passexcept OSError as e:@@ -216,7 +241,5 @@ async def delete_file(path: Path, to_trash: bool = True) -> bool:msg = str(e)if "File not found" not in msg:raise- else:- return Truereturn False