Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: directory hashing (#1361)
NTFSvolume2784df218519
diff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.pyindex 0fd78da2b..c4cda80eb 100644--- a/cyberdrop_dl/clients/hash_client.py+++ b/cyberdrop_dl/clients/hash_client.py@@ -49,6 +49,7 @@ class HashClient:async def hash_directory(self, path: Path) -> None:path = Path(path)with self.manager.live_manager.get_hash_live(stop=True):+ self.manager.progress_manager.hash_progress.base_dir = pathif not await asyncio.to_thread(path.is_dir):raise NotADirectoryErrorfor file in path.rglob("*"):@@ -81,7 +82,7 @@ class HashClient:self, file: Path | str, original_filename: str | None = None, referer: URL | None = None) -> str | None:file = Path(file)- if file.suffix == ".part":+ if file.suffix in (".cdl_hls", ".cdl_hsl", ".part"):returnif not await asyncio.to_thread(get_size_or_none, file):return@@ -112,7 +113,7 @@ class HashClient:original_filename,referer,)- self.manager.progress_manager.hash_progress.add_new_completed_hash()+ self.manager.progress_manager.hash_progress.add_new_completed_hash(hash_type)else:self.manager.progress_manager.hash_progress.add_prev_hash()await self.manager.db_manager.hash_table.insert_or_update_hash_db(diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.pyindex f62ad17bd..6007ad075 100644--- a/cyberdrop_dl/director.py+++ b/cyberdrop_dl/director.py@@ -231,10 +231,9 @@ def _setup_main_logger(manager: Manager, config_name: str) -> None:if not manager.parsed_args.cli_only_args.fullscreen_ui:constants.CONSOLE_LEVEL = settings_data.runtime_options.console_log_level+ constants.console_handler = LogHandler(level=constants.CONSOLE_LEVEL)- constants.console_handler = console_handler = LogHandler(level=constants.CONSOLE_LEVEL)- logger.addHandler(console_handler)-+ logger.addHandler(constants.console_handler)file_handler = LogHandler(level=log_level, file=file_io, width=settings_data.logs.log_line_width)queued_logger = QueuedLogger(manager, file_handler)logger.addHandler(queued_logger.handler)diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex 6755a20eb..d67cbff94 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -264,7 +264,7 @@ class Downloader:def create_segments() -> Generator[HlsSegment]:for index, segment in enumerate(m3u8.segments, 1):assert segment.uri- name = f"{index:0{padding}d}.cdl_hsl"+ name = f"{index:0{padding}d}.cdl_hls"yield HlsSegment(segment.title, name, parse_url(segment.absolute_uri))async def download_segment(segment: HlsSegment):diff --git a/cyberdrop_dl/managers/live_manager.py b/cyberdrop_dl/managers/live_manager.pyindex 57a327988..1a5493dea 100644--- a/cyberdrop_dl/managers/live_manager.py+++ b/cyberdrop_dl/managers/live_manager.py@@ -25,7 +25,7 @@ class LiveManager:self.fullscreen = f = self.manager.parsed_args.cli_only_args.fullscreen_uiself.refresh_rate = rate = self.manager.config_manager.global_settings_data.ui_options.refresh_rateself.live = Live(refresh_per_second=rate, transient=True, screen=f, auto_refresh=True)- self.current_layout: str+ self.current_layout: str = ""@contextmanagerdef get_live(self, name: str, stop: bool = False) -> Generator[Live | None]:@@ -60,20 +60,30 @@ class LiveManager:def live_context_manager(self, layout: RenderableType | None, stop: bool = False) -> Generator[Live | None]:stop_event = asyncio.Event()orientation_task = None- og_console = constants.console_handler.console+try:- constants.console_handler.console = self.live.console- self.live.start()- if layout:- self.live.update(layout, refresh=True)- if self.current_layout in ("vertical_layout", "horizontal_layout"):- orientation_task = asyncio.create_task(self.watch_orientation(stop_event))- yield self.live+ with self.replace_console():+ self.live.start()+ if layout:+ self.live.update(layout, refresh=True)+ if self.current_layout in ("vertical_layout", "horizontal_layout"):+ orientation_task = asyncio.create_task(self.watch_orientation(stop_event))+ yield self.livefinally:- constants.console_handler.console = og_consolestop_event.set()if orientation_task:orientation_task.cancel()if stop:self.live.update("")self.live.stop()++ @contextmanager+ def replace_console(self) -> Generator[None]:+ """Disable the default console, replacing it with the internal live's console"""++ default_console = constants.console_handler.console+ try:+ constants.console_handler.console = self.live.console+ yield+ finally:+ constants.console_handler.console = default_consolediff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex 31013deb0..955196a5c 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -22,7 +22,7 @@ from cyberdrop_dl.managers.progress_manager import ProgressManagerfrom cyberdrop_dl.managers.storage_manager import StorageManagerfrom cyberdrop_dl.utils import ffmpegfrom cyberdrop_dl.utils.args import ParsedArgs, parse_args-from cyberdrop_dl.utils.logger import QueuedLogger, log+from cyberdrop_dl.utils.logger import LogHandler, QueuedLogger, logfrom cyberdrop_dl.utils.utilities import close_if_defined, get_system_informationif TYPE_CHECKING:@@ -68,6 +68,8 @@ class Manager:self.args = argsself.states: AsyncioEvents+ constants.console_handler = LogHandler(level=constants.CONSOLE_LEVEL)+@propertydef config(self):return self.config_manager.settings_datadiff --git a/cyberdrop_dl/ui/progress/hash_progress.py b/cyberdrop_dl/ui/progress/hash_progress.pyindex 99192b387..d9cd92e1f 100644--- a/cyberdrop_dl/ui/progress/hash_progress.py+++ b/cyberdrop_dl/ui/progress/hash_progress.py@@ -5,41 +5,75 @@ from typing import TYPE_CHECKINGfrom pydantic import ByteSizefrom rich.console import Group+from rich.markup import escapefrom rich.panel import Panel-from rich.progress import BarColumn, Progress+from rich.progress import BarColumn, Progress, TaskIDif TYPE_CHECKING:from cyberdrop_dl.managers.manager import Manager+def _generic_progress() -> Progress:+ return Progress("[progress.description]{task.description}", BarColumn(bar_width=None), "{task.completed:,}")++class HashProgress:"""Class that keeps track of hashed files."""def __init__(self, manager: Manager) -> None:self.manager = manager- self.hash_progress = self.create_generic_progress()- self.remove_progress = self.create_generic_progress()- self.match_progress = self.create_generic_progress()- self.current_hashing_text = Progress("{task.description}")+ self._hash_progress = _generic_progress()+ self._remove_progress = _generic_progress()+ self._match_progress = _generic_progress()+ self._file_info = Progress("{task.description}")+ self._base_dir: Path# hashing- self.hashed_files = self.prev_hashed_files = 0- self.hash_progress_group = Group(self.current_hashing_text, self.hash_progress)- self.hashed_files_task_id = self.hash_progress.add_task("[green]Hashed", total=None)- self.prev_hashed_files_task_id = self.hash_progress.add_task("[green]Previously Hashed", total=None)- self.currently_hashing_task_id = self.current_hashing_text.add_task("")- self.currently_hashing_size_task_id = self.current_hashing_text.add_task("")+ self._computed_hashes = self._prev_hashed = 0+ self.hash_progress_group = Group(self._file_info, self._hash_progress)++ self._tasks: dict[str, TaskID] = {}++ def add_hashed_task(hash_type: str) -> None:+ desc = "[green]Hashed " + escape(f"[{hash_type}]")+ self._tasks[hash_type] = self._hash_progress.add_task(desc, total=None)++ add_hashed_task("xxh128")+ if manager.config.dupe_cleanup_options.add_md5_hash:+ add_hashed_task("md5")+ if manager.config.dupe_cleanup_options.add_sha256_hash:+ add_hashed_task("sha256")++ self.prev_hashed_files_task_id = self._hash_progress.add_task("[green]Previously Hashed", total=None)++ self._base_dir_task_id = self._file_info.add_task("")+ self._file_task_id = self._file_info.add_task("")# removeself.removed_files = 0- self.removed_progress_group = Group(self.match_progress, self.remove_progress)- self.removed_files_task_id = self.remove_progress.add_task(+ self.removed_progress_group = Group(self._match_progress, self._remove_progress)+ self.removed_files_task_id = self._remove_progress.add_task("[green]Removed From Downloaded Files",total=None,)- def create_generic_progress(self) -> Progress:- return Progress("[progress.description]{task.description}", BarColumn(bar_width=None), "{task.completed:,}")+ @property+ def base_dir(self) -> Path:+ return self._base_dir++ @base_dir.setter+ def base_dir(self, path: Path) -> None:+ self._base_dir = path+ desc = "[green]Base dir: [blue]" + escape(f"{self._base_dir}")+ self._file_info.update(self._base_dir_task_id, description=desc)++ @property+ def hashed_files(self) -> int:+ return int(self._computed_hashes / len(self._tasks))++ @property+ def prev_hashed_files(self) -> int:+ return int(self._prev_hashed / len(self._tasks))def get_renderable(self) -> Panel:"""Returns the progress bar."""@@ -54,34 +88,34 @@ class HashProgress:"""Returns the progress bar."""return Panel(self.removed_progress_group, border_style="green", padding=(1, 1))- def update_currently_hashing(self, file: Path | str) -> None:- self.current_hashing_text.update(self.currently_hashing_task_id, description=f"[blue]{file}")+ def update_currently_hashing(self, file: Path) -> None:file_size = ByteSize(Path(file).stat().st_size)- self.current_hashing_text.update(- self.currently_hashing_size_task_id,- description=f"[blue]{file_size.human_readable(decimal=True)}",- )+ size_text = file_size.human_readable(decimal=True)+ path = file.relative_to(self._base_dir)+ desc = "[green]Current file: [blue]" + escape(f"{path}") + f" [green]({size_text})"+ self._file_info.update(self._file_task_id, description=desc)- def add_new_completed_hash(self) -> None:+ def add_new_completed_hash(self, hash_type: str) -> None:"""Adds a completed file to the progress bar."""- self.hash_progress.advance(self.hashed_files_task_id, 1)- self.hashed_files += 1+ self._hash_progress.advance(self._tasks[hash_type], 1)+ self._computed_hashes += 1def add_prev_hash(self) -> None:"""Adds a completed file to the progress bar."""- self.hash_progress.advance(self.prev_hashed_files_task_id, 1)- self.prev_hashed_files += 1+ self._hash_progress.advance(self.prev_hashed_files_task_id, 1)+ self._prev_hashed += 1def add_removed_file(self) -> None:- """Adds a completed file to the progress bar."""- self.remove_progress.advance(self.removed_files_task_id, 1)+ """Adds a removed file to the progress bar."""+ self._remove_progress.advance(self.removed_files_task_id, 1)self.removed_files += 1def reset(self):"""Resets the progress bar."""- self.hash_progress.reset(self.hashed_files_task_id)- self.hash_progress.reset(self.prev_hashed_files_task_id)- self.hashed_files = self.prev_hashed_files = 0+ for task in self._tasks.values():+ self._hash_progress.reset(task)+ self._hash_progress.reset(self.prev_hashed_files_task_id)+ self._computed_hashes = self._prev_hashed = 0- self.remove_progress.reset(self.removed_files_task_id)+ self._remove_progress.reset(self.removed_files_task_id)self.removed_files = 0diff --git a/cyberdrop_dl/ui/prompts/basic_prompts.py b/cyberdrop_dl/ui/prompts/basic_prompts.pyindex 78fadcc4d..86462a157 100644--- a/cyberdrop_dl/ui/prompts/basic_prompts.py+++ b/cyberdrop_dl/ui/prompts/basic_prompts.py@@ -1,4 +1,5 @@# type: ignore[reportPrivateImportUsage]+import sysfrom pathlib import Pathfrom InquirerPy import inquirer@@ -77,6 +78,8 @@ def ask_toggle(message: str = "enable", **kwargs):def enter_to_continue(message: str = "Press <ENTER> to continue", **kwargs):+ if "pytest" in sys.modules:+ returnoptions = DEFAULT_OPTIONS | {"long_instruction": "ENTER: continue"} | kwargsmsg = f"\n{message}"return inquirer.confirm(message=msg, qmark="", **options).execute()diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.pyindex ed30691dd..f244862a6 100644--- a/cyberdrop_dl/utils/sorting.py+++ b/cyberdrop_dl/utils/sorting.py@@ -111,9 +111,8 @@ class Sorter:for file in files:ext = file.suffix.lower()- if ".part" in ext:- continue- if ".cdl_hsl" in ext:++ if ext in (".cdl_hls", ".cdl_hsl", ".part"):continueif ext in FILE_FORMATS["Audio"]:await self.sort_audio(file, folder_name)diff --git a/tests/test_hashing.py b/tests/test_hashing.pynew file mode 100644index 000000000..0ae9498de--- /dev/null+++ b/tests/test_hashing.py@@ -0,0 +1,68 @@+from __future__ import annotations++import sqlite3+from collections import Counter+from typing import TYPE_CHECKING++import pytest++from cyberdrop_dl.clients.hash_client import hash_directory_scanner++if TYPE_CHECKING:+ from pathlib import Path++ from cyberdrop_dl.managers.manager import Manager+++def get_hashes(path: Path) -> set[tuple[str, str]]:+ query = "SELECT hash_type, hash FROM hash "+ conn = sqlite3.connect(path)+ try:+ cursor = conn.execute(query)+ return set(cursor.fetchall())+ finally:+ conn.close()+++def create_files(path: Path, number: int) -> None:+ for i in range(number):+ file = path / f"file_{i}"+ file.write_text(str(i), encoding="utf-8")+++ "expected_results",+ [+ {+ ("xxh128", "cb358fcee0dfde56fb95a7322f5da314"),+ ("xxh128", "df3ce784d856334d65cd25028f98f158"),+ ("xxh128", "ce8f15881282a4001982e3a7bb241055"),+ },+ {+ ("xxh128", "ce8f15881282a4001982e3a7bb241055"),+ ("md5", "cfcd208495d565ef66e7dff9f98764da"),+ },+ {+ ("xxh128", "ce8f15881282a4001982e3a7bb241055"),+ ("md5", "cfcd208495d565ef66e7dff9f98764da"),+ ("sha256", "5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"),+ },+ ],+)+def test_hash_directory_scanner(manager: Manager, expected_results: set[tuple[str, str]]) -> None:+ count = Counter(x[0] for x in expected_results)+ n_files = max(count.values())+ algos = count.keys()+ assert len(expected_results) == len(algos) * n_files+ manager.config.dupe_cleanup_options.add_md5_hash = "md5" in algos+ manager.config.dupe_cleanup_options.add_sha256_hash = "sha256" in algos++ manager.path_manager.download_folder.mkdir(parents=True)+ db_path = manager.path_manager.history_db+ hash_directory_scanner(manager, manager.path_manager.download_folder)+ assert not get_hashes(db_path)+ create_files(manager.path_manager.download_folder, n_files)+ hash_directory_scanner(manager, manager.path_manager.download_folder)+ results = get_hashes(db_path)+ assert len(results) == len(expected_results)+ assert results == expected_results