Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: directory hashing (#1361)

NTFSvolume2784df218519
diff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.py
index 0fd78da2b..c4cda80eb 100644
--- a/cyberdrop_dl/clients/hash_client.py
+++ b/cyberdrop_dl/clients/hash_client.py
@@ -49,6 +49,7 @@ class HashClient:
async def hash_directory(self, path: Path) -> None:
path = Path(path)
with self.manager.live_manager.get_hash_live(stop=True):
+ self.manager.progress_manager.hash_progress.base_dir = path
if not await asyncio.to_thread(path.is_dir):
raise NotADirectoryError
for file in path.rglob("*"):
@@ -81,7 +82,7 @@ class HashClient:
self, file: Path | str, original_filename: str | None = None, referer: URL | None = None
) -> str | None:
file = Path(file)
- if file.suffix == ".part":
+ if file.suffix in (".cdl_hls", ".cdl_hsl", ".part"):
return
if not await asyncio.to_thread(get_size_or_none, file):
return
@@ -112,7 +113,7 @@ class HashClient:
original_filename,
referer,
)
- self.manager.progress_manager.hash_progress.add_new_completed_hash()
+ self.manager.progress_manager.hash_progress.add_new_completed_hash(hash_type)
else:
self.manager.progress_manager.hash_progress.add_prev_hash()
await self.manager.db_manager.hash_table.insert_or_update_hash_db(
diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.py
index f62ad17bd..6007ad075 100644
--- a/cyberdrop_dl/director.py
+++ b/cyberdrop_dl/director.py
@@ -231,10 +231,9 @@ def _setup_main_logger(manager: Manager, config_name: str) -> None:
if not manager.parsed_args.cli_only_args.fullscreen_ui:
constants.CONSOLE_LEVEL = settings_data.runtime_options.console_log_level
+ constants.console_handler = LogHandler(level=constants.CONSOLE_LEVEL)
- constants.console_handler = console_handler = LogHandler(level=constants.CONSOLE_LEVEL)
- logger.addHandler(console_handler)
-
+ logger.addHandler(constants.console_handler)
file_handler = LogHandler(level=log_level, file=file_io, width=settings_data.logs.log_line_width)
queued_logger = QueuedLogger(manager, file_handler)
logger.addHandler(queued_logger.handler)
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index 6755a20eb..d67cbff94 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -264,7 +264,7 @@ class Downloader:
def create_segments() -> Generator[HlsSegment]:
for index, segment in enumerate(m3u8.segments, 1):
assert segment.uri
- name = f"{index:0{padding}d}.cdl_hsl"
+ name = f"{index:0{padding}d}.cdl_hls"
yield HlsSegment(segment.title, name, parse_url(segment.absolute_uri))
async def download_segment(segment: HlsSegment):
diff --git a/cyberdrop_dl/managers/live_manager.py b/cyberdrop_dl/managers/live_manager.py
index 57a327988..1a5493dea 100644
--- a/cyberdrop_dl/managers/live_manager.py
+++ b/cyberdrop_dl/managers/live_manager.py
@@ -25,7 +25,7 @@ class LiveManager:
self.fullscreen = f = self.manager.parsed_args.cli_only_args.fullscreen_ui
self.refresh_rate = rate = self.manager.config_manager.global_settings_data.ui_options.refresh_rate
self.live = Live(refresh_per_second=rate, transient=True, screen=f, auto_refresh=True)
- self.current_layout: str
+ self.current_layout: str = ""
@contextmanager
def get_live(self, name: str, stop: bool = False) -> Generator[Live | None]:
@@ -60,20 +60,30 @@ class LiveManager:
def live_context_manager(self, layout: RenderableType | None, stop: bool = False) -> Generator[Live | None]:
stop_event = asyncio.Event()
orientation_task = None
- og_console = constants.console_handler.console
+
try:
- constants.console_handler.console = self.live.console
- self.live.start()
- if layout:
- self.live.update(layout, refresh=True)
- if self.current_layout in ("vertical_layout", "horizontal_layout"):
- orientation_task = asyncio.create_task(self.watch_orientation(stop_event))
- yield self.live
+ with self.replace_console():
+ self.live.start()
+ if layout:
+ self.live.update(layout, refresh=True)
+ if self.current_layout in ("vertical_layout", "horizontal_layout"):
+ orientation_task = asyncio.create_task(self.watch_orientation(stop_event))
+ yield self.live
finally:
- constants.console_handler.console = og_console
stop_event.set()
if orientation_task:
orientation_task.cancel()
if stop:
self.live.update("")
self.live.stop()
+
+ @contextmanager
+ def replace_console(self) -> Generator[None]:
+ """Disable the default console, replacing it with the internal live's console"""
+
+ default_console = constants.console_handler.console
+ try:
+ constants.console_handler.console = self.live.console
+ yield
+ finally:
+ constants.console_handler.console = default_console
diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
index 31013deb0..955196a5c 100644
--- a/cyberdrop_dl/managers/manager.py
+++ b/cyberdrop_dl/managers/manager.py
@@ -22,7 +22,7 @@ from cyberdrop_dl.managers.progress_manager import ProgressManager
from cyberdrop_dl.managers.storage_manager import StorageManager
from cyberdrop_dl.utils import ffmpeg
from cyberdrop_dl.utils.args import ParsedArgs, parse_args
-from cyberdrop_dl.utils.logger import QueuedLogger, log
+from cyberdrop_dl.utils.logger import LogHandler, QueuedLogger, log
from cyberdrop_dl.utils.utilities import close_if_defined, get_system_information
if TYPE_CHECKING:
@@ -68,6 +68,8 @@ class Manager:
self.args = args
self.states: AsyncioEvents
+ constants.console_handler = LogHandler(level=constants.CONSOLE_LEVEL)
+
@property
def config(self):
return self.config_manager.settings_data
diff --git a/cyberdrop_dl/ui/progress/hash_progress.py b/cyberdrop_dl/ui/progress/hash_progress.py
index 99192b387..d9cd92e1f 100644
--- a/cyberdrop_dl/ui/progress/hash_progress.py
+++ b/cyberdrop_dl/ui/progress/hash_progress.py
@@ -5,41 +5,75 @@ from typing import TYPE_CHECKING
from pydantic import ByteSize
from rich.console import Group
+from rich.markup import escape
from rich.panel import Panel
-from rich.progress import BarColumn, Progress
+from rich.progress import BarColumn, Progress, TaskID
if TYPE_CHECKING:
from cyberdrop_dl.managers.manager import Manager
+def _generic_progress() -> Progress:
+ return Progress("[progress.description]{task.description}", BarColumn(bar_width=None), "{task.completed:,}")
+
+
class HashProgress:
"""Class that keeps track of hashed files."""
def __init__(self, manager: Manager) -> None:
self.manager = manager
- self.hash_progress = self.create_generic_progress()
- self.remove_progress = self.create_generic_progress()
- self.match_progress = self.create_generic_progress()
- self.current_hashing_text = Progress("{task.description}")
+ self._hash_progress = _generic_progress()
+ self._remove_progress = _generic_progress()
+ self._match_progress = _generic_progress()
+ self._file_info = Progress("{task.description}")
+ self._base_dir: Path
# hashing
- self.hashed_files = self.prev_hashed_files = 0
- self.hash_progress_group = Group(self.current_hashing_text, self.hash_progress)
- self.hashed_files_task_id = self.hash_progress.add_task("[green]Hashed", total=None)
- self.prev_hashed_files_task_id = self.hash_progress.add_task("[green]Previously Hashed", total=None)
- self.currently_hashing_task_id = self.current_hashing_text.add_task("")
- self.currently_hashing_size_task_id = self.current_hashing_text.add_task("")
+ self._computed_hashes = self._prev_hashed = 0
+ self.hash_progress_group = Group(self._file_info, self._hash_progress)
+
+ self._tasks: dict[str, TaskID] = {}
+
+ def add_hashed_task(hash_type: str) -> None:
+ desc = "[green]Hashed " + escape(f"[{hash_type}]")
+ self._tasks[hash_type] = self._hash_progress.add_task(desc, total=None)
+
+ add_hashed_task("xxh128")
+ if manager.config.dupe_cleanup_options.add_md5_hash:
+ add_hashed_task("md5")
+ if manager.config.dupe_cleanup_options.add_sha256_hash:
+ add_hashed_task("sha256")
+
+ self.prev_hashed_files_task_id = self._hash_progress.add_task("[green]Previously Hashed", total=None)
+
+ self._base_dir_task_id = self._file_info.add_task("")
+ self._file_task_id = self._file_info.add_task("")
# remove
self.removed_files = 0
- self.removed_progress_group = Group(self.match_progress, self.remove_progress)
- self.removed_files_task_id = self.remove_progress.add_task(
+ self.removed_progress_group = Group(self._match_progress, self._remove_progress)
+ self.removed_files_task_id = self._remove_progress.add_task(
"[green]Removed From Downloaded Files",
total=None,
)
- def create_generic_progress(self) -> Progress:
- return Progress("[progress.description]{task.description}", BarColumn(bar_width=None), "{task.completed:,}")
+ @property
+ def base_dir(self) -> Path:
+ return self._base_dir
+
+ @base_dir.setter
+ def base_dir(self, path: Path) -> None:
+ self._base_dir = path
+ desc = "[green]Base dir: [blue]" + escape(f"{self._base_dir}")
+ self._file_info.update(self._base_dir_task_id, description=desc)
+
+ @property
+ def hashed_files(self) -> int:
+ return int(self._computed_hashes / len(self._tasks))
+
+ @property
+ def prev_hashed_files(self) -> int:
+ return int(self._prev_hashed / len(self._tasks))
def get_renderable(self) -> Panel:
"""Returns the progress bar."""
@@ -54,34 +88,34 @@ class HashProgress:
"""Returns the progress bar."""
return Panel(self.removed_progress_group, border_style="green", padding=(1, 1))
- def update_currently_hashing(self, file: Path | str) -> None:
- self.current_hashing_text.update(self.currently_hashing_task_id, description=f"[blue]{file}")
+ def update_currently_hashing(self, file: Path) -> None:
file_size = ByteSize(Path(file).stat().st_size)
- self.current_hashing_text.update(
- self.currently_hashing_size_task_id,
- description=f"[blue]{file_size.human_readable(decimal=True)}",
- )
+ size_text = file_size.human_readable(decimal=True)
+ path = file.relative_to(self._base_dir)
+ desc = "[green]Current file: [blue]" + escape(f"{path}") + f" [green]({size_text})"
+ self._file_info.update(self._file_task_id, description=desc)
- def add_new_completed_hash(self) -> None:
+ def add_new_completed_hash(self, hash_type: str) -> None:
"""Adds a completed file to the progress bar."""
- self.hash_progress.advance(self.hashed_files_task_id, 1)
- self.hashed_files += 1
+ self._hash_progress.advance(self._tasks[hash_type], 1)
+ self._computed_hashes += 1
def add_prev_hash(self) -> None:
"""Adds a completed file to the progress bar."""
- self.hash_progress.advance(self.prev_hashed_files_task_id, 1)
- self.prev_hashed_files += 1
+ self._hash_progress.advance(self.prev_hashed_files_task_id, 1)
+ self._prev_hashed += 1
def add_removed_file(self) -> None:
- """Adds a completed file to the progress bar."""
- self.remove_progress.advance(self.removed_files_task_id, 1)
+ """Adds a removed file to the progress bar."""
+ self._remove_progress.advance(self.removed_files_task_id, 1)
self.removed_files += 1
def reset(self):
"""Resets the progress bar."""
- self.hash_progress.reset(self.hashed_files_task_id)
- self.hash_progress.reset(self.prev_hashed_files_task_id)
- self.hashed_files = self.prev_hashed_files = 0
+ for task in self._tasks.values():
+ self._hash_progress.reset(task)
+ self._hash_progress.reset(self.prev_hashed_files_task_id)
+ self._computed_hashes = self._prev_hashed = 0
- self.remove_progress.reset(self.removed_files_task_id)
+ self._remove_progress.reset(self.removed_files_task_id)
self.removed_files = 0
diff --git a/cyberdrop_dl/ui/prompts/basic_prompts.py b/cyberdrop_dl/ui/prompts/basic_prompts.py
index 78fadcc4d..86462a157 100644
--- a/cyberdrop_dl/ui/prompts/basic_prompts.py
+++ b/cyberdrop_dl/ui/prompts/basic_prompts.py
@@ -1,4 +1,5 @@
# type: ignore[reportPrivateImportUsage]
+import sys
from pathlib import Path
from InquirerPy import inquirer
@@ -77,6 +78,8 @@ def ask_toggle(message: str = "enable", **kwargs):
def enter_to_continue(message: str = "Press <ENTER> to continue", **kwargs):
+ if "pytest" in sys.modules:
+ return
options = DEFAULT_OPTIONS | {"long_instruction": "ENTER: continue"} | kwargs
msg = f"\n{message}"
return inquirer.confirm(message=msg, qmark="", **options).execute()
diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py
index ed30691dd..f244862a6 100644
--- a/cyberdrop_dl/utils/sorting.py
+++ b/cyberdrop_dl/utils/sorting.py
@@ -111,9 +111,8 @@ class Sorter:
for file in files:
ext = file.suffix.lower()
- if ".part" in ext:
- continue
- if ".cdl_hsl" in ext:
+
+ if ext in (".cdl_hls", ".cdl_hsl", ".part"):
continue
if ext in FILE_FORMATS["Audio"]:
await self.sort_audio(file, folder_name)
diff --git a/tests/test_hashing.py b/tests/test_hashing.py
new file mode 100644
index 000000000..0ae9498de
--- /dev/null
+++ b/tests/test_hashing.py
@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import sqlite3
+from collections import Counter
+from typing import TYPE_CHECKING
+
+import pytest
+
+from cyberdrop_dl.clients.hash_client import hash_directory_scanner
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+ from cyberdrop_dl.managers.manager import Manager
+
+
+def get_hashes(path: Path) -> set[tuple[str, str]]:
+ query = "SELECT hash_type, hash FROM hash "
+ conn = sqlite3.connect(path)
+ try:
+ cursor = conn.execute(query)
+ return set(cursor.fetchall())
+ finally:
+ conn.close()
+
+
+def create_files(path: Path, number: int) -> None:
+ for i in range(number):
+ file = path / f"file_{i}"
+ file.write_text(str(i), encoding="utf-8")
+
+
+ "expected_results",
+ [
+ {
+ ("xxh128", "cb358fcee0dfde56fb95a7322f5da314"),
+ ("xxh128", "df3ce784d856334d65cd25028f98f158"),
+ ("xxh128", "ce8f15881282a4001982e3a7bb241055"),
+ },
+ {
+ ("xxh128", "ce8f15881282a4001982e3a7bb241055"),
+ ("md5", "cfcd208495d565ef66e7dff9f98764da"),
+ },
+ {
+ ("xxh128", "ce8f15881282a4001982e3a7bb241055"),
+ ("md5", "cfcd208495d565ef66e7dff9f98764da"),
+ ("sha256", "5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"),
+ },
+ ],
+)
+def test_hash_directory_scanner(manager: Manager, expected_results: set[tuple[str, str]]) -> None:
+ count = Counter(x[0] for x in expected_results)
+ n_files = max(count.values())
+ algos = count.keys()
+ assert len(expected_results) == len(algos) * n_files
+ manager.config.dupe_cleanup_options.add_md5_hash = "md5" in algos
+ manager.config.dupe_cleanup_options.add_sha256_hash = "sha256" in algos
+
+ manager.path_manager.download_folder.mkdir(parents=True)
+ db_path = manager.path_manager.history_db
+ hash_directory_scanner(manager, manager.path_manager.download_folder)
+ assert not get_hashes(db_path)
+ create_files(manager.path_manager.download_folder, n_files)
+ hash_directory_scanner(manager, manager.path_manager.download_folder)
+ results = get_hashes(db_path)
+ assert len(results) == len(expected_results)
+ assert results == expected_results