Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: make manager a context manager (#1774)

NTFSvolume164744d425f2
diff --git a/cyberdrop_dl/__main__.py b/cyberdrop_dl/__main__.py
index 1b2d32ade..2059c9d89 100644
--- a/cyberdrop_dl/__main__.py
+++ b/cyberdrop_dl/__main__.py
@@ -15,7 +15,6 @@ from cyberdrop_dl.config import Config
from cyberdrop_dl.logs import log_spacer, set_console_level, setup_console_logging, setup_file_logging
from cyberdrop_dl.managers.manager import AppData, Manager
from cyberdrop_dl.models.types import HttpURL
-from cyberdrop_dl.progress import REFRESH_RATE, TUI_DISABLED
from cyberdrop_dl.scrape_mapper import ScrapeMapper
from cyberdrop_dl.updates import check_latest_pypi
from cyberdrop_dl.utils import apprise, check_partials_and_empty_folders
@@ -28,10 +27,7 @@ async def _scrape(manager: Manager) -> None:
manager.config.settings.logs.main_log,
level=manager.config.settings.runtime_options.effective_log_level,
):
- await manager.async_startup()
- REFRESH_RATE.set(manager.config.global_settings.ui_options.refresh_rate)
- TUI_DISABLED.set(manager.cli_args.ui.is_disabled)
-
+ manager.log_config_settings()
log_spacer()
async with manager.database:
log_spacer()
@@ -81,12 +77,11 @@ async def _post_runtime(manager: Manager) -> None:
def _main(manager: Manager) -> None:
set_console_level(manager.config.settings.runtime_options.effective_console_log_level)
- manager.resolve_paths()
- if not manager.cli_args.download:
- program_ui.run(manager)
-
try:
- aio.run(_scrape(manager))
+ with manager():
+ if not manager.cli_args.download:
+ program_ui.run(manager)
+ aio.run(_scrape(manager))
except KeyboardInterrupt:
logger.info("Exiting (Ctrl + C) ...")
diff --git a/cyberdrop_dl/hasher.py b/cyberdrop_dl/hasher.py
index 7c4574527..eb50e815a 100644
--- a/cyberdrop_dl/hasher.py
+++ b/cyberdrop_dl/hasher.py
@@ -46,13 +46,26 @@ def _compute_hash(file: Path, algorithm: Literal["xxh128", "md5", "sha256"]) ->
async def hash_directory_scanner(manager: Manager, path: Path) -> None:
- manager.async_db_hash_startup()
async with manager.database:
- stats = await manager.hasher.hash_directory(path)
+ stats = await hash_directory(manager, path)
manager.print_hashing_stats(stats)
+async def hash_directory(manager: Manager, path: Path) -> HashingStats:
+ hasher = Hasher(manager)
+ hasher._tui = tui = HashingUI(path)
+ with tui():
+ if not await aio.is_dir(path):
+ raise NotADirectoryError(None, path)
+
+ async with asyncio.TaskGroup() as tg:
+ async for file in aio.rglob(path, "*"):
+ _ = tg.create_task(hasher.update_db_and_retrive_hash(file))
+
+ return tui.stats
+
+
@dataclasses.dataclass(slots=True)
class Hasher:
manager: Manager
@@ -86,24 +99,6 @@ class Hasher:
file_path = self._cwd / filename
return await asyncio.to_thread(_compute_hash, file_path, hash_type)
- async def hash_directory(self, path: Path) -> HashingStats:
- path = Path(path)
- tui = HashingUI(path)
- old_tui = self._tui
- with tui():
- if not await aio.is_dir(path):
- raise NotADirectoryError(None, path)
-
- try:
- self._tui = tui
- async with asyncio.TaskGroup() as tg:
- async for file in aio.rglob(path, "*"):
- _ = tg.create_task(self.update_db_and_retrive_hash(file))
- finally:
- self._tui = old_tui
-
- return tui.stats
-
async def hash_item(self, media_item: MediaItem) -> None:
if media_item.is_segment:
return
diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
index 1634beed1..c155cbeba 100644
--- a/cyberdrop_dl/managers/manager.py
+++ b/cyberdrop_dl/managers/manager.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import contextlib
import dataclasses
import logging
import os
@@ -17,14 +18,15 @@ from cyberdrop_dl.config import Config
from cyberdrop_dl.database import Database
from cyberdrop_dl.dedupe import Czkawka
from cyberdrop_dl.hasher import Hasher
-from cyberdrop_dl.logs import capture_logs, log_spacer
+from cyberdrop_dl.logs import _enter_context, capture_logs, log_spacer
from cyberdrop_dl.managers.client_manager import ClientManager
from cyberdrop_dl.managers.logs import LogManager
+from cyberdrop_dl.progress import REFRESH_RATE, TUI_DISABLED
from cyberdrop_dl.sorter import Sorter
from cyberdrop_dl.utils import get_system_information
if TYPE_CHECKING:
- from collections.abc import Sequence
+ from collections.abc import Generator, Sequence
from os import PathLike
from cyberdrop_dl.progress.dedupe import DedupeStats
@@ -71,24 +73,28 @@ class Manager:
self._config = Config.from_manager(self)
return self._config
- def resolve_paths(self) -> None:
+ def __resolve_paths(self) -> None:
self.appdata.mkdirs()
self.config.settings.resolve_paths()
self.logs = LogManager.from_manager(self)
self.logs.delete_old_logs()
- async def __aenter__(self) -> Self:
- cache_file = self.appdata.cache_file
- try:
- self.cache.update(yaml.load(cache_file))
- except FileNotFoundError:
- cache_file.parent.mkdir(exist_ok=True, parents=True)
- cache_file.touch()
- return self
-
- async def __aexit__(self, *_) -> None:
- self.cache["version"] = __version__
- yaml.save(self.appdata.cache_file, self.cache)
+ @contextlib.contextmanager
+ def __call__(self) -> Generator[Self]:
+ self.__resolve_paths()
+ self.database = Database(
+ self.appdata.db_file,
+ self.config.settings.runtime_options.ignore_history,
+ )
+ self.deduper = Czkawka.from_manager(self)
+ self.sorter = Sorter.from_manager(self)
+ self.client_manager = ClientManager(self)
+ with (
+ _cache_context(self.appdata.cache_file, self.cache),
+ _enter_context(REFRESH_RATE, self.config.global_settings.ui_options.refresh_rate),
+ _enter_context(TUI_DISABLED, self.cli_args.ui.is_disabled),
+ ):
+ yield self
def add_completed(self, media_item: MediaItem) -> None:
if media_item.is_segment:
@@ -99,21 +105,7 @@ class Manager:
def completed_downloads(self) -> list[MediaItem]:
return self._completed_downloads
- async def async_startup(self) -> None:
- self._log_config_settings()
- self.async_db_hash_startup()
-
- self.client_manager = ClientManager(self)
-
- def async_db_hash_startup(self) -> None:
- self.database = Database(
- self.appdata.db_file,
- self.config.settings.runtime_options.ignore_history,
- )
- self.deduper = Czkawka.from_manager(self)
- self.sorter = Sorter.from_manager(self)
-
- def _log_config_settings(self) -> None:
+ def log_config_settings(self) -> None:
auth = {site: all(credentials.values()) for site, credentials in self.config.auth.model_dump().items()}
config_settings = self.config.settings.model_copy()
config_settings.runtime_options.deep_scrape = self.config.deep_scrape
@@ -173,11 +165,11 @@ class Manager:
log_spacer()
with capture_logs() as stream:
- self._print_stats(stats)
+ self.__print_stats(stats)
return stream.getvalue()
- def _print_stats(self, stats: ScrapeStats) -> None:
+ def __print_stats(self, stats: ScrapeStats) -> None:
elapsed = timedelta(seconds=int(time.monotonic() - stats.start_time))
total_data_written = ByteSize(self.scrape_mapper.tui.downloads.bytes_downloaded).human_readable(decimal=True)
@@ -218,7 +210,10 @@ class Manager:
self.print_hashing_stats(self.hasher.stats)
self.print_dedupe_stats(self.deduper.stats)
self.print_sort_stats(self.sorter.stats)
- self.print_errors()
+ _log_errors(
+ tuple(self.scrape_mapper.tui.scrape_errors),
+ tuple(self.scrape_mapper.tui.download_errors),
+ )
def print_sort_stats(self, stats: SortStats):
log_spacer()
@@ -228,12 +223,6 @@ class Manager:
logger.info(f" Videos: {stats.videos:,}")
logger.info(f" Other files: {stats.others:,}")
- def print_errors(self) -> None:
- _log_errors(
- tuple(self.scrape_mapper.tui.scrape_errors),
- tuple(self.scrape_mapper.tui.download_errors),
- )
-
def print_hashing_stats(self, stats: HashingStats) -> None:
log_spacer()
logger.info("Checksum Stats:", extra={"color": "cyan"})
@@ -269,6 +258,21 @@ def _log_errors(scrape_errors: Sequence[UIError], download_errors: Sequence[UIEr
logger.info(f" {error.format(padding)}", extra={"color": "red"})
+def _cache_context(cache_file: Path, cache: dict[str, Any]) -> Generator[None]:
+ try:
+ cache.update(yaml.load(cache_file))
+ except FileNotFoundError:
+ cache_file.parent.mkdir(exist_ok=True, parents=True)
+ cache_file.touch()
+
+ try:
+ yield
+ finally:
+ cache["version"] = __version__
+ yaml.save(cache_file, cache)
+
+
@dataclasses.dataclass(slots=True, frozen=True, kw_only=True)
class AppData:
path: Path
diff --git a/tests/conftest.py b/tests/conftest.py
index 885b675cf..217f4f4fa 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -63,16 +63,13 @@ async def logs(caplog: pytest.LogCaptureFixture) -> pytest.LogCaptureFixture:
return caplog
[email protected](scope="function", name="manager")
-def post_startup_manager() -> Manager:
- manager = Manager()
- manager.resolve_paths()
- return manager
[email protected](scope="function")
+def manager() -> Generator[Manager]:
+ with Manager()() as manager:
+ yield manager
@pytest.fixture(scope="function")
async def running_manager(manager: Manager) -> AsyncGenerator[Manager]:
- await manager.async_startup()
-
async with manager.database:
yield manager
diff --git a/tests/test_cache.py b/tests/test_cache.py
index 0d0456920..15bcb9a96 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -1,20 +1,24 @@
+from pathlib import Path
+from typing import Any
+
from cyberdrop_dl import __version__, yaml
-from cyberdrop_dl.managers.manager import Manager
+from cyberdrop_dl.managers.manager import Manager, _cache_context
-def test_cache_file_is_not_saved_outside_ctx(manager: Manager) -> None:
+def test_cache_file_is_not_saved_outside_ctx() -> None:
+ manager = Manager()
cache_file = manager.appdata.cache_file
manager.cache["test"] = 1
-
assert manager.cache == {"test": 1}
assert not cache_file.exists()
-async def test_cache_file_is_saved_in_ctx(manager: Manager) -> None:
- cache_file = manager.appdata.cache_file
- async with manager:
- manager.cache["test"] = 1
- assert manager.cache == {"test": 1}
+def test_cache_file_is_saved_in_ctx(tmp_path: Path) -> None:
+ cache_file = tmp_path / "cache_file.txt"
+ cache: dict[str, Any] = {}
+ with _cache_context(cache_file, cache):
+ cache["test"] = 1
+ assert cache == {"test": 1}
assert cache_file.is_file()
assert cache_file.is_file()
diff --git a/tests/test_manager.py b/tests/test_manager.py
index b610a4efd..cc93efb23 100644
--- a/tests/test_manager.py
+++ b/tests/test_manager.py
@@ -4,11 +4,17 @@ from typing import TYPE_CHECKING, Any, TypeVar
import pytest
+from cyberdrop_dl.config import Config
+from cyberdrop_dl.database import Database
+from cyberdrop_dl.dedupe import Czkawka
+from cyberdrop_dl.managers.client_manager import ClientManager
+from cyberdrop_dl.managers.manager import Manager
+from cyberdrop_dl.progress import REFRESH_RATE
+from cyberdrop_dl.sorter import Sorter
+
if TYPE_CHECKING:
from pydantic import BaseModel
- from cyberdrop_dl.managers.manager import Manager
-
_M = TypeVar("_M", bound=BaseModel)
@@ -28,7 +34,7 @@ def test_args_logging_should_censor_webhook(
) -> None:
logs_model = running_manager.config.settings.logs
running_manager.config.settings.logs = update_model(logs_model, webhook=webhook)
- running_manager._log_config_settings()
+ running_manager.log_config_settings()
assert logs.messages
assert "Running cyberdrop-dl " in logs.text
assert webhook not in logs.text
@@ -36,3 +42,21 @@ def test_args_logging_should_censor_webhook(
_, _, webhook_text = webhook_line.partition(":")
webhook_url = webhook_text.strip().split(" ")[0].replace('"', "").strip()
assert output == webhook_url
+
+
+def test_manager_context() -> None:
+ config = Config.parse_args(["--refresh-rate", "40"])
+ manager = Manager(config=config)
+
+ for attr in ("database", "deduper", "sorter", "client_manager"):
+ with pytest.raises(AttributeError):
+ getattr(manager, attr)
+
+ assert REFRESH_RATE.get() == 10
+
+ with manager():
+ assert type(manager.database) is Database
+ assert type(manager.deduper) is Czkawka
+ assert type(manager.sorter) is Sorter
+ assert type(manager.client_manager) is ClientManager
+ assert REFRESH_RATE.get() == 40