Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: make manager a context manager (#1774)
NTFSvolume164744d425f2
diff --git a/cyberdrop_dl/__main__.py b/cyberdrop_dl/__main__.pyindex 1b2d32ade..2059c9d89 100644--- a/cyberdrop_dl/__main__.py+++ b/cyberdrop_dl/__main__.py@@ -15,7 +15,6 @@ from cyberdrop_dl.config import Configfrom cyberdrop_dl.logs import log_spacer, set_console_level, setup_console_logging, setup_file_loggingfrom cyberdrop_dl.managers.manager import AppData, Managerfrom cyberdrop_dl.models.types import HttpURL-from cyberdrop_dl.progress import REFRESH_RATE, TUI_DISABLEDfrom cyberdrop_dl.scrape_mapper import ScrapeMapperfrom cyberdrop_dl.updates import check_latest_pypifrom cyberdrop_dl.utils import apprise, check_partials_and_empty_folders@@ -28,10 +27,7 @@ async def _scrape(manager: Manager) -> None:manager.config.settings.logs.main_log,level=manager.config.settings.runtime_options.effective_log_level,):- await manager.async_startup()- REFRESH_RATE.set(manager.config.global_settings.ui_options.refresh_rate)- TUI_DISABLED.set(manager.cli_args.ui.is_disabled)-+ manager.log_config_settings()log_spacer()async with manager.database:log_spacer()@@ -81,12 +77,11 @@ async def _post_runtime(manager: Manager) -> None:def _main(manager: Manager) -> None:set_console_level(manager.config.settings.runtime_options.effective_console_log_level)- manager.resolve_paths()- if not manager.cli_args.download:- program_ui.run(manager)-try:- aio.run(_scrape(manager))+ with manager():+ if not manager.cli_args.download:+ program_ui.run(manager)+ aio.run(_scrape(manager))except KeyboardInterrupt:logger.info("Exiting (Ctrl + C) ...")diff --git a/cyberdrop_dl/hasher.py b/cyberdrop_dl/hasher.pyindex 7c4574527..eb50e815a 100644--- a/cyberdrop_dl/hasher.py+++ b/cyberdrop_dl/hasher.py@@ -46,13 +46,26 @@ def _compute_hash(file: Path, algorithm: Literal["xxh128", "md5", "sha256"]) ->async def hash_directory_scanner(manager: Manager, path: Path) -> None:- manager.async_db_hash_startup()async with manager.database:- stats = await manager.hasher.hash_directory(path)+ stats = await hash_directory(manager, path)manager.print_hashing_stats(stats)+async def hash_directory(manager: Manager, path: Path) -> HashingStats:+ hasher = Hasher(manager)+ hasher._tui = tui = HashingUI(path)+ with tui():+ if not await aio.is_dir(path):+ raise NotADirectoryError(None, path)++ async with asyncio.TaskGroup() as tg:+ async for file in aio.rglob(path, "*"):+ _ = tg.create_task(hasher.update_db_and_retrive_hash(file))++ return tui.stats++@dataclasses.dataclass(slots=True)class Hasher:manager: Manager@@ -86,24 +99,6 @@ class Hasher:file_path = self._cwd / filenamereturn await asyncio.to_thread(_compute_hash, file_path, hash_type)- async def hash_directory(self, path: Path) -> HashingStats:- path = Path(path)- tui = HashingUI(path)- old_tui = self._tui- with tui():- if not await aio.is_dir(path):- raise NotADirectoryError(None, path)-- try:- self._tui = tui- async with asyncio.TaskGroup() as tg:- async for file in aio.rglob(path, "*"):- _ = tg.create_task(self.update_db_and_retrive_hash(file))- finally:- self._tui = old_tui-- return tui.stats-async def hash_item(self, media_item: MediaItem) -> None:if media_item.is_segment:returndiff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex 1634beed1..c155cbeba 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -1,5 +1,6 @@from __future__ import annotations+import contextlibimport dataclassesimport loggingimport os@@ -17,14 +18,15 @@ from cyberdrop_dl.config import Configfrom cyberdrop_dl.database import Databasefrom cyberdrop_dl.dedupe import Czkawkafrom cyberdrop_dl.hasher import Hasher-from cyberdrop_dl.logs import capture_logs, log_spacer+from cyberdrop_dl.logs import _enter_context, capture_logs, log_spacerfrom cyberdrop_dl.managers.client_manager import ClientManagerfrom cyberdrop_dl.managers.logs import LogManager+from cyberdrop_dl.progress import REFRESH_RATE, TUI_DISABLEDfrom cyberdrop_dl.sorter import Sorterfrom cyberdrop_dl.utils import get_system_informationif TYPE_CHECKING:- from collections.abc import Sequence+ from collections.abc import Generator, Sequencefrom os import PathLikefrom cyberdrop_dl.progress.dedupe import DedupeStats@@ -71,24 +73,28 @@ class Manager:self._config = Config.from_manager(self)return self._config- def resolve_paths(self) -> None:+ def __resolve_paths(self) -> None:self.appdata.mkdirs()self.config.settings.resolve_paths()self.logs = LogManager.from_manager(self)self.logs.delete_old_logs()- async def __aenter__(self) -> Self:- cache_file = self.appdata.cache_file- try:- self.cache.update(yaml.load(cache_file))- except FileNotFoundError:- cache_file.parent.mkdir(exist_ok=True, parents=True)- cache_file.touch()- return self-- async def __aexit__(self, *_) -> None:- self.cache["version"] = __version__- yaml.save(self.appdata.cache_file, self.cache)+ @contextlib.contextmanager+ def __call__(self) -> Generator[Self]:+ self.__resolve_paths()+ self.database = Database(+ self.appdata.db_file,+ self.config.settings.runtime_options.ignore_history,+ )+ self.deduper = Czkawka.from_manager(self)+ self.sorter = Sorter.from_manager(self)+ self.client_manager = ClientManager(self)+ with (+ _cache_context(self.appdata.cache_file, self.cache),+ _enter_context(REFRESH_RATE, self.config.global_settings.ui_options.refresh_rate),+ _enter_context(TUI_DISABLED, self.cli_args.ui.is_disabled),+ ):+ yield selfdef add_completed(self, media_item: MediaItem) -> None:if media_item.is_segment:@@ -99,21 +105,7 @@ class Manager:def completed_downloads(self) -> list[MediaItem]:return self._completed_downloads- async def async_startup(self) -> None:- self._log_config_settings()- self.async_db_hash_startup()-- self.client_manager = ClientManager(self)-- def async_db_hash_startup(self) -> None:- self.database = Database(- self.appdata.db_file,- self.config.settings.runtime_options.ignore_history,- )- self.deduper = Czkawka.from_manager(self)- self.sorter = Sorter.from_manager(self)-- def _log_config_settings(self) -> None:+ def log_config_settings(self) -> None:auth = {site: all(credentials.values()) for site, credentials in self.config.auth.model_dump().items()}config_settings = self.config.settings.model_copy()config_settings.runtime_options.deep_scrape = self.config.deep_scrape@@ -173,11 +165,11 @@ class Manager:log_spacer()with capture_logs() as stream:- self._print_stats(stats)+ self.__print_stats(stats)return stream.getvalue()- def _print_stats(self, stats: ScrapeStats) -> None:+ def __print_stats(self, stats: ScrapeStats) -> None:elapsed = timedelta(seconds=int(time.monotonic() - stats.start_time))total_data_written = ByteSize(self.scrape_mapper.tui.downloads.bytes_downloaded).human_readable(decimal=True)@@ -218,7 +210,10 @@ class Manager:self.print_hashing_stats(self.hasher.stats)self.print_dedupe_stats(self.deduper.stats)self.print_sort_stats(self.sorter.stats)- self.print_errors()+ _log_errors(+ tuple(self.scrape_mapper.tui.scrape_errors),+ tuple(self.scrape_mapper.tui.download_errors),+ )def print_sort_stats(self, stats: SortStats):log_spacer()@@ -228,12 +223,6 @@ class Manager:logger.info(f" Videos: {stats.videos:,}")logger.info(f" Other files: {stats.others:,}")- def print_errors(self) -> None:- _log_errors(- tuple(self.scrape_mapper.tui.scrape_errors),- tuple(self.scrape_mapper.tui.download_errors),- )-def print_hashing_stats(self, stats: HashingStats) -> None:log_spacer()logger.info("Checksum Stats:", extra={"color": "cyan"})@@ -269,6 +258,21 @@ def _log_errors(scrape_errors: Sequence[UIError], download_errors: Sequence[UIErlogger.info(f" {error.format(padding)}", extra={"color": "red"})+def _cache_context(cache_file: Path, cache: dict[str, Any]) -> Generator[None]:+ try:+ cache.update(yaml.load(cache_file))+ except FileNotFoundError:+ cache_file.parent.mkdir(exist_ok=True, parents=True)+ cache_file.touch()++ try:+ yield+ finally:+ cache["version"] = __version__+ yaml.save(cache_file, cache)++@dataclasses.dataclass(slots=True, frozen=True, kw_only=True)class AppData:path: Pathdiff --git a/tests/conftest.py b/tests/conftest.pyindex 885b675cf..217f4f4fa 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -63,16 +63,13 @@ async def logs(caplog: pytest.LogCaptureFixture) -> pytest.LogCaptureFixture:return caplog[email protected](scope="function", name="manager")-def post_startup_manager() -> Manager:- manager = Manager()- manager.resolve_paths()- return manager[email protected](scope="function")+def manager() -> Generator[Manager]:+ with Manager()() as manager:+ yield manager@pytest.fixture(scope="function")async def running_manager(manager: Manager) -> AsyncGenerator[Manager]:- await manager.async_startup()-async with manager.database:yield managerdiff --git a/tests/test_cache.py b/tests/test_cache.pyindex 0d0456920..15bcb9a96 100644--- a/tests/test_cache.py+++ b/tests/test_cache.py@@ -1,20 +1,24 @@+from pathlib import Path+from typing import Any+from cyberdrop_dl import __version__, yaml-from cyberdrop_dl.managers.manager import Manager+from cyberdrop_dl.managers.manager import Manager, _cache_context-def test_cache_file_is_not_saved_outside_ctx(manager: Manager) -> None:+def test_cache_file_is_not_saved_outside_ctx() -> None:+ manager = Manager()cache_file = manager.appdata.cache_filemanager.cache["test"] = 1-assert manager.cache == {"test": 1}assert not cache_file.exists()-async def test_cache_file_is_saved_in_ctx(manager: Manager) -> None:- cache_file = manager.appdata.cache_file- async with manager:- manager.cache["test"] = 1- assert manager.cache == {"test": 1}+def test_cache_file_is_saved_in_ctx(tmp_path: Path) -> None:+ cache_file = tmp_path / "cache_file.txt"+ cache: dict[str, Any] = {}+ with _cache_context(cache_file, cache):+ cache["test"] = 1+ assert cache == {"test": 1}assert cache_file.is_file()assert cache_file.is_file()diff --git a/tests/test_manager.py b/tests/test_manager.pyindex b610a4efd..cc93efb23 100644--- a/tests/test_manager.py+++ b/tests/test_manager.py@@ -4,11 +4,17 @@ from typing import TYPE_CHECKING, Any, TypeVarimport pytest+from cyberdrop_dl.config import Config+from cyberdrop_dl.database import Database+from cyberdrop_dl.dedupe import Czkawka+from cyberdrop_dl.managers.client_manager import ClientManager+from cyberdrop_dl.managers.manager import Manager+from cyberdrop_dl.progress import REFRESH_RATE+from cyberdrop_dl.sorter import Sorter+if TYPE_CHECKING:from pydantic import BaseModel- from cyberdrop_dl.managers.manager import Manager-_M = TypeVar("_M", bound=BaseModel)@@ -28,7 +34,7 @@ def test_args_logging_should_censor_webhook() -> None:logs_model = running_manager.config.settings.logsrunning_manager.config.settings.logs = update_model(logs_model, webhook=webhook)- running_manager._log_config_settings()+ running_manager.log_config_settings()assert logs.messagesassert "Running cyberdrop-dl " in logs.textassert webhook not in logs.text@@ -36,3 +42,21 @@ def test_args_logging_should_censor_webhook(_, _, webhook_text = webhook_line.partition(":")webhook_url = webhook_text.strip().split(" ")[0].replace('"', "").strip()assert output == webhook_url+++def test_manager_context() -> None:+ config = Config.parse_args(["--refresh-rate", "40"])+ manager = Manager(config=config)++ for attr in ("database", "deduper", "sorter", "client_manager"):+ with pytest.raises(AttributeError):+ getattr(manager, attr)++ assert REFRESH_RATE.get() == 10++ with manager():+ assert type(manager.database) is Database+ assert type(manager.deduper) is Czkawka+ assert type(manager.sorter) is Sorter+ assert type(manager.client_manager) is ClientManager+ assert REFRESH_RATE.get() == 40