Skip to content

Bulk asynchronous downloader for multiple file hosts

tests: add tests for the manager (#982)

NTFSvolume9b64dd84d40f
diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.py
index a690a34b0..56a698f93 100644
--- a/cyberdrop_dl/clients/scraper_client.py
+++ b/cyberdrop_dl/clients/scraper_client.py
@@ -5,7 +5,6 @@ import functools
import inspect
from collections.abc import Callable
from contextlib import asynccontextmanager
-from dataclasses import Field, field
from datetime import datetime
from functools import wraps
from json import dumps as json_dumps
@@ -20,9 +19,10 @@ from bs4 import BeautifulSoup
import cyberdrop_dl.constants as constants
from cyberdrop_dl import env
+from cyberdrop_dl.constants import NOT_DEFINED
from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, InvalidContentTypeError, ScrapeError
from cyberdrop_dl.utils.logger import log_debug
-from cyberdrop_dl.utils.utilities import get_soup_no_error, sanitize_filename
+from cyberdrop_dl.utils.utilities import close_if_defined, get_soup_no_error, sanitize_filename
if TYPE_CHECKING:
from collections.abc import Callable, Coroutine
@@ -108,10 +108,10 @@ class ScraperClient:
# folder len + date_prefix len + 10 [suffix (.html) + 1 OS separator + 4 (padding)]
min_html_file_path_len = len(str(self._pages_folder)) + len(constants.STARTUP_TIME_STR) + 10
self._max_html_stem_len = 245 - min_html_file_path_len
- self._session: CachedSession = field(init=False)
- self._curl_session: AsyncSession = field(init=False)
+ self._session: CachedSession = NOT_DEFINED
+ self._curl_session: AsyncSession = NOT_DEFINED
- def startup(self):
+ def startup(self) -> None:
add_request_log_hooks(self._trace_configs)
self._session = CachedSession(
headers=self._headers,
@@ -135,9 +135,8 @@ class ScraperClient:
)
async def close(self) -> None:
- await self._session.close()
- if not isinstance(self._curl_session, Field):
- await self._curl_session.close()
+ await close_if_defined(self._session)
+ await close_if_defined(self._curl_session)
def is_ddos(self, soup: BeautifulSoup) -> bool:
return self.client_manager.check_ddos_guard(soup) or self.client_manager.check_cloudflare(soup)
diff --git a/cyberdrop_dl/config_definitions/config_settings.py b/cyberdrop_dl/config_definitions/config_settings.py
index 0bf68ca5d..cf90a20a0 100644
--- a/cyberdrop_dl/config_definitions/config_settings.py
+++ b/cyberdrop_dl/config_definitions/config_settings.py
@@ -5,7 +5,7 @@ from pathlib import Path
from pydantic import BaseModel, ByteSize, Field, NonNegativeInt, PositiveInt, field_serializer, field_validator
-from cyberdrop_dl.constants import APP_STORAGE, BROWSERS, DOWNLOAD_STORAGE
+from cyberdrop_dl.constants import BROWSERS, DEFAULT_APP_STORAGE, DEFAULT_DOWNLOAD_STORAGE
from cyberdrop_dl.data_structures.hash import Hashing
from cyberdrop_dl.data_structures.supported_domains import SUPPORTED_SITES_DOMAINS
from cyberdrop_dl.types import (
@@ -43,9 +43,9 @@ class DownloadOptions(BaseModel):
class Files(AliasModel):
- download_folder: Path = Field(validation_alias="d", default=DOWNLOAD_STORAGE)
+ download_folder: Path = Field(validation_alias="d", default=DEFAULT_DOWNLOAD_STORAGE)
dump_json: bool = Field(default=False, validation_alias="j")
- input_file: Path = Field(validation_alias="i", default=APP_STORAGE / "Configs" / "{config}" / "URLs.txt")
+ input_file: Path = Field(validation_alias="i", default=DEFAULT_APP_STORAGE / "Configs" / "{config}" / "URLs.txt")
save_pages_html: bool = False
@@ -56,7 +56,7 @@ class Logs(AliasModel):
last_forum_post: LogPath = Field(
default=Path("Last_Scraped_Forum_Posts.csv"), validation_alias="last_forum_post_filename"
)
- log_folder: Path = APP_STORAGE / "Configs" / "{config}" / "Logs"
+ log_folder: Path = DEFAULT_APP_STORAGE / "Configs" / "{config}" / "Logs"
log_line_width: PositiveInt = Field(default=240, ge=50)
logs_expire_after: timedelta | None = None
main_log: MainLogPath = Field(default=Path("downloader.log"), validation_alias="main_log_filename")
@@ -151,7 +151,7 @@ class RuntimeOptions(BaseModel):
class Sorting(BaseModel):
scan_folder: PathOrNone = None
sort_downloads: bool = False
- sort_folder: Path = DOWNLOAD_STORAGE / "Cyberdrop-DL Sorted Downloads"
+ sort_folder: Path = DEFAULT_DOWNLOAD_STORAGE / "Cyberdrop-DL Sorted Downloads"
sort_incrementer_format: NonEmptyStr = " ({i})"
sorted_audio: NonEmptyStrOrNone = "{sort_dir}/{base_dir}/Audio/{filename}{ext}"
sorted_image: NonEmptyStrOrNone = "{sort_dir}/{base_dir}/Images/{filename}{ext}"
diff --git a/cyberdrop_dl/constants.py b/cyberdrop_dl/constants.py
index 536852cc3..370f45661 100644
--- a/cyberdrop_dl/constants.py
+++ b/cyberdrop_dl/constants.py
@@ -1,4 +1,5 @@
import re
+from dataclasses import field
from datetime import UTC, datetime
from enum import Enum, IntEnum, StrEnum, auto
from pathlib import Path
@@ -52,9 +53,10 @@ class CustomHTTPStatus(IntEnum):
BLOCKED_DOMAINS = ("facebook", "instagram", "fbcdn")
-APP_STORAGE = Path("./AppData")
-DOWNLOAD_STORAGE = Path("./Downloads")
+DEFAULT_APP_STORAGE = Path("./AppData")
+DEFAULT_DOWNLOAD_STORAGE = Path("./Downloads")
RESERVED_CONFIG_NAMES = ["all", "default"]
+NOT_DEFINED = field(init=False)
class HashType(StrEnum):
diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.py
index 93def7016..24302635c 100644
--- a/cyberdrop_dl/director.py
+++ b/cyberdrop_dl/director.py
@@ -312,7 +312,7 @@ def _startup_logging(*, first_time_setup: bool = False) -> Generator:
sys.exit(exit_code)
-def _setup_manager() -> Manager:
+def _setup_manager(args: tuple[str, ...] | None = None) -> Manager:
"""Starts the program and returns the manager.
This will also run the UI for the program
@@ -320,7 +320,7 @@ def _setup_manager() -> Manager:
"""
with _startup_logging(first_time_setup=True):
try:
- manager = Manager()
+ manager = Manager(args)
manager.startup()
if manager.parsed_args.cli_only_args.multiconfig:
startup_logger.info("validating all configs, please wait...")
@@ -346,12 +346,12 @@ def _setup_manager() -> Manager:
class Director:
"""Creates a manager and runs it"""
- def __init__(self) -> None:
+ def __init__(self, args: tuple[str, ...] | None = None) -> None:
if os.name == "nt":
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
- self.manager = _setup_manager()
+ self.manager = _setup_manager(args)
# For future use with textual
# fullscreen = f = m.parsed_args.cli_only_args.fullscreen_ui
# self.use_textual = m.parsed_args.cli_only_args.textual_ui and fullscreen
diff --git a/cyberdrop_dl/main.py b/cyberdrop_dl/main.py
index b76217d57..fa8e5bd1d 100644
--- a/cyberdrop_dl/main.py
+++ b/cyberdrop_dl/main.py
@@ -15,11 +15,11 @@ def actual_main() -> None:
sys.exit(run())
-def run() -> str | int:
+def run(args: tuple[str, ...] | None = None) -> str | int:
try:
from cyberdrop_dl.director import Director
- director = Director()
+ director = Director(args)
return director.run()
except KeyboardInterrupt:
return "\nKeyboardInterrupt"
diff --git a/cyberdrop_dl/managers/config_manager.py b/cyberdrop_dl/managers/config_manager.py
index 5e0b7cc0f..aa0b75f2d 100644
--- a/cyberdrop_dl/managers/config_manager.py
+++ b/cyberdrop_dl/managers/config_manager.py
@@ -114,13 +114,16 @@ class ConfigManager:
if posible_fields == set_fields and not needs_update and self.pydantic_config:
return
else:
- from cyberdrop_dl import constants
-
self.settings_data = ConfigSettings()
- self.settings_data.files.input_file = constants.APP_STORAGE / "Configs" / self.loaded_config / "URLs.txt"
- self.settings_data.files.download_folder = constants.DOWNLOAD_STORAGE / "Cyberdrop-DL Downloads"
- self.settings_data.logs.log_folder = constants.APP_STORAGE / "Configs" / self.loaded_config / "Logs"
- self.settings_data.sorting.sort_folder = constants.DOWNLOAD_STORAGE / "Cyberdrop-DL Sorted Downloads"
+ self.settings_data.files.input_file = (
+ self.manager.path_manager.appdata / "Configs" / self.loaded_config / "URLs.txt"
+ )
+ downloads = self.manager.path_manager.cwd / "Downloads"
+ self.settings_data.sorting.sort_folder = downloads / "Cyberdrop-DL Sorted Downloads"
+ self.settings_data.files.download_folder = downloads / "Cyberdrop-DL Downloads"
+ self.settings_data.logs.log_folder = (
+ self.manager.path_manager.appdata / "Configs" / self.loaded_config / "Logs"
+ )
yaml.save(self.settings, self.settings_data)
diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
index fa53b3540..e798375c2 100644
--- a/cyberdrop_dl/managers/manager.py
+++ b/cyberdrop_dl/managers/manager.py
@@ -1,14 +1,14 @@
from __future__ import annotations
import asyncio
-import inspect
import json
from dataclasses import Field, field
from time import perf_counter
-from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Protocol, TypeVar
+from typing import TYPE_CHECKING, Literal, NamedTuple, TypeVar
+
+from pydantic import BaseModel
from cyberdrop_dl import __version__, constants
-from cyberdrop_dl.config_definitions import ConfigSettings, GlobalSettings
from cyberdrop_dl.managers.cache_manager import CacheManager
from cyberdrop_dl.managers.client_manager import ClientManager
from cyberdrop_dl.managers.config_manager import ConfigManager
@@ -22,38 +22,18 @@ from cyberdrop_dl.managers.progress_manager import ProgressManager
from cyberdrop_dl.managers.realdebrid_manager import RealDebridManager
from cyberdrop_dl.managers.storage_manager import StorageManager
from cyberdrop_dl.ui.textual import TextualUI
-from cyberdrop_dl.utils.args import ParsedArgs
+from cyberdrop_dl.utils.args import ParsedArgs, parse_args
from cyberdrop_dl.utils.ffmpeg import FFmpeg, get_ffmpeg_version
from cyberdrop_dl.utils.logger import QueuedLogger, log
from cyberdrop_dl.utils.transfer import transfer_v5_db_to_v6
-from cyberdrop_dl.utils.utilities import get_system_information
+from cyberdrop_dl.utils.utilities import close_if_defined, get_system_information
if TYPE_CHECKING:
import queue
from asyncio import TaskGroup
from cyberdrop_dl.scraper.scrape_mapper import ScrapeMapper
-
-
-class HasClose(Protocol):
- def close(self): ...
-
-
-class HasAsyncClose(Protocol):
- async def close(self): ...
-
-
-C = TypeVar("C", bound=HasAsyncClose | HasClose)
-
-
-def unset() -> Any:
- return field(init=False)
-
-
-async def close_if_set(obj: C) -> C:
- if not isinstance(obj, Field):
- await obj.close() if inspect.iscoroutinefunction(obj.close) else obj.close()
- return unset()
+ from cyberdrop_dl.types import AnyDict
class AsyncioEvents(NamedTuple):
@@ -62,7 +42,7 @@ class AsyncioEvents(NamedTuple):
class Manager:
- def __init__(self) -> None:
+ def __init__(self, args: tuple[str, ...] | None = None) -> None:
self.parsed_args: ParsedArgs = field(init=False)
self.cache_manager: CacheManager = CacheManager(self)
self.path_manager: PathManager = field(init=False)
@@ -96,6 +76,7 @@ class Manager:
self.multiconfig: bool = False
self.loggers: dict[str, QueuedLogger] = {}
self.states = AsyncioEvents()
+ self.args = args
@property
def config(self):
@@ -120,7 +101,7 @@ class Manager:
"""Startup process for the manager."""
if isinstance(self.parsed_args, Field):
- self.parsed_args = ParsedArgs.parse_args()
+ self.parsed_args = parse_args(self.args)
self.path_manager = PathManager(self)
self.path_manager.pre_startup()
@@ -215,62 +196,21 @@ class Manager:
cli_ignore_options = self.parsed_args.config_settings.ignore_options
config_skip_hosts = self.config_manager.settings_data.ignore_options.skip_hosts
config_only_hosts = self.config_manager.settings_data.ignore_options.only_hosts
- exclude = {"+", "-"}
-
- def add(config_list: list[str], cli_list: list[str]) -> list[str]:
- new_list_as_set = set(config_list + cli_list)
- return sorted(new_list_as_set - exclude)
-
- def remove(config_list: list[str], cli_list: list[str]) -> list[str]:
- new_list_as_set = set(config_list) - set(cli_list)
- return sorted(new_list_as_set - exclude)
- def add_or_remove(config_list: list[str], cli_list: list[str]) -> list[str]:
- if cli_list:
- if cli_list[0] == "+":
- return add(config_list, cli_list)
- if cli_list[0] == "-":
- return remove(config_list, cli_list)
- return cli_list
-
- cli_ignore_options.skip_hosts = add_or_remove(config_skip_hosts, cli_ignore_options.skip_hosts)
- cli_ignore_options.only_hosts = add_or_remove(config_only_hosts, cli_ignore_options.only_hosts)
+ cli_ignore_options.skip_hosts = add_or_remove_lists(config_skip_hosts, cli_ignore_options.skip_hosts)
+ cli_ignore_options.only_hosts = add_or_remove_lists(config_only_hosts, cli_ignore_options.only_hosts)
def args_consolidation(self) -> None:
"""Consolidates runtime arguments with config values."""
self.process_additive_args()
- cli_config_settings = self.parsed_args.config_settings.model_dump(exclude_unset=True)
- cli_global_settings = self.parsed_args.global_settings.model_dump(exclude_unset=True)
-
- current_config_settings = self.config_manager.settings_data.model_dump()
- current_global_settings = self.config_manager.global_settings_data.model_dump()
-
- merged_config_settings = self.merge_dicts(current_config_settings, cli_config_settings)
- merged_global_settings = self.merge_dicts(current_global_settings, cli_global_settings)
-
- updated_config_settings = ConfigSettings.model_validate(merged_config_settings)
- updated_global_settings = GlobalSettings.model_validate(merged_global_settings)
- self.config_manager.settings_data = updated_config_settings
- self.config_manager.global_settings_data = updated_global_settings
- self.config_manager.deep_scrape = (
- self.parsed_args.config_settings.runtime_options.deep_scrape or self.config_manager.deep_scrape
- )
+ conf = merge_models(self.config_manager.settings_data, self.parsed_args.config_settings)
+ global_conf = merge_models(self.config_manager.global_settings_data, self.parsed_args.global_settings)
+ deep_scrape = self.parsed_args.config_settings.runtime_options.deep_scrape or self.config_manager.deep_scrape
- def merge_dicts(self, dict1: dict, dict2: dict):
- for key, val in dict1.items():
- if isinstance(val, dict):
- if key in dict2 and isinstance(dict2[key], dict):
- self.merge_dicts(dict1[key], dict2[key])
- else:
- if key in dict2:
- dict1[key] = dict2[key]
-
- for key, val in dict2.items():
- if key not in dict1:
- dict1[key] = val
-
- return dict1
+ self.config_manager.settings_data = conf
+ self.config_manager.global_settings_data = global_conf
+ self.config_manager.deep_scrape = deep_scrape
def args_logging(self) -> None:
"""Logs the runtime arguments."""
@@ -303,8 +243,8 @@ class Manager:
async def async_db_close(self) -> None:
"Partial shutdown for managers used for hash directory scanner"
- self.db_manager = await close_if_set(self.db_manager)
- self.hash_manager = unset()
+ self.db_manager = await close_if_defined(self.db_manager)
+ self.hash_manager = constants.NOT_DEFINED
self.progress_manager.hash_progress.reset()
async def close(self) -> None:
@@ -313,9 +253,9 @@ class Manager:
await self.async_db_close()
- self.client_manager = await close_if_set(self.client_manager)
- self.storage_manager = await close_if_set(self.storage_manager)
- self.cache_manager = await close_if_set(self.cache_manager)
+ self.client_manager = await close_if_defined(self.client_manager)
+ self.storage_manager = await close_if_defined(self.storage_manager)
+ self.cache_manager = await close_if_defined(self.cache_manager)
while self.loggers:
_, queued_logger = self.loggers.popitem()
@@ -334,3 +274,49 @@ class Manager:
rewrite constants after config/arg manager have loaded
"""
constants.DISABLE_CACHE = self.parsed_args.cli_only_args.disable_cache
+
+
+def add_or_remove_lists(old_list: list[str], new_list: list[str]) -> list[str]:
+ exclude = {"+", "-"}
+
+ def add(config_list: list[str], cli_list: list[str]) -> list[str]:
+ new_list_as_set = set(config_list + cli_list)
+ return sorted(new_list_as_set - exclude)
+
+ def remove(config_list: list[str], cli_list: list[str]) -> list[str]:
+ new_list_as_set = set(config_list) - set(cli_list)
+ return sorted(new_list_as_set - exclude)
+
+ if new_list:
+ if new_list[0] == "+":
+ return add(old_list, new_list)
+ if new_list[0] == "-":
+ return remove(old_list, new_list)
+ return new_list
+
+
+def merge_dicts(dict1: AnyDict, dict2: AnyDict) -> AnyDict:
+ for key, val in dict1.items():
+ if isinstance(val, dict):
+ if key in dict2 and isinstance(dict2[key], dict):
+ merge_dicts(dict1[key], dict2[key])
+ else:
+ if key in dict2:
+ dict1[key] = dict2[key]
+
+ for key, val in dict2.items():
+ if key not in dict1:
+ dict1[key] = val
+
+ return dict1
+
+
+M = TypeVar("M", bound=BaseModel)
+
+
+def merge_models(default: M, new: M) -> M:
+ default_dict = default.model_dump()
+ new_dict = new.model_dump(exclude_unset=True)
+
+ updated_dict = merge_dicts(default_dict, new_dict)
+ return default.model_validate(updated_dict)
diff --git a/cyberdrop_dl/managers/path_manager.py b/cyberdrop_dl/managers/path_manager.py
index 4a30912e4..607eac1a4 100644
--- a/cyberdrop_dl/managers/path_manager.py
+++ b/cyberdrop_dl/managers/path_manager.py
@@ -1,11 +1,11 @@
from __future__ import annotations
-from dataclasses import field
+from dataclasses import Field, field
from datetime import datetime
from pathlib import Path
from typing import TYPE_CHECKING
-from cyberdrop_dl import constants, env
+from cyberdrop_dl import env
from cyberdrop_dl.utils.utilities import purge_dir_tree
if TYPE_CHECKING:
@@ -13,12 +13,6 @@ if TYPE_CHECKING:
from cyberdrop_dl.managers.manager import Manager
-if env.RUNNING_IN_IDE and Path.cwd().name == "cyberdrop_dl":
- """This is for testing purposes only"""
- constants.APP_STORAGE = Path("../AppData")
- constants.DOWNLOAD_STORAGE = Path("../Downloads")
-
-
class PathManager:
def __init__(self, manager: Manager) -> None:
self.manager = manager
@@ -55,16 +49,30 @@ class PathManager:
"download_error_urls",
"scrape_error_urls",
]
+ self._appdata: Path = field(init=False)
- def pre_startup(self) -> None:
- if self.manager.parsed_args.cli_only_args.appdata_folder:
- constants.APP_STORAGE = self.manager.parsed_args.cli_only_args.appdata_folder / "AppData"
+ @property
+ def cwd(self) -> Path:
+ if env.RUNNING_IN_IDE and Path.cwd().name == "cyberdrop_dl":
+ # This is for testing purposes only"""
+ return Path("..").resolve()
+ return Path().resolve()
- constants.APP_STORAGE = constants.APP_STORAGE.resolve()
+ @property
+ def appdata(self) -> Path:
+ if isinstance(self._appdata, Field):
+ if self.manager.parsed_args.cli_only_args.appdata_folder:
+ path = self.manager.parsed_args.cli_only_args.appdata_folder / "AppData"
+ self._appdata = self.cwd / path
+ else:
+ self._appdata = self.cwd / "AppData"
+
+ return self._appdata
- self.cache_folder = constants.APP_STORAGE / "Cache"
- self.config_folder = constants.APP_STORAGE / "Configs"
- self.cookies_dir = constants.APP_STORAGE / "Cookies"
+ def pre_startup(self) -> None:
+ self.cache_folder = self.appdata / "Cache"
+ self.config_folder = self.appdata / "Configs"
+ self.cookies_dir = self.appdata / "Cookies"
self.cache_db = self.cache_folder / "request_cache.db"
self.cache_folder.mkdir(parents=True, exist_ok=True)
@@ -78,7 +86,8 @@ class PathManager:
current_config = self.manager.config_manager.loaded_config
def replace(path: Path) -> Path:
- return Path(str(path).replace("{config}", current_config)).resolve()
+ path_w_config = str(path).replace("{config}", current_config)
+ return self.cwd.joinpath(Path(path_w_config))
self.download_folder = replace(settings_data.files.download_folder)
self.sorted_folder = replace(settings_data.sorting.sort_folder)
diff --git a/cyberdrop_dl/utils/args.py b/cyberdrop_dl/utils/args.py
index 261bec984..141a93388 100644
--- a/cyberdrop_dl/utils/args.py
+++ b/cyberdrop_dl/utils/args.py
@@ -3,7 +3,7 @@ import time
import warnings
from argparse import SUPPRESS, ArgumentParser, BooleanOptionalAction, RawDescriptionHelpFormatter
from argparse import _ArgumentGroup as ArgGroup
-from collections.abc import Iterable
+from collections.abc import Iterable, Sequence
from datetime import date
from enum import StrEnum, auto
from pathlib import Path
@@ -158,11 +158,6 @@ class ParsedArgs(AliasModel):
sys.exit(1)
time.sleep(WARNING_TIMEOUT)
- @staticmethod
- def parse_args() -> "ParsedArgs":
- """Parses the command line arguments passed into the program. Returns an instance of `ParsedArgs`"""
- return parse_args()
-
def prepare_warnings(self) -> set[str]:
warnings_to_emit = set()
@@ -267,17 +262,17 @@ def make_parser() -> tuple[ArgumentParser, dict[str, list[ArgGroup]]]:
return parser, group_lists
-def get_parsed_args_dict() -> dict[str, AnyDict]:
+def get_parsed_args_dict(args: Sequence[str] | None = None) -> dict[str, AnyDict]:
parser, group_lists = make_parser()
- args = parser.parse_intermixed_args()
+ namespace = parser.parse_intermixed_args(args)
parsed_args: dict[str, AnyDict] = {}
for name, groups in group_lists.items():
parsed_args[name] = {}
for group in groups:
group_dict = {
- arg.dest: getattr(args, arg.dest)
+ arg.dest: getattr(namespace, arg.dest)
for arg in group._group_actions
- if getattr(args, arg.dest, None) is not None
+ if getattr(namespace, arg.dest, None) is not None
}
if group_dict:
assert group.title
@@ -289,9 +284,9 @@ def get_parsed_args_dict() -> dict[str, AnyDict]:
return parsed_args
-def parse_args() -> ParsedArgs:
+def parse_args(args: Sequence[str] | None = None) -> ParsedArgs:
"""Parses the command line arguments passed into the program."""
- parsed_args_dict = get_parsed_args_dict()
+ parsed_args_dict = get_parsed_args_dict(args)
try:
parsed_args_model = ParsedArgs.model_validate(parsed_args_dict)
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index e444cdebc..2ee7648f0 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import contextlib
+import inspect
import json
import os
import platform
@@ -10,7 +11,7 @@ import shutil
import subprocess
import sys
import unicodedata
-from dataclasses import dataclass, fields
+from dataclasses import Field, dataclass, fields
from functools import lru_cache, partial, wraps
from pathlib import Path
from stat import S_ISREG
@@ -546,6 +547,23 @@ def get_size_or_none(path: Path) -> int | None:
return None
+class HasClose(Protocol):
+ def close(self): ...
+
+
+class HasAsyncClose(Protocol):
+ async def close(self): ...
+
+
+C = TypeVar("C", bound=HasAsyncClose | HasClose)
+
+
+async def close_if_defined(obj: C) -> C:
+ if not isinstance(obj, Field):
+ await obj.close() if inspect.iscoroutinefunction(obj.close) else obj.close()
+ return constants.NOT_DEFINED
+
+
def with_suffix_encoded(url: AnyURL, suffix: str) -> AnyURL:
name = Path(url.raw_name).with_suffix(suffix)
return url.parent.joinpath(str(name), encoded=True).with_query(url.query).with_fragment(url.fragment)
diff --git a/pyproject.toml b/pyproject.toml
index ac1f1b8c1..3694bad13 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -48,6 +48,20 @@ Changelog = "https://github.com/jbsparrow/CyberDropDownloader/blob/master/CHANGE
Documentation = "https://script-ware.gitbook.io/cyberdrop-dl/"
Repository = "https://github.com/jbsparrow/CyberDropDownloader"
+[tool.coverage.report]
+exclude_also = [
+ '@(abc\.)?abstractmethod',
+ 'class .*\bProtocol\):',
+ 'if __name__ == .__main__.:',
+ 'if env.DEBUG_VAR:',
+ 'if TYPE_CHECKING:'
+]
+
+[tool.coverage.run]
+omit = [
+ "tests/*"
+]
+
[tool.poetry]
include = ["CHANGELOG.md"]
packages = [{include = "cyberdrop_dl"}]
@@ -67,7 +81,8 @@ pytest-cov = ">=6.0.0,<7"
pytest-mock = "*"
[tool.pytest.ini_options]
-asyncio_default_fixture_loop_scope = "module"
+addopts = ["-s"]
+asyncio_default_fixture_loop_scope = "function"
asyncio_mode = "auto"
minversion = "8.3"
testpaths = ["tests"]
diff --git a/tests/conftest.py b/tests/conftest.py
index ee56eb9e3..42203bceb 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,8 +1,21 @@
+from __future__ import annotations
+
from pathlib import Path
+from typing import TYPE_CHECKING
import pytest
+if TYPE_CHECKING:
+ from pathlib import Path
+
[email protected](autouse=True)
-def test_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+def tmp_cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
monkeypatch.chdir(tmp_path)
+ return tmp_path
+
+
+async def logs(caplog: pytest.LogCaptureFixture) -> pytest.LogCaptureFixture:
+ caplog.set_level(10)
+ return caplog
diff --git a/tests/test_apprise.py b/tests/test_apprise.py
index c083b0461..fe815b27f 100644
--- a/tests/test_apprise.py
+++ b/tests/test_apprise.py
@@ -20,9 +20,12 @@ FAKE_MANAGER.cache_manager = FakeCacheManager(FAKE_MANAGER)
FAKE_MANAGER.config_manager = ConfigManager(FAKE_MANAGER)
URL_FAIL = "mailto://test_user:[email protected]"
-URL_SUCCESS = os.environ.get("APPRISE_TEST_EMAIL_URL", "")
+APPRISE_ENV_VAR = "APPRISE_TEST_EMAIL_URL"
+URL_SUCCESS = os.environ.get(APPRISE_ENV_VAR, "")
URL_SUCCESS_ATTACH_LOGS = f"attach_logs={URL_SUCCESS}"
+pytestmark = pytest.mark.skipif(not URL_SUCCESS, reason=f"{APPRISE_ENV_VAR} environment variable is not set")
+
@dataclass
class AppriseTestCase:
@@ -34,7 +37,7 @@ class AppriseTestCase:
name: str = "Testing Apprise"
-def test_get_apprise_urls():
+def test_get_apprise_urls() -> None:
with pytest.raises(ValueError):
apprise.get_apprise_urls()
@@ -77,8 +80,7 @@ def test_get_apprise_urls():
assert got == expected, f"Parsed URL: {got.raw_url}, Expected URL: {expected.raw_url}"
-async def send_notification(test_case: AppriseTestCase):
- assert URL_SUCCESS, "Apprise URL should be set on enviroment"
+async def send_notification(test_case: AppriseTestCase) -> None:
FAKE_MANAGER.config_manager.apprise_urls = []
if test_case.urls and any(test_case.urls):
FAKE_MANAGER.config_manager.apprise_urls = apprise.get_apprise_urls(urls=test_case.urls)
@@ -98,7 +100,7 @@ async def send_notification(test_case: AppriseTestCase):
assert not any(match in logs_as_str for match in test_case.exclude), "Logs should not match exclude pattern"
-async def test_failed_notification():
+async def test_failed_notification() -> None:
test_case = AppriseTestCase(
urls=[URL_FAIL],
result=NotificationResult.FAILED,
@@ -107,7 +109,7 @@ async def test_failed_notification():
await send_notification(test_case)
-async def test_successful_notification():
+async def test_successful_notification() -> None:
test_case = AppriseTestCase(
urls=[URL_SUCCESS],
result=NotificationResult.SUCCESS,
@@ -117,7 +119,7 @@ async def test_successful_notification():
await send_notification(test_case)
-async def test_successful_notification_with_successful_attachment():
+async def test_successful_notification_with_successful_attachment() -> None:
test_case = AppriseTestCase(
urls=[URL_SUCCESS_ATTACH_LOGS],
result=NotificationResult.SUCCESS,
@@ -127,7 +129,7 @@ async def test_successful_notification_with_successful_attachment():
await send_notification(test_case)
-async def test_successful_notification_with_failed_attachment():
+async def test_successful_notification_with_failed_attachment() -> None:
test_case = AppriseTestCase(
urls=[URL_SUCCESS_ATTACH_LOGS],
result=NotificationResult.SUCCESS,
@@ -138,7 +140,7 @@ async def test_successful_notification_with_failed_attachment():
await send_notification(test_case)
-async def test_none_notification():
+async def test_none_notification() -> None:
test_case = AppriseTestCase(
urls=[""],
result=NotificationResult.NONE,
@@ -148,7 +150,7 @@ async def test_none_notification():
await send_notification(test_case)
-async def test_partial_notification():
+async def test_partial_notification() -> None:
test_case = AppriseTestCase(
urls=[URL_SUCCESS_ATTACH_LOGS, URL_FAIL],
result=NotificationResult.PARTIAL,
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 5a8e5c568..94e00f825 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -1,17 +1,22 @@
-import sys
+from pathlib import Path
import pytest
from cyberdrop_dl.main import run
-def test_help(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
- # This is just to test that cyberdrop is able to run in the current python version
- monkeypatch.setattr(sys, "argv", ["pytest", "--help"])
+ "command, text",
+ [
+ ("--help", "Bulk asynchronous downloader for multiple file hosts"),
+ ("--show-supported-sites", "for a details about supported paths"),
+ ],
+)
+def test_command_by_console_output(tmp_cwd: Path, capsys: pytest.CaptureFixture[str], command: str, text: str) -> None:
try:
- run()
+ run(tuple(command.split()))
except SystemExit:
pass
captured = capsys.readouterr()
output = captured.out
- assert "Bulk asynchronous downloader for multiple file hosts" in output
+ assert text in output
diff --git a/tests/test_manager.py b/tests/test_manager.py
new file mode 100644
index 000000000..7ea111c0c
--- /dev/null
+++ b/tests/test_manager.py
@@ -0,0 +1,137 @@
+from dataclasses import Field
+from pathlib import Path
+from typing import Any, TypeVar
+
+import pytest
+from pydantic import BaseModel
+
+from cyberdrop_dl.managers.manager import Manager, merge_dicts
+
+M = TypeVar("M", bound=BaseModel)
+
+
+def update_model(model: M, **kwargs: Any) -> M:
+ return model.model_validate(model.model_dump() | kwargs)
+
+
[email protected](scope="function", name="manager")
+def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Manager:
+ appdata = str(tmp_path)
+ downloads = str(tmp_path / "Downloads")
+ monkeypatch.chdir(tmp_path)
+ bare_manager = Manager(("--appdata-folder", appdata, "-d", downloads))
+ bare_manager.startup()
+ bare_manager.path_manager.startup()
+ bare_manager.log_manager.startup()
+ return bare_manager
+
+
+class TestMergeDicts:
+ def test_overwrite(self) -> None:
+ dict1 = {"a": 1, "b": 2}
+ dict2 = {"b": 3, "c": 4}
+ expected = {"a": 1, "b": 3, "c": 4}
+ assert merge_dicts(dict1, dict2) == expected
+
+ def test_merge_with_new_keys(self) -> None:
+ dict1 = {"a": 1}
+ dict2 = {"b": 2, "c": 3}
+ expected = {"a": 1, "b": 2, "c": 3}
+ assert merge_dicts(dict1, dict2) == expected
+
+ def test_merge_recursive(self) -> None:
+ dict1 = {
+ "a": {
+ "b": 1,
+ "c": 2,
+ },
+ "d": 3,
+ }
+ dict2 = {
+ "a": {
+ "b": 4,
+ "e": 4,
+ "f": {
+ "g": 5,
+ "h": 6,
+ },
+ },
+ "i": 7,
+ }
+ expected = {
+ "a": {
+ "b": 4,
+ "c": 2,
+ "e": 4,
+ "f": {
+ "g": 5,
+ "h": 6,
+ },
+ },
+ "d": 3,
+ "i": 7,
+ }
+ assert merge_dicts(dict1, dict2) == expected
+
+ def test_merge_with_empty_dict1(self) -> None:
+ dict1 = {}
+ dict2 = {"a": 1, "b": 2}
+ expected = {"a": 1, "b": 2}
+ assert merge_dicts(dict1, dict2) == expected
+
+ def test_merge_with_empty_dict2(self) -> None:
+ dict1 = {"a": 1, "b": 2}
+ dict2 = {}
+ expected = {"a": 1, "b": 2}
+ assert merge_dicts(dict1, dict2) == expected
+
+ def test_merge_with_both_empty_dicts(self) -> None:
+ dict1 = {}
+ dict2 = {}
+ expected = {}
+ assert merge_dicts(dict1, dict2) == expected
+
+ def test_dict_overwrites_value(self) -> None:
+ dict1 = {"a": 1}
+ dict2 = {"a": {"x": 1}}
+ expected = {"a": {"x": 1}}
+ assert merge_dicts(dict1, dict2) == expected
+
+ def test_value_should_not_overwrite_dict(self) -> None:
+ dict1 = {"a": {"x": 1}}
+ dict2 = {"a": 1}
+ expected = {"a": {"x": 1}}
+ assert merge_dicts(dict1, dict2) == expected
+
+
+ "webhook, output",
+ [
+ ("https://example.com", "**********"),
+ ("attach_logs=https://example.com", "attach_logs=**********"),
+ ],
+)
+def test_args_logging_should_censor_webhook(
+ manager: Manager, logs: pytest.LogCaptureFixture, webhook: str, output: str
+) -> None:
+ logs_model = manager.config_manager.settings_data.logs
+ manager.config_manager.settings_data.logs = update_model(logs_model, webhook=webhook)
+ manager.args_logging()
+ assert logs.messages
+ assert "Starting Cyberdrop-DL Process" in logs.text
+ assert webhook not in logs.text
+ webhook_line = next(msg for msg in logs.text.splitlines() if '"webhook"' in msg)
+ _, _, webhook_text = webhook_line.partition(":")
+ webhook_url = webhook_text.strip().split(" ")[0].replace('"', "").strip()
+ assert output == webhook_url
+
+
+async def test_async_db_close(manager: Manager) -> None:
+ await manager.async_startup()
+ assert not isinstance(manager.db_manager, Field)
+ assert not isinstance(manager.hash_manager, Field)
+ assert "overwrite" not in str(manager.log_manager.main_log)
+ await manager.async_db_close()
+ assert isinstance(manager.db_manager, Field)
+ assert isinstance(manager.hash_manager, Field)
+ await manager.close()
diff --git a/tests/test_startup.py b/tests/test_startup.py
index 6ec1fe29c..1470379b2 100644
--- a/tests/test_startup.py
+++ b/tests/test_startup.py
@@ -1,27 +1,27 @@
+from pathlib import Path
+
import pytest
from cyberdrop_dl.main import run
from cyberdrop_dl.ui.program_ui import ProgramUI
-def test_startup(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
+def test_startup(tmp_cwd: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
# This is just to test that cyberdrop is able to run in the current python version
msg = "main UI started successfully"
def main_ui(*_) -> None:
print(msg)
- monkeypatch.setattr("sys.argv", ["pytest", "--disable-cache"])
monkeypatch.setattr(ProgramUI, "__init__", main_ui)
- run()
+ run(())
captured = capsys.readouterr()
output = captured.out
assert msg in output
-def test_async_startup(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:
- monkeypatch.setattr("sys.argv", ["pytest", "--download"])
- run()
+def test_async_startup(tmp_cwd: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ run(("--download",))
captured = capsys.readouterr()
output = captured.out
assert "Finished downloading. Enjoy :)" in output
diff --git a/tests/test_unicode_emoji_sanitization.py b/tests/test_unicode_emoji_sanitization.py
index 6ef005306..027c27448 100644
--- a/tests/test_unicode_emoji_sanitization.py
+++ b/tests/test_unicode_emoji_sanitization.py
@@ -11,6 +11,8 @@ IS_MACOS = sys.platform == "darwin"
FOREIGN_LENGUAGE_STRINGS = ["これは単なるテストです", "это всего лишь тест", "این فقط یک آزمایش است", "هذا مجرد اختبار"]
INVALID_UNICODE9_STRING = "🫧Bubblz🫧"
+pytestmark = pytest.mark.filterwarnings("ignore:invalid escape sequence.*::SyntaxWarning")
+
def test_sanitize_macos_problematic_unicode_symbol() -> None:
assert sanitize_unicode_emojis_and_symbols(INVALID_UNICODE9_STRING) == "Bubblz"