Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
tests: add tests for the manager (#982)
NTFSvolume9b64dd84d40f
diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.pyindex a690a34b0..56a698f93 100644--- a/cyberdrop_dl/clients/scraper_client.py+++ b/cyberdrop_dl/clients/scraper_client.py@@ -5,7 +5,6 @@ import functoolsimport inspectfrom collections.abc import Callablefrom contextlib import asynccontextmanager-from dataclasses import Field, fieldfrom datetime import datetimefrom functools import wrapsfrom json import dumps as json_dumps@@ -20,9 +19,10 @@ from bs4 import BeautifulSoupimport cyberdrop_dl.constants as constantsfrom cyberdrop_dl import env+from cyberdrop_dl.constants import NOT_DEFINEDfrom cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, InvalidContentTypeError, ScrapeErrorfrom cyberdrop_dl.utils.logger import log_debug-from cyberdrop_dl.utils.utilities import get_soup_no_error, sanitize_filename+from cyberdrop_dl.utils.utilities import close_if_defined, get_soup_no_error, sanitize_filenameif TYPE_CHECKING:from collections.abc import Callable, Coroutine@@ -108,10 +108,10 @@ class ScraperClient:# folder len + date_prefix len + 10 [suffix (.html) + 1 OS separator + 4 (padding)]min_html_file_path_len = len(str(self._pages_folder)) + len(constants.STARTUP_TIME_STR) + 10self._max_html_stem_len = 245 - min_html_file_path_len- self._session: CachedSession = field(init=False)- self._curl_session: AsyncSession = field(init=False)+ self._session: CachedSession = NOT_DEFINED+ self._curl_session: AsyncSession = NOT_DEFINED- def startup(self):+ def startup(self) -> None:add_request_log_hooks(self._trace_configs)self._session = CachedSession(headers=self._headers,@@ -135,9 +135,8 @@ class ScraperClient:)async def close(self) -> None:- await self._session.close()- if not isinstance(self._curl_session, Field):- await self._curl_session.close()+ await close_if_defined(self._session)+ await close_if_defined(self._curl_session)def is_ddos(self, soup: BeautifulSoup) -> bool:return self.client_manager.check_ddos_guard(soup) or self.client_manager.check_cloudflare(soup)diff --git a/cyberdrop_dl/config_definitions/config_settings.py b/cyberdrop_dl/config_definitions/config_settings.pyindex 0bf68ca5d..cf90a20a0 100644--- a/cyberdrop_dl/config_definitions/config_settings.py+++ b/cyberdrop_dl/config_definitions/config_settings.py@@ -5,7 +5,7 @@ from pathlib import Pathfrom pydantic import BaseModel, ByteSize, Field, NonNegativeInt, PositiveInt, field_serializer, field_validator-from cyberdrop_dl.constants import APP_STORAGE, BROWSERS, DOWNLOAD_STORAGE+from cyberdrop_dl.constants import BROWSERS, DEFAULT_APP_STORAGE, DEFAULT_DOWNLOAD_STORAGEfrom cyberdrop_dl.data_structures.hash import Hashingfrom cyberdrop_dl.data_structures.supported_domains import SUPPORTED_SITES_DOMAINSfrom cyberdrop_dl.types import (@@ -43,9 +43,9 @@ class DownloadOptions(BaseModel):class Files(AliasModel):- download_folder: Path = Field(validation_alias="d", default=DOWNLOAD_STORAGE)+ download_folder: Path = Field(validation_alias="d", default=DEFAULT_DOWNLOAD_STORAGE)dump_json: bool = Field(default=False, validation_alias="j")- input_file: Path = Field(validation_alias="i", default=APP_STORAGE / "Configs" / "{config}" / "URLs.txt")+ input_file: Path = Field(validation_alias="i", default=DEFAULT_APP_STORAGE / "Configs" / "{config}" / "URLs.txt")save_pages_html: bool = False@@ -56,7 +56,7 @@ class Logs(AliasModel):last_forum_post: LogPath = Field(default=Path("Last_Scraped_Forum_Posts.csv"), validation_alias="last_forum_post_filename")- log_folder: Path = APP_STORAGE / "Configs" / "{config}" / "Logs"+ log_folder: Path = DEFAULT_APP_STORAGE / "Configs" / "{config}" / "Logs"log_line_width: PositiveInt = Field(default=240, ge=50)logs_expire_after: timedelta | None = Nonemain_log: MainLogPath = Field(default=Path("downloader.log"), validation_alias="main_log_filename")@@ -151,7 +151,7 @@ class RuntimeOptions(BaseModel):class Sorting(BaseModel):scan_folder: PathOrNone = Nonesort_downloads: bool = False- sort_folder: Path = DOWNLOAD_STORAGE / "Cyberdrop-DL Sorted Downloads"+ sort_folder: Path = DEFAULT_DOWNLOAD_STORAGE / "Cyberdrop-DL Sorted Downloads"sort_incrementer_format: NonEmptyStr = " ({i})"sorted_audio: NonEmptyStrOrNone = "{sort_dir}/{base_dir}/Audio/{filename}{ext}"sorted_image: NonEmptyStrOrNone = "{sort_dir}/{base_dir}/Images/{filename}{ext}"diff --git a/cyberdrop_dl/constants.py b/cyberdrop_dl/constants.pyindex 536852cc3..370f45661 100644--- a/cyberdrop_dl/constants.py+++ b/cyberdrop_dl/constants.py@@ -1,4 +1,5 @@import re+from dataclasses import fieldfrom datetime import UTC, datetimefrom enum import Enum, IntEnum, StrEnum, autofrom pathlib import Path@@ -52,9 +53,10 @@ class CustomHTTPStatus(IntEnum):BLOCKED_DOMAINS = ("facebook", "instagram", "fbcdn")-APP_STORAGE = Path("./AppData")-DOWNLOAD_STORAGE = Path("./Downloads")+DEFAULT_APP_STORAGE = Path("./AppData")+DEFAULT_DOWNLOAD_STORAGE = Path("./Downloads")RESERVED_CONFIG_NAMES = ["all", "default"]+NOT_DEFINED = field(init=False)class HashType(StrEnum):diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.pyindex 93def7016..24302635c 100644--- a/cyberdrop_dl/director.py+++ b/cyberdrop_dl/director.py@@ -312,7 +312,7 @@ def _startup_logging(*, first_time_setup: bool = False) -> Generator:sys.exit(exit_code)-def _setup_manager() -> Manager:+def _setup_manager(args: tuple[str, ...] | None = None) -> Manager:"""Starts the program and returns the manager.This will also run the UI for the program@@ -320,7 +320,7 @@ def _setup_manager() -> Manager:"""with _startup_logging(first_time_setup=True):try:- manager = Manager()+ manager = Manager(args)manager.startup()if manager.parsed_args.cli_only_args.multiconfig:startup_logger.info("validating all configs, please wait...")@@ -346,12 +346,12 @@ def _setup_manager() -> Manager:class Director:"""Creates a manager and runs it"""- def __init__(self) -> None:+ def __init__(self, args: tuple[str, ...] | None = None) -> None:if os.name == "nt":asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())self.loop = asyncio.new_event_loop()asyncio.set_event_loop(self.loop)- self.manager = _setup_manager()+ self.manager = _setup_manager(args)# For future use with textual# fullscreen = f = m.parsed_args.cli_only_args.fullscreen_ui# self.use_textual = m.parsed_args.cli_only_args.textual_ui and fullscreendiff --git a/cyberdrop_dl/main.py b/cyberdrop_dl/main.pyindex b76217d57..fa8e5bd1d 100644--- a/cyberdrop_dl/main.py+++ b/cyberdrop_dl/main.py@@ -15,11 +15,11 @@ def actual_main() -> None:sys.exit(run())-def run() -> str | int:+def run(args: tuple[str, ...] | None = None) -> str | int:try:from cyberdrop_dl.director import Director- director = Director()+ director = Director(args)return director.run()except KeyboardInterrupt:return "\nKeyboardInterrupt"diff --git a/cyberdrop_dl/managers/config_manager.py b/cyberdrop_dl/managers/config_manager.pyindex 5e0b7cc0f..aa0b75f2d 100644--- a/cyberdrop_dl/managers/config_manager.py+++ b/cyberdrop_dl/managers/config_manager.py@@ -114,13 +114,16 @@ class ConfigManager:if posible_fields == set_fields and not needs_update and self.pydantic_config:returnelse:- from cyberdrop_dl import constants-self.settings_data = ConfigSettings()- self.settings_data.files.input_file = constants.APP_STORAGE / "Configs" / self.loaded_config / "URLs.txt"- self.settings_data.files.download_folder = constants.DOWNLOAD_STORAGE / "Cyberdrop-DL Downloads"- self.settings_data.logs.log_folder = constants.APP_STORAGE / "Configs" / self.loaded_config / "Logs"- self.settings_data.sorting.sort_folder = constants.DOWNLOAD_STORAGE / "Cyberdrop-DL Sorted Downloads"+ self.settings_data.files.input_file = (+ self.manager.path_manager.appdata / "Configs" / self.loaded_config / "URLs.txt"+ )+ downloads = self.manager.path_manager.cwd / "Downloads"+ self.settings_data.sorting.sort_folder = downloads / "Cyberdrop-DL Sorted Downloads"+ self.settings_data.files.download_folder = downloads / "Cyberdrop-DL Downloads"+ self.settings_data.logs.log_folder = (+ self.manager.path_manager.appdata / "Configs" / self.loaded_config / "Logs"+ )yaml.save(self.settings, self.settings_data)diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex fa53b3540..e798375c2 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -1,14 +1,14 @@from __future__ import annotationsimport asyncio-import inspectimport jsonfrom dataclasses import Field, fieldfrom time import perf_counter-from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Protocol, TypeVar+from typing import TYPE_CHECKING, Literal, NamedTuple, TypeVar++from pydantic import BaseModelfrom cyberdrop_dl import __version__, constants-from cyberdrop_dl.config_definitions import ConfigSettings, GlobalSettingsfrom cyberdrop_dl.managers.cache_manager import CacheManagerfrom cyberdrop_dl.managers.client_manager import ClientManagerfrom cyberdrop_dl.managers.config_manager import ConfigManager@@ -22,38 +22,18 @@ from cyberdrop_dl.managers.progress_manager import ProgressManagerfrom cyberdrop_dl.managers.realdebrid_manager import RealDebridManagerfrom cyberdrop_dl.managers.storage_manager import StorageManagerfrom cyberdrop_dl.ui.textual import TextualUI-from cyberdrop_dl.utils.args import ParsedArgs+from cyberdrop_dl.utils.args import ParsedArgs, parse_argsfrom cyberdrop_dl.utils.ffmpeg import FFmpeg, get_ffmpeg_versionfrom cyberdrop_dl.utils.logger import QueuedLogger, logfrom cyberdrop_dl.utils.transfer import transfer_v5_db_to_v6-from cyberdrop_dl.utils.utilities import get_system_information+from cyberdrop_dl.utils.utilities import close_if_defined, get_system_informationif TYPE_CHECKING:import queuefrom asyncio import TaskGroupfrom cyberdrop_dl.scraper.scrape_mapper import ScrapeMapper---class HasClose(Protocol):- def close(self): ...---class HasAsyncClose(Protocol):- async def close(self): ...---C = TypeVar("C", bound=HasAsyncClose | HasClose)---def unset() -> Any:- return field(init=False)---async def close_if_set(obj: C) -> C:- if not isinstance(obj, Field):- await obj.close() if inspect.iscoroutinefunction(obj.close) else obj.close()- return unset()+ from cyberdrop_dl.types import AnyDictclass AsyncioEvents(NamedTuple):@@ -62,7 +42,7 @@ class AsyncioEvents(NamedTuple):class Manager:- def __init__(self) -> None:+ def __init__(self, args: tuple[str, ...] | None = None) -> None:self.parsed_args: ParsedArgs = field(init=False)self.cache_manager: CacheManager = CacheManager(self)self.path_manager: PathManager = field(init=False)@@ -96,6 +76,7 @@ class Manager:self.multiconfig: bool = Falseself.loggers: dict[str, QueuedLogger] = {}self.states = AsyncioEvents()+ self.args = args@propertydef config(self):@@ -120,7 +101,7 @@ class Manager:"""Startup process for the manager."""if isinstance(self.parsed_args, Field):- self.parsed_args = ParsedArgs.parse_args()+ self.parsed_args = parse_args(self.args)self.path_manager = PathManager(self)self.path_manager.pre_startup()@@ -215,62 +196,21 @@ class Manager:cli_ignore_options = self.parsed_args.config_settings.ignore_optionsconfig_skip_hosts = self.config_manager.settings_data.ignore_options.skip_hostsconfig_only_hosts = self.config_manager.settings_data.ignore_options.only_hosts- exclude = {"+", "-"}-- def add(config_list: list[str], cli_list: list[str]) -> list[str]:- new_list_as_set = set(config_list + cli_list)- return sorted(new_list_as_set - exclude)-- def remove(config_list: list[str], cli_list: list[str]) -> list[str]:- new_list_as_set = set(config_list) - set(cli_list)- return sorted(new_list_as_set - exclude)- def add_or_remove(config_list: list[str], cli_list: list[str]) -> list[str]:- if cli_list:- if cli_list[0] == "+":- return add(config_list, cli_list)- if cli_list[0] == "-":- return remove(config_list, cli_list)- return cli_list-- cli_ignore_options.skip_hosts = add_or_remove(config_skip_hosts, cli_ignore_options.skip_hosts)- cli_ignore_options.only_hosts = add_or_remove(config_only_hosts, cli_ignore_options.only_hosts)+ cli_ignore_options.skip_hosts = add_or_remove_lists(config_skip_hosts, cli_ignore_options.skip_hosts)+ cli_ignore_options.only_hosts = add_or_remove_lists(config_only_hosts, cli_ignore_options.only_hosts)def args_consolidation(self) -> None:"""Consolidates runtime arguments with config values."""self.process_additive_args()- cli_config_settings = self.parsed_args.config_settings.model_dump(exclude_unset=True)- cli_global_settings = self.parsed_args.global_settings.model_dump(exclude_unset=True)-- current_config_settings = self.config_manager.settings_data.model_dump()- current_global_settings = self.config_manager.global_settings_data.model_dump()-- merged_config_settings = self.merge_dicts(current_config_settings, cli_config_settings)- merged_global_settings = self.merge_dicts(current_global_settings, cli_global_settings)-- updated_config_settings = ConfigSettings.model_validate(merged_config_settings)- updated_global_settings = GlobalSettings.model_validate(merged_global_settings)- self.config_manager.settings_data = updated_config_settings- self.config_manager.global_settings_data = updated_global_settings- self.config_manager.deep_scrape = (- self.parsed_args.config_settings.runtime_options.deep_scrape or self.config_manager.deep_scrape- )+ conf = merge_models(self.config_manager.settings_data, self.parsed_args.config_settings)+ global_conf = merge_models(self.config_manager.global_settings_data, self.parsed_args.global_settings)+ deep_scrape = self.parsed_args.config_settings.runtime_options.deep_scrape or self.config_manager.deep_scrape- def merge_dicts(self, dict1: dict, dict2: dict):- for key, val in dict1.items():- if isinstance(val, dict):- if key in dict2 and isinstance(dict2[key], dict):- self.merge_dicts(dict1[key], dict2[key])- else:- if key in dict2:- dict1[key] = dict2[key]-- for key, val in dict2.items():- if key not in dict1:- dict1[key] = val-- return dict1+ self.config_manager.settings_data = conf+ self.config_manager.global_settings_data = global_conf+ self.config_manager.deep_scrape = deep_scrapedef args_logging(self) -> None:"""Logs the runtime arguments."""@@ -303,8 +243,8 @@ class Manager:async def async_db_close(self) -> None:"Partial shutdown for managers used for hash directory scanner"- self.db_manager = await close_if_set(self.db_manager)- self.hash_manager = unset()+ self.db_manager = await close_if_defined(self.db_manager)+ self.hash_manager = constants.NOT_DEFINEDself.progress_manager.hash_progress.reset()async def close(self) -> None:@@ -313,9 +253,9 @@ class Manager:await self.async_db_close()- self.client_manager = await close_if_set(self.client_manager)- self.storage_manager = await close_if_set(self.storage_manager)- self.cache_manager = await close_if_set(self.cache_manager)+ self.client_manager = await close_if_defined(self.client_manager)+ self.storage_manager = await close_if_defined(self.storage_manager)+ self.cache_manager = await close_if_defined(self.cache_manager)while self.loggers:_, queued_logger = self.loggers.popitem()@@ -334,3 +274,49 @@ class Manager:rewrite constants after config/arg manager have loaded"""constants.DISABLE_CACHE = self.parsed_args.cli_only_args.disable_cache+++def add_or_remove_lists(old_list: list[str], new_list: list[str]) -> list[str]:+ exclude = {"+", "-"}++ def add(config_list: list[str], cli_list: list[str]) -> list[str]:+ new_list_as_set = set(config_list + cli_list)+ return sorted(new_list_as_set - exclude)++ def remove(config_list: list[str], cli_list: list[str]) -> list[str]:+ new_list_as_set = set(config_list) - set(cli_list)+ return sorted(new_list_as_set - exclude)++ if new_list:+ if new_list[0] == "+":+ return add(old_list, new_list)+ if new_list[0] == "-":+ return remove(old_list, new_list)+ return new_list+++def merge_dicts(dict1: AnyDict, dict2: AnyDict) -> AnyDict:+ for key, val in dict1.items():+ if isinstance(val, dict):+ if key in dict2 and isinstance(dict2[key], dict):+ merge_dicts(dict1[key], dict2[key])+ else:+ if key in dict2:+ dict1[key] = dict2[key]++ for key, val in dict2.items():+ if key not in dict1:+ dict1[key] = val++ return dict1+++M = TypeVar("M", bound=BaseModel)+++def merge_models(default: M, new: M) -> M:+ default_dict = default.model_dump()+ new_dict = new.model_dump(exclude_unset=True)++ updated_dict = merge_dicts(default_dict, new_dict)+ return default.model_validate(updated_dict)diff --git a/cyberdrop_dl/managers/path_manager.py b/cyberdrop_dl/managers/path_manager.pyindex 4a30912e4..607eac1a4 100644--- a/cyberdrop_dl/managers/path_manager.py+++ b/cyberdrop_dl/managers/path_manager.py@@ -1,11 +1,11 @@from __future__ import annotations-from dataclasses import field+from dataclasses import Field, fieldfrom datetime import datetimefrom pathlib import Pathfrom typing import TYPE_CHECKING-from cyberdrop_dl import constants, env+from cyberdrop_dl import envfrom cyberdrop_dl.utils.utilities import purge_dir_treeif TYPE_CHECKING:@@ -13,12 +13,6 @@ if TYPE_CHECKING:from cyberdrop_dl.managers.manager import Manager-if env.RUNNING_IN_IDE and Path.cwd().name == "cyberdrop_dl":- """This is for testing purposes only"""- constants.APP_STORAGE = Path("../AppData")- constants.DOWNLOAD_STORAGE = Path("../Downloads")--class PathManager:def __init__(self, manager: Manager) -> None:self.manager = manager@@ -55,16 +49,30 @@ class PathManager:"download_error_urls","scrape_error_urls",]+ self._appdata: Path = field(init=False)- def pre_startup(self) -> None:- if self.manager.parsed_args.cli_only_args.appdata_folder:- constants.APP_STORAGE = self.manager.parsed_args.cli_only_args.appdata_folder / "AppData"+ @property+ def cwd(self) -> Path:+ if env.RUNNING_IN_IDE and Path.cwd().name == "cyberdrop_dl":+ # This is for testing purposes only"""+ return Path("..").resolve()+ return Path().resolve()- constants.APP_STORAGE = constants.APP_STORAGE.resolve()+ @property+ def appdata(self) -> Path:+ if isinstance(self._appdata, Field):+ if self.manager.parsed_args.cli_only_args.appdata_folder:+ path = self.manager.parsed_args.cli_only_args.appdata_folder / "AppData"+ self._appdata = self.cwd / path+ else:+ self._appdata = self.cwd / "AppData"++ return self._appdata- self.cache_folder = constants.APP_STORAGE / "Cache"- self.config_folder = constants.APP_STORAGE / "Configs"- self.cookies_dir = constants.APP_STORAGE / "Cookies"+ def pre_startup(self) -> None:+ self.cache_folder = self.appdata / "Cache"+ self.config_folder = self.appdata / "Configs"+ self.cookies_dir = self.appdata / "Cookies"self.cache_db = self.cache_folder / "request_cache.db"self.cache_folder.mkdir(parents=True, exist_ok=True)@@ -78,7 +86,8 @@ class PathManager:current_config = self.manager.config_manager.loaded_configdef replace(path: Path) -> Path:- return Path(str(path).replace("{config}", current_config)).resolve()+ path_w_config = str(path).replace("{config}", current_config)+ return self.cwd.joinpath(Path(path_w_config))self.download_folder = replace(settings_data.files.download_folder)self.sorted_folder = replace(settings_data.sorting.sort_folder)diff --git a/cyberdrop_dl/utils/args.py b/cyberdrop_dl/utils/args.pyindex 261bec984..141a93388 100644--- a/cyberdrop_dl/utils/args.py+++ b/cyberdrop_dl/utils/args.py@@ -3,7 +3,7 @@ import timeimport warningsfrom argparse import SUPPRESS, ArgumentParser, BooleanOptionalAction, RawDescriptionHelpFormatterfrom argparse import _ArgumentGroup as ArgGroup-from collections.abc import Iterable+from collections.abc import Iterable, Sequencefrom datetime import datefrom enum import StrEnum, autofrom pathlib import Path@@ -158,11 +158,6 @@ class ParsedArgs(AliasModel):sys.exit(1)time.sleep(WARNING_TIMEOUT)- @staticmethod- def parse_args() -> "ParsedArgs":- """Parses the command line arguments passed into the program. Returns an instance of `ParsedArgs`"""- return parse_args()-def prepare_warnings(self) -> set[str]:warnings_to_emit = set()@@ -267,17 +262,17 @@ def make_parser() -> tuple[ArgumentParser, dict[str, list[ArgGroup]]]:return parser, group_lists-def get_parsed_args_dict() -> dict[str, AnyDict]:+def get_parsed_args_dict(args: Sequence[str] | None = None) -> dict[str, AnyDict]:parser, group_lists = make_parser()- args = parser.parse_intermixed_args()+ namespace = parser.parse_intermixed_args(args)parsed_args: dict[str, AnyDict] = {}for name, groups in group_lists.items():parsed_args[name] = {}for group in groups:group_dict = {- arg.dest: getattr(args, arg.dest)+ arg.dest: getattr(namespace, arg.dest)for arg in group._group_actions- if getattr(args, arg.dest, None) is not None+ if getattr(namespace, arg.dest, None) is not None}if group_dict:assert group.title@@ -289,9 +284,9 @@ def get_parsed_args_dict() -> dict[str, AnyDict]:return parsed_args-def parse_args() -> ParsedArgs:+def parse_args(args: Sequence[str] | None = None) -> ParsedArgs:"""Parses the command line arguments passed into the program."""- parsed_args_dict = get_parsed_args_dict()+ parsed_args_dict = get_parsed_args_dict(args)try:parsed_args_model = ParsedArgs.model_validate(parsed_args_dict)diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex e444cdebc..2ee7648f0 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport contextlib+import inspectimport jsonimport osimport platform@@ -10,7 +11,7 @@ import shutilimport subprocessimport sysimport unicodedata-from dataclasses import dataclass, fields+from dataclasses import Field, dataclass, fieldsfrom functools import lru_cache, partial, wrapsfrom pathlib import Pathfrom stat import S_ISREG@@ -546,6 +547,23 @@ def get_size_or_none(path: Path) -> int | None:return None+class HasClose(Protocol):+ def close(self): ...+++class HasAsyncClose(Protocol):+ async def close(self): ...+++C = TypeVar("C", bound=HasAsyncClose | HasClose)+++async def close_if_defined(obj: C) -> C:+ if not isinstance(obj, Field):+ await obj.close() if inspect.iscoroutinefunction(obj.close) else obj.close()+ return constants.NOT_DEFINED++def with_suffix_encoded(url: AnyURL, suffix: str) -> AnyURL:name = Path(url.raw_name).with_suffix(suffix)return url.parent.joinpath(str(name), encoded=True).with_query(url.query).with_fragment(url.fragment)diff --git a/pyproject.toml b/pyproject.tomlindex ac1f1b8c1..3694bad13 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -48,6 +48,20 @@ Changelog = "https://github.com/jbsparrow/CyberDropDownloader/blob/master/CHANGEDocumentation = "https://script-ware.gitbook.io/cyberdrop-dl/"Repository = "https://github.com/jbsparrow/CyberDropDownloader"+[tool.coverage.report]+exclude_also = [+ '@(abc\.)?abstractmethod',+ 'class .*\bProtocol\):',+ 'if __name__ == .__main__.:',+ 'if env.DEBUG_VAR:',+ 'if TYPE_CHECKING:'+]++[tool.coverage.run]+omit = [+ "tests/*"+]+[tool.poetry]include = ["CHANGELOG.md"]packages = [{include = "cyberdrop_dl"}]@@ -67,7 +81,8 @@ pytest-cov = ">=6.0.0,<7"pytest-mock = "*"[tool.pytest.ini_options]-asyncio_default_fixture_loop_scope = "module"+addopts = ["-s"]+asyncio_default_fixture_loop_scope = "function"asyncio_mode = "auto"minversion = "8.3"testpaths = ["tests"]diff --git a/tests/conftest.py b/tests/conftest.pyindex ee56eb9e3..42203bceb 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -1,8 +1,21 @@+from __future__ import annotations+from pathlib import Path+from typing import TYPE_CHECKINGimport pytest+if TYPE_CHECKING:+ from pathlib import Path+[email protected](autouse=True)-def test_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+def tmp_cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:monkeypatch.chdir(tmp_path)+ return tmp_path+++async def logs(caplog: pytest.LogCaptureFixture) -> pytest.LogCaptureFixture:+ caplog.set_level(10)+ return caplogdiff --git a/tests/test_apprise.py b/tests/test_apprise.pyindex c083b0461..fe815b27f 100644--- a/tests/test_apprise.py+++ b/tests/test_apprise.py@@ -20,9 +20,12 @@ FAKE_MANAGER.cache_manager = FakeCacheManager(FAKE_MANAGER)FAKE_MANAGER.config_manager = ConfigManager(FAKE_MANAGER)URL_FAIL = "mailto://test_user:[email protected]"-URL_SUCCESS = os.environ.get("APPRISE_TEST_EMAIL_URL", "")+APPRISE_ENV_VAR = "APPRISE_TEST_EMAIL_URL"+URL_SUCCESS = os.environ.get(APPRISE_ENV_VAR, "")URL_SUCCESS_ATTACH_LOGS = f"attach_logs={URL_SUCCESS}"+pytestmark = pytest.mark.skipif(not URL_SUCCESS, reason=f"{APPRISE_ENV_VAR} environment variable is not set")+@dataclassclass AppriseTestCase:@@ -34,7 +37,7 @@ class AppriseTestCase:name: str = "Testing Apprise"-def test_get_apprise_urls():+def test_get_apprise_urls() -> None:with pytest.raises(ValueError):apprise.get_apprise_urls()@@ -77,8 +80,7 @@ def test_get_apprise_urls():assert got == expected, f"Parsed URL: {got.raw_url}, Expected URL: {expected.raw_url}"-async def send_notification(test_case: AppriseTestCase):- assert URL_SUCCESS, "Apprise URL should be set on enviroment"+async def send_notification(test_case: AppriseTestCase) -> None:FAKE_MANAGER.config_manager.apprise_urls = []if test_case.urls and any(test_case.urls):FAKE_MANAGER.config_manager.apprise_urls = apprise.get_apprise_urls(urls=test_case.urls)@@ -98,7 +100,7 @@ async def send_notification(test_case: AppriseTestCase):assert not any(match in logs_as_str for match in test_case.exclude), "Logs should not match exclude pattern"-async def test_failed_notification():+async def test_failed_notification() -> None:test_case = AppriseTestCase(urls=[URL_FAIL],result=NotificationResult.FAILED,@@ -107,7 +109,7 @@ async def test_failed_notification():await send_notification(test_case)-async def test_successful_notification():+async def test_successful_notification() -> None:test_case = AppriseTestCase(urls=[URL_SUCCESS],result=NotificationResult.SUCCESS,@@ -117,7 +119,7 @@ async def test_successful_notification():await send_notification(test_case)-async def test_successful_notification_with_successful_attachment():+async def test_successful_notification_with_successful_attachment() -> None:test_case = AppriseTestCase(urls=[URL_SUCCESS_ATTACH_LOGS],result=NotificationResult.SUCCESS,@@ -127,7 +129,7 @@ async def test_successful_notification_with_successful_attachment():await send_notification(test_case)-async def test_successful_notification_with_failed_attachment():+async def test_successful_notification_with_failed_attachment() -> None:test_case = AppriseTestCase(urls=[URL_SUCCESS_ATTACH_LOGS],result=NotificationResult.SUCCESS,@@ -138,7 +140,7 @@ async def test_successful_notification_with_failed_attachment():await send_notification(test_case)-async def test_none_notification():+async def test_none_notification() -> None:test_case = AppriseTestCase(urls=[""],result=NotificationResult.NONE,@@ -148,7 +150,7 @@ async def test_none_notification():await send_notification(test_case)-async def test_partial_notification():+async def test_partial_notification() -> None:test_case = AppriseTestCase(urls=[URL_SUCCESS_ATTACH_LOGS, URL_FAIL],result=NotificationResult.PARTIAL,diff --git a/tests/test_cli.py b/tests/test_cli.pyindex 5a8e5c568..94e00f825 100644--- a/tests/test_cli.py+++ b/tests/test_cli.py@@ -1,17 +1,22 @@-import sys+from pathlib import Pathimport pytestfrom cyberdrop_dl.main import run-def test_help(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:- # This is just to test that cyberdrop is able to run in the current python version- monkeypatch.setattr(sys, "argv", ["pytest", "--help"])+ "command, text",+ [+ ("--help", "Bulk asynchronous downloader for multiple file hosts"),+ ("--show-supported-sites", "for a details about supported paths"),+ ],+)+def test_command_by_console_output(tmp_cwd: Path, capsys: pytest.CaptureFixture[str], command: str, text: str) -> None:try:- run()+ run(tuple(command.split()))except SystemExit:passcaptured = capsys.readouterr()output = captured.out- assert "Bulk asynchronous downloader for multiple file hosts" in output+ assert text in outputdiff --git a/tests/test_manager.py b/tests/test_manager.pynew file mode 100644index 000000000..7ea111c0c--- /dev/null+++ b/tests/test_manager.py@@ -0,0 +1,137 @@+from dataclasses import Field+from pathlib import Path+from typing import Any, TypeVar++import pytest+from pydantic import BaseModel++from cyberdrop_dl.managers.manager import Manager, merge_dicts++M = TypeVar("M", bound=BaseModel)+++def update_model(model: M, **kwargs: Any) -> M:+ return model.model_validate(model.model_dump() | kwargs)++[email protected](scope="function", name="manager")+def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Manager:+ appdata = str(tmp_path)+ downloads = str(tmp_path / "Downloads")+ monkeypatch.chdir(tmp_path)+ bare_manager = Manager(("--appdata-folder", appdata, "-d", downloads))+ bare_manager.startup()+ bare_manager.path_manager.startup()+ bare_manager.log_manager.startup()+ return bare_manager+++class TestMergeDicts:+ def test_overwrite(self) -> None:+ dict1 = {"a": 1, "b": 2}+ dict2 = {"b": 3, "c": 4}+ expected = {"a": 1, "b": 3, "c": 4}+ assert merge_dicts(dict1, dict2) == expected++ def test_merge_with_new_keys(self) -> None:+ dict1 = {"a": 1}+ dict2 = {"b": 2, "c": 3}+ expected = {"a": 1, "b": 2, "c": 3}+ assert merge_dicts(dict1, dict2) == expected++ def test_merge_recursive(self) -> None:+ dict1 = {+ "a": {+ "b": 1,+ "c": 2,+ },+ "d": 3,+ }+ dict2 = {+ "a": {+ "b": 4,+ "e": 4,+ "f": {+ "g": 5,+ "h": 6,+ },+ },+ "i": 7,+ }+ expected = {+ "a": {+ "b": 4,+ "c": 2,+ "e": 4,+ "f": {+ "g": 5,+ "h": 6,+ },+ },+ "d": 3,+ "i": 7,+ }+ assert merge_dicts(dict1, dict2) == expected++ def test_merge_with_empty_dict1(self) -> None:+ dict1 = {}+ dict2 = {"a": 1, "b": 2}+ expected = {"a": 1, "b": 2}+ assert merge_dicts(dict1, dict2) == expected++ def test_merge_with_empty_dict2(self) -> None:+ dict1 = {"a": 1, "b": 2}+ dict2 = {}+ expected = {"a": 1, "b": 2}+ assert merge_dicts(dict1, dict2) == expected++ def test_merge_with_both_empty_dicts(self) -> None:+ dict1 = {}+ dict2 = {}+ expected = {}+ assert merge_dicts(dict1, dict2) == expected++ def test_dict_overwrites_value(self) -> None:+ dict1 = {"a": 1}+ dict2 = {"a": {"x": 1}}+ expected = {"a": {"x": 1}}+ assert merge_dicts(dict1, dict2) == expected++ def test_value_should_not_overwrite_dict(self) -> None:+ dict1 = {"a": {"x": 1}}+ dict2 = {"a": 1}+ expected = {"a": {"x": 1}}+ assert merge_dicts(dict1, dict2) == expected+++ "webhook, output",+ [+ ("https://example.com", "**********"),+ ("attach_logs=https://example.com", "attach_logs=**********"),+ ],+)+def test_args_logging_should_censor_webhook(+ manager: Manager, logs: pytest.LogCaptureFixture, webhook: str, output: str+) -> None:+ logs_model = manager.config_manager.settings_data.logs+ manager.config_manager.settings_data.logs = update_model(logs_model, webhook=webhook)+ manager.args_logging()+ assert logs.messages+ assert "Starting Cyberdrop-DL Process" in logs.text+ assert webhook not in logs.text+ webhook_line = next(msg for msg in logs.text.splitlines() if '"webhook"' in msg)+ _, _, webhook_text = webhook_line.partition(":")+ webhook_url = webhook_text.strip().split(" ")[0].replace('"', "").strip()+ assert output == webhook_url+++async def test_async_db_close(manager: Manager) -> None:+ await manager.async_startup()+ assert not isinstance(manager.db_manager, Field)+ assert not isinstance(manager.hash_manager, Field)+ assert "overwrite" not in str(manager.log_manager.main_log)+ await manager.async_db_close()+ assert isinstance(manager.db_manager, Field)+ assert isinstance(manager.hash_manager, Field)+ await manager.close()diff --git a/tests/test_startup.py b/tests/test_startup.pyindex 6ec1fe29c..1470379b2 100644--- a/tests/test_startup.py+++ b/tests/test_startup.py@@ -1,27 +1,27 @@+from pathlib import Path+import pytestfrom cyberdrop_dl.main import runfrom cyberdrop_dl.ui.program_ui import ProgramUI-def test_startup(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:+def test_startup(tmp_cwd: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:# This is just to test that cyberdrop is able to run in the current python versionmsg = "main UI started successfully"def main_ui(*_) -> None:print(msg)- monkeypatch.setattr("sys.argv", ["pytest", "--disable-cache"])monkeypatch.setattr(ProgramUI, "__init__", main_ui)- run()+ run(())captured = capsys.readouterr()output = captured.outassert msg in output-def test_async_startup(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None:- monkeypatch.setattr("sys.argv", ["pytest", "--download"])- run()+def test_async_startup(tmp_cwd: Path, capsys: pytest.CaptureFixture[str]) -> None:+ run(("--download",))captured = capsys.readouterr()output = captured.outassert "Finished downloading. Enjoy :)" in outputdiff --git a/tests/test_unicode_emoji_sanitization.py b/tests/test_unicode_emoji_sanitization.pyindex 6ef005306..027c27448 100644--- a/tests/test_unicode_emoji_sanitization.py+++ b/tests/test_unicode_emoji_sanitization.py@@ -11,6 +11,8 @@ IS_MACOS = sys.platform == "darwin"FOREIGN_LENGUAGE_STRINGS = ["これは単なるテストです", "это всего лишь тест", "این فقط یک آزمایش است", "هذا مجرد اختبار"]INVALID_UNICODE9_STRING = "🫧Bubblz🫧"+pytestmark = pytest.mark.filterwarnings("ignore:invalid escape sequence.*::SyntaxWarning")+def test_sanitize_macos_problematic_unicode_symbol() -> None:assert sanitize_unicode_emojis_and_symbols(INVALID_UNICODE9_STRING) == "Bubblz"