Skip to content

Bulk asynchronous downloader for multiple file hosts

14.3 KB
Raw Blame
from __future__ import annotations

import datetime as dt
import json
import os
import time
import warnings
from pathlib import Path

import pytest
import yaml
from cyclopts.exceptions import UnknownOptionError
from pydantic import BaseModel

import cyberdrop_dl.commands.scrape
from cyberdrop_dl.config import Config, Files, _resolve_paths, settings
from cyberdrop_dl.config.appdata import AppData
from cyberdrop_dl.config.auth import Authentication, Notifications
from cyberdrop_dl.config.filters import Filters, _FileFilter
from cyberdrop_dl.exceptions import CDLConfigRuntimeErrorsGroup
from cyberdrop_dl.models import AppriseURL, merge_additive_args, merge_dicts


def test_config_equality() -> None:
    config1 = Config()
    time.sleep(0.1)
    config2 = Config()
    assert config1.__dict__ == config2.__dict__
    assert config1 == config2
    assert config1.model_dump() == config2.model_dump()
    config1.resolve_paths()
    config2.resolve_paths()
    assert config1 == config2
    assert config1.model_dump() == config2.model_dump()


def test_parse_config_from_args() -> None:
    assert Config() == Config.parse_args([])


def test_parse_config_from_args2() -> None:
    config = Config.model_validate({"download_folder": "downloads"})
    assert config == Config.parse_args(["--download-folder", "downloads"])
    assert config == Config.parse_args(["-o", "downloads"])
    with pytest.raises(UnknownOptionError):
        _ = Config.parse_args(["-i", "test.txt"])


def test_logs_equality() -> None:
    logs1 = settings.Logs()
    time.sleep(0.1)
    logs2 = settings.Logs()
    assert logs1._created_at != logs2._created_at
    assert logs1.__dict__ == logs2.__dict__
    assert logs1 == logs2


class TestMergeDicts:
    def test_overwrite(self) -> None:
        dict1 = {"a": 1, "b": 2}
        dict2 = {"b": 3, "c": 4}
        expected = {"a": 1, "b": 3, "c": 4}
        assert merge_dicts(dict1, dict2) == expected

    def test_merge_with_new_keys(self) -> None:
        dict1 = {"a": 1}
        dict2 = {"b": 2, "c": 3}
        expected = {"a": 1, "b": 2, "c": 3}
        assert merge_dicts(dict1, dict2) == expected

    def test_merge_recursive(self) -> None:
        dict1 = {
            "a": {
                "b": 1,
                "c": 2,
            },
            "d": 3,
        }
        dict2 = {
            "a": {
                "b": 4,
                "e": 4,
                "f": {
                    "g": 5,
                    "h": 6,
                },
            },
            "i": 7,
        }
        expected = {
            "a": {
                "b": 4,
                "c": 2,
                "e": 4,
                "f": {
                    "g": 5,
                    "h": 6,
                },
            },
            "d": 3,
            "i": 7,
        }
        assert merge_dicts(dict1, dict2) == expected

    def test_merge_with_empty_dict1(self) -> None:
        dict1 = {}
        dict2 = {"a": 1, "b": 2}
        expected = {"a": 1, "b": 2}
        assert merge_dicts(dict1, dict2) == expected

    def test_merge_with_empty_dict2(self) -> None:
        dict1 = {"a": 1, "b": 2}
        dict2 = {}
        expected = {"a": 1, "b": 2}
        assert merge_dicts(dict1, dict2) == expected

    def test_merge_with_both_empty_dicts(self) -> None:
        dict1 = {}
        dict2 = {}
        expected = {}
        assert merge_dicts(dict1, dict2) == expected

    def test_dict_overwrites_value(self) -> None:
        dict1 = {"a": 1}
        dict2 = {"a": {"x": 1}}
        expected = {"a": {"x": 1}}
        assert merge_dicts(dict1, dict2) == expected

    def test_value_should_not_overwrite_dict(self) -> None:
        dict1 = {"a": {"x": 1}}
        dict2 = {"a": 1}
        expected = {"a": {"x": 1}}
        assert merge_dicts(dict1, dict2) == expected

    def test_additive_args_add(self) -> None:
        dict1 = {"a": {"b": [1, 2]}}
        dict2 = {"a": {"b": ["+", 3, 4]}}
        additive_keys = (("a", "b"),)
        assert merge_dicts(dict1, dict2, additive_keys) == {"a": {"b": [1, 2, 3, 4]}}

    def test_additive_args_remove(self) -> None:
        dict1 = {"a": {"b": [1, 2]}}
        dict2 = {"a": {"b": ["-", 1]}}
        additive_keys = (("a", "b"),)
        assert merge_dicts(dict1, dict2, additive_keys) == {"a": {"b": [2]}}

    def test_additive_args_override(self) -> None:
        dict1 = {"a": {"b": [1, 2]}}
        dict2 = {"a": {"b": [3, 1]}}
        additive_keys = (("a", "b"),)
        assert merge_dicts(dict1, dict2, additive_keys) == {"a": {"b": [3, 1]}}


class TestRuntimeLogsConfig:
    @staticmethod
    def parse(level: object, console_level: object) -> settings.Logs:
        return settings.Logs.model_validate({"level": level, "console_level": console_level})

    def test_default(self) -> None:
        default = settings.Logs()
        expected = settings.Logs(level="debug", console_level="debug")  # pyright: ignore[reportArgumentType]

        assert default.effective_level == 10
        assert default.effective_console_level == 10
        assert default.effective_level == expected.effective_level
        assert default.effective_console_level == expected.effective_console_level

    def test_falsy_console_log_level(self) -> None:
        for name, level, console_level in [("warning", 30, None), ("error", 40, ""), ("info", 20, "none")]:
            logs = self.parse(name, console_level)
            assert logs.level == name.upper()
            assert logs.console_level is None
            assert logs.effective_level == level
            assert logs.effective_console_level == level


def test_default_config_does_not_need_ffmpeg() -> None:
    cyberdrop_dl.commands.scrape._check_ffmpeg(Config())


def test_media_durations_need_ffmpeg() -> None:
    config = Config.parse_args(["--video.duration.max", "20 seconds"])
    duration = config.filters.duration.video.max
    assert duration
    assert duration.total_seconds() == 20
    assert config.filters.duration.needs_ffmpeg
    with pytest.raises(CDLConfigRuntimeErrorsGroup) as exc:
        cyberdrop_dl.commands.scrape._check_ffmpeg(config)

    assert len(exc.value.exceptions) == 1
    assert type(exc.value.exceptions[0]) is RuntimeError
    assert str(exc.value.exceptions[0]) == "Filtering files by duration requires 'ffmpeg' to be installed"


def test_resolve_paths(tmp_cwd: Path) -> None:

    class SubConfig(BaseModel):
        path: Path

    class FakeConfig(BaseModel):
        name: str
        child: SubConfig

    config = FakeConfig(
        name="foo",
        child=SubConfig(
            path=Path("/home/user/cdl/{config}/settings.yml"),
        ),
    )

    with pytest.raises(CDLConfigRuntimeErrorsGroup, match="Invalid config") as exc_info:
        _resolve_paths(config)

    assert len(exc_info.value.exceptions) == 1
    first = exc_info.value.exceptions[0]
    assert type(first) is ValueError
    assert "Using '{config}' as reference on a path is no longer supported" in str(first)

    config.child.path = Path("URLs.txt")
    _resolve_paths(config)
    assert config.child.path == tmp_cwd / "URLs.txt"


@pytest.mark.parametrize(
    ("config_args", "cli_args", "expected"),
    [
        (
            [],
            ["a", "b", "c"],
            ["a", "b", "c"],
        ),
        (
            ("e"),
            ("a", "b", "c"),
            ("a", "b", "c"),
        ),
        (
            ("b", "d", "e"),
            ("+", "a", "b", "c"),
            ("a", "b", "c", "d", "e"),
        ),
        (
            ("a", "d", "e"),
            ("-", "a", "b", "c"),
            ("d", "e"),
        ),
        (
            ("drive.google.com", "facebook.com", "youtube.com"),
            ("instagram.com",),
            ("instagram.com",),
        ),
        (
            ("drive.google.com", "facebook.com", "youtube.com"),
            ("+", "instagram.com"),
            ("drive.google.com", "facebook.com", "instagram.com", "youtube.com"),
        ),
        (
            ("drive.google.com", "facebook.com", "youtube.com"),
            ("-", "youtube.com"),
            ("drive.google.com", "facebook.com"),
        ),
    ],
)
def test_additive_args(
    config_args: list[str] | tuple[str, ...],
    cli_args: list[str] | tuple[str, ...],
    expected: list[str] | tuple[str, ...],
) -> None:
    result = merge_additive_args(config_args, cli_args)
    assert type(result) is type(expected)
    assert result == expected


def test_config_from_file(tmp_cwd: Path) -> None:
    config_file = tmp_cwd / "cdl_config.txt"
    config_1 = Config.from_file(config_file)
    assert config_1.source is None
    config_file.touch()
    config_2 = Config.from_file(config_file)
    assert config_2.source == config_file
    assert config_2._sources == (config_file,)
    config_2._sources = ()
    assert config_1 == config_2


@pytest.mark.skipif(os.name == "nt", reason="pydantic can't generate schema w pathlib.WindowsPath defaults")
def test_schema_has_not_changed() -> None:
    schema = Config.model_json_schema()
    expected_schema = json.loads(Files.SCHEMA.read_text(encoding="utf8"))
    assert schema == expected_schema, "Validation schema changed"


def test_round_trip_parsed_the_same_config() -> None:
    config = Config()
    for mode in ("python", "json"):
        new_config = Config.model_validate(config.model_dump(mode=mode))
        assert vars(new_config) == vars(config)
        assert new_config == config


@pytest.mark.skipif(os.name == "nt", reason="OS separators make paths different on Windows")
def test_config_default_has_not_changed() -> None:
    config = Config().model_dump(mode="json")
    expected = yaml.safe_load(Files.DEFAULT.read_text(encoding="utf8"))
    assert config == expected, "Config serialization changed"


def test_config_can_be_serialized_as_json() -> None:
    Config().model_dump_json()


def test_config_defaults_are_valid() -> None:
    class StrictConfig(Config, validate_default=True, validation_error_cause=True): ...

    StrictConfig()


class TestCensoredConfig:
    def test_auth_is_censored_on_repr(self) -> None:
        api_key = "my_api_key"
        auth = Authentication.model_validate({"gofile": {"api_key": api_key}})
        assert api_key not in repr(auth)
        assert auth.gofile.api_key == api_key
        assert auth.model_dump()["gofile"]["api_key"] == api_key
        assert auth.censored_dump()["gofile"] is True
        assert auth.model_dump(mode="json")["gofile"]["api_key"] == api_key

    def test_apprise_urls_are_censored(self) -> None:
        url = AppriseURL.model_validate({"url": "https://example.com/a"})
        assert str(url) == "no_logs=https://example.com/a"
        assert repr(url) == "AppriseURL(url=Secret('**********'), tags={'no_logs'})"
        assert url.model_dump() == "no_logs=https://example.com/a"
        assert url.model_dump(mode="json") == "no_logs=**********"

    def test_notifications_are_censored(self) -> None:
        url = AppriseURL.model_validate({"url": "https://example.com/a"})
        noti = Notifications(apprise=(url,))
        assert "example.com" not in repr(noti)
        assert "example.com" not in str(noti)


class TestAppData:
    def test_default_names(self) -> None:
        appdata = AppData.default()
        assert appdata.cache_file.name == "cache.json"
        assert appdata.db_file.name == "cyberdrop.db"
        assert appdata.config_file.name == "config.yaml"

    def test_default_log_folder_name(self) -> None:
        appdata = AppData.default()
        assert appdata.logs_folder.name == "Logs" if os.name == "nt" else "logs"

    def test_create_w_no_args_is_default(self) -> None:
        assert AppData.create() == AppData.default()


def test_log_folder_after_resolution(tmp_cwd: Path) -> None:
    logs = settings.Logs()
    assert logs.folder is None
    tmp_log_folder = tmp_cwd / "logs"
    logs.resolve_filenames(tmp_log_folder)
    assert logs.folder == tmp_log_folder


def test_config_union_preserves_source() -> None:
    config_1 = Config()
    config_1._sources = (Path("config.yaml"),)
    config_2 = config_1 | Config()
    assert config_1 is not config_2
    assert config_1.source == config_2.source


def test_configs_with_same_values_but_diferrent_sources_are_not_equal() -> None:
    config_1 = Config()
    config_1._sources = (Path("config.yaml"),)
    config_2 = Config()
    config_2._sources = (Path("config2.yaml"),)
    assert config_1 != config_2


def test_config_union() -> None:
    config = Config(ignore_history=True, dump_json=True)

    assert config | Config() == config
    assert config | Config(deep_scrape=True) == Config(ignore_history=True, dump_json=True, deep_scrape=True)

    filters = Filters(files=_FileFilter(audio=False), before=dt.date.today())  # noqa: DTZ011
    config_2 = Config(filters=filters)
    assert config.filters.files.audio is True
    assert config_2.filters.files.audio is False
    assert config | config_2 == Config(ignore_history=True, dump_json=True, filters=filters)


def test_additive_args_list() -> None:
    args = set(Config()._additive_args())
    assert args == {
        ("crawlers", "bandcamp", "formats"),
        ("crawlers", "disabled"),
        ("crawlers", "pornhub", "profile_paths"),
        ("filters", "only_hosts"),
        ("filters", "skip_hosts"),
        ("hashing", "algorithms"),
        ("jdownloader", "whitelist"),
        ("restrict_path",),
    }


def test_merge_does_not_mark_unset_fields_as_set() -> None:
    merged = Config() | Config.parse_args(["--download-folder", "downloads"])
    assert merged.network.model_fields_set == set()
    assert merged.model_fields_set == {"download_folder"}


def test_merge_does_not_warn_about_untouched_deprecated_options(
    caplog: pytest.LogCaptureFixture,
) -> None:
    from cyberdrop_dl.models import _warned

    _warned.clear()
    start = time.monotonic()
    with caplog.at_level("WARNING", logger="cyberdrop_dl.models"):
        _ = Config() | Config.parse_args(["--download-folder", "downloads"])

    assert "deprecated" not in caplog.text
    assert time.monotonic() - start < 1


def test_deprecated_option_set_by_the_user_still_warns(
    caplog: pytest.LogCaptureFixture,
) -> None:
    from cyberdrop_dl.models import _warned

    _warned.clear()
    with caplog.at_level("WARNING", logger="cyberdrop_dl.models"):
        config = Config.load({"network": {"ssl_context": "certifi"}})

    assert "'ssl_context' config option is deprecated" in caplog.text
    with warnings.catch_warnings(action="ignore"):
        assert config.network.ssl_context == "certifi"
        assert (config | Config.parse_args([])).network.ssl_context == "certifi"