Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: remove yaml module (#1983)

NTFSvolumed189cfbed1db
diff --git a/cyberdrop_dl/__init__.py b/cyberdrop_dl/__init__.py
index e1d4c33c2..f8144818f 100644
--- a/cyberdrop_dl/__init__.py
+++ b/cyberdrop_dl/__init__.py
@@ -1,8 +1,5 @@
import importlib.metadata
import re
-import sys
-
-_ = sys.path.pop(0)
__dist_name__ = "cyberdrop-dl-patched"
__version__ = importlib.metadata.version(__dist_name__)
diff --git a/cyberdrop_dl/config/__init__.py b/cyberdrop_dl/config/__init__.py
index fd580fa3b..2bb2e53e3 100644
--- a/cyberdrop_dl/config/__init__.py
+++ b/cyberdrop_dl/config/__init__.py
@@ -4,14 +4,14 @@ import logging
from pathlib import Path
from typing import TYPE_CHECKING, Annotated, Literal, final
+import yaml
from cyclopts import App, Parameter
from cyclopts.bind import normalize_tokens
from pydantic import AfterValidator, BaseModel, Field, NonNegativeInt, PositiveInt
-from cyberdrop_dl import yaml
from cyberdrop_dl.config.appdata import AppData
from cyberdrop_dl.constants import DEFAULT_DOWNLOAD_PATH
-from cyberdrop_dl.exceptions import CDLConfigRuntimeErrorsGroup
+from cyberdrop_dl.exceptions import CDLConfigRuntimeErrorsGroup, InvalidYamlError
from cyberdrop_dl.models import ConfigModel
from cyberdrop_dl.models.types import ByteSizeSerilized, FalsyAsTuple # noqa: TC001
from cyberdrop_dl.models.validators import to_bytesize
@@ -34,7 +34,7 @@ _app: App | None = None
@final
class Files:
- DEFAULT: Path = MODULE_PATH / "default.json"
+ DEFAULT: Path = MODULE_PATH / "default.yaml"
SCHEMA: Path = MODULE_PATH / "schema.json"
@staticmethod
@@ -42,7 +42,7 @@ class Files:
import json
Files.SCHEMA.write_text(json.dumps(Config.model_json_schema(), indent=2, ensure_ascii=False))
- Files.DEFAULT.write_text(Config().model_dump_json(indent=2))
+ Config().save_to(Files.DEFAULT)
@Parameter(name="*")
@@ -120,19 +120,26 @@ class Config(ConfigModel, title="cyberdrop-dl config"):
return self._source
def save_to(self, file: Path) -> None:
- yaml.save(file, self.model_dump(mode="json"))
+ file.parent.mkdir(parents=True, exist_ok=True)
+ with file.open("w", encoding="utf8") as file_io:
+ yaml.safe_dump(self.model_dump(mode="json"), file_io, default_flow_style=False)
@staticmethod
def from_file(file: Path, *, _save_if_not_found: bool = False) -> Config:
try:
- content = yaml.load(file)
+ content = file.read_text()
except FileNotFoundError:
default = Config()
if _save_if_not_found:
default.save_to(file)
return default
- config = Config.model_validate(content)
+ try:
+ data = yaml.safe_load(content) or {}
+ except yaml.YAMLError as e:
+ raise InvalidYamlError(file, e) from e
+
+ config = Config.model_validate(data)
config._source = file
return config
diff --git a/cyberdrop_dl/config/appdata.py b/cyberdrop_dl/config/appdata.py
index f738cfa51..2156993db 100644
--- a/cyberdrop_dl/config/appdata.py
+++ b/cyberdrop_dl/config/appdata.py
@@ -3,6 +3,8 @@ from __future__ import annotations
import dataclasses
import logging
import os
+import sys
+import tempfile
from pathlib import Path
from typing import TYPE_CHECKING, final
@@ -13,6 +15,7 @@ logger = logging.getLogger(__name__)
_win_appdata: Path | None = None
_default_app_dirs: AppDirs | None = None
_appname = "cyberdrop-dl"
+_logs_folder = "Logs" if os.name == "nt" else "logs"
def _windows_appdata() -> Path:
@@ -80,22 +83,9 @@ class AppDirs:
if _default_app_dirs is not None:
return _default_app_dirs
- if os.name == "nt":
- appdata = _windows_appdata()
- _default_app_dirs = AppDirs(
- cache=appdata,
- config=appdata,
- data=appdata,
- logs=appdata / "Logs",
- )
-
- else:
- _default_app_dirs = AppDirs(
- cache=_resolve(XDG.CACHE_HOME) / _appname,
- config=_resolve(XDG.CONFIG_HOME) / _appname,
- data=_resolve(XDG.DATA_HOME) / _appname,
- logs=_resolve(XDG.STATE_HOME) / _appname / "logs",
- )
+ app_dirs = _default_app_dir()
+ object.__setattr__(app_dirs, "logs", app_dirs.logs / _logs_folder)
+ _default_app_dirs = app_dirs
return _default_app_dirs
def __iter__(self) -> Iterator[tuple[str, Path]]:
@@ -105,6 +95,22 @@ class AppDirs:
return {k: str(v) for k, v in self}
+def _default_app_dir() -> AppDirs:
+ if "pytest" in sys.modules:
+ temp_dir = Path(tempfile.TemporaryDirectory(prefix="cdl_", delete=False).name)
+ return AppDirs.from_path(temp_dir)
+
+ if os.name == "nt":
+ return AppDirs.from_path(_windows_appdata())
+
+ return AppDirs(
+ cache=_resolve(XDG.CACHE_HOME) / _appname,
+ config=_resolve(XDG.CONFIG_HOME) / _appname,
+ data=_resolve(XDG.DATA_HOME) / _appname,
+ logs=_resolve(XDG.STATE_HOME) / _appname,
+ )
+
+
@final
@dataclasses.dataclass(slots=True, frozen=True, kw_only=True)
class AppData:
diff --git a/cyberdrop_dl/config/default.json b/cyberdrop_dl/config/default.json
deleted file mode 100644
index f676c2181..000000000
--- a/cyberdrop_dl/config/default.json
+++ /dev/null
@@ -1,187 +0,0 @@
-{
- "auth": {
- "gofile": {
- "api_key": null
- },
- "jdownloader": {
- "username": null,
- "password": null,
- "device": null
- },
- "mega_nz": {
- "email": null,
- "password": null
- },
- "pixeldrain": {
- "api_key": null
- },
- "real_debrid": {
- "api_key": null
- }
- },
- "cookies": null,
- "crawlers": {
- "disabled": [],
- "generic": {
- "wordpress_media": [],
- "wordpress_html": [],
- "discourse": [],
- "chevereto": []
- },
- "kemono": {
- "ignore_ads": false,
- "ignore_post_content": true
- },
- "coomer": {
- "ignore_ads": false,
- "ignore_post_content": true
- },
- "nekohouse": {
- "ignore_ads": false,
- "ignore_post_content": true
- },
- "tiktok": {
- "original": false
- }
- },
- "deep_scrape": false,
- "delete_empty_folders": true,
- "delete_partial_files": false,
- "download_folder": "Downloads",
- "downloads": {
- "concurrency": 15,
- "concurrency_per_domain": 5,
- "attempts": 2,
- "delay": 0.0,
- "slow_speed": "0B",
- "speed_limit": "0B",
- "jitter": 0.0,
- "skip_and_mark_completed": false,
- "concurrent_segments": 10
- },
- "dump_json": false,
- "filters": {
- "files": {
- "audio": true,
- "images": true,
- "videos": true,
- "non_media": true
- },
- "sizes": {
- "image": {
- "min": "0B",
- "max": "0B"
- },
- "video": {
- "min": "0B",
- "max": "0B"
- },
- "audio": {
- "min": "0B",
- "max": "0B"
- },
- "non_media": {
- "min": "0B",
- "max": "0B"
- }
- },
- "duration": {
- "video": {
- "min": "0:00:00",
- "max": "0:00:00"
- },
- "audio": {
- "min": "0:00:00",
- "max": "0:00:00"
- }
- },
- "before": null,
- "after": null,
- "filename_regex": null,
- "only_hosts": [],
- "skip_hosts": [],
- "allow_files_with_no_extension": false
- },
- "hashing": {
- "mode": "in_place",
- "algorithms": [
- "xxh128",
- "md5",
- "sha256"
- ],
- "dedupe": {
- "enabled": true,
- "use_trash_bin": true
- }
- },
- "ignore_history": false,
- "jdownloader": {
- "enabled": false,
- "autostart": false,
- "download_dir": null,
- "whitelist": []
- },
- "logs": {
- "level": "DEBUG",
- "console_level": null,
- "files": {
- "main": "downloader.log",
- "download_errors": "Download_Error_URLs.csv",
- "scrape_errors": "Scrape_Error_URLs.csv",
- "unsupported": "Unsupported_URLs.csv"
- },
- "folder": null,
- "expire_after": null,
- "rotate": false
- },
- "max_children": [],
- "max_file_name_length": 95,
- "max_folder_name_length": 60,
- "max_thread_depth": 0,
- "max_thread_folder_depth": null,
- "min_free_space": "5.0GB",
- "mtime": true,
- "network": {
- "dump_responses": false,
- "flaresolverr": null,
- "proxy": null,
- "rate_limit": 25.0,
- "connection_timeout": 15.0,
- "read_timeout": 300.0,
- "ssl_context": "truststore+certifi",
- "user_agent": "Mozilla/5.0 (X11; Linux x86_64; rv:150.0) Gecko/20100101 Firefox/150.0",
- "impersonate": null
- },
- "notifications": {
- "apprise": [],
- "webhook": null
- },
- "show_stats": true,
- "sort": {
- "enabled": false,
- "input_folder": null,
- "output_folder": "Downloads/Cyberdrop-DL Sorted Downloads",
- "formats": {
- "audio": "{sort_dir}/{base_dir}/Audio/{filename}{ext}",
- "image": "{sort_dir}/{base_dir}/Images/{filename}{ext}",
- "non_media": "{sort_dir}/{base_dir}/Other/{filename}{ext}",
- "video": "{sort_dir}/{base_dir}/Videos/{filename}{ext}",
- "incrementer": " ({i})"
- }
- },
- "subfolders": {
- "create": true,
- "include": {
- "album_id": false,
- "thread_id": false,
- "domain": true
- },
- "separate_posts_format": "{default}",
- "separate_posts": false
- },
- "ui": {
- "mode": "fullscreen",
- "portrait": false,
- "refresh_rate": 10.0
- }
-}
diff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yaml
new file mode 100644
index 000000000..eef55375e
--- /dev/null
+++ b/cyberdrop_dl/config/default.yaml
@@ -0,0 +1,149 @@
+auth:
+ gofile:
+ api_key: null
+ jdownloader:
+ device: null
+ password: null
+ username: null
+ mega_nz:
+ email: null
+ password: null
+ pixeldrain:
+ api_key: null
+ real_debrid:
+ api_key: null
+cookies: null
+crawlers:
+ coomer:
+ ignore_ads: false
+ ignore_post_content: true
+ disabled: []
+ generic:
+ chevereto: []
+ discourse: []
+ wordpress_html: []
+ wordpress_media: []
+ kemono:
+ ignore_ads: false
+ ignore_post_content: true
+ nekohouse:
+ ignore_ads: false
+ ignore_post_content: true
+ tiktok:
+ original: false
+deep_scrape: false
+delete_empty_folders: true
+delete_partial_files: false
+download_folder: Downloads
+downloads:
+ attempts: 2
+ concurrency: 15
+ concurrency_per_domain: 5
+ concurrent_segments: 10
+ delay: 0.0
+ jitter: 0.0
+ skip_and_mark_completed: false
+ slow_speed: 0B
+ speed_limit: 0B
+dump_json: false
+filters:
+ after: null
+ allow_files_with_no_extension: false
+ before: null
+ duration:
+ audio:
+ max: 0:00:00
+ min: 0:00:00
+ video:
+ max: 0:00:00
+ min: 0:00:00
+ filename_regex: null
+ files:
+ audio: true
+ images: true
+ non_media: true
+ videos: true
+ only_hosts: []
+ sizes:
+ audio:
+ max: 0B
+ min: 0B
+ image:
+ max: 0B
+ min: 0B
+ non_media:
+ max: 0B
+ min: 0B
+ video:
+ max: 0B
+ min: 0B
+ skip_hosts: []
+hashing:
+ algorithms:
+ - xxh128
+ - md5
+ - sha256
+ dedupe:
+ enabled: true
+ use_trash_bin: true
+ mode: in_place
+ignore_history: false
+jdownloader:
+ autostart: false
+ download_dir: null
+ enabled: false
+ whitelist: []
+logs:
+ console_level: null
+ expire_after: null
+ files:
+ download_errors: Download_Error_URLs.csv
+ main: downloader.log
+ scrape_errors: Scrape_Error_URLs.csv
+ unsupported: Unsupported_URLs.csv
+ folder: null
+ level: DEBUG
+ rotate: false
+max_children: []
+max_file_name_length: 95
+max_folder_name_length: 60
+max_thread_depth: 0
+max_thread_folder_depth: null
+min_free_space: 5.0GB
+mtime: true
+network:
+ connection_timeout: 15.0
+ dump_responses: false
+ flaresolverr: null
+ impersonate: null
+ proxy: null
+ rate_limit: 25.0
+ read_timeout: 300.0
+ ssl_context: truststore+certifi
+ user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:150.0) Gecko/20100101 Firefox/150.0
+notifications:
+ apprise: []
+ webhook: null
+show_stats: true
+sort:
+ enabled: false
+ formats:
+ audio: '{sort_dir}/{base_dir}/Audio/{filename}{ext}'
+ image: '{sort_dir}/{base_dir}/Images/{filename}{ext}'
+ incrementer: ' ({i})'
+ non_media: '{sort_dir}/{base_dir}/Other/{filename}{ext}'
+ video: '{sort_dir}/{base_dir}/Videos/{filename}{ext}'
+ input_folder: null
+ output_folder: Downloads/Cyberdrop-DL Sorted Downloads
+subfolders:
+ create: true
+ include:
+ album_id: false
+ domain: true
+ thread_id: false
+ separate_posts: false
+ separate_posts_format: '{default}'
+ui:
+ mode: fullscreen
+ portrait: false
+ refresh_rate: 10.0
diff --git a/cyberdrop_dl/manager.py b/cyberdrop_dl/manager.py
index e62928603..0afdce890 100644
--- a/cyberdrop_dl/manager.py
+++ b/cyberdrop_dl/manager.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import contextlib
import datetime
+import json
import logging
import os
import sys
@@ -11,7 +12,7 @@ from typing import TYPE_CHECKING, Any, Self, final
from pydantic.types import ByteSize
-from cyberdrop_dl import ALL_DEPENDENCIES, __version__, aio, env, ffmpeg, stats, yaml
+from cyberdrop_dl import ALL_DEPENDENCIES, __version__, aio, env, ffmpeg, stats
from cyberdrop_dl.clients.downloads import DownloadClient
from cyberdrop_dl.clients.http import HTTPClient
from cyberdrop_dl.config import Config
@@ -185,16 +186,20 @@ class Manager:
@contextlib.contextmanager
def _cache_context(cache_file: Path, cache: dict[str, Any]) -> Generator[None]:
try:
- cache.update(yaml.load(cache_file))
+ content = cache_file.read_text()
except FileNotFoundError:
cache_file.parent.mkdir(exist_ok=True, parents=True)
cache_file.touch()
-
+ else:
+ data = json.loads(content)
+ assert type(data) is dict
+ cache.update(data)
try:
yield
finally:
cache["version"] = __version__
- yaml.save(cache_file, cache)
+ cache_file.parent.mkdir(parents=True, exist_ok=True)
+ cache_file.write_text(json.dumps(cache, indent=2, ensure_ascii=False, sort_keys=True))
def _log_dependencies() -> None:
diff --git a/cyberdrop_dl/yaml.py b/cyberdrop_dl/yaml.py
deleted file mode 100644
index adc029e9c..000000000
--- a/cyberdrop_dl/yaml.py
+++ /dev/null
@@ -1,55 +0,0 @@
-from __future__ import annotations
-
-import datetime
-from enum import Enum
-from pathlib import Path, PurePath
-from typing import TYPE_CHECKING, Any
-
-import yaml
-import yarl
-
-from cyberdrop_dl.exceptions import InvalidYamlError
-
-if TYPE_CHECKING:
- from collections.abc import Iterable
-
- from yaml.nodes import ScalarNode
-
-
-def _str(dumper: yaml.Dumper, value: object) -> ScalarNode:
- return dumper.represent_str(str(value))
-
-
-def _enum(dumper: yaml.Dumper, value: Enum) -> ScalarNode:
- return dumper.represent_str(value.name)
-
-
-def _date(dumper: yaml.Dumper, value: datetime.date) -> ScalarNode:
- return dumper.represent_str(value.isoformat())
-
-
-def _list(dumper: yaml.Dumper, value: Iterable[Any]):
- return dumper.represent_list(value)
-
-
-yaml.add_multi_representer(PurePath, _str)
-yaml.add_multi_representer(Enum, _enum)
-yaml.add_multi_representer(datetime.date, _date)
-yaml.add_representer(tuple, _list)
-yaml.add_representer(datetime.timedelta, _str)
-yaml.add_representer(yarl.URL, _str)
-
-
-def save(file: Path, /, data: dict[str, Any]) -> None:
- file.parent.mkdir(parents=True, exist_ok=True)
- with file.open("w", encoding="utf8") as file_io:
- yaml.dump(data, file_io, default_flow_style=False)
-
-
-def load(file: Path, /) -> dict[str, Any]:
- try:
- with file.open("r", encoding="utf8") as file_io:
- return yaml.safe_load(file_io) or {}
-
- except yaml.YAMLError as e:
- raise InvalidYamlError(file, e) from e
diff --git a/tests/conftest.py b/tests/conftest.py
index 5f963a4ac..6584ac541 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -62,12 +62,13 @@ async def logs(caplog: pytest.LogCaptureFixture) -> pytest.LogCaptureFixture:
@pytest.fixture
-def manager(tmp_cwd: Path) -> Generator[Manager]:
- with Manager(
- appdata=AppData.from_dirs(
- AppDirs.from_path(tmp_cwd / "pytest_appdata"),
- )
- )() as manager:
+def appdata(tmp_cwd: Path) -> AppData:
+ return AppData.from_dirs(AppDirs.from_path(tmp_cwd / "pytest_appdata"))
+
+
+def manager(appdata: AppData) -> Generator[Manager]:
+ with Manager(appdata=appdata)() as manager:
yield manager
diff --git a/tests/crawlers/test_xenforo.py b/tests/crawlers/test_xenforo.py
index 12ebb0f0d..bdbc6c934 100644
--- a/tests/crawlers/test_xenforo.py
+++ b/tests/crawlers/test_xenforo.py
@@ -1,8 +1,11 @@
from __future__ import annotations
+from pathlib import Path
+
import pytest
from bs4 import BeautifulSoup
+from cyberdrop_dl.config.appdata import AppData, AppDirs
from cyberdrop_dl.crawlers import _forum
from cyberdrop_dl.crawlers import xenforo as crawlers
from cyberdrop_dl.crawlers.xenforo import xenforo
@@ -15,7 +18,8 @@ def _item(url: str) -> ScrapeItem:
return ScrapeItem.from_url(url)
-manager = Manager()
+manager = Manager(appdata=AppData.from_dirs(AppDirs.from_path(Path(__file__).parent / "xenforo_appdata")))
+
scrape_item = _item("https://xenforo.com/community")
crawler_instances = {crawler: crawler(manager) for crawler in crawlers.XF_CRAWLERS}
TEST_CRAWLER = crawler_instances[crawlers.CelebForumCrawler]
diff --git a/tests/test_cache.py b/tests/test_cache.py
index a970e5b15..fd5e47534 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -1,12 +1,14 @@
from pathlib import Path
from typing import Any
-from cyberdrop_dl import __version__, yaml
+from cyberdrop_dl import __version__
+from cyberdrop_dl.config.appdata import AppData
from cyberdrop_dl.manager import Manager, _cache_context
+from cyberdrop_dl.utils import json
-def test_cache_file_is_not_saved_outside_ctx() -> None:
- manager = Manager()
+def test_cache_file_is_not_saved_outside_ctx(appdata: AppData) -> None:
+ manager = Manager(appdata=appdata)
cache_file = manager.appdata.cache_file
manager.cache["test"] = 1
assert manager.cache == {"test": 1}
@@ -22,4 +24,4 @@ def test_cache_file_is_saved_in_ctx(tmp_path: Path) -> None:
assert cache_file.is_file()
assert cache_file.is_file()
- assert yaml.load(cache_file) == {"test": 1, "version": __version__}
+ assert json.loads(cache_file.read_text()) == {"test": 1, "version": __version__}
diff --git a/tests/test_config.py b/tests/test_config.py
index ec7146279..01c36857d 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -6,6 +6,7 @@ import time
from pathlib import Path
import pytest
+import yaml
from cyclopts.exceptions import UnknownOptionError
from pydantic import BaseModel
@@ -278,7 +279,7 @@ def test_round_trip_parsed_the_same_config() -> None:
@pytest.mark.skipif(os.name == "nt", reason="OS separators make paths different on Windows")
def test_config_default_has_not_changed() -> None:
config = Config().model_dump(mode="json")
- expected = json.loads(Files.DEFAULT.read_text())
+ expected = yaml.safe_load(Files.DEFAULT.read_text())
assert config == expected, "Config serialization changed"
diff --git a/tests/test_manager.py b/tests/test_manager.py
index 3e8e36155..5575f11ac 100644
--- a/tests/test_manager.py
+++ b/tests/test_manager.py
@@ -15,6 +15,8 @@ from cyberdrop_dl.sorter import Sorter
if TYPE_CHECKING:
from pydantic import BaseModel
+ from cyberdrop_dl.config.appdata import AppData
+
def update_model[M: BaseModel](model: M, **kwargs: Any) -> M:
return model.model_validate(model.model_dump() | kwargs)
@@ -42,9 +44,9 @@ def test_args_logging_should_censor_webhook(
assert output == webhook_url
-def test_manager_context() -> None:
+def test_manager_context(appdata: AppData) -> None:
config = Config.parse_args(["--refresh-rate", "40"])
- manager = Manager(config=config)
+ manager = Manager(config=config, appdata=appdata)
for attr in ("database", "deduper", "sorter"):
with pytest.raises(AttributeError):