Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: remove yaml module (#1983)
NTFSvolumed189cfbed1db
diff --git a/cyberdrop_dl/__init__.py b/cyberdrop_dl/__init__.pyindex e1d4c33c2..f8144818f 100644--- a/cyberdrop_dl/__init__.py+++ b/cyberdrop_dl/__init__.py@@ -1,8 +1,5 @@import importlib.metadataimport re-import sys--_ = sys.path.pop(0)__dist_name__ = "cyberdrop-dl-patched"__version__ = importlib.metadata.version(__dist_name__)diff --git a/cyberdrop_dl/config/__init__.py b/cyberdrop_dl/config/__init__.pyindex fd580fa3b..2bb2e53e3 100644--- a/cyberdrop_dl/config/__init__.py+++ b/cyberdrop_dl/config/__init__.py@@ -4,14 +4,14 @@ import loggingfrom pathlib import Pathfrom typing import TYPE_CHECKING, Annotated, Literal, final+import yamlfrom cyclopts import App, Parameterfrom cyclopts.bind import normalize_tokensfrom pydantic import AfterValidator, BaseModel, Field, NonNegativeInt, PositiveInt-from cyberdrop_dl import yamlfrom cyberdrop_dl.config.appdata import AppDatafrom cyberdrop_dl.constants import DEFAULT_DOWNLOAD_PATH-from cyberdrop_dl.exceptions import CDLConfigRuntimeErrorsGroup+from cyberdrop_dl.exceptions import CDLConfigRuntimeErrorsGroup, InvalidYamlErrorfrom cyberdrop_dl.models import ConfigModelfrom cyberdrop_dl.models.types import ByteSizeSerilized, FalsyAsTuple # noqa: TC001from cyberdrop_dl.models.validators import to_bytesize@@ -34,7 +34,7 @@ _app: App | None = None@finalclass Files:- DEFAULT: Path = MODULE_PATH / "default.json"+ DEFAULT: Path = MODULE_PATH / "default.yaml"SCHEMA: Path = MODULE_PATH / "schema.json"@staticmethod@@ -42,7 +42,7 @@ class Files:import jsonFiles.SCHEMA.write_text(json.dumps(Config.model_json_schema(), indent=2, ensure_ascii=False))- Files.DEFAULT.write_text(Config().model_dump_json(indent=2))+ Config().save_to(Files.DEFAULT)@Parameter(name="*")@@ -120,19 +120,26 @@ class Config(ConfigModel, title="cyberdrop-dl config"):return self._sourcedef save_to(self, file: Path) -> None:- yaml.save(file, self.model_dump(mode="json"))+ file.parent.mkdir(parents=True, exist_ok=True)+ with file.open("w", encoding="utf8") as file_io:+ yaml.safe_dump(self.model_dump(mode="json"), file_io, default_flow_style=False)@staticmethoddef from_file(file: Path, *, _save_if_not_found: bool = False) -> Config:try:- content = yaml.load(file)+ content = file.read_text()except FileNotFoundError:default = Config()if _save_if_not_found:default.save_to(file)return default- config = Config.model_validate(content)+ try:+ data = yaml.safe_load(content) or {}+ except yaml.YAMLError as e:+ raise InvalidYamlError(file, e) from e++ config = Config.model_validate(data)config._source = filereturn configdiff --git a/cyberdrop_dl/config/appdata.py b/cyberdrop_dl/config/appdata.pyindex f738cfa51..2156993db 100644--- a/cyberdrop_dl/config/appdata.py+++ b/cyberdrop_dl/config/appdata.py@@ -3,6 +3,8 @@ from __future__ import annotationsimport dataclassesimport loggingimport os+import sys+import tempfilefrom pathlib import Pathfrom typing import TYPE_CHECKING, final@@ -13,6 +15,7 @@ logger = logging.getLogger(__name__)_win_appdata: Path | None = None_default_app_dirs: AppDirs | None = None_appname = "cyberdrop-dl"+_logs_folder = "Logs" if os.name == "nt" else "logs"def _windows_appdata() -> Path:@@ -80,22 +83,9 @@ class AppDirs:if _default_app_dirs is not None:return _default_app_dirs- if os.name == "nt":- appdata = _windows_appdata()- _default_app_dirs = AppDirs(- cache=appdata,- config=appdata,- data=appdata,- logs=appdata / "Logs",- )-- else:- _default_app_dirs = AppDirs(- cache=_resolve(XDG.CACHE_HOME) / _appname,- config=_resolve(XDG.CONFIG_HOME) / _appname,- data=_resolve(XDG.DATA_HOME) / _appname,- logs=_resolve(XDG.STATE_HOME) / _appname / "logs",- )+ app_dirs = _default_app_dir()+ object.__setattr__(app_dirs, "logs", app_dirs.logs / _logs_folder)+ _default_app_dirs = app_dirsreturn _default_app_dirsdef __iter__(self) -> Iterator[tuple[str, Path]]:@@ -105,6 +95,22 @@ class AppDirs:return {k: str(v) for k, v in self}+def _default_app_dir() -> AppDirs:+ if "pytest" in sys.modules:+ temp_dir = Path(tempfile.TemporaryDirectory(prefix="cdl_", delete=False).name)+ return AppDirs.from_path(temp_dir)++ if os.name == "nt":+ return AppDirs.from_path(_windows_appdata())++ return AppDirs(+ cache=_resolve(XDG.CACHE_HOME) / _appname,+ config=_resolve(XDG.CONFIG_HOME) / _appname,+ data=_resolve(XDG.DATA_HOME) / _appname,+ logs=_resolve(XDG.STATE_HOME) / _appname,+ )++@final@dataclasses.dataclass(slots=True, frozen=True, kw_only=True)class AppData:diff --git a/cyberdrop_dl/config/default.json b/cyberdrop_dl/config/default.jsondeleted file mode 100644index f676c2181..000000000--- a/cyberdrop_dl/config/default.json+++ /dev/null@@ -1,187 +0,0 @@-{- "auth": {- "gofile": {- "api_key": null- },- "jdownloader": {- "username": null,- "password": null,- "device": null- },- "mega_nz": {- "email": null,- "password": null- },- "pixeldrain": {- "api_key": null- },- "real_debrid": {- "api_key": null- }- },- "cookies": null,- "crawlers": {- "disabled": [],- "generic": {- "wordpress_media": [],- "wordpress_html": [],- "discourse": [],- "chevereto": []- },- "kemono": {- "ignore_ads": false,- "ignore_post_content": true- },- "coomer": {- "ignore_ads": false,- "ignore_post_content": true- },- "nekohouse": {- "ignore_ads": false,- "ignore_post_content": true- },- "tiktok": {- "original": false- }- },- "deep_scrape": false,- "delete_empty_folders": true,- "delete_partial_files": false,- "download_folder": "Downloads",- "downloads": {- "concurrency": 15,- "concurrency_per_domain": 5,- "attempts": 2,- "delay": 0.0,- "slow_speed": "0B",- "speed_limit": "0B",- "jitter": 0.0,- "skip_and_mark_completed": false,- "concurrent_segments": 10- },- "dump_json": false,- "filters": {- "files": {- "audio": true,- "images": true,- "videos": true,- "non_media": true- },- "sizes": {- "image": {- "min": "0B",- "max": "0B"- },- "video": {- "min": "0B",- "max": "0B"- },- "audio": {- "min": "0B",- "max": "0B"- },- "non_media": {- "min": "0B",- "max": "0B"- }- },- "duration": {- "video": {- "min": "0:00:00",- "max": "0:00:00"- },- "audio": {- "min": "0:00:00",- "max": "0:00:00"- }- },- "before": null,- "after": null,- "filename_regex": null,- "only_hosts": [],- "skip_hosts": [],- "allow_files_with_no_extension": false- },- "hashing": {- "mode": "in_place",- "algorithms": [- "xxh128",- "md5",- "sha256"- ],- "dedupe": {- "enabled": true,- "use_trash_bin": true- }- },- "ignore_history": false,- "jdownloader": {- "enabled": false,- "autostart": false,- "download_dir": null,- "whitelist": []- },- "logs": {- "level": "DEBUG",- "console_level": null,- "files": {- "main": "downloader.log",- "download_errors": "Download_Error_URLs.csv",- "scrape_errors": "Scrape_Error_URLs.csv",- "unsupported": "Unsupported_URLs.csv"- },- "folder": null,- "expire_after": null,- "rotate": false- },- "max_children": [],- "max_file_name_length": 95,- "max_folder_name_length": 60,- "max_thread_depth": 0,- "max_thread_folder_depth": null,- "min_free_space": "5.0GB",- "mtime": true,- "network": {- "dump_responses": false,- "flaresolverr": null,- "proxy": null,- "rate_limit": 25.0,- "connection_timeout": 15.0,- "read_timeout": 300.0,- "ssl_context": "truststore+certifi",- "user_agent": "Mozilla/5.0 (X11; Linux x86_64; rv:150.0) Gecko/20100101 Firefox/150.0",- "impersonate": null- },- "notifications": {- "apprise": [],- "webhook": null- },- "show_stats": true,- "sort": {- "enabled": false,- "input_folder": null,- "output_folder": "Downloads/Cyberdrop-DL Sorted Downloads",- "formats": {- "audio": "{sort_dir}/{base_dir}/Audio/{filename}{ext}",- "image": "{sort_dir}/{base_dir}/Images/{filename}{ext}",- "non_media": "{sort_dir}/{base_dir}/Other/{filename}{ext}",- "video": "{sort_dir}/{base_dir}/Videos/{filename}{ext}",- "incrementer": " ({i})"- }- },- "subfolders": {- "create": true,- "include": {- "album_id": false,- "thread_id": false,- "domain": true- },- "separate_posts_format": "{default}",- "separate_posts": false- },- "ui": {- "mode": "fullscreen",- "portrait": false,- "refresh_rate": 10.0- }-}diff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yamlnew file mode 100644index 000000000..eef55375e--- /dev/null+++ b/cyberdrop_dl/config/default.yaml@@ -0,0 +1,149 @@+auth:+ gofile:+ api_key: null+ jdownloader:+ device: null+ password: null+ username: null+ mega_nz:+ email: null+ password: null+ pixeldrain:+ api_key: null+ real_debrid:+ api_key: null+cookies: null+crawlers:+ coomer:+ ignore_ads: false+ ignore_post_content: true+ disabled: []+ generic:+ chevereto: []+ discourse: []+ wordpress_html: []+ wordpress_media: []+ kemono:+ ignore_ads: false+ ignore_post_content: true+ nekohouse:+ ignore_ads: false+ ignore_post_content: true+ tiktok:+ original: false+deep_scrape: false+delete_empty_folders: true+delete_partial_files: false+download_folder: Downloads+downloads:+ attempts: 2+ concurrency: 15+ concurrency_per_domain: 5+ concurrent_segments: 10+ delay: 0.0+ jitter: 0.0+ skip_and_mark_completed: false+ slow_speed: 0B+ speed_limit: 0B+dump_json: false+filters:+ after: null+ allow_files_with_no_extension: false+ before: null+ duration:+ audio:+ max: 0:00:00+ min: 0:00:00+ video:+ max: 0:00:00+ min: 0:00:00+ filename_regex: null+ files:+ audio: true+ images: true+ non_media: true+ videos: true+ only_hosts: []+ sizes:+ audio:+ max: 0B+ min: 0B+ image:+ max: 0B+ min: 0B+ non_media:+ max: 0B+ min: 0B+ video:+ max: 0B+ min: 0B+ skip_hosts: []+hashing:+ algorithms:+ - xxh128+ - md5+ - sha256+ dedupe:+ enabled: true+ use_trash_bin: true+ mode: in_place+ignore_history: false+jdownloader:+ autostart: false+ download_dir: null+ enabled: false+ whitelist: []+logs:+ console_level: null+ expire_after: null+ files:+ download_errors: Download_Error_URLs.csv+ main: downloader.log+ scrape_errors: Scrape_Error_URLs.csv+ unsupported: Unsupported_URLs.csv+ folder: null+ level: DEBUG+ rotate: false+max_children: []+max_file_name_length: 95+max_folder_name_length: 60+max_thread_depth: 0+max_thread_folder_depth: null+min_free_space: 5.0GB+mtime: true+network:+ connection_timeout: 15.0+ dump_responses: false+ flaresolverr: null+ impersonate: null+ proxy: null+ rate_limit: 25.0+ read_timeout: 300.0+ ssl_context: truststore+certifi+ user_agent: Mozilla/5.0 (X11; Linux x86_64; rv:150.0) Gecko/20100101 Firefox/150.0+notifications:+ apprise: []+ webhook: null+show_stats: true+sort:+ enabled: false+ formats:+ audio: '{sort_dir}/{base_dir}/Audio/{filename}{ext}'+ image: '{sort_dir}/{base_dir}/Images/{filename}{ext}'+ incrementer: ' ({i})'+ non_media: '{sort_dir}/{base_dir}/Other/{filename}{ext}'+ video: '{sort_dir}/{base_dir}/Videos/{filename}{ext}'+ input_folder: null+ output_folder: Downloads/Cyberdrop-DL Sorted Downloads+subfolders:+ create: true+ include:+ album_id: false+ domain: true+ thread_id: false+ separate_posts: false+ separate_posts_format: '{default}'+ui:+ mode: fullscreen+ portrait: false+ refresh_rate: 10.0diff --git a/cyberdrop_dl/manager.py b/cyberdrop_dl/manager.pyindex e62928603..0afdce890 100644--- a/cyberdrop_dl/manager.py+++ b/cyberdrop_dl/manager.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport contextlibimport datetime+import jsonimport loggingimport osimport sys@@ -11,7 +12,7 @@ from typing import TYPE_CHECKING, Any, Self, finalfrom pydantic.types import ByteSize-from cyberdrop_dl import ALL_DEPENDENCIES, __version__, aio, env, ffmpeg, stats, yaml+from cyberdrop_dl import ALL_DEPENDENCIES, __version__, aio, env, ffmpeg, statsfrom cyberdrop_dl.clients.downloads import DownloadClientfrom cyberdrop_dl.clients.http import HTTPClientfrom cyberdrop_dl.config import Config@@ -185,16 +186,20 @@ class Manager:@contextlib.contextmanagerdef _cache_context(cache_file: Path, cache: dict[str, Any]) -> Generator[None]:try:- cache.update(yaml.load(cache_file))+ content = cache_file.read_text()except FileNotFoundError:cache_file.parent.mkdir(exist_ok=True, parents=True)cache_file.touch()-+ else:+ data = json.loads(content)+ assert type(data) is dict+ cache.update(data)try:yieldfinally:cache["version"] = __version__- yaml.save(cache_file, cache)+ cache_file.parent.mkdir(parents=True, exist_ok=True)+ cache_file.write_text(json.dumps(cache, indent=2, ensure_ascii=False, sort_keys=True))def _log_dependencies() -> None:diff --git a/cyberdrop_dl/yaml.py b/cyberdrop_dl/yaml.pydeleted file mode 100644index adc029e9c..000000000--- a/cyberdrop_dl/yaml.py+++ /dev/null@@ -1,55 +0,0 @@-from __future__ import annotations--import datetime-from enum import Enum-from pathlib import Path, PurePath-from typing import TYPE_CHECKING, Any--import yaml-import yarl--from cyberdrop_dl.exceptions import InvalidYamlError--if TYPE_CHECKING:- from collections.abc import Iterable-- from yaml.nodes import ScalarNode---def _str(dumper: yaml.Dumper, value: object) -> ScalarNode:- return dumper.represent_str(str(value))---def _enum(dumper: yaml.Dumper, value: Enum) -> ScalarNode:- return dumper.represent_str(value.name)---def _date(dumper: yaml.Dumper, value: datetime.date) -> ScalarNode:- return dumper.represent_str(value.isoformat())---def _list(dumper: yaml.Dumper, value: Iterable[Any]):- return dumper.represent_list(value)---yaml.add_multi_representer(PurePath, _str)-yaml.add_multi_representer(Enum, _enum)-yaml.add_multi_representer(datetime.date, _date)-yaml.add_representer(tuple, _list)-yaml.add_representer(datetime.timedelta, _str)-yaml.add_representer(yarl.URL, _str)---def save(file: Path, /, data: dict[str, Any]) -> None:- file.parent.mkdir(parents=True, exist_ok=True)- with file.open("w", encoding="utf8") as file_io:- yaml.dump(data, file_io, default_flow_style=False)---def load(file: Path, /) -> dict[str, Any]:- try:- with file.open("r", encoding="utf8") as file_io:- return yaml.safe_load(file_io) or {}-- except yaml.YAMLError as e:- raise InvalidYamlError(file, e) from ediff --git a/tests/conftest.py b/tests/conftest.pyindex 5f963a4ac..6584ac541 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -62,12 +62,13 @@ async def logs(caplog: pytest.LogCaptureFixture) -> pytest.LogCaptureFixture:@pytest.fixture-def manager(tmp_cwd: Path) -> Generator[Manager]:- with Manager(- appdata=AppData.from_dirs(- AppDirs.from_path(tmp_cwd / "pytest_appdata"),- )- )() as manager:+def appdata(tmp_cwd: Path) -> AppData:+ return AppData.from_dirs(AppDirs.from_path(tmp_cwd / "pytest_appdata"))+++def manager(appdata: AppData) -> Generator[Manager]:+ with Manager(appdata=appdata)() as manager:yield managerdiff --git a/tests/crawlers/test_xenforo.py b/tests/crawlers/test_xenforo.pyindex 12ebb0f0d..bdbc6c934 100644--- a/tests/crawlers/test_xenforo.py+++ b/tests/crawlers/test_xenforo.py@@ -1,8 +1,11 @@from __future__ import annotations+from pathlib import Path+import pytestfrom bs4 import BeautifulSoup+from cyberdrop_dl.config.appdata import AppData, AppDirsfrom cyberdrop_dl.crawlers import _forumfrom cyberdrop_dl.crawlers import xenforo as crawlersfrom cyberdrop_dl.crawlers.xenforo import xenforo@@ -15,7 +18,8 @@ def _item(url: str) -> ScrapeItem:return ScrapeItem.from_url(url)-manager = Manager()+manager = Manager(appdata=AppData.from_dirs(AppDirs.from_path(Path(__file__).parent / "xenforo_appdata")))+scrape_item = _item("https://xenforo.com/community")crawler_instances = {crawler: crawler(manager) for crawler in crawlers.XF_CRAWLERS}TEST_CRAWLER = crawler_instances[crawlers.CelebForumCrawler]diff --git a/tests/test_cache.py b/tests/test_cache.pyindex a970e5b15..fd5e47534 100644--- a/tests/test_cache.py+++ b/tests/test_cache.py@@ -1,12 +1,14 @@from pathlib import Pathfrom typing import Any-from cyberdrop_dl import __version__, yaml+from cyberdrop_dl import __version__+from cyberdrop_dl.config.appdata import AppDatafrom cyberdrop_dl.manager import Manager, _cache_context+from cyberdrop_dl.utils import json-def test_cache_file_is_not_saved_outside_ctx() -> None:- manager = Manager()+def test_cache_file_is_not_saved_outside_ctx(appdata: AppData) -> None:+ manager = Manager(appdata=appdata)cache_file = manager.appdata.cache_filemanager.cache["test"] = 1assert manager.cache == {"test": 1}@@ -22,4 +24,4 @@ def test_cache_file_is_saved_in_ctx(tmp_path: Path) -> None:assert cache_file.is_file()assert cache_file.is_file()- assert yaml.load(cache_file) == {"test": 1, "version": __version__}+ assert json.loads(cache_file.read_text()) == {"test": 1, "version": __version__}diff --git a/tests/test_config.py b/tests/test_config.pyindex ec7146279..01c36857d 100644--- a/tests/test_config.py+++ b/tests/test_config.py@@ -6,6 +6,7 @@ import timefrom pathlib import Pathimport pytest+import yamlfrom cyclopts.exceptions import UnknownOptionErrorfrom pydantic import BaseModel@@ -278,7 +279,7 @@ def test_round_trip_parsed_the_same_config() -> None:@pytest.mark.skipif(os.name == "nt", reason="OS separators make paths different on Windows")def test_config_default_has_not_changed() -> None:config = Config().model_dump(mode="json")- expected = json.loads(Files.DEFAULT.read_text())+ expected = yaml.safe_load(Files.DEFAULT.read_text())assert config == expected, "Config serialization changed"diff --git a/tests/test_manager.py b/tests/test_manager.pyindex 3e8e36155..5575f11ac 100644--- a/tests/test_manager.py+++ b/tests/test_manager.py@@ -15,6 +15,8 @@ from cyberdrop_dl.sorter import Sorterif TYPE_CHECKING:from pydantic import BaseModel+ from cyberdrop_dl.config.appdata import AppData+def update_model[M: BaseModel](model: M, **kwargs: Any) -> M:return model.model_validate(model.model_dump() | kwargs)@@ -42,9 +44,9 @@ def test_args_logging_should_censor_webhook(assert output == webhook_url-def test_manager_context() -> None:+def test_manager_context(appdata: AppData) -> None:config = Config.parse_args(["--refresh-rate", "40"])- manager = Manager(config=config)+ manager = Manager(config=config, appdata=appdata)for attr in ("database", "deduper", "sorter"):with pytest.raises(AttributeError):