Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
misc: update crawler test gen script
NTFSvolume04bdbca266f7
diff --git a/scripts/tools/make_test_cases.py b/scripts/tools/make_test_cases.pyindex 93ecdac8e..47c65fd39 100644--- a/scripts/tools/make_test_cases.py+++ b/scripts/tools/make_test_cases.py@@ -8,12 +8,13 @@ from pathlib import Pathfrom typing import TYPE_CHECKING, Anyfrom cyberdrop_dl.__main__ import run_cdl+from cyberdrop_dl.config.appdata import AppData, AppDirsfrom cyberdrop_dl.crawlers import SKIP_DOWNLOADif TYPE_CHECKING:from collections.abc import Generator-KEYS = ("url", "filename", "debrid_link", "original_filename", "referer", "album_id", "uploaded_at", "download_folder")+KEYS = ("url", "filename", "debrid_url", "original_filename", "referer", "album_id", "uploaded_at", "download_folder")ROOT = Path(__file__).resolve().parents[2]TEST_FOLDER = ROOT / "tests/crawlers/test_cases"@@ -21,7 +22,7 @@ TestCase = dict[str, Any]def parse_jsonl(file: Path) -> Generator[tuple[str, str, TestCase]]:- base = Path.cwd() / "Downloads"+ base = Path.cwd() / "downloads"for line in file.read_text(encoding="utf-8").splitlines():media = json.loads(line)url = media["parents"][0] if media["parents"] else media["referer"]@@ -31,15 +32,20 @@ def parse_jsonl(file: Path) -> Generator[tuple[str, str, TestCase]]:def run(url_txt: Path, main_log: Path) -> None:with tempfile.TemporaryDirectory() as temp:+ appdata = AppData.from_dirs(AppDirs.from_path(Path(temp) / "test_appdata"))+ appdata.config_file.parent.mkdir(parents=True, exist_ok=True)+ appdata.config_file.touch()_ = SKIP_DOWNLOAD.set(True)_ = run_cdl([- "--download",- "--appdata-folder",- temp,+ "download",+ "--database-file",+ str(appdata.db_file),+ "--config-file",+ str(appdata.config_file),"--input-file",str(url_txt),- "--main-log",+ "--log-file",str(main_log),"--dump-json","--ui",