Skip to content

Bulk asynchronous downloader for multiple file hosts

ci/cd: auto generate crawler tests (#134)

NTFSvolume94de0db7e16e
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 4e6767e60..26d5edb56 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -11,10 +11,11 @@ import re
import weakref
from abc import ABC, abstractmethod
from collections import Counter
+from contextvars import ContextVar
from functools import wraps
from pathlib import Path
from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, ClassVar, Concatenate, Final, Literal, ParamSpec, Self, TypeAlias, TypeVar, final
+from typing import TYPE_CHECKING, Any, ClassVar, Concatenate, Final, Literal, ParamSpec, Self, TypeVar, final
from aiolimiter import AsyncLimiter
from typing_extensions import deprecated
@@ -54,10 +55,11 @@ _P = ParamSpec("_P")
_R = TypeVar("_R")
_T = TypeVar("_T")
-OneOrTuple: TypeAlias = _T | tuple[_T, ...]
-SupportedPaths: TypeAlias = dict[str, OneOrTuple[str]]
-SupportedDomains: TypeAlias = OneOrTuple[str]
+OneOrTuple = _T | tuple[_T, ...]
+SupportedPaths = dict[str, OneOrTuple[str]]
+SupportedDomains = OneOrTuple[str]
RateLimit = tuple[float, float]
+SKIP_DOWNLOAD: ContextVar[bool] = ContextVar("SKIP_DOWNLOAD", default=False)
_HASH_PREFIXES = "md5:", "sha1:", "sha256:", "xxh128:"
@@ -446,6 +448,8 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
@final
async def _download(self, media_item: MediaItem, m3u8: m3u8.Rendition | None) -> None:
try:
+ if SKIP_DOWNLOAD.get():
+ return
if m3u8:
await self.downloader.download_hls(media_item, m3u8)
else:
diff --git a/scripts/tools/make_test_cases.py b/scripts/tools/make_test_cases.py
new file mode 100644
index 000000000..dd3288547
--- /dev/null
+++ b/scripts/tools/make_test_cases.py
@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from cyberdrop_dl.__main__ import main as run_cdl
+from cyberdrop_dl.crawlers.crawler import SKIP_DOWNLOAD
+
+if TYPE_CHECKING:
+ from collections.abc import Generator
+
+KEYS = ("url", "filename", "debrid_link", "original_filename", "referer", "album_id", "uploaded_at", "download_folder")
+ROOT = Path(__file__).resolve().parents[2]
+TEST_FOLDER = ROOT / "tests/crawlers/test_cases"
+
+TestCase = dict[str, Any]
+
+
+def parse_jsonl(file: Path) -> Generator[tuple[str, str, TestCase]]:
+ for line in file.read_text().splitlines():
+ media = json.loads(line)
+ url = media["parents"][0] if media["parents"] else media["referer"]
+ media["download_folder"] = "re:" + media["download_folder"].split("/")[-1]
+ yield media["domain"], url, {key: media[key] for key in KEYS}
+
+
+def run(url_txt: Path, main_log: Path) -> None:
+ with tempfile.TemporaryDirectory() as temp:
+ _ = SKIP_DOWNLOAD.set(True)
+ _ = run_cdl(
+ [
+ "--download",
+ "--appdata-folder",
+ temp,
+ "--input-file",
+ str(url_txt),
+ "--main-log",
+ str(main_log),
+ "--dump-json",
+ "--ui",
+ "simple",
+ ]
+ )
+
+
+def create_test_files(file: Path) -> None:
+ all_results: dict[str, dict[str, list[TestCase]]] = {}
+ for domain, url, results in parse_jsonl(file):
+ site_tests = all_results.setdefault(domain, {})
+ site_tests.setdefault(url, []).append(results)
+
+ test_files: list[Path] = []
+ for domain, cases in all_results.items():
+ domain = domain.replace(".", "_")
+ test_cases = [(url, results) for url, results in cases.items()]
+ test_file = TEST_FOLDER / f"test_case_{domain}.py"
+ _ = test_file.write_text(f"DOMAIN = {domain!r}\nTEST_CASES = {test_cases}")
+ test_files.append(test_file)
+
+ if test_files:
+ _ = subprocess.run(["ruff", "format", *test_files], check=False)
+
+
+if __name__ == "__main__":
+ main_log, url_txt = [Path(file).resolve() for file in (*sys.argv[1:], "test_run.log", "URLs.txt")[:2]]
+ run(url_txt, main_log)
+ json_l = main_log.with_suffix(".results.jsonl")
+ create_test_files(json_l)