Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
ci/cd: auto generate crawler tests (#134)
NTFSvolume94de0db7e16e
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 4e6767e60..26d5edb56 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -11,10 +11,11 @@ import reimport weakreffrom abc import ABC, abstractmethodfrom collections import Counter+from contextvars import ContextVarfrom functools import wrapsfrom pathlib import Pathfrom types import MappingProxyType-from typing import TYPE_CHECKING, Any, ClassVar, Concatenate, Final, Literal, ParamSpec, Self, TypeAlias, TypeVar, final+from typing import TYPE_CHECKING, Any, ClassVar, Concatenate, Final, Literal, ParamSpec, Self, TypeVar, finalfrom aiolimiter import AsyncLimiterfrom typing_extensions import deprecated@@ -54,10 +55,11 @@ _P = ParamSpec("_P")_R = TypeVar("_R")_T = TypeVar("_T")-OneOrTuple: TypeAlias = _T | tuple[_T, ...]-SupportedPaths: TypeAlias = dict[str, OneOrTuple[str]]-SupportedDomains: TypeAlias = OneOrTuple[str]+OneOrTuple = _T | tuple[_T, ...]+SupportedPaths = dict[str, OneOrTuple[str]]+SupportedDomains = OneOrTuple[str]RateLimit = tuple[float, float]+SKIP_DOWNLOAD: ContextVar[bool] = ContextVar("SKIP_DOWNLOAD", default=False)_HASH_PREFIXES = "md5:", "sha1:", "sha256:", "xxh128:"@@ -446,6 +448,8 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):@finalasync def _download(self, media_item: MediaItem, m3u8: m3u8.Rendition | None) -> None:try:+ if SKIP_DOWNLOAD.get():+ returnif m3u8:await self.downloader.download_hls(media_item, m3u8)else:diff --git a/scripts/tools/make_test_cases.py b/scripts/tools/make_test_cases.pynew file mode 100644index 000000000..dd3288547--- /dev/null+++ b/scripts/tools/make_test_cases.py@@ -0,0 +1,72 @@+from __future__ import annotations++import json+import subprocess+import sys+import tempfile+from pathlib import Path+from typing import TYPE_CHECKING, Any++from cyberdrop_dl.__main__ import main as run_cdl+from cyberdrop_dl.crawlers.crawler import SKIP_DOWNLOAD++if TYPE_CHECKING:+ from collections.abc import Generator++KEYS = ("url", "filename", "debrid_link", "original_filename", "referer", "album_id", "uploaded_at", "download_folder")+ROOT = Path(__file__).resolve().parents[2]+TEST_FOLDER = ROOT / "tests/crawlers/test_cases"++TestCase = dict[str, Any]+++def parse_jsonl(file: Path) -> Generator[tuple[str, str, TestCase]]:+ for line in file.read_text().splitlines():+ media = json.loads(line)+ url = media["parents"][0] if media["parents"] else media["referer"]+ media["download_folder"] = "re:" + media["download_folder"].split("/")[-1]+ yield media["domain"], url, {key: media[key] for key in KEYS}+++def run(url_txt: Path, main_log: Path) -> None:+ with tempfile.TemporaryDirectory() as temp:+ _ = SKIP_DOWNLOAD.set(True)+ _ = run_cdl(+ [+ "--download",+ "--appdata-folder",+ temp,+ "--input-file",+ str(url_txt),+ "--main-log",+ str(main_log),+ "--dump-json",+ "--ui",+ "simple",+ ]+ )+++def create_test_files(file: Path) -> None:+ all_results: dict[str, dict[str, list[TestCase]]] = {}+ for domain, url, results in parse_jsonl(file):+ site_tests = all_results.setdefault(domain, {})+ site_tests.setdefault(url, []).append(results)++ test_files: list[Path] = []+ for domain, cases in all_results.items():+ domain = domain.replace(".", "_")+ test_cases = [(url, results) for url, results in cases.items()]+ test_file = TEST_FOLDER / f"test_case_{domain}.py"+ _ = test_file.write_text(f"DOMAIN = {domain!r}\nTEST_CASES = {test_cases}")+ test_files.append(test_file)++ if test_files:+ _ = subprocess.run(["ruff", "format", *test_files], check=False)+++if __name__ == "__main__":+ main_log, url_txt = [Path(file).resolve() for file in (*sys.argv[1:], "test_run.log", "URLs.txt")[:2]]+ run(url_txt, main_log)+ json_l = main_log.with_suffix(".results.jsonl")+ create_test_files(json_l)