Skip to content

Bulk asynchronous downloader for multiple file hosts

3.7 KB
Raw Blame
from __future__ import annotations

import json
import subprocess
import tempfile
from pathlib import Path
from typing import TYPE_CHECKING, Any

from cyclopts import App

from cyberdrop_dl.__main__ import run_cdl
from cyberdrop_dl.config import Config
from cyberdrop_dl.config.appdata import AppData, AppDirs
from cyberdrop_dl.crawlers import SKIP_DOWNLOAD

if TYPE_CHECKING:
    from collections.abc import Generator, Sequence

KEYS = (
    "url",
    "filename",
    "debrid_url",
    "original_filename",
    "referer",
    "album_id",
    "uploaded_at",
    "download_folder",
    "thumbnail",
    "size",
    "duration",
    "parents",
    "metadata",
    "extra_info",
    "id",
    "base64_id",
    "headers",
)
ROOT = Path(__file__).resolve().parents[2]
TEST_FOLDER = ROOT / "tests/crawlers/test_cases"

TestCase = dict[str, Any]


def parse_jsonl(file: Path, keys: Sequence[str]) -> Generator[tuple[str, str, TestCase]]:
    base = Path.cwd() / Config().download_folder
    for line in file.read_text(encoding="utf-8").splitlines():
        media = json.loads(line)
        if media["url"].startswith("metadata:"):
            continue

        url = media["parents"][0] if media["parents"] else media["referer"]
        media["download_folder"] = "re:" + str(Path(media["download_folder"]).relative_to(base))
        yield media["domain"], url, {key: media[key] for key in keys}


def run_test(url_txt: Path, main_log: Path) -> None:
    with tempfile.TemporaryDirectory(prefix="cdl_test") as temp:
        appdata = AppData.from_dirs(AppDirs.from_path(Path(temp) / "test_appdata"))
        appdata.config_file.parent.mkdir(parents=True, exist_ok=True)
        appdata.config_file.touch()
        appdata.db_file.touch()
        _ = SKIP_DOWNLOAD.set(True)
        _ = run_cdl(
            [
                "download",
                "--database-file",
                str(appdata.db_file),
                "--config-file",
                str(appdata.config_file),
                "--cookies",
                "cookies.txt",
                "--input-file",
                str(url_txt),
                "--log-file",
                str(main_log),
                "--flaresolverr",
                "http://localhost:8191",
                "--no-flaresolverr-use-session",
                "--flaresolverr-concurrency",
                "3",
                "--dump-json",
                "--ui",
                "simple",
            ]
        )


def create_test_files(file: Path) -> None:
    all_results: dict[str, dict[str, list[TestCase]]] = {}
    if not file.exists():
        return
    for domain, url, results in parse_jsonl(file, KEYS):
        site_tests = all_results.setdefault(domain, {})
        site_tests.setdefault(url, []).append(results)

    test_files: list[Path] = []
    for domain, cases in all_results.items():
        test_cases = [
            {
                "url": url,
                "results": results,
                "count": len(results),
            }
            for url, results in cases.items()
        ]
        test_file = TEST_FOLDER / f"test_case_{domain.replace('.', '_')}.py"
        _ = test_file.write_text(f"DOMAIN = {domain!r}\nTEST_CASES = {test_cases}", encoding="utf8")
        test_files.append(test_file)

    if test_files:
        _ = subprocess.run(["ruff", "format", *test_files], check=False)


app = App()


@app.default()
def run(main_log: Path = Path("test_run.log"), url_txt: Path = Path("URLs.txt")) -> None:  # pyright: ignore[reportCallInDefaultInitializer]
    main_log, url_txt = map(Path.resolve, (main_log, url_txt))
    run_test(url_txt, main_log)
    json_l = main_log.with_suffix(".results.jsonl")
    create_test_files(json_l)


if __name__ == "__main__":
    app()