Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

8.2 KB
Raw Blame
#!/usr/bin/env python3
"""联网录制来源回放夹具:对一站、一个番号跑一次真实 `query`,把交互存进 `tests/fixtures/source-replay/`。

只读:不写 ledger,不写生产的来源冷却。代理设置与冷却记录从 `--secrets`(默认本机数据根的 secrets)
复制进一个临时根再用:代理按用户现有设置走,冷却期内的站照样停下,录制中撞出的新冷却只落在临时根。
结局是连接失败、冷却、预算、限流、验证页、封禁或地区限制的,不落盘,记为「未取得」。

默认不带这一站的 Cookie:登录态的页面带着账号编号与收藏入口。只有不登录就拿不到作品页的站才加
`--with-credentials`,这时 Cookie 值、CSRF 令牌与 `uid`、`formhash` 一类账号标识都换成固定假值。
落盘前去掉与解析无关的网页区块;脱敏与瘦身各回放一次,解析结局与联网那一次不一致就不采用这一步。
录制格式见 `docs/TESTING.md`「来源录制回放」。

    python scripts/record_source_fixtures.py javbus:SSIS-001 avbase:SSIS-001
    python scripts/record_source_fixtures.py --update javbus:SSIS-001
    python scripts/record_source_fixtures.py --with-credentials javdb:SSIS-001
"""
from __future__ import annotations

import argparse
import json
import os
import re
import shutil
import sys
import tempfile
import time
from datetime import date
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parent.parent
SRC_DIR = PROJECT_ROOT / "src"
if str(SRC_DIR) not in sys.path:
    sys.path.insert(0, str(SRC_DIR))

import httpx

from peach.config import SECRETS_DIR
from peach.jav_cover_fetch import DeadlineExceeded, HostLimitedTransport
from peach.scraping_access import SourceBudgetExhausted, SourcePaused, SourceTransport
from peach.sources import SITE_SOURCES, FailureReason
from peach.sources.replay import (Cassette, RecordingTransport, ReplayMiss, ReplayTransport,
                                  minimize, outcome_of, redact, rewrite_texts, secret_pieces)

FIXTURES = PROJECT_ROOT / "tests" / "fixtures" / "source-replay"
REPORT = PROJECT_ROOT / "build" / "source-replay" / "record-report.json"
#: 与作品有关的结局才落盘;其余是这台机器此刻的网络状况,换个出口、等一会儿结果就变。
RECORDABLE = {"ok", FailureReason.NOT_FOUND.value, FailureReason.GONE.value, FailureReason.PARSE_ERROR.value,
              FailureReason.NO_USABLE_METADATA.value, FailureReason.AUTH_REQUIRED.value}
#: 一次录制最多发几条请求:javdb 搜索加详情两条,DMM 猜 cid、搜索再试几个 cid 也不过十条。
MAX_REQUESTS = 12
WINDOWS_PATH = re.compile(r"[A-Za-z]:\\\\(?:Users|Desktop)", re.I)


def fixture_path(source: str, code: str) -> Path:
    return FIXTURES / source / f"{code}.json"


def scratch_secrets(secrets: Path, source: str, *, credentials: bool) -> tuple[Path, list[str]]:
    """把代理设置、这一站的连接设置与冷却记录复制进临时根,交回临时根与要替换的凭据片段。

    不带凭据时这一站的设置只留 `network`(直连还是走代理),Cookie 那几栏不进临时根。
    """
    root = Path(tempfile.mkdtemp(prefix="peach-record-"))
    (root / "follow").mkdir()
    proxy = secrets / "follow" / "peach-proxy.json"
    if proxy.is_file():
        shutil.copy2(proxy, root / "follow" / proxy.name)
    values: list[str] = []
    site = secrets / "follow" / f"scraping-{source}.json"
    if site.is_file():
        payload = json.loads(site.read_text(encoding="utf-8"))
        kept = payload if credentials else {key: payload[key] for key in ("network",) if key in payload}
        values = [str(value) for key, value in kept.items() if key != "network" and isinstance(value, str)]
        (root / "follow" / site.name).write_text(json.dumps(kept, ensure_ascii=False), encoding="utf-8")
    cooldown = secrets / f"scraping-{source}.cooldown.json"
    if cooldown.is_file():
        shutil.copy2(cooldown, root / cooldown.name)
    return root, secret_pieces(values)


def replays_same(cassette: Cassette, expected: tuple) -> bool:
    try:
        return outcome_of(cassette.source, cassette.code, ReplayTransport(cassette)) == expected
    except ReplayMiss:
        return False


def record(source: str, code: str, *, secrets: Path, credentials: bool, note: str) -> dict:
    root, pieces = scratch_secrets(secrets, source, credentials=credentials)
    cassette = Cassette(source=source, code=code, recorded=date.today().isoformat(), note=note)
    inner = SourceTransport(root, max_requests=MAX_REQUESTS)
    transport = HostLimitedTransport(RecordingTransport(inner, cassette), SITE_SOURCES[source].DEFAULT.interval)
    started = time.monotonic()
    try:
        live = outcome_of(source, code, transport)
    except (SourcePaused, SourceBudgetExhausted, DeadlineExceeded, httpx.TransportError) as error:
        return {"status": "未取得", "reason": type(error).__name__, "message": str(error)}
    finally:
        inner.close()
        shutil.rmtree(root, ignore_errors=True)
    outcome = live[0]
    if outcome not in RECORDABLE:
        return {"status": "未取得", "reason": outcome, "message": (live[2] or {}).get("message", "")}
    cassette.outcome, cassette.payload, cassette.failure = live
    if any(piece in row["url"] for row in cassette.interactions for piece in pieces):
        return {"status": "拒绝", "reason": "凭据出现在请求地址里", "message": ""}
    safe = rewrite_texts(cassette, lambda text: redact(text, pieces), urls=True)
    if not replays_same(safe, live):
        return {"status": "拒绝", "reason": "替换凭据后解析结局变了,凭据值进了解析字段", "message": ""}
    small = rewrite_texts(safe, minimize)
    trimmed = replays_same(small, live)
    final = small if trimmed else safe
    text = final.dump()
    if any(piece in text for piece in pieces) or WINDOWS_PATH.search(text):
        return {"status": "拒绝", "reason": "录制里仍有凭据或本机路径", "message": ""}
    path = fixture_path(source, code)
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_bytes(text.encode("utf-8"))
    return {"status": "已录制", "outcome": outcome, "path": path.relative_to(PROJECT_ROOT).as_posix(),
            "bytes": len(text.encode("utf-8")), "raw_bytes": len(cassette.dump().encode("utf-8")),
            "trimmed": trimmed, "requests": len(cassette.interactions),
            "seconds": round(time.monotonic() - started, 1)}


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0])
    parser.add_argument("samples", nargs="+", metavar="来源:番号")
    parser.add_argument("--secrets", type=Path, default=SECRETS_DIR, help="凭据与代理设置所在的 secrets 根,只读")
    parser.add_argument("--update", action="store_true", help="已有录制也重录")
    parser.add_argument("--with-credentials", action="store_true", help="带上这一站的 Cookie;不登录拿不到作品页时才用")
    parser.add_argument("--note", default="", help="写进录制的场景说明")
    args = parser.parse_args(argv)
    if os.environ.get("CI") or os.environ.get("GITHUB_ACTIONS"):
        print("CI 里只回放,不录制", file=sys.stderr)
        return 2
    results = []
    for sample in args.samples:
        source, _, code = sample.partition(":")
        if source not in SITE_SOURCES or not code:
            print(f"无法识别 {sample}:写成「来源:番号」,来源取 {', '.join(SITE_SOURCES)}", file=sys.stderr)
            return 2
        if fixture_path(source, code).exists() and not args.update:
            result = {"status": "已有录制"}
        else:
            result = record(source, code, secrets=args.secrets, credentials=args.with_credentials,
                            note=args.note)
        result = {"source": source, "code": code, **result}
        results.append(result)
        print(json.dumps(result, ensure_ascii=False))
    REPORT.parent.mkdir(parents=True, exist_ok=True)
    REPORT.write_text(json.dumps(results, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
    return 0 if all(row["status"] in {"已录制", "已有录制", "未取得"} for row in results) else 1


if __name__ == "__main__":
    raise SystemExit(main())