Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

13.0 KB
Raw Blame
"""Peach 与 amane 刮削站点之间的薄桥:一次子进程,stdout 只有一行 JSON。

形状与 Peach 调 Javinizer-Go 的方式一致(`peach.metadata.JavinizerGoProvider`):无端口、
无状态、无常驻,每次由 Peach 起一个子进程、读一行 JSON、看退出码。amane 那边只用爬虫层
与网络层——`amane.crawlers.sites.<站>`、`amane.crawlers.http`、`amane.net.*`;不碰
`amane.aggregate`(它把配置层与数据库层一起拖进来,聚合逻辑 Peach 自己在
`metadata_policy` 里做),也不经 `observability.invoke_source`(失败原因会被它吞进
Recorder,返回值只剩站点名)。这里自己 catch `SourceError`,把 `FailureReason` 的分类
原样写进 JSON,冷却与分档由 Peach 一侧决定。

已知代价:`amane.crawlers` 的包 `__init__` 会连带 SQLAlchemy 等模块(实测导入约 0.6~1 秒),
从子模块进也绕不开;这是每次子进程的固定开销,接受它换来的是不改上游一行。

amane 为 GPL-3.0(<https://github.com/sqzw-x/amane>);本文件属于 Peach(AGPL-3.0-or-later),
两者以进程边界相接,见 ADR-0043。运行环境由 `pyproject.toml` / `uv.lock` 钉死。

退出码:0 至少一站取到资料;2 问的站都明确说没有;3 没有一站取到且至少一站出错;
4 参数错误(未知站名、番号为空)。
"""
from __future__ import annotations

import argparse
import asyncio
import json
import logging
import sys
import time
from dataclasses import dataclass
from typing import Any, Callable

#: 桥认得的站点:站名 → (`amane.crawlers.sites` 下的模块名,类名)。站名是 Peach 的来源名,
#: 多数与 amane 同名;`makers` 是 amane 的 `official`:它按番号的系列前缀查自带的片商表
#: (`MANUFACTURER_SERIES`),路由到二十九家片商官网的作品页,前缀不在表里就不发请求、直接答没有,
#: 所以不带 `SiteConfig` 也跑得起来。Peach 那边 `official` 是来源分级的名字,换个站名免得两义。
#: 不列 `r18dev`(要本地 PostgreSQL 镜像)与 `theporndb`(要 API token):两者没有 `SiteConfig`
#: 就跑不起来,而桥刻意不带配置。
SITES: dict[str, tuple[str, str]] = {
    "airav": ("airav", "AiravCrawler"),
    "avsox": ("avsox", "AvsoxCrawler"),
    "dahlia": ("dahlia", "DahliaCrawler"),
    "dmm": ("dmm", "DmmCrawler"),
    "faleno": ("faleno", "FalenoCrawler"),
    "fc2": ("fc2", "FC2Crawler"),
    "fc2club": ("fc2club", "FC2ClubCrawler"),
    "freejavbt": ("freejavbt", "FreejavbtCrawler"),
    "getchu": ("getchu", "GetchuCrawler"),
    "giga": ("giga", "GigaCrawler"),
    "iqqtv": ("iqqtv", "IqqtvCrawler"),
    "jav321": ("jav321", "Jav321Crawler"),
    "javbus": ("javbus", "JavBusCrawler"),
    "javdb": ("javdb", "JavDBCrawler"),
    "javlibrary": ("javlibrary", "JavLibraryCrawler"),
    "kin8": ("kin8", "Kin8Crawler"),
    "makers": ("official", "OfficialCrawler"),
    "mgstage": ("mgstage", "MGStageCrawler"),
    "prestige": ("prestige", "PrestigeCrawler"),
    "xcity": ("xcity", "XCityCrawler"),
}

LANGUAGES = ("jp", "zh_cn", "zh_tw", "en")

EXIT_FOUND, EXIT_NOT_FOUND, EXIT_FAILED, EXIT_USAGE = 0, 2, 3, 4

#: 上游 `FailureReason` 的全部取值,来自 `amane/net/errors.py`。桥不解释它们,只保证写进
#: JSON 的 `reason` 落在这张表里;上游加档时这里要跟着改,Peach 一侧的映射也是。
FAILURE_REASONS = (
    "http_error", "api_error", "not_found", "rate_limited", "server_error", "timeout", "network",
    "cloudflare_challenge", "cloudflare_blocked", "ip_banned", "geo_restricted",
    "age_verification", "login_required", "empty_response", "no_usable_metadata", "parse_error",
    "crawler_unavailable", "unexpected",
)


@dataclass(frozen=True)
class Runtime:
    """桥用到的 amane 那几样东西。真实实现由 `load_runtime()` 给;单测塞假的进来。

    `crawler(site)` 返回爬虫类;`make_client(proxy, timeout, rate)` 返回传给爬虫构造函数的
    `HttpClient`;`query(number)` 与 `options(language)` 构造 `SearchQuery` / `FetchOptions`;
    `source_error` 是要 catch 的异常类型,实例上有 `reason` / `http_status` / `detail` / `url`。
    """

    crawler: Callable[[str], type]
    make_client: Callable[[str | None, float, float], Any]
    query: Callable[[str], Any]
    options: Callable[[str | None], Any]
    source_error: type[BaseException]
    version: str = ""


def load_runtime() -> Runtime:
    """真实 amane。只在这里 import,单测不必装它。"""
    import importlib

    import structlog

    # amane 的日志走 structlog,默认打到 stdout——那正是 JSON 要独占的那条流。
    structlog.configure(logger_factory=structlog.PrintLoggerFactory(file=sys.stderr))
    logging.basicConfig(stream=sys.stderr, level=logging.WARNING)

    from amane.crawlers.http import HttpClient
    from amane.crawlers.models import FetchOptions, SearchQuery
    from amane.enums import Language
    from amane.net.errors import SourceError
    from amane.net.http import RateLimiters, WebClient

    from importlib.metadata import PackageNotFoundError, version as installed_version

    try:
        version = installed_version("amane")
    except PackageNotFoundError:  # 以源码路径而不是安装件运行时没有 dist-info
        version = ""

    def crawler(site: str) -> type:
        module_name, class_name = SITES[site]
        module = importlib.import_module(f"amane.crawlers.sites.{module_name}")
        return getattr(module, class_name)

    def make_client(proxy: str | None, timeout: float, rate: float):
        # 上游 `max_retries=1` 限普通重试;403/406 另可轮换三次指纹,单个地址最多五次请求。
        # Peach 用 60 秒子进程总时限约束整次查询。不带 BrowserClient:挑战页报
        # `cloudflare_challenge`,不切浏览器渲染。
        limiters = RateLimiters(default_rate=rate)
        return HttpClient(WebClient(limiters=limiters, proxy=proxy, timeout=timeout, max_retries=1))

    return Runtime(
        crawler=crawler, make_client=make_client,
        query=lambda number: SearchQuery(number=number),
        options=lambda language: FetchOptions(language=Language(language)) if language else None,
        source_error=SourceError, version=str(version or ""),
    )


def _dump(metadata: Any) -> dict:
    """`MediaMetadata` → JSON 形状。pydantic 的 `model_dump` 优先,假对象退回 `vars`。"""
    if hasattr(metadata, "model_dump"):
        return metadata.model_dump(mode="json")
    if isinstance(metadata, dict):
        return dict(metadata)
    return dict(vars(metadata))


def failure_record(error: BaseException) -> dict:
    """一次 `SourceError` 写成 JSON 里的一条。`reason` 不在表里就按 `unexpected` 记并把原值留在 `detail`。"""
    reason = str(getattr(error, "reason", "") or "")
    detail = str(getattr(error, "detail", "") or "")
    if reason not in FAILURE_REASONS:
        detail = f"{reason}: {detail}" if reason else detail
        reason = "unexpected"
    record: dict[str, Any] = {"status": "failed", "reason": reason}
    status = getattr(error, "http_status", None)
    if status is not None:
        record["http_status"] = int(status)
    if detail:
        record["detail"] = detail[:500]
    url = getattr(error, "url", None)
    if url:
        record["url"] = str(url)
    return record


async def query_site(runtime: Runtime, site: str, client: Any, number: str,
                     language: str | None) -> dict:
    """问一站,返回这一站的 JSON 记录;异常全部收成 `failed`,不让一站的崩溃带走整批。"""
    started = time.monotonic()
    record: dict[str, Any]
    try:
        crawler = runtime.crawler(site)(client=client, config=None)
        metadata = await crawler.fetch(runtime.query(number), runtime.options(language))
        if metadata is None:
            # 上游把「搜不到」与「页面解析不出资料」都返回 None,不抛异常。
            record = {"status": "not_found", "reason": "not_found"}
        else:
            record = {"status": "found", "metadata": _dump(metadata)}
    except runtime.source_error as error:
        record = failure_record(error)
    except Exception as error:  # noqa: BLE001 - 一站的意外不能拖垮整批
        record = {"status": "failed", "reason": "unexpected",
                  "detail": f"{type(error).__name__}: {error}"[:500]}
    record["elapsed_s"] = round(time.monotonic() - started, 2)
    return record


async def run(runtime: Runtime, number: str, sites: list[str], *, language: str | None,
              proxy: str | None, timeout: float, rate: float, direct_sites: tuple[str, ...] = ()) -> dict:
    """并发问每一站;限速器按主机计,不同站互不排队。"""
    client = runtime.make_client(proxy, timeout, rate)
    direct = runtime.make_client(None, timeout, rate) if direct_sites and proxy else client
    records = await asyncio.gather(
        *(query_site(runtime, site, direct if site in direct_sites else client, number, language) for site in sites))
    return {
        "number": number,
        "language": language or "",
        "sites": dict(zip(sites, records)),
        "amane": {"version": runtime.version},
        "transport": {"stdout_encoding": sys.stdout.encoding,
                      "stderr_encoding": sys.stderr.encoding,
                      "frozen": bool(getattr(sys, 'frozen', False))},
    }


def exit_code(report: dict) -> int:
    statuses = [record.get("status") for record in report.get("sites", {}).values()]
    if "found" in statuses:
        return EXIT_FOUND
    if statuses and all(status == "not_found" for status in statuses):
        return EXIT_NOT_FOUND
    return EXIT_FAILED


def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
    parser.add_argument("--self-check", action="store_true", help="离线载入运行时与全部站点类")
    parser.add_argument("--number", default="", help="规范化番号")
    parser.add_argument("--sites", default="", help="逗号分隔的站名,见 SITES")
    parser.add_argument("--language", default="jp", choices=(*LANGUAGES, ""),
                        help="amane 的 Language;空串表示不指定")
    parser.add_argument("--proxy", default=None,
                        help="代理地址;不给就不走代理(libcurl 仍会读环境变量)")
    parser.add_argument("--direct-sites", default="", help="本次查询中强制直连的站名,逗号分隔")
    parser.add_argument("--timeout", type=float, default=30.0, help="单次请求超时(秒)")
    parser.add_argument("--rate", type=float, default=0.5, help="每主机每秒请求数上限")
    parser.add_argument("--output", default="json", choices=("json",))
    return parser.parse_args(argv)


def main(argv: list[str] | None = None, runtime: Runtime | None = None,
         out=None) -> int:
    out = out or sys.stdout
    try:
        args = parse_args(argv)
    except SystemExit as exc:
        return EXIT_USAGE if exc.code else 0
    if args.self_check:
        try:
            active = runtime or load_runtime()
            for site in SITES:
                active.crawler(site)
            report = {"ok": True, "amane": {"version": active.version}, "sites": list(SITES)}
            code = EXIT_FOUND
        except Exception as error:
            report = {"ok": False, "error": f"amane 加载失败:{type(error).__name__}: {error}"}
            code = EXIT_FAILED
        print(json.dumps(report, ensure_ascii=False), file=out, flush=True)
        return code
    number = str(args.number or "").strip()
    sites = [part.strip() for part in str(args.sites).split(",") if part.strip()]
    unknown = [site for site in sites if site not in SITES]
    direct_sites = tuple(part.strip() for part in args.direct_sites.split(",") if part.strip())
    unknown.extend(site for site in direct_sites if site not in sites)
    if not number or not sites or unknown:
        message = ("未知站名:" + "、".join(unknown)) if unknown else "番号与站名都不能为空"
        print(json.dumps({"error": message}, ensure_ascii=False), file=out, flush=True)
        return EXIT_USAGE
    try:
        active = runtime or load_runtime()
    except Exception as error:  # noqa: BLE001 - venv 坏了也要以 JSON 报出去
        print(json.dumps({"error": f"amane 加载失败:{type(error).__name__}: {error}"},
                         ensure_ascii=False), file=out, flush=True)
        return EXIT_FAILED
    report = asyncio.run(run(active, number, sites, language=args.language or None,
                             proxy=args.proxy or None, timeout=args.timeout, rate=args.rate,
                             direct_sites=direct_sites))
    print(json.dumps(report, ensure_ascii=False, default=str), file=out, flush=True)
    return exit_code(report)


if __name__ == "__main__":
    # 冻结程序不读取 PYTHONIOENCODING;重定向后的默认编码可能仍是 Windows GBK。
    sys.stdout.reconfigure(encoding="utf-8")
    sys.stderr.reconfigure(encoding="utf-8")
    sys.exit(main())