Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

18.7 KB
Raw Blame
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
r"""从 fc2cmadb 抓 FC2 作品元数据与评论区的人工标记。

FC2 没有 JAV 那样的番号体系,官方页面只有卖家自己填的标题,演员基本不写。
fc2cmadb 的价值不在正文而在**评论区**:那里有用户长期维护的两类标记。

演员标记,`<video_id> <名前>`,全角空格分隔,一行可以有多个演员:

    2724256 未歩なな 皐月
    1934545 ゆう

等价标记,把同一段内容在不同 video_id 下的发布对应起来:

    3312576-4 = 2471432 = 3090722-1 = 4605413

等价关系是合集的判据,也是本脚本存在的主要理由。`FC2PPV-3312576` 是一个
21 段的合集,本地就按 `FC2PPV-3312576-1.mp4` 分片存着。**合集封面绝不能套给
每个分片**——那会让 21 个不同内容的视频显示同一张图。所以判定为合集时
`cover_url` 留空并在 note 里写明,让分片回落到自己的缩略图。

页面是 Laravel + Inertia,数据在 `<script type="application/json">` 里,
不用解析 HTML。请求带用户在采集设置里贴的 Cookie,按登录用户看到的那一页取。页面地址、props 与评论标记的解析
在站点解析器 `peach.sources.fc2cmadb`,本脚本只管汇总、收获表与复核候选。

评论是匿名用户写的,一律只作候选:产出 CSV 交人工复核,不碰真相字段。
同一个演员名被两条以上独立评论提到时置信度更高,写在 `performer_votes` 里。

**只抓库里有的作品页,但每页评论全量留存。** 一页评论往往给几十个 video_id
标了演员,本地只对上其中两三个;只留对得上的那几条,等于把评论区的价值丢掉。
所以产出三份:按本地资产的候选 CSV(复核页用)、按 video_id 汇总的全量收获
CSV、以及原始评论 JSONL。留原文是因为解析标记的那些正则一定会漏掉某种写法,
有原文就不必为此重爬一遍。
"""
from __future__ import annotations

import argparse
import hashlib
import json
import re
import time
import sys
from pathlib import Path

import httpx

PROJECT_ROOT = Path(__file__).resolve().parent.parent
SRC_DIR = PROJECT_ROOT / "src"
if str(SRC_DIR) not in sys.path:
    sys.path.insert(0, str(SRC_DIR))

from peach.catalog_rules import normalise_code_key
from peach.config import SECRETS_DIR
from peach.scraping_access import client_for
from peach.genre_decisions import load_genre_decisions
from peach.genre_taxonomy import map_genres
from peach.review_csv import read_rows, write_rows
from peach.scripting import open_readonly
from peach.config import DATABASE_PATH, GENERATED_DIR
from peach.sources.fc2 import canonical_code
from peach.sources.fc2cmadb import (VIDEO_ID, Fc2cmadbSource, collection_parts, inertia_props,
                                    page_comments, parse_equivalences, parse_performers)

FC2_COVER_WIDTH = 1200
#: 判定合集所需的最少分片映射数。一两条可能只是同一段内容的重复投稿。
COLLECTION_MIN_PARTS = 3

METADATA_POLICY_VERSION = "fc2cmadb-article-v1"
METADATA_FIELDS = (
    "item_key", "code", "query", "field", "field_label", "current_value",
    "candidates_json", "source_count", "source_profile", "policy_version",
    "status", "size_gb", "videos", "fetched_at",
)


def high_resolution_cover_url(url: str) -> str:
    """Use FC2's measured 1200px CDN rendition instead of the 276px listing thumb."""
    return re.sub(r"(/w)\d+(/)", rf"\g<1>{FC2_COVER_WIDTH}\2", str(url or ""), count=1)


def article_url(video_id: str) -> str:
    return Fc2cmadbSource().article_url(canonical_code(video_id))


def fetch_article(client: httpx.Client, video_id: str) -> dict:
    response = client.get(article_url(video_id), timeout=30)
    if response.status_code == 404:
        raise LookupError("站上无此作品")
    response.raise_for_status()
    props = inertia_props(response.text)
    if not props:
        raise LookupError("页面没有 Inertia 数据,可能被挡在登录或人机验证外")
    return props


def summarise(video_id: str, props: dict) -> dict:
    article = props.get("article") or {}
    comments = page_comments(props)

    votes: dict[str, int] = {}
    equivalences: list[list[tuple[str, str]]] = []
    for comment in comments:
        body = str(comment.get("body") or "")
        for name in parse_performers(body).get(video_id, []):
            votes[name] = votes.get(name, 0) + 1
        equivalences.extend(parse_equivalences(body, subject=video_id))

    parts = collection_parts(video_id, equivalences)
    is_collection = len(parts) >= COLLECTION_MIN_PARTS
    # 同一段内容在别的 video_id 下的发布,供跨号去重与演员回填使用。
    aliases = sorted({tok[0] for group in equivalences for tok in group
                      if any(t[0] == video_id for t in group) and tok[0] != video_id})

    ranked = sorted(votes.items(), key=lambda kv: (-kv[1], kv[0]))
    writer = article.get("writer") or {}
    return {
        "title": article.get("title") or "",
        "release_date": article.get("release_date") or "",
        "duration": article.get("duration") or "",
        "censored": article.get("censored") or "",
        "writer": writer.get("name") or "",
        "writer_slug": writer.get("slug") or "",
        "tags": " ".join(str(t.get("name") or t) for t in (article.get("tags") or [])),
        "performers": " ".join(name for name, _ in ranked),
        "performer_votes": " ".join(f"{name}:{n}" for name, n in ranked),
        "is_collection": "1" if is_collection else "",
        "collection_parts": str(len(parts)) if is_collection else "",
        "equivalents": " ".join(aliases),
        # 合集封面套给每个分片会让 21 个不同内容显示同一张图,所以留空。
        "cover_url": "" if is_collection else high_resolution_cover_url(
            article.get("image_url") or ""
        ),
        "note": (f"合集,{len(parts)} 个分片各自独立,封面不下发" if is_collection
                 else ("FC2 官方页已下架" if article.get("not_found") else "")),
    }


HARVEST_FIELDS = ("video_id", "owned", "performers", "performer_votes",
                  "equivalents", "seen_on")


def harvest(video_id: str, props: dict, into: dict) -> None:
    """把一页评论里**所有** video_id 的标记并进全局收获表。

    本地只对得上其中两三个,其余的先存着:库随时会加片子,重爬一遍页面要几小时。
    """
    for comment in page_comments(props):
        body = str(comment.get("body") or "")
        for other_id, names in parse_performers(body).items():
            slot = into.setdefault(other_id, {"votes": {}, "equiv": set(), "seen": set()})
            for name in names:
                slot["votes"][name] = slot["votes"].get(name, 0) + 1
            slot["seen"].add(video_id)
        for group in parse_equivalences(body, subject=video_id):
            members = {token[0] for token in group}
            for other_id in members:
                slot = into.setdefault(other_id, {"votes": {}, "equiv": set(), "seen": set()})
                slot["equiv"].update(members - {other_id})
                slot["seen"].add(video_id)


def harvest_rows(collected: dict, owned: set[str]) -> list[dict]:
    rows = []
    for video_id in sorted(collected):
        slot = collected[video_id]
        ranked = sorted(slot["votes"].items(), key=lambda kv: (-kv[1], kv[0]))
        rows.append({
            "video_id": video_id,
            "owned": "1" if video_id in owned else "",
            "performers": " ".join(name for name, _ in ranked),
            "performer_votes": " ".join(f"{name}:{n}" for name, n in ranked),
            "equivalents": " ".join(sorted(slot["equiv"])),
            "seen_on": " ".join(sorted(slot["seen"])),
        })
    return rows


def backfill(rows: list[dict], collected: dict) -> list[dict]:
    """用全站收获补候选行的演员与等价关系。

    一个作品的演员常常标在**别的作品页**的评论里:`4176112` 那条「剛毛マキちゃん +
    四条链接」同时认领了 `3701252` 和 `4078398`。只看本页评论,这两条候选就会写着
    「无演员标记」,而收获表里明明有。
    """
    for row in rows:
        slot = collected.get(row.get("video_id"))
        if not slot:
            continue
        ranked = sorted(slot["votes"].items(), key=lambda kv: (-kv[1], kv[0]))
        if ranked:
            row["performers"] = " ".join(name for name, _ in ranked)
            row["performer_votes"] = " ".join(f"{name}:{n}" for name, n in ranked)
        merged = set(slot["equiv"]) | set((row.get("equivalents") or "").split())
        if merged:
            row["equivalents"] = " ".join(sorted(merged))
    return rows


FIELDS = ("code", "video_id", "result", "title", "release_date", "duration",
          "censored", "writer", "writer_slug", "tags", "performers",
          "performer_votes", "is_collection", "collection_parts",
          "equivalents", "cover_url", "note")


def _candidate_key(code: str, field: str, value: object) -> str:
    canonical = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
    digest = hashlib.sha256(f"{code}\0{field}\0fc2\0{canonical}".encode()).hexdigest()[:20]
    return f"{code}:{field}:fc2:{digest}"


def translated_tags(raw: str, genre_decisions=None) -> list[str]:
    """Conservatively project archived FC2 labels into Peach's reviewed taxonomy.

    词表和 JAV 官方来源共用 `peach.genre_taxonomy`:同一个日文标签在 FC2 和
    dmm/mgstage 上必须投影到同一个 Peach 标签,两份表迟早会漂移。用户在复核页
    收录下来的那批也共用,不然收录一次只对其中一条抓取链生效。
    """
    return map_genres(str(raw or "").split(), genre_decisions)[0]


def metadata_candidate_rows(rows: list[dict], database: Path, *, raw_snapshot: Path,
                            fetched_at: str) -> list[dict]:
    """Turn archived article facts into the normal review queue without writing ledger."""
    connection = open_readonly(database)
    output = []
    try:
        # 用户在复核页收录过的 genre 这一批就当已知词,不再作为未收录回来问一遍。
        genre_decisions = load_genre_decisions(connection)
        for row in rows:
            if row.get("result") != "取得":
                continue
            code = normalise_code_key(str(row.get("code") or ""))
            assets = connection.execute(
                "SELECT id,size,catalog_title FROM asset WHERE medium='video' "
                "AND upper(trim(code))=upper(?) AND (disposal IS NULL OR disposal<>'trash')",
                (code,),
            ).fetchall()
            if not assets:
                continue
            current_tags = sorted({str(item[0]).strip() for item in connection.execute(
                "SELECT DISTINCT t.tag FROM asset a JOIN asset_tag t ON t.asset_id=a.id "
                "WHERE a.medium='video' AND upper(trim(a.code))=upper(?) "
                "AND t.tag IS NOT NULL AND trim(t.tag)<>''",
                (code,),
            )})
            source_url = article_url(str(row.get("video_id") or ""))
            evidence = {}
            for field, value in (
                ("runtime", row.get("duration")),
                ("cover_url", high_resolution_cover_url(row.get("cover_url") or "")),
            ):
                if value:
                    evidence[field] = {"value": value, "display_value": str(value), "warnings": []}
            warnings = ["FC2CMADB 归档镜像,需人工复核"]
            values = []
            title = " ".join(str(row.get("title") or "").split())
            if title:
                values.append(("title", "标题", title, title,
                               next((str(item["catalog_title"] or "").strip()
                                     for item in assets if item["catalog_title"]), "")))
            tags = translated_tags(str(row.get("tags") or ""), genre_decisions)
            if tags:
                values.append(("tags", "内容标签", tags, "、".join(tags),
                               "、".join(current_tags)))
            for field, label, value, display_value, current in values:
                candidate = {
                    "candidate_key": _candidate_key(code, field, value),
                    "provider": "fc2cmadb", "source": "fc2", "source_url": source_url,
                    "confidence": 0.8, "profile": "fc2", "policy_version": METADATA_POLICY_VERSION,
                    "field_rank": 1, "source_kind": "official_mirror", "official": True,
                    "provider_id": str(row.get("video_id") or ""),
                    "content_id": str(row.get("video_id") or ""),
                    "value": value, "display_value": display_value, "warnings": warnings,
                    "catalog_evidence": evidence, "raw_snapshot": str(raw_snapshot),
                }
                output.append({
                    "item_key": f"{code}:{field}", "code": code, "query": code,
                    "field": field, "field_label": label, "current_value": current,
                    "candidates_json": json.dumps([candidate], ensure_ascii=False,
                                                   separators=(",", ":")),
                    "source_count": 1, "source_profile": "fc2",
                    "policy_version": METADATA_POLICY_VERSION, "status": "candidate",
                    "size_gb": round(sum(int(item["size"] or 0) for item in assets) / 1e9, 2),
                    "videos": len(assets), "fetched_at": fetched_at,
                })
    finally:
        connection.close()
    return output


def pending(database: Path, limit: int) -> list[tuple[str, str]]:
    """库里的 FC2 资产,按 (code, video_id) 去重。"""
    connection = open_readonly(database)
    try:
        rows = connection.execute(
            "SELECT DISTINCT code FROM asset "
            "WHERE code LIKE 'FC2%' AND code IS NOT NULL AND code<>'' ORDER BY code"
        ).fetchall()
    finally:
        connection.close()
    seen: dict[str, str] = {}
    for (code,) in rows:
        match = VIDEO_ID.search(code or "")
        if match:
            seen.setdefault(match.group(0), code)
    ordered = [(code, video_id) for video_id, code in seen.items()]
    return ordered[:limit] if limit else ordered


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--db", type=Path, default=DATABASE_PATH)
    parser.add_argument("--log", type=Path,
                        default=GENERATED_DIR / "fc2-candidate-log.csv")
    parser.add_argument("--harvest", type=Path,
                        default=GENERATED_DIR / "fc2-comment-harvest.csv",
                        help="评论里出现过的所有 video_id,不限于库里有的")
    parser.add_argument("--raw", type=Path,
                        default=GENERATED_DIR / "fc2-comments-raw.jsonl",
                        help="原始评论存档;解析规则以后改了不必重爬")
    parser.add_argument("--metadata-log", type=Path,
                        default=GENERATED_DIR / "fc2-metadata-field-candidates.csv",
                        help="标题与标签的统一元数据复核候选")
    parser.add_argument("--rebuild-metadata-only", action="store_true",
                        help="不联网,只用既有 --log 重建元数据候选")
    parser.add_argument("--limit", type=int, default=0)
    parser.add_argument("--delay", type=float, default=2.0)
    return parser


def run(args: argparse.Namespace) -> int:
    if args.rebuild_metadata_only:
        rows = read_rows(args.log)
        fetched_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(args.log.stat().st_mtime))
        candidates = metadata_candidate_rows(
            rows, args.db, raw_snapshot=args.log, fetched_at=fetched_at,
        )
        write_rows(args.metadata_log, METADATA_FIELDS, candidates)
        print(f"完成:{len(candidates)} 个 FC2 元数据字段候选 -> {args.metadata_log}")
        return 0
    todo = pending(args.db, args.limit)
    owned = {video_id for _, video_id in todo}
    rows: list[dict] = []
    collected: dict = {}
    stats = {"hit": 0, "miss": 0}
    args.raw.parent.mkdir(parents=True, exist_ok=True)
    raw_log = args.raw.open("w", encoding="utf-8")
    print(f"待抓 {len(todo)} 个 FC2 作品", flush=True)
    with client_for(SECRETS_DIR, "fc2cmadb", session=True) as client:
        for index, (code, video_id) in enumerate(todo, 1):
            try:
                props = fetch_article(client, video_id)
                data = summarise(video_id, props)
            # 长跑任务不能因为一个作品的网络问题整体退出;按条记「未取得」继续。
            # Ctrl-C 是 BaseException,不受影响。
            except Exception as exc:
                stats["miss"] += 1
                reason = f"{type(exc).__name__}: {exc}"
                rows.append({"code": code, "video_id": video_id,
                             "result": "未取得", "note": reason[:120]})
                print(f"[{index}/{len(todo)}] 未取得 {code}:{reason[:70]}", flush=True)
            else:
                stats["hit"] += 1
                rows.append({"code": code, "video_id": video_id,
                             "result": "取得", **data})
                harvest(video_id, props, collected)
                # 原文照存。这些是用户多年攒下的标记,我的正则漏掉哪种写法都不该
                # 意味着要重爬一遍。
                raw_log.write(json.dumps({"video_id": video_id, "code": code,
                                          "comments": page_comments(props)},
                                         ensure_ascii=False) + "\n")
                raw_log.flush()
                mark = "合集" if data["is_collection"] else (data["performers"] or "无演员标记")
                print(f"[{index}/{len(todo)}] 取得 {code} {mark}", flush=True)
            # 每条都落盘:上次抓取死在半路时进度全丢,重来一遍是三小时。
            write_rows(args.log, FIELDS, backfill(rows, collected))
            write_rows(args.harvest, HARVEST_FIELDS, harvest_rows(collected, owned))
            write_rows(args.metadata_log, METADATA_FIELDS, metadata_candidate_rows(
                backfill(rows, collected), args.db, raw_snapshot=args.log,
                fetched_at=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
            ))
            time.sleep(args.delay)
    raw_log.close()
    extra = sum(1 for vid in collected if vid not in owned)
    print(f"完成:取得 {stats['hit']},未取得 {stats['miss']} -> {args.log}")
    print(f"评论收获 {len(collected)} 个 video_id,其中 {extra} 个库里还没有 -> {args.harvest}")
    return 0


def main(argv: list[str] | None = None) -> int:
    return run(build_parser().parse_args(argv))


if __name__ == "__main__":
    raise SystemExit(main())