Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

6.2 KB
Raw Blame
#!/usr/bin/env python3
"""只读审计全库艺人的 MISSAV 入口与中文名证据缺口。"""
from __future__ import annotations

import argparse
import json
import re
import sqlite3
import sys
from collections import Counter, defaultdict
from pathlib import Path
from urllib.parse import quote

PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT_ROOT))
sys.path.insert(0, str(PROJECT_ROOT / "src"))

from peach import entry_links  # noqa: E402
from peach.config import DATABASE_PATH, DATA_ROOT, STATE_DIR  # noqa: E402
from peach.review_csv import write_rows  # noqa: E402
from peach.scripting import open_readonly  # noqa: E402
from localize_performer_names import RELEASE_SOURCES  # noqa: E402

FIELDS = ("entity_id", "current_name", "assets", "jav_directory", "previous_name",
          "entry_name", "previous_url", "url", "name_changed", "cn_status", "cn_aliases",
          "aliases", "directory_status", "link_evidence")
KANA = re.compile(r"[\u3040-\u30ff]")
HAN = re.compile(r"[\u3400-\u9fff]")
LATIN = re.compile(r"[A-Za-z]")


def _name_localization(raw: str):
    try:
        metadata = json.loads(raw or "{}")
    except (ValueError, TypeError):
        return None
    return metadata.get("name_localization") if isinstance(metadata, dict) else None


def _directory_evidence(jav: bool, directory, selected: str, name: str) -> tuple[str, str]:
    """名录匹配与目标页面请求是两个独立的证据状态。"""
    if not jav:
        return "不适用", "不适用"
    path = directory.path(selected, name) if directory else None
    if path is None:
        return "未取得", "未取得:未取得名录条目,目标未逐页请求"
    if not path:
        return "需复核", "未取得:名录无唯一匹配"
    return "已匹配", "已取得:名录 href,目标未逐页请求"


def collect(connection: sqlite3.Connection, directory=None) -> list[dict]:
    """每位 performer 一行;入口生成正确与远端页面可访问分别留证。"""
    connection.row_factory = sqlite3.Row
    aliases: dict[int, list[dict]] = defaultdict(list)
    refs: dict[int, list[dict]] = defaultdict(list)
    sources: dict[int, set[str]] = defaultdict(set)
    assets: dict[int, set[int]] = defaultdict(set)
    for row in connection.execute("SELECT entity_id,alias,source FROM entity_alias ORDER BY entity_id,alias"):
        aliases[int(row["entity_id"])].append(dict(row))
    for row in connection.execute("SELECT entity_id,provider,external_kind,external_id FROM entity_external_ref"):
        refs[int(row["entity_id"])].append(dict(row))
    for row in connection.execute("SELECT entity_id,asset_id,source FROM asset_entity"):
        sources[int(row["entity_id"])].add(str(row["source"]))
        assets[int(row["entity_id"])].add(int(row["asset_id"]))
    out = []
    for entity in connection.execute("SELECT id,canonical_name,metadata_json FROM entity WHERE kind='performer' ORDER BY id"):
        entity_id = int(entity["id"])
        name = str(entity["canonical_name"])
        chain = aliases[entity_id]
        jav = entry_links.is_jav_performer(refs[entity_id])
        previous = entry_links.japanese_name("", chain) or name
        selected = entry_links.japanese_name(name, chain, _name_localization(entity["metadata_json"]))
        directory_status, link_evidence = _directory_evidence(jav, directory, selected, name)
        chinese = sorted({str(row["alias"]) for row in chain
                          if HAN.search(str(row["alias"])) and not KANA.search(str(row["alias"]))
                          and not LATIN.search(str(row["alias"]))})
        release = bool(RELEASE_SOURCES.intersection(sources[entity_id])) or jav
        status = ("非发行艺人,不判中译" if not release else
                  "假名规范名,有汉字别名待复核" if KANA.search(name) and chinese else
                  "假名规范名,中文来源未取得" if KANA.search(name) else
                  "拉丁规范名,需区分艺名与转写" if LATIN.search(name) else
                  "无假名规范名,通用中译覆盖未验证")
        out.append({"entity_id": entity_id, "current_name": name,
                    "assets": len(assets[entity_id]), "jav_directory": int(jav),
                    "previous_name": previous if jav else "", "entry_name": selected if jav else "",
                    "previous_url": f"https://missav.ws/cn/actresses/{quote(previous, safe='')}" if jav else "",
                    "url": next((row["url"] for row in entry_links.build(
                        entry_links.defaults(), name, refs[entity_id], chain,
                        _name_localization(entity["metadata_json"]), directory) if row["site"] == "missav"), ""),
                    "name_changed": int(jav and previous != selected), "cn_status": status,
                    "cn_aliases": "|".join(chinese), "aliases": "|".join(row["alias"] for row in chain),
                    "directory_status": directory_status,
                    "link_evidence": link_evidence})
    return out


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--db", type=Path, default=DATABASE_PATH)
    parser.add_argument("--out", type=Path, default=DATA_ROOT / "review" / "performer-entry-name-audit.csv")
    parser.add_argument("--missav-directory", type=Path, default=STATE_DIR / entry_links.DIRECTORY_FILENAME)
    args = parser.parse_args(argv)
    directory = None
    if args.missav_directory.is_file():
        directory = entry_links.directory_index(json.loads(
            args.missav_directory.read_text(encoding="utf-8")))
    with open_readonly(args.db) as connection:
        rows = collect(connection, directory)
    write_rows(args.out, FIELDS, rows)
    print(json.dumps({"performers": len(rows), "missav_entries": sum(row["jav_directory"] for row in rows),
                      "entry_names_changed": sum(row["name_changed"] for row in rows),
                      "directory_status": dict(Counter(row["directory_status"] for row in rows)),
                      "cn_status": dict(Counter(row["cn_status"] for row in rows)),
                      "review_csv": str(args.out)}, ensure_ascii=False))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())