Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

7.9 KB
Raw Blame
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
r"""把一格装了多个艺名的别名拆成一人多名。

r18.dev 的罗马字字段本身就是 `现用名 (曾用名, 曾用名)` 这个渲染格式,导入时一个字段
写一行,于是 `Ako Momona (Kou Akemi, Mari Koizumi)` 整串成了一条别名。它做别名是死的:
按任何一段都搜不到,选成统称更不成立。同一条实体的假名和汉字写法本来就各自成行,缺的
只是这几个罗马字,所以拆开是补信息,不是丢信息。

**只拆罗马字人名这一种形状**(判据见 `peach.entities.COMPOSITE_PERSON_NAME`)。账本里
「名字后面跟一对括号」的还有四类完全不同的东西,一律不动,只写进复核 CSV 交人工:

- `AV DEBUT(本物人妻)`:括号里是厂牌,用来把三个同名系列分开。拆了就撞成一个。
- `アスナ(SAO)`:角色标签的出处消歧,账本里有五百多条。
- `快慢扳机(接稿中)`、`るかぴあ(X:復帰しました)`:括号里是接稿状态或公告。
- `kitty(1)`、`Mana(23)`:去重后缀或数字。

形状分不开的那些交人工:在复核 CSV 里把 `verdict` 改成 `split`(别名拆成几条)或
`strip`(规范名换成 `target`,旧名留作别名),填好 `target`,再用 `--from-review` 跑。

拆分不可逆。默认只产出复核 CSV,`--apply` 才写 ledger 且必须给 `--backup`。
"""
from __future__ import annotations

import argparse
import sqlite3
import sys
import time
from collections import Counter
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parent.parent
SRC_DIR = PROJECT_ROOT / "src"
if str(SRC_DIR) not in sys.path:
    sys.path.insert(0, str(SRC_DIR))

from peach.config import GENERATED_DIR   # noqa: E402
from peach.entities import (   # noqa: E402
    normalize_entity_name, rewrite_flat_projection,
)
from peach.review_csv import read_rows, write_rows   # noqa: E402
from peach.scripting import (   # noqa: E402
    add_ledger_write_args, counts_of, open_for_write, verify_after_write,
)

#: 只有人名实体才可能出现「一个人的多个艺名」。系列、厂牌和标签的括号是别的意思。
from peach.composite_alias_repair import (FIELDS, CLEANUP_SOURCE, EXTRA_COUNTS, collect, apply_rows, _blocked)


def apply_review(connection: sqlite3.Connection,
                 rows: list[dict[str, str]]) -> dict[str, object]:
    """按人工填好的 `verdict` 与 `target` 改名字。

    `split`:把一条别名换成 `target` 里那几条,来源不变。
    `strip`:把规范名换成 `target` 的第一段,旧名留作别名,扁平投影跟着改。
    """
    done = {"split": 0, "strip": 0, "flat": 0, "blocked": []}
    for row in rows:
        verdict = str(row.get("verdict", "")).strip()
        targets = [part.strip() for part in str(row.get("target", "")).split("|")
                   if part.strip()]
        if verdict not in {"split", "strip"} or not targets:
            continue
        entity_id, value = int(row["entity_id"]), str(row["value"])
        blocked = _blocked(connection, entity_id, targets)
        if blocked:
            done["blocked"].append(f"{value}:{blocked}")
            continue
        if verdict == "split":
            source = str(row.get("source", ""))
            for target in targets:
                connection.execute(
                    "INSERT OR IGNORE INTO entity_alias"
                    "(entity_id,alias,normalized_alias,source) VALUES(?,?,?,?)",
                    (entity_id, target, normalize_entity_name(target), source))
            connection.execute(
                "DELETE FROM entity_alias WHERE entity_id=? AND alias=? AND source=?",
                (entity_id, value, source))
            done["split"] += 1
            continue
        target = targets[0]
        stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
        connection.execute(
            "UPDATE entity SET canonical_name=?,normalized_name=?,updated_at=? WHERE id=?",
            (target, normalize_entity_name(target), stamp, entity_id))
        # 旧名留作别名:它是这条实体真的用过的写法,也是回退的入口。
        connection.execute(
            "INSERT OR IGNORE INTO entity_alias"
            "(entity_id,alias,normalized_alias,source,confidence) VALUES(?,?,?,?,1.0)",
            (entity_id, value, normalize_entity_name(value), CLEANUP_SOURCE))
        done["flat"] += rewrite_flat_projection(
            connection, str(row["kind"]), entity_id, value, target)
        done["strip"] += 1
    return done


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=__doc__,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    add_ledger_write_args(parser)
    parser.add_argument("--review-csv", type=Path,
                        default=GENERATED_DIR / "review" / "composite-names.csv",
                        help="复核 CSV 输出路径")
    parser.add_argument("--from-review", type=Path,
                        help="按这份人工判定过的 CSV 执行 split / strip,不再重新扫描")
    return parser


def _report(before: dict[str, int], after: dict[str, int],
            connection: sqlite3.Connection) -> int:
    _integrity, violations = verify_after_write(connection)
    for key in before:
        mark = "" if before[key] == after[key] else "  <-- 变化"
        print(f"    {key}: {before[key]} -> {after[key]}{mark}")
    print(f"  foreign_key_check 违规 {violations} 条")
    return 1 if violations else 0


def run_review(args: argparse.Namespace) -> int:
    connection = open_for_write(args)
    try:
        rows = read_rows(args.from_review)
        acting = [row for row in rows
                  if str(row.get("verdict", "")).strip() in {"split", "strip"}]
        print(f"人工判定 {len(acting)} 行:{args.from_review}")
        for row in acting:
            print(f"    {row['verdict']} [{row['entity_id']}] {row['value']}"
                  f" -> {row['target']}")
        if not args.apply:
            print("  未写 ledger(加 --apply --backup 才写)")
            return 0
        print(f"  已备份到 {args.backup}")
        before = counts_of(connection, EXTRA_COUNTS)
        with connection:
            done = apply_review(connection, acting)
        after = counts_of(connection, EXTRA_COUNTS)
        print("  结果:", done)
        return _report(before, after, connection)
    finally:
        connection.close()


def run(args: argparse.Namespace) -> int:
    if args.from_review:
        return run_review(args)
    connection = open_for_write(args)
    try:
        rows = collect(connection)
        write_rows(args.review_csv, FIELDS, rows)
        verdicts = Counter(str(row["verdict"]) for row in rows)
        print(f"带括号的名字 {len(rows)} 条,复核 CSV:{args.review_csv}")
        print("  判定分布:", dict(verdicts))
        for row in rows:
            if row["verdict"] == "auto":
                print(f"    拆 [{row['entity_id']}] {row['canonical_name']}:"
                      f"{row['value']} -> {row['parts']}")
        by_kind = Counter(f"{row['kind']}/{row['field']}" for row in rows
                          if row["verdict"] != "auto")
        print("  交人工:", dict(by_kind))
        if not args.apply:
            print("  未写 ledger(加 --apply --backup 才写)")
            return 0

        print(f"  已备份到 {args.backup}")
        before = counts_of(connection, EXTRA_COUNTS)
        with connection:
            moved = apply_rows(connection, rows)
        after = counts_of(connection, EXTRA_COUNTS)
        print("  拆分结果:", moved)
        return _report(before, after, connection)
    finally:
        connection.close()


def main() -> int:
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
    return run(build_parser().parse_args())


if __name__ == "__main__":
    raise SystemExit(main())