Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

5.3 KB
Raw Blame
r"""已落库的刮削归属,来源交回的作品认不认得出所查番号(只读,只出复核 CSV)。

`metadata.identifies_code` 在候选落库处核验来源身份;它上线前落下的归属没人回头查过。
实测样例:目录名 `WX17` 被规范成 `WX-017` 去问 javbus,来源交回 `WXSD-017`,片名和标签
落到了 266 个无关视频上。本脚本按归属里留下的 `provider_id`/`content_id`/`source_url`
用同一道闸重判,分三档:

- `错配`:来源交回的是另一部作品(`WX-017` → `WXSD-017`)。
- `前缀未证实`:去掉三位数字前缀就对得上(`476MLA-234` → `MLA-234`)。数字前缀属于作品
  身份(`390JAC-040` 配信与 `JAC-040` DVD 合集是两个商品,见 `docs/SOURCING.md`),这一档
  只说明认不出,不说明错;账本里的标题、日期往往就取自同一份快照,不能拿来互证。
- `来源无编号`:来源没给 id,只有 `source_url`,缺证据不是反证,只计数不进 CSV。

复核 CSV 每行一组(资产 code、复核番号、来源编号),写库另走 `revert_misread_code.py`
或 `peach-ledger-write` 的单独授权。
"""
from __future__ import annotations

import argparse
import json
import re
import sys
from collections import Counter
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))

from peach.config import DATABASE_PATH, GENERATED_DIR  # noqa: E402
from peach.metadata import identifies_code  # noqa: E402
from peach.review_csv import write_rows  # noqa: E402
from peach.scripting import open_readonly  # noqa: E402

MISMATCH, PREFIX, NO_ID = "错配", "前缀未证实", "来源无编号"
FIELDS = ("tier", "asset_code", "review_code", "provider_id", "content_id", "source_url",
          "asset_count", "link_count", "sources", "asset_ids", "sample_path")
_MAKER_PREFIX = re.compile(r"^\d{3}(?=[A-Z])")


def classify(code: str, payload: dict) -> str | None:
    """认得出返回 None;认不出按档位返回。"""
    if identifies_code(code, payload):
        return None
    stripped = _MAKER_PREFIX.sub("", code.upper())
    if stripped != code.upper() and identifies_code(stripped, payload):
        return PREFIX
    if not (payload.get("id") or payload.get("content_id")):
        return NO_ID
    return MISMATCH


def collect(connection) -> tuple[list[dict], Counter]:
    groups: dict[tuple, dict] = {}
    tiers: Counter = Counter()
    for asset_id, code, path, source, meta in connection.execute(
            "SELECT a.id,a.code,a.path,ae.source,ae.metadata_json FROM asset a "
            "JOIN asset_entity ae ON ae.asset_id=a.id "
            "WHERE ae.source LIKE 'javinizer:%' AND trim(COALESCE(a.code,''))<>'' ORDER BY a.id"):
        data = json.loads(meta or "{}")
        payload = {"id": data.get("provider_id"), "content_id": data.get("content_id"),
                   "source_url": data.get("source_url")}
        if not any(payload.values()):
            continue
        review = str(data.get("review_item") or "").split(":", 1)[0]
        review = code if review in {"", "asset"} else review
        tier = classify(review, payload)
        if tier is None:
            continue
        tiers[tier] += 1
        group = groups.setdefault((tier, code, review), {
            "tier": tier, "asset_code": code, "review_code": review, "provider_id": "",
            "content_id": "", "source_url": "", "assets": set(), "links": 0,
            "sources": set(), "sample_path": path})
        for field, value in (("provider_id", payload["id"]), ("content_id", payload["content_id"]),
                             ("source_url", payload["source_url"])):
            group[field] = group[field] or value or ""
        group["assets"].add(asset_id)
        group["links"] += 1
        group["sources"].add(source)
    rows = [{**{k: v for k, v in group.items() if k not in {"assets", "links", "sources"}},
             "asset_count": len(group["assets"]), "link_count": group["links"],
             "sources": " ".join(sorted(group["sources"])),
             "asset_ids": " ".join(map(str, sorted(group["assets"])))}
            for group in groups.values() if group["tier"] != NO_ID]
    rows.sort(key=lambda row: (row["tier"] != MISMATCH, -row["asset_count"], row["asset_code"]))
    return rows, tiers


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="重判已落库刮削归属的来源身份;只读,只出复核 CSV")
    parser.add_argument("--db", type=Path, default=DATABASE_PATH)
    parser.add_argument("--review-csv", type=Path,
                        default=GENERATED_DIR / "scraped-identity-review.csv")
    return parser


def main(argv: list[str] | None = None) -> int:
    args = build_parser().parse_args(argv)
    connection = open_readonly(args.db)
    try:
        rows, tiers = collect(connection)
    finally:
        connection.close()
    print("认不出所查番号的归属行:" + (
        ",".join(f"{tier} {tiers[tier]}" for tier in (MISMATCH, PREFIX, NO_ID)) or "无"))
    for row in rows:
        print(f"  {row['tier']} {row['asset_code']} 查 {row['review_code']} → "
              f"{row['provider_id'] or row['content_id']}:资产 {row['asset_count']} 条")
    write_rows(args.review_csv, FIELDS, rows)
    print(f"复核 CSV → {args.review_csv}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())