Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

50.1 KB
Raw Blame
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
r"""社媒存在感扩张:从已核实的 X 账号走到集链页,扩张全部社媒链接并取最高清头像。

规范化流程(2026-09-02 与用户约定):社媒链接与头像是同一条链路的两个产出——

    entity_link 里的 X/Twitter(minnano-av 管线已装、人工复核过的)
      → X 登出页:og:image 给本人头像、og:title 给显示名(名字闸门的证据)、
        简介文本里的明文外链给集链页
      → 集链页(lit.link / linktr.ee / allmylinks / twpf.jp):服务端直出的
        全部外链——Instagram、TikTok、Threads、Fantia、DMM/MGStage 检索……
      → 各社媒头像同图去重,取最大的最清晰的一张。

产出三份东西:
  1) 链接安装队列 CSV(喂 install_entity_links.py;只含名字闸门通过的行);
  2) 闸门没过的行进复核 CSV,只记录不安装;
  3) 头像候选 CSV(performer-avatar-candidate- 前缀,复核页照常可读),其中质量、
     身份都过关的赢家在 `--apply` 下装进 generated/avatars——用户已明确授权
     「不同社媒头像一般是一样的,取最大的最清晰的;不行就加入复核」。
     达不到自动安装线的照样进复核队列,由人在复核页批准。

缺省是空跑:只产上面三份 CSV,一个字节都不落盘。装错的是人脸,而人脸正是复核
队列存在的理由(KRONE 那次「人对、页面错」就是这一类),所以本脚本和仓库里其余
写入脚本一样,写盘必须显式 `--apply`。

`--apply` 只往好里换:落盘前先按同一把尺(`rank_key`)量一次盘上那张,脸没有更大就
不覆盖。头像目录是几条管线共用的,本脚本的候选池未必比别人装的强。

头像源(登出可取;取不到的只记链接,不产头像):
  X          profile_images 原图(无尺寸后缀),失败退 _400x400 / _200x200
  lit.link   creators/<uuid>/icons/<uuid>.jpe
  babepedia  /pics/<Babe>.jpg —— creator 实体走 babepedia-candidates.csv 的命中行
  jae        Japan Adult Expo 名录里片商自己交的人像 —— 走
             harvest_directory_links.py 产的 jae-performer-links-portraits.csv 命中行
Instagram 登出页拿不到头像图;TikTok 的 oembed 头像只有百来像素——都只记链接。

身份闸门是链式核验:minnano-av 重定向装进账本 → X 页 og:title 命中名字链 →
集链页标题/正文命中名字链。每一跳都有名字证据,断在哪一跳,那一跳之后的产出
就只进复核不自动写入。
"""
from __future__ import annotations

import argparse
import hashlib
import html as html_mod
import json
import re
import sqlite3
import sys
import time
import urllib.parse
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))

from peach.avatar_face import (   # noqa: E402
    FaceProbe, face_px_width, read_sidecar,
)
from peach.avatar_provider import (   # noqa: E402
    POLICY_VERSION,
    AvatarCandidateCache,
    inspect_avatar,
    install_entity_avatar,
    provenance_now,
)
from peach.config import (   # noqa: E402
    DATABASE_PATH, GENERATED_DIR, REVIEW_DIR, STATE_DIR, COVER_DIR,
)
from peach.entities import resolve_entity_id  # noqa: E402
from peach.http import HttpRequest, HttpTransport, HttpxTransport   # noqa: E402
from peach.review_csv import read_rows, write_rows   # noqa: E402
from peach.scripting import (   # noqa: E402
    USER_AGENT, HostLimiter, host_under, hostname_of, open_readonly,
)
from peach.social_links import twimg_tiers   # noqa: E402


X_HOSTS = ("x.com", "twitter.com", "mobile.twitter.com")
AGGREGATOR_HOSTS = ("lit.link", "linktr.ee", "allmylinks.com", "twpf.jp")
BLOG_HOSTS = ("ameblo.jp", "lineblog.me", "note.com", "livedoor.jp", "hatenablog.com")
#: 候选的 provider 归到哪个内容寻址缓存。
PROVIDER_CACHES = {"social-web": "social", "babepedia": "babepedia", "jae": "jae", "cover-fallback": "cover"}
SOCIAL_LABELS = {
    ("instagram.com",): "Instagram",
    ("threads.net", "threads.com"): "Threads",
    ("tiktok.com",): "TikTok",
    ("youtube.com",): "YouTube",
    ("fantia.jp",): "Fantia",
    ("fansly.com",): "Fansly",
    ("dmfan.jp",): "DMFan",
    ("booth.pm",): "BOOTH",
    ("pixiv.net",): "pixiv",
    ("twitch.tv",): "Twitch",
    ("onlyfans.com",): "OnlyFans",
    ("line.me",): "LINE",
}
CATALOG_LABELS = {
    ("dmm.co.jp", "dmm.com"): "DMM 检索",
    ("mgstage.com",): "MGStage 检索",
    ("av-event.jp",): "AV活动检索",
    ("javlibrary.com",): "JavLibrary",
}
#: 简介扫描认得的主机。必须精确到主机本身:`x.com` 是 `netflix.com` 的子串、
#: `twitter.com` 是 `ads-twitter.com` 的后缀,松一档就会把广告域当成这个人的主页。
BIO_HOSTS = frozenset(X_HOSTS + AGGREGATOR_HOSTS + BLOG_HOSTS
                      + tuple(h for hosts in SOCIAL_LABELS for h in hosts))
BIO_URL = re.compile(
    r"(?:https?://)?(?:www\.)?("
    + "|".join(re.escape(h) for h in sorted(BIO_HOSTS, key=len, reverse=True))
    + r")(/[^\s\"'<>\\,,。]+)", re.I)

X_OG_TITLE = re.compile(r'property="og:title"\s+content="([^"]+)"')
X_OG_IMAGE = re.compile(r'property="og:image"\s+content="([^"]+)"')
#: profile_images 的尺寸档位;基名是档位前缀,原图(无后缀)最大。
PBS_SUFFIX = re.compile(r"_(?:normal|bigger|mini|x96|[0-9]+x[0-9]+)$")
PBS_URL = re.compile(
    r"https://pbs\.twimg\.com/profile_images/(\d+)/([A-Za-z0-9_-]+)\.(?:jpg|jpeg|png|webp)",
    re.I)
ANCHOR_HREF = re.compile(r'<a\s[^>]*href=["\']([^"\']+)["\']', re.I)
PAGE_TITLE = re.compile(r"<title[^>]*>(.*?)</title>", re.S | re.I)
OG_IMAGE = re.compile(r'property="og:image"\s+content="([^"]+)"')
LIT_ICON = re.compile(
    r"https://prd\.storage\.lit\.link/images/creators/[0-9a-f-]+/icons/[0-9a-f-]+\.jpe",
    re.I)
TAG = re.compile(r"<[^>]+>")

HANDLE = re.compile(r"^[A-Za-z0-9_]{1,20}$")
#: X 登录页保留路径。页脚(/tos、/privacy、/sitemap.xml、/articles/...)与功能页
#: (/intent、/i)都在 HTML 里以真锚点出现,不拦会把 X 的服务条款装成她的社交链接。
X_RESERVED = frozenset({
    "home", "i", "intent", "hashtag", "search", "messages", "explore",
    "settings", "personalization", "tos", "privacy", "rules", "jobs", "press",
    "legal", "help", "features", "moments", "dev", "developers", "support",
    "safety", "verification", "communities", "compose", "notifications",
    "bookmarks", "login", "signup", "register", "welcome", "sitemap",
})

LINK_FIELDS = ("entity_id", "kind", "name", "link_kind", "label", "url", "evidence",
               "verdict", "reason")
CANDIDATE_FIELDS = (
    "entity_id", "current_name", "matched_name", "name_source", "provider",
    "source_kind", "source_url", "external_id", "gfriends_category",
    "gfriends_file", "width", "height", "face_width", "mime_type", "sha256", "cache_path",
    "provenance_path", "policy_version", "verdict", "avatar_url", "evidence",
)


def fetch(http: HttpTransport, url: str, timeout: float, limiter: HostLimiter,
          accept: str = "text/html,application/xhtml+xml,*/*;q=0.8"):
    """取一次;网络层异常降级为 None,单个来源的 TLS 抖动不拖垮整批。"""
    limiter.wait(url)
    try:
        return http(HttpRequest("GET", url, {"User-Agent": USER_AGENT, "Accept": accept}),
                    timeout, 16 * 1024 * 1024)
    except Exception:
        return None


def visible_text(body: str, limit: int = 20000) -> str:
    return html_mod.unescape(re.sub(r"\s+", " ", TAG.sub(" ", body)))[:limit]


# ---------------------------------------------------------------- 名字链与闸门

def load_entity_chain(connection: sqlite3.Connection, entity_id: int) -> dict:
    row = connection.execute(
        "SELECT kind, canonical_name, metadata_json FROM entity WHERE id=?",
        (entity_id,)).fetchone()
    if row is None:
        return {}
    kind, canonical, raw = row
    jp = ""
    try:
        localization = (json.loads(raw or "{}").get("name_localization") or {})
        jp = localization.get("jp") or ""
    except (TypeError, ValueError):
        jp = ""
    aliases = [r[0] for r in connection.execute(
        "SELECT alias FROM entity_alias WHERE entity_id=? ORDER BY confidence DESC",
        (entity_id,)) if r[0].strip()]
    names: list[str] = []
    for name in [canonical, *aliases, jp]:
        name = (name or "").strip()
        if name and name.casefold() not in {n.casefold() for n in names}:
            names.append(name)
    return {"entity_id": entity_id, "kind": kind, "canonical": canonical or "",
            "names": names}


def identity_match(names: list[str], text: str) -> str:
    """返回名字链里第一个出现在文本中的写法;都不在则空串。"""
    folded = text.casefold()
    for name in names:
        if len(name) >= 2 and name.casefold() in folded:
            return name
    return ""


# ---------------------------------------------------------------- 链接分类

def classify(url: str) -> tuple[str, str]:
    """(link_kind, label)。label 要让人在资料页上不点就知道去的是什么。"""
    host = hostname_of(url)
    if host_under(host, X_HOSTS):
        handle = urllib.parse.urlsplit(url).path.strip("/").split("?")[0]
        return "social", (f"X @{handle}" if handle and HANDLE.match(handle) else "X")
    for hosts, platform in SOCIAL_LABELS.items():
        if host_under(host, hosts):
            handle = urllib.parse.urlsplit(url).path.strip("/").lstrip("@")
            return "social", (f"{platform} @{handle}" if handle else platform)
    if host_under(host, AGGREGATOR_HOSTS):
        return "social", f"链接集({host})"
    if host_under(host, BLOG_HOSTS):
        return "social", "博客"
    for hosts, label in CATALOG_LABELS.items():
        if host_under(host, hosts):
            return "catalog", label
    return "official", "官方网站"


def profile_shaped(url: str) -> bool:
    """这个社媒 URL 是不是指向个人主页而不是平台功能页。

    集链页与简介里混着 /intent、/hashtag、/explore、/p/<帖子> 这类地址,它们指向
    平台而不是这个人。X 域内只认「单段 handle 形」路径——X 的页脚与帮助页全是
    单段或两段真锚点,靠枚举保留字加单段约束一起拦;youtu.be 没有个人主页形态。
    """
    host = hostname_of(url)
    path = urllib.parse.urlsplit(url).path
    if host_under(host, X_HOSTS):
        segments = path.strip("/").split("/")
        return (len(segments) == 1 and HANDLE.match(segments[0])
                and segments[0].lower() not in X_RESERVED)
    if host_under(host, ("instagram.com",)):
        return not re.search(r"^/(p|reel|reels|explore|stories)(\b|/)", path)
    if host_under(host, ("youtube.com",)):
        return path.startswith(("/@", "/channel/", "/c/", "/user/"))
    if host_under(host, ("youtu.be",)):
        return False
    if host_under(host, ("tiktok.com",)):
        return path.startswith("/@")
    return True


# ---------------------------------------------------------------- X 登出页

def parse_x_profile(body: str) -> dict:
    """从 X 登出页 HTML 里取 (显示名, 本人头像基名, 简介外链)。

    本人的头像以 og:image 为准——页面里混着大量 pbs.twimg.com 地址(嵌入模块、
    引用卡片),谁的出现次数多都说明不了归属;og:image 是 X 自己声明的本人头像。
    og:image 缺席时退回「去档位后唯一基名」:出现两个以上基名就放弃,绝不从
    「推荐关注」里替人挑头像。
    """
    title = X_OG_TITLE.search(body)
    display = html_mod.unescape(title.group(1)).strip() if title else ""
    og = X_OG_IMAGE.search(body)
    base = ""
    if og and "profile_images" in og.group(1):
        path = html_mod.unescape(og.group(1)).split("profile_images/", 1)[1]
        path = path.split("?", 1)[0]
        # og:image 通常带档位后缀(_200x200.jpg);先去扩展名再剥档位,
        # 否则锚定行尾的尺寸正则永远碰不到后缀。
        stem = PBS_SUFFIX.sub("", path.rsplit(".", 1)[0])
        base = "https://pbs.twimg.com/profile_images/" + stem
    if not base:
        stripped = {image_id + "/" + PBS_SUFFIX.sub("", stem)
                    for image_id, stem in PBS_URL.findall(body)}
        if len(stripped) == 1:
            base = "https://pbs.twimg.com/profile_images/" + stripped.pop()
    # 档位表由 `peach.social_links.twimg_tiers` 给(厂牌 Logo 那条路共用同一份)。
    avatars: list[str] = twimg_tiers(base + ".jpg") if base else []
    bio_urls: list[str] = []
    for match in BIO_URL.finditer(html_mod.unescape(body)):
        url = "https://" + match.group(1).removeprefix("www.").lower() + match.group(2)
        url = url.rstrip("),.;]")
        if profile_shaped(url):
            bio_urls.append(url)
    return {"display_name": display, "avatars": avatars, "bio_urls": bio_urls}


# ---------------------------------------------------------------- 集链页

def parse_link_page(url: str, body: str) -> dict:
    """集链页(lit.link 等)→ (标题, 头像图, 全部外链)。

    lit.link 是服务端渲染,锚点直出;linktr.ee 对爬虫回 403(安装器的可达性
    记录里有同一笔账),取不到就由调用方记「未取得」,不假装没有。
    """
    title_match = PAGE_TITLE.search(body)
    title = html_mod.unescape(title_match.group(1)).strip() if title_match else ""
    icon = ""
    icon_match = LIT_ICON.search(body)
    if icon_match:
        icon = icon_match.group(0)
    else:
        og = OG_IMAGE.search(body)
        if og:
            icon = html_mod.unescape(og.group(1))
    self_host = hostname_of(url)
    anchors: list[str] = []
    for href in ANCHOR_HREF.findall(body):
        href = html_mod.unescape(href).strip()
        if href.startswith("http") and hostname_of(href) != self_host:
            anchors.append(href)
    return {"title": title, "icon": icon, "anchors": anchors}


# ---------------------------------------------------------------- ledger 读取

def x_handle(url: str) -> str:
    """X/Twitter URL → handle;功能页与带查询串的返回空。"""
    path = urllib.parse.urlsplit(url).path.strip("/")
    handle = path.split("/")[0]
    return handle if HANDLE.match(handle) and handle.lower() not in (
        "home", "intent", "hashtag", "search", "i", "explore", "settings") else ""


def load_babepedia_rows(path: Path) -> dict[int, dict]:
    """babepedia-candidates.csv → {entity_id: 行},只留命中与需人工确认。"""
    if not path.is_file():
        return {}
    out = {}
    for row in read_rows(path):
        if str(row.get("verdict") or "") not in ("命中", "需人工确认"):
            continue
        if str(row.get("entity_id") or "").isdigit() and row.get("portrait_url"):
            out[int(row["entity_id"])] = row
    return out


def load_jae_rows(path: Path) -> dict[int, list[dict]]:
    """jae 人像候选表 → {entity_id: [行]},只留唯一命中的。

    2014/2015/2017 三届各有一张,同一个人可能占三行;三张一起进竞选,谁大用谁。
    """
    out: dict[int, list[dict]] = {}
    if not path.is_file():
        return out
    for row in read_rows(path):
        if str(row.get("verdict") or "") != "命中" or not row.get("portrait_url"):
            continue
        if str(row.get("entity_id") or "").isdigit():
            out.setdefault(int(row["entity_id"]), []).append(row)
    return out


def load_targets(connection: sqlite3.Connection, avatar_dir: Path,
                 entities: list[int], force: bool, *, include_cover_targets: bool = False) -> list[dict]:
    """本轮要跑的人:缺头像、且至少有一条可扩张的路径。

    performer 走社媒路线(账本里有 X 链接)与 jae 名录路线(jae 人像候选表的命中行);
    creator 走 babepedia 路线(babepedia-candidates.csv 的命中行)。几条路都有就都走——
    头像竞选不看路线,谁的最大最清晰用谁。
    """
    babe = {}
    for old_id, row in load_babepedia_rows(GENERATED_DIR / "babepedia-candidates.csv").items():
        target = (old_id if connection.execute('SELECT 1 FROM entity WHERE id=?', (old_id,)).fetchone()
                  else resolve_entity_id(connection, old_id))
        if target is not None:
            babe.setdefault(target, row)
    jae = load_jae_rows(REVIEW_DIR / "jae-performer-links-portraits.csv")
    x_links: dict[int, str] = {}
    for entity_id, url in connection.execute(
            "SELECT entity_id, url FROM entity_link WHERE hostname IN "
            "('x.com','twitter.com','mobile.twitter.com') ORDER BY entity_id"):
        handle = x_handle(url)
        if handle:
            x_links.setdefault(int(entity_id), handle)

    if entities:
        ids = entities
    else:
        ids = [int(row[0]) for row in connection.execute(
            "SELECT id FROM entity WHERE kind IN ('performer','creator') ORDER BY id")]
    targets = []
    for entity_id in ids:
        info = load_entity_chain(connection, entity_id)
        if not info:
            continue
        kind = info["kind"]
        if (avatar_dir / f"{kind}-{entity_id}.img").exists() and not force:
            continue
        routes: dict[str, object] = {}
        if entity_id in x_links:
            routes["x"] = x_links[entity_id]
        if entity_id in babe:
            routes["babepedia"] = babe[entity_id]
        if entity_id in jae:
            routes["jae"] = jae[entity_id]
        if not routes and not (include_cover_targets and kind == "performer"):
            continue
        targets.append({**info, "routes": routes})
    return targets


# ---------------------------------------------------------------- 头像竞选

def passes_auto_bar(width: int, height: int) -> bool:
    """自动安装线:方图 400×400 起(X/lit.link 原图的原生档位),竖构图人像沿用
    Gfriends 的 500/300 门槛。低于线的赢家照常进复核队列,由人批准。"""
    return ((width == height and min(width, height) >= 400)
            or (max(width, height) >= 500 and min(width, height) >= 300))


def shows_a_real_face(candidate: dict) -> bool:
    """这张图里有没有一张真人的脸。检不出就当没有。

    官方社媒头像也可能是一张画:本人挂动漫头像的不在少数。而头像这一格要
    回答的是「这是谁」,画出来的那张脸回答不了,来源档再高也一样。

    判据借 YuNet,它只认真人脸。这把尺会把真人的侧脸、低头一并判成 False——
    那不是误伤,检不出脸的图放进 112px 的圆框里本来也认不出人,退回复核
    队列由人看一眼正是它存在的理由。全批检不出(模型没下到)时整批退回复核,
    比按画布尺寸盲装安全。
    """
    return bool(candidate.get("face_width"))


def fetch_image(http: HttpTransport, url: str, timeout: float, limiter: HostLimiter):
    """取一张并完整解码校验;失败返回 None。"""
    response = fetch(http, url, timeout, limiter, accept="image/*")
    if response is None or response.status != 200:
        return None
    inspected = inspect_avatar(response.body)
    if inspected is None:
        return None
    return response.body, inspected


def harvest_entity(record: dict, http: HttpTransport, limiter: HostLimiter,
                   timeout: float,
                   caches: dict[str, AvatarCandidateCache]) -> dict:
    """跑一个人的全部路线,返回 {links, candidates, notes}。

    links:发现的全部链接(已过 profile_shaped),每条带 gated 标记——
    False 的行只进复核 CSV。candidates:全部有效头像(入内容寻址缓存),
    调用方做同图去重、竞选与安装。
    """
    names = record["names"]
    links: list[dict] = []
    candidates: list[dict] = []
    notes: list[str] = []

    x_identity = ""
    x_handle_value = record["routes"].get("x")
    if x_handle_value:
        response = fetch(http, f"https://x.com/{x_handle_value}", timeout, limiter)
        if response is None or response.status != 200:
            notes.append(f"X @{x_handle_value} 未取得"
                         f"(HTTP {response.status if response else 'transport'})")
        else:
            body = response.body.decode("utf-8", "replace")
            parsed = parse_x_profile(body)
            page_text = parsed["display_name"] + " " + visible_text(body, 8000)
            x_identity = identity_match(names, page_text)
            if not x_identity:
                notes.append(f"X @{x_handle_value} 页面没有命中名字链,此跳不采信")
            for url in parsed["bio_urls"]:
                kind, label = classify(url)
                links.append({"link_kind": kind, "label": label, "url": url,
                              "evidence": f"X @{x_handle_value} 简介外链",
                              "gated": bool(x_identity)})
            for url in parsed["avatars"]:
                got = fetch_image(http, url, timeout, limiter)
                if got is None:
                    continue
                _, inspected = got
                object_path = caches["social"].store(url, got[0], inspected)
                candidates.append({
                    "provider": "social-web", "source_kind": "official_profile",
                    "source_url": url, "external_id": f"x:{x_handle_value}",
                    "width": inspected.width, "height": inspected.height,
                    "mime_type": inspected.mime_type, "sha256": inspected.sha256,
                    "object_path": object_path,
                    "matched": x_identity, "name_source": "social_identity_gate",
                    "evidence": f"X @{x_handle_value} 头像 "
                                f"{inspected.width}×{inspected.height}",
                })
                break

    # 集链页:X 简介里给的入口。闸门用集链页自己的标题/正文再验一遍——
    # 就算 X 一跳过了,聚合页也可能被别人注册同 handle 抢走。
    aggregator_urls = [link["url"] for link in links
                       if host_under(hostname_of(link["url"]), AGGREGATOR_HOSTS)]
    for agg_url in list(dict.fromkeys(u.rstrip("/") for u in aggregator_urls))[:3]:
        response = fetch(http, agg_url, timeout, limiter)
        if response is None or response.status != 200:
            notes.append(f"集链页未取得:{agg_url}"
                         f"(HTTP {response.status if response else 'transport'})")
            continue
        body = response.body.decode("utf-8", "replace")
        page = parse_link_page(agg_url, body)
        agg_identity = identity_match(names, page["title"] + " " + visible_text(body))
        if not agg_identity:
            notes.append(f"集链页标题与正文没有命中名字链,只记复核:{agg_url}")
        for href in page["anchors"]:
            if not profile_shaped(href):
                continue
            kind, label = classify(href)
            if kind == "official":
                # 集链页上的不明外链不进 official——那是给事务所这类已核实来源的;
                # 用主机名当标签让人一眼看清它其实是什么。
                kind, label = "social", hostname_of(href)
            links.append({"link_kind": kind, "label": label, "url": href,
                          "evidence": (f"{hostname_of(agg_url)} 页面锚点"
                                       + (f",标题「{page['title'][:40]}」"
                                          if page["title"] else "")),
                          "gated": bool(agg_identity)})
        if page["icon"] and agg_identity:
            got = fetch_image(http, page["icon"], timeout, limiter)
            if got is not None:
                _, inspected = got
                object_path = caches["social"].store(page["icon"], got[0], inspected)
                candidates.append({
                    "provider": "social-web", "source_kind": "official_profile",
                    "source_url": page["icon"],
                    "external_id": hostname_of(agg_url),
                    "width": inspected.width, "height": inspected.height,
                    "mime_type": inspected.mime_type, "sha256": inspected.sha256,
                    "object_path": object_path,
                    "matched": agg_identity, "name_source": "social_identity_gate",
                    "evidence": f"{hostname_of(agg_url)} 头像 "
                                f"{inspected.width}×{inspected.height}",
                })

    # babepedia 路线(creator)。身份结论沿用回配时的判定,这里不重新猜人。
    babe_row = record["routes"].get("babepedia")
    if babe_row:
        portrait = str(babe_row.get("portrait_url") or "")
        got = fetch_image(http, portrait, timeout, limiter)
        if got is None:
            notes.append("babepedia 档案像未取得或不是可用图片")
        else:
            _, inspected = got
            object_path = caches["babepedia"].store(portrait, got[0], inspected)
            candidates.append({
                "provider": "babepedia", "source_kind": "external_media_library",
                "source_url": portrait,
                "external_id": f"babepedia:{babe_row.get('babepedia_name', '')}",
                "width": inspected.width, "height": inspected.height,
                "mime_type": inspected.mime_type, "sha256": inspected.sha256,
                "object_path": object_path,
                "matched": str(babe_row.get("babepedia_name") or ""),
                "name_source": "babepedia_match",
                "identity_verified": str(babe_row.get("verdict") or "") == "命中",
                "evidence": (f"babepedia「{babe_row.get('babepedia_name', '')}」档案像 "
                             f"{inspected.width}×{inspected.height},"
                             f"回配判定「{babe_row.get('verdict', '')}」"),
            })

    # jae 路线(performer)。名字判定在 harvest_directory_links.py 那一步做完,只收 `命中`
    # 的行,这里不重新猜人——头像装错人和链接装错人是同一个错误。
    for jae_row in record["routes"].get("jae") or ():
        portrait = str(jae_row.get("portrait_url") or "")
        got = fetch_image(http, portrait, timeout, limiter)
        if got is None:
            notes.append(f"jae 名录人像未取得或不是可用图片:{portrait}")
            continue
        _, inspected = got
        object_path = caches["jae"].store(portrait, got[0], inspected)
        page = str(jae_row.get("page") or "")
        matched = str(jae_row.get("matched_name") or "")
        candidates.append({
            "provider": "jae", "source_kind": "official_directory",
            "source_url": portrait,
            "external_id": f"jae:{page.rsplit('/', 1)[-1] or page}",
            "width": inspected.width, "height": inspected.height,
            "mime_type": inspected.mime_type, "sha256": inspected.sha256,
            "object_path": object_path,
            "matched": matched, "name_source": "jae_directory",
            "identity_verified": str(jae_row.get("verdict") or "") == "命中",
            "evidence": (f"jae 名录「{matched}」人像 "
                         f"{inspected.width}×{inspected.height},页面 {page}"),
        })

    return {"links": links, "candidates": candidates, "notes": notes}


def cover_fallback(connection, record: dict, cache: AvatarCandidateCache,
                   cover_root: Path = COVER_DIR, probe=None) -> dict | None:
    """仅使用已关联的单人作品封面,保留完整大图和来源。

    封面是这条链路上脸最容易被画布口径坑到的一档:JAV 封面普遍是 800×538 的双联版式,
    右半幅整幅剧照里那张脸常常只有几十像素,而左半幅的文字块一个像素也不给脸。挑封面
    和挑社媒头像用同一把尺——先看脸有多少像素。
    """
    if record["kind"] != "performer":
        return None
    candidates = []
    for (code,) in connection.execute(
            "SELECT DISTINCT a.code FROM asset a JOIN asset_entity ae ON ae.asset_id=a.id "
            "WHERE ae.entity_id=? AND ae.role='performer' AND coalesce(a.code,'')<>'' "
            "AND (a.disposal IS NULL OR a.disposal<>'trash') "
            "AND NOT EXISTS(SELECT 1 FROM asset_entity other WHERE other.asset_id=a.id "
            "AND other.role='performer' AND other.entity_id<>ae.entity_id)", (record["entity_id"],)):
        if not re.fullmatch(r'[A-Za-z0-9-]+', code):
            continue
        path = cover_root / f'{code}.jpg'
        if not path.is_file():
            continue
        data = path.read_bytes()
        inspected = inspect_avatar(data)
        if inspected is None or not passes_auto_bar(inspected.width, inspected.height):
            continue
        source = path.resolve().as_uri()
        stored = cache.store(source, data, inspected)
        candidates.append(dict(provider='cover-fallback', source_kind='single_performer_cover',
                               source_url=source, external_id=code, width=inspected.width,
                               height=inspected.height, mime_type=inspected.mime_type,
                               sha256=inspected.sha256, object_path=stored,
                               matched=record['canonical'], name_source='ledger-performer',
                               identity_verified=False,
                               evidence=f'单人作品 {code} 完整封面'))
    if not candidates:
        return None
    if probe is not None:
        measure_faces(candidates, probe)
    return max(candidates, key=rank_key)


def measure_faces(candidates: list[dict], probe) -> None:
    """就地给每枚候选检一次脸。同一张图(SHA 相同)只检一次。

    落在候选上的是两样东西:`face_width` 是选图判据——量的是像素不是比例,因为
    `rank_key` 要回答的是「装进圆框后看不看得清这张脸」,而那取决于脸有多少像素可用,
    与画布多大无关;`face_record` 是赢家落盘时一并写出的取景 sidecar,同一次检出直接
    复用,不让页面读的脸框和挑图用的脸框来自两次不同的检测。
    """
    seen: dict[str, dict | None] = {}
    for candidate in candidates:
        digest = candidate.get("sha256", "")
        if digest not in seen:
            path = candidate.get("object_path")
            seen[digest] = probe(path) if path else None
        record = seen[digest]
        candidate["face_record"] = record
        candidate["face_width"] = face_px_width(record)


#: 挑图的第一层判据:这张图是拍来做人像的,还是拍来卖片的。脸有多少像素是第二层。
#:
#: 作品封面是这里唯一被单独压一档的来源。它不是拍来做人像的:JAV 封面普遍是双联版式,
#: 左半幅整块留给标题和宣传文字,右半幅那张脸还常被角标和腰封压住。把它放大成一个
#: 64–160 px 的圆头像,圆框里出现的往往是半行日文标题,而不是这个人。像素再多也换不来
#: 一张能认人的脸——这一档差的是构图,不是清晰度,所以必须在比脸宽之前就分开。
#: 缩略图在它下面:那是同一张图被裁小的一份,除了没有别的可用之外没有任何理由选它。
PORTRAIT_TIER = 2
COVER_TIER = 1
THUMBNAIL_TIER = 0
SOURCE_TIERS = {
    # 候选行用的是 source_kind
    "official_profile": PORTRAIT_TIER,        # 本人社媒头像,脸是自己挑的
    "official_directory": PORTRAIT_TIER,      # 事务所与名录的宣材人像
    "official_mirror": PORTRAIT_TIER,
    "external_media_library": PORTRAIT_TIER,  # 媒体库整版人像
    "single_performer_cover": COVER_TIER,
    "thumbnail": THUMBNAIL_TIER,
    # 盘上那张只留下 provenance 里的 provider,两套词汇在这里并成一张表
    "social-web": PORTRAIT_TIER,
    "jae": PORTRAIT_TIER,
    "babepedia": PORTRAIT_TIER,
    "verified-photo-page": PORTRAIT_TIER,
    "cover-fallback": COVER_TIER,
}


def source_tier(row: dict) -> int:
    """认不出来源就当人像档。

    这个默认方向是有意的:认不出来的多半是别的管线装的整版人像(gfriends 那 434 张、
    Stash 那 14 张的 provenance 里根本没有 provider 字段),把它们当封面会让本脚本那些
    400×400 的社媒头像一路把它们顶掉——正是 2026-09-06 那趟 `--force` 的教训。
    """
    for key in (row.get("source_kind"), row.get("provider")):
        if key in SOURCE_TIERS:
            return SOURCE_TIERS[key]
    return PORTRAIT_TIER


def rank_key(row: dict) -> tuple[int, int, int, int, int]:
    """先看这张图是不是拍来做人像的,再看脸的像素宽,画布只是它们都没得比时的退路。

    来源档在最前面,所以一张 400×400 社媒头像胜过 800×538 的作品封面,哪怕封面上
    那张脸更大:封面上和脸抢地方的是标题文字,那不是清晰度问题。同一档之内才轮到
    脸宽说话。

    检不出脸的候选脸宽是 0,自然排到有脸的后面;全都检不出(侧脸、低头、模型缺席)
    时整批都是 0,排序原样落回画布口径:先比短边再比长边——竖构图人像与方图头像
    用同一把尺会偏袒长边。
    """
    return (source_tier(row),
            row.get("face_width") or 0,
            min(row["width"], row["height"]),
            max(row["width"], row["height"]),
            row["width"] * row["height"])


def select_winner(candidates: list[dict]) -> tuple[dict | None, list[dict]]:
    """同图去重(SHA-256 相同 = 同一张图的不同档位/不同平台),取脸最清楚的那一张。

    **脸的像素数先说话,画布只是它没得比时的退路。** 头像最终显示在一个 64–160 px
    的圆框里,能不能认出是谁,取决于那张脸有多少像素,不取决于这张图有多大。
    实测 `performer-8711`:装着的是 640×960 的全身站姿照,脸只有 67 px 宽,在资料页
    圆框里放到源图 1:1 也只占两成;同一个人另有一张 540×810 的半身照,画布小了 15%,
    脸却有 160 px 上下——按画布挑,赢的是前者,而它是两张里唯一看不清脸的。
    判据本身见 `rank_key`。
    """
    by_sha: dict[str, dict] = {}
    for candidate in candidates:
        kept = by_sha.get(candidate["sha256"])
        if kept is None:
            by_sha[candidate["sha256"]] = candidate
            continue
        larger = max((kept, candidate),
                     key=lambda row: (row["width"], row["height"]))
        larger["evidence"] += ";" + (candidate if larger is kept else kept)["evidence"] \
            + "(同图)"
        by_sha[candidate["sha256"]] = larger
    if not by_sha:
        return None, []
    ranked = sorted(by_sha.values(), key=rank_key)
    return ranked[-1], ranked[:-1]


def incumbent_provider(destination: Path) -> str:
    """盘上那张是谁装的。只有 provenance 记得住这件事,图本身不带来源。

    没有 provenance 的按认不出处理,`source_tier` 会把它当人像档——434 张 gfriends
    和 14 张 Stash 头像的 provenance 里就没有 provider 这个字段。
    """
    try:
        record = json.loads(Path(f"{destination}.provenance.json").read_text(encoding="utf-8"))
    except (OSError, ValueError):
        return ""
    return str(record.get("provider") or "")


def incumbent_row(destination: Path, probe) -> dict | None:
    """盘上那张的选图口径,用来和赢家比。没有图、读不出都返回 None。

    先用现成的取景 sidecar,那本来就是同一次 YuNet 检出的结果;没有 sidecar 才真去
    检一遍。525 张头像各检一次要好几分钟,而绝大多数的 sidecar 早就算好了。
    """
    if not destination.is_file():
        return None
    record = read_sidecar(destination)
    if not (record and record.get("px")):
        record = probe(destination)
    if not (record and record.get("px")):
        return None
    width, height = record["px"][0], record["px"][1]
    if not (width > 0 and height > 0):
        return None
    return {"width": width, "height": height,
            "face_width": face_px_width(record), "face_record": record,
            "provider": incumbent_provider(destination)}


def outranks_incumbent(winner: dict, incumbent: dict | None) -> bool:
    """赢家值不值得覆盖盘上那张。用的是同一把尺 `rank_key`。

    这道闸门是 `--force` 的安全带。`--force` 的语义是「已有头像也重跑」,覆盖的却是
    **所有**已装头像,包括别的管线装的——而本脚本的候选池(X 的 400×400、jae 的
    320×500、作品封面)比 `verified-photo-page` 那类整版人像差一大截。2026-09-06 实测:
    一趟 `--force --apply` 换掉 350 张,其中 282 张脸变小,脸宽中位数从 262 px 掉到
    139 px。缺头像时这道闸门是空操作,只在真要覆盖时才拦。
    """
    if incumbent is None:
        return True
    return rank_key(winner) > rank_key(incumbent)


def install_avatar(avatar_dir: Path, kind: str, entity_id: int, winner: dict) -> Path:
    """赢家直接落盘到 /entity-image 真正读的目录,带 .ct、provenance 与取景 sidecar。"""
    if source_tier(winner) == COVER_TIER:
        raise ValueError("作品封面只作为候选留档,不安装为人物头像")
    data = winner["object_path"].read_bytes()
    if hashlib.sha256(data).hexdigest() != winner["sha256"]:
        raise ValueError("缓存对象与竞选记录的哈希不一致,拒绝安装")
    # 脸在竞选时就检过了(`measure_faces`),同一张图不必再跑一次模型。
    return install_entity_avatar(
        avatar_dir, kind, entity_id, data, winner["mime_type"], {
            "source": "social avatar harvest",
            "provider": winner["provider"],
            "source_url": winner["source_url"],
            "external_id": winner["external_id"],
            "matched_name": winner.get("matched") or "",
            "name_source": winner.get("name_source") or "",
            "sha256": winner["sha256"],
            "width": winner["width"],
            "height": winner["height"],
            "policy_version": POLICY_VERSION,
        }, winner.get("face_record"), probe_face=False)


def provenance_for(cache: AvatarCandidateCache, entity_id: int, candidate: dict):
    return provenance_now(
        entity_id=int(entity_id), provider=candidate["provider"],
        source_kind=candidate["source_kind"], matched_name=candidate.get("matched") or "",
        name_source=candidate.get("name_source") or "",
        external_id=candidate["external_id"], upstream_url=candidate["source_url"],
        width=candidate["width"], height=candidate["height"],
        mime_type=candidate["mime_type"], sha256=candidate["sha256"],
        cache_path=str(candidate["object_path"].relative_to(cache.root)),
    )


def merge_prior_pending(db_path: Path, new_path: Path, rows: list[dict]) -> None:
    """把上一批候选里还没被判过的行并进本轮,别让换批次把复核队列清空。

    复核页每个类别只读 mtime 最新的一份 CSV;不并的话,2026-08-25 批次里
    还没人判过的 17 条 Gfriends 候选会从界面上凭空消失。已判过的不再并——
    判过即终局,装不装都已经有结论。
    """
    decided: set[str] = set()
    try:
        connection = open_readonly(db_path)
        decided = {str(row[0]) for row in connection.execute(
            "SELECT item_key FROM review_decision WHERE category='performer_avatars'")}
        connection.close()
    except sqlite3.Error:
        pass
    prior_files = sorted(GENERATED_DIR.glob("performer-avatar-candidate-*.csv"),
                         key=lambda p: p.stat().st_mtime_ns)
    prior = [p for p in prior_files if p.resolve() != new_path.resolve()]
    if not prior:
        return
    fresh_keys = {str(row["entity_id"]) for row in rows}
    carried = 0
    for old in read_rows(prior[-1]):
        key = str(old.get("entity_id") or "")
        if not key or key in fresh_keys or key in decided:
            continue
        rows.append(old)
        carried += 1
    if carried:
        print(f"并入上一批未判定候选 {carried} 条({prior[-1].name})")


def build_parser() -> argparse.ArgumentParser:
    stamp = time.strftime("%Y%m%d-%H%M%S")
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--db", type=Path, default=DATABASE_PATH)
    parser.add_argument("--avatars", type=Path, default=GENERATED_DIR / "avatars")
    parser.add_argument("--cache-root", type=Path,
                        default=GENERATED_DIR / "provider-cache" / "performer-avatars")
    parser.add_argument("--candidates", type=Path,
                        default=GENERATED_DIR / f"performer-avatar-candidate-social-{stamp}.csv")
    parser.add_argument("--out-links", type=Path,
                        default=GENERATED_DIR / f"social-links-{stamp}.csv")
    parser.add_argument("--review-links", type=Path,
                        default=GENERATED_DIR / f"social-links-unverified-{stamp}.csv")
    parser.add_argument("--entities", type=str, default="",
                        help="逗号分隔的 entity id;缺省跑全部缺头像且有路线的人")
    parser.add_argument("--force", action="store_true",
                        help="已有头像也参加竞选(默认跳过);只有脸更大才覆盖")
    parser.add_argument("--apply", action="store_true",
                        help="把过线的赢家头像落盘到 --avatars;缺省只产 CSV 不写盘")
    parser.add_argument("--timeout", type=float, default=25.0)
    parser.add_argument("--limit", type=int, default=0)
    parser.add_argument("--lock", type=Path, default=STATE_DIR / ".social-avatars.lock")
    return parser


def run(args) -> int:
    limiter = HostLimiter({
        "x.com": 2.0, "pbs.twimg.com": 0.3, "lit.link": 1.5,
        "babepedia.com": 3.0, "linktr.ee": 2.0, "jae.tokyo": 1.2,
    })
    connection = open_readonly(args.db)
    http = HttpxTransport()
    caches = {
        "social": AvatarCandidateCache(args.cache_root / "social"),
        "babepedia": AvatarCandidateCache(args.cache_root / "babepedia"),
        "jae": AvatarCandidateCache(args.cache_root / "jae"),
        "cover": AvatarCandidateCache(args.cache_root / "cover"),
    }
    candidate_rows: list[dict] = []
    install_rows: list[dict] = []
    review_rows: list[dict] = []
    installed = 0
    kept = 0
    probe = FaceProbe()
    try:
        entities = [int(v) for v in str(args.entities).split(",") if v.strip().isdigit()]
        targets = load_targets(connection, args.avatars, entities, args.force, include_cover_targets=True)
        if args.limit:
            targets = targets[:args.limit]
        print(f"待跑 {len(targets)} 人:"
              + "、".join(f"{r['entity_id']}({r['kind'][0]})" for r in targets[:15])
              + ("…" if len(targets) > 15 else ""), flush=True)

        for record in targets:
            entity_id, kind = record["entity_id"], record["kind"]
            result = harvest_entity(record, http, limiter, args.timeout, caches)
            # 同一地址会同时出现在 X 简介与集链页锚点里,去重保序。键用主机+路径:
            # Instagram/YouTube 的分享链接每次抓都会换 igsh/si 跟踪参数,按完整
            # URL 去重会把同一条链接装两遍。
            seen: set[tuple[str, str]] = set()
            deduped = []
            for link in result["links"]:
                split = urllib.parse.urlsplit(link["url"])
                key = (split.hostname or "", split.path.rstrip("/"))
                if key in seen:
                    continue
                seen.add(key)
                deduped.append(link)
            for link in deduped:
                row = {"entity_id": entity_id, "kind": kind,
                       "name": record["canonical"], "link_kind": link["link_kind"],
                       "label": link["label"], "url": link["url"],
                       "evidence": link["evidence"]}
                if link["gated"]:
                    row.update(verdict="确认", reason="")
                    install_rows.append(row)
                else:
                    row.update(verdict="需人工确认", reason="名字闸门未过")
                    review_rows.append(row)

            # 先量脸再挑:赢家由脸的像素数定,画布只是没得比时的退路。
            measure_faces(result["candidates"], probe)
            winner, runners_up = select_winner(result["candidates"])
            if (winner is None or not winner.get('matched')
                    or not passes_auto_bar(winner['width'], winner['height'])):
                fallback = cover_fallback(connection, record, caches['cover'],
                                          probe=probe)
                if fallback:
                    if winner:
                        runners_up.append(winner)
                    winner = fallback
            # 盘上那张只量一次,赢家拿它当门槛:新的没有更好就不动它。
            incumbent = incumbent_row(args.avatars / f"{kind}-{entity_id}.img", probe)
            for candidate in ([winner] if winner else []) + runners_up:
                cache = caches[PROVIDER_CACHES[candidate["provider"]]]
                provenance_path = cache.store_provenance(
                    provenance_for(cache, entity_id, candidate))
                identity_ok = candidate.get("source_kind") != 'single_performer_cover' and candidate.get("identity_verified", True) and bool(
                    candidate.get("matched"))
                is_winner = candidate is winner
                note = ""
                if is_winner and identity_ok and not shows_a_real_face(candidate):
                    note = "竞选赢家但检不出真人脸(多半是动漫头像),不自动安装,待复核"
                elif is_winner and identity_ok:
                    note = "竞选赢家;质量与身份过关"
                    # 「没过线」和「空跑」是两件事,别都写成待复核:前者要人去看,
                    # 后者只差一个 --apply。
                    if not passes_auto_bar(candidate["width"], candidate["height"]):
                        note += ",未达自动安装线,待复核"
                    elif not args.apply:
                        note += ",已过自动安装线;加 --apply 才落盘"
                    elif not outranks_incumbent(candidate, incumbent):
                        note += (f",盘上那张的脸更大({incumbent['face_width']} px "
                                 f"vs {candidate.get('face_width') or 0} px),不覆盖")
                        kept += 1
                    else:
                        destination = install_avatar(args.avatars, kind, entity_id,
                                                     candidate)
                        note += f",已安装 {destination.name}"
                        installed += 1
                elif is_winner:
                    note = "竞选赢家但身份未核实;先核链接再装"
                else:
                    note = "竞选落选(同图或脸更小);留档备查"
                candidate_rows.append({
                    "entity_id": entity_id, "current_name": record["canonical"],
                    "matched_name": candidate.get("matched") or "",
                    "name_source": candidate.get("name_source") or "",
                    "provider": candidate["provider"],
                    "source_kind": candidate["source_kind"],
                    "source_url": candidate["source_url"],
                    "external_id": candidate["external_id"],
                    "gfriends_category": "", "gfriends_file": "",
                    "width": candidate["width"], "height": candidate["height"],
                    # 复核的人据此一眼看出「这张脸够不够大」,比画布尺寸有用得多。
                    "face_width": candidate.get("face_width") or 0,
                    "mime_type": candidate["mime_type"],
                    "sha256": candidate["sha256"],
                    "cache_path": candidate["object_path"].name,
                    "provenance_path": provenance_path.name,
                    "policy_version": POLICY_VERSION,
                    "verdict": ("ok" if identity_ok and shows_a_real_face(candidate)
                                else "no_real_face" if identity_ok
                                else "identity_unverified"),
                    "avatar_url": candidate["source_url"],
                    "evidence": candidate["evidence"] + ";" + note,
                })
            for note in result["notes"]:
                print(f"  [{entity_id}] {note}", flush=True)
            print(f"[{entity_id}] {record['canonical']}:链接 {len(deduped)} 条"
                  f"(自动 {sum(1 for l in deduped if l['gated'])})、"
                  f"候选 {len(result['candidates'])} 张", flush=True)
    finally:
        http.close()
        connection.close()

    merge_prior_pending(args.db, args.candidates, candidate_rows)
    write_rows(args.candidates, CANDIDATE_FIELDS, candidate_rows, atomic=True)
    write_rows(args.out_links, LINK_FIELDS, install_rows, atomic=True)
    write_rows(args.review_links, LINK_FIELDS, review_rows, atomic=True)
    if probe.unavailable:
        # 说清这一轮是按什么挑的。不说的话,一次「今天没网下不到 ONNX」会静悄悄地
        # 把整批退回画布口径,产出看着完全正常,脸却全挑小了。
        print(f"人脸检测未取得({probe.unavailable}):本轮按画布尺寸挑图,"
              "face_width 一列全是 0;模型可用后加 --force 重跑该批。")
    if not args.apply:
        print("空跑:以上一张头像都没落盘;确认候选无误后加 --apply 重跑。")
    print({"候选": len(candidate_rows), "自动安装": installed, "在位的更好未覆盖": kept,
           "带脸候选": sum(1 for r in candidate_rows if int(r.get("face_width") or 0) > 0),
           "链接待装": len(install_rows), "链接待复核": len(review_rows),
           "候选 CSV": str(args.candidates), "链接 CSV": str(args.out_links),
           "复核链接 CSV": str(args.review_links)})
    print("链接装账:python scripts/install_entity_links.py --database <db> "
          f"--input {args.out_links.name} --apply --backup <备份路径>")
    return 0


if __name__ == "__main__":
    from peach.jobs import job_main
    raise SystemExit(job_main(build_parser, run))