Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

10.3 KB
Raw Blame
"""从厂牌官网找出其社交账号,穿过 18+ 年龄门。

只采信**厂牌自有域名页面上**的社交链接。维基外链里混着引用来源的新闻站,
直接抓会把新闻站自己的账号当成厂牌的——实测 `妄想族` 条目就抓出了
`@news_postseven` 与 `@taishurxjp`。

AV 厂牌官网普遍先给一个年龄确认页:肯定链接写「はい(入室する)」指向站内,
否定链接指向 dmm.com。不穿过它只能拿到约 10 KB 的空壳页。
"""
from __future__ import annotations

import argparse
import hashlib
import re
import sqlite3
import sys
import time
from pathlib import Path
from urllib.parse import urljoin, urlsplit

from bs4 import BeautifulSoup

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))

from peach.http import HttpRequest, HttpxTransport, body_text   # noqa: E402
from peach.review_csv import read_rows, write_rows
from peach.scripting import USER_AGENT
from peach.social_links import NON_HANDLES, handle

SOCIAL = re.compile(r'https?://(?:www\.|mobile\.)?(?:twitter\.com|x\.com)/([A-Za-z0-9_]{1,15})/?(?:[?#].*)?$', re.I)
AFFIRMATIVE = re.compile(r"はい|入室|入場|入る|同意|年齢認証|(?<![a-z])(?:ENTER|YES)(?![a-z])"
                         r"|18\s*歳以上|over\s*18", re.I)
NEGATIVE = re.compile(r"いいえ|退出|戻る|(?<![a-z])NO\b|18\s*歳未満", re.I)
MAX_HOPS = 2
SCAN_FIELDS = ("studio", "site", "final_url", "handles", "note")
REVIEW_FIELDS = ("entity_id", "kind", "name", "link_kind", "label", "url", "evidence", "context", "review")


def affirmative_link(html: str, base: str) -> str | None:
    """返回年龄确认页的「进入」链接;不是年龄门就返回 None。

    判据是锚文本而不是 URL:否定链接通常指向站外(dmm.com),
    肯定链接指向站内,两者的 href 本身看不出区别。
    """
    host = urlsplit(base).hostname or ""
    for anchor in BeautifulSoup(html, "html.parser").select("a[href]"):
        text = anchor.get_text(" ", strip=True) + " ".join(
            str(img.get("alt", "")) for img in anchor.select("img"))
        text = re.sub(r"は\s+い", "はい", text)
        if not AFFIRMATIVE.search(text) or NEGATIVE.search(text):
            continue
        target = urljoin(base, str(anchor["href"]))
        if urlsplit(target).scheme in {"http", "https"} and (urlsplit(target).hostname or "") == host:
            return target
    return None


def handles_in(html: str) -> set[str]:
    return {item["handle"] for item in accounts_in(html)}


def accounts_in(html: str) -> list[dict[str, str]]:
    """提取账号主页锚点及原文;帖子、分享组件与脚本内容不算账号入口。"""
    found = {}
    for anchor in BeautifulSoup(html, "html.parser").select("a[href]"):
        url = str(anchor.get("href", "")).strip()
        if url.startswith("//"):
            url = "https:" + url
        match = SOCIAL.fullmatch(url)
        if not match or match[1].casefold() in NON_HANDLES:
            continue
        shown = match[1]
        text = anchor.get_text(" ", strip=True)
        alt = " ".join(str(img.get("alt", "")) for img in anchor.select("img"))
        item = {"handle": shown, "anchor": (text or alt).strip(),
                "context": anchor.parent.get_text(" ", strip=True)[:600]}
        previous = found.get(shown.casefold())
        if previous is None or (not previous["anchor"] and item["anchor"]):
            found[shown.casefold()] = item
    return list(found.values())


def scan(http, site: str, timeout: float, *, evidence: list | None = None) -> tuple[set[str], str, str]:
    """返回(找到的 handle, 最终 URL, 说明)。"""
    current, note = site, ""
    found: set[str] = set()
    visited = set()
    host = (urlsplit(site).hostname or "").removeprefix("www.")
    for hop in range(MAX_HOPS + 1):
        visited.add(current)
        response = http(HttpRequest("GET", current, {"User-Agent": USER_AGENT}), timeout, 8 << 20)
        current = response.url or current
        if response.status != 200:
            return found, current, f"未取得:HTTP {response.status}"
        if (urlsplit(current).hostname or "").removeprefix("www.") != host:
            return found, current, "未取得:重定向离开官网域名"
        html = body_text(response.body, response.headers)
        accounts = accounts_in(html)
        found.update(item["handle"] for item in accounts)
        if evidence is not None:
            for item in accounts:
                evidence.append({**item, "page": current,
                                 "sha256": hashlib.sha256(response.body).hexdigest()})
        following = affirmative_link(html, current)
        if following is None or following in visited or hop == MAX_HOPS:
            return found, current, note or ("官网页面直接给出社交链接" if found else "页面无社交链接")
        note = f"穿过 {hop + 1} 层年龄门"
        current = following
    return found, current, note


def ledger_sites(connection: sqlite3.Connection) -> list[dict]:
    """读取厂牌官网及已收录的 X 账号;有社媒的厂牌同样核查多账号遗漏。"""
    rows = []
    for eid, name, site in connection.execute(
            "SELECT e.id,e.canonical_name,l.url FROM entity e JOIN entity_link l ON l.entity_id=e.id "
            "WHERE e.kind='studio' AND l.link_kind='official' ORDER BY e.id,l.id"):
        known = {handle(url) for (url,) in connection.execute(
            "SELECT url FROM entity_link WHERE entity_id=? AND link_kind='social'", (eid,))
                 if (urlsplit(url).hostname or "").removeprefix("www.") in {"x.com", "twitter.com"}}
        rows.append({"entity_id": eid, "studio": name, "site": site, "known": known})
    return rows


def review_rows(row: dict, evidence: list[dict]) -> list[dict]:
    """生成安装器可读的待复核表;归属须结合锚文本确认,不能整页自动采信。"""
    known = set(row.get("known", ()))
    result = []
    for item in evidence:
        if item["handle"].casefold() in known:
            continue
        known.add(item["handle"].casefold())
        result.append({"entity_id": row.get("entity_id", ""), "kind": "studio",
                       "name": row["studio"], "link_kind": "social",
                       "label": f"X @{item['handle']}", "url": f"https://x.com/{item['handle']}",
                       "evidence": f"{item['page']} ; sha256={item['sha256']} ; {item['anchor']}",
                       "context": item["context"], "review": "待复核账号归属"})
    return result


def parse_args():
    """校验采集输入及进度文件,避免覆盖来源。"""
    parser = argparse.ArgumentParser()
    source = parser.add_mutually_exclusive_group(required=True)
    source.add_argument("--input", type=Path, help="含 studio,site 两列的 CSV")
    source.add_argument("--db", type=Path, help="只读账本里的厂牌官网")
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("--review-output", type=Path, help="待复核链接表,确认归属后交 install_entity_links.py")
    parser.add_argument("--resume", action="store_true", help="复用有账号证据的行;空结果与失败重新检查")
    parser.add_argument("--interval", type=float, default=1.5)
    parser.add_argument("--timeout", type=float, default=25.0)
    args = parser.parse_args()
    source_path = (args.db or args.input).resolve()
    if any(path and path.resolve() == source_path for path in (args.output, args.review_output)):
        parser.error("输出文件不能覆盖输入文件或账本")
    if args.review_output and args.output.resolve() == args.review_output.resolve():
        parser.error("扫描表与复核表必须使用不同路径")
    if args.resume and args.output.exists() and args.review_output and not args.review_output.exists():
        parser.error("续跑缺少原复核表;去掉 --resume 可重新采集")
    return args


def input_rows(args) -> list[dict]:
    """从只读账本或官网清单取得本轮范围。"""
    if args.db:
        with sqlite3.connect(args.db.resolve().as_uri() + "?mode=ro", uri=True) as connection:
            return ledger_sites(connection)
    return [row for row in read_rows(args.input) if (row.get("site") or "").strip()]


def main() -> int:
    args = parse_args()
    rows = input_rows(args)
    results = read_rows(args.output) if args.resume and args.output.exists() else []
    reviews = read_rows(args.review_output) if args.resume and args.review_output and args.review_output.exists() else []
    settled = {(r["studio"], r["site"]) for r in results
               if r["handles"] and not r["note"].startswith("未取得")}
    http = HttpxTransport()
    last = 0.0
    try:
        for row in rows:
            studio, site = (row.get("studio") or "").strip(), (row["site"]).strip()
            if (studio, site) in settled:
                continue
            wait = args.interval - (time.monotonic() - last)
            if wait > 0:
                time.sleep(wait)
            last = time.monotonic()
            evidence: list[dict] = []
            try:
                found, final, note = scan(http, site, args.timeout, evidence=evidence)
            except Exception as exc:
                found, final, note = set(), "", f"未取得:{type(exc).__name__}"
            results = [r for r in results if (r["studio"], r["site"]) != (studio, site)]
            results.append({"studio": studio, "site": site, "final_url": final,
                            "handles": "|".join(sorted(found)), "note": note})
            seen = {(str(r["entity_id"]), r["name"], r["url"].casefold()) for r in reviews}
            reviews.extend(r for r in review_rows(row, evidence)
                           if (str(r["entity_id"]), r["name"], r["url"].casefold()) not in seen)
            write_rows(args.output, SCAN_FIELDS, results)
            if args.review_output:
                write_rows(args.review_output, REVIEW_FIELDS, reviews)
            print(f"{studio:<20} {'|'.join(sorted(found)) or '未取得':<28} {note}")
    finally:
        http.close()

    write_rows(args.output, SCAN_FIELDS, results)
    if args.review_output:
        write_rows(args.review_output, REVIEW_FIELDS, reviews)
    print({"total": len(results), "with_handles": sum(1 for r in results if r["handles"]),
           "output": str(args.output)})
    return 0


if __name__ == "__main__":
    raise SystemExit(main())