Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

12.9 KB
Raw Blame
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
r"""给已入库的关注图片补上固有宽高。

图片墙靠 `<img width height>` 在图落地前占好比例;没有这两个数的卡片按 1:1
占位,图一到就整墙重排。只有图片视图摆成瀑布流,所以只补图片卡面那一张。尺寸只有
fanbox 一家一直在记(imageMap 自带),其余来源的存量行都是空的。这一趟分两路补:

- **归档**:rule34.xxx 的 dapi 响应本来就带 `width`/`height`,每次检查更新的
  原始响应都归档在 `sources/follow/rule34xxx/` 下。不用再发一个请求,把归档里
  每个帖子的尺寸按帖子号对回条目即可。
- **探测**:归档站(kemono、coomer、pawchive)、paheal、论坛附件这些来源不报
  尺寸,只能问文件本身——但只问文件头:`Range: bytes=0-65535`,PNG/GIF/WebP/
  AVIF 几十到几百字节就够,JPEG 的 SOF 段在 EXIF 之后,64 KiB 覆盖绝大多数。
  先问卡片实际显示的公开缩略图(归档站的原文件主机对脚本直接 403),再按界面
  同一条 `FollowMediaResolver` 拿媒体地址与请求头,凭据与 Referer 同一口径。
  每个主机各自限速。

判据只有 `peach.follow_image_dims` 一份;落库走 `FollowStore.set_image_dims`,
只补空缺,条数不增不减。默认 dry-run。`--apply` 必须同时给 `--backup`,与本仓库
其它真实写入脚本一致。可反复运行:已经有尺寸的条目直接跳过,中断后接着跑就是。
"""
from __future__ import annotations

import argparse
import json
import sqlite3
import sys
from pathlib import Path

import httpx

PROJECT_ROOT = Path(__file__).resolve().parent.parent
SRC_DIR = PROJECT_ROOT / "src"
if str(SRC_DIR) not in sys.path:
    sys.path.insert(0, str(SRC_DIR))

from peach.config import GENERATED_DIR, SECRETS_DIR, SHARED_CREDENTIAL_ROOT, SOURCES_DIR
from peach.follow_image_dims import ImageDimsUnavailable, positive_dims, probe_image_dims
from peach.follow_secrets import credential_store_for
from peach.follow_sources import display_thumb_url, f95_attachment_media_items
from peach.follow_store import FollowItemRow, FollowStore
from peach.follow_stream import FollowMediaResolver, FollowMediaUnavailable
from peach.http import HttpTransport, HttpxTransport
from peach.review_csv import write_rows
from peach.scripting import (
    BACKUP_REQUIRED, HostLimiter, add_ledger_write_args, open_for_write, open_readonly,
)
# 「这条是不是图片」与「卡面是清单里哪一张」都与界面同一判据:卡片按卡面那张图预留
# 比例,这里就按同一张去问尺寸。
from peach.web_follow import _media_items as visible_media_of, _media_kind as media_kind_of

FIELDS = ("item_id", "provider", "external_id", "media", "mode", "result",
          "width", "height", "note")

#: 同一主机两次探测的最小间隔。归档站对匿名请求限流很紧,一秒两次已经算快。
DEFAULT_DELAY = 0.5
#: 单次探测的超时。只取文件头,等得久多半是被挂起,不值得等。
PROBE_TIMEOUT = 15.0
#: 每写这么多条提交一次并落一次 CSV,中断也不白跑。
FLUSH_EVERY = 50


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="补齐关注图片的固有宽高")
    add_ledger_write_args(parser)
    parser.add_argument("--out", type=Path,
                        default=GENERATED_DIR / "follow-image-dims-backfill.csv")
    parser.add_argument("--archives", type=Path, default=SOURCES_DIR / "follow",
                        help="原始响应归档根目录(各来源一个子目录)")
    parser.add_argument("--mode", choices=("all", "archives", "probe"), default="all",
                        help="archives 只对归档、probe 只发请求,默认两路都走")
    parser.add_argument("--provider", action="append", default=[],
                        help="只处理这些来源,可重复给")
    parser.add_argument("--limit", type=int, default=0, help="只处理前 N 张,0 为全部")
    parser.add_argument("--delay", type=float, default=DEFAULT_DELAY)
    return parser


def archived_dims(root: Path) -> dict[str, tuple[int, int]]:
    """rule34.xxx 归档响应里每个帖子的宽高,按帖子号索引。

    同一帖子在多次响应里都出现时任取一份:尺寸是文件属性,不随时间变。认不出的
    文件(非 JSON、不是列表)跳过,不报错——归档目录里还有 `.json` 元数据。
    """
    dims: dict[str, tuple[int, int]] = {}
    directory = root / "rule34xxx"
    if not directory.is_dir():
        return dims
    for path in sorted(directory.rglob("*.raw")):
        try:
            posts = json.loads(path.read_bytes())
        except (OSError, ValueError):
            continue
        if not isinstance(posts, list):
            continue
        for post in posts:
            if not isinstance(post, dict) or post.get("id") is None:
                continue
            pair = positive_dims(post.get("width"), post.get("height"))
            if pair is not None:
                dims.setdefault(str(post["id"]), pair)
    return dims


def pending_targets(items: tuple[FollowItemRow, ...],
                    providers: set[str]) -> list[tuple[FollowItemRow, int | None, dict | None]]:
    """待补的(条目,媒体序号,媒体)。序号 None 是条目级那张直链图。

    瀑布流只按卡面那张占位,一个条目最多出一条:有媒体清单的条目问可见清单里的第一张
    图(界面卡面取的就是它,隐藏的不算),其余几张只在详情里翻,不需要比例;F95 的
    附件清单是读时合成的,不在 metadata 里,尺寸没有地方落,跳过。没有清单的条目
    只问图片,视频不进图片墙。
    """
    targets: list[tuple[FollowItemRow, int | None, dict | None]] = []
    for item in items:
        if providers and item.provider not in providers:
            continue
        media_items = item.metadata.get("media_items")
        if isinstance(media_items, list) and media_items:
            index = next((media["index"] for media in visible_media_of(item)
                          if media["media_kind"] == "image"), None)
            if index is not None and positive_dims(
                    media_items[index].get("width"), media_items[index].get("height")) is None:
                targets.append((item, index, media_items[index]))
            continue
        if item.provider == "f95zone" and f95_attachment_media_items(item.metadata):
            continue
        if media_kind_of(item) != "image":
            continue
        if positive_dims(item.metadata.get("width"), item.metadata.get("height")) is None:
            targets.append((item, None, None))
    return targets


def probe_attempts(resolver: FollowMediaResolver, item: FollowItemRow, index: int | None,
                   media: dict | None) -> tuple[list[tuple[str, dict, str]], list[str]]:
    """一张图按顺序可以去问的地址:(地址,请求头,来路),以及排不上的原因。

    先问卡片实际显示的那张——公开缩略图,不带请求头,浏览器也是这么取的,它的
    比例才是排版要的那一个;归档站的原文件主机对脚本请求直接 403(pawchive 回的
    是一句「stop before I block」),缩略图主机照常给。缩略图不行再按界面同一条
    解析器拿媒体地址与请求头(凭据、Referer 都在里面)。
    """
    attempts: list[tuple[str, dict, str]] = []
    notes: list[str] = []
    thumb = str((media or {}).get("thumb_url") or "") if media is not None \
        else str(display_thumb_url(item) or "")
    if thumb.startswith("https://"):
        attempts.append((thumb, {}, "缩略图"))
    try:
        resolved = resolver.resolve(item, index)
    except FollowMediaUnavailable as error:
        notes.append(f"解析器拒收:{error}")
    else:
        headers = dict(resolved.headers or {})
        if resolved.referer:
            headers.setdefault("Referer", resolved.referer)
        if resolved.url != thumb:
            attempts.append((resolved.url, headers, "媒体地址"))
    return attempts, notes


def probe_target(resolver: FollowMediaResolver, transport: HttpTransport,
                 limiter: HostLimiter, item: FollowItemRow, index: int | None,
                 media: dict | None) -> tuple[tuple[int, int] | None, str]:
    """问一张图的尺寸。返回(尺寸或 None,说明);每条地址各限速一次。"""
    attempts, notes = probe_attempts(resolver, item, index, media)
    for url, headers, via in attempts:
        limiter.wait(url)
        try:
            return probe_image_dims(transport, url, headers, timeout=PROBE_TIMEOUT), via
        except (ImageDimsUnavailable, OSError, httpx.HTTPError) as error:
            notes.append(f"{via}:{type(error).__name__}: {error}"[:120])
    return None, ";".join(notes) or "没有可问的地址"


def run(args: argparse.Namespace, transport: HttpTransport | None = None) -> int:
    if args.apply and not args.backup:
        print(BACKUP_REQUIRED, file=sys.stderr)
        return 2
    providers = set(args.provider)
    reader = open_readonly(args.db)
    try:
        items = FollowStore(lambda: reader).items(limit=10_000_000)
    finally:
        reader.close()
    targets = pending_targets(items, providers)
    if args.limit:
        targets = targets[:args.limit]
    print(f"关注条目 {len(items)} 条,待补尺寸 {len(targets)} 张", flush=True)
    if not targets:
        return 0

    archive = archived_dims(args.archives) if args.mode in ("all", "archives") else {}
    if args.mode in ("all", "archives"):
        print(f"归档里有尺寸的 rule34.xxx 帖子 {len(archive)} 个", flush=True)

    writer: sqlite3.Connection | None = None
    store: FollowStore | None = None
    before = after = 0
    if args.apply:
        writer = open_for_write(args)
        print(f"备份:{args.backup}", flush=True)
        before = writer.execute("SELECT COUNT(*) FROM follow_item").fetchone()[0]
        store = FollowStore(lambda: writer)

    resolver: FollowMediaResolver | None = None
    limiter = HostLimiter({}, default_interval=args.delay)
    if args.mode in ("all", "probe"):
        transport = transport or HttpxTransport()
        resolver = FollowMediaResolver(transport).with_credential_loader(
            lambda provider: credential_store_for(
                SECRETS_DIR, shared_root=SHARED_CREDENTIAL_ROOT).load(provider))

    log: list[dict] = []
    found = written = 0
    since_flush = 0

    def flush() -> None:
        nonlocal since_flush
        if writer is not None:
            writer.commit()
        args.out.parent.mkdir(parents=True, exist_ok=True)
        write_rows(args.out, FIELDS, log, atomic=True)
        since_flush = 0

    try:
        for position, (item, index, media) in enumerate(targets, 1):
            row = {"item_id": item.id, "provider": item.provider,
                   "external_id": item.external_id,
                   "media": "" if index is None else index,
                   "mode": "", "result": "未取得", "width": "", "height": "", "note": ""}
            dims: tuple[int, int] | None = None
            if index is None and item.provider == "rule34xxx" and archive:
                dims = archive.get(str(item.external_id))
                row["mode"] = "归档"
                if dims is None:
                    row["note"] = "归档里没有这个帖子"
            if dims is None and resolver is not None:
                dims, note = probe_target(resolver, transport, limiter, item, index, media)
                row["mode"] = "探测"
                row["note"] = note if dims is None else note
            if dims is not None:
                found += 1
                row["result"] = "取得"
                row["width"], row["height"] = dims
                if store is not None:
                    if store.set_image_dims(item.id, *dims, media_index=index):
                        written += 1
                        since_flush += 1
                    else:
                        row["note"] = "已有尺寸,未改动"
            log.append(row)
            if since_flush >= FLUSH_EVERY or position % 200 == 0:
                flush()
                print(f"{position}/{len(targets)}:已取得 {found},已写入 {written}", flush=True)
    finally:
        flush()

    print(f"取得 {found}/{len(targets)} 张 → {args.out}", flush=True)
    if writer is None:
        print(f"dry-run:可写入 {found} 张;加 --apply --backup 才真正写入")
        return 0
    after = writer.execute("SELECT COUNT(*) FROM follow_item").fetchone()[0]
    writer.close()
    # 条数必须不变:这一趟只补 metadata 里的两个数,不新增也不删除条目。
    print(f"已写入 {written} 张;follow_item {before} → {after}")
    return 0 if before == after else 1


def main(argv: list[str] | None = None) -> int:
    return run(build_parser().parse_args(argv))


if __name__ == "__main__":
    raise SystemExit(main())