Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

11.3 KB
Raw Blame
#!/usr/bin/env python3
"""按创作者跨作品抽样并生成 3×3 风格板。"""
from __future__ import annotations

import argparse
import collections
import concurrent.futures as futures
import hashlib
import random
import re
import subprocess
import threading
import time
import sys
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parent.parent
SRC_DIR = PROJECT_ROOT / "src"
if str(SRC_DIR) not in sys.path:
    sys.path.insert(0, str(SRC_DIR))

from peach.config import DATABASE_PATH, FFMPEG_DIR, GENERATED_DIR, STATE_DIR
from peach.scripting import open_readonly
from peach.classification import is_probable_mainstream_release, is_structural_creator
from peach.ffmpeg import FFmpegResolver
from peach.jobs import (
    ACTIVE_ASSET_SQL,
    DiskGuard,
    DiskSpaceDenied,
    JobPolicyError,
    SourceAccessPolicy,
    require_free_space,
    job_main,
)
from peach.platform import system_volume
from peach.media import normalized_path


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="生成创作者跨作品视觉风格板")
    parser.add_argument("--top", type=int, default=40)
    parser.add_argument("--per", type=int, default=9)
    parser.add_argument("--workers", type=int, default=16)
    parser.add_argument("--from-video", action="store_true")
    parser.add_argument("--location")
    parser.add_argument("--allow-metered", action="store_true")
    parser.add_argument("--min-free", type=float, default=40.0)
    parser.add_argument(
        "--disk-check-secs",
        type=float,
        default=20.0,
        help="运行期复查磁盘余量的间隔;0 表示每条都查",
    )
    parser.add_argument("--db", type=Path, default=DATABASE_PATH)
    parser.add_argument("--output-dir", type=Path, default=GENERATED_DIR / "boards")
    parser.add_argument("--lock", type=Path, default=STATE_DIR / ".creator-boards.lock")
    return parser


def safe_name(name: str) -> str:
    return re.sub(r"[^\w\u4e00-\u9fff-]", "_", name)[:40]


def cell_height(ffprobe: str, path: Path) -> int | None:
    result = subprocess.run(
        [
            ffprobe, "-v", "error", "-select_streams", "v",
            "-show_entries", "stream=width,height", "-of", "csv=p=0", str(path),
        ],
        capture_output=True,
        text=True,
        encoding="utf-8",
        errors="replace",
    )
    try:
        width, height = (int(value) for value in result.stdout.strip().split(","))
    except ValueError:
        return None
    if width != 1440:
        return None
    return height if height < 1000 else height // 3


def grab_frame(ffmpeg: str, cache: Path, job: tuple[str, float]) -> Path | None:
    path, duration = job
    destination = cache / (
        hashlib.sha1(path.encode("utf-8", "ignore")).hexdigest()[:16] + ".jpg"
    )
    if destination.is_file() and destination.stat().st_size > 0:
        return destination
    seek = int((duration or 60) * 0.3)
    try:
        subprocess.run(
            [
                ffmpeg, "-y", "-v", "error", "-ss", str(seek), "-i", path,
                "-frames:v", "1", "-vf", "scale=480:-1", str(destination),
            ],
            capture_output=True,
            timeout=240,
        )
    except subprocess.TimeoutExpired:
        return None
    return destination if destination.is_file() and destination.stat().st_size > 0 else None


def build_board(ffmpeg: str, ffprobe: str, paths: list[Path], destination: Path) -> bool:
    inputs: list[str] = []
    filters: list[str] = []
    labels: list[str] = []
    for source in paths[:9]:
        height = cell_height(ffprobe, source)
        index = len(labels)
        inputs.extend(("-i", str(source)))
        crop = f"crop=480:{height}:0:0," if height else ""
        filters.append(
            f"[{index}:v]{crop}scale=360:360:force_original_aspect_ratio=decrease,"
            f"pad=360:360:(ow-iw)/2:(oh-ih)/2:color=black[v{index}]"
        )
        labels.append(f"[v{index}]")
    if len(labels) < 4:
        return False
    while len(labels) < 9:
        filters.append(f"color=c=black:s=360x360:d=1[v{len(labels)}]")
        labels.append(f"[v{len(labels)}]")
    layout = "|".join(f"{column*360}_{row*360}" for row in range(3) for column in range(3))
    chain = ";".join(filters) + ";" + "".join(labels) + f"xstack=inputs=9:layout={layout}[out]"
    result = subprocess.run(
        [
            ffmpeg, "-y", "-v", "error", *inputs, "-filter_complex", chain,
            "-map", "[out]", "-frames:v", "1", "-q:v", "4", str(destination),
        ],
        capture_output=True,
        text=True,
        encoding="utf-8",
        errors="replace",
    )
    if result.returncode != 0:
        print("  ffmpeg:", result.stderr.strip()[:160])
        return False
    return destination.is_file()


def _existing_video_boards(output_dir: Path) -> set[str]:
    names = set()
    for path in output_dir.glob("v*_*.jpg"):
        match = re.match(r"^v\d+_(.+)_\d+\.jpg$", path.name)
        if match:
            names.add(match.group(1))
    return names


def build_from_videos(args: argparse.Namespace, ffmpeg: str, ffprobe: str) -> int:
    require_free_space(system_volume(), args.min_free)
    source_sql, source_parameters = SourceAccessPolicy().sql_filter(
        args.location, args.allow_metered
    )
    frame_cache = args.output_dir / "_frames"
    frame_cache.mkdir(parents=True, exist_ok=True)
    connection = open_readonly(args.db)
    rows = connection.execute(
        "SELECT creator,path,name,duration FROM asset WHERE medium='video' "
        "AND creator IS NOT NULL AND creator<>'' "
        f"AND {ACTIVE_ASSET_SQL} "
        "AND id NOT IN (SELECT asset_id FROM asset_tag)" + source_sql,
        source_parameters,
    ).fetchall()
    connection.close()

    by_creator: dict[str, list[tuple[str, float]]] = collections.defaultdict(list)
    for creator, path, name, duration in rows:
        if is_structural_creator(creator) or is_probable_mainstream_release(name, path):
            continue
        by_creator[creator].append((path, duration or 60))
    ranked = sorted(by_creator.items(), key=lambda item: -len(item[1]))[:args.top]
    existing = _existing_video_boards(args.output_dir)
    ranked = [(creator, items) for creator, items in ranked if safe_name(creator) not in existing]
    print(f"待出板 {len(ranked)} 位;并发 {args.workers}")

    jobs: list[tuple[str, float]] = []
    for creator, items in ranked:
        random.Random(creator).shuffle(items)
        jobs.extend(items[:args.per])
    captured: dict[tuple[str, float], Path] = {}
    started = time.time()
    # 抽帧触发的云盘块缓存落在系统盘,起跑线检查拦不住;边跑边看,触线让未开始的帧直接跳过。
    guard = DiskGuard(system_volume(), args.min_free, args.disk_check_secs)
    stop = threading.Event()
    stopped_reason = ""

    def grab(job: tuple[str, float]) -> Path | None:
        if stop.is_set():
            return None
        return grab_frame(ffmpeg, frame_cache, job)

    with futures.ThreadPoolExecutor(args.workers) as pool:
        outputs = pool.map(grab, jobs)
        for index, (job, output) in enumerate(zip(jobs, outputs), 1):
            if output:
                captured[job] = output
            if index % 25 == 0:
                rate = index / max(time.time() - started, 1) * 60
                print(f"  {index}/{len(jobs)} 帧 成功 {len(captured)} {rate:.1f} 帧/分", flush=True)
            if not stop.is_set():
                try:
                    guard.check()
                except JobPolicyError as exc:
                    stopped_reason = str(exc)
                    print(f"  [stop] {exc}", flush=True)
                    stop.set()

    if stopped_reason:
        # 已抽到的帧留在缓存里,续跑时不会重复下载;但板子不出,避免用残缺样本代表创作者。
        print(f"磁盘闸门中止:{stopped_reason}")
        print(f"已缓存 {len(captured)} 帧,未出板;恢复磁盘后重跑即可续上。")
        return DiskSpaceDenied.exit_code

    made = 0
    for rank, (creator, items) in enumerate(ranked, 1):
        frames = [captured[job] for job in items[:args.per] if job in captured]
        if len(frames) < 4:
            print(f"  {rank:>2}. {creator[:26]:<26} 可用帧仅 {len(frames)},跳过")
            continue
        destination = args.output_dir / f"v{rank:02d}_{safe_name(creator)}_{len(items)}.jpg"
        if build_board(ffmpeg, ffprobe, frames, destination):
            made += 1
            print(f"  {rank:>2}. {creator[:26]:<26} 视频{len(items):>4} 帧{len(frames)} → {destination.name}")
    print(f"生成 {made} 张 → {args.output_dir}")
    return 0


def build_from_snapshots(args: argparse.Namespace, ffmpeg: str, ffprobe: str) -> int:
    connection = open_readonly(args.db)
    rows = connection.execute(
        "SELECT creator,name,path,snapshot_path FROM asset WHERE medium='video' "
        "AND creator IS NOT NULL AND creator<>'' AND snapshot_path IS NOT NULL "
        f"AND {ACTIVE_ASSET_SQL} "
        "AND id NOT IN (SELECT asset_id FROM asset_tag)"
    ).fetchall()
    count_rows = connection.execute(
            "SELECT creator,name,path FROM asset WHERE medium='video' AND creator IS NOT NULL "
            f"AND creator<>'' AND {ACTIVE_ASSET_SQL} "
            "AND id NOT IN (SELECT asset_id FROM asset_tag)"
        ).fetchall()
    counts = collections.Counter(
        creator for creator, name, path in count_rows
        if not is_structural_creator(creator)
        and not is_probable_mainstream_release(name, path)
    )
    connection.close()
    by_creator: dict[str, list[Path]] = collections.defaultdict(list)
    for creator, name, asset_path, snapshot in rows:
        if is_structural_creator(creator) or is_probable_mainstream_release(name, asset_path):
            continue
        path = normalized_path(snapshot)
        if path.is_file():
            by_creator[creator].append(path)
    ranked = [(creator, count) for creator, count in counts.most_common() if len(by_creator.get(creator, [])) >= 4][:args.top]
    print(f"可出板创作者 {len(ranked)} 位,覆盖无标签视频 {sum(count for _, count in ranked)} 条")
    made = 0
    for rank, (creator, count) in enumerate(ranked, 1):
        candidates = by_creator[creator]
        random.Random(creator).shuffle(candidates)
        destination = args.output_dir / f"{rank:02d}_{safe_name(creator)}_{count}.jpg"
        if build_board(ffmpeg, ffprobe, candidates[:args.per], destination):
            made += 1
            print(f"  {rank:>2}. {creator[:26]:<26} 视频{count:>4} 取样{min(len(candidates), args.per)} → {destination.name}")
    print(f"生成 {made} 张 → {args.output_dir}")
    return 0


def run(args: argparse.Namespace) -> int:
    if args.top < 1 or args.per < 4 or args.workers < 1:
        raise SystemExit("top/workers 必须大于 0,per 至少为 4")
    resolver = FFmpegResolver(FFMPEG_DIR)
    ffmpeg = resolver.ffmpeg()
    ffprobe = resolver.ffprobe()
    if ffmpeg is None or ffprobe is None:
        raise RuntimeError("ffmpeg/ffprobe unavailable")
    args.output_dir.mkdir(parents=True, exist_ok=True)
    if args.from_video:
        return build_from_videos(args, str(ffmpeg.path), str(ffprobe.path))
    return build_from_snapshots(args, str(ffmpeg.path), str(ffprobe.path))


def main(argv: list[str] | None = None) -> int:
    return job_main(build_parser, run, argv)


if __name__ == "__main__":
    raise SystemExit(main())