Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

13.1 KB
Raw Blame
#!/usr/bin/env python3
"""全库关键帧九宫格:可续跑、显式计费授权、产物落地后登记。"""
from __future__ import annotations

import argparse
import hashlib
import queue
import shutil
import sqlite3
import tempfile
import threading
import time
import sys
from collections import Counter
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parent.parent
SRC_DIR = PROJECT_ROOT / "src"
if str(SRC_DIR) not in sys.path:
    sys.path.insert(0, str(SRC_DIR))

from peach import frame_capture, junk
from peach.config import DATABASE_PATH, FFMPEG_DIR, GENERATED_DIR, LOG_DIR, STATE_DIR
from peach.ffmpeg import FFmpegResolver
from peach.jobs import (
    ACTIVE_ASSET_SQL,
    DiskGuard,
    DiskSpaceDenied,
    JobPolicyError,
    SourceAccessPolicy,
    require_free_space,
    job_main,
)
from peach.platform import system_volume
from peach.media import resolve_case_insensitive
from peach.media_processing_policy import metadata_only
from peach.strm import background_rows, skip_background_read


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="生成视频关键帧九宫格")
    parser.add_argument("--workers", type=int, default=4)
    parser.add_argument("--location")
    parser.add_argument("--frames", type=int, default=9)
    parser.add_argument("--limit", type=int, default=0)
    parser.add_argument("--min-free", type=float, default=40.0)
    parser.add_argument(
        "--disk-check-secs",
        type=float,
        default=20.0,
        help="运行期复查磁盘余量的间隔;0 表示每条都查",
    )
    parser.add_argument(
        "--asset",
        type=int,
        action="append",
        help="按 id 只抽指定资产,绕过来源与批量筛选。用于修复个别失败条目,"
             "避免为一条重跑整个来源的待抽队列",
    )
    parser.add_argument("--allow-metered", action="store_true")
    parser.add_argument("--db", type=Path, default=DATABASE_PATH)
    parser.add_argument("--output-root", type=Path, default=GENERATED_DIR / "snapshots" / "cloud")
    parser.add_argument("--log-dir", type=Path, default=LOG_DIR)
    parser.add_argument("--lock", type=Path, default=STATE_DIR / ".sheets.lock")
    return parser


def output_path(output_root: Path, location: str, path: str) -> Path:
    digest = hashlib.sha1(path.encode("utf-8", "ignore")).hexdigest()[:16]
    directory = output_root / location / digest[:2]
    directory.mkdir(parents=True, exist_ok=True)
    return directory / f"{digest}.jpg"


COLOR_OVERRIDE = frame_capture.COLOR_OVERRIDE
COLOR_METADATA_ERROR = frame_capture.COLOR_METADATA_ERROR


def _capture_frame(ffmpeg: str, path: str, timestamp: float, destination: Path,
                   color_override: bool) -> tuple[bool, str]:
    """抽一帧,返回(是否成功, stderr)。stderr 用于判断值不值得重试。

    命令本身在 `peach.frame_capture`,时间轴预览走的是同一条。色彩元数据那条重试判据
    只写一处:分成两份之后,坏元数据的那批片子会只在其中一份里被认出来,而失败形态是
    「这些片子没有图」,看不出是哪一份少了一条。
    """
    return frame_capture.capture_frame(ffmpeg, path, timestamp, destination,
                                       width=480, color_override=color_override)


def make_sheet(ffmpeg: str, path: str, duration: float, destination: Path,
               frames: int) -> tuple[bool, str]:
    """输入端 seek 抽帧后调用 FFmpeg tile;不线性解码整片。

    失败时返回可区分的原因,不能只返回「失败」:asset 12510 与 18349 都失败了一整天,
    一个是片源头损坏、一个是账本时长记错,光看计数分不出来,也就没法决定该修哪一边。
    """
    if skip_background_read(path):
        return False, "unsupported_source"
    if not duration or duration < 2:
        return False, "no_duration"
    temporary = Path(tempfile.mkdtemp(prefix="sheet_"))
    try:
        captured: list[Path] = []
        for index in range(frames):
            if metadata_only():
                return False, "previews_disabled"
            timestamp = duration * (0.03 + 0.94 * (index + 0.5) / frames)
            frame = temporary / f"{index:02d}.jpg"
            ok, stderr = _capture_frame(ffmpeg, path, timestamp, frame, color_override=False)
            if not ok and COLOR_METADATA_ERROR.search(stderr):
                ok, _ = _capture_frame(ffmpeg, path, timestamp, frame, color_override=True)
            if ok:
                captured.append(frame)
        if len(captured) < 2:
            # 一帧都解不出来是片源问题;解得出一部分说明片源可用,是账本时长比真实文件长,
            # 采样点落到了文件末尾之后。两者要修的地方不同。
            return False, "broken_source" if not captured else "duration_mismatch"
        for index, frame in enumerate(captured):
            target = temporary / f"s{index:02d}.jpg"
            if frame != target:
                frame.replace(target)
        rows = (len(captured) + 2) // 3
        if frame_capture.tile_frames(ffmpeg, str(temporary / "s%02d.jpg"), 3, rows,
                                     destination):
            return True, ""
        return False, "tile_failed"
    finally:
        shutil.rmtree(temporary, ignore_errors=True)


def _require_previews_enabled() -> None:
    if metadata_only():
        raise JobPolicyError("视频预览生成已关闭,请先在媒体库设置中开启")


def _preview_active(stop, reasons) -> bool:
    if stop.is_set():
        return False
    if metadata_only():
        reasons.append("视频预览生成已关闭")
        stop.set()
        return False
    return True


def run(args: argparse.Namespace) -> int:
    _require_previews_enabled()
    if args.workers < 1 or args.frames < 2 or args.limit < 0:
        raise SystemExit("workers 必须大于 0,frames 至少为 2,limit 不能为负数")
    resolver = FFmpegResolver(FFMPEG_DIR)
    ffmpeg = resolver.ffmpeg()
    if ffmpeg is None:
        raise RuntimeError("ffmpeg unavailable")
    require_free_space(system_volume(), args.min_free)
    source_sql, source_parameters = SourceAccessPolicy().sql_filter(
        args.location, args.allow_metered
    )
    args.output_root.mkdir(parents=True, exist_ok=True)
    args.log_dir.mkdir(parents=True, exist_ok=True)
    log_path = args.log_dir / f"sheets-{time.strftime('%Y%m%d-%H%M%S')}.log"
    lock = threading.RLock()

    with log_path.open("w", encoding="utf-8", buffering=1) as log_file:
        def log(message: str) -> None:
            with lock:
                line = f"[{time.strftime('%H:%M:%S')}] {message}"
                print(line, flush=True)
                log_file.write(line + "\n")

        if args.allow_metered:
            log("已显式允许计费来源")
        connection = sqlite3.connect(args.db)
        if args.asset:
            # 点名重抽时不套 snapshot_path IS NULL:修完时长要重抽的那条,可能已经带着
            # 上一次的错误结果。计费来源和 online 的边界照旧生效。
            placeholders = ",".join("?" for _ in args.asset)
            sql = (
                "SELECT id,location,path,duration FROM asset WHERE medium='video' "
                f"AND id IN ({placeholders}) AND location != 'online' AND duration > 2"
                + source_sql + " ORDER BY size DESC"
            )
            parameters: tuple[object, ...] = tuple(args.asset) + source_parameters
        else:
            sql = (
                "SELECT id,location,path,duration FROM asset WHERE medium='video' "
                "AND snapshot_path IS NULL AND location != 'online' AND duration > 2"
                f" AND {ACTIVE_ASSET_SQL}"
                + source_sql + " ORDER BY size DESC"
            )
            parameters = source_parameters
        if args.limit:
            sql += " LIMIT ?"
            parameters += (args.limit,)
        tasks = background_rows(connection.execute(sql, parameters).fetchall(), 2)
        connection.close()
        checked = junk.isolate(args.db, asset_ids=[row[0] for row in tasks])
        tasks = [row for row in tasks if row[0] not in checked['ids']]
        log(f"垃圾检查:隔离 {checked['isolated']} 个,待复核 {checked['review']} 个")
        total = len(tasks)
        log(f"待处理 {total:,} 个视频 workers={args.workers} frames={args.frames} 日志={log_path}")
        if not total:
            log("没有待处理项;可能需要先运行 probe")
            return 0

        pending: queue.Queue = queue.Queue()
        results: queue.Queue = queue.Queue()
        for task in tasks:
            pending.put(task)
        counters = {"done": 0, "failed": 0, "existing": 0}
        failure_reasons: Counter[str] = Counter()
        started = time.time()
        # 起跑线检查拦不住运行期把盘吃光的第三方缓存;这里边跑边看。
        guard = DiskGuard(system_volume(), args.min_free, args.disk_check_secs)
        stop = threading.Event()
        stop_reason: list[str] = []

        def worker() -> None:
            while _preview_active(stop, stop_reason):
                try:
                    asset_id, location, path, duration = pending.get_nowait()
                except queue.Empty:
                    return
                destination = output_path(args.output_root, location, path)
                try:
                    if (not args.asset and destination.is_file()
                            and destination.stat().st_size > 4096):
                        results.put((str(destination), asset_id))
                        with lock:
                            counters["existing"] += 1
                    else:
                        ok, reason = make_sheet(
                            str(ffmpeg.path), resolve_case_insensitive(path), duration,
                            destination, args.frames,
                        )
                        if ok:
                            results.put((str(destination), asset_id))
                        else:
                            with lock:
                                counters["failed"] += 1
                                failure_reasons[reason] += 1
                            log(f"[fail] asset {asset_id} {reason} {path}")
                except Exception as exc:
                    with lock:
                        counters["failed"] += 1
                        failure_reasons["exception"] += 1
                    log(f"[fail] asset {asset_id} exception {type(exc).__name__} {path}")
                finally:
                    with lock:
                        counters["done"] += 1
                        if counters["done"] % 100 == 0:
                            elapsed = time.time() - started
                            rate = counters["done"] / elapsed if elapsed else 0
                            eta = (total - counters["done"]) / rate / 3600 if rate else 0
                            log(f"{counters['done']:,}/{total:,} 失败 {counters['failed']} 已存在 {counters['existing']} 剩余 {eta:.1f} 小时")
                        try:
                            guard.check()
                        except JobPolicyError as exc:
                            if not stop.is_set():
                                stop_reason.append(str(exc))
                                log(f"[stop] {exc}")
                            stop.set()

        threads = [threading.Thread(target=worker, daemon=True) for _ in range(args.workers)]
        for thread in threads:
            thread.start()

        connection = sqlite3.connect(args.db, timeout=60)
        buffer = []
        while any(thread.is_alive() for thread in threads) or not results.empty():
            # 等待要短:循环条件看到线程还活着、下一刻线程就退出的话,这一次 get 要等满
            # 超时才回来。等 2 秒的话每次收尾都白站 2 秒,一次一张的重拍尤其明显。
            try:
                buffer.append(results.get(timeout=0.25))
            except queue.Empty:
                pass
            if len(buffer) >= 50:
                connection.executemany("UPDATE asset SET snapshot_path=? WHERE id=?", buffer)
                connection.commit()
                buffer.clear()
        if buffer:
            connection.executemany("UPDATE asset SET snapshot_path=? WHERE id=?", buffer)
            connection.commit()
        elapsed = time.time() - started
        log(f"完成 {counters['done']:,},失败 {counters['failed']},已存在 {counters['existing']},耗时 {elapsed/3600:.2f} 小时")
        if failure_reasons:
            breakdown = ",".join(f"{reason} {count}" for reason, count in failure_reasons.most_common())
            log(f"失败原因:{breakdown}")
        connection.close()
        if stop_reason:
            # 已完成的部分都已入库;磁盘闸门中止不能报成正常完成,否则续跑决策会被误导。
            log(f"任务中止:{stop_reason[0]}")
            return DiskSpaceDenied.exit_code
    return 0


def main(argv: list[str] | None = None) -> int:
    return job_main(build_parser, run, argv)


if __name__ == "__main__":
    raise SystemExit(main())