Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

14.7 KB
Raw Blame
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
r"""只读抽取视频首尾帧并生成片尾出处/不完整版候选。

帧和 OCR sidecar 只写 generated/endcard-evidence;ledger、媒体文件、quality goal 与复核决定
都不写。默认必须点名 --asset 或给正数 --limit,避免无意启动全库云盘批处理。
"""
from __future__ import annotations

import argparse
import json
import os
import sqlite3
import subprocess
import time
import sys
from datetime import datetime
from typing import Callable
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parent.parent
SRC_DIR = PROJECT_ROOT / "src"
if str(SRC_DIR) not in sys.path:
    sys.path.insert(0, str(SRC_DIR))

from peach.config import DATABASE_PATH, FFMPEG_DIR, GENERATED_DIR, STATE_DIR
from peach.scripting import open_readonly
from peach.endcard import POLICY_VERSION, detect_endcard
from peach.ffmpeg import FFmpegResolver
from peach.jobs import (
    DiskGuard,
    JobPolicyError,
    SourceAccessPolicy,
    require_free_space,
    job_main,
)
from peach.media import resolve_case_insensitive
from peach.platform import system_volume
from peach.review_csv import write_rows


CANDIDATE_FIELDS = (
    "candidate_key", "asset_id", "name", "code", "location", "sample_kind",
    "timestamp_seconds", "frame_key", "ocr_text", "verdict", "detected_urls",
    "detected_handles", "confidence", "reason", "policy_version", "status",
)
HEALTH_FIELDS = (
    "source", "policy_version", "attempted", "frames_requested", "frame_cache_reused",
    "captured", "capture_errors", "ocr_cache_reused", "ocr_succeeded", "ocr_errors",
    "candidates", "incomplete_candidates", "source_evidence", "elapsed_ms",
    "last_error_kind", "last_error_message",
)


def build_parser() -> argparse.ArgumentParser:
    stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--db", type=Path, default=DATABASE_PATH)
    parser.add_argument("--asset", type=int, action="append")
    parser.add_argument("--location")
    parser.add_argument("--limit", type=int, default=0)
    parser.add_argument("--allow-metered", action="store_true")
    parser.add_argument("--head-offsets", default="0.5,2,5")
    parser.add_argument("--tail-offsets", default="0.5,2,5,10,15")
    parser.add_argument("--min-free", type=float, default=40.0)
    parser.add_argument(
        "--evidence-root", type=Path, default=GENERATED_DIR / "endcard-evidence",
    )
    parser.add_argument(
        "--out", type=Path,
        default=GENERATED_DIR / f"video-endcard-candidate-{stamp}.csv",
    )
    parser.add_argument(
        "--health", type=Path,
        default=GENERATED_DIR / f"video-endcard-health-{stamp}.csv",
    )
    parser.add_argument(
        "--ocr-script", type=Path,
        default=Path(__file__).resolve().with_name("windows_ocr.ps1"),
    )
    parser.add_argument("--lock", type=Path, default=STATE_DIR / ".endcard-audit.lock")
    return parser


def parse_offsets(raw: str) -> tuple[float, ...]:
    try:
        values = tuple(dict.fromkeys(float(value.strip()) for value in raw.split(",")))
    except ValueError as error:
        raise ValueError("抽帧偏移必须是逗号分隔的秒数") from error
    if not values or any(value <= 0 for value in values):
        raise ValueError("抽帧偏移必须全部大于 0")
    return values


def sample_points(
    duration: float, head_offsets: tuple[float, ...], tail_offsets: tuple[float, ...],
) -> list[tuple[str, float]]:
    points: list[tuple[str, float]] = []
    seen: set[int] = set()
    for kind, offsets in (("head", head_offsets), ("tail", tail_offsets)):
        for offset in offsets:
            timestamp = offset if kind == "head" else duration - offset
            millis = round(timestamp * 1000)
            if timestamp <= 0 or timestamp >= duration or millis in seen:
                continue
            seen.add(millis)
            points.append((kind, timestamp))
    return points


def load_assets(args: argparse.Namespace) -> list[dict]:
    if not args.asset and args.limit <= 0:
        raise ValueError("必须点名 --asset,或给正数 --limit 才会启动")
    source_sql, source_parameters = SourceAccessPolicy().sql_filter(
        args.location, args.allow_metered,
    )
    sql = (
        "SELECT id,location,path,name,code,duration,size FROM asset "
        "WHERE medium='video' AND location<>'online' AND duration>2"
        + source_sql
    )
    parameters: tuple[object, ...] = source_parameters
    if args.asset:
        marks = ",".join("?" for _ in args.asset)
        sql += f" AND id IN ({marks})"
        parameters += tuple(args.asset)
    sql += " ORDER BY size DESC,id"
    if args.limit > 0:
        sql += " LIMIT ?"
        parameters += (args.limit,)
    connection = open_readonly(args.db)
    try:
        return [dict(row) for row in connection.execute(sql, parameters)]
    finally:
        connection.close()


def frame_path(root: Path, asset_id: int, kind: str, timestamp: float) -> Path:
    return root / str(asset_id) / f"{kind}-{round(timestamp * 1000):09d}.png"


def capture_frame(ffmpeg: str, source: str, timestamp: float, destination: Path) -> bool:
    destination.parent.mkdir(parents=True, exist_ok=True)
    temporary = destination.with_name(f".{destination.name}.{os.getpid()}.tmp.png")
    command = [
        ffmpeg, "-y", "-v", "error", "-rw_timeout", "8000000",
        "-ss", f"{timestamp:.3f}", "-i", source, "-frames:v", "1",
        "-vf", "scale=1280:-1", str(temporary),
    ]
    try:
        completed = subprocess.run(command, capture_output=True, timeout=45)
        if completed.returncode != 0 or not temporary.is_file() or temporary.stat().st_size <= 1024:
            return False
        os.replace(temporary, destination)
        return True
    except (OSError, subprocess.TimeoutExpired):
        return False
    finally:
        temporary.unlink(missing_ok=True)


class WindowsOcrProvider:
    def __init__(self, script: Path):
        self.script = script
        self.powershell = Path(os.environ.get("SystemRoot", r"C:\Windows")) / (
            r"System32\WindowsPowerShell\v1.0\powershell.exe"
        )

    def recognize(self, paths: list[Path]) -> dict[str, dict]:
        if os.name != "nt" or not self.powershell.is_file() or not self.script.is_file():
            raise RuntimeError("Windows OCR provider unavailable")
        results: dict[str, dict] = {}
        # Windows PowerShell 5.1 对命令行 string[] 的绑定有歧义:第二个裸路径会被
        # 绑定到后面的 Language。逐张调用固定 -File 虽有少量启动成本,但路径不会
        # 被拼进脚本文本,也不会串到别的参数。
        for path in paths:
            command = [
                str(self.powershell), "-NoProfile", "-ExecutionPolicy", "Bypass",
                "-File", str(self.script), "-Paths", str(path.resolve()),
            ]
            completed = subprocess.run(
                command, capture_output=True, text=True, encoding="utf-8",
                errors="replace", timeout=30,
            )
            if completed.returncode != 0:
                raise RuntimeError((completed.stderr or completed.stdout or "OCR failed")[-800:])
            try:
                payload = json.loads(completed.stdout.lstrip("\ufeff"))
            except ValueError as error:
                raise RuntimeError("Windows OCR returned invalid JSON") from error
            for row in payload:
                results[str(Path(row["path"]).resolve())] = row
        return results


def atomic_json(path: Path, payload: dict) -> None:
    temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
    try:
        temporary.write_text(
            json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8",
        )
        os.replace(temporary, path)
    except BaseException:
        temporary.unlink(missing_ok=True)
        raise


def atomic_csv(path: Path, fields: tuple[str, ...], rows: list[dict]) -> None:
    write_rows(path, fields, rows, atomic=True, fill_missing=True)


def run(
    args: argparse.Namespace,
    *,
    capture: Callable[[str, str, float, Path], bool] = capture_frame,
    ocr=None,
    ffmpeg_path: str | None = None,
) -> int:
    started = time.perf_counter()
    health = {
        "source": "ffmpeg+windows_ocr", "policy_version": POLICY_VERSION,
        "attempted": 0, "frames_requested": 0, "frame_cache_reused": 0,
        "captured": 0, "capture_errors": 0, "ocr_cache_reused": 0,
        "ocr_succeeded": 0, "ocr_errors": 0, "candidates": 0,
        "incomplete_candidates": 0, "source_evidence": 0, "elapsed_ms": 0,
        "last_error_kind": "", "last_error_message": "",
    }
    try:
        assets = load_assets(args)
        head = parse_offsets(args.head_offsets)
        tail = parse_offsets(args.tail_offsets)
        require_free_space(system_volume(), args.min_free)
        guard = DiskGuard(system_volume(), args.min_free, 0)
        if ffmpeg_path is None:
            resolver = FFmpegResolver(FFMPEG_DIR)
            ffmpeg = resolver.ffmpeg()
            if ffmpeg is None:
                raise RuntimeError("ffmpeg unavailable")
            active_ffmpeg = str(ffmpeg.path)
        else:
            active_ffmpeg = ffmpeg_path
        provider = ocr or WindowsOcrProvider(args.ocr_script)
        observations: dict[int, list[dict]] = {int(asset["id"]): [] for asset in assets}
        uncached: list[Path] = []
        frame_context: dict[str, tuple[dict, str, float, Path]] = {}
        for asset in assets:
            health["attempted"] += 1
            for kind, timestamp in sample_points(float(asset["duration"]), head, tail):
                health["frames_requested"] += 1
                guard.check(force=True)
                frame = frame_path(args.evidence_root, int(asset["id"]), kind, timestamp)
                sidecar = frame.with_suffix(".ocr.json")
                if frame.is_file() and frame.stat().st_size > 1024:
                    health["frame_cache_reused"] += 1
                else:
                    ok = capture(
                        active_ffmpeg, resolve_case_insensitive(asset["path"]),
                        timestamp, frame,
                    )
                    if not ok:
                        health["capture_errors"] += 1
                        continue
                    health["captured"] += 1
                key = str(frame.resolve())
                frame_context[key] = (asset, kind, timestamp, sidecar)
                if sidecar.is_file():
                    try:
                        result = json.loads(sidecar.read_text(encoding="utf-8"))
                    except (OSError, ValueError):
                        uncached.append(frame)
                    else:
                        observations[int(asset["id"])].append({
                            "asset": asset, "kind": kind, "timestamp": timestamp,
                            "frame": frame, **result,
                        })
                        health["ocr_cache_reused"] += 1
                else:
                    uncached.append(frame)
        if uncached:
            results = provider.recognize(uncached)
            for frame in uncached:
                key = str(frame.resolve())
                asset, kind, timestamp, sidecar = frame_context[key]
                result = results.get(key) or {"text": "", "lines": [], "error": "missing OCR row"}
                atomic_json(sidecar, result)
                observations[int(asset["id"])].append({
                    "asset": asset, "kind": kind, "timestamp": timestamp,
                    "frame": frame, **result,
                })
                if result.get("error"):
                    health["ocr_errors"] += 1
                    health["last_error_kind"] = "ocr_item"
                    health["last_error_message"] = str(result["error"])[:500]
                else:
                    health["ocr_succeeded"] += 1
        candidates = []
        priority = {"incomplete_candidate": 2, "source_evidence": 1, "none": 0}
        for asset_id, rows in observations.items():
            classified = [(detect_endcard(row.get("text") or ""), row) for row in rows]
            classified = [item for item in classified if item[0].verdict != "none"]
            if not classified:
                continue
            detection, evidence = max(
                classified,
                key=lambda item: (priority[item[0].verdict], item[0].confidence,
                                  item[1]["kind"] == "tail", item[1]["timestamp"]),
            )
            asset = evidence["asset"]
            candidates.append({
                "candidate_key": str(asset_id), "asset_id": asset_id,
                "name": asset.get("name") or "", "code": asset.get("code") or "",
                "location": asset.get("location") or "", "sample_kind": evidence["kind"],
                "timestamp_seconds": round(float(evidence["timestamp"]), 3),
                "frame_key": str(evidence["frame"].relative_to(args.evidence_root)).replace("\\", "/"),
                "ocr_text": str(evidence.get("text") or "")[:2000],
                "verdict": detection.verdict,
                "detected_urls": " ".join(detection.urls),
                "detected_handles": " ".join(detection.handles),
                "confidence": detection.confidence, "reason": detection.reason,
                "policy_version": POLICY_VERSION, "status": "candidate",
            })
        candidates.sort(key=lambda row: (-float(row["confidence"]), int(row["asset_id"])))
        health["candidates"] = len(candidates)
        health["incomplete_candidates"] = sum(
            row["verdict"] == "incomplete_candidate" for row in candidates
        )
        health["source_evidence"] = sum(
            row["verdict"] == "source_evidence" for row in candidates
        )
        atomic_csv(args.out, CANDIDATE_FIELDS, candidates)
    except (OSError, sqlite3.Error, subprocess.SubprocessError, ValueError, RuntimeError,
            JobPolicyError) as error:
        health["last_error_kind"] = type(error).__name__
        health["last_error_message"] = str(error)[:500]
        health["elapsed_ms"] = round((time.perf_counter() - started) * 1000)
        atomic_csv(args.health, HEALTH_FIELDS, [health])
        print(f"首尾帧审计失败:{error}")
        print(f"健康报告 → {args.health}")
        return getattr(error, "exit_code", 2)
    health["elapsed_ms"] = round((time.perf_counter() - started) * 1000)
    atomic_csv(args.health, HEALTH_FIELDS, [health])
    print(f"首尾帧候选 {health['candidates']} 条 → {args.out}")
    print(f"来源健康 → {args.health}")
    print("只生成候选:未修改 ledger、quality goal、媒体文件或复核决定。")
    return 0


def main(argv: list[str] | None = None) -> int:
    return job_main(build_parser, run, argv)


if __name__ == "__main__":
    raise SystemExit(main())