Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

9.6 KB
Raw Blame
#!/usr/bin/env python3
"""全库 ffprobe:可续跑、单写者、显式计费授权。

新文件入库时已经探过一次(`peach.media_probe`),这里补的是历史空缺、失败重探和点名
重探,以及入库时不碰的计费来源。
"""
from __future__ import annotations

import argparse
import queue
import sqlite3
import threading
import time
import sys
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parent.parent
SRC_DIR = PROJECT_ROOT / "src"
if str(SRC_DIR) not in sys.path:
    sys.path.insert(0, str(SRC_DIR))

from peach import junk, record_rehome
from peach.config import DATABASE_PATH, FFMPEG_DIR, LOG_DIR, STATE_DIR
from peach.ffmpeg import FFmpegResolver
from peach.jobs import (
    ACTIVE_ASSET_SQL,
    DiskGuard,
    DiskSpaceDenied,
    JobPolicyError,
    SourceAccessPolicy,
    require_free_space,
    job_main,
)
from peach.platform import system_volume
from peach.media import resolve_case_insensitive
from peach.media_probe import UPDATE, context_fields, probe_file


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="补全视频事实层和情境层")
    parser.add_argument("--workers", type=int, default=6)
    parser.add_argument("--location")
    parser.add_argument("--interval", type=float, default=0.15)
    parser.add_argument("--limit", type=int, default=0)
    parser.add_argument("--min-free", type=float, default=40.0)
    parser.add_argument(
        "--disk-check-secs",
        type=float,
        default=20.0,
        help="运行期复查磁盘余量的间隔;0 表示每条都查",
    )
    parser.add_argument("--timeout", type=float, default=20.0)
    parser.add_argument(
        "--redo",
        choices=("none", "zero", "failed", "all"),
        default="none",
        help="额外重探已记录但无效的时长:zero=0(软失败)、failed=-1(硬失败)、all=两者",
    )
    parser.add_argument(
        "--asset",
        type=int,
        action="append",
        help="按 id 重探指定资产,绕过时长筛选。用于账本时长与真实文件不符的个案:"
             "这类值看着正常,--redo 的 0/-1 判据够不着",
    )
    parser.add_argument("--allow-metered", action="store_true")
    parser.add_argument("--db", type=Path, default=DATABASE_PATH)
    parser.add_argument("--log-dir", type=Path, default=LOG_DIR)
    parser.add_argument("--lock", type=Path, default=STATE_DIR / ".probe.lock")
    return parser


def duration_selection(redo: str) -> str:
    """未探测和"探测过但没拿到时长"是两种状态;后者必须显式要求才重跑。

    软失败历史上被写成 duration=0,既不在 `duration IS NULL` 的待办里,也过不了
    抽帧的 duration>2 门槛,于是永远卡住。硬失败写成 -1。
    """
    return {
        "none": "duration IS NULL",
        "zero": "(duration IS NULL OR duration=0)",
        "failed": "(duration IS NULL OR duration<0)",
        "all": "(duration IS NULL OR duration<=0)",
    }[redo]


def run(args: argparse.Namespace) -> int:
    if args.workers < 1 or args.interval < 0 or args.limit < 0:
        raise SystemExit("workers 必须大于 0;interval/limit 不能为负数")
    if args.timeout <= 0:
        raise SystemExit("timeout 必须大于 0")
    choice = FFmpegResolver(FFMPEG_DIR).ffprobe()
    if choice is None:
        raise RuntimeError("ffprobe unavailable")
    require_free_space(system_volume(), args.min_free)
    source_sql, source_parameters = SourceAccessPolicy().sql_filter(
        args.location, args.allow_metered
    )

    args.log_dir.mkdir(parents=True, exist_ok=True)
    log_path = args.log_dir / f"probe-{time.strftime('%Y%m%d-%H%M%S')}.log"
    lock = threading.RLock()
    with log_path.open("w", encoding="utf-8", buffering=1) as log_file:
        def log(message: str) -> None:
            with lock:
                line = f"[{time.strftime('%H:%M:%S')}] {message}"
                print(line, flush=True)
                log_file.write(line + "\n")

        if args.allow_metered:
            log("已显式允许计费来源")
        if args.redo != "none":
            log(f"重探已失败记录:--redo {args.redo}")
        connection = sqlite3.connect(args.db)
        if args.asset:
            # 显式点名就不再套时长筛选,但计费来源和 online 的边界照旧生效。
            placeholders = ",".join("?" for _ in args.asset)
            sql = (
                f"SELECT id,path FROM asset WHERE medium='video' AND id IN ({placeholders})"
                " AND location != 'online'" + source_sql + " ORDER BY size ASC"
            )
            parameters: tuple[object, ...] = tuple(args.asset) + source_parameters
        else:
            sql = (
                "SELECT id,path FROM asset WHERE medium='video' AND "
                + duration_selection(args.redo)
                + f" AND location != 'online' AND {ACTIVE_ASSET_SQL}"
                + source_sql + " ORDER BY size ASC"
            )
            parameters = source_parameters
        if args.limit:
            sql += " LIMIT ?"
            parameters += (args.limit,)
        from peach.strm import background_rows
        tasks = background_rows(connection.execute(sql, parameters).fetchall(), 1)
        connection.close()
        isolated = junk.isolate(args.db, asset_ids=[row[0] for row in tasks])
        tasks = [row for row in tasks if row[0] not in isolated['ids']]
        total = len(tasks)
        log(f"待处理 {total:,} 个视频 workers={args.workers} interval={args.interval}s 日志={log_path}")
        if not total:
            return 0

        pending: queue.Queue = queue.Queue()
        results: queue.Queue = queue.Queue()
        for task in tasks:
            pending.put(task)
        counters = {"done": 0, "failed": 0}
        measured_ids = []
        started = time.time()
        # 探测本身产物很小,但它触发的云盘块缓存会落在系统盘;边跑边看,不只查起跑线。
        guard = DiskGuard(system_volume(), args.min_free, args.disk_check_secs)
        stop = threading.Event()
        stop_reason: list[str] = []

        def worker() -> None:
            while not stop.is_set():
                try:
                    asset_id, path = pending.get_nowait()
                except queue.Empty:
                    return
                try:
                    duration, width, height, codec, fps, audio = probe_file(
                        str(choice.path), resolve_case_insensitive(path), args.timeout
                    )
                    if duration <= 0:
                        with lock:
                            counters["failed"] += 1
                    length, orientation, quality = context_fields(width, height, duration)
                    results.put((duration, width, height, codec, fps, audio, length, orientation, quality, asset_id))
                except Exception:
                    results.put((-1, 0, 0, None, 0, 0, None, None, None, asset_id))
                    with lock:
                        counters["failed"] += 1
                finally:
                    with lock:
                        counters["done"] += 1
                        if counters["done"] % 50 == 0:
                            elapsed = time.time() - started
                            rate = counters["done"] / elapsed if elapsed else 0
                            eta = (total - counters["done"]) / rate / 60 if rate else 0
                            log(f"{counters['done']:,}/{total:,} 失败 {counters['failed']} {rate:.1f} 个/秒 剩余 {eta:.0f} 分钟")
                        try:
                            guard.check()
                        except JobPolicyError as exc:
                            if not stop.is_set():
                                stop_reason.append(str(exc))
                                log(f"[stop] {exc}")
                            stop.set()
                    time.sleep(args.interval)

        threads = [threading.Thread(target=worker, daemon=True) for _ in range(args.workers)]
        for thread in threads:
            thread.start()

        connection = sqlite3.connect(args.db, timeout=120)
        connection.execute("PRAGMA journal_mode=WAL")
        connection.execute("PRAGMA busy_timeout=120000")
        update = UPDATE
        buffer = []
        while any(thread.is_alive() for thread in threads) or not results.empty():
            try:
                buffer.append(results.get(timeout=2))
            except queue.Empty:
                pass
            if len(buffer) >= 25:
                connection.executemany(update, buffer)
                connection.commit()
                measured_ids.extend(row[-1] for row in buffer)
                buffer.clear()
        if buffer:
            connection.executemany(update, buffer)
            connection.commit()
            measured_ids.extend(row[-1] for row in buffer)
        elapsed = time.time() - started
        log(f"完成 {counters['done']:,} 个,失败 {counters['failed']},耗时 {elapsed/60:.1f} 分钟")
        record_rehome.reattach(connection, measured_ids)
        connection.commit()
        connection.close()
        checked = junk.isolate(args.db, asset_ids=measured_ids)
        log(f"垃圾检查:隔离 {isolated['isolated'] + checked['isolated']} 个,待复核 {checked['review']} 个")
        if stop_reason:
            # 已探测的部分都已入库;中止必须体现在退出码上,不能报成正常完成。
            log(f"磁盘闸门中止:{stop_reason[0]}")
            return DiskSpaceDenied.exit_code
    return 0


def main(argv: list[str] | None = None) -> int:
    return job_main(build_parser, run, argv)


if __name__ == "__main__":
    raise SystemExit(main())