Skip to content

Local video editor for coding agents: 42 FFmpeg tools with a machine-readable contract and an MCP server, for Claude Code, Cursor and Codex. Cut, captions, reframe, loudness, platform exports. No cloud, no API keys.

17.2 KB
Raw Blame
#!/usr/bin/env python3
"""Find scene changes and loud moments, and propose highlight candidates so the
agent can plan an edit or a digest without watching the whole file.

Scene cuts come from ffmpeg's scdet; energy peaks from a 0.5 s RMS envelope
of the audio. Highlight candidates are scenes ranked by --rank-by: "audio"
(default, loudest first) or "duration" (longest first). Both are proxies,
not a judgement of what matters: "audio" misses a quiet but important
moment (a confession, a punchline landing in silence) and can surface pure
crowd noise; "duration" just finds long unbroken takes. Neither replaces
watching the contact sheet (--sheet) before committing to a cut.

Examples:
  python3 scenes.py talk.mp4                                # scenes + peaks, JSON
  python3 scenes.py event.mp4 --highlights 5 --target 60   # 5 candidate ranges summing to ~60 s
  python3 scenes.py event.mp4 --highlights 4 --edl picks.txt   # cut.py --segments compatible list
  python3 scenes.py event.mp4 --sheet scenes.png             # one thumbnail per scene
  python3 scenes.py talk.mp4 --highlights 5 --rank-by duration  # longest unbroken scenes, not loudest
"""
import argparse
import math
import sys
from typing import Dict, List, Optional, Tuple

# `detect_scenes` moved into _common/probe.py in 1.16.0 (see silence.py); the body is unchanged.
from _common import (detect_scenes, STATE, add_common, apply_common, beat_grid, default_font_file, die, emit,
                     escape_filter_path, ffmpeg_base, info, print_json, probe, run, decode_pcm_mono,
                     rms_envelope, BEAT_MIN_CONFIDENCE, decode_gray_frames, frame_flow, label_shot_flow)

SHOT_FPS = 4.0          # frames/second sampled for --shots' flow estimate
SHOT_W, SHOT_H = 48, 27  # decode size for --shots (16:9-ish; enough blocks, still a few KB/shot)
SPEECH_STEP_S = 1.0      # --speech energy-ratio window



def audio_envelope(path: str, step_s: float, *, rate: int = 8000,
                   samples: "Optional[List[float]]" = None) -> List[float]:
    """RMS level per step_s window, absolute (a loud scene scores higher); [] when the audio
    cannot be decoded (the cut scoring then runs on the picture alone).

    `samples` reuses PCM a caller already decoded rather than decoding the same file twice --
    --beats needs a 10 ms envelope and the scene scoring a 0.5 s one, and 0.5 s is an integer
    multiple of 10 ms, so both come from one pass.
    """
    if samples is None:
        samples = decode_pcm_mono(path, rate, check=False)
    return rms_envelope(samples, int(rate * step_s))


def parse_beat_range(text: str) -> "tuple":
    """`--beat-range 60-200` as (lo, hi) BPM."""
    try:
        lo, hi = (float(p) for p in str(text).replace(" ", "").split("-", 1))
    except ValueError:
        die(f"--beat-range: expected LO-HI in BPM (e.g. 60-200), got {text!r}", kind="input")
    if not (0 < lo < hi):
        die(f"--beat-range {text}: LO must be above 0 and below HI", kind="input")
    return (lo, hi)


def shot_flow_label(path: str, start: float, end: float) -> Dict:
    """{start, end, label, flow_magnitude} for one shot: decode it at SHOT_FPS/SHOT_W x SHOT_H
    and block-match consecutive frames (see _common.decision.frame_flow / label_shot_flow). A
    shot under two sampled frames has nothing to compare and is reported static with
    flow_magnitude 0 -- there is no motion measurement to make on a single frame."""
    frames = decode_gray_frames(path, SHOT_FPS, SHOT_W, SHOT_H, start=start, seconds=max(0.0, end - start), check=False)
    flows = [frame_flow(frames[i], frames[i + 1], SHOT_W, SHOT_H) for i in range(len(frames) - 1)]
    label = label_shot_flow(flows)
    return {"start": round(start, 3), "end": round(end, 3), "label": label["label"], "flow_magnitude": label["flow_magnitude"]}


def audio_peaks_db(samples: "List[float]", rate: int, step_s: float = 0.25) -> List[Dict]:
    """--audio-peaks: local maxima of the loudness envelope reported as measured dBFS, not the
    unitless RMS scenes.py has always put in the (unconditional) `audio_peaks` key -- a
    different unit needs a different key so the existing one keeps meaning what it always has."""
    env = rms_envelope(samples, int(rate * step_s))
    peaks = []
    for i, val in enumerate(env):
        if val <= 1e-6:
            continue
        if (i == 0 or env[i - 1] <= val) and (i == len(env) - 1 or env[i + 1] <= val):
            level = round(20 * math.log10(val), 1)
            peaks.append({"time": round(i * step_s, 2), "level": level})
    return peaks


def speech_music_ratio(samples: "List[float]", rate: int, step_s: float = SPEECH_STEP_S) -> List[Dict]:
    """--speech: a per-second zero-crossing-rate ratio, reported as a measured number, not a
    speech/music label. Speech's rapid consonant transients drive the zero-crossing rate up and
    make it jump window to window; sustained tones (music, a held note, room tone) cross zero at
    a steadier rate. ratio = this window's ZCR / the file's median ZCR, so 1.0 is "typical for
    this file" regardless of its overall noisiness -- a proxy, in the same spirit as scenes.py's
    --rank-by, not a classifier: nothing here decides what is speech."""
    step = max(1, int(rate * step_s))
    zcrs: List[float] = []
    for i in range(0, len(samples) - step + 1, step):
        block = samples[i:i + step]
        crossings = sum(1 for a, b in zip(block, block[1:]) if (a >= 0) != (b >= 0))
        zcrs.append(crossings / max(1, len(block) - 1))
    if not zcrs:
        return []
    med = sorted(zcrs)[len(zcrs) // 2] or 1e-9
    return [{"time": round(i * step_s, 2), "speech_music_ratio": round(z / med, 3)} for i, z in enumerate(zcrs)]


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("input")
    ap.add_argument("--threshold", type=float, default=8.0, help="minimum scdet score for a cut, 0-100 (default 8)")
    ap.add_argument("--ratio", type=float, default=3.0, help="a cut must exceed this multiple of the neighbouring frames' median score (default 3; lower = more cuts)")
    ap.add_argument("--min-scene", type=float, default=1.0, help="ignore cuts closer than this in seconds (default 1)")
    ap.add_argument("--highlights", type=int, default=0, help="number of highlight ranges to propose")
    ap.add_argument("--rank-by", choices=["audio", "duration"], default="audio",
                    help="how to rank scenes for --highlights: audio energy (default) or scene duration")
    ap.add_argument("--target", type=float, help="with --highlights: total seconds the picks should add up to (trims long scenes)")
    ap.add_argument("--max-scene", type=float, default=15.0, help="cap a highlight range at this many seconds (default 15)")
    ap.add_argument("--edl", help="write highlight ranges as START-END lines (cut.py --segments format)")
    ap.add_argument("--beats", action="store_true",
                    help="measure the music's beat grid (tempo, beat times, confidence) and report it; "
                         "a measurement, not a proposal -- no cut is made and no beat is invented")
    ap.add_argument("--beat-step", type=float, default=0.01,
                    help="envelope resolution in seconds for the onset pass with --beats (default 0.01)")
    ap.add_argument("--beat-range", default="60-200",
                    help="tempo search range in BPM for --beats (default 60-200)")
    ap.add_argument("--min-confidence", type=float, default=BEAT_MIN_CONFIDENCE,
                    help="with --beats: below this confidence the grid is still reported, marked "
                         f"usable: false (default {BEAT_MIN_CONFIDENCE})")
    ap.add_argument("--shots", action="store_true",
                    help="label each detected shot static / pan / motion by a measured optical-flow "
                         "proxy (lightweight block matching over sampled frames); reports flow_magnitude too")
    ap.add_argument("--audio-peaks", action="store_true",
                    help="report loudness peaks as {time, level} in measured dBFS (separate from the "
                         "always-on `audio_peaks` RMS list used to pick --highlights)")
    ap.add_argument("--speech", action="store_true",
                    help="report a per-second speech-vs-music energy ratio (zero-crossing-rate proxy, "
                         "a measurement, not a speech/music classification)")
    ap.add_argument("--sheet", help="write a contact sheet PNG with the first frame of every scene")
    ap.add_argument("--no-timecode", action="store_true", help="--sheet without the burnt-in timecode stamp (a way out if drawtext itself is unusable, see doctor)")
    add_common(ap)
    args = ap.parse_args()
    apply_common(args)

    meta = probe(args.input)
    if not meta.get("video"):
        die("input has no video stream")
    # Only parsed when it is going to be used: --beat-range is a --beats flag, and a run that
    # never asked for beats should not be able to die on one.
    beat_range = parse_beat_range(args.beat_range) if args.beats else (60.0, 200.0)
    if args.beats:
        if not meta.get("audio"):
            die("--beats needs an audio stream; this file has none", kind="input")
        if args.beat_step <= 0:
            die("--beat-step must be greater than 0", kind="input")
    dur = meta.get("duration") or 0.0
    cuts = detect_scenes(args.input, args.threshold, args.min_scene, dur, args.ratio)
    bounds = cuts + [dur]
    step_s = 0.5
    # With --beats the file is decoded once, at the finer rate, and both envelopes come from that
    # one pass: the 0.5 s scene blocks are an exact multiple of the 10 ms onset blocks.
    beat_rate = 22050
    need_fine = args.beats or args.audio_peaks or args.speech
    fine_samples = decode_pcm_mono(args.input, beat_rate, check=False) if (need_fine and meta.get("audio")) else None
    if fine_samples is not None:
        env = audio_envelope(args.input, step_s, rate=beat_rate, samples=fine_samples)
    else:
        env = audio_envelope(args.input, step_s) if meta.get("audio") else []

    scenes = []
    for i in range(len(bounds) - 1):
        s, e = bounds[i], bounds[i + 1]
        if e - s <= 0.05:
            continue
        seg = env[int(s / step_s): max(int(s / step_s) + 1, int(e / step_s))] if env else []
        energy = (sum(seg) / len(seg)) if seg else 0.0
        peak = max(seg) if seg else 0.0
        scenes.append({"index": len(scenes), "start": round(s, 3), "end": round(e, 3), "duration": round(e - s, 3),
                       "audio_rms": round(energy, 4), "audio_peak": round(peak, 4)})
    peaks = []
    if env:
        thr = sorted(env)[int(len(env) * 0.9)] if len(env) > 10 else max(env)
        for i, val in enumerate(env):
            if val >= thr and val > 0.02 and (i == 0 or env[i - 1] < val) and (i == len(env) - 1 or env[i + 1] <= val):
                peaks.append({"time": round(i * step_s, 2), "rms": round(val, 4)})
        peaks = sorted(peaks, key=lambda p: -p["rms"])[:20]
        peaks.sort(key=lambda p: p["time"])

    result: Dict = {"file": args.input, "duration": round(dur, 3), "scene_count": len(scenes), "scenes": scenes, "audio_peaks": peaks}
    info(f"{len(scenes)} scenes, {len(peaks)} audio peaks over {dur:.1f}s")

    if args.shots:
        shots = [shot_flow_label(args.input, s, e) for s, e in zip(bounds[:-1], bounds[1:]) if e - s > 0.05]
        result["shots"] = shots
        counts = {}
        for sh in shots:
            counts[sh["label"]] = counts.get(sh["label"], 0) + 1
        info(f"--shots: {len(shots)} shots (" + ", ".join(f"{v} {k}" for k, v in sorted(counts.items())) + ")")

    if args.audio_peaks:
        if not meta.get("audio"):
            result["audio_peaks_db"] = []
            info("--audio-peaks: no audio stream, nothing to measure")
        else:
            db_samples = fine_samples if fine_samples is not None else decode_pcm_mono(args.input, 22050, check=False)
            result["audio_peaks_db"] = audio_peaks_db(db_samples, 22050 if fine_samples is not None else 22050)
            info(f"--audio-peaks: {len(result['audio_peaks_db'])} peaks")

    if args.speech:
        if not meta.get("audio"):
            result["speech"] = []
            info("--speech: no audio stream, nothing to measure")
        else:
            sp_samples = fine_samples if fine_samples is not None else decode_pcm_mono(args.input, 22050, check=False)
            result["speech"] = speech_music_ratio(sp_samples, 22050)
            info(f"--speech: {len(result['speech'])} one-second windows")

    if args.beats:
        # A beat grid is a measurement of the music's periodicity, not a statement about where a
        # cut belongs. scenes.py reports what it measured, including a low confidence: reporting a
        # weak measurement is honest, and only a tool that CHANGES a file refuses to act on one.
        fine = rms_envelope(fine_samples or [], max(1, int(round(beat_rate * args.beat_step))))
        grid = beat_grid(fine, args.beat_step, bpm_range=beat_range,
                         min_confidence=args.min_confidence, duration=dur)
        result["beats"] = grid["beats"]
        result["beat_grid"] = {
            # The regular grid AND the subset a measured onset supports. A tool that MOVES
            # something (cut.py --snap beats) may only use the subset; scenes.py reports both,
            # because here the regular grid is the measurement being made.
            "supported_beats": grid["supported_beats"],
            "tempo_bpm": grid["tempo_bpm"], "interval": grid["interval"],
            "confidence": grid["confidence"], "phase": grid["phase"],
            "onsets": len(grid["onsets"]), "supported": grid["supported"],
            "unsupported": grid["unsupported"], "method": grid["method"],
            "step_s": grid["step_s"], "range_bpm": grid["range_bpm"], "usable": grid["usable"],
        }
        if grid["tempo_bpm"] is None:
            info(f"--beats: no steady pulse in this audio (confidence {grid['confidence']:.2f}) -- "
                 "speech, ambience or rubato has no tempo to measure")
        else:
            info(f"--beats: {grid['tempo_bpm']:.1f} BPM, {len(grid['beats'])} beats, confidence "
                 f"{grid['confidence']:.2f} ({grid['supported']} of {len(grid['beats'])} grid points "
                 f"have a measured onset)"
                 + ("" if grid["usable"] else f" -- below --min-confidence {args.min_confidence}, usable: false"))

    if args.highlights:
        if args.rank_by == "duration":
            rank_key = lambda sc: (-sc["duration"], sc["start"])
        else:
            rank_key = lambda sc: (-sc["audio_rms"], sc["start"])
        ranked = sorted(scenes, key=rank_key)[: args.highlights]
        picks: List[Tuple[float, float]] = []
        budget = args.target if args.target else None
        per = (budget / max(1, len(ranked))) if budget else args.max_scene
        for sc in ranked:
            length = min(sc["duration"], per, args.max_scene)
            # take the loudest window inside the scene
            best_s = sc["start"]
            if env and length < sc["duration"]:
                best, best_s = -1.0, sc["start"]
                win = max(1, int(length / step_s))
                lo, hi = int(sc["start"] / step_s), max(int(sc["start"] / step_s) + 1, int(sc["end"] / step_s) - win)
                for i in range(lo, hi + 1):
                    val = sum(env[i:i + win])
                    if val > best:
                        best, best_s = val, i * step_s
            picks.append((round(best_s, 2), round(min(sc["end"], best_s + length), 2)))
        picks.sort()
        result["highlights"] = [{"start": s, "end": e, "duration": round(e - s, 2)} for s, e in picks]
        result["highlights_total"] = round(sum(e - s for s, e in picks), 2)
        result["highlights_rank_by"] = args.rank_by
        info(f"proposed {len(picks)} highlight ranges totalling {result['highlights_total']:.1f}s")
        if args.edl:
            if not STATE.dry_run:  # the contract says --edl is not written under --dry-run
                with open(args.edl, "w", encoding="utf-8") as fh:
                    for s, e in picks:
                        fh.write(f"{s:.2f}-{e:.2f}\n")
            info(f"wrote {args.edl}")

    if args.sheet:
        n = len(scenes)
        cols = min(4, max(1, n))
        rows = max(1, math.ceil(n / cols))
        tile_w = 1280 // cols // 2 * 2
        # exactly one frame per scene: the frame index at the scene start
        fps = meta["video"].get("fps") or 30.0
        expr = "+".join(f"eq(n\\,{int(round(sc['start'] * fps))})" for sc in scenes)
        stamp = ""
        if not args.no_timecode:
            default_font = default_font_file("DejaVu Sans")
            font_prefix = f"fontfile={escape_filter_path(default_font)}:" if default_font else ""
            stamp = f",drawtext=text='%{{pts\\:hms}}':{font_prefix}fontcolor=white:fontsize=h/14:box=1:[email protected]:boxborderw=4:x=6:y=6"
        vf = (f"select='{expr}',scale={tile_w}:-2{stamp},"
              f"tile={cols}x{rows}:padding=2:margin=2:color=0x202020")
        run(ffmpeg_base() + ["-i", args.input, "-vf", vf, "-frames:v", "1", "-fps_mode", "vfr", args.sheet])
        info(f"wrote {args.sheet}")
        result["sheet"] = args.sheet

    if args.json:
        emit(None, **result)
    else:
        print_json(result)
    return 0


if __name__ == "__main__":
    sys.exit(main())