World's first open-source, agentic video production system. 12 production pipelines, 100+ tools, 700+ agent skill and production-knowledge files. Turn your AI coding assistant into a full video production studio.
Python agentagentic-aiaiclaudecopilotcursorelevenlabsffmpegfluximage-generationopen-sourceopenaipythonremotionstable-diffusiontext-to-speechtext-to-videovideo-generationvideo-production
Fix video_understand sampling only the opening seconds of a clip
Daniel Liu45f45855da1f
diff --git a/tests/tools/test_video_understand_sampling.py b/tests/tools/test_video_understand_sampling.pynew file mode 100644index 00000000..9afce86b--- /dev/null+++ b/tests/tools/test_video_understand_sampling.py@@ -0,0 +1,121 @@+"""Frame sampling in video_understand.++No network and no ffmpeg: subprocess.run is faked, so these assert the commands+the tool builds rather than what a particular ffmpeg build happens to return.+"""++from __future__ import annotations++import types+from pathlib import Path++import pytest++from tools.analysis.video_understand import VideoUnderstand+++class FakePIL:+ """Stand-in for PIL.Image.open(...).convert('RGB')."""++ def __init__(self, path):+ self.path = path++ def convert(self, _mode):+ return self+++def fake_ffmpeg(monkeypatch):+ """Record every ffmpeg invocation and materialise the files it promises."""+ calls: list[list[str]] = []++ def fake_run(cmd, **kwargs):+ calls.append(list(cmd))+ if cmd and cmd[0] == "ffmpeg":+ target = Path(cmd[-4])+ if "%04d" in target.name:+ for index in range(3):+ target.with_name(target.name.replace("%04d", f"{index:04d}")).write_bytes(b"x")+ else:+ target.write_bytes(b"x")+ return types.SimpleNamespace(returncode=0, stdout="", stderr="")++ monkeypatch.setattr("tools.analysis.video_understand.subprocess.run", fake_run)++ fake_image_module = types.SimpleNamespace(open=FakePIL)+ monkeypatch.setitem(+ __import__("sys").modules, "PIL",+ types.SimpleNamespace(Image=fake_image_module),+ )+ return calls+++def _seek_positions(calls: list[list[str]]) -> list[float]:+ return [float(cmd[cmd.index("-ss") + 1]) for cmd in calls if "-ss" in cmd]+++def test_samples_span_the_whole_clip(monkeypatch, fake_ffmpeg, tmp_path):+ """The regression this file exists for.++ `-vf thumbnail=N -frames:v N` reads as even sampling and is not: thumbnail=N+ picks the most representative frame out of each consecutive N-frame batch,+ so it never looks past the opening seconds. A four-second clip that is+ solid red for 2s then solid blue for 2s came back as four red frames, and+ the caption pass called it a video in which nothing changes.+ """+ monkeypatch.setattr(+ VideoUnderstand, "_video_duration_seconds", staticmethod(lambda _p: 100.0)+ )++ frames = VideoUnderstand()._extract_video_frames(tmp_path / "clip.mp4", None, 4)++ assert len(frames) == 4+ assert _seek_positions(fake_ffmpeg) == [12.5, 37.5, 62.5, 87.5]+ # The property the batching bug violated: the last sample is near the end.+ assert _seek_positions(fake_ffmpeg)[-1] > 75.0+ assert not any("thumbnail=" in " ".join(cmd) for cmd in fake_ffmpeg)+++def test_explicit_frame_indices_still_use_the_select_filter(fake_ffmpeg, tmp_path):+ """That branch was already correct and must not change."""+ VideoUnderstand()._extract_video_frames(tmp_path / "clip.mp4", [3, 9, 27], 3)++ joined = " ".join(fake_ffmpeg[0])+ assert "select=" in joined+ assert "eq(n\\,3)" in joined and "eq(n\\,27)" in joined+ assert "-ss" not in fake_ffmpeg[0]+++def test_falls_back_when_duration_is_unknown(monkeypatch, fake_ffmpeg, tmp_path):+ """A stream, or no ffprobe on PATH, must still yield frames."""+ monkeypatch.setattr(+ VideoUnderstand, "_video_duration_seconds", staticmethod(lambda _p: None)+ )++ frames = VideoUnderstand()._extract_video_frames(tmp_path / "stream.mp4", None, 3)++ assert frames, "the fallback must produce frames, not an empty list"+ assert not _seek_positions(fake_ffmpeg), "no duration means no seek positions"++[email protected]("probe_output, expected", [+ ("12.5\n", 12.5),+ ("N/A\n", None),+ ("", None),+ ("0\n", None), # a zero duration would divide the sampler by zero+ ("-3\n", None),+])+def test_duration_probe_rejects_unusable_values(monkeypatch, tmp_path, probe_output, expected):+ monkeypatch.setattr(+ "tools.analysis.video_understand.subprocess.run",+ lambda *a, **k: types.SimpleNamespace(returncode=0, stdout=probe_output, stderr=""),+ )+ assert VideoUnderstand._video_duration_seconds(tmp_path / "c.mp4") == expected+++def test_duration_probe_survives_a_missing_ffprobe(monkeypatch, tmp_path):+ def boom(*_a, **_k):+ raise FileNotFoundError("ffprobe")++ monkeypatch.setattr("tools.analysis.video_understand.subprocess.run", boom)+ assert VideoUnderstand._video_duration_seconds(tmp_path / "c.mp4") is Nonediff --git a/tools/analysis/video_understand.py b/tools/analysis/video_understand.pyindex 50027593..1cc9c362 100644--- a/tools/analysis/video_understand.py+++ b/tools/analysis/video_understand.py@@ -252,48 +252,81 @@ class VideoUnderstand(BaseTool):return self._extract_video_frames(input_path, frame_indices, max_frames)+ @staticmethod+ def _video_duration_seconds(video_path: Path) -> float | None:+ """Duration via ffprobe, or None when it cannot be determined."""+ try:+ out = subprocess.run(+ ["ffprobe", "-v", "error", "-show_entries", "format=duration",+ "-of", "default=noprint_wrappers=1:nokey=1", str(video_path)],+ capture_output=True, text=True, timeout=30,+ ).stdout.strip()+ duration = float(out)+ return duration if duration > 0 else None+ except (ValueError, OSError, subprocess.SubprocessError):+ return None+def _extract_video_frames(self,video_path: Path,frame_indices: list[int] | None,max_frames: int,) -> list:- """Extract frames from a video file using ffmpeg."""+ """Extract frames from a video file using ffmpeg.++ Without explicit frame_indices, frames are sampled by TIMESTAMP across+ the whole clip. The obvious spelling, `-vf thumbnail=N -frames:v N`,+ looks like even sampling and is not: thumbnail=N picks the most+ representative frame out of each consecutive N-frame batch, so paired+ with -frames:v N it stops after the opening seconds and never sees the+ rest. A four-second clip that is solid red for 2s then solid blue for+ 2s yielded four red frames, and the caption pass then described it as+ a video in which nothing changes.+ """from PIL import Imagewith tempfile.TemporaryDirectory() as tmp_dir:tmp = Path(tmp_dir)if frame_indices:- # Extract specific frames using select filter+ # Explicit indices: the select filter is exactly right here.frames_to_extract = frame_indices[:max_frames]select_expr = "+".join(f"eq(n\\,{idx})" for idx in frames_to_extract)- cmd = [- "ffmpeg", "-i", str(video_path),- "-vf", f"select='{select_expr}'",- "-vsync", "vfr",- str(tmp / "frame_%04d.png"),- "-y", "-loglevel", "error",- ]+ subprocess.run(+ ["ffmpeg", "-i", str(video_path),+ "-vf", f"select='{select_expr}'",+ "-vsync", "vfr",+ str(tmp / "frame_%04d.png"),+ "-y", "-loglevel", "error"],+ capture_output=True, text=True, timeout=60,+ )+ elif (duration := self._video_duration_seconds(video_path)) is not None:+ # Land inside each 1/N slice rather than on its edge, so a cut+ # exactly on a boundary does not sample the frame before or+ # after it depending on rounding.+ for index in range(max_frames):+ stamp = duration * (index + 0.5) / max_frames+ # -ss ahead of -i seeks by keyframe: fast, and accurate+ # enough for sampling, which is not frame-exact anyway.+ subprocess.run(+ ["ffmpeg", "-ss", f"{stamp:.3f}", "-i", str(video_path),+ "-frames:v", "1",+ str(tmp / f"frame_{index:04d}.png"),+ "-y", "-loglevel", "error"],+ capture_output=True, text=True, timeout=60,+ )else:- # Get total frame count first- probe_cmd = [- "ffmpeg", "-i", str(video_path),- "-map", "0:v:0", "-c", "copy", "-f", "null", "-",- ]- # Sample at even intervals using fps filter- # Use a select filter that picks frames at even intervals- cmd = [- "ffmpeg", "-i", str(video_path),- "-frames:v", str(max_frames),- "-vf", f"thumbnail={max_frames}",- str(tmp / "frame_%04d.png"),- "-y", "-loglevel", "error",- ]-- subprocess.run(cmd, capture_output=True, text=True, timeout=60)+ # No duration — a stream, or no ffprobe on PATH. Fall back to+ # an even pass over the file rather than returning nothing.+ subprocess.run(+ ["ffmpeg", "-i", str(video_path),+ "-vsync", "vfr", "-frames:v", str(max_frames),+ str(tmp / "frame_%04d.png"),+ "-y", "-loglevel", "error"],+ capture_output=True, text=True, timeout=60,+ )# Load extracted framesframe_files = sorted(tmp.glob("frame_*.png"))