Skip to content

World's first open-source, agentic video production system. 12 production pipelines, 100+ tools, 700+ agent skill and production-knowledge files. Turn your AI coding assistant into a full video production studio.

Fix video_understand sampling only the opening seconds of a clip

Daniel Liu45f45855da1f
diff --git a/tests/tools/test_video_understand_sampling.py b/tests/tools/test_video_understand_sampling.py
new file mode 100644
index 00000000..9afce86b
--- /dev/null
+++ b/tests/tools/test_video_understand_sampling.py
@@ -0,0 +1,121 @@
+"""Frame sampling in video_understand.
+
+No network and no ffmpeg: subprocess.run is faked, so these assert the commands
+the tool builds rather than what a particular ffmpeg build happens to return.
+"""
+
+from __future__ import annotations
+
+import types
+from pathlib import Path
+
+import pytest
+
+from tools.analysis.video_understand import VideoUnderstand
+
+
+class FakePIL:
+ """Stand-in for PIL.Image.open(...).convert('RGB')."""
+
+ def __init__(self, path):
+ self.path = path
+
+ def convert(self, _mode):
+ return self
+
+
+def fake_ffmpeg(monkeypatch):
+ """Record every ffmpeg invocation and materialise the files it promises."""
+ calls: list[list[str]] = []
+
+ def fake_run(cmd, **kwargs):
+ calls.append(list(cmd))
+ if cmd and cmd[0] == "ffmpeg":
+ target = Path(cmd[-4])
+ if "%04d" in target.name:
+ for index in range(3):
+ target.with_name(target.name.replace("%04d", f"{index:04d}")).write_bytes(b"x")
+ else:
+ target.write_bytes(b"x")
+ return types.SimpleNamespace(returncode=0, stdout="", stderr="")
+
+ monkeypatch.setattr("tools.analysis.video_understand.subprocess.run", fake_run)
+
+ fake_image_module = types.SimpleNamespace(open=FakePIL)
+ monkeypatch.setitem(
+ __import__("sys").modules, "PIL",
+ types.SimpleNamespace(Image=fake_image_module),
+ )
+ return calls
+
+
+def _seek_positions(calls: list[list[str]]) -> list[float]:
+ return [float(cmd[cmd.index("-ss") + 1]) for cmd in calls if "-ss" in cmd]
+
+
+def test_samples_span_the_whole_clip(monkeypatch, fake_ffmpeg, tmp_path):
+ """The regression this file exists for.
+
+ `-vf thumbnail=N -frames:v N` reads as even sampling and is not: thumbnail=N
+ picks the most representative frame out of each consecutive N-frame batch,
+ so it never looks past the opening seconds. A four-second clip that is
+ solid red for 2s then solid blue for 2s came back as four red frames, and
+ the caption pass called it a video in which nothing changes.
+ """
+ monkeypatch.setattr(
+ VideoUnderstand, "_video_duration_seconds", staticmethod(lambda _p: 100.0)
+ )
+
+ frames = VideoUnderstand()._extract_video_frames(tmp_path / "clip.mp4", None, 4)
+
+ assert len(frames) == 4
+ assert _seek_positions(fake_ffmpeg) == [12.5, 37.5, 62.5, 87.5]
+ # The property the batching bug violated: the last sample is near the end.
+ assert _seek_positions(fake_ffmpeg)[-1] > 75.0
+ assert not any("thumbnail=" in " ".join(cmd) for cmd in fake_ffmpeg)
+
+
+def test_explicit_frame_indices_still_use_the_select_filter(fake_ffmpeg, tmp_path):
+ """That branch was already correct and must not change."""
+ VideoUnderstand()._extract_video_frames(tmp_path / "clip.mp4", [3, 9, 27], 3)
+
+ joined = " ".join(fake_ffmpeg[0])
+ assert "select=" in joined
+ assert "eq(n\\,3)" in joined and "eq(n\\,27)" in joined
+ assert "-ss" not in fake_ffmpeg[0]
+
+
+def test_falls_back_when_duration_is_unknown(monkeypatch, fake_ffmpeg, tmp_path):
+ """A stream, or no ffprobe on PATH, must still yield frames."""
+ monkeypatch.setattr(
+ VideoUnderstand, "_video_duration_seconds", staticmethod(lambda _p: None)
+ )
+
+ frames = VideoUnderstand()._extract_video_frames(tmp_path / "stream.mp4", None, 3)
+
+ assert frames, "the fallback must produce frames, not an empty list"
+ assert not _seek_positions(fake_ffmpeg), "no duration means no seek positions"
+
+
[email protected]("probe_output, expected", [
+ ("12.5\n", 12.5),
+ ("N/A\n", None),
+ ("", None),
+ ("0\n", None), # a zero duration would divide the sampler by zero
+ ("-3\n", None),
+])
+def test_duration_probe_rejects_unusable_values(monkeypatch, tmp_path, probe_output, expected):
+ monkeypatch.setattr(
+ "tools.analysis.video_understand.subprocess.run",
+ lambda *a, **k: types.SimpleNamespace(returncode=0, stdout=probe_output, stderr=""),
+ )
+ assert VideoUnderstand._video_duration_seconds(tmp_path / "c.mp4") == expected
+
+
+def test_duration_probe_survives_a_missing_ffprobe(monkeypatch, tmp_path):
+ def boom(*_a, **_k):
+ raise FileNotFoundError("ffprobe")
+
+ monkeypatch.setattr("tools.analysis.video_understand.subprocess.run", boom)
+ assert VideoUnderstand._video_duration_seconds(tmp_path / "c.mp4") is None
diff --git a/tools/analysis/video_understand.py b/tools/analysis/video_understand.py
index 50027593..1cc9c362 100644
--- a/tools/analysis/video_understand.py
+++ b/tools/analysis/video_understand.py
@@ -252,48 +252,81 @@ class VideoUnderstand(BaseTool):
return self._extract_video_frames(input_path, frame_indices, max_frames)
+ @staticmethod
+ def _video_duration_seconds(video_path: Path) -> float | None:
+ """Duration via ffprobe, or None when it cannot be determined."""
+ try:
+ out = subprocess.run(
+ ["ffprobe", "-v", "error", "-show_entries", "format=duration",
+ "-of", "default=noprint_wrappers=1:nokey=1", str(video_path)],
+ capture_output=True, text=True, timeout=30,
+ ).stdout.strip()
+ duration = float(out)
+ return duration if duration > 0 else None
+ except (ValueError, OSError, subprocess.SubprocessError):
+ return None
+
def _extract_video_frames(
self,
video_path: Path,
frame_indices: list[int] | None,
max_frames: int,
) -> list:
- """Extract frames from a video file using ffmpeg."""
+ """Extract frames from a video file using ffmpeg.
+
+ Without explicit frame_indices, frames are sampled by TIMESTAMP across
+ the whole clip. The obvious spelling, `-vf thumbnail=N -frames:v N`,
+ looks like even sampling and is not: thumbnail=N picks the most
+ representative frame out of each consecutive N-frame batch, so paired
+ with -frames:v N it stops after the opening seconds and never sees the
+ rest. A four-second clip that is solid red for 2s then solid blue for
+ 2s yielded four red frames, and the caption pass then described it as
+ a video in which nothing changes.
+ """
from PIL import Image
with tempfile.TemporaryDirectory() as tmp_dir:
tmp = Path(tmp_dir)
if frame_indices:
- # Extract specific frames using select filter
+ # Explicit indices: the select filter is exactly right here.
frames_to_extract = frame_indices[:max_frames]
select_expr = "+".join(
f"eq(n\\,{idx})" for idx in frames_to_extract
)
- cmd = [
- "ffmpeg", "-i", str(video_path),
- "-vf", f"select='{select_expr}'",
- "-vsync", "vfr",
- str(tmp / "frame_%04d.png"),
- "-y", "-loglevel", "error",
- ]
+ subprocess.run(
+ ["ffmpeg", "-i", str(video_path),
+ "-vf", f"select='{select_expr}'",
+ "-vsync", "vfr",
+ str(tmp / "frame_%04d.png"),
+ "-y", "-loglevel", "error"],
+ capture_output=True, text=True, timeout=60,
+ )
+ elif (duration := self._video_duration_seconds(video_path)) is not None:
+ # Land inside each 1/N slice rather than on its edge, so a cut
+ # exactly on a boundary does not sample the frame before or
+ # after it depending on rounding.
+ for index in range(max_frames):
+ stamp = duration * (index + 0.5) / max_frames
+ # -ss ahead of -i seeks by keyframe: fast, and accurate
+ # enough for sampling, which is not frame-exact anyway.
+ subprocess.run(
+ ["ffmpeg", "-ss", f"{stamp:.3f}", "-i", str(video_path),
+ "-frames:v", "1",
+ str(tmp / f"frame_{index:04d}.png"),
+ "-y", "-loglevel", "error"],
+ capture_output=True, text=True, timeout=60,
+ )
else:
- # Get total frame count first
- probe_cmd = [
- "ffmpeg", "-i", str(video_path),
- "-map", "0:v:0", "-c", "copy", "-f", "null", "-",
- ]
- # Sample at even intervals using fps filter
- # Use a select filter that picks frames at even intervals
- cmd = [
- "ffmpeg", "-i", str(video_path),
- "-frames:v", str(max_frames),
- "-vf", f"thumbnail={max_frames}",
- str(tmp / "frame_%04d.png"),
- "-y", "-loglevel", "error",
- ]
-
- subprocess.run(cmd, capture_output=True, text=True, timeout=60)
+ # No duration — a stream, or no ffprobe on PATH. Fall back to
+ # an even pass over the file rather than returning nothing.
+ subprocess.run(
+ ["ffmpeg", "-i", str(video_path),
+ "-vsync", "vfr", "-frames:v", str(max_frames),
+ str(tmp / "frame_%04d.png"),
+ "-y", "-loglevel", "error"],
+ capture_output=True, text=True, timeout=60,
+ )
# Load extracted frames
frame_files = sorted(tmp.glob("frame_*.png"))