Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: timestamp every other sampled video frame for GGUF chat (#14254)

Daniel Hanf7ec00f5f75a
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 4b582dc77eb..706dbde8cce 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -6031,21 +6031,41 @@ _MMPROJ_OFFLOAD_NEG_ENV_VAR = "LLAMA_ARG_NO_MMPROJ_OFFLOAD"
# Gemma 4 expects video at up to 1 fps; llama-server defaults to 4.
_LLAMA_VIDEO_FPS = "1"
_VIDEO_FPS_ENV_VAR = "LLAMA_ARG_VIDEO_FPS"
+_LLAMA_SERVER_DEFAULT_VIDEO_FPS = 4.0
+_VIDEO_TIMESTAMP_ENV_VAR = "LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL"
def _video_fps_flags(
- server_caps: Mapping[str, object], env: Optional[Mapping[str, str]] = None
+ server_caps: Mapping[str, object],
+ env: Optional[Mapping[str, str]] = None,
+ extra_args: Optional[Sequence[str]] = None,
) -> list[str]:
- """Return the default FPS flag unless unsupported or set in the environment.
+ """Return the default FPS and timestamp flags unless unsupported or set by the user.
User extras are appended later and retain their last-wins override.
"""
- if not server_caps.get("supports_video_fps"):
- return []
source = os.environ if env is None else env
- if source.get(_VIDEO_FPS_ENV_VAR) is not None:
- return []
- return ["--video-fps", _LLAMA_VIDEO_FPS]
+ flags: list[str] = []
+ if server_caps.get("supports_video_fps") and source.get(_VIDEO_FPS_ENV_VAR) is None:
+ flags += ["--video-fps", _LLAMA_VIDEO_FPS]
+ if (
+ server_caps.get("supports_video_timestamp_interval")
+ and source.get(_VIDEO_TIMESTAMP_ENV_VAR) is None
+ and not _extra_args_set_any_flag(extra_args, {"--video-timestamp-interval"})
+ ):
+ # llama-server writes a timestamp only every 5000 ms by default, so the model snaps
+ # every event to a multiple of 5 s. One label per 2 sampled frames: Qwen-VL merges
+ # frame pairs and a label between them splits the pair (about 1.7x the prompt), and
+ # Gemma 4 placed events best at this spacing. Floor, so drift never skips a label.
+ fps = requested_video_fps(extra_args, source)
+ if fps is None:
+ fps = (
+ float(_LLAMA_VIDEO_FPS)
+ if server_caps.get("supports_video_fps")
+ else _LLAMA_SERVER_DEFAULT_VIDEO_FPS
+ )
+ flags += ["--video-timestamp-interval", str(max(1, int(2000 / fps)))]
+ return flags
def requested_video_fps(
@@ -10525,6 +10545,7 @@ class LlamaCppBackend:
"supports_no_mmproj_offload": False,
"supports_trace_verbosity": False,
"supports_video_fps": False,
+ "supports_video_timestamp_interval": False,
"supports_load_mode": False,
"supports_moe_cache": False,
"supports_moe_cache_auto": False,
@@ -10584,6 +10605,7 @@ class LlamaCppBackend:
supports_reasoning_flag = False
supports_no_mmproj_offload = False
supports_video_fps = False
+ supports_video_timestamp_interval = False
supports_load_mode = False
supports_moe_cache = False
supports_moe_cache_auto = False
@@ -10834,6 +10856,7 @@ class LlamaCppBackend:
supports_reasoning_flag = _is_real("--reasoning")
supports_no_mmproj_offload = _is_real("--no-mmproj-offload")
supports_video_fps = _is_real("--video-fps")
+ supports_video_timestamp_interval = _is_real("--video-timestamp-interval")
# --load-mode supersedes --mlock / --no-mmap, which are deprecated.
# Pre-initialised above: a failed probe must fall back, not raise.
supports_load_mode = _is_real("--load-mode")
@@ -10949,6 +10972,7 @@ class LlamaCppBackend:
"supports_no_mmproj_offload": supports_no_mmproj_offload,
"supports_trace_verbosity": supports_trace_verbosity,
"supports_video_fps": supports_video_fps,
+ "supports_video_timestamp_interval": supports_video_timestamp_interval,
"supports_load_mode": supports_load_mode,
"supports_moe_cache": supports_moe_cache,
"supports_moe_cache_auto": supports_moe_cache_auto,
@@ -30335,7 +30359,7 @@ class LlamaCppBackend:
env = self._llama_server_env_for_binary(binary)
# On every launch, not only where Studio emits --mmproj: a projector
# from the extras or LLAMA_ARG_MMPROJ takes video too.
- cmd.extend(_video_fps_flags(server_caps))
+ cmd.extend(_video_fps_flags(server_caps, extra_args = extra_args))
# Report a clean public model id (matching GET /v1/models) rather
# than the raw -m path in llama-server's own /v1/models and the
diff --git a/studio/backend/tests/test_llama_video_frame_budget.py b/studio/backend/tests/test_llama_video_frame_budget.py
index 89ef7750b44..f55d23a82fe 100644
--- a/studio/backend/tests/test_llama_video_frame_budget.py
+++ b/studio/backend/tests/test_llama_video_frame_budget.py
@@ -133,6 +133,104 @@ def test_the_probe_reads_the_flag_from_help(monkeypatch, tmp_path):
assert without["supports_video_fps"] is False
+def _timestamp_launch(
+ tmp_path,
+ monkeypatch,
+ extra_args = None,
+ **env,
+):
+ for name in ("LLAMA_ARG_VIDEO_FPS", "LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL"):
+ monkeypatch.delenv(name, raising = False)
+ for name, value in env.items():
+ monkeypatch.setenv(name, value)
+ backend, gguf = _video_backend(
+ tmp_path, supports_video_fps = True, supports_video_timestamp_interval = True
+ )
+ return _launch(backend, gguf, extra_args = extra_args)["cmd"]
+
+
+def _last_value(cmd, flag):
+ positions = [i for i, token in enumerate(cmd) if token == flag]
+ return cmd[positions[-1] + 1] if positions else None
+
+
+def test_every_other_sampled_frame_gets_a_timestamp(tmp_path, monkeypatch):
+ cmd = _timestamp_launch(tmp_path, monkeypatch)
+
+ assert cmd.count("--video-timestamp-interval") == 1
+ assert _last_value(cmd, "--video-timestamp-interval") == "2000"
+
+
+def test_the_timestamp_interval_follows_a_hand_typed_frame_rate(tmp_path, monkeypatch):
+ cmd = _timestamp_launch(tmp_path, monkeypatch, extra_args = ["--video-fps", "2"])
+
+ assert _last_value(cmd, "--video-fps") == "2"
+ assert _last_value(cmd, "--video-timestamp-interval") == "1000"
+
+
+def test_the_timestamp_interval_follows_an_inherited_frame_rate(tmp_path, monkeypatch):
+ cmd = _timestamp_launch(tmp_path, monkeypatch, LLAMA_ARG_VIDEO_FPS = "3")
+
+ assert "--video-fps" not in cmd
+ assert _last_value(cmd, "--video-timestamp-interval") == "666"
+
+
+def test_a_hand_typed_timestamp_interval_is_left_alone(tmp_path, monkeypatch):
+ cmd = _timestamp_launch(
+ tmp_path, monkeypatch, extra_args = ["--video-timestamp-interval", "5000"]
+ )
+
+ assert cmd.count("--video-timestamp-interval") == 1
+ assert _last_value(cmd, "--video-timestamp-interval") == "5000"
+
+
+def test_an_inherited_timestamp_interval_is_not_overridden(tmp_path, monkeypatch):
+ cmd = _timestamp_launch(tmp_path, monkeypatch, LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL = "2000")
+
+ assert "--video-timestamp-interval" not in cmd
+
+
+def test_a_build_without_the_timestamp_flag_is_not_sent_it(tmp_path, monkeypatch):
+ monkeypatch.delenv("LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL", raising = False)
+ backend, gguf = _video_backend(tmp_path, supports_video_fps = True)
+
+ assert "--video-timestamp-interval" not in _launch(backend, gguf)["cmd"]
+
+
+def test_the_timestamp_lands_on_every_other_frame():
+ """llama-server labels frame k when k / fps * 1000 reaches a threshold that grows by
+ the interval per label. Even frames only keeps Qwen-VL's frame pairs whole."""
+ caps = {"supports_video_fps": True, "supports_video_timestamp_interval": True}
+ for fps in ("1", "2", "3", "4", "0.5", "7"):
+ flags = _video_fps_flags(caps, env = {}, extra_args = ["--video-fps", fps])
+ interval = float(flags[flags.index("--video-timestamp-interval") + 1])
+ threshold, labelled = 0.0, []
+ for k in range(120):
+ if k / float(fps) * 1000.0 >= threshold:
+ labelled.append(k)
+ threshold += interval
+ assert labelled == list(range(0, 120, 2)), fps
+ old_build = {"supports_video_timestamp_interval": True}
+ assert _video_fps_flags(old_build, env = {}) == ["--video-timestamp-interval", "500"]
+
+
+def test_the_probe_reads_the_timestamp_flag_from_help(monkeypatch, tmp_path):
+ with_flag = _probe_with_help(
+ monkeypatch,
+ tmp_path,
+ "--video-timestamp-interval N interval in milliseconds between text "
+ "timestamps (default: 5000)\n",
+ )
+ assert with_flag["supports_video_timestamp_interval"] is True
+
+ older = tmp_path / "older"
+ older.mkdir()
+ without = _probe_with_help(
+ monkeypatch, older, "--video-fps N target video frame rate\n"
+ )
+ assert without["supports_video_timestamp_interval"] is False
+
+
def _clip(
tmp_path: Path,
name: str,