Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: timestamp every other sampled video frame for GGUF chat (#14254)
Daniel Hanf7ec00f5f75a
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 4b582dc77eb..706dbde8cce 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -6031,21 +6031,41 @@ _MMPROJ_OFFLOAD_NEG_ENV_VAR = "LLAMA_ARG_NO_MMPROJ_OFFLOAD"# Gemma 4 expects video at up to 1 fps; llama-server defaults to 4._LLAMA_VIDEO_FPS = "1"_VIDEO_FPS_ENV_VAR = "LLAMA_ARG_VIDEO_FPS"+_LLAMA_SERVER_DEFAULT_VIDEO_FPS = 4.0+_VIDEO_TIMESTAMP_ENV_VAR = "LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL"def _video_fps_flags(- server_caps: Mapping[str, object], env: Optional[Mapping[str, str]] = None+ server_caps: Mapping[str, object],+ env: Optional[Mapping[str, str]] = None,+ extra_args: Optional[Sequence[str]] = None,) -> list[str]:- """Return the default FPS flag unless unsupported or set in the environment.+ """Return the default FPS and timestamp flags unless unsupported or set by the user.User extras are appended later and retain their last-wins override."""- if not server_caps.get("supports_video_fps"):- return []source = os.environ if env is None else env- if source.get(_VIDEO_FPS_ENV_VAR) is not None:- return []- return ["--video-fps", _LLAMA_VIDEO_FPS]+ flags: list[str] = []+ if server_caps.get("supports_video_fps") and source.get(_VIDEO_FPS_ENV_VAR) is None:+ flags += ["--video-fps", _LLAMA_VIDEO_FPS]+ if (+ server_caps.get("supports_video_timestamp_interval")+ and source.get(_VIDEO_TIMESTAMP_ENV_VAR) is None+ and not _extra_args_set_any_flag(extra_args, {"--video-timestamp-interval"})+ ):+ # llama-server writes a timestamp only every 5000 ms by default, so the model snaps+ # every event to a multiple of 5 s. One label per 2 sampled frames: Qwen-VL merges+ # frame pairs and a label between them splits the pair (about 1.7x the prompt), and+ # Gemma 4 placed events best at this spacing. Floor, so drift never skips a label.+ fps = requested_video_fps(extra_args, source)+ if fps is None:+ fps = (+ float(_LLAMA_VIDEO_FPS)+ if server_caps.get("supports_video_fps")+ else _LLAMA_SERVER_DEFAULT_VIDEO_FPS+ )+ flags += ["--video-timestamp-interval", str(max(1, int(2000 / fps)))]+ return flagsdef requested_video_fps(@@ -10525,6 +10545,7 @@ class LlamaCppBackend:"supports_no_mmproj_offload": False,"supports_trace_verbosity": False,"supports_video_fps": False,+ "supports_video_timestamp_interval": False,"supports_load_mode": False,"supports_moe_cache": False,"supports_moe_cache_auto": False,@@ -10584,6 +10605,7 @@ class LlamaCppBackend:supports_reasoning_flag = Falsesupports_no_mmproj_offload = Falsesupports_video_fps = False+ supports_video_timestamp_interval = Falsesupports_load_mode = Falsesupports_moe_cache = Falsesupports_moe_cache_auto = False@@ -10834,6 +10856,7 @@ class LlamaCppBackend:supports_reasoning_flag = _is_real("--reasoning")supports_no_mmproj_offload = _is_real("--no-mmproj-offload")supports_video_fps = _is_real("--video-fps")+ supports_video_timestamp_interval = _is_real("--video-timestamp-interval")# --load-mode supersedes --mlock / --no-mmap, which are deprecated.# Pre-initialised above: a failed probe must fall back, not raise.supports_load_mode = _is_real("--load-mode")@@ -10949,6 +10972,7 @@ class LlamaCppBackend:"supports_no_mmproj_offload": supports_no_mmproj_offload,"supports_trace_verbosity": supports_trace_verbosity,"supports_video_fps": supports_video_fps,+ "supports_video_timestamp_interval": supports_video_timestamp_interval,"supports_load_mode": supports_load_mode,"supports_moe_cache": supports_moe_cache,"supports_moe_cache_auto": supports_moe_cache_auto,@@ -30335,7 +30359,7 @@ class LlamaCppBackend:env = self._llama_server_env_for_binary(binary)# On every launch, not only where Studio emits --mmproj: a projector# from the extras or LLAMA_ARG_MMPROJ takes video too.- cmd.extend(_video_fps_flags(server_caps))+ cmd.extend(_video_fps_flags(server_caps, extra_args = extra_args))# Report a clean public model id (matching GET /v1/models) rather# than the raw -m path in llama-server's own /v1/models and thediff --git a/studio/backend/tests/test_llama_video_frame_budget.py b/studio/backend/tests/test_llama_video_frame_budget.pyindex 89ef7750b44..f55d23a82fe 100644--- a/studio/backend/tests/test_llama_video_frame_budget.py+++ b/studio/backend/tests/test_llama_video_frame_budget.py@@ -133,6 +133,104 @@ def test_the_probe_reads_the_flag_from_help(monkeypatch, tmp_path):assert without["supports_video_fps"] is False+def _timestamp_launch(+ tmp_path,+ monkeypatch,+ extra_args = None,+ **env,+):+ for name in ("LLAMA_ARG_VIDEO_FPS", "LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL"):+ monkeypatch.delenv(name, raising = False)+ for name, value in env.items():+ monkeypatch.setenv(name, value)+ backend, gguf = _video_backend(+ tmp_path, supports_video_fps = True, supports_video_timestamp_interval = True+ )+ return _launch(backend, gguf, extra_args = extra_args)["cmd"]+++def _last_value(cmd, flag):+ positions = [i for i, token in enumerate(cmd) if token == flag]+ return cmd[positions[-1] + 1] if positions else None+++def test_every_other_sampled_frame_gets_a_timestamp(tmp_path, monkeypatch):+ cmd = _timestamp_launch(tmp_path, monkeypatch)++ assert cmd.count("--video-timestamp-interval") == 1+ assert _last_value(cmd, "--video-timestamp-interval") == "2000"+++def test_the_timestamp_interval_follows_a_hand_typed_frame_rate(tmp_path, monkeypatch):+ cmd = _timestamp_launch(tmp_path, monkeypatch, extra_args = ["--video-fps", "2"])++ assert _last_value(cmd, "--video-fps") == "2"+ assert _last_value(cmd, "--video-timestamp-interval") == "1000"+++def test_the_timestamp_interval_follows_an_inherited_frame_rate(tmp_path, monkeypatch):+ cmd = _timestamp_launch(tmp_path, monkeypatch, LLAMA_ARG_VIDEO_FPS = "3")++ assert "--video-fps" not in cmd+ assert _last_value(cmd, "--video-timestamp-interval") == "666"+++def test_a_hand_typed_timestamp_interval_is_left_alone(tmp_path, monkeypatch):+ cmd = _timestamp_launch(+ tmp_path, monkeypatch, extra_args = ["--video-timestamp-interval", "5000"]+ )++ assert cmd.count("--video-timestamp-interval") == 1+ assert _last_value(cmd, "--video-timestamp-interval") == "5000"+++def test_an_inherited_timestamp_interval_is_not_overridden(tmp_path, monkeypatch):+ cmd = _timestamp_launch(tmp_path, monkeypatch, LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL = "2000")++ assert "--video-timestamp-interval" not in cmd+++def test_a_build_without_the_timestamp_flag_is_not_sent_it(tmp_path, monkeypatch):+ monkeypatch.delenv("LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL", raising = False)+ backend, gguf = _video_backend(tmp_path, supports_video_fps = True)++ assert "--video-timestamp-interval" not in _launch(backend, gguf)["cmd"]+++def test_the_timestamp_lands_on_every_other_frame():+ """llama-server labels frame k when k / fps * 1000 reaches a threshold that grows by+ the interval per label. Even frames only keeps Qwen-VL's frame pairs whole."""+ caps = {"supports_video_fps": True, "supports_video_timestamp_interval": True}+ for fps in ("1", "2", "3", "4", "0.5", "7"):+ flags = _video_fps_flags(caps, env = {}, extra_args = ["--video-fps", fps])+ interval = float(flags[flags.index("--video-timestamp-interval") + 1])+ threshold, labelled = 0.0, []+ for k in range(120):+ if k / float(fps) * 1000.0 >= threshold:+ labelled.append(k)+ threshold += interval+ assert labelled == list(range(0, 120, 2)), fps+ old_build = {"supports_video_timestamp_interval": True}+ assert _video_fps_flags(old_build, env = {}) == ["--video-timestamp-interval", "500"]+++def test_the_probe_reads_the_timestamp_flag_from_help(monkeypatch, tmp_path):+ with_flag = _probe_with_help(+ monkeypatch,+ tmp_path,+ "--video-timestamp-interval N interval in milliseconds between text "+ "timestamps (default: 5000)\n",+ )+ assert with_flag["supports_video_timestamp_interval"] is True++ older = tmp_path / "older"+ older.mkdir()+ without = _probe_with_help(+ monkeypatch, older, "--video-fps N target video frame rate\n"+ )+ assert without["supports_video_timestamp_interval"] is False++def _clip(tmp_path: Path,name: str,