Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep a still-advancing llama.cpp prefill from timing out (#10172)
oobabooga4e64d1729617
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex fbea7138785..ecb4bc92e36 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -27581,18 +27581,38 @@ class LlamaCppBackend:yield response, first_token_deadline@staticmethod- def _sse_event_has_generated_output(event: str) -> bool:- """Return true when a complete SSE event carries model-generated output."""+ def _sse_event_payload(event: str) -> Optional[dict]:+ """Decode one complete SSE event's ``data:`` payload, or None."""payload_lines = [line[5:].lstrip() for line in event.splitlines() if line.startswith("data:")]if not payload_lines:- return False+ return Nonetry:data = json.loads("\n".join(payload_lines))except (TypeError, json.JSONDecodeError):- return False- if not isinstance(data, dict):+ return None+ return data if isinstance(data, dict) else None++ @staticmethod+ def _sse_event_prefill_progress(event: str) -> Optional[float]:+ """Return the reported processed prompt tokens, if present."""+ data = LlamaCppBackend._sse_event_payload(event)+ if data is None:+ return None+ progress = data.get("prompt_progress")+ if not isinstance(progress, dict):+ return None+ try:+ return float(progress.get("processed"))+ except (TypeError, ValueError):+ return None++ @staticmethod+ def _sse_event_has_generated_output(event: str) -> bool:+ """Return true when a complete SSE event carries model-generated output."""+ data = LlamaCppBackend._sse_event_payload(event)+ if data is None:return Falseif data.get("type") == "diffusion_frame":return True@@ -27624,6 +27644,7 @@ class LlamaCppBackend:if first_token_deadline is None:first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_Slast_chunk_at: Optional[float] = None+ last_prefill_progress: Optional[float] = Noneprefill_sse_buffer = ""while True:if cancel_event is not None and cancel_event.is_set():@@ -27646,6 +27667,13 @@ class LlamaCppBackend:starts_output = Trueprefill_sse_buffer = ""break+ # Renew only on increasing progress so repeated events still time out.+ processed = LlamaCppBackend._sse_event_prefill_progress(event)+ if processed is not None and (+ last_prefill_progress is None or processed > last_prefill_progress+ ):+ last_prefill_progress = processed+ first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_Sif chunk and starts_output:if last_chunk_at is None and post_first_chunk_read_timeout_s is not None:LlamaCppBackend._set_stream_read_timeout(@@ -28056,8 +28084,9 @@ class LlamaCppBackend:retry_max_tokens = max_tokensretry_context_overflow = context_overflowretry_preflight_context_length = None+ # Progress events let advancing prefills renew the first-token deadline.+ payload["return_progress"] = Trueif perf_callback is not None:- payload["return_progress"] = Truepayload["timings_per_token"] = Trueif logit_bias:payload["logit_bias"] = logit_bias@@ -29125,8 +29154,9 @@ class LlamaCppBackend:"frequency_penalty": frequency_penalty,}+ # Progress events feed the first-token deadline; timings stay opt-in.+ payload["return_progress"] = Trueif perf_callback is not None:- payload["return_progress"] = Truepayload["timings_per_token"] = Trueif logit_bias:payload["logit_bias"] = logit_bias@@ -31630,8 +31660,9 @@ class LlamaCppBackend:_apply_seeded_llama_request(stream_payload, seed)stream_payload["stream_options"] = {"include_usage": True}+ # Progress events feed the first-token deadline; timings stay opt-in.+ stream_payload["return_progress"] = Trueif perf_callback is not None:- stream_payload["return_progress"] = Truestream_payload["timings_per_token"] = True_final_respawn_truncations: list[dict] = []diff --git a/studio/backend/tests/test_llama_cpp_stall_timeout.py b/studio/backend/tests/test_llama_cpp_stall_timeout.pyindex 044d15b82be..37f214f13e0 100644--- a/studio/backend/tests/test_llama_cpp_stall_timeout.py+++ b/studio/backend/tests/test_llama_cpp_stall_timeout.py@@ -31,6 +31,7 @@ sys.modules.setdefault("loggers", _loggers_stub)sys.modules.setdefault("structlog", _types.ModuleType("structlog"))import httpcore # noqa: E402+import httpx # noqa: E402from core.inference import llama_cpp as llama_cpp_mod # noqa: E402from core.inference.llama_cpp import LlamaCppBackend # noqa: E402@@ -137,6 +138,72 @@ def test_prompt_progress_keeps_the_prefill_read_timeout():iterator.close()+_OUTPUT_EVENT = 'data: {"choices":[{"delta":{"content":"x"}}]}\n\n'+++def _progress_event(processed: int) -> str:+ return (+ 'data: {"choices":[{"delta":{"role":"assistant","content":null}}],'+ f'"prompt_progress":{{"processed":{processed},"cache":0,"time_ms":1}}}}\n\n'+ )+++def _run(events, clock):+ """Drain _iter_text_cancellable over ``events``, 200s of fake time apart."""++ class Response:+ request = _types.SimpleNamespace(extensions = {"timeout": {"read": _PREFILL_TIMEOUT}})++ @staticmethod+ def iter_text():+ for event in events:+ clock["t"] += 200.0+ yield event++ @staticmethod+ def close():+ return None++ return list(+ LlamaCppBackend._iter_text_cancellable(+ Response(),+ first_token_deadline = clock["t"] + _PREFILL_TIMEOUT,+ post_first_chunk_read_timeout_s = _STALL_TIMEOUT,+ )+ )+++def test_advancing_prefill_progress_extends_the_first_token_deadline(monkeypatch):+ """A prompt that keeps prefilling past the window is not a hang (#9037)."""+ clock = {"t": 0.0}+ monkeypatch.setattr(llama_cpp_mod.time, "monotonic", lambda: clock["t"])++ # Progress every 200s keeps a 4000s prefill alive.+ events = [_progress_event(i * 4096) for i in range(1, 21)] + [_OUTPUT_EVENT]++ assert _run(events, clock)[-1] == _OUTPUT_EVENT+ assert clock["t"] > _PREFILL_TIMEOUT+++def test_repeated_prefill_progress_still_times_out(monkeypatch):+ """A server replaying one progress value is stuck, not prefilling."""+ clock = {"t": 0.0}+ monkeypatch.setattr(llama_cpp_mod.time, "monotonic", lambda: clock["t"])++ with pytest.raises(httpx.ReadTimeout):+ _run([_progress_event(4096)] * 20 + [_OUTPUT_EVENT], clock)+++def test_prefill_without_progress_events_still_times_out(monkeypatch):+ """Content-free events alone are not proof of life: the window still applies."""+ clock = {"t": 0.0}+ monkeypatch.setattr(llama_cpp_mod.time, "monotonic", lambda: clock["t"])++ keepalive = 'data: {"choices":[{"delta":{"role":"assistant","content":null}}]}\n\n'+ with pytest.raises(httpx.ReadTimeout):+ _run([keepalive] * 20 + [_OUTPUT_EVENT], clock)++def test_prefill_timeout_used_when_no_live_override(monkeypatch):"""Without a lowered live timeout, the wrapper honors the passed prefill timeout, so the normal first-token wait is unchanged."""clock = {"t": 0.0}