Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep a still-advancing llama.cpp prefill from timing out (#10172)

oobabooga4e64d1729617
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index fbea7138785..ecb4bc92e36 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -27581,18 +27581,38 @@ class LlamaCppBackend:
yield response, first_token_deadline
@staticmethod
- def _sse_event_has_generated_output(event: str) -> bool:
- """Return true when a complete SSE event carries model-generated output."""
+ def _sse_event_payload(event: str) -> Optional[dict]:
+ """Decode one complete SSE event's ``data:`` payload, or None."""
payload_lines = [
line[5:].lstrip() for line in event.splitlines() if line.startswith("data:")
]
if not payload_lines:
- return False
+ return None
try:
data = json.loads("\n".join(payload_lines))
except (TypeError, json.JSONDecodeError):
- return False
- if not isinstance(data, dict):
+ return None
+ return data if isinstance(data, dict) else None
+
+ @staticmethod
+ def _sse_event_prefill_progress(event: str) -> Optional[float]:
+ """Return the reported processed prompt tokens, if present."""
+ data = LlamaCppBackend._sse_event_payload(event)
+ if data is None:
+ return None
+ progress = data.get("prompt_progress")
+ if not isinstance(progress, dict):
+ return None
+ try:
+ return float(progress.get("processed"))
+ except (TypeError, ValueError):
+ return None
+
+ @staticmethod
+ def _sse_event_has_generated_output(event: str) -> bool:
+ """Return true when a complete SSE event carries model-generated output."""
+ data = LlamaCppBackend._sse_event_payload(event)
+ if data is None:
return False
if data.get("type") == "diffusion_frame":
return True
@@ -27624,6 +27644,7 @@ class LlamaCppBackend:
if first_token_deadline is None:
first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_S
last_chunk_at: Optional[float] = None
+ last_prefill_progress: Optional[float] = None
prefill_sse_buffer = ""
while True:
if cancel_event is not None and cancel_event.is_set():
@@ -27646,6 +27667,13 @@ class LlamaCppBackend:
starts_output = True
prefill_sse_buffer = ""
break
+ # Renew only on increasing progress so repeated events still time out.
+ processed = LlamaCppBackend._sse_event_prefill_progress(event)
+ if processed is not None and (
+ last_prefill_progress is None or processed > last_prefill_progress
+ ):
+ last_prefill_progress = processed
+ first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_S
if chunk and starts_output:
if last_chunk_at is None and post_first_chunk_read_timeout_s is not None:
LlamaCppBackend._set_stream_read_timeout(
@@ -28056,8 +28084,9 @@ class LlamaCppBackend:
retry_max_tokens = max_tokens
retry_context_overflow = context_overflow
retry_preflight_context_length = None
+ # Progress events let advancing prefills renew the first-token deadline.
+ payload["return_progress"] = True
if perf_callback is not None:
- payload["return_progress"] = True
payload["timings_per_token"] = True
if logit_bias:
payload["logit_bias"] = logit_bias
@@ -29125,8 +29154,9 @@ class LlamaCppBackend:
"frequency_penalty": frequency_penalty,
}
+ # Progress events feed the first-token deadline; timings stay opt-in.
+ payload["return_progress"] = True
if perf_callback is not None:
- payload["return_progress"] = True
payload["timings_per_token"] = True
if logit_bias:
payload["logit_bias"] = logit_bias
@@ -31630,8 +31660,9 @@ class LlamaCppBackend:
_apply_seeded_llama_request(stream_payload, seed)
stream_payload["stream_options"] = {"include_usage": True}
+ # Progress events feed the first-token deadline; timings stay opt-in.
+ stream_payload["return_progress"] = True
if perf_callback is not None:
- stream_payload["return_progress"] = True
stream_payload["timings_per_token"] = True
_final_respawn_truncations: list[dict] = []
diff --git a/studio/backend/tests/test_llama_cpp_stall_timeout.py b/studio/backend/tests/test_llama_cpp_stall_timeout.py
index 044d15b82be..37f214f13e0 100644
--- a/studio/backend/tests/test_llama_cpp_stall_timeout.py
+++ b/studio/backend/tests/test_llama_cpp_stall_timeout.py
@@ -31,6 +31,7 @@ sys.modules.setdefault("loggers", _loggers_stub)
sys.modules.setdefault("structlog", _types.ModuleType("structlog"))
import httpcore # noqa: E402
+import httpx # noqa: E402
from core.inference import llama_cpp as llama_cpp_mod # noqa: E402
from core.inference.llama_cpp import LlamaCppBackend # noqa: E402
@@ -137,6 +138,72 @@ def test_prompt_progress_keeps_the_prefill_read_timeout():
iterator.close()
+_OUTPUT_EVENT = 'data: {"choices":[{"delta":{"content":"x"}}]}\n\n'
+
+
+def _progress_event(processed: int) -> str:
+ return (
+ 'data: {"choices":[{"delta":{"role":"assistant","content":null}}],'
+ f'"prompt_progress":{{"processed":{processed},"cache":0,"time_ms":1}}}}\n\n'
+ )
+
+
+def _run(events, clock):
+ """Drain _iter_text_cancellable over ``events``, 200s of fake time apart."""
+
+ class Response:
+ request = _types.SimpleNamespace(extensions = {"timeout": {"read": _PREFILL_TIMEOUT}})
+
+ @staticmethod
+ def iter_text():
+ for event in events:
+ clock["t"] += 200.0
+ yield event
+
+ @staticmethod
+ def close():
+ return None
+
+ return list(
+ LlamaCppBackend._iter_text_cancellable(
+ Response(),
+ first_token_deadline = clock["t"] + _PREFILL_TIMEOUT,
+ post_first_chunk_read_timeout_s = _STALL_TIMEOUT,
+ )
+ )
+
+
+def test_advancing_prefill_progress_extends_the_first_token_deadline(monkeypatch):
+ """A prompt that keeps prefilling past the window is not a hang (#9037)."""
+ clock = {"t": 0.0}
+ monkeypatch.setattr(llama_cpp_mod.time, "monotonic", lambda: clock["t"])
+
+ # Progress every 200s keeps a 4000s prefill alive.
+ events = [_progress_event(i * 4096) for i in range(1, 21)] + [_OUTPUT_EVENT]
+
+ assert _run(events, clock)[-1] == _OUTPUT_EVENT
+ assert clock["t"] > _PREFILL_TIMEOUT
+
+
+def test_repeated_prefill_progress_still_times_out(monkeypatch):
+ """A server replaying one progress value is stuck, not prefilling."""
+ clock = {"t": 0.0}
+ monkeypatch.setattr(llama_cpp_mod.time, "monotonic", lambda: clock["t"])
+
+ with pytest.raises(httpx.ReadTimeout):
+ _run([_progress_event(4096)] * 20 + [_OUTPUT_EVENT], clock)
+
+
+def test_prefill_without_progress_events_still_times_out(monkeypatch):
+ """Content-free events alone are not proof of life: the window still applies."""
+ clock = {"t": 0.0}
+ monkeypatch.setattr(llama_cpp_mod.time, "monotonic", lambda: clock["t"])
+
+ keepalive = 'data: {"choices":[{"delta":{"role":"assistant","content":null}}]}\n\n'
+ with pytest.raises(httpx.ReadTimeout):
+ _run([keepalive] * 20 + [_OUTPUT_EVENT], clock)
+
+
def test_prefill_timeout_used_when_no_live_override(monkeypatch):
"""Without a lowered live timeout, the wrapper honors the passed prefill timeout, so the normal first-token wait is unchanged."""
clock = {"t": 0.0}