Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: renew the chat-run lease while long prefill is still advancing (#12523)

Indrajit swaind8740d38772f
diff --git a/studio/backend/core/inference/chat_generation_runs.py b/studio/backend/core/inference/chat_generation_runs.py
index 953de0a5f28..c01a48bfbd1 100644
--- a/studio/backend/core/inference/chat_generation_runs.py
+++ b/studio/backend/core/inference/chat_generation_runs.py
@@ -36,9 +36,10 @@ _SHUTDOWN_CANCEL_SECONDS = 5.0
# force-kills the backend after five seconds.
_SWEEP_SHUTDOWN_SECONDS = 0.5
# A durable run sets cancel_on_disconnect=False, so reaping is keyed on progress rather than on connectedness. The
-# default matches llama_cpp._DEFAULT_FIRST_TOKEN_TIMEOUT_S, the request path's own first-token budget: a lease older
-# than that cannot be legitimate prefill, and slow decode is safe at any speed. A century: clear of any real lease, far
-# below where integer milliseconds overflow.
+# default matches llama_cpp._DEFAULT_FIRST_TOKEN_TIMEOUT_S: that path renews on advancing ``prompt_progress``, and this
+# lease renews on the ``: prefill-progress`` stand-in the routes emit when those frames are dropped for the UI. Slow
+# decode is safe at any speed via streamed chunks. A century: clear of any real lease, far below where integer
+# milliseconds overflow.
_MAX_ENV_SECONDS = 100.0 * 365.0 * 24.0 * 60.0 * 60.0
# The longest admission keep-alive cadence worth deriving a lease from. A day already means the queue never reports,
# and tripling it stays far inside _MAX_ENV_SECONDS.
@@ -402,6 +403,9 @@ _ADMISSION_WAIT_MARKER = ": admission-wait"
_ADMISSION_DONE_MARKER = ": admission-done"
# A server-side tool still running. Rate limited like the wait marker.
_TOOL_HEARTBEAT_MARKER = ": tool-heartbeat"
+# Prefill still advancing while prompt_progress frames are dropped for the UI. Same lease
+# contract as the tool heartbeat; not a stall keep-alive.
+_PREFILL_PROGRESS_MARKER = ": prefill-progress"
def _minimum_lease_seconds() -> float:
@@ -804,15 +808,22 @@ class ChatGenerationSupervisor:
break
next_raw_task = asyncio.create_task(iterator.__anext__())
text = raw.decode("utf-8", "replace") if isinstance(raw, bytes) else str(raw)
- # Admission comments are progress; plain keep-alives are not. A queued run only emits `:
- # admission-wait`, which _SSEDecoder drops, so nothing renewed the lease and a healthy queue reaped its
- # own runs. `: keep-alive` is the opposite signal, emitted when the generator has produced NOTHING, so
- # renewing on any byte would keep a wedged run alive forever. Rate limited because chunk traffic already
+ # Admission / tool / prefill comments are progress; plain keep-alives are not. A
+ # queued run only emits `: admission-wait`, which _SSEDecoder drops, so nothing
+ # renewed the lease and a healthy queue reaped its own runs. `: keep-alive` is the
+ # opposite signal, emitted when the generator has produced NOTHING, so renewing on
+ # any byte would keep a wedged run alive forever. Prefill drops ``prompt_progress``
+ # for clients that did not ask for it and stands in with `: prefill-progress`
+ # (same idea as `: tool-heartbeat`). Rate limited because chunk traffic already
# renews through append_events.
if _ADMISSION_DONE_MARKER in text:
last_keepalive = time.monotonic()
await self._try_touch_progress(run_id)
- elif _ADMISSION_WAIT_MARKER in text or _TOOL_HEARTBEAT_MARKER in text:
+ elif (
+ _ADMISSION_WAIT_MARKER in text
+ or _TOOL_HEARTBEAT_MARKER in text
+ or _PREFILL_PROGRESS_MARKER in text
+ ):
now_s = time.monotonic()
if now_s - last_keepalive >= _renew_interval_seconds():
last_keepalive = now_s
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index f7d0183458e..df2474e520e 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -6979,6 +6979,11 @@ def _llama_chunk_has_generated_output(data: dict) -> bool:
return False
+def _perf_callback_wants_timings(callback) -> bool:
+ """A callback that only tracks prefill progress opts out of per-token timings."""
+ return callback is not None and getattr(callback, "wants_timings", True)
+
+
def _report_live_llama_timings(callback, chunk) -> None:
"""Report request-scoped llama.cpp progress without altering the public stream."""
if callback is None or not isinstance(chunk, dict):
@@ -35917,7 +35922,7 @@ class LlamaCppBackend:
retry_preflight_context_length = None
# Progress events let advancing prefills renew the first-token deadline.
payload["return_progress"] = True
- if perf_callback is not None:
+ if _perf_callback_wants_timings(perf_callback):
payload["timings_per_token"] = True
if logit_bias:
payload["logit_bias"] = logit_bias
@@ -37052,7 +37057,7 @@ class LlamaCppBackend:
if on_decode_slot is not None:
payload["verbose"] = True
payload["response_fields"] = ["id_slot"]
- if perf_callback is not None:
+ if _perf_callback_wants_timings(perf_callback):
payload["timings_per_token"] = True
if logit_bias:
payload["logit_bias"] = logit_bias
@@ -39786,7 +39791,7 @@ class LlamaCppBackend:
# Progress events feed the first-token deadline; timings stay opt-in.
stream_payload["return_progress"] = True
- if perf_callback is not None:
+ if _perf_callback_wants_timings(perf_callback):
stream_payload["timings_per_token"] = True
_final_respawn_truncations: list[dict] = []
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 5dd1cc7992b..04d66c1c97b 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -870,7 +870,9 @@ def _is_prefill_progress_only(data) -> bool:
class _ProgressKeepalive:
- """Dropped progress still resets the relay's idle timer, so stand in for the keepalive it starved."""
+ """Dropped ``prompt_progress`` still needs a stand-in frame: the relay's idle timer, and the
+ durable-run lease (which ignores plain ``: keep-alive``). Emit ``: prefill-progress`` on the
+ same cadence."""
def __init__(self, interval_s: Optional[float]):
self._interval_s = interval_s
@@ -1907,6 +1909,10 @@ _OPENAI_ADMISSION_SSE_WAIT = ": admission-wait\n\n"
_OPENAI_ADMISSION_SSE_DONE = ": admission-done\n\n"
# A server-side tool still running, unlike a stall keep-alive: durable runs renew their lease on it.
_OPENAI_TOOL_HEARTBEAT_SSE = ": tool-heartbeat\n\n"
+# Prefill still advancing with nothing to stream: ``prompt_progress`` dropped for a client that did
+# not ask for it, or a native GGUF generator still in prefill. Same lease contract as the tool
+# heartbeat: not a stall keep-alive.
+_OPENAI_PREFILL_PROGRESS_SSE = ": prefill-progress\n\n"
_OPENAI_LLAMA_ADMISSION_POLL_S = 0.25
# Cap on waiting for a cancelled teardown task. Request.is_disconnected() can swallow
# cancel() (#7617), so teardown abandons the task rather than hold the response, and
@@ -7084,6 +7090,45 @@ def _monitor_perf_callback(monitor_id: Optional[str], context_length):
return _callback
+class _PrefillProgressSignal:
+ """Wrap a GGUF perf callback to count advancing ``prompt_progress`` from the backend thread.
+
+ The native GGUF generators consume progress themselves and yield nothing until the first
+ token, so the route's stall loop would only ever send ``: keep-alive``, which the durable
+ lease ignores. The stall loop polls ``advanced()`` to send ``: prefill-progress`` instead."""
+
+ def __init__(self, inner):
+ self._inner = inner
+ self._last_processed = None
+ self._advances = 0
+ self._seen = 0
+ # Only ask llama-server for per-token timings when a monitor actually consumes them.
+ self.wants_timings = inner is not None
+
+ @property
+ def needs_phase(self) -> bool:
+ return self._inner is not None and getattr(self._inner, "needs_phase", True)
+
+ def __call__(self, sample: dict) -> None:
+ progress = sample.get("prompt_progress")
+ if isinstance(progress, dict):
+ processed = progress.get("processed")
+ # Any change counts, so a new prefill round (lower count) renews too; a repeat does not.
+ if processed is not None and processed != self._last_processed:
+ self._last_processed = processed
+ self._advances += 1
+ if self._inner is not None:
+ self._inner(sample)
+
+ def advanced(self) -> bool:
+ """True once per stall tick when prefill moved since the previous call."""
+ advances = self._advances
+ if advances == self._seen:
+ return False
+ self._seen = advances
+ return True
+
+
def _monitor_call_text(name: Any, arguments: Any = None) -> str:
call_name = str(name or "tool")
if arguments is None or arguments == "":
@@ -28286,11 +28331,14 @@ async def produce_openai_chat_completions(
)
)
- _gguf_perf_callback = (
+ _gguf_monitor_callback = (
_monitor_perf_callback(monitor_id, llama_backend.context_length)
if not _wants_multiple_choices(payload)
else None
)
+ # Only a stream has a stall loop to feed; elsewhere the monitor callback (or None) stands.
+ _gguf_prefill_signal = _PrefillProgressSignal(_gguf_monitor_callback)
+ _gguf_perf_callback = _gguf_prefill_signal if payload.stream else _gguf_monitor_callback
def _gguf_chat_delta_line(delta: ChoiceDelta, finish_reason = None) -> str:
if delta.reasoning_content is not None and delta.content is None:
@@ -28779,7 +28827,12 @@ async def produce_openai_chat_completions(
)
if done_tasks:
break
- yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE
+ # Advancing prefill renews the durable lease; a bare keep-alive does not.
+ yield (
+ _OPENAI_PREFILL_PROGRESS_SSE
+ if _gguf_prefill_signal.advanced()
+ else _OPENAI_PASSTHROUGH_SSE_KEEPALIVE
+ )
approval_flush_pending = False
wait_timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S
event = next_task.result()
@@ -29474,7 +29527,12 @@ async def produce_openai_chat_completions(
)
if done_tasks:
break
- yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE
+ # Advancing prefill renews the durable lease; a bare keep-alive does not.
+ yield (
+ _OPENAI_PREFILL_PROGRESS_SSE
+ if _gguf_prefill_signal.advanced()
+ else _OPENAI_PASSTHROUGH_SSE_KEEPALIVE
+ )
cumulative = next_task.result()
finally:
if next_task.done():
@@ -33383,7 +33441,8 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge
)
):
if progress_keepalive.due():
- yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE.encode()
+ # Distinct from stall keep-alive: durable runs renew their lease on it.
+ yield _OPENAI_PREFILL_PROGRESS_SSE.encode()
continue
out = _cmpl_stream_event_out(event, _include_usage)
if out is not None:
@@ -41810,7 +41869,8 @@ async def _openai_passthrough_stream_admitted(
if not client_wants_progress and _is_prefill_progress_only(chunk_data):
_monitor_openai_sse_line(monitor_id, raw_line, llama_backend.context_length)
if progress_keepalive.due():
- yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE
+ # Distinct from stall keep-alive: durable runs renew their lease on it.
+ yield _OPENAI_PREFILL_PROGRESS_SSE
continue
# With healing active, a content-bearing line may be replaced by
# held/promoted chunks; otherwise the single (already
diff --git a/studio/backend/tests/test_chat_generation_lease_compat.py b/studio/backend/tests/test_chat_generation_lease_compat.py
index be82ee93011..bd5b6f2e1a2 100644
--- a/studio/backend/tests/test_chat_generation_lease_compat.py
+++ b/studio/backend/tests/test_chat_generation_lease_compat.py
@@ -705,11 +705,15 @@ def test_the_admission_marker_matches_the_route_that_emits_it():
assert inference_route._OPENAI_ADMISSION_SSE_WAIT.startswith(runs_mod._ADMISSION_WAIT_MARKER)
assert inference_route._OPENAI_ADMISSION_SSE_DONE.startswith(runs_mod._ADMISSION_DONE_MARKER)
assert inference_route._OPENAI_TOOL_HEARTBEAT_SSE.startswith(runs_mod._TOOL_HEARTBEAT_MARKER)
+ assert inference_route._OPENAI_PREFILL_PROGRESS_SSE.startswith(
+ runs_mod._PREFILL_PROGRESS_MARKER
+ )
# And none may match the stall keep-alive, which is the opposite signal.
for marker in (
runs_mod._ADMISSION_WAIT_MARKER,
runs_mod._ADMISSION_DONE_MARKER,
runs_mod._TOOL_HEARTBEAT_MARKER,
+ runs_mod._PREFILL_PROGRESS_MARKER,
):
assert not inference_route._OPENAI_PASSTHROUGH_SSE_KEEPALIVE.startswith(marker)
# The two must stay distinct, or the done branch would swallow every wait.
diff --git a/studio/backend/tests/test_chat_generation_supervisor.py b/studio/backend/tests/test_chat_generation_supervisor.py
index 2829e533a3e..4c4321f098d 100644
--- a/studio/backend/tests/test_chat_generation_supervisor.py
+++ b/studio/backend/tests/test_chat_generation_supervisor.py
@@ -445,9 +445,13 @@ async def test_a_prefill_reporting_only_progress_renews_the_lease(durable_run, m
@pytest.mark.asyncio
@pytest.mark.parametrize(
"comment, reaped",
- [("_OPENAI_TOOL_HEARTBEAT_SSE", []), ("_OPENAI_PASSTHROUGH_SSE_KEEPALIVE", ["run-1"])],
+ [
+ ("_OPENAI_TOOL_HEARTBEAT_SSE", []),
+ ("_OPENAI_PREFILL_PROGRESS_SSE", []),
+ ("_OPENAI_PASSTHROUGH_SSE_KEEPALIVE", ["run-1"]),
+ ],
)
-async def test_a_silent_tool_holds_the_lease_but_a_stalled_stream_does_not(
+async def test_progress_comments_hold_the_lease_but_a_stalled_stream_does_not(
durable_run, monkeypatch, comment, reaped
):
comment = getattr(inference, comment)
@@ -474,6 +478,144 @@ async def test_a_silent_tool_holds_the_lease_but_a_stalled_stream_does_not(
assert sampled["reaped"] == reaped
[email protected]("tools", [False, True], ids = ["chat", "tool-loop"])
[email protected]("reports_progress, reaped", [(True, []), (False, ["run-1"])])
+async def test_native_gguf_prefill_progress_holds_the_lease(
+ durable_run, monkeypatch, tools, reports_progress, reaped
+):
+ """The Studio UI never takes the passthrough: its GGUF chat and tool loop run the native
+ generators, which consume prompt_progress and stay silent until the first token. Advancing
+ progress must still renew the lease through the stall loop; a silent generator must not."""
+ from core.inference.llama_cpp import _report_live_llama_timings
+
+ entered, release = threading.Event(), threading.Event()
+
+ def _prefill(perf_callback):
+ entered.set()
+ processed = 0
+ while not release.wait(0.005):
+ if reports_progress:
+ processed += 512
+ # The same hook the real generators feed from each llama-server progress frame.
+ _report_live_llama_timings(
+ perf_callback,
+ {"choices": [], "prompt_progress": {"total": 10**6, "processed": processed}},
+ )
+
+ def generate(*, perf_callback = None, **_kwargs):
+ _prefill(perf_callback)
+ yield "Hi"
+ yield {"type": "metadata", "finish_reason": "stop"}
+
+ def generate_with_tools(*, perf_callback = None, **_kwargs):
+ _prefill(perf_callback)
+ yield {"type": "content", "text": "Hi"}
+ yield {"type": "metadata", "finish_reason": "stop"}
+
+ llama = SimpleNamespace(
+ is_loaded = True,
+ model_identifier = "local.gguf",
+ base_url = "http://llama.test",
+ effective_parallel_slots = 1,
+ supports_tools = tools,
+ supports_tool_passthrough = tools,
+ is_vision = False,
+ _is_audio = False,
+ context_length = None,
+ generate_chat_completion = generate,
+ generate_chat_completion_with_tools = generate_with_tools,
+ release_idle_chat_slot = lambda *_a, **_k: False,
+ count_chat_tokens = lambda *_a, **_k: 16,
+ _request_reasoning_kwargs = lambda *_a, **_k: None,
+ )
+ monkeypatch.setattr(inference, "get_llama_cpp_backend", lambda: llama)
+ monkeypatch.setattr(inference, "_automatic_model_load_may_run", lambda: False)
+
+ async def no_switch(*_a, **_k):
+ return None
+
+ monkeypatch.setattr(inference, "_maybe_auto_switch_model", no_switch)
+ monkeypatch.setattr(inference, "_LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S", 0.02)
+ if not tools:
+ monkeypatch.setattr(inference, "_effective_enable_tools", lambda _payload: False)
+ now = {"ms": runs_db.now_ms()}
+ monkeypatch.setattr(runs_db, "now_ms", lambda: now["ms"])
+ real_produce = inference.produce_openai_chat_completions
+ sampled = {"comments": set()}
+
+ async def wrapped(payload, request, subject, *, cancel_on_disconnect):
+ if tools:
+ # What the UI sends with the web-search pill lit: server-side tools, no client tools.
+ payload.enable_tools = True
+ payload.enabled_tools = ["web_search"]
+ response = await real_produce(
+ payload, request, subject, cancel_on_disconnect = cancel_on_disconnect
+ )
+ inner = response.body_iterator
+ # Preparation has already read its own interval; only the stream's rate limit is lifted.
+ monkeypatch.setattr(chat_generation_runs, "_renew_interval_seconds", lambda: 0.0)
+
+ async def body():
+ comments = 0
+ async for raw in inner:
+ text = raw.decode() if isinstance(raw, bytes) else str(raw)
+ if text.startswith(":") and entered.is_set():
+ comments += 1
+ sampled["comments"].add(text.strip())
+ if comments == 2:
+ now["ms"] += 21 * 60_000
+ elif comments == 5 and "reaped" not in sampled:
+ sampled["reaped"] = runs_db.reconcile_runs(stale_after_ms = 1_200_000)
+ release.set()
+ yield raw
+
+ response.body_iterator = body()
+ return response
+
+ monkeypatch.setattr(inference, "produce_openai_chat_completions", wrapped)
+ try:
+ supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
+ await asyncio.wait_for(supervisor._produce("run-1"), 30)
+ finally:
+ release.set()
+ assert sampled["reaped"] == reaped
+ expected = ": prefill-progress" if reports_progress else ": keep-alive"
+ assert expected in sampled["comments"], sampled["comments"]
+ if not reports_progress:
+ assert ": prefill-progress" not in sampled["comments"]
+
+
+def test_prefill_signal_counts_only_moving_progress():
+ from core.inference.llama_cpp import _perf_callback_wants_timings
+
+ forwarded = []
+ bare = inference._PrefillProgressSignal(None)
+ # Without a monitor the wrapper must not switch llama-server to per-token timings.
+ assert _perf_callback_wants_timings(bare) is False
+ assert bare.needs_phase is False
+ assert bare.advanced() is False
+ bare({"prompt_progress": {"processed": 10}})
+ bare({"prompt_progress": {"processed": 10}})
+ assert bare.advanced() is True
+ assert bare.advanced() is False
+ # A new prefill round restarts the count lower; that is still movement.
+ bare({"prompt_progress": {"processed": 2}})
+ assert bare.advanced() is True
+ bare({"timings": {"predicted_n": 1}})
+ assert bare.advanced() is False
+
+ def monitor(sample):
+ forwarded.append(sample)
+
+ monitor.needs_phase = False
+ wrapped = inference._PrefillProgressSignal(monitor)
+ assert _perf_callback_wants_timings(wrapped) is True
+ assert wrapped.needs_phase is False
+ wrapped({"prompt_progress": {"processed": 1}})
+ assert forwarded == [{"prompt_progress": {"processed": 1}}]
+
+
async def _subscriber_sequences(after = 0):
response = await run_routes.chat_generation_events(
"run-1",
diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py
index d5a0259e53d..b67eeb85921 100644
--- a/studio/backend/tests/test_openai_tool_passthrough.py
+++ b/studio/backend/tests/test_openai_tool_passthrough.py
@@ -7501,7 +7501,9 @@ class TestApiMonitorProviderAndCompletionStreams:
body = b"".join([chunk async for chunk in response.body_iterator])
assert upstream_bodies[0]["return_progress"] is True
- assert (b": keep-alive" in body) is not client_progress
+ # Dropped progress stands in with ``: prefill-progress`` (lease renewal), not stall keep-alive.
+ assert (b": prefill-progress" in body) is not client_progress
+ assert (b": keep-alive" in body) is False
assert upstream_bodies[0]["stream_options"]["include_usage"] is True
assert b'"usage"' not in body
assert (b"prompt_progress" in body) is client_progress