Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: renew the chat-run lease while long prefill is still advancing (#12523)
Indrajit swaind8740d38772f
diff --git a/studio/backend/core/inference/chat_generation_runs.py b/studio/backend/core/inference/chat_generation_runs.pyindex 953de0a5f28..c01a48bfbd1 100644--- a/studio/backend/core/inference/chat_generation_runs.py+++ b/studio/backend/core/inference/chat_generation_runs.py@@ -36,9 +36,10 @@ _SHUTDOWN_CANCEL_SECONDS = 5.0# force-kills the backend after five seconds._SWEEP_SHUTDOWN_SECONDS = 0.5# A durable run sets cancel_on_disconnect=False, so reaping is keyed on progress rather than on connectedness. The-# default matches llama_cpp._DEFAULT_FIRST_TOKEN_TIMEOUT_S, the request path's own first-token budget: a lease older-# than that cannot be legitimate prefill, and slow decode is safe at any speed. A century: clear of any real lease, far-# below where integer milliseconds overflow.+# default matches llama_cpp._DEFAULT_FIRST_TOKEN_TIMEOUT_S: that path renews on advancing ``prompt_progress``, and this+# lease renews on the ``: prefill-progress`` stand-in the routes emit when those frames are dropped for the UI. Slow+# decode is safe at any speed via streamed chunks. A century: clear of any real lease, far below where integer+# milliseconds overflow._MAX_ENV_SECONDS = 100.0 * 365.0 * 24.0 * 60.0 * 60.0# The longest admission keep-alive cadence worth deriving a lease from. A day already means the queue never reports,# and tripling it stays far inside _MAX_ENV_SECONDS.@@ -402,6 +403,9 @@ _ADMISSION_WAIT_MARKER = ": admission-wait"_ADMISSION_DONE_MARKER = ": admission-done"# A server-side tool still running. Rate limited like the wait marker._TOOL_HEARTBEAT_MARKER = ": tool-heartbeat"+# Prefill still advancing while prompt_progress frames are dropped for the UI. Same lease+# contract as the tool heartbeat; not a stall keep-alive.+_PREFILL_PROGRESS_MARKER = ": prefill-progress"def _minimum_lease_seconds() -> float:@@ -804,15 +808,22 @@ class ChatGenerationSupervisor:breaknext_raw_task = asyncio.create_task(iterator.__anext__())text = raw.decode("utf-8", "replace") if isinstance(raw, bytes) else str(raw)- # Admission comments are progress; plain keep-alives are not. A queued run only emits `:- # admission-wait`, which _SSEDecoder drops, so nothing renewed the lease and a healthy queue reaped its- # own runs. `: keep-alive` is the opposite signal, emitted when the generator has produced NOTHING, so- # renewing on any byte would keep a wedged run alive forever. Rate limited because chunk traffic already+ # Admission / tool / prefill comments are progress; plain keep-alives are not. A+ # queued run only emits `: admission-wait`, which _SSEDecoder drops, so nothing+ # renewed the lease and a healthy queue reaped its own runs. `: keep-alive` is the+ # opposite signal, emitted when the generator has produced NOTHING, so renewing on+ # any byte would keep a wedged run alive forever. Prefill drops ``prompt_progress``+ # for clients that did not ask for it and stands in with `: prefill-progress`+ # (same idea as `: tool-heartbeat`). Rate limited because chunk traffic already# renews through append_events.if _ADMISSION_DONE_MARKER in text:last_keepalive = time.monotonic()await self._try_touch_progress(run_id)- elif _ADMISSION_WAIT_MARKER in text or _TOOL_HEARTBEAT_MARKER in text:+ elif (+ _ADMISSION_WAIT_MARKER in text+ or _TOOL_HEARTBEAT_MARKER in text+ or _PREFILL_PROGRESS_MARKER in text+ ):now_s = time.monotonic()if now_s - last_keepalive >= _renew_interval_seconds():last_keepalive = now_sdiff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex f7d0183458e..df2474e520e 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -6979,6 +6979,11 @@ def _llama_chunk_has_generated_output(data: dict) -> bool:return False+def _perf_callback_wants_timings(callback) -> bool:+ """A callback that only tracks prefill progress opts out of per-token timings."""+ return callback is not None and getattr(callback, "wants_timings", True)++def _report_live_llama_timings(callback, chunk) -> None:"""Report request-scoped llama.cpp progress without altering the public stream."""if callback is None or not isinstance(chunk, dict):@@ -35917,7 +35922,7 @@ class LlamaCppBackend:retry_preflight_context_length = None# Progress events let advancing prefills renew the first-token deadline.payload["return_progress"] = True- if perf_callback is not None:+ if _perf_callback_wants_timings(perf_callback):payload["timings_per_token"] = Trueif logit_bias:payload["logit_bias"] = logit_bias@@ -37052,7 +37057,7 @@ class LlamaCppBackend:if on_decode_slot is not None:payload["verbose"] = Truepayload["response_fields"] = ["id_slot"]- if perf_callback is not None:+ if _perf_callback_wants_timings(perf_callback):payload["timings_per_token"] = Trueif logit_bias:payload["logit_bias"] = logit_bias@@ -39786,7 +39791,7 @@ class LlamaCppBackend:# Progress events feed the first-token deadline; timings stay opt-in.stream_payload["return_progress"] = True- if perf_callback is not None:+ if _perf_callback_wants_timings(perf_callback):stream_payload["timings_per_token"] = True_final_respawn_truncations: list[dict] = []diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 5dd1cc7992b..04d66c1c97b 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -870,7 +870,9 @@ def _is_prefill_progress_only(data) -> bool:class _ProgressKeepalive:- """Dropped progress still resets the relay's idle timer, so stand in for the keepalive it starved."""+ """Dropped ``prompt_progress`` still needs a stand-in frame: the relay's idle timer, and the+ durable-run lease (which ignores plain ``: keep-alive``). Emit ``: prefill-progress`` on the+ same cadence."""def __init__(self, interval_s: Optional[float]):self._interval_s = interval_s@@ -1907,6 +1909,10 @@ _OPENAI_ADMISSION_SSE_WAIT = ": admission-wait\n\n"_OPENAI_ADMISSION_SSE_DONE = ": admission-done\n\n"# A server-side tool still running, unlike a stall keep-alive: durable runs renew their lease on it._OPENAI_TOOL_HEARTBEAT_SSE = ": tool-heartbeat\n\n"+# Prefill still advancing with nothing to stream: ``prompt_progress`` dropped for a client that did+# not ask for it, or a native GGUF generator still in prefill. Same lease contract as the tool+# heartbeat: not a stall keep-alive.+_OPENAI_PREFILL_PROGRESS_SSE = ": prefill-progress\n\n"_OPENAI_LLAMA_ADMISSION_POLL_S = 0.25# Cap on waiting for a cancelled teardown task. Request.is_disconnected() can swallow# cancel() (#7617), so teardown abandons the task rather than hold the response, and@@ -7084,6 +7090,45 @@ def _monitor_perf_callback(monitor_id: Optional[str], context_length):return _callback+class _PrefillProgressSignal:+ """Wrap a GGUF perf callback to count advancing ``prompt_progress`` from the backend thread.++ The native GGUF generators consume progress themselves and yield nothing until the first+ token, so the route's stall loop would only ever send ``: keep-alive``, which the durable+ lease ignores. The stall loop polls ``advanced()`` to send ``: prefill-progress`` instead."""++ def __init__(self, inner):+ self._inner = inner+ self._last_processed = None+ self._advances = 0+ self._seen = 0+ # Only ask llama-server for per-token timings when a monitor actually consumes them.+ self.wants_timings = inner is not None++ @property+ def needs_phase(self) -> bool:+ return self._inner is not None and getattr(self._inner, "needs_phase", True)++ def __call__(self, sample: dict) -> None:+ progress = sample.get("prompt_progress")+ if isinstance(progress, dict):+ processed = progress.get("processed")+ # Any change counts, so a new prefill round (lower count) renews too; a repeat does not.+ if processed is not None and processed != self._last_processed:+ self._last_processed = processed+ self._advances += 1+ if self._inner is not None:+ self._inner(sample)++ def advanced(self) -> bool:+ """True once per stall tick when prefill moved since the previous call."""+ advances = self._advances+ if advances == self._seen:+ return False+ self._seen = advances+ return True++def _monitor_call_text(name: Any, arguments: Any = None) -> str:call_name = str(name or "tool")if arguments is None or arguments == "":@@ -28286,11 +28331,14 @@ async def produce_openai_chat_completions())- _gguf_perf_callback = (+ _gguf_monitor_callback = (_monitor_perf_callback(monitor_id, llama_backend.context_length)if not _wants_multiple_choices(payload)else None)+ # Only a stream has a stall loop to feed; elsewhere the monitor callback (or None) stands.+ _gguf_prefill_signal = _PrefillProgressSignal(_gguf_monitor_callback)+ _gguf_perf_callback = _gguf_prefill_signal if payload.stream else _gguf_monitor_callbackdef _gguf_chat_delta_line(delta: ChoiceDelta, finish_reason = None) -> str:if delta.reasoning_content is not None and delta.content is None:@@ -28779,7 +28827,12 @@ async def produce_openai_chat_completions()if done_tasks:break- yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE+ # Advancing prefill renews the durable lease; a bare keep-alive does not.+ yield (+ _OPENAI_PREFILL_PROGRESS_SSE+ if _gguf_prefill_signal.advanced()+ else _OPENAI_PASSTHROUGH_SSE_KEEPALIVE+ )approval_flush_pending = Falsewait_timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_Sevent = next_task.result()@@ -29474,7 +29527,12 @@ async def produce_openai_chat_completions()if done_tasks:break- yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE+ # Advancing prefill renews the durable lease; a bare keep-alive does not.+ yield (+ _OPENAI_PREFILL_PROGRESS_SSE+ if _gguf_prefill_signal.advanced()+ else _OPENAI_PASSTHROUGH_SSE_KEEPALIVE+ )cumulative = next_task.result()finally:if next_task.done():@@ -33383,7 +33441,8 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge)):if progress_keepalive.due():- yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE.encode()+ # Distinct from stall keep-alive: durable runs renew their lease on it.+ yield _OPENAI_PREFILL_PROGRESS_SSE.encode()continueout = _cmpl_stream_event_out(event, _include_usage)if out is not None:@@ -41810,7 +41869,8 @@ async def _openai_passthrough_stream_admitted(if not client_wants_progress and _is_prefill_progress_only(chunk_data):_monitor_openai_sse_line(monitor_id, raw_line, llama_backend.context_length)if progress_keepalive.due():- yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE+ # Distinct from stall keep-alive: durable runs renew their lease on it.+ yield _OPENAI_PREFILL_PROGRESS_SSEcontinue# With healing active, a content-bearing line may be replaced by# held/promoted chunks; otherwise the single (alreadydiff --git a/studio/backend/tests/test_chat_generation_lease_compat.py b/studio/backend/tests/test_chat_generation_lease_compat.pyindex be82ee93011..bd5b6f2e1a2 100644--- a/studio/backend/tests/test_chat_generation_lease_compat.py+++ b/studio/backend/tests/test_chat_generation_lease_compat.py@@ -705,11 +705,15 @@ def test_the_admission_marker_matches_the_route_that_emits_it():assert inference_route._OPENAI_ADMISSION_SSE_WAIT.startswith(runs_mod._ADMISSION_WAIT_MARKER)assert inference_route._OPENAI_ADMISSION_SSE_DONE.startswith(runs_mod._ADMISSION_DONE_MARKER)assert inference_route._OPENAI_TOOL_HEARTBEAT_SSE.startswith(runs_mod._TOOL_HEARTBEAT_MARKER)+ assert inference_route._OPENAI_PREFILL_PROGRESS_SSE.startswith(+ runs_mod._PREFILL_PROGRESS_MARKER+ )# And none may match the stall keep-alive, which is the opposite signal.for marker in (runs_mod._ADMISSION_WAIT_MARKER,runs_mod._ADMISSION_DONE_MARKER,runs_mod._TOOL_HEARTBEAT_MARKER,+ runs_mod._PREFILL_PROGRESS_MARKER,):assert not inference_route._OPENAI_PASSTHROUGH_SSE_KEEPALIVE.startswith(marker)# The two must stay distinct, or the done branch would swallow every wait.diff --git a/studio/backend/tests/test_chat_generation_supervisor.py b/studio/backend/tests/test_chat_generation_supervisor.pyindex 2829e533a3e..4c4321f098d 100644--- a/studio/backend/tests/test_chat_generation_supervisor.py+++ b/studio/backend/tests/test_chat_generation_supervisor.py@@ -445,9 +445,13 @@ async def test_a_prefill_reporting_only_progress_renews_the_lease(durable_run, m@pytest.mark.asyncio@pytest.mark.parametrize("comment, reaped",- [("_OPENAI_TOOL_HEARTBEAT_SSE", []), ("_OPENAI_PASSTHROUGH_SSE_KEEPALIVE", ["run-1"])],+ [+ ("_OPENAI_TOOL_HEARTBEAT_SSE", []),+ ("_OPENAI_PREFILL_PROGRESS_SSE", []),+ ("_OPENAI_PASSTHROUGH_SSE_KEEPALIVE", ["run-1"]),+ ],)-async def test_a_silent_tool_holds_the_lease_but_a_stalled_stream_does_not(+async def test_progress_comments_hold_the_lease_but_a_stalled_stream_does_not(durable_run, monkeypatch, comment, reaped):comment = getattr(inference, comment)@@ -474,6 +478,144 @@ async def test_a_silent_tool_holds_the_lease_but_a_stalled_stream_does_not(assert sampled["reaped"] == reaped[email protected]("tools", [False, True], ids = ["chat", "tool-loop"])[email protected]("reports_progress, reaped", [(True, []), (False, ["run-1"])])+async def test_native_gguf_prefill_progress_holds_the_lease(+ durable_run, monkeypatch, tools, reports_progress, reaped+):+ """The Studio UI never takes the passthrough: its GGUF chat and tool loop run the native+ generators, which consume prompt_progress and stay silent until the first token. Advancing+ progress must still renew the lease through the stall loop; a silent generator must not."""+ from core.inference.llama_cpp import _report_live_llama_timings++ entered, release = threading.Event(), threading.Event()++ def _prefill(perf_callback):+ entered.set()+ processed = 0+ while not release.wait(0.005):+ if reports_progress:+ processed += 512+ # The same hook the real generators feed from each llama-server progress frame.+ _report_live_llama_timings(+ perf_callback,+ {"choices": [], "prompt_progress": {"total": 10**6, "processed": processed}},+ )++ def generate(*, perf_callback = None, **_kwargs):+ _prefill(perf_callback)+ yield "Hi"+ yield {"type": "metadata", "finish_reason": "stop"}++ def generate_with_tools(*, perf_callback = None, **_kwargs):+ _prefill(perf_callback)+ yield {"type": "content", "text": "Hi"}+ yield {"type": "metadata", "finish_reason": "stop"}++ llama = SimpleNamespace(+ is_loaded = True,+ model_identifier = "local.gguf",+ base_url = "http://llama.test",+ effective_parallel_slots = 1,+ supports_tools = tools,+ supports_tool_passthrough = tools,+ is_vision = False,+ _is_audio = False,+ context_length = None,+ generate_chat_completion = generate,+ generate_chat_completion_with_tools = generate_with_tools,+ release_idle_chat_slot = lambda *_a, **_k: False,+ count_chat_tokens = lambda *_a, **_k: 16,+ _request_reasoning_kwargs = lambda *_a, **_k: None,+ )+ monkeypatch.setattr(inference, "get_llama_cpp_backend", lambda: llama)+ monkeypatch.setattr(inference, "_automatic_model_load_may_run", lambda: False)++ async def no_switch(*_a, **_k):+ return None++ monkeypatch.setattr(inference, "_maybe_auto_switch_model", no_switch)+ monkeypatch.setattr(inference, "_LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S", 0.02)+ if not tools:+ monkeypatch.setattr(inference, "_effective_enable_tools", lambda _payload: False)+ now = {"ms": runs_db.now_ms()}+ monkeypatch.setattr(runs_db, "now_ms", lambda: now["ms"])+ real_produce = inference.produce_openai_chat_completions+ sampled = {"comments": set()}++ async def wrapped(payload, request, subject, *, cancel_on_disconnect):+ if tools:+ # What the UI sends with the web-search pill lit: server-side tools, no client tools.+ payload.enable_tools = True+ payload.enabled_tools = ["web_search"]+ response = await real_produce(+ payload, request, subject, cancel_on_disconnect = cancel_on_disconnect+ )+ inner = response.body_iterator+ # Preparation has already read its own interval; only the stream's rate limit is lifted.+ monkeypatch.setattr(chat_generation_runs, "_renew_interval_seconds", lambda: 0.0)++ async def body():+ comments = 0+ async for raw in inner:+ text = raw.decode() if isinstance(raw, bytes) else str(raw)+ if text.startswith(":") and entered.is_set():+ comments += 1+ sampled["comments"].add(text.strip())+ if comments == 2:+ now["ms"] += 21 * 60_000+ elif comments == 5 and "reaped" not in sampled:+ sampled["reaped"] = runs_db.reconcile_runs(stale_after_ms = 1_200_000)+ release.set()+ yield raw++ response.body_iterator = body()+ return response++ monkeypatch.setattr(inference, "produce_openai_chat_completions", wrapped)+ try:+ supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))+ await asyncio.wait_for(supervisor._produce("run-1"), 30)+ finally:+ release.set()+ assert sampled["reaped"] == reaped+ expected = ": prefill-progress" if reports_progress else ": keep-alive"+ assert expected in sampled["comments"], sampled["comments"]+ if not reports_progress:+ assert ": prefill-progress" not in sampled["comments"]+++def test_prefill_signal_counts_only_moving_progress():+ from core.inference.llama_cpp import _perf_callback_wants_timings++ forwarded = []+ bare = inference._PrefillProgressSignal(None)+ # Without a monitor the wrapper must not switch llama-server to per-token timings.+ assert _perf_callback_wants_timings(bare) is False+ assert bare.needs_phase is False+ assert bare.advanced() is False+ bare({"prompt_progress": {"processed": 10}})+ bare({"prompt_progress": {"processed": 10}})+ assert bare.advanced() is True+ assert bare.advanced() is False+ # A new prefill round restarts the count lower; that is still movement.+ bare({"prompt_progress": {"processed": 2}})+ assert bare.advanced() is True+ bare({"timings": {"predicted_n": 1}})+ assert bare.advanced() is False++ def monitor(sample):+ forwarded.append(sample)++ monitor.needs_phase = False+ wrapped = inference._PrefillProgressSignal(monitor)+ assert _perf_callback_wants_timings(wrapped) is True+ assert wrapped.needs_phase is False+ wrapped({"prompt_progress": {"processed": 1}})+ assert forwarded == [{"prompt_progress": {"processed": 1}}]++async def _subscriber_sequences(after = 0):response = await run_routes.chat_generation_events("run-1",diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.pyindex d5a0259e53d..b67eeb85921 100644--- a/studio/backend/tests/test_openai_tool_passthrough.py+++ b/studio/backend/tests/test_openai_tool_passthrough.py@@ -7501,7 +7501,9 @@ class TestApiMonitorProviderAndCompletionStreams:body = b"".join([chunk async for chunk in response.body_iterator])assert upstream_bodies[0]["return_progress"] is True- assert (b": keep-alive" in body) is not client_progress+ # Dropped progress stands in with ``: prefill-progress`` (lease renewal), not stall keep-alive.+ assert (b": prefill-progress" in body) is not client_progress+ assert (b": keep-alive" in body) is Falseassert upstream_bodies[0]["stream_options"]["include_usage"] is Trueassert b'"usage"' not in bodyassert (b"prompt_progress" in body) is client_progress