Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep the live generation line from moving the branch picker, and update it during short replies (#16449)
oobabooga0102e22d94fa
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 8947f99a351..62b4fca476e 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -1815,8 +1815,7 @@ _TEARDOWN_TASK_RECANCEL_S = 0.01_LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S = 15.0# How often a UI stream looks for new prefill progress while the generator is silent._LIVE_PROGRESS_POLL_S = 0.25-# Minimum gap between two live decode frames: the client only redraws one line of text.-_LIVE_DECODE_FRAME_INTERVAL_S = 1.0+_LIVE_DECODE_FRAME_INTERVAL_S = 0.25_APPROVAL_CACHE_RECLAIM_POLL_S = 0.25@@ -7163,11 +7162,7 @@ def _live_rate(value: Any) -> Optional[float]:class _PrefillProgressSignal:- """Counts advancing ``prompt_progress`` so the stall loop can send ``: prefill-progress``.-- Native GGUF generators yield nothing until the first token, and ``: keep-alive`` does not renew the lease.-- With ``live`` it also feeds ``live_frame``, the chat UI's ``generation_progress`` frames."""+ """Renew the lease during silent GGUF prefill and optionally report live generation figures."""def __init__(self,@@ -7184,6 +7179,7 @@ class _PrefillProgressSignal:self._prefill_sent: Optional[dict] = Noneself._decode: Optional[dict] = Noneself._decode_sent_at = 0.0+ self._decode_started = Falseself.wants_timings = inner is not None or live@property@@ -7219,6 +7215,7 @@ class _PrefillProgressSignal:cached = _live_count(progress.get("cache"))# A new prompt pass (the next round of a tool loop): the last round's decode figures must not follow it.self._decode = None+ self._decode_started = Falseself._prefill = {"type": "generation_progress","phase": "prefill",@@ -7234,11 +7231,18 @@ class _PrefillProgressSignal:tokens = _live_count(sample.get("predicted_n"))if not tokens:return+ if not self._decode_started:+ # The first token has no decode rate; allow a full interval for speculative timing to settle.+ self._decode_started = True+ self._decode_sent_at = time.monotonic()+ rate = _live_rate(sample.get("predicted_per_second"))+ if rate is None:+ returnframe = {"type": "generation_progress","phase": "decode","tokens": tokens,- "tok_per_s": _live_rate(sample.get("predicted_per_second")),+ "tok_per_s": rate,}draft_n = _live_count(sample.get("draft_n"))draft_accepted = _live_count(sample.get("draft_n_accepted"))diff --git a/studio/backend/tests/test_live_generation_progress.py b/studio/backend/tests/test_live_generation_progress.pyindex 8ff0c82ba26..fa981375f00 100644--- a/studio/backend/tests/test_live_generation_progress.py+++ b/studio/backend/tests/test_live_generation_progress.py@@ -74,6 +74,8 @@ def test_decode_frames_are_throttled(monkeypatch):signal = route._PrefillProgressSignal(None, live = True)signal({"predicted_n": 10, "predicted_per_second": 48.7})+ assert signal.live_frame() is None, "the pass's first sample opens the interval"+ clock["now"] += route._LIVE_DECODE_FRAME_INTERVAL_Sassert _frame(signal.live_frame()) == {"type": "generation_progress","phase": "decode",@@ -87,6 +89,26 @@ def test_decode_frames_are_throttled(monkeypatch):assert signal.live_frame() is None, "the same sample twice"+def test_the_first_frame_waits_an_interval_past_the_first_token(monkeypatch):+ clock = {"now": 1000.0}+ monkeypatch.setattr(route.time, "monotonic", lambda: clock["now"])+ signal = route._PrefillProgressSignal(None, live = True)+ signal({"predicted_n": 1, "predicted_per_second": 0.0})+ assert signal.live_frame() is None, "a lone '1 token' held the line for a whole interval"+ clock["now"] += 0.01+ signal({"predicted_n": 2, "predicted_per_second": 12.5})+ assert signal.live_frame() is None, "one step's rate is noise"+ clock["now"] += route._LIVE_DECODE_FRAME_INTERVAL_S+ signal({"predicted_n": 31, "predicted_per_second": 118.0})+ assert _frame(signal.live_frame())["tokens"] == 31+ signal({"prompt_progress": {"total": 3000, "processed": 3000}})+ signal.live_frame()+ clock["now"] += route._LIVE_DECODE_FRAME_INTERVAL_S+ signal({"predicted_n": 1, "predicted_per_second": 0.0})+ signal({"predicted_n": 2, "predicted_per_second": 40.0})+ assert signal.live_frame() is None++def test_a_new_prompt_pass_drops_the_last_rounds_decode_figures():signal = route._PrefillProgressSignal(None, live = True)signal({"predicted_n": 50, "predicted_per_second": 40.0})@@ -95,7 +117,8 @@ def test_a_new_prompt_pass_drops_the_last_rounds_decode_figures():assert signal.live_frame() is None, "round 1's decode frame must not follow round 2's prefill"-def test_decode_frame_carries_draft_acceptance():+def test_decode_frame_carries_draft_acceptance(monkeypatch):+ monkeypatch.setattr(route, "_LIVE_DECODE_FRAME_INTERVAL_S", 0.0)signal = route._PrefillProgressSignal(None, live = True)signal({"predicted_n": 40, "predicted_per_second": 90.0, "draft_n": 23, "draft_n_accepted": 14})frame = _frame(signal.live_frame())@@ -148,8 +171,9 @@ def test_stall_wait_sends_progress_while_the_generator_is_silent(monkeypatch):assert [_frame(line)["processed"] for line in lines] == [2048]-def test_a_step_faster_than_the_poll_still_sends_what_it_recorded():+def test_a_step_faster_than_the_poll_still_sends_what_it_recorded(monkeypatch):# Fast tokens (or tool-call markup stripped to no text) never reach the poll.+ monkeypatch.setattr(route, "_LIVE_DECODE_FRAME_INTERVAL_S", 0.0)signal = route._PrefillProgressSignal(None, live = True)async def scenario():diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsxindex e40f26d52e7..125ee4dee14 100644--- a/studio/frontend/src/components/assistant-ui/thread.tsx+++ b/studio/frontend/src/components/assistant-ui/thread.tsx@@ -8287,9 +8287,9 @@ const AssistantMessage: FC = () => {<div className="aui-assistant-message-footer mt-1.5 -ml-[var(--icon-btn-inset)] flex min-h-8 min-w-0 flex-wrap items-center gap-y-1">{/* First, so the action bar mounting on hover never moves it. */}<AssistantMessageTime />- {/* Only while streaming, where the action bar is hidden; the timing badge takes over after. */}+ {/* Match the row height so wrapping live text cannot move the picker. */}+ <BranchPicker className="mr-0.5 h-8" /><LiveGenerationLine />- <BranchPicker className="mr-0.5" /><AssistantActionBar /></div>{/* Renders nothing. `If last` keeps the hook off the other N-1. */}diff --git a/studio/frontend/tests/generation-details.test.ts b/studio/frontend/tests/generation-details.test.tsindex 264d86687bc..76ac5e8e9f5 100644--- a/studio/frontend/tests/generation-details.test.ts+++ b/studio/frontend/tests/generation-details.test.ts@@ -250,6 +250,16 @@ test("Generation details is off by default and wired into Settings > Chat > Dispassert.match(readSrc("features/settings/settings-search.ts"), /"settings\.chat\.showGenerationDetailsKeywords"/);});+test("the live line follows the branch picker, so its growing text never moves the picker", () => {+ const thread = readSrc("components/assistant-ui/thread.tsx");+ const footer = thread.indexOf("aui-assistant-message-footer");+ const picker = thread.indexOf("<BranchPicker", footer);+ const line = thread.indexOf("<LiveGenerationLine />", footer);+ const bar = thread.indexOf("<AssistantActionBar />", footer);+ assert.ok(footer !== -1 && picker > footer, "the picker sits in the footer");+ assert.ok(line > picker && bar > line, "picker, then the live line, then the action bar");+});+test("the live line gates on the preference before any assistant-ui selector", () => {const line = readSrc("components/assistant-ui/live-generation-line.tsx");const gate = line.slice(line.indexOf("export const LiveGenerationLine"), line.indexOf("const RunningGate"));