Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep the live generation line from moving the branch picker, and update it during short replies (#16449)

oobabooga0102e22d94fa
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 8947f99a351..62b4fca476e 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -1815,8 +1815,7 @@ _TEARDOWN_TASK_RECANCEL_S = 0.01
_LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S = 15.0
# How often a UI stream looks for new prefill progress while the generator is silent.
_LIVE_PROGRESS_POLL_S = 0.25
-# Minimum gap between two live decode frames: the client only redraws one line of text.
-_LIVE_DECODE_FRAME_INTERVAL_S = 1.0
+_LIVE_DECODE_FRAME_INTERVAL_S = 0.25
_APPROVAL_CACHE_RECLAIM_POLL_S = 0.25
@@ -7163,11 +7162,7 @@ def _live_rate(value: Any) -> Optional[float]:
class _PrefillProgressSignal:
- """Counts advancing ``prompt_progress`` so the stall loop can send ``: prefill-progress``.
-
- Native GGUF generators yield nothing until the first token, and ``: keep-alive`` does not renew the lease.
-
- With ``live`` it also feeds ``live_frame``, the chat UI's ``generation_progress`` frames."""
+ """Renew the lease during silent GGUF prefill and optionally report live generation figures."""
def __init__(
self,
@@ -7184,6 +7179,7 @@ class _PrefillProgressSignal:
self._prefill_sent: Optional[dict] = None
self._decode: Optional[dict] = None
self._decode_sent_at = 0.0
+ self._decode_started = False
self.wants_timings = inner is not None or live
@property
@@ -7219,6 +7215,7 @@ class _PrefillProgressSignal:
cached = _live_count(progress.get("cache"))
# A new prompt pass (the next round of a tool loop): the last round's decode figures must not follow it.
self._decode = None
+ self._decode_started = False
self._prefill = {
"type": "generation_progress",
"phase": "prefill",
@@ -7234,11 +7231,18 @@ class _PrefillProgressSignal:
tokens = _live_count(sample.get("predicted_n"))
if not tokens:
return
+ if not self._decode_started:
+ # The first token has no decode rate; allow a full interval for speculative timing to settle.
+ self._decode_started = True
+ self._decode_sent_at = time.monotonic()
+ rate = _live_rate(sample.get("predicted_per_second"))
+ if rate is None:
+ return
frame = {
"type": "generation_progress",
"phase": "decode",
"tokens": tokens,
- "tok_per_s": _live_rate(sample.get("predicted_per_second")),
+ "tok_per_s": rate,
}
draft_n = _live_count(sample.get("draft_n"))
draft_accepted = _live_count(sample.get("draft_n_accepted"))
diff --git a/studio/backend/tests/test_live_generation_progress.py b/studio/backend/tests/test_live_generation_progress.py
index 8ff0c82ba26..fa981375f00 100644
--- a/studio/backend/tests/test_live_generation_progress.py
+++ b/studio/backend/tests/test_live_generation_progress.py
@@ -74,6 +74,8 @@ def test_decode_frames_are_throttled(monkeypatch):
signal = route._PrefillProgressSignal(None, live = True)
signal({"predicted_n": 10, "predicted_per_second": 48.7})
+ assert signal.live_frame() is None, "the pass's first sample opens the interval"
+ clock["now"] += route._LIVE_DECODE_FRAME_INTERVAL_S
assert _frame(signal.live_frame()) == {
"type": "generation_progress",
"phase": "decode",
@@ -87,6 +89,26 @@ def test_decode_frames_are_throttled(monkeypatch):
assert signal.live_frame() is None, "the same sample twice"
+def test_the_first_frame_waits_an_interval_past_the_first_token(monkeypatch):
+ clock = {"now": 1000.0}
+ monkeypatch.setattr(route.time, "monotonic", lambda: clock["now"])
+ signal = route._PrefillProgressSignal(None, live = True)
+ signal({"predicted_n": 1, "predicted_per_second": 0.0})
+ assert signal.live_frame() is None, "a lone '1 token' held the line for a whole interval"
+ clock["now"] += 0.01
+ signal({"predicted_n": 2, "predicted_per_second": 12.5})
+ assert signal.live_frame() is None, "one step's rate is noise"
+ clock["now"] += route._LIVE_DECODE_FRAME_INTERVAL_S
+ signal({"predicted_n": 31, "predicted_per_second": 118.0})
+ assert _frame(signal.live_frame())["tokens"] == 31
+ signal({"prompt_progress": {"total": 3000, "processed": 3000}})
+ signal.live_frame()
+ clock["now"] += route._LIVE_DECODE_FRAME_INTERVAL_S
+ signal({"predicted_n": 1, "predicted_per_second": 0.0})
+ signal({"predicted_n": 2, "predicted_per_second": 40.0})
+ assert signal.live_frame() is None
+
+
def test_a_new_prompt_pass_drops_the_last_rounds_decode_figures():
signal = route._PrefillProgressSignal(None, live = True)
signal({"predicted_n": 50, "predicted_per_second": 40.0})
@@ -95,7 +117,8 @@ def test_a_new_prompt_pass_drops_the_last_rounds_decode_figures():
assert signal.live_frame() is None, "round 1's decode frame must not follow round 2's prefill"
-def test_decode_frame_carries_draft_acceptance():
+def test_decode_frame_carries_draft_acceptance(monkeypatch):
+ monkeypatch.setattr(route, "_LIVE_DECODE_FRAME_INTERVAL_S", 0.0)
signal = route._PrefillProgressSignal(None, live = True)
signal({"predicted_n": 40, "predicted_per_second": 90.0, "draft_n": 23, "draft_n_accepted": 14})
frame = _frame(signal.live_frame())
@@ -148,8 +171,9 @@ def test_stall_wait_sends_progress_while_the_generator_is_silent(monkeypatch):
assert [_frame(line)["processed"] for line in lines] == [2048]
-def test_a_step_faster_than_the_poll_still_sends_what_it_recorded():
+def test_a_step_faster_than_the_poll_still_sends_what_it_recorded(monkeypatch):
# Fast tokens (or tool-call markup stripped to no text) never reach the poll.
+ monkeypatch.setattr(route, "_LIVE_DECODE_FRAME_INTERVAL_S", 0.0)
signal = route._PrefillProgressSignal(None, live = True)
async def scenario():
diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx
index e40f26d52e7..125ee4dee14 100644
--- a/studio/frontend/src/components/assistant-ui/thread.tsx
+++ b/studio/frontend/src/components/assistant-ui/thread.tsx
@@ -8287,9 +8287,9 @@ const AssistantMessage: FC = () => {
<div className="aui-assistant-message-footer mt-1.5 -ml-[var(--icon-btn-inset)] flex min-h-8 min-w-0 flex-wrap items-center gap-y-1">
{/* First, so the action bar mounting on hover never moves it. */}
<AssistantMessageTime />
- {/* Only while streaming, where the action bar is hidden; the timing badge takes over after. */}
+ {/* Match the row height so wrapping live text cannot move the picker. */}
+ <BranchPicker className="mr-0.5 h-8" />
<LiveGenerationLine />
- <BranchPicker className="mr-0.5" />
<AssistantActionBar />
</div>
{/* Renders nothing. `If last` keeps the hook off the other N-1. */}
diff --git a/studio/frontend/tests/generation-details.test.ts b/studio/frontend/tests/generation-details.test.ts
index 264d86687bc..76ac5e8e9f5 100644
--- a/studio/frontend/tests/generation-details.test.ts
+++ b/studio/frontend/tests/generation-details.test.ts
@@ -250,6 +250,16 @@ test("Generation details is off by default and wired into Settings > Chat > Disp
assert.match(readSrc("features/settings/settings-search.ts"), /"settings\.chat\.showGenerationDetailsKeywords"/);
});
+test("the live line follows the branch picker, so its growing text never moves the picker", () => {
+ const thread = readSrc("components/assistant-ui/thread.tsx");
+ const footer = thread.indexOf("aui-assistant-message-footer");
+ const picker = thread.indexOf("<BranchPicker", footer);
+ const line = thread.indexOf("<LiveGenerationLine />", footer);
+ const bar = thread.indexOf("<AssistantActionBar />", footer);
+ assert.ok(footer !== -1 && picker > footer, "the picker sits in the footer");
+ assert.ok(line > picker && bar > line, "picker, then the live line, then the action bar");
+});
+
test("the live line gates on the preference before any assistant-ui selector", () => {
const line = readSrc("components/assistant-ui/live-generation-line.tsx");
const gate = line.slice(line.indexOf("export const LiveGenerationLine"), line.indexOf("const RunningGate"));