Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio chat: publish held stream text on the next frame instead of waiting for the next chunk (#15998)

Daniel Han35310d51fd5c
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index 9db85dc085f..8833ec10e63 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -372,7 +372,11 @@ import {
createTrailingPlaceholderWatch,
stripTrailingTemplatePlaceholder,
} from "../utils/trailing-template-placeholder";
-import { createStreamPublishGate } from "../utils/stream-pacing";
+import {
+ TRAILING_PUBLISH,
+ createStreamPublishGate,
+ withTrailingPublish,
+} from "../utils/stream-pacing";
import {
countReasoningGroups,
createReasoningDurationTracker,
@@ -7101,7 +7105,7 @@ export function createOpenAIStreamAdapter(
}
};
// The window lets a length-stop tell user Max Tokens from the backend's stand-in for Max.
- const stream =
+ const sourceStream =
generationDecision === "durable"
? durableStream()
: streamChatCompletions(
@@ -7111,8 +7115,25 @@ export function createOpenAIStreamAdapter(
);
// Per run, not per module: two turns must not share a cycle.
const canPublish = createStreamPublishGate();
+ const stream = withTrailingPublish(sourceStream, canPublish);
for await (const chunk of stream) {
+ if (chunk === TRAILING_PUBLISH) {
+ if (canPublish(streamedChars)) {
+ yield {
+ content: liveAssistantContent(),
+ metadata: {
+ timing: buildTiming(
+ streamStartTime,
+ totalChunks,
+ firstTokenTime,
+ ),
+ custom: liveCustom(),
+ },
+ };
+ }
+ continue;
+ }
const chunkModel = (chunk as { model?: unknown }).model;
if (typeof chunkModel === "string" && chunkModel.length > 0) {
responseModelId = chunkModel;
@@ -7373,6 +7394,7 @@ export function createOpenAIStreamAdapter(
if (toolEvent.type === "context_window_exceeded") {
contextWindowExceeded = true;
// assistant-ui saves the last STREAMED yield, so publish why the turn ended now.
+ canPublish.published();
yield {
content: liveAssistantContent(),
metadata: {
@@ -7442,6 +7464,7 @@ export function createOpenAIStreamAdapter(
),
} as PositionedToolCallPart;
// A stop before the next chunk saves only what was yielded.
+ canPublish.published();
yield {
content: liveAssistantContent(),
metadata: {
@@ -7776,6 +7799,7 @@ export function createOpenAIStreamAdapter(
};
}
}
+ canPublish.published();
yield {
content: liveAssistantContent(),
metadata: {
@@ -8374,6 +8398,7 @@ export function createOpenAIStreamAdapter(
const forcePublish =
addedToolCall || replayStateChanged || stampedProvenance;
if (forcePublish || canPublish(streamedChars)) {
+ canPublish.published();
yield {
content: liveAssistantContent(),
metadata: {
@@ -8395,6 +8420,7 @@ export function createOpenAIStreamAdapter(
if (replayStateChanged && !delta && !reasoning) {
const replayContent = liveAssistantContent();
if (replayContent.length > 0) {
+ canPublish.published();
yield {
content: replayContent,
metadata: {
@@ -8485,6 +8511,7 @@ export function createOpenAIStreamAdapter(
}
if (assistantContent.length > 0) {
+ canPublish.published();
yield {
content: assistantContent,
metadata: {
diff --git a/studio/frontend/src/features/chat/utils/stream-pacing.ts b/studio/frontend/src/features/chat/utils/stream-pacing.ts
index 415b56b2240..c6348bcfff6 100644
--- a/studio/frontend/src/features/chat/utils/stream-pacing.ts
+++ b/studio/frontend/src/features/chat/utils/stream-pacing.ts
@@ -7,14 +7,29 @@ export const UNPAINTED_REOPEN_MS = 500;
/** assistant-ui drops output after an abort, so this bounds what Stop can discard. */
export const MAX_HELD_CHARS = 256;
+export type StreamPublishGate = ((streamed: number) => boolean) & {
+ /** Resolves when the gate reopens while text it refused is still unpublished; null when nothing is held. */
+ heldUntilReopen(): Promise<void> | null;
+ /** An unpaced publish sent the held text. */
+ published(): void;
+ /** Cancels the pending frame and timer; the gate stays open. */
+ dispose(): void;
+};
+
/** One publish per frame, but publishes anyway once MAX_HELD_CHARS have arrived. */
-export function createStreamPublishGate(): (streamed: number) => boolean {
+export function createStreamPublishGate(): StreamPublishGate {
let open = true;
let publishedAt = 0;
- return (streamed: number) => {
+ let held = false;
+ let cancelCycle: (() => void) | null = null;
+ let reopenWaiter: Promise<void> | null = null;
+ let wakeWaiter: (() => void) | null = null;
+ const canPublish = (streamed: number) => {
if (!open && streamed - publishedAt < MAX_HELD_CHARS) {
+ held = true;
return false;
}
+ held = false;
publishedAt = streamed;
if (open) {
open = false;
@@ -25,9 +40,9 @@ export function createStreamPublishGate(): (streamed: number) => boolean {
frame?: number;
timer?: ReturnType<typeof setTimeout>;
} = {};
- const reopen = () => {
+ const settle = () => {
if (reopened) {
- return;
+ return false;
}
reopened = true;
if (handles.frame !== undefined) {
@@ -37,10 +52,98 @@ export function createStreamPublishGate(): (streamed: number) => boolean {
clearTimeout(handles.timer);
}
open = true;
+ cancelCycle = null;
+ return true;
+ };
+ const reopen = () => {
+ if (!settle()) {
+ return;
+ }
+ const wake = wakeWaiter;
+ reopenWaiter = null;
+ wakeWaiter = null;
+ wake?.();
};
+ cancelCycle = settle;
handles.frame = requestAnimationFrame(reopen);
handles.timer = setTimeout(reopen, UNPAINTED_REOPEN_MS);
}
return true;
};
+ return Object.assign(canPublish, {
+ heldUntilReopen(): Promise<void> | null {
+ if (!held) {
+ return null;
+ }
+ if (open) {
+ return Promise.resolve();
+ }
+ reopenWaiter ??= new Promise<void>((resolve) => {
+ wakeWaiter = resolve;
+ });
+ return reopenWaiter;
+ },
+ published() {
+ held = false;
+ },
+ dispose() {
+ cancelCycle?.();
+ reopenWaiter = null;
+ wakeWaiter = null;
+ held = false;
+ },
+ });
+}
+
+export const TRAILING_PUBLISH: unique symbol = Symbol("trailing publish");
+
+/** Adds TRAILING_PUBLISH when the gate reopens with text held; a ready item always wins, so order holds. */
+export async function* withTrailingPublish<T>(
+ source: AsyncIterable<T>,
+ gate: StreamPublishGate,
+): AsyncGenerator<T | typeof TRAILING_PUBLISH, void, undefined> {
+ const iterator = source[Symbol.asyncIterator]();
+ let pending: Promise<IteratorResult<T>> | null = null;
+ let exhausted = false;
+ try {
+ while (true) {
+ pending ??= iterator.next();
+ const reopened = gate.heldUntilReopen();
+ const next = reopened
+ ? await Promise.race([
+ pending,
+ reopened.then((): typeof TRAILING_PUBLISH => TRAILING_PUBLISH),
+ ])
+ : await pending;
+ if (next === TRAILING_PUBLISH) {
+ yield TRAILING_PUBLISH;
+ continue;
+ }
+ pending = null;
+ if (next.done) {
+ exhausted = true;
+ return;
+ }
+ yield next.value;
+ }
+ } catch (error) {
+ exhausted = true;
+ throw error;
+ } finally {
+ gate.dispose();
+ if (!exhausted) {
+ if (pending) {
+ // return() would queue behind the in-flight read, so don't block the caller's exit on it.
+ const inFlight = pending;
+ void inFlight
+ .then(
+ () => iterator.return?.(),
+ () => undefined,
+ )
+ .catch(() => undefined);
+ } else {
+ await iterator.return?.();
+ }
+ }
+ }
}
diff --git a/studio/frontend/tests/chat-stream-trailing-publish.test.ts b/studio/frontend/tests/chat-stream-trailing-publish.test.ts
new file mode 100644
index 00000000000..edcb4cf4438
--- /dev/null
+++ b/studio/frontend/tests/chat-stream-trailing-publish.test.ts
@@ -0,0 +1,371 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import * as pacing from "../src/features/chat/utils/stream-pacing.ts";
+import { readSrc } from "./helpers/kit.ts";
+
+const { MAX_HELD_CHARS, createStreamPublishGate } = pacing;
+// Looked up rather than imported by name, so a missing export fails the test instead of the file.
+const withTrailingPublish = (pacing as Record<string, unknown>)
+ .withTrailingPublish as typeof pacing.withTrailingPublish | undefined;
+const TRAILING_PUBLISH = (pacing as Record<string, unknown>).TRAILING_PUBLISH as
+ | typeof pacing.TRAILING_PUBLISH
+ | undefined;
+
+type Clock = {
+ frame(): void;
+ framesQueued(): number;
+ cancelledFrames: number[];
+ timersArmed: number;
+ timersCleared: number;
+};
+
+async function withFakeClock(body: (clock: Clock) => Promise<void>) {
+ const globals = globalThis as unknown as Record<string, unknown>;
+ const real = {
+ requestAnimationFrame: globals.requestAnimationFrame,
+ cancelAnimationFrame: globals.cancelAnimationFrame,
+ setTimeout: globals.setTimeout,
+ clearTimeout: globals.clearTimeout,
+ };
+ let nextHandle = 1;
+ let frames = new Map<number, () => void>();
+ const clock: Clock = {
+ frame() {
+ const due = frames;
+ frames = new Map();
+ for (const cb of due.values()) cb();
+ },
+ framesQueued: () => frames.size,
+ cancelledFrames: [],
+ timersArmed: 0,
+ timersCleared: 0,
+ };
+ globals.requestAnimationFrame = (cb: () => void) => {
+ const handle = nextHandle++;
+ frames.set(handle, cb);
+ return handle;
+ };
+ globals.cancelAnimationFrame = (handle: number) => {
+ clock.cancelledFrames.push(handle);
+ frames.delete(handle);
+ };
+ globals.setTimeout = () => {
+ clock.timersArmed += 1;
+ return nextHandle++;
+ };
+ globals.clearTimeout = () => {
+ clock.timersCleared += 1;
+ };
+ try {
+ await body(clock);
+ } finally {
+ Object.assign(globals, real);
+ }
+}
+
+const settle = () => new Promise<void>((resolve) => setImmediate(resolve));
+
+function channel<T>() {
+ const queue: T[] = [];
+ let ended = false;
+ let failure: unknown = null;
+ let waiter: ((r: IteratorResult<T>) => void) | null = null;
+ let rejecter: ((e: unknown) => void) | null = null;
+ let returned = 0;
+ const flush = () => {
+ if (!waiter) return;
+ const resolve = waiter;
+ const reject = rejecter!;
+ if (queue.length > 0) {
+ waiter = rejecter = null;
+ resolve({ value: queue.shift()!, done: false });
+ } else if (failure !== null) {
+ waiter = rejecter = null;
+ reject(failure);
+ } else if (ended) {
+ waiter = rejecter = null;
+ resolve({ value: undefined, done: true });
+ }
+ };
+ const iterable: AsyncIterable<T> = {
+ [Symbol.asyncIterator]() {
+ return {
+ next: () =>
+ new Promise<IteratorResult<T>>((resolve, reject) => {
+ waiter = resolve;
+ rejecter = reject;
+ flush();
+ }),
+ return: async () => {
+ returned += 1;
+ return { value: undefined, done: true };
+ },
+ };
+ },
+ };
+ return {
+ iterable,
+ push(...items: T[]) {
+ queue.push(...items);
+ flush();
+ },
+ end() {
+ ended = true;
+ flush();
+ },
+ fail(error: unknown) {
+ failure = error;
+ flush();
+ },
+ get returned() {
+ return returned;
+ },
+ };
+}
+
+/** The adapter's loop in miniature: text publishes through the gate, held text on TRAILING_PUBLISH. */
+function consume(source: AsyncIterable<string>, clock: Clock) {
+ assert.ok(withTrailingPublish, "stream-pacing has no trailing publish");
+ const gate = createStreamPublishGate();
+ const published: { text: string; frame: number }[] = [];
+ let frameNo = 0;
+ const realFrame = clock.frame;
+ clock.frame = () => {
+ frameNo += 1;
+ realFrame();
+ };
+ let text = "";
+ const done = (async () => {
+ for await (const item of withTrailingPublish(source, gate)) {
+ if (item === TRAILING_PUBLISH) {
+ if (gate(text.length)) published.push({ text, frame: frameNo });
+ continue;
+ }
+ text += item as string;
+ if (gate(text.length)) published.push({ text, frame: frameNo });
+ }
+ published.push({ text, frame: frameNo }); // the adapter's final yield
+ })();
+ return {
+ published,
+ done,
+ get text() {
+ return text;
+ },
+ };
+}
+
+test("held text is published on the next frame without waiting for another chunk", async () => {
+ await withFakeClock(async (clock) => {
+ const source = channel<string>();
+ const run = consume(source.iterable, clock);
+
+ source.push("a", "b", "c");
+ await settle();
+ assert.deepEqual(
+ run.published.map((p) => p.text),
+ ["a"],
+ );
+
+ clock.frame();
+ await settle();
+ assert.deepEqual(
+ run.published.map((p) => p.text),
+ ["a", "abc"],
+ "the held tail waited for the next chunk instead of the next frame",
+ );
+
+ source.end();
+ await run.done;
+ assert.equal(run.published.at(-1)?.text, "abc");
+ });
+});
+
+test("at most one paced publish per frame, and text and order are unchanged", async () => {
+ await withFakeClock(async (clock) => {
+ const source = channel<string>();
+ const run = consume(source.iterable, clock);
+ // Deterministic bursty arrival pattern: 0-4 chunks between frames, some frames idle.
+ const pattern = [3, 0, 1, 4, 0, 0, 2, 1, 3, 0, 4, 1, 0, 2, 2, 0, 1, 3];
+ const sent: string[] = [];
+ let n = 0;
+ for (const burst of pattern) {
+ for (let i = 0; i < burst; i += 1) {
+ const token = `t${n++} `;
+ sent.push(token);
+ source.push(token);
+ await settle();
+ }
+ clock.frame();
+ await settle();
+ // Every burst is fully visible by the frame after it: nothing waits for a later chunk.
+ assert.equal(
+ run.published.at(-1)?.text ?? "",
+ sent.join(""),
+ `text sent before frame ${run.published.length} is still held after it`,
+ );
+ }
+ source.end();
+ await run.done;
+
+ const paced = run.published.slice(0, -1);
+ const perFrame = new Map<number, number>();
+ for (const p of paced)
+ perFrame.set(p.frame, (perFrame.get(p.frame) ?? 0) + 1);
+ for (const [frame, count] of perFrame) {
+ assert.ok(count <= 1, `frame ${frame} published ${count} times`);
+ }
+ // Each publish is a prefix of the next, ending with everything sent, in order.
+ const full = sent.join("");
+ let previous = "";
+ for (const p of run.published) {
+ assert.ok(p.text.startsWith(previous) && full.startsWith(p.text));
+ previous = p.text;
+ }
+ assert.equal(run.text, full);
+ assert.equal(run.published.at(-1)?.text, full);
+ });
+});
+
+test("nothing held means no trailing publish and no extra render", async () => {
+ await withFakeClock(async (clock) => {
+ const source = channel<string>();
+ const run = consume(source.iterable, clock);
+ // Slower than the frame rate: every chunk lands on an open gate.
+ for (let i = 0; i < 5; i += 1) {
+ source.push(`w${i}`);
+ await settle();
+ clock.frame();
+ await settle();
+ clock.frame();
+ await settle();
+ }
+ source.end();
+ await run.done;
+ assert.deepEqual(
+ run.published.map((p) => p.text),
+ ["w0", "w0w1", "w0w1w2", "w0w1w2w3", "w0w1w2w3w4", "w0w1w2w3w4"],
+ );
+ });
+});
+
+test("a publish the gate did not pace clears the held text", async () => {
+ await withFakeClock(async () => {
+ assert.ok(withTrailingPublish);
+ const gate = createStreamPublishGate();
+ assert.equal(gate(1), true);
+ assert.equal(gate(2), false);
+ assert.ok(gate.heldUntilReopen(), "refused text is held");
+ gate.published();
+ assert.equal(
+ gate.heldUntilReopen(),
+ null,
+ "a forced publish already showed it",
+ );
+ });
+});
+
+test("the cap still publishes mid-frame and leaves nothing held", async () => {
+ await withFakeClock(async () => {
+ const gate = createStreamPublishGate();
+ gate(0);
+ assert.equal(gate(1), false);
+ assert.equal(gate(MAX_HELD_CHARS), true);
+ assert.equal(gate.heldUntilReopen(), null);
+ });
+});
+
+test("an early exit cancels the pending frame and does not wait on an in-flight read", async () => {
+ await withFakeClock(async (clock) => {
+ assert.ok(withTrailingPublish && TRAILING_PUBLISH);
+ const source = channel<string>();
+ const gate = createStreamPublishGate();
+ const iterator = withTrailingPublish(source.iterable, gate);
+ source.push("a", "b");
+ assert.equal((await iterator.next()).value, "a");
+ gate(1);
+ assert.equal((await iterator.next()).value, "b");
+ assert.equal(gate(2), false, "held");
+ const trailing = iterator.next();
+ clock.frame();
+ assert.equal((await trailing).value, TRAILING_PUBLISH);
+ gate(2); // the trailing publish closes the gate and arms a new frame
+ assert.equal(clock.framesQueued(), 1);
+ // The consumer leaves (Stop / thread switch) while a read is in flight.
+ await iterator.return(undefined);
+ assert.equal(
+ clock.framesQueued(),
+ 0,
+ "the pending frame was not cancelled",
+ );
+ assert.equal(
+ source.returned,
+ 0,
+ "return waits for the in-flight read, off the exit path",
+ );
+ source.push("c");
+ await settle();
+ assert.equal(
+ source.returned,
+ 1,
+ "the source is closed once its read settles",
+ );
+ });
+});
+
+test("a source error and a clean end pass through unchanged", async () => {
+ await withFakeClock(async (clock) => {
+ const source = channel<string>();
+ const run = consume(source.iterable, clock);
+ source.push("x", "y");
+ await settle();
+ source.fail(new Error("boom"));
+ await assert.rejects(run.done, /boom/);
+ assert.equal(clock.framesQueued(), 0, "the frame outlived the stream");
+ });
+});
+
+test("the adapter loop publishes held text when the gate reopens", () => {
+ const adapter = readSrc("features/chat/api/chat-adapter.ts");
+ const gate = adapter.indexOf("const canPublish = createStreamPublishGate();");
+ const wrap = adapter.indexOf(
+ "const stream = withTrailingPublish(sourceStream, canPublish);",
+ );
+ const loop = adapter.indexOf("for await (const chunk of stream) {");
+ assert.ok(
+ gate !== -1 && wrap > gate && loop > wrap,
+ "the stream is not wrapped",
+ );
+ const head = adapter.slice(loop, loop + 700);
+ assert.match(
+ head,
+ /if \(chunk === TRAILING_PUBLISH\) \{\s*if \(canPublish\(streamedChars\)\) \{\s*yield \{\s*content: liveAssistantContent\(\)/,
+ "the trailing publish must go through the gate",
+ );
+});
+
+test("every live publish in the loop is paced or clears the held text", () => {
+ const adapter = readSrc("features/chat/api/chat-adapter.ts");
+ const start = adapter.indexOf("for await (const chunk of stream) {");
+ const end = adapter.indexOf("} catch (streamError) {", start);
+ assert.ok(start !== -1 && end > start, "the stream loop moved");
+ const loop = adapter.slice(start, end);
+ const yields = [...loop.matchAll(/yield \{\s*content:/g)];
+ assert.ok(
+ yields.length >= 7,
+ `expected the live yields, found ${yields.length}`,
+ );
+ for (const match of yields) {
+ const before = loop.slice(Math.max(0, match.index - 240), match.index);
+ // An unpaced publish that leaves `held` set is repeated, unchanged, on the next frame.
+ assert.ok(
+ before.includes("canPublish.published();") ||
+ /if \(canPublish\(streamedChars\)\) \{\s*$/.test(before),
+ `a live yield neither paced nor marked published: ${loop.slice(match.index, match.index + 60)}`,
+ );
+ }
+});