Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep queued prompts when the model is reloaded or switched (#13147)

Daniel Hana167b1324893
diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx
index b53fd44bd64..dbe029f3f68 100644
--- a/studio/frontend/src/components/assistant-ui/thread.tsx
+++ b/studio/frontend/src/components/assistant-ui/thread.tsx
@@ -469,6 +469,8 @@ type PromptQueueTarget = {
complete: () => void;
cancel: () => void;
cancelActiveRun: () => void;
+ /** Forget the captured local model so a held prompt resolves the one loaded when it is sent. */
+ releaseModel: () => void;
isIndexing: () => boolean;
usesThreadDocuments: boolean;
usesLocalModel: boolean;
@@ -1586,16 +1588,19 @@ function stopLocalPromptQueueRun(run: PromptQueueRun) {
})),
run.index,
);
- if (plan.retainedItemIndexes.length === run.items.length) {
+ if (!plan.cancelActiveItem && plan.heldItemIndexes.length === 0) {
return;
}
+ const heldItems = plan.heldItemIndexes.map((index) => run.items[index]);
run.items = plan.retainedItemIndexes.map((index) => run.items[index]);
+ for (const item of heldItems) {
+ // Held like a failed load: the row stays editable and Resume sends it to the model loaded then.
+ item.blockedByModelFailure = true;
+ item.target.releaseModel();
+ }
if (!getActivePromptQueueItem(run)) {
deletePromptQueueRun(run);
- if (!plan.cancelActiveItem) {
- return;
- }
try {
activeItem?.target.cancel();
} catch {
@@ -1603,27 +1608,33 @@ function stopLocalPromptQueueRun(run: PromptQueueRun) {
}
return;
}
- if (plan.activeItemRemoved) {
- clearPromptQueueRetryTimer(run);
- }
if (plan.cancelActiveItem) {
waitForPromptQueueTargetIdle(run);
try {
- activeItem?.target.cancel();
+ // A permanent cancel would also void the held prompts that share this target.
+ if (run.items.some((item) => item.target === activeItem?.target)) {
+ activeItem?.target.cancelActiveRun();
+ } else {
+ activeItem?.target.cancel();
+ }
} catch {
// The active local run may have already ended.
}
refreshPromptQueueTargetIdleWait(run);
return;
}
- syncPromptQueueUI();
- if (plan.refreshTargetIdleWait) {
- refreshPromptQueueTargetIdleWait(run);
- return;
- }
- if (plan.activeItemRemoved && run.index >= 0 && !run.waitingForTargetIdle) {
- requestPromptQueuePump(50);
+ if (getActivePromptQueueItem(run)?.blockedByModelFailure) {
+ // Invalidate an attempt already underway for the now-held item.
+ run.generation += 1;
+ clearPromptQueueRetryTimer(run);
+ promptQueueDispatchingRunIds.delete(run.id);
+ promptQueueActiveRunIds.delete(run.id);
+ // The cleared timer may be the poll that sees the run this queue waits behind end.
+ if (shouldPollPromptQueueTargetState(run)) {
+ refreshPromptQueueTargetIdleWait(run);
+ }
}
+ syncPromptQueueUI();
}
function stopLocalPromptQueueRunsForThreadIds(threadIds: string[]) {
@@ -4006,7 +4017,7 @@ const Composer: FC<{
const usesKnowledgeBaseAtQueueStart =
chatStateAtQueueStart.ragEnabled &&
chatStateAtQueueStart.ragSource.type === "kb";
- const deferModelResolution =
+ let deferModelResolution =
chatStateAtQueueStart.modelLoading &&
parseExternalModelId(
chatStateAtQueueStart.loadingModelPick &&
@@ -4263,6 +4274,12 @@ const Composer: FC<{
discardOldestPendingSettings();
getThreadRuntime()?.cancelRun();
},
+ releaseModel: () => {
+ // The same snapshot a queue started during a model load takes.
+ deferModelResolution = true;
+ runSettingsAtQueueStart.params.checkpoint = "";
+ runSettingsAtQueueStart.activeGgufVariant = null;
+ },
isIndexing: () =>
promptQueueTargetMountedRef.current &&
isTargetCurrentThread() &&
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index fd90146481c..e6406ffb606 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -95,7 +95,7 @@ import {
} from "../utils/confirm-stop-running-chats";
import {
requestLocalPromptQueueStop,
- requestPromptQueueStop,
+ requestScopedLocalPromptQueueStop,
notifyLocalPromptQueueLoadFailed,
} from "../utils/prompt-queue-boundary";
import { cancelPreStreamRunReservations } from "../utils/pre-stream-run-reservation";
@@ -642,7 +642,7 @@ function publishLoadedModels(
function stopQueuedRuns(decision: StopRunningChatsDecision, scoped: boolean): void {
if (scoped) {
- requestPromptQueueStop(decision.promptQueueThreadIds);
+ requestScopedLocalPromptQueueStop(decision.promptQueueThreadIds);
return;
}
cancelPreStreamRunReservations(decision.preStreamRunTokens);
diff --git a/studio/frontend/src/features/chat/utils/prompt-queue-boundary.ts b/studio/frontend/src/features/chat/utils/prompt-queue-boundary.ts
index 79a557ede38..f1ef5549baf 100644
--- a/studio/frontend/src/features/chat/utils/prompt-queue-boundary.ts
+++ b/studio/frontend/src/features/chat/utils/prompt-queue-boundary.ts
@@ -64,6 +64,19 @@ export function requestLocalPromptQueueStop(
);
}
+/** The same local stop for the chats on one model of several: holds their unsent prompts without
+ * advancing the boundary, which would also void queue starts on the models left loaded. */
+export function requestScopedLocalPromptQueueStop(threadIds: string[]) {
+ if (typeof window === "undefined" || threadIds.length === 0) {
+ return;
+ }
+ window.dispatchEvent(
+ new CustomEvent<PromptQueueStopEventDetail>(PROMPT_QUEUE_STOP_EVENT, {
+ detail: { threadIds, localOnly: true },
+ }),
+ );
+}
+
export function requestTemporaryPromptQueueStop() {
const threadIds = [
...new Set(
diff --git a/studio/frontend/src/features/chat/utils/prompt-queue-model-boundary.ts b/studio/frontend/src/features/chat/utils/prompt-queue-model-boundary.ts
index 663e60e2579..728a73b90ff 100644
--- a/studio/frontend/src/features/chat/utils/prompt-queue-model-boundary.ts
+++ b/studio/frontend/src/features/chat/utils/prompt-queue-model-boundary.ts
@@ -22,35 +22,32 @@ export type PromptQueueModelStopItem = {
export type LocalPromptQueueStopPlan = {
cancelActiveItem: boolean;
- activeItemRemoved: boolean;
- refreshTargetIdleWait: boolean;
retainedItemIndexes: number[];
+ /** Indexes into the original items of unsent local prompts to hold until Resume. */
+ heldItemIndexes: number[];
};
-/** Preserve external-provider work when the singleton local model changes. Only local items depend
- * on the outgoing model. A dispatched local item must be cancelled, but external follow-ups
- * remain valid and can resume once the thread runtime becomes idle. */
+/** A local model change ends the local reply in flight, so that item is cancelled and dropped as a
+ * Stop drops it. Unsent local prompts are held for Resume instead of discarded (#10428), and
+ * external-provider items are untouched since they never used the outgoing model. */
export function planLocalPromptQueueStop(
items: readonly PromptQueueModelStopItem[],
runIndex: number,
): LocalPromptQueueStopPlan {
const activeIndex = Math.max(runIndex, 0);
const activeItem = items[activeIndex];
- const retainedItemIndexes = items.flatMap((item, index) =>
- index < activeIndex || !item.usesLocalModel ? [index] : [],
+ const cancelActiveItem = Boolean(
+ activeItem?.usesLocalModel && activeItem.dispatched,
);
- return {
- cancelActiveItem: Boolean(
- activeItem?.usesLocalModel && activeItem.dispatched,
- ),
- activeItemRemoved: Boolean(activeItem?.usesLocalModel),
- refreshTargetIdleWait: Boolean(
- runIndex < 0 &&
- activeItem?.usesLocalModel &&
- retainedItemIndexes.length > 0,
- ),
- retainedItemIndexes,
- };
+ const retainedItemIndexes = items.flatMap((_, index) =>
+ cancelActiveItem && index === activeIndex ? [] : [index],
+ );
+ const heldItemIndexes = items.flatMap((item, index) =>
+ index >= activeIndex && item.usesLocalModel && !item.dispatched
+ ? [index]
+ : [],
+ );
+ return { cancelActiveItem, retainedItemIndexes, heldItemIndexes };
}
export function shouldAbortPendingQueueForModelBoundary({
diff --git a/studio/frontend/tests/prompt-queue-model-boundary.test.ts b/studio/frontend/tests/prompt-queue-model-boundary.test.ts
index 5af3923a4c9..eeabe5d95fa 100644
--- a/studio/frontend/tests/prompt-queue-model-boundary.test.ts
+++ b/studio/frontend/tests/prompt-queue-model-boundary.test.ts
@@ -8,7 +8,7 @@ import {
shouldAbortPendingQueueForSettingsChange,
} from "../src/features/chat/utils/prompt-queue-model-boundary.ts";
-test("a local model stop preserves an active external item", () => {
+test("a local model stop preserves an active external item and holds local follow-ups", () => {
assert.deepEqual(
planLocalPromptQueueStop(
[
@@ -20,9 +20,8 @@ test("a local model stop preserves an active external item", () => {
),
{
cancelActiveItem: false,
- activeItemRemoved: false,
- refreshTargetIdleWait: false,
- retainedItemIndexes: [0, 2],
+ retainedItemIndexes: [0, 1, 2],
+ heldItemIndexes: [1],
},
);
});
@@ -38,14 +37,48 @@ test("a dispatched local item is cancelled without losing external follow-ups",
),
{
cancelActiveItem: true,
- activeItemRemoved: true,
- refreshTargetIdleWait: false,
retainedItemIndexes: [1],
+ heldItemIndexes: [],
},
);
});
-test("an undispatched local item is dropped without losing external follow-ups", () => {
+test("a reload mid-reply drops only the reply in flight and holds the queued prompts (#10428)", () => {
+ assert.deepEqual(
+ planLocalPromptQueueStop(
+ [
+ { usesLocalModel: true, dispatched: true },
+ { usesLocalModel: true, dispatched: false },
+ { usesLocalModel: true, dispatched: false },
+ ],
+ 0,
+ ),
+ {
+ cancelActiveItem: true,
+ retainedItemIndexes: [1, 2],
+ heldItemIndexes: [1, 2],
+ },
+ );
+});
+
+test("a paused queue keeps every prompt across a reload (#10428)", () => {
+ assert.deepEqual(
+ planLocalPromptQueueStop(
+ [
+ { usesLocalModel: true, dispatched: false },
+ { usesLocalModel: true, dispatched: false },
+ ],
+ 0,
+ ),
+ {
+ cancelActiveItem: false,
+ retainedItemIndexes: [0, 1],
+ heldItemIndexes: [0, 1],
+ },
+ );
+});
+
+test("a queue waiting on a direct send is held, not dropped", () => {
assert.deepEqual(
planLocalPromptQueueStop(
[
@@ -56,14 +89,13 @@ test("an undispatched local item is dropped without losing external follow-ups",
),
{
cancelActiveItem: false,
- activeItemRemoved: true,
- refreshTargetIdleWait: true,
- retainedItemIndexes: [1],
+ retainedItemIndexes: [0, 1],
+ heldItemIndexes: [0],
},
);
});
-test("completed queue history is preserved when pending local work is dropped", () => {
+test("completed queue history is preserved and only pending local work is held", () => {
assert.deepEqual(
planLocalPromptQueueStop(
[
@@ -76,9 +108,8 @@ test("completed queue history is preserved when pending local work is dropped",
),
{
cancelActiveItem: false,
- activeItemRemoved: false,
- refreshTargetIdleWait: false,
- retainedItemIndexes: [0, 1, 3],
+ retainedItemIndexes: [0, 1, 2, 3],
+ heldItemIndexes: [2],
},
);
});
diff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.ts
index 60a1943592f..875e32011c6 100644
--- a/studio/frontend/tests/resident-config-match.test.ts
+++ b/studio/frontend/tests/resident-config-match.test.ts
@@ -2012,7 +2012,7 @@ test("a pick asks the status about its own model and keeps or replaces the other
test("ejects stop only the ejected model's chats; eject all asks once and unloads the others first", () => {
assert.match(
USE_CHAT_MODEL_RUNTIME,
- /function stopQueuedRuns\(decision: StopRunningChatsDecision, scoped: boolean\): void \{\s*if \(scoped\) \{\s*requestPromptQueueStop\(decision\.promptQueueThreadIds\);\s*return;\s*\}\s*cancelPreStreamRunReservations\(decision\.preStreamRunTokens\);\s*requestLocalPromptQueueStop\(decision\.promptQueueThreadIds\);/,
+ /function stopQueuedRuns\(decision: StopRunningChatsDecision, scoped: boolean\): void \{\s*if \(scoped\) \{\s*requestScopedLocalPromptQueueStop\(decision\.promptQueueThreadIds\);\s*return;\s*\}\s*cancelPreStreamRunReservations\(decision\.preStreamRunTokens\);\s*requestLocalPromptQueueStop\(decision\.promptQueueThreadIds\);/,
);
assert.match(
USE_CHAT_MODEL_RUNTIME,
diff --git a/tests/studio/test_multi_chat_prompt_queue_contract.py b/tests/studio/test_multi_chat_prompt_queue_contract.py
index 096988e6034..c85151ae4f6 100644
--- a/tests/studio/test_multi_chat_prompt_queue_contract.py
+++ b/tests/studio/test_multi_chat_prompt_queue_contract.py
@@ -602,7 +602,9 @@ def test_queued_settings_are_thread_scoped_without_cross_chat_fallback():
"function stopLocalPromptQueueRun(run: PromptQueueRun)",
"function stopLocalPromptQueueRunsForThreadIds(threadIds: string[])",
)
- assert "if (plan.refreshTargetIdleWait)" in local_queue_stop
+ # #10428: a model change holds unsent local prompts for Resume instead of dropping them.
+ assert "item.blockedByModelFailure = true;" in local_queue_stop
+ assert "item.target.releaseModel();" in local_queue_stop
assert "refreshPromptQueueTargetIdleWait(run);" in local_queue_stop
assert "claimPreStreamRunReservation(reservationToken);" in RUNTIME_PROVIDER
assert "if (!reservation.claimed)" in PRE_STREAM_RESERVATION