Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep queued prompts when the model is reloaded or switched (#13147)
Daniel Hana167b1324893
diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsxindex b53fd44bd64..dbe029f3f68 100644--- a/studio/frontend/src/components/assistant-ui/thread.tsx+++ b/studio/frontend/src/components/assistant-ui/thread.tsx@@ -469,6 +469,8 @@ type PromptQueueTarget = {complete: () => void;cancel: () => void;cancelActiveRun: () => void;+ /** Forget the captured local model so a held prompt resolves the one loaded when it is sent. */+ releaseModel: () => void;isIndexing: () => boolean;usesThreadDocuments: boolean;usesLocalModel: boolean;@@ -1586,16 +1588,19 @@ function stopLocalPromptQueueRun(run: PromptQueueRun) {})),run.index,);- if (plan.retainedItemIndexes.length === run.items.length) {+ if (!plan.cancelActiveItem && plan.heldItemIndexes.length === 0) {return;}+ const heldItems = plan.heldItemIndexes.map((index) => run.items[index]);run.items = plan.retainedItemIndexes.map((index) => run.items[index]);+ for (const item of heldItems) {+ // Held like a failed load: the row stays editable and Resume sends it to the model loaded then.+ item.blockedByModelFailure = true;+ item.target.releaseModel();+ }if (!getActivePromptQueueItem(run)) {deletePromptQueueRun(run);- if (!plan.cancelActiveItem) {- return;- }try {activeItem?.target.cancel();} catch {@@ -1603,27 +1608,33 @@ function stopLocalPromptQueueRun(run: PromptQueueRun) {}return;}- if (plan.activeItemRemoved) {- clearPromptQueueRetryTimer(run);- }if (plan.cancelActiveItem) {waitForPromptQueueTargetIdle(run);try {- activeItem?.target.cancel();+ // A permanent cancel would also void the held prompts that share this target.+ if (run.items.some((item) => item.target === activeItem?.target)) {+ activeItem?.target.cancelActiveRun();+ } else {+ activeItem?.target.cancel();+ }} catch {// The active local run may have already ended.}refreshPromptQueueTargetIdleWait(run);return;}- syncPromptQueueUI();- if (plan.refreshTargetIdleWait) {- refreshPromptQueueTargetIdleWait(run);- return;- }- if (plan.activeItemRemoved && run.index >= 0 && !run.waitingForTargetIdle) {- requestPromptQueuePump(50);+ if (getActivePromptQueueItem(run)?.blockedByModelFailure) {+ // Invalidate an attempt already underway for the now-held item.+ run.generation += 1;+ clearPromptQueueRetryTimer(run);+ promptQueueDispatchingRunIds.delete(run.id);+ promptQueueActiveRunIds.delete(run.id);+ // The cleared timer may be the poll that sees the run this queue waits behind end.+ if (shouldPollPromptQueueTargetState(run)) {+ refreshPromptQueueTargetIdleWait(run);+ }}+ syncPromptQueueUI();}function stopLocalPromptQueueRunsForThreadIds(threadIds: string[]) {@@ -4006,7 +4017,7 @@ const Composer: FC<{const usesKnowledgeBaseAtQueueStart =chatStateAtQueueStart.ragEnabled &&chatStateAtQueueStart.ragSource.type === "kb";- const deferModelResolution =+ let deferModelResolution =chatStateAtQueueStart.modelLoading &&parseExternalModelId(chatStateAtQueueStart.loadingModelPick &&@@ -4263,6 +4274,12 @@ const Composer: FC<{discardOldestPendingSettings();getThreadRuntime()?.cancelRun();},+ releaseModel: () => {+ // The same snapshot a queue started during a model load takes.+ deferModelResolution = true;+ runSettingsAtQueueStart.params.checkpoint = "";+ runSettingsAtQueueStart.activeGgufVariant = null;+ },isIndexing: () =>promptQueueTargetMountedRef.current &&isTargetCurrentThread() &&diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.tsindex fd90146481c..e6406ffb606 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts@@ -95,7 +95,7 @@ import {} from "../utils/confirm-stop-running-chats";import {requestLocalPromptQueueStop,- requestPromptQueueStop,+ requestScopedLocalPromptQueueStop,notifyLocalPromptQueueLoadFailed,} from "../utils/prompt-queue-boundary";import { cancelPreStreamRunReservations } from "../utils/pre-stream-run-reservation";@@ -642,7 +642,7 @@ function publishLoadedModels(function stopQueuedRuns(decision: StopRunningChatsDecision, scoped: boolean): void {if (scoped) {- requestPromptQueueStop(decision.promptQueueThreadIds);+ requestScopedLocalPromptQueueStop(decision.promptQueueThreadIds);return;}cancelPreStreamRunReservations(decision.preStreamRunTokens);diff --git a/studio/frontend/src/features/chat/utils/prompt-queue-boundary.ts b/studio/frontend/src/features/chat/utils/prompt-queue-boundary.tsindex 79a557ede38..f1ef5549baf 100644--- a/studio/frontend/src/features/chat/utils/prompt-queue-boundary.ts+++ b/studio/frontend/src/features/chat/utils/prompt-queue-boundary.ts@@ -64,6 +64,19 @@ export function requestLocalPromptQueueStop();}+/** The same local stop for the chats on one model of several: holds their unsent prompts without+ * advancing the boundary, which would also void queue starts on the models left loaded. */+export function requestScopedLocalPromptQueueStop(threadIds: string[]) {+ if (typeof window === "undefined" || threadIds.length === 0) {+ return;+ }+ window.dispatchEvent(+ new CustomEvent<PromptQueueStopEventDetail>(PROMPT_QUEUE_STOP_EVENT, {+ detail: { threadIds, localOnly: true },+ }),+ );+}+export function requestTemporaryPromptQueueStop() {const threadIds = [...new Set(diff --git a/studio/frontend/src/features/chat/utils/prompt-queue-model-boundary.ts b/studio/frontend/src/features/chat/utils/prompt-queue-model-boundary.tsindex 663e60e2579..728a73b90ff 100644--- a/studio/frontend/src/features/chat/utils/prompt-queue-model-boundary.ts+++ b/studio/frontend/src/features/chat/utils/prompt-queue-model-boundary.ts@@ -22,35 +22,32 @@ export type PromptQueueModelStopItem = {export type LocalPromptQueueStopPlan = {cancelActiveItem: boolean;- activeItemRemoved: boolean;- refreshTargetIdleWait: boolean;retainedItemIndexes: number[];+ /** Indexes into the original items of unsent local prompts to hold until Resume. */+ heldItemIndexes: number[];};-/** Preserve external-provider work when the singleton local model changes. Only local items depend- * on the outgoing model. A dispatched local item must be cancelled, but external follow-ups- * remain valid and can resume once the thread runtime becomes idle. */+/** A local model change ends the local reply in flight, so that item is cancelled and dropped as a+ * Stop drops it. Unsent local prompts are held for Resume instead of discarded (#10428), and+ * external-provider items are untouched since they never used the outgoing model. */export function planLocalPromptQueueStop(items: readonly PromptQueueModelStopItem[],runIndex: number,): LocalPromptQueueStopPlan {const activeIndex = Math.max(runIndex, 0);const activeItem = items[activeIndex];- const retainedItemIndexes = items.flatMap((item, index) =>- index < activeIndex || !item.usesLocalModel ? [index] : [],+ const cancelActiveItem = Boolean(+ activeItem?.usesLocalModel && activeItem.dispatched,);- return {- cancelActiveItem: Boolean(- activeItem?.usesLocalModel && activeItem.dispatched,- ),- activeItemRemoved: Boolean(activeItem?.usesLocalModel),- refreshTargetIdleWait: Boolean(- runIndex < 0 &&- activeItem?.usesLocalModel &&- retainedItemIndexes.length > 0,- ),- retainedItemIndexes,- };+ const retainedItemIndexes = items.flatMap((_, index) =>+ cancelActiveItem && index === activeIndex ? [] : [index],+ );+ const heldItemIndexes = items.flatMap((item, index) =>+ index >= activeIndex && item.usesLocalModel && !item.dispatched+ ? [index]+ : [],+ );+ return { cancelActiveItem, retainedItemIndexes, heldItemIndexes };}export function shouldAbortPendingQueueForModelBoundary({diff --git a/studio/frontend/tests/prompt-queue-model-boundary.test.ts b/studio/frontend/tests/prompt-queue-model-boundary.test.tsindex 5af3923a4c9..eeabe5d95fa 100644--- a/studio/frontend/tests/prompt-queue-model-boundary.test.ts+++ b/studio/frontend/tests/prompt-queue-model-boundary.test.ts@@ -8,7 +8,7 @@ import {shouldAbortPendingQueueForSettingsChange,} from "../src/features/chat/utils/prompt-queue-model-boundary.ts";-test("a local model stop preserves an active external item", () => {+test("a local model stop preserves an active external item and holds local follow-ups", () => {assert.deepEqual(planLocalPromptQueueStop([@@ -20,9 +20,8 @@ test("a local model stop preserves an active external item", () => {),{cancelActiveItem: false,- activeItemRemoved: false,- refreshTargetIdleWait: false,- retainedItemIndexes: [0, 2],+ retainedItemIndexes: [0, 1, 2],+ heldItemIndexes: [1],},);});@@ -38,14 +37,48 @@ test("a dispatched local item is cancelled without losing external follow-ups",),{cancelActiveItem: true,- activeItemRemoved: true,- refreshTargetIdleWait: false,retainedItemIndexes: [1],+ heldItemIndexes: [],},);});-test("an undispatched local item is dropped without losing external follow-ups", () => {+test("a reload mid-reply drops only the reply in flight and holds the queued prompts (#10428)", () => {+ assert.deepEqual(+ planLocalPromptQueueStop(+ [+ { usesLocalModel: true, dispatched: true },+ { usesLocalModel: true, dispatched: false },+ { usesLocalModel: true, dispatched: false },+ ],+ 0,+ ),+ {+ cancelActiveItem: true,+ retainedItemIndexes: [1, 2],+ heldItemIndexes: [1, 2],+ },+ );+});++test("a paused queue keeps every prompt across a reload (#10428)", () => {+ assert.deepEqual(+ planLocalPromptQueueStop(+ [+ { usesLocalModel: true, dispatched: false },+ { usesLocalModel: true, dispatched: false },+ ],+ 0,+ ),+ {+ cancelActiveItem: false,+ retainedItemIndexes: [0, 1],+ heldItemIndexes: [0, 1],+ },+ );+});++test("a queue waiting on a direct send is held, not dropped", () => {assert.deepEqual(planLocalPromptQueueStop([@@ -56,14 +89,13 @@ test("an undispatched local item is dropped without losing external follow-ups",),{cancelActiveItem: false,- activeItemRemoved: true,- refreshTargetIdleWait: true,- retainedItemIndexes: [1],+ retainedItemIndexes: [0, 1],+ heldItemIndexes: [0],},);});-test("completed queue history is preserved when pending local work is dropped", () => {+test("completed queue history is preserved and only pending local work is held", () => {assert.deepEqual(planLocalPromptQueueStop([@@ -76,9 +108,8 @@ test("completed queue history is preserved when pending local work is dropped",),{cancelActiveItem: false,- activeItemRemoved: false,- refreshTargetIdleWait: false,- retainedItemIndexes: [0, 1, 3],+ retainedItemIndexes: [0, 1, 2, 3],+ heldItemIndexes: [2],},);});diff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.tsindex 60a1943592f..875e32011c6 100644--- a/studio/frontend/tests/resident-config-match.test.ts+++ b/studio/frontend/tests/resident-config-match.test.ts@@ -2012,7 +2012,7 @@ test("a pick asks the status about its own model and keeps or replaces the othertest("ejects stop only the ejected model's chats; eject all asks once and unloads the others first", () => {assert.match(USE_CHAT_MODEL_RUNTIME,- /function stopQueuedRuns\(decision: StopRunningChatsDecision, scoped: boolean\): void \{\s*if \(scoped\) \{\s*requestPromptQueueStop\(decision\.promptQueueThreadIds\);\s*return;\s*\}\s*cancelPreStreamRunReservations\(decision\.preStreamRunTokens\);\s*requestLocalPromptQueueStop\(decision\.promptQueueThreadIds\);/,+ /function stopQueuedRuns\(decision: StopRunningChatsDecision, scoped: boolean\): void \{\s*if \(scoped\) \{\s*requestScopedLocalPromptQueueStop\(decision\.promptQueueThreadIds\);\s*return;\s*\}\s*cancelPreStreamRunReservations\(decision\.preStreamRunTokens\);\s*requestLocalPromptQueueStop\(decision\.promptQueueThreadIds\);/,);assert.match(USE_CHAT_MODEL_RUNTIME,diff --git a/tests/studio/test_multi_chat_prompt_queue_contract.py b/tests/studio/test_multi_chat_prompt_queue_contract.pyindex 096988e6034..c85151ae4f6 100644--- a/tests/studio/test_multi_chat_prompt_queue_contract.py+++ b/tests/studio/test_multi_chat_prompt_queue_contract.py@@ -602,7 +602,9 @@ def test_queued_settings_are_thread_scoped_without_cross_chat_fallback():"function stopLocalPromptQueueRun(run: PromptQueueRun)","function stopLocalPromptQueueRunsForThreadIds(threadIds: string[])",)- assert "if (plan.refreshTargetIdleWait)" in local_queue_stop+ # #10428: a model change holds unsent local prompts for Resume instead of dropping them.+ assert "item.blockedByModelFailure = true;" in local_queue_stop+ assert "item.target.releaseModel();" in local_queue_stopassert "refreshPromptQueueTargetIdleWait(run);" in local_queue_stopassert "claimPreStreamRunReservation(reservationToken);" in RUNTIME_PROVIDERassert "if (!reservation.claimed)" in PRE_STREAM_RESERVATION