Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
// Reads are independent so a missing runtime does not blank other rows. Feature modules are
// imported directly because their indexes re-export pages kept out of the eager bundle.
import { authFetch, isAccountOwner } from "@/features/auth";
import { confirmStopRunningChatsIfNeeded } from "@/features/chat/utils/confirm-stop-running-chats";
import { getInferenceStatus, unloadModel } from "@/features/chat/api/chat-api";
import { isExternalModelId } from "@/features/chat/external-providers";
import { requestScopedLocalPromptQueueStop } from "@/features/chat/utils/prompt-queue-boundary";
import { resolveInferenceCheckpointId } from "@/features/chat/lib/apply-inference-status-to-store";
import { useChatRuntimeStore } from "@/features/chat/stores/chat-runtime-store";
import { disposableTimeoutSignal } from "@/features/hub/lib/abort-signals";
import { modelIdsMatch } from "@/features/hub/lib/model-identity";
import {
getDiffusionStatus,
unloadDiffusionModel,
} from "@/features/images/api";
import { getVideoStatus, unloadVideoModel } from "@/features/video/api";
import { unloadEmbeddingModel } from "@/features/settings/api/embedding-model";
import { useEmbeddingModelStore } from "@/features/settings/stores/embedding-model-store";
import { notifyModelEjected } from "@/lib/model-lifecycle-events";
import { ejectChatModel } from "./eject-chat-model";
import {
embeddingRowName,
type EmbeddingStatusResponse,
type LoadedModelEntry,
type LoadedModelSource,
type SttStatusResponse,
describeDiffusionStatus,
describeEmbeddingStatus,
describeInferenceStatus,
describeSttStatus,
describeVideoStatus,
heldByAnotherAccount,
mergeLoadedModels,
sttEngineStatus,
verifyResident,
withServingResident,
} from "./loaded-models-sources";
type InferenceStatus = NonNullable<
Parameters<typeof describeInferenceStatus>[0]
>;
async function readInferenceStatus(
signal?: AbortSignal,
model?: string,
): Promise<InferenceStatus | null> {
const query = model ? `?model=${encodeURIComponent(model)}` : "";
const response = await authFetch(`/api/inference/status${query}`, { signal });
if (!response.ok) return null;
return (await response.json()) as InferenceStatus;
}
async function readSttStatus(
signal?: AbortSignal,
): Promise<SttStatusResponse | null> {
const response = await authFetch("/api/inference/audio/stt/status", {
signal,
});
if (!response.ok) return null;
return (await response.json()) as SttStatusResponse;
}
// Owner-only route: a managed account would only collect 403s, so it is not asked.
async function readEmbeddingStatus(
signal?: AbortSignal,
): Promise<EmbeddingStatusResponse | null> {
const response = await authFetch("/api/settings/embedding-model", {
signal,
});
if (!response.ok) return null;
return (await response.json()) as EmbeddingStatusResponse;
}
// A runtime that accepts the connection and never answers would leave the whole batch pending
// forever, and with it the in-flight guard gating every later refresh. Well past a cold status
// probe, so a slow-but-healthy read still lands and only a real hang trips it.
const READ_TIMEOUT_MS = 10_000;
async function settled<T>(
read: (signal: AbortSignal) => Promise<T>,
): Promise<T | null> {
const timeout = disposableTimeoutSignal(READ_TIMEOUT_MS);
try {
return await read(timeout.signal);
} catch {
return null;
} finally {
timeout.dispose();
}
}
/**
* Bounds eject reads but rethrows. Not applied to unloads: they wait on the generate lock
* for tens of seconds, and aborting would not cancel the teardown.
*/
async function bounded<T>(
read: (signal: AbortSignal) => Promise<T>,
): Promise<T> {
const timeout = disposableTimeoutSignal(READ_TIMEOUT_MS);
try {
return await read(timeout.signal);
} finally {
timeout.dispose();
}
}
/**
* A failed read is not evidence the runtime is empty: an unreadable source keeps its last
* rows, a readable one is always replaced (even by an empty answer).
*/
export type LoadedModelsRead = {
entries: LoadedModelEntry[];
unreadable: LoadedModelSource[];
heldByOther: boolean;
};
export async function readLoadedModels(
previous: readonly LoadedModelEntry[] = [],
): Promise<LoadedModelsRead> {
const owner = isAccountOwner();
const [inference, diffusion, video, stt, embedding] = await Promise.all([
settled((signal) => readInferenceStatus(signal)),
settled(getDiffusionStatus),
settled(getVideoStatus),
settled(readSttStatus),
owner ? settled(readEmbeddingStatus) : Promise.resolve(null),
]);
const kept = (source: LoadedModelSource) =>
previous.filter((row) => row.source === source);
const unreadable: LoadedModelSource[] = [];
const group = <T>(
source: LoadedModelSource,
status: T | null,
describe: (value: T) => LoadedModelEntry[],
) => {
if (status !== null) return describe(status);
unreadable.push(source);
return kept(source);
};
// /status sizes only the primary slot; each kept model is asked for on its own.
const chat = await Promise.all(
group("chat", inference, describeInferenceStatus).map(async (entry) =>
entry.servingCheckpoint
? withServingResident(
entry,
await settled((signal) =>
readInferenceStatus(signal, entry.servingCheckpoint),
),
)
: entry,
),
);
const entries = mergeLoadedModels([
chat,
group("image", diffusion, describeDiffusionStatus),
group("video", video, describeVideoStatus),
group("stt", stt, describeSttStatus),
owner ? group("embedding", embedding, describeEmbeddingStatus) : [],
]);
const heldByOther = heldByAnotherAccount([
inference as { yours?: boolean } | null,
diffusion as { yours?: boolean } | null,
video as { yours?: boolean } | null,
]);
return { entries, unreadable, heldByOther };
}
async function ejectChatRow(entry: LoadedModelEntry): Promise<EjectOutcome> {
if (entry.servingCheckpoint) return ejectServingRow(entry);
const decision = await confirmStopRunningChatsIfNeeded(
"Unloading this model",
"unloadOne",
entry.name,
);
if (!decision.proceed) return { status: "declined" };
const { unloadedAliases, stillResident, replacedBy } = await ejectChatModel(
entry.name,
{
readResident: async () => {
const status = await bounded(getInferenceStatus);
const checkpoint = resolveInferenceCheckpointId(status);
if (!checkpoint) return null;
return {
checkpoint,
aliases: [checkpoint, status.active_model].filter(
(alias): alias is string => alias != null,
),
};
},
// Only once an unload is really sent: a row replaced since the poll unloads nothing.
unload: (modelPath) => {
requestScopedLocalPromptQueueStop(decision.promptQueueThreadIds);
return unloadModel({
model_path: modelPath,
force_cancel_active: decision.forceCancelActive,
});
},
matches: modelIdsMatch,
cachedRow: entry.inactive === true,
readCached: async () => (await bounded(getInferenceStatus)).loaded ?? [],
},
);
if (stillResident) return { status: "stillResident", model: stillResident };
if (replacedBy) return { status: "replaced", resident: replacedBy };
if (unloadedAliases.length === 0) return { status: "alreadyFree" };
// Only when the model is really gone: a reload during the run leaves it usable.
clearChatSelectionFor(unloadedAliases);
return { status: "ejected" };
}
async function ejectServingRow(entry: LoadedModelEntry): Promise<EjectOutcome> {
const checkpoint = entry.servingCheckpoint ?? entry.name;
const decision = await confirmStopRunningChatsIfNeeded(
"Unloading this model",
"unloadOne",
checkpoint,
);
if (!decision.proceed) return { status: "declined" };
const servedCheckpoints = async () =>
(await bounded(getInferenceStatus)).serving_checkpoints ?? [];
const { unloadedAliases, stillResident } = await ejectChatModel(checkpoint, {
// Never the active model: that row has no servingCheckpoint.
readResident: async () => null,
unload: (modelPath) => {
requestScopedLocalPromptQueueStop(decision.promptQueueThreadIds);
return unloadModel({
model_path: modelPath,
force_cancel_active: decision.forceCancelActive,
});
},
matches: (left, right) => left === right,
cachedRow: true,
readCached: servedCheckpoints,
});
if (stillResident) return { status: "stillResident", model: stillResident };
if (unloadedAliases.length === 0) return { status: "alreadyFree" };
clearChatSelectionFor([checkpoint, entry.name]);
return { status: "ejected" };
}
/** Clear the picker only when it names something this eject released: chat can
* hold an external selection while a local model is resident. */
function clearChatSelectionFor(aliases: string[]): void {
const store = useChatRuntimeStore.getState();
const selected = store.params.checkpoint;
if (!selected || isExternalModelId(selected)) return;
if (aliases.some((alias) => modelIdsMatch(selected, alias))) {
store.clearCheckpoint();
}
}
/**
* `replaced`: the runtime holds another model, so nothing was unloaded (these endpoints
* carry no model id). `alreadyFree`: it held nothing.
*/
export type EjectOutcome =
| { status: "ejected" }
| { status: "alreadyFree" }
| { status: "declined" }
| { status: "replaced"; resident: string }
| { status: "stillResident"; model: string }
// Unload accepted but the confirming read did not answer: neither done nor failed.
| { status: "unverified" };
const UNVERIFIED = Symbol("unverified");
/** Narrows the stale-row window to the round trip; only a model-id unload could close it. */
async function ejectRuntimeRow(
entry: LoadedModelEntry,
resident: string | null,
unload: () => Promise<string | null | typeof UNVERIFIED>,
): Promise<EjectOutcome> {
const verdict = verifyResident(entry.name, resident, modelIdsMatch);
if (!resident) return { status: "alreadyFree" };
if (verdict !== "match") return { status: "replaced", resident };
const stillResident = await unload();
if (stillResident === UNVERIFIED) return { status: "unverified" };
return stillResident
? { status: "stillResident", model: stillResident }
: { status: "ejected" };
}
/** Release one row, after checking the runtime still holds what the row names. */
export async function ejectLoadedModel(
entry: LoadedModelEntry,
): Promise<EjectOutcome> {
switch (entry.source) {
case "chat":
return ejectChatRow(entry);
case "image": {
const before = await bounded(getDiffusionStatus);
return ejectRuntimeRow(
entry,
before.loaded ? before.repo_id : null,
async () => {
const after = await unloadDiffusionModel();
notifyModelEjected("image");
return after.loaded ? (after.repo_id ?? entry.name) : null;
},
);
}
case "video": {
const before = await bounded(getVideoStatus);
return ejectRuntimeRow(
entry,
before.loaded ? before.repo_id : null,
async () => {
const after = await unloadVideoModel();
notifyModelEjected("video");
return after.loaded ? (after.repo_id ?? entry.name) : null;
},
);
}
case "embedding": {
const before = await bounded(readEmbeddingStatus);
if (!before) {
throw new Error(
"Could not read the embedding model status, so nothing was ejected.",
);
}
if (!(before.backend_loaded ?? before.loaded))
return { status: "alreadyFree" };
// The unload route releases whatever is resident: a model swapped in since the poll stays.
const resident = embeddingRowName(before);
if (resident !== entry.name) return { status: "replaced", resident };
await useEmbeddingModelStore
.getState()
.applyResidency(unloadEmbeddingModel);
const after = await bounded(readEmbeddingStatus);
if (!after) return { status: "unverified" };
return (after.backend_loaded ?? after.loaded)
? { status: "stillResident", model: entry.name }
: { status: "ejected" };
}
case "stt": {
const engine = entry.sttEngine;
if (!engine) throw new Error("This row names no dictation engine.");
// Dictation loads and releases on its own, so the resident model can change unprompted.
const before = await bounded(readSttStatus);
if (!before) {
throw new Error(
"Could not read dictation status, so nothing was ejected.",
);
}
const resident = sttEngineStatus(before, engine);
return ejectRuntimeRow(
entry,
resident?.loaded_model ?? null,
async () => {
const query = new URLSearchParams({ engine }).toString();
const response = await authFetch(
`/api/inference/audio/stt/unload?${query}`,
{ method: "POST" },
);
if (!response.ok) throw new Error(await readErrorDetail(response));
// The unload body is fixed and the backend may serve `gguf` from transformers, so re-read.
const after = await bounded(readSttStatus);
// A failed read is null too; treating it as empty would hide a model still holding memory.
if (!after) return UNVERIFIED;
return sttEngineStatus(after, engine)?.loaded_model ?? null;
},
);
}
}
}
async function readErrorDetail(response: Response): Promise<string> {
try {
const body = (await response.json()) as { detail?: unknown };
if (typeof body.detail === "string") return body.detail;
} catch {
// non-JSON error body
}
return `Request failed (${response.status})`;
}