Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { authFetch } from "@/features/auth";
import { bumpInventoryVersion } from "@/features/hub/stores/inventory-events";
// Leaf module, not the barrel: tests stub the barrel down to the cache bump.
import { hubTokenHeader } from "@/features/hub/lib/hub-token-header";
import { readFastApiError } from "@/lib/format-fastapi-error";
export type EmbeddingModelSettings = {
embeddingModel: string;
embeddingGgufRepo: string;
defaultEmbeddingModel: string;
defaultEmbeddingGgufRepo: string;
isCustom: boolean;
loaded: boolean;
/** ANY embedder is resident; saving a new model does not release the old one. */
backendLoaded: boolean;
/** Where the embedder runs, for every embedding model. */
embeddingDevice: EmbeddingDevice;
/** RAG_EMBED_DEVICE names a device, so the choice is read-only here. */
embeddingDeviceLocked: boolean;
};
export type EmbeddingDevice = "auto" | "cpu" | "gpu";
type ApiEmbeddingModelSettings = {
// biome-ignore lint/style/useNamingConvention: API schema
embedding_model: string;
// biome-ignore lint/style/useNamingConvention: API schema
embedding_gguf_repo: string;
// biome-ignore lint/style/useNamingConvention: API schema
default_embedding_model: string;
// biome-ignore lint/style/useNamingConvention: API schema
default_embedding_gguf_repo: string;
// biome-ignore lint/style/useNamingConvention: API schema
is_custom: boolean;
loaded?: boolean;
// biome-ignore lint/style/useNamingConvention: API schema
backend_loaded?: boolean;
// biome-ignore lint/style/useNamingConvention: API schema
embedding_device?: EmbeddingDevice;
// biome-ignore lint/style/useNamingConvention: API schema
embedding_device_locked?: boolean;
};
/** 409: not verifiable as an embedding model; retry with force to save anyway. */
export class EmbeddingModelVerificationError extends Error {}
/** 403: flagged unsafe by HF; force cannot bypass it. */
export class EmbeddingModelBlockedError extends Error {}
function fromApi(settings: ApiEmbeddingModelSettings): EmbeddingModelSettings {
return {
embeddingModel: settings.embedding_model,
embeddingGgufRepo: settings.embedding_gguf_repo,
defaultEmbeddingModel: settings.default_embedding_model,
defaultEmbeddingGgufRepo: settings.default_embedding_gguf_repo,
isCustom: settings.is_custom,
loaded: settings.loaded ?? false,
// Older backends only report the selected model.
backendLoaded: settings.backend_loaded ?? settings.loaded ?? false,
embeddingDevice: settings.embedding_device ?? "auto",
embeddingDeviceLocked: settings.embedding_device_locked ?? false,
};
}
export async function loadEmbeddingModelSettings(): Promise<EmbeddingModelSettings> {
const res = await authFetch("/api/settings/embedding-model");
if (!res.ok) {
throw new Error(
await readFastApiError(res, "Failed to load embedding model setting"),
);
}
return fromApi(await res.json());
}
export async function updateEmbeddingModelSettings(
embeddingModel: string,
options?: {
hfToken?: string;
force?: boolean;
ggufRepo?: string | null;
backend?: EmbeddingModelResolution["backend"] | null;
},
): Promise<EmbeddingModelSettings> {
const res = await authFetch("/api/settings/embedding-model", {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
// biome-ignore lint/style/useNamingConvention: API schema
embedding_model: embeddingModel,
// biome-ignore lint/style/useNamingConvention: API schema
hf_token: options?.hfToken || null,
// biome-ignore lint/style/useNamingConvention: API schema
gguf_repo: options?.ggufRepo ?? null,
backend: options?.backend ?? null,
force: options?.force ?? false,
}),
});
if (res.status === 403) {
throw new EmbeddingModelBlockedError(
await readFastApiError(res, "This model is blocked by a security scan"),
);
}
if (res.status === 409) {
throw new EmbeddingModelVerificationError(
await readFastApiError(res, "Could not verify the embedding model"),
);
}
if (!res.ok) {
throw new Error(
await readFastApiError(res, "Failed to save embedding model"),
);
}
const settings = fromApi(await res.json());
bumpInventoryVersion();
return settings;
}
/** Moves the embedder to another device; the next index or search reloads it there. */
export async function updateEmbeddingDevice(
device: EmbeddingDevice,
): Promise<EmbeddingModelSettings> {
const res = await authFetch("/api/settings/embedding-model/device", {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ device }),
});
if (!res.ok) {
throw new Error(
await readFastApiError(res, "Failed to save the embedding device"),
);
}
return fromApi(await res.json());
}
/** What saving a model would need fetched, and whether it is already on disk. */
export type EmbeddingModelResolution = {
embeddingModel: string;
backend: "llama" | "sentence-transformers";
downloadRepo: string | null;
files: string[] | null;
cached: boolean;
sizeBytes: number | null;
error: string | null;
};
type ApiEmbeddingModelResolution = {
// biome-ignore lint/style/useNamingConvention: API schema
embedding_model: string;
backend: "llama" | "sentence-transformers";
// biome-ignore lint/style/useNamingConvention: API schema
download_repo: string | null;
files: string[] | null;
cached: boolean;
// biome-ignore lint/style/useNamingConvention: API schema
size_bytes: number | null;
error: string | null;
};
export async function resolveEmbeddingModel(
embeddingModel: string,
options?: { hfToken?: string },
): Promise<EmbeddingModelResolution> {
const params = new URLSearchParams({ model: embeddingModel });
const res = await authFetch(
`/api/settings/embedding-model/resolve?${params}`,
// Header keeps a gated repo's token out of the URL.
{ headers: hubTokenHeader(options?.hfToken) },
);
if (!res.ok) {
throw new Error(
await readFastApiError(res, "Failed to check the embedding model"),
);
}
const body = (await res.json()) as ApiEmbeddingModelResolution;
return {
embeddingModel: body.embedding_model,
backend: body.backend,
downloadRepo: body.download_repo,
files: body.files,
cached: body.cached,
sizeBytes: body.size_bytes,
error: body.error,
};
}
export async function unloadEmbeddingModel(): Promise<EmbeddingModelSettings> {
const res = await authFetch("/api/settings/embedding-model/unload", {
method: "POST",
});
if (!res.ok) {
throw new Error(
await readFastApiError(res, "Failed to unload the embedding model"),
);
}
return fromApi(await res.json());
}
export async function resetEmbeddingModelSettings(): Promise<EmbeddingModelSettings> {
const res = await authFetch("/api/settings/embedding-model", {
method: "DELETE",
});
if (!res.ok) {
throw new Error(
await readFastApiError(res, "Failed to reset embedding model"),
);
}
const settings = fromApi(await res.json());
bumpInventoryVersion();
return settings;
}