Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import {
forgetThreadReads,
inFlightThreadReads,
type InFlightThreadRead,
} from "./thread-read-cache.ts";
import { skillLoadCardEvent } from "./skill-load-event";
import { authFetch, getAuthSessionEpoch } from "@/features/auth";
import { prepareHfTokenForUse } from "@/features/hf-auth";
// These helpers are deliberately API-layer-only, not part of their features' public barrels.
// eslint-disable-next-line no-restricted-imports
import {
abortError,
combineAbortSignals,
disposableTimeoutSignal,
} from "@/features/hub/lib/abort-signals";
// eslint-disable-next-line no-restricted-imports
import { hubTokenHeader } from "@/features/hub/lib/hub-token-header";
// eslint-disable-next-line no-restricted-imports
import { isHuggingFaceOffline } from "@/features/hub/lib/network";
import { dismissCarveoutAdviceForModel, showCarveoutAdvice } from "@/features/igpu-carveout";
// eslint-disable-next-line no-restricted-imports
import { consumeNativePathToken } from "@/features/native-intents/api";
// eslint-disable-next-line no-restricted-imports
import { checkDiskSpace } from "@/features/settings/low-disk-check";
import { formatApiErrorBody } from "@/lib/format-fastapi-error";
import {
type ModelRuntime,
withModelLoadNotice,
} from "@/lib/model-lifecycle-events";
import type { AgentsMdRecord } from "../utils/agents-md";
import { showLoadWarning } from "../utils/load-warning-toast";
import { readAdvertisedPatchGuards } from "../utils/openapi-support";
import { validateRequestBody, wireLoadRequest } from "../lib/load-request";
import type {
MessageRecord,
ModelType,
ProjectRecord,
ThreadRecord,
} from "../types";
import type {
ApiMonitorEntry,
ApiMonitorResponse,
AudioGenerationResponse,
GgufVariantsResponse,
InferenceStatusResponse,
ListLorasResponse,
ListModelsResponse,
LoadModelRequest,
LoadModelResponse,
OpenAIChatChunk,
OpenAIChatCompletionsRequest,
UnloadModelRequest,
ValidateModelResponse,
} from "../types/api";
import {
CHAT_HISTORY_REVISION_KEY,
publishChatHistoryRevision,
} from "../utils/chat-history-revision";
import {
type GgufVariantsRequestOptions,
ggufVariantsQuery,
runBoundedVariantsRequest,
} from "./gguf-variants-request";
import { assertCompletedPaddedBody } from "./padded-response";
import {
type LengthStopCause,
lengthStopCause,
windowEvidenceCount,
} from "./generation-length.ts";
export const CHAT_HISTORY_UPDATED_EVENT = "unsloth-chat-history-updated";
// Bumped with that event so other tabs, which never receive it, can drop stale caches.
export { CHAT_HISTORY_REVISION_KEY } from "../utils/chat-history-revision";
export const CHAT_PROJECTS_UPDATED_EVENT = "unsloth-chat-projects-updated";
export type ChatHistoryUpdatedDetail = {
thread?: ThreadRecord;
coalesce?: boolean;
};
// bounds the request itself so a wedged socket cannot stall every reader waiting on the write
const THREAD_WRITE_TIMEOUT_MS = 30_000;
async function threadWriteFetch(
input: string,
init: RequestInit,
affected: readonly string[] | null,
caller?: AbortSignal,
): Promise<Response> {
const timeout = disposableTimeoutSignal(THREAD_WRITE_TIMEOUT_MS);
if (caller === undefined) {
try {
return await authFetch(input, { ...init, signal: timeout.signal });
} finally {
timeout.dispose();
forgetThreadReads(affected);
}
}
// Either reason ends the request. Linked by hand rather than with AbortSignal.any, which Safari only got in 17.4.
const controller = new AbortController();
const abort = () => controller.abort();
if (caller.aborted) abort();
caller.addEventListener("abort", abort);
timeout.signal.addEventListener("abort", abort);
try {
return await authFetch(input, { ...init, signal: controller.signal });
} finally {
caller.removeEventListener("abort", abort);
timeout.dispose();
forgetThreadReads(affected);
}
}
/** Stream ended without [DONE] or a finish_reason: the connection dropped mid-generation. */
export class StreamInterruptedError extends Error {
constructor() {
super(
"Response interrupted: the connection dropped before the model finished. " +
"Use Retry to regenerate.",
);
this.name = "StreamInterruptedError";
}
}
const LENGTH_STOP_ADVICE: Record<LengthStopCause, string> = {
max_tokens:
"The model reached the Max Tokens limit before producing a final answer. " +
"Increase Max Tokens or disable thinking, then retry.",
// The context window can bind even when Max Tokens is finite.
context_length:
"The model ran out of room to answer: thinking used what the context window " +
"had left after the prompt, before any answer was written. Raising Max " +
"Tokens cannot create room the window does not have -- increase the " +
"Context Length in Model settings, or disable thinking, then retry.",
// No Studio setting controls a connected model's window.
context_window:
"The conversation filled the model's context window before an answer was " +
"written. Start a new chat, or shorten this one, then retry.",
unknown:
"The model hit Max Tokens or its context window before answering. Increase " +
"Max Tokens, disable thinking, or start a new chat.",
};
/** A reasoning model spent its output budget before emitting any standard content. */
export class GenerationLengthError extends Error {
readonly stopCause: LengthStopCause;
constructor(stopCause: LengthStopCause) {
super(LENGTH_STOP_ADVICE[stopCause]);
this.name = "GenerationLengthError";
this.stopCause = stopCause;
}
}
/** Notifies this document and, via localStorage, other tabs. `coalesce` is for per-chunk
* streaming saves only; structural changes must not use it. */
export function notifyChatHistoryUpdated(
detail: ChatHistoryUpdatedDetail = {},
): void {
if (typeof window !== "undefined") {
const coalesce = detail.coalesce === true;
// detail lets a listener tell a chunk save from a structural change (isCoalescedHistoryEvent)
window.dispatchEvent(
new CustomEvent<ChatHistoryUpdatedDetail>(CHAT_HISTORY_UPDATED_EVENT, {
detail: { ...detail, coalesce },
}),
);
// The event above is same-document; a storage write is what crosses.
publishChatHistoryRevision(coalesce);
}
}
export function subscribeChatHistoryUpdated(
listener: (detail: ChatHistoryUpdatedDetail) => void,
): () => void {
if (typeof window === "undefined") return () => {};
const onEvent = (event: Event) =>
listener((event as CustomEvent<ChatHistoryUpdatedDetail>).detail ?? {});
window.addEventListener(CHAT_HISTORY_UPDATED_EVENT, onEvent);
return () => window.removeEventListener(CHAT_HISTORY_UPDATED_EVENT, onEvent);
}
const THREAD_READ_JOIN_MS = 2_000;
if (typeof window !== "undefined") {
window.addEventListener("storage", (event) => {
if (event.key === CHAT_HISTORY_REVISION_KEY) forgetThreadReads(null);
});
}
async function readThreadRecord(
threadId: string,
signal?: AbortSignal,
): Promise<ThreadRecord | null> {
const response = await authFetch(
`/api/chat/threads/${encodeURIComponent(threadId)}`,
signal ? { signal } : undefined,
);
if (response.status === 404) return null;
return parseJsonOrThrow<ThreadRecord>(response);
}
function shareThreadRead(
threadId: string,
signal?: AbortSignal,
): Promise<ThreadRecord | null> {
if (signal?.aborted) return Promise.reject(abortError(signal));
let shared = inFlightThreadReads.get(threadId);
const authEpoch = getAuthSessionEpoch();
if (
!shared
|| shared.authEpoch !== authEpoch
|| performance.now() - shared.startedAt > THREAD_READ_JOIN_MS
) {
const controller = new AbortController();
const started: InFlightThreadRead = {
controller,
readers: 0,
startedAt: performance.now(),
authEpoch,
promise: readThreadRecord(threadId, controller.signal),
};
const forget = () => {
if (inFlightThreadReads.get(threadId) === started) {
inFlightThreadReads.delete(threadId);
}
};
started.promise.then(forget, forget);
inFlightThreadReads.set(threadId, started);
shared = started;
}
shared.readers += 1;
if (!signal) return shared.promise;
const joined = shared;
return new Promise<ThreadRecord | null>((resolve, reject) => {
const onAbort = () => {
joined.readers -= 1;
if (inFlightThreadReads.get(threadId) === joined) {
inFlightThreadReads.delete(threadId);
}
if (joined.readers === 0) joined.controller.abort(signal.reason);
reject(abortError(signal));
};
signal.addEventListener("abort", onAbort, { once: true });
joined.promise.then(resolve, reject).finally(() => {
signal.removeEventListener("abort", onAbort);
});
});
}
export function notifyChatProjectsUpdated(): void {
notifyChatHistoryUpdated();
if (typeof window !== "undefined") {
window.dispatchEvent(new Event(CHAT_PROJECTS_UPDATED_EVENT));
}
}
function parseErrorText(status: number, body: unknown): string {
return formatApiErrorBody(body) ?? `Request failed (${status})`;
}
/** `/api/inference/load` and `/unload` pad their body so a proxy cannot time the request out,
* which commits the status early: a later failure can only arrive as `_deferred_error`. */
function deferredError(
body: unknown,
): { status: number; message: string; detail: unknown } | null {
const deferred =
body && typeof body === "object"
? (
body as {
_deferred_error?: { status_code?: unknown; detail?: unknown };
}
)._deferred_error
: undefined;
if (!deferred || typeof deferred !== "object") return null;
const status =
typeof deferred.status_code === "number" ? deferred.status_code : 500;
return {
status,
message: parseErrorText(status, { detail: deferred.detail }),
detail: deferred.detail,
};
}
/** Keeps the server's `detail`, so a structured audio.cpp failure (`{code, message}`) reaches the caller. */
export class LoadModelError extends Error {
readonly status: number;
readonly detail: unknown;
constructor(message: string, status: number, detail: unknown) {
super(message);
this.status = status;
this.detail = detail;
}
}
/** `paddedLabel` opts a caller into `assertCompletedPaddedBody`; only the two padded routes may,
* since a truncated body means unfinished there but is legitimate elsewhere. */
async function parseJsonOrThrow<T>(
response: Response,
paddedLabel?: string,
): Promise<T> {
const body = await response.json().catch(() => null);
if (!response.ok) {
const detail = (body as { detail?: unknown } | null)?.detail;
throw new LoadModelError(parseErrorText(response.status, body), response.status, detail);
}
const deferred = deferredError(body);
if (deferred) {
throw new LoadModelError(deferred.message, deferred.status, deferred.detail);
}
if (paddedLabel !== undefined) {
assertCompletedPaddedBody(body, paddedLabel);
}
return body as T;
}
export async function listModels(): Promise<ListModelsResponse> {
const response = await authFetch("/api/models/list");
return parseJsonOrThrow<ListModelsResponse>(response);
}
export async function listLoras(
outputsDir?: string,
): Promise<ListLorasResponse> {
const query = outputsDir
? `?${new URLSearchParams({ outputs_dir: outputsDir }).toString()}`
: "";
const response = await authFetch(`/api/models/loras${query}`);
return parseJsonOrThrow<ListLorasResponse>(response);
}
export async function getInferenceStatus(
signal?: AbortSignal,
model?: string,
): Promise<InferenceStatusResponse> {
const query = model ? `?${new URLSearchParams({ model }).toString()}` : "";
const response = await authFetch(`/api/inference/status${query}`, { signal });
return parseJsonOrThrow<InferenceStatusResponse>(response);
}
export async function getApiMonitor(): Promise<ApiMonitorResponse> {
const response = await authFetch("/api/inference/monitor");
return parseJsonOrThrow<ApiMonitorResponse>(response);
}
export async function getApiMonitorEntry(
id: string,
includePrompt = true,
): Promise<ApiMonitorEntry> {
const response = await authFetch(
`/api/inference/monitor/${encodeURIComponent(id)}${includePrompt ? "" : "?include_prompt=false"}`,
);
return parseJsonOrThrow<ApiMonitorEntry>(response);
}
export async function clearApiMonitor(): Promise<void> {
const response = await authFetch("/api/inference/monitor", {
method: "DELETE",
});
await parseJsonOrThrow<{ cleared: boolean }>(response);
}
export interface ActiveGenerationsResponse {
count: number;
/** Conversations with a generation in flight. Shorter than `count` when a first turn started
* before its thread id was persisted. */
thread_ids: string[];
/** One per in-flight request; `kind` is "chat" unless it is a non-conversation call. */
active?: { thread_id: string | null; kind?: string }[];
parallel_slots: number;
}
/** Chats generating on the backend right now. Authoritative where `runningByThreadId` is not: that
* map is per-tab, empty after a reload and blind to a second tab, and /load 409s on these. */
export async function getActiveGenerations(
model?: string,
): Promise<ActiveGenerationsResponse> {
const query = model ? `?model=${encodeURIComponent(model)}` : "";
const response = await authFetch(`/api/inference/active-generations${query}`);
return parseJsonOrThrow<ActiveGenerationsResponse>(response);
}
export async function loadModel(
payload: LoadModelRequest,
options?: {
signal?: AbortSignal;
onRequestStart?: () => void;
/** Awaited once the load is going out, after the token prompt and abort check. */
beforeRequest?: () => Promise<void>;
/** What is taking the slot. Chat ignores its own loads when reconciling, so an Audio load
* announced as "chat" left chat naming a model it had evicted. */
runtime?: ModelRuntime;
},
): Promise<LoadModelResponse> {
const preparedToken = await prepareHfTokenForUse(payload.hf_token);
// Tagged so auto-load can tell a user cancellation from a backend rejection.
if (!preparedToken.proceed)
throw Object.assign(new Error("Model load cancelled."), {
unslothUserCancelled: true,
});
if (options?.signal?.aborted)
throw options.signal.reason ?? new DOMException("Aborted", "AbortError");
await options?.beforeRequest?.();
options?.onRequestStart?.();
// Announced after the token prompt so a cancelled load never shows a row.
return withModelLoadNotice(
options?.runtime ?? "chat",
payload.model_path ?? null,
async () => {
// An uncached model selected here is downloaded by the backend inside this request, bypassing
// the download manager's disk check, so check here (throttled).
void checkDiskSpace();
try {
const response = await authFetch("/api/inference/load", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(wireLoadRequest(payload, preparedToken.token)),
signal: options?.signal,
});
const loaded = await parseJsonOrThrow<LoadModelResponse>(response, "Model load");
// Pass both identities: a cached Hub candidate loads by loadId while the runtime keeps `loaded.model`.
showCarveoutAdvice(loaded.carveout_advice, loaded.model, payload.model_path);
showLoadWarning(loaded.memory_warning);
return loaded;
} finally {
// Forced, and in finally: the reading must follow the write, and a failed load is the likeliest
// to have filled the disk.
void checkDiskSpace({ force: true });
}
},
);
}
export async function countChatInputTokens(payload: {
model: string;
messages: OpenAIChatCompletionsRequest["messages"];
enable_thinking?: boolean;
reasoning_effort?: OpenAIChatCompletionsRequest["reasoning_effort"];
preserve_thinking?: boolean;
enable_tools?: boolean;
enabled_tools?: string[];
mcp_enabled?: boolean;
rag_scope?: Record<string, unknown>;
auto_heal_tool_calls?: boolean;
studio_tool_history?: boolean;
run_tools_locally?: boolean;
// `model` is informational: the endpoint counts with whatever is resident and reports which.
}, serializedPayload = JSON.stringify(payload)): Promise<{ input_tokens: number; model?: string }> {
const response = await authFetch("/api/inference/chat/count_tokens", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: serializedPayload,
});
return parseJsonOrThrow<{ input_tokens: number; model?: string }>(response);
}
export async function validateModel(
payload: LoadModelRequest,
options?: { signal?: AbortSignal },
): Promise<ValidateModelResponse> {
const preparedToken = await prepareHfTokenForUse(payload.hf_token);
if (!preparedToken.proceed)
throw Object.assign(new Error("Model load cancelled."), {
unslothUserCancelled: true,
});
options?.signal?.throwIfAborted();
const response = await authFetch("/api/inference/validate", {
method: "POST",
headers: { "Content-Type": "application/json" },
signal: options?.signal,
body: JSON.stringify(validateRequestBody(payload, preparedToken.token)),
});
return parseJsonOrThrow<ValidateModelResponse>(response);
}
/** Reads GGUF header dims locally; all null when not downloaded, not a GGUF, or gated. */
export async function fetchGgufStagedMetadata(payload: {
model_path: string;
gguf_variant?: string | null;
hf_token?: string | null;
nativePathToken?: string | null;
}): Promise<{
contextLength: number | null;
layerCount: number | null;
moeLayerCount: number | null;
isDiffusion: boolean;
/** Unclassifiable, so `isDiffusion: false` above means "not known to be diffusion": callers
* picking a GPU split must assume possibly-diffusion. */
diffusionUnknown: boolean;
}> {
let nativePathLease: string | null = null;
if (payload.nativePathToken) {
try {
nativePathLease = (
await consumeNativePathToken(payload.nativePathToken, "validate-model")
).nativePathLease;
} catch {
// Lease expired: return no metadata (load can re-mint); diffusion is unknown, not false.
return {
contextLength: null,
layerCount: null,
moeLayerCount: null,
isDiffusion: false,
diffusionUnknown: true,
};
}
}
const response = await authFetch("/api/inference/validate", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
model_path: payload.model_path,
gguf_variant: payload.gguf_variant ?? null,
hf_token: payload.hf_token ?? null,
native_path_lease: nativePathLease,
include_context_length: true,
}),
});
const res = await parseJsonOrThrow<ValidateModelResponse>(response);
return {
contextLength: res.context_length ?? null,
layerCount: res.layer_count ?? null,
moeLayerCount: res.moe_layer_count ?? null,
isDiffusion: res.is_diffusion ?? false,
// Absent on a pre-#7575 backend, which never reported the inconclusive case.
diffusionUnknown: res.diffusion_unknown ?? false,
};
}
export async function unloadModel(payload: UnloadModelRequest): Promise<void> {
const response = await authFetch("/api/inference/unload", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
});
await parseJsonOrThrow<unknown>(response, "Model unload");
// Only after a confirmed unload; a rejected one leaves the model resident.
dismissCarveoutAdviceForModel(payload.model_path);
}
/** The approval is no longer waiting (expired, cancelled, backend restarted); unlike a failed
* post, retrying can never succeed. */
export class ToolApprovalGoneError extends Error {
constructor(message = "No pending tool call confirmation") {
super(message);
this.name = "ToolApprovalGoneError";
}
}
/** True only when the backend matched a pending call; throws `ToolApprovalGoneError` if gone. */
export async function resolveToolConfirmation(
sessionId: string,
approvalId: string,
decision: "allow" | "deny",
fullAccess = false,
): Promise<boolean> {
const epoch = getAuthSessionEpoch();
const response = await authFetch("/api/inference/tool-confirm", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
session_id: sessionId,
approval_id: approvalId,
decision,
...(fullAccess ? { permission_mode: "full" } : {}),
}),
});
// Ahead of parseJsonOrThrow, which folds every non-ok status into one bare Error.
if (response.status === 404) throw new ToolApprovalGoneError();
const parsed = await parseJsonOrThrow<{ resolved?: boolean; full_access_granted?: boolean }>(response);
// An older server may allow this call while ignoring the requested upgrade.
return getAuthSessionEpoch() === epoch && parsed.resolved === true &&
(!fullAccess || parsed.full_access_granted === true);
}
export interface CachedGgufRepo {
repo_id: string;
load_id?: string | null;
size_bytes: number;
cache_path: string;
last_modified?: number;
/** True when the repo ships an mmproj adapter (image inputs). Optional for older-backend compatibility. */
has_vision?: boolean;
/** HF pipeline task from the GGUF architecture, so the Images picker shows diffusion only. */
task?: string | null;
audio_type?: string | null;
audio_workflows?: string[] | null;
umbrella_folder?: boolean;
/** True when some quant has a download manifest or cancel marker. Optional for older-backend compatibility. */
has_variant_state?: boolean;
partial?: boolean;
/** Whether that partial can be continued byte for byte. False on a GGUF repo row by design:
* transport is per quant, so the repo cannot answer for all of them. */
partial_resumable?: boolean;
capabilities?: CachedRepoCapabilities | null;
}
export interface CachedRepoCapabilities {
can_chat?: boolean;
}
export async function getGgufDownloadProgress(
repoId: string,
variant: string,
expectedBytes: number,
hfToken?: string | null,
): Promise<{
downloaded_bytes: number;
expected_bytes: number;
progress: number;
}> {
const params = new URLSearchParams({
repo_id: repoId,
variant,
expected_bytes: String(expectedBytes),
});
const response = await authFetch(
`/api/models/gguf-download-progress?${params}`,
{ headers: hubTokenHeader(hfToken) },
);
return parseJsonOrThrow(response);
}
export interface DownloadProgressResponse {
downloaded_bytes: number;
/** Finalized-blob bytes only. Bytes still landing in a `.incomplete` blob count toward
* `downloaded_bytes` but not here, so the two are equal exactly when nothing is in flight. */
completed_bytes: number;
/** True once the backend verified a usable snapshot on disk; `progress` is capped at 0.99 until then. */
complete_on_disk: boolean;
expected_bytes: number;
progress: number;
/** On-disk path of the snapshot dir (or the cache repo root if there is no snapshot yet); null
* when nothing has been written to the cache for this repo. */
cache_path: string | null;
}
export async function getDownloadProgress(
repoId: string,
hfToken?: string | null,
mlxLoad = false,
): Promise<DownloadProgressResponse> {
const params = new URLSearchParams({ repo_id: repoId });
if (mlxLoad) params.set("mlx_load", "true");
const response = await authFetch(`/api/models/download-progress?${params}`, {
headers: hubTokenHeader(hfToken),
});
return parseJsonOrThrow(response);
}
export async function getDatasetDownloadProgress(
repoId: string,
hfToken?: string | null,
): Promise<DownloadProgressResponse> {
const params = new URLSearchParams({ repo_id: repoId });
const response = await authFetch(
`/api/hub/datasets/download-progress?${params}`,
{ headers: hubTokenHeader(hfToken) },
);
return parseJsonOrThrow(response);
}
export type ModelLoadPhase = "mmap" | "ready" | null;
export interface LoadProgressResponse {
/** Load phase: "mmap" while llama-server pages weight shards into RAM, "ready" once healthy, or
* null when no load is in flight. */
phase: ModelLoadPhase | "starting" | "loading_weights" | "warming_up";
bytes_loaded: number;
bytes_total: number;
fraction: number;
}
/** GGUF load mmap/upload progress, covering the download-to-ready window for large models. */
export async function getLoadProgress(): Promise<LoadProgressResponse> {
const response = await authFetch("/api/inference/load-progress");
return parseJsonOrThrow(response);
}
export interface LocalModelInfo {
id: string;
display_name: string;
path: string;
source: "models_dir" | "hf_cache" | "lmstudio" | "omlx" | "ollama" | "hermes" | "custom";
model_id?: string | null;
// Backend-detected weights format ("gguf" when known), for folders whose name lacks -GGUF.
model_format?: string | null;
opaque?: boolean;
// Set when a cached snapshot holds an incomplete download, so consumers skip unloadable weights.
partial?: boolean;
updated_at?: number | null;
task?: string | null;
audio_type?: string | null;
audio_workflows?: string[] | null;
}
interface LocalModelListResponse {
models_dir: string;
hf_cache_dir?: string | null;
lmstudio_dirs: string[];
models: LocalModelInfo[];
}
export async function listLocalModels(
signal?: AbortSignal,
): Promise<LocalModelListResponse> {
const response = await authFetch("/api/models/local", { signal });
return parseJsonOrThrow<LocalModelListResponse>(response);
}
export async function listCachedGguf(
signal?: AbortSignal,
): Promise<CachedGgufRepo[]> {
const response = await authFetch("/api/hub/cached-gguf", { signal });
const data = await parseJsonOrThrow<{ cached: CachedGgufRepo[] }>(response);
return data.cached;
}
export interface CachedModelRepo {
repo_id: string;
load_id?: string | null;
size_bytes: number;
/** Weights format; "adapter" is a LoRA with no base weights of its own. Optional for older-backend compatibility. */
model_format?: string | null;
opaque?: boolean;
last_modified?: number;
/** HF pipeline task; "text-to-image" lets the chat picker hide it. Absent = chat. */
task?: string | null;
audio_type?: string | null;
audio_workflows?: string[] | null;
/** Incomplete snapshot: must not count as downloaded, or a click re-downloads everything. */
partial?: boolean;
partial_resumable?: boolean;
/** Single-file diffusion checkpoint (no model_index.json); not a pipeline load unless curated. */
single_file?: boolean;
/** sd.cpp companion mirror (VAE/text encoders): listed for deletion, never loadable. */
companion?: boolean;
/** Owning cache dir; sent so a delete targets this copy, not the active cache. */
cache_path?: string | null;
capabilities?: CachedRepoCapabilities | null;
tags?: string[];
library_name?: string | null;
}
export async function listCachedModels(
hfToken?: string | null,
signal?: AbortSignal,
): Promise<CachedModelRepo[]> {
const response = await authFetch("/api/hub/cached-models", {
headers: hubTokenHeader(hfToken),
signal,
});
const data = await parseJsonOrThrow<{ cached: CachedModelRepo[] }>(response);
return data.cached;
}
export interface CachedModelPath {
path: string;
is_dir: boolean;
}
export async function getCachedModelPath(
repoId: string,
variant?: string,
): Promise<CachedModelPath> {
const params = new URLSearchParams({ repo_id: repoId });
if (variant) params.set("variant", variant);
const response = await authFetch(
`/api/models/cached-model-path?${params.toString()}`,
);
return parseJsonOrThrow<CachedModelPath>(response);
}
export async function revealCachedModel(
repoId: string,
variant?: string,
): Promise<void> {
const payload: Record<string, string> = { repo_id: repoId };
if (variant) payload.variant = variant;
const response = await authFetch("/api/models/reveal-cached-model", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload),
});
await parseJsonOrThrow<unknown>(response);
}
export async function revealFineTunedModel(
modelPath: string,
source: "training" | "exported",
): Promise<void> {
const response = await authFetch("/api/library/items/reveal", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ id: `model:${source}:${modelPath}` }),
});
await parseJsonOrThrow<unknown>(response);
}
export async function deleteFineTunedModel(args: {
modelPath: string;
source: "training" | "exported";
exportType?: "lora" | "merged" | "gguf";
ggufVariant?: string;
}): Promise<void> {
const response = await authFetch("/api/models/delete-finetuned", {
method: "DELETE",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
model_path: args.modelPath,
source: args.source,
export_type: args.exportType ?? null,
gguf_variant: args.ggufVariant ?? null,
}),
});
await parseJsonOrThrow<unknown>(response);
}
let advertisedPatchGuards: boolean | null = null;
export function advertisedThreadPatchGuards(): boolean | null {
return advertisedPatchGuards;
}
function noteAdvertisedPatchGuards(data: unknown): void {
const advertised = readAdvertisedPatchGuards(data);
if (advertised !== null) advertisedPatchGuards = advertised;
}
export async function listChatThreads(
args: {
modelType?: ModelType;
pairId?: string;
projectId?: string | null;
includeArchived?: boolean;
} = {},
): Promise<ThreadRecord[]> {
const params = new URLSearchParams();
if (args.modelType) params.set("model_type", args.modelType);
if (args.pairId) params.set("pair_id", args.pairId);
if (args.projectId) params.set("project_id", args.projectId);
if (args.includeArchived !== undefined) {
params.set("include_archived", String(args.includeArchived));
}
const qs = params.toString();
const response = await authFetch(`/api/chat/threads${qs ? `?${qs}` : ""}`);
const data = await parseJsonOrThrow<{ threads: ThreadRecord[] }>(response);
noteAdvertisedPatchGuards(data);
// Always hand back an array: an older or misbehaving backend may omit it or send a non-array.
return Array.isArray(data.threads) ? data.threads : [];
}
export interface SidebarRowRecord {
id: string;
type: "single" | "compare";
threadIds: string[];
title: string;
createdAt: number;
updatedAt: number;
modifiedAt?: number | null;
modelIds?: string[] | null;
isFork?: boolean | null;
isScheduled?: boolean | null;
/** Compare rows: each pane's own title, by `threadIds` position. */
threadTitles?: string[] | null;
projectId?: string | null;
}
export interface SidebarRowsRequest {
archived: boolean;
projectId: string | null;
unfiledOnly: boolean;
includeIds: string[] | null;
excludeIds: string[];
sort: "updated" | "manual";
manualOrder: string[] | null;
limit: number;
cursor: string | null;
}
export interface SidebarRowsPage {
rows: SidebarRowRecord[];
nextCursor: string | null;
hasMore: boolean;
total?: number | null;
}
export class SidebarCursorRejectedError extends Error {
readonly cursorRejected = true;
constructor() {
super("Sidebar page cursor rejected");
this.name = "SidebarCursorRejectedError";
}
}
/** Null on a server without the route: the caller falls back to the full list. */
export async function fetchSidebarRows(
request: SidebarRowsRequest,
): Promise<SidebarRowsPage | null> {
if (request.includeIds && request.includeIds.length === 0) {
return { rows: [], nextCursor: null, hasMore: false, total: 0 };
}
const body: SidebarRowsRequest = {
archived: request.archived,
projectId: request.projectId,
unfiledOnly: request.unfiledOnly,
includeIds: request.includeIds,
excludeIds: request.excludeIds,
sort: request.sort,
manualOrder: request.manualOrder,
limit: request.limit,
cursor: request.cursor,
};
const response = await authFetch("/api/chat/sidebar/rows", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(body),
});
if (response.status === 404 || response.status === 405) return null;
if (response.status === 400 && request.cursor) throw new SidebarCursorRejectedError();
const data = await parseJsonOrThrow<SidebarRowsPage>(response);
noteAdvertisedPatchGuards(data);
return {
rows: Array.isArray(data.rows) ? data.rows : [],
nextCursor: data.nextCursor ?? null,
hasMore: Boolean(data.hasMore),
total: typeof data.total === "number" ? data.total : null,
};
}
// under the route's 500 cap: 500 ids make a ~24 KB request line, past h11's 16 KiB head limit
const SIDEBAR_THREAD_ROWS_MAX = 200;
export async function fetchSidebarThreadRows(
threadIds: string[],
): Promise<Record<string, string> | null> {
const out: Record<string, string> = {};
for (let start = 0; start < threadIds.length; start += SIDEBAR_THREAD_ROWS_MAX) {
const params = new URLSearchParams();
for (const id of threadIds.slice(start, start + SIDEBAR_THREAD_ROWS_MAX)) {
params.append("thread_ids", id);
}
const response = await authFetch(`/api/chat/sidebar/thread-rows?${params}`);
if (response.status === 404 || response.status === 405) return null;
const data = await parseJsonOrThrow<{ rowIdByThreadId?: Record<string, string> }>(response);
for (const [threadId, rowId] of Object.entries(data.rowIdByThreadId ?? {})) {
if (typeof rowId === "string" && rowId) out[threadId] = rowId;
}
}
return out;
}
/** One chat message attachment, as listed for the settings uploaded-files view. */
export interface ChatAttachmentRecord {
id: string;
messageId: string;
threadId: string;
pairId?: string | null;
threadTitle?: string | null;
name: string;
type?: string | null;
contentType?: string | null;
sizeBytes?: number | null;
createdAt?: number | null;
}
export interface ChatAttachmentPage {
attachments: ChatAttachmentRecord[];
nextOffset: number | null;
}
export async function listChatAttachments(
offset = 0,
limit = 50,
): Promise<ChatAttachmentPage> {
const params = new URLSearchParams({
limit: String(limit),
offset: String(offset),
});
const response = await authFetch(`/api/chat/attachments?${params}`);
const data = await parseJsonOrThrow<{
attachments: ChatAttachmentRecord[];
nextOffset: number | null;
}>(response);
return {
attachments: Array.isArray(data.attachments) ? data.attachments : [],
nextOffset:
typeof data.nextOffset === "number" && Number.isFinite(data.nextOffset)
? data.nextOffset
: null,
};
}
export async function uploadChatAttachmentOriginal(
file: File,
epoch = getAuthSessionEpoch(),
): Promise<{ sha256: string; sizeBytes: number }> {
const sameAccount = () => {
if (getAuthSessionEpoch() !== epoch) throw new Error("The account changed during the upload.");
};
sameAccount();
const form = new FormData();
form.append("file", file, file.name);
const response = await authFetch(
"/api/chat/attachment-originals",
{ method: "POST", body: form },
{ beforeRetry: sameAccount },
);
sameAccount();
return parseJsonOrThrow<{ sha256: string; sizeBytes: number }>(response);
}
export async function fetchChatAttachmentBlob(
messageId: string,
attachmentId: string,
): Promise<Blob> {
const response = await authFetch(
`/api/chat/attachments/${encodeURIComponent(messageId)}/${encodeURIComponent(attachmentId)}/file`,
);
if (!response.ok) {
const body = await response.json().catch(() => null);
throw new Error(parseErrorText(response.status, body));
}
return response.blob();
}
export async function deleteChatAttachment(
messageId: string,
attachmentId: string,
): Promise<void> {
const response = await authFetch(
`/api/chat/attachments/${encodeURIComponent(messageId)}/${encodeURIComponent(attachmentId)}`,
{ method: "DELETE" },
);
await parseJsonOrThrow<{ ok: boolean }>(response);
}
export async function getChatThread(
threadId: string,
options: {
bounded?: boolean;
timeoutMs?: number;
signal?: AbortSignal;
share?: boolean;
} = {},
): Promise<ThreadRecord | null> {
// Bounded so a delete reconciliation cannot hang; `timeoutMs` and `signal` let callers end it sooner.
const timeout =
options.bounded || options.timeoutMs !== undefined
? disposableTimeoutSignal(options.timeoutMs ?? THREAD_WRITE_TIMEOUT_MS)
: null;
const combined =
timeout && options.signal
? combineAbortSignals([timeout.signal, options.signal])
: null;
const signal = combined?.signal ?? timeout?.signal ?? options.signal;
try {
return options.share
? await shareThreadRead(threadId, signal)
: await readThreadRecord(threadId, signal);
} finally {
combined?.dispose();
timeout?.dispose();
}
}
export class ChatThreadDeletedError extends Error {
constructor(message: string) {
super(message);
this.name = "ChatThreadDeletedError";
}
}
/** Carries the status so writers can tell a rejected row from an unreachable backend. */
export class ChatThreadWriteError extends Error {
readonly status: number;
constructor(message: string, status: number) {
super(message);
this.name = "ChatThreadWriteError";
this.status = status;
}
}
export async function saveChatThread(
thread: ThreadRecord,
): Promise<ThreadRecord> {
const response = await threadWriteFetch(
"/api/chat/threads",
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(thread),
},
[thread.id],
);
if (response.status === 410) {
const body = await response.json().catch(() => null);
throw new ChatThreadDeletedError(parseErrorText(response.status, body));
}
if (!response.ok) {
const body = await response.json().catch(() => null);
throw new ChatThreadWriteError(
parseErrorText(response.status, body),
response.status,
);
}
const savedThread = await parseJsonOrThrow<ThreadRecord>(response);
notifyChatHistoryUpdated({ thread: savedThread });
return savedThread;
}
export interface UpdateChatThreadOptions {
/** Apply only while the row still holds this title, else 409. */
expectedTitle?: string;
/** Regenerate a title without changing conversation recency. */
preserveTitleModifiedAt?: boolean;
/** And only while this is still the thread's opening user message. */
expectedOpeningMessageId?: string;
/** Off inside bulk actions, which announce once: each notification wakes every other tab. */
notify?: boolean;
/** Give up on the write; used to stand a superseded settings PATCH down. */
signal?: AbortSignal;
}
/** `settings` replaces the chat's whole snapshot; `settingsPatch` applies only the fields it
* names, for a writer that knows what changed but not what else the row holds. */
export type ChatThreadWritePatch = Partial<ThreadRecord> & {
settingsPatch?: ThreadRecord["settings"];
/** Orders a writer's snapshot writes against its own earlier ones, never across tabs. */
settingsSeq?: number;
settingsWriter?: string;
};
export async function updateChatThread(
threadId: string,
patch: ChatThreadWritePatch,
options: UpdateChatThreadOptions = {},
): Promise<ThreadRecord> {
const body: Record<string, unknown> = { ...patch };
if (options.expectedTitle !== undefined) {
body.expectedTitle = options.expectedTitle;
}
if (options.preserveTitleModifiedAt !== undefined) {
body.preserveTitleModifiedAt = options.preserveTitleModifiedAt;
}
if (options.expectedOpeningMessageId !== undefined) {
body.expectedOpeningMessageId = options.expectedOpeningMessageId;
}
const response = await threadWriteFetch(
`/api/chat/threads/${encodeURIComponent(threadId)}`,
{
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
},
[threadId],
options.signal,
);
const thread = await parseJsonOrThrow<ThreadRecord>(response);
if (options.notify !== false) notifyChatHistoryUpdated({ thread });
return thread;
}
export interface ForkChatThreadResult {
thread: ThreadRecord;
messages: MessageRecord[];
containerSnapshotWarning: string | null;
}
export async function forkChatThread(
threadId: string,
/** Omit `messageId` to fork at the tip, which the route resolves after its own check that
* the chat is not generating. */
args: { messageId?: string; newThreadId: string; createdAt: number },
): Promise<ForkChatThreadResult> {
let response: Response;
try {
response = await authFetch(
`/api/chat/threads/${encodeURIComponent(threadId)}/fork`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(args),
},
);
} finally {
forgetThreadReads([threadId, args.newThreadId]);
}
const data = await parseJsonOrThrow<{
thread: ThreadRecord;
messages: MessageRecord[];
containerSnapshotWarning: string | null;
}>(response);
notifyChatHistoryUpdated({ thread: data.thread });
return data;
}
/** Fork counts for a whole thread, keyed by message id. One request per thread, not per message. */
export async function getThreadForkCounts(
threadId: string,
): Promise<ReadonlyMap<string, number>> {
const response = await authFetch(
`/api/chat/threads/${encodeURIComponent(threadId)}/forks`,
);
if (response.status === 404) return new Map();
const data = await parseJsonOrThrow<{ counts?: Record<string, number> }>(
response,
);
return new Map(Object.entries(data.counts ?? {}));
}
/** Thread ids whose sandbox still holds files, for a caller that never asked. */
export async function deleteChatThreads(
threadIds: string[],
args: { deleteFiles?: boolean } = {},
): Promise<string[]> {
if (threadIds.length === 0) return [];
const response = await threadWriteFetch(
"/api/chat/threads",
{
method: "DELETE",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ ids: threadIds, delete_files: !!args.deleteFiles }),
},
threadIds,
);
const data = await parseJsonOrThrow<{ sandboxes_kept?: string[] }>(response);
notifyChatHistoryUpdated();
return Array.isArray(data?.sandboxes_kept) ? data.sandboxes_kept : [];
}
export async function listChatProjects(
args: { includeArchived?: boolean } = {},
): Promise<ProjectRecord[]> {
const params = new URLSearchParams();
if (args.includeArchived !== undefined) {
params.set("include_archived", String(args.includeArchived));
}
const qs = params.toString();
const response = await authFetch(`/api/chat/projects${qs ? `?${qs}` : ""}`);
const data = await parseJsonOrThrow<{ projects: ProjectRecord[] }>(response);
return Array.isArray(data.projects) ? data.projects : [];
}
export async function getChatProject(
projectId: string,
): Promise<ProjectRecord | null> {
const response = await authFetch(
`/api/chat/projects/${encodeURIComponent(projectId)}`,
);
if (response.status === 404) return null;
return parseJsonOrThrow<ProjectRecord>(response);
}
export async function getChatAgentsMd(
projectId: string | null,
): Promise<AgentsMdRecord> {
const query = projectId ? `?project_id=${encodeURIComponent(projectId)}` : "";
const response = await authFetch(`/api/chat/agents-md${query}`);
return parseJsonOrThrow<AgentsMdRecord>(response);
}
export async function saveChatProject(
project: ProjectRecord,
nativePathLease?: string,
): Promise<ProjectRecord> {
const response = await authFetch("/api/chat/projects", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ ...project, ...(nativePathLease ? { nativePathLease } : {}) }),
});
const saved = await parseJsonOrThrow<ProjectRecord>(response);
notifyChatProjectsUpdated();
return saved;
}
export async function updateChatProject(
projectId: string,
patch: Partial<ProjectRecord>,
): Promise<ProjectRecord> {
const response = await authFetch(
`/api/chat/projects/${encodeURIComponent(projectId)}`,
{
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(patch),
},
);
const project = await parseJsonOrThrow<ProjectRecord>(response);
notifyChatProjectsUpdated();
return project;
}
/** A null lease reverts to Studio's folder; the user's folder is never moved or deleted. */
export async function setChatProjectWorkingFolder(
projectId: string,
nativePathLease: string | null,
): Promise<ProjectRecord> {
const response = await authFetch(
`/api/chat/projects/${encodeURIComponent(projectId)}/working-folder`,
nativePathLease
? {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ nativePathLease }),
}
: { method: "DELETE" },
);
const project = await parseJsonOrThrow<ProjectRecord>(response);
notifyChatProjectsUpdated();
return project;
}
/** Member thread ids whose sandbox still holds files, from the route. */
export async function deleteChatProject(
projectId: string,
args: { deleteFiles?: boolean } = {},
): Promise<string[]> {
const params = new URLSearchParams();
if (args.deleteFiles) params.set("delete_files", "true");
const qs = params.toString();
let response: Response;
try {
response = await authFetch(
`/api/chat/projects/${encodeURIComponent(projectId)}${qs ? `?${qs}` : ""}`,
{ method: "DELETE" },
);
} finally {
forgetThreadReads(null);
}
const data = await parseJsonOrThrow<
ProjectRecord & { sandboxes_kept?: string[] }
>(response);
notifyChatProjectsUpdated();
return Array.isArray(data?.sandboxes_kept) ? data.sandboxes_kept : [];
}
export async function listChatMessages(
threadId: string,
): Promise<MessageRecord[]> {
const response = await authFetch(
`/api/chat/threads/${encodeURIComponent(threadId)}/messages`,
);
if (response.status === 404) return [];
const data = await parseJsonOrThrow<{ messages: MessageRecord[] }>(response);
return data.messages;
}
/**
* Batch fetch; falls back to per-thread listChatMessages on 404/405 from older servers.
* view "search" trims messages to what the search index reads; older backends ignore it.
*/
export async function batchListChatMessages(
threadIds: string[],
options: { view?: "search" } = {},
): Promise<Map<string, MessageRecord[]>> {
const out = new Map<string, MessageRecord[]>();
if (threadIds.length === 0) return out;
const response = await authFetch("/api/chat/messages:batch", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(
options.view ? { threadIds, view: options.view } : { threadIds },
),
});
if (response.status === 404 || response.status === 405) {
const per = await Promise.all(
threadIds.map(async (id) => [id, await listChatMessages(id)] as const),
);
for (const [id, msgs] of per) out.set(id, msgs);
return out;
}
const data = await parseJsonOrThrow<{
messagesByThreadId: Record<string, MessageRecord[]>;
}>(response);
for (const id of threadIds) {
out.set(id, data.messagesByThreadId[id] ?? []);
}
return out;
}
export async function batchCountChatMessages(
threadIds: string[],
): Promise<Map<string, number> | null> {
const out = new Map<string, number>();
if (threadIds.length === 0) return out;
const response = await authFetch("/api/chat/messages:counts", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ threadIds }),
});
if (response.status === 404 || response.status === 405) return null;
const data = await parseJsonOrThrow<{ countsByThreadId: Record<string, number> }>(response);
for (const id of threadIds) out.set(id, data.countsByThreadId[id] ?? 0);
return out;
}
export async function getChatMessage(
threadId: string,
messageId: string,
): Promise<MessageRecord | null> {
const response = await authFetch(
`/api/chat/threads/${encodeURIComponent(threadId)}/messages/${encodeURIComponent(messageId)}`,
);
if (response.status === 404) return null;
return parseJsonOrThrow<MessageRecord>(response);
}
/** Set by routes/chat_history.py; exposed through the CORS middleware in main.py. */
const CONFLICT_KIND_HEADER = "X-Unsloth-Conflict-Kind";
const CONFLICT_KIND_PROTECTED = "protected";
/** The server owns this message and rejects every save; callers must stop, not back off. */
export class ChatMessageProtectedError extends Error {
readonly messageId: string;
readonly threadId: string;
constructor(threadId: string, messageId: string, detail?: string) {
// Keep the server's wording: a manual edit surfaces this text to the user.
super(detail || `Message ${messageId} is server-managed and cannot be edited`);
this.name = "ChatMessageProtectedError";
this.threadId = threadId;
this.messageId = messageId;
}
}
export async function saveChatMessage(
message: MessageRecord,
options: { allowGenerationEdit?: boolean; coalesce?: boolean } = {},
): Promise<MessageRecord> {
const editQuery = options.allowGenerationEdit
? "?allowGenerationEdit=true"
: "";
const response = await threadWriteFetch(
`/api/chat/threads/${encodeURIComponent(message.threadId)}/messages/${encodeURIComponent(message.id)}${editQuery}`,
{
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(message),
},
[message.threadId],
);
// Two failures share this status: a protected message, where the autosave must stop, and a
// thread-id collision, which the caller must see. Only the header separates them.
if (
response.status === 409 &&
response.headers?.get(CONFLICT_KIND_HEADER) === CONFLICT_KIND_PROTECTED
) {
// Read here, not in parseJsonOrThrow: a body is single-use.
const body = await response.json().catch(() => null);
throw new ChatMessageProtectedError(
message.threadId,
message.id,
formatApiErrorBody(body) ?? undefined,
);
}
const savedMessage = await parseJsonOrThrow<MessageRecord>(response);
// Only the per-chunk streaming autosave coalesces; manual edits publish at once.
notifyChatHistoryUpdated({ coalesce: options.coalesce === true });
return savedMessage;
}
export async function syncChatMessages(
threadId: string,
messages: MessageRecord[],
options: { pruneMissing?: boolean; deletedMessageIds?: string[] } = {},
): Promise<MessageRecord[]> {
const response = await threadWriteFetch(
`/api/chat/threads/${encodeURIComponent(threadId)}/messages`,
{
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
messages,
pruneMissing: options.pruneMissing ?? false,
deletedMessageIds: options.deletedMessageIds ?? [],
}),
},
[threadId],
);
const data = await parseJsonOrThrow<{ messages: MessageRecord[] }>(response);
// Pruning deletes messages, which other tabs must see immediately, so it does not coalesce.
notifyChatHistoryUpdated({ coalesce: options.pruneMissing !== true });
return data.messages;
}
export async function countBackendChats(): Promise<number> {
const response = await authFetch("/api/chat/count");
const data = await parseJsonOrThrow<{ count: number }>(response);
return data.count;
}
/** Thread ids whose sandbox still holds files, passed through from the route. */
export async function clearBackendChats(
options: {
notify?: boolean;
operationId?: string;
tombstoneThreadIds?: string[];
deleteFiles?: boolean;
} = {},
): Promise<{ deletedThreadIds: string[]; sandboxesKept: string[] }> {
const response = await threadWriteFetch(
`/api/chat${options.deleteFiles ? "?delete_files=true" : ""}`,
{
method: "DELETE",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
ids: options.tombstoneThreadIds ?? [],
operationId: options.operationId,
}),
},
null,
);
const data = await parseJsonOrThrow<{
deletedThreadIds?: string[];
sandboxes_kept?: string[];
}>(response);
if (options.notify !== false) {
notifyChatHistoryUpdated();
}
return {
deletedThreadIds: Array.isArray(data?.deletedThreadIds)
? data.deletedThreadIds
: [],
sandboxesKept: Array.isArray(data?.sandboxes_kept)
? data.sandboxes_kept
: [],
};
}
export async function buildBackendChatExport(): Promise<{
exportedAt: string;
version: number;
threadCount: number;
projects?: ProjectRecord[];
threads: ThreadRecord[];
messages: MessageRecord[];
}> {
const response = await authFetch("/api/chat/export");
const data = await parseJsonOrThrow<Awaited<
ReturnType<typeof buildBackendChatExport>
> | null>(response);
if (!data) throw new Error("The chat export did not download completely.");
return data;
}
// Server-side legacy-Dexie import ledger, so a studio.db wipe keeps the import recoverable.
export async function listChatImportLedger(): Promise<Set<string>> {
const response = await authFetch("/api/chat/import-ledger");
// Old backends act as an empty ledger; syncChatMessages upserts prevent duplicates.
if (response.status === 404 || response.status === 405) return new Set();
const data = await parseJsonOrThrow<{ threadIds: string[] }>(response);
return new Set(data.threadIds);
}
export interface RecordChatImportLedgerResult {
accepted: number;
inserted: number;
// False on old backends so the caller does not poison the localStorage perf hint.
supported: boolean;
}
export async function recordChatImportLedger(
threadIds: string[],
): Promise<RecordChatImportLedgerResult> {
if (threadIds.length === 0) {
return { accepted: 0, inserted: 0, supported: true };
}
const response = await authFetch("/api/chat/import-ledger", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ threadIds }),
});
if (
response.status === 404 ||
response.status === 405 ||
response.status === 501
) {
return { accepted: 0, inserted: 0, supported: false };
}
const data = await parseJsonOrThrow<{ accepted: number; inserted: number }>(
response,
);
return {
accepted: data.accepted,
inserted: data.inserted,
supported: true,
};
}
export interface BrowseEntry {
name: string;
has_models: boolean;
hidden: boolean;
}
export interface BrowseFoldersResponse {
current: string;
parent: string | null;
entries: BrowseEntry[];
suggestions: string[];
truncated?: boolean;
model_files_here?: number;
}
export async function listRecommendedFolders(): Promise<string[]> {
const response = await authFetch("/api/models/recommended-folders");
const data = await parseJsonOrThrow<{ folders: string[] }>(response);
return data.folders;
}
export async function browseFolders(
path?: string,
showHidden = false,
signal?: AbortSignal,
): Promise<BrowseFoldersResponse> {
const params = new URLSearchParams();
if (path !== undefined && path !== null) params.set("path", path);
if (showHidden) params.set("show_hidden", "true");
const qs = params.toString();
// Forward the signal so a cancelled navigation also cancels the server-side walk.
const response = await authFetch(
`/api/models/browse-folders${qs ? `?${qs}` : ""}`,
signal ? { signal } : undefined,
);
return parseJsonOrThrow<BrowseFoldersResponse>(response);
}
export async function listGgufVariants(
repoId: string,
hfToken?: string,
options?: GgufVariantsRequestOptions,
): Promise<GgufVariantsResponse> {
const params = ggufVariantsQuery(repoId, options, isHuggingFaceOffline());
return runBoundedVariantsRequest(options?.signal, async (signal) => {
const response = await authFetch(`/api/models/gguf-variants?${params}`, {
headers: hubTokenHeader(hfToken),
signal,
});
return parseJsonOrThrow<GgufVariantsResponse>(response);
});
}
export interface KvCacheEstimate {
kv_bytes: number | null;
weights_bytes: number | null;
/** Host / disk lookup tables, part of weights_bytes but never on the GPU; null when none. */
lookup_table_bytes?: number | null;
native_context: number | null;
/** Extra MTP draft reserve; null for ngram or a model with no MTP head. */
spec_bytes: number | null;
/** Context the estimate was computed at, which is the native length when the request omitted one. */
n_ctx: number | null;
/** Vision projector bytes at worst-case VRAM multiple; null when absent or disabled. */
projector_bytes: number | null;
/** True when the configured speculative mode attaches a drafter the route did not price (dspark/
* dflash, and Auto where it promotes to one). The total is then a floor, not an answer. */
spec_unpriced: boolean;
/** The share of kv_bytes llama.cpp keeps in HOST heap rather than on the card: the SWA checkpoint
* snapshots. Included in kv_bytes, so a VRAM figure has to subtract it. */
kv_checkpoint_bytes: number | null;
/** The share of spec_bytes no shorter context can reduce, being the separate drafter's resident
* weights. Auto-fit softening must not cover it. */
spec_fixed_bytes: number | null;
/** Planner compute buffers reserved on top of weights and cache; scale with slots and batch. */
compute_bytes: number | null;
/** The planner's complete GPU-resident figure, and its everything-total. */
gpu_bytes: number | null;
total_bytes: number | null;
/** What still lands on the card at the shortest context: the share no context reduction can recover. */
gpu_floor_bytes: number | null;
/** False only when the loader is free to shrink the context to fit. An inherited
* LLAMA_ARG_CTX_SIZE is kept, not fitted. */
context_is_pinned: boolean | null;
/** An inherited LLAMA_ARG_DEVICE confines the launch to the cards it names, so an aggregate VRAM
* budget describes a pool it will not open. */
inherited_device_pin: boolean | null;
}
export interface KvCacheEstimateOptions {
cacheTypeKv?: string | null;
nParallel?: number | null;
speculativeType?: string | null;
/** --spec-draft-n-max; a Hybrid Mamba target keeps one rollback state per drafted token, which
* dominates its reserve. */
specDraftNMax?: number | null;
/** Draft KV dtype, quantized independently of the main cache. */
specDraftCacheType?: string | null;
/** --ctx-checkpoints; each adds an SWA snapshot per slot. */
ctxCheckpoints?: number | null;
nBatch?: number | null;
nUbatch?: number | null;
/** Tensor mode replicates buffers on every device in the pool. */
tensorParallel?: boolean | null;
/** Vision off frees the projector, so it is not charged. */
disableVision?: boolean;
signal?: AbortSignal;
}
/** Omit `nCtx` to size against the model's own context length; the response says which. */
export async function estimateKvCache(
repoId: string,
quant: string,
nCtx?: number,
options: KvCacheEstimateOptions = {},
): Promise<KvCacheEstimate> {
const {
cacheTypeKv,
nParallel,
speculativeType,
specDraftNMax,
specDraftCacheType,
ctxCheckpoints,
nBatch,
nUbatch,
tensorParallel,
disableVision,
signal,
} = options;
const params = new URLSearchParams({ repo_id: repoId, quant });
if (nCtx && nCtx > 0) params.set("n_ctx", String(nCtx));
if (cacheTypeKv) params.set("cache_type_kv", cacheTypeKv);
// Send any positive value including 1: omission means the server's multi-slot default.
if (nParallel && nParallel > 0) params.set("n_parallel", String(nParallel));
if (speculativeType) params.set("speculative_type", speculativeType);
// Zero is a real choice for both of these, so they are sent whenever set rather than when truthy.
if (specDraftNMax != null && specDraftNMax >= 0)
params.set("spec_draft_n_max", String(specDraftNMax));
if (specDraftCacheType)
params.set("spec_draft_cache_type", specDraftCacheType);
if (ctxCheckpoints != null && ctxCheckpoints >= 0)
params.set("ctx_checkpoints", String(ctxCheckpoints));
// Compute buffers scale with these; omitting them underprices a raised config.
if (nBatch && nBatch > 0) params.set("n_batch", String(nBatch));
if (nUbatch && nUbatch > 0) params.set("n_ubatch", String(nUbatch));
if (tensorParallel) params.set("tensor_parallel", "true");
if (disableVision) params.set("disable_vision", "true");
const response = await authFetch(
`/api/models/kv-cache-estimate?${params}`,
signal ? { signal } : undefined,
);
return parseJsonOrThrow<KvCacheEstimate>(response);
}
function parseSseEvent(rawEvent: string): string[] {
const dataLines: string[] = [];
for (const line of rawEvent.split(/\r?\n/)) {
if (line.startsWith("data:")) {
dataLines.push(line.slice(5).trimStart());
}
}
return dataLines;
}
function hasNonWhitespaceText(value: unknown): boolean {
if (typeof value === "string") {
return value.trim().length > 0;
}
if (Array.isArray(value)) {
return value.some((item) => hasNonWhitespaceText(item));
}
if (!value || typeof value !== "object") {
return false;
}
const record = value as Record<string, unknown>;
return ["thinking", "text", "content", "reasoning", "summary"].some(
(key) => key in record && hasNonWhitespaceText(record[key]),
);
}
function classifyStructuredDeltaContent(content: unknown): {
hasAssistantContent: boolean;
hasReasoningContent: boolean;
} {
if (typeof content === "string") {
return {
hasAssistantContent: hasNonWhitespaceText(content),
hasReasoningContent: false,
};
}
if (!Array.isArray(content)) {
return {
hasAssistantContent: false,
hasReasoningContent: false,
};
}
let hasAssistantContent = false;
let hasReasoningContent = false;
for (const part of content) {
if (typeof part === "string") {
hasAssistantContent ||= hasNonWhitespaceText(part);
continue;
}
if (!part || typeof part !== "object") {
continue;
}
const record = part as Record<string, unknown>;
if (record.type === "thinking" || record.type === "reasoning") {
hasReasoningContent ||= hasNonWhitespaceText(record);
} else if (record.type === "text" || record.type === "output_text") {
const text =
typeof record.text === "string" ? record.text : record.content;
hasAssistantContent ||= hasNonWhitespaceText(text);
}
}
return { hasAssistantContent, hasReasoningContent };
}
export async function* streamChatCompletions(
payload: OpenAIChatCompletionsRequest,
signal: AbortSignal,
/** Served context window for length-stop attribution; null or omitted when unknown. */
loadedContextLength?: number | null,
): AsyncGenerator<OpenAIChatChunk> {
const response = await authFetch("/v1/chat/completions", {
method: "POST",
headers: {
"Content-Type": "application/json",
// Opt into UI control frames; the endpoint defaults to a clean OpenAI stream.
"X-Unsloth-Events": "1",
},
body: JSON.stringify(payload),
signal,
});
if (!response.ok) {
const body = await response.json().catch(() => null);
throw new Error(parseErrorText(response.status, body));
}
if (!response.body) {
throw new Error("Stream response missing body");
}
const reader = response.body.getReader();
const decoder = new TextDecoder();
let buffer = "";
let completed = false;
// EOF without `[DONE]` or a finish_reason chunk means the stream was cut mid-generation.
let sawTerminalSignal = false;
let terminalFinishReason: string | null = null;
let sawAssistantContent = false;
let sawReasoningContent = false;
// Needed to tell the walls apart: a Max Tokens below context did not necessarily stop it.
let promptTokens: number | null = null;
let windowCount: number | null = null;
// Anthropic's explicit model_context_window_exceeded signal.
let providerReportedWindow = false;
const throwIfReasoningOnlyLength = () => {
if (
terminalFinishReason === "length" &&
sawReasoningContent &&
!sawAssistantContent
) {
// Backend substitutes the context length for Max Tokens "Max", so equal means unset.
throw new GenerationLengthError(
providerReportedWindow
? "context_window"
: lengthStopCause({
cap: payload.max_tokens ?? null,
contextLength: loadedContextLength ?? null,
promptTokens,
completionTokens: windowCount,
}),
);
}
};
try {
while (true) {
const { done, value } = await reader.read();
if (done) {
completed = true;
if (!sawTerminalSignal) {
throw new StreamInterruptedError();
}
throwIfReasoningOnlyLength();
break;
}
buffer += decoder.decode(value, { stream: true });
let separatorIndex = buffer.search(/\r?\n\r?\n/);
while (separatorIndex >= 0) {
const rawEvent = buffer.slice(0, separatorIndex);
const separatorLength = buffer[separatorIndex] === "\r" ? 4 : 2;
buffer = buffer.slice(separatorIndex + separatorLength);
const dataLines = parseSseEvent(rawEvent);
if (dataLines.length === 0) {
separatorIndex = buffer.search(/\r?\n\r?\n/);
continue;
}
const dataText = dataLines.join("\n");
if (dataText === "[DONE]") {
completed = true;
sawTerminalSignal = true;
throwIfReasoningOnlyLength();
return;
}
const parsed = JSON.parse(dataText) as
| OpenAIChatChunk
| { type?: string; content?: string; error?: { message?: string } };
if ("error" in parsed && parsed.error) {
throw new Error(parsed.error.message || "Stream error");
}
if ("type" in parsed && parsed.type === "skill_load") {
yield { _toolEvent: skillLoadCardEvent(parsed) } as unknown as OpenAIChatChunk;
separatorIndex = buffer.search(/\r?\n\r?\n/);
continue;
}
if ("type" in parsed && parsed.type === "tool_status") {
yield {
_toolStatus: parsed.content ?? "",
} as unknown as OpenAIChatChunk;
separatorIndex = buffer.search(/\r?\n\r?\n/);
continue;
}
if ("type" in parsed && parsed.type === "diffusion_frame") {
yield {
_diffusionFrame: parsed,
} as unknown as OpenAIChatChunk;
separatorIndex = buffer.search(/\r?\n\r?\n/);
continue;
}
if (
"type" in parsed &&
(parsed.type === "tool_start" ||
parsed.type === "tool_end" ||
parsed.type === "tool_output" ||
parsed.type === "tool_args" ||
parsed.type === "tool_model_note" ||
parsed.type === "tool_approved")
) {
yield { _toolEvent: parsed } as unknown as OpenAIChatChunk;
separatorIndex = buffer.search(/\r?\n\r?\n/);
continue;
}
if (
parsed &&
typeof parsed === "object" &&
"type" in parsed &&
parsed.type === "reasoning_summary"
) {
yield {
_reasoningDurationMs: (parsed as { duration_ms?: number })
.duration_ms,
} as unknown as OpenAIChatChunk;
separatorIndex = buffer.search(/\r?\n\r?\n/);
continue;
}
// A tool loop's context as it hands off to a tool, so the meter moves mid-turn.
if ("type" in parsed && parsed.type === "context_usage") {
yield { _contextUsage: parsed } as unknown as OpenAIChatChunk;
separatorIndex = buffer.search(/\r?\n\r?\n/);
continue;
}
if ("type" in parsed && parsed.type === "generation_progress") {
yield { _generationProgress: parsed } as unknown as OpenAIChatChunk;
separatorIndex = buffer.search(/\r?\n\r?\n/);
continue;
}
const {
usage: parsedUsage,
timings: parsedTimings,
_toolEvent: parsedToolEvent,
} = parsed as {
usage?: { prompt_tokens?: number };
timings?: { predicted_n?: number };
_toolEvent?: { type?: string };
};
if (typeof parsedUsage?.prompt_tokens === "number") {
promptTokens = parsedUsage.prompt_tokens;
}
windowCount = windowEvidenceCount(parsedTimings) ?? windowCount;
if (parsedToolEvent?.type === "context_window_exceeded") {
providerReportedWindow = true;
}
// finish_reason is a valid terminal signal for providers that close without a [DONE] sentinel.
const parsedChoices = (
parsed as {
choices?: Array<{
delta?: Record<string, unknown>;
finish_reason?: string | null;
}>;
}
).choices;
for (const choice of parsedChoices ?? []) {
const delta = choice.delta;
if (delta) {
const contentState = classifyStructuredDeltaContent(delta.content);
sawAssistantContent ||= contentState.hasAssistantContent;
sawReasoningContent ||= contentState.hasReasoningContent;
const reasoning =
delta.reasoning_content ??
delta.reasoning ??
delta.reasoning_details;
sawReasoningContent ||= hasNonWhitespaceText(reasoning);
}
if (choice.finish_reason) {
terminalFinishReason = choice.finish_reason;
}
}
const finishReason = parsedChoices?.[0]?.finish_reason;
if (finishReason) {
sawTerminalSignal = true;
}
yield parsed as OpenAIChatChunk;
separatorIndex = buffer.search(/\r?\n\r?\n/);
}
}
} finally {
// Abort only on early exit: cancelling after [DONE] can mark a finished request cancelled.
if (!completed) {
try {
await reader.cancel();
} catch {
// already closed
}
}
}
}
export async function generateAudio(
payload: OpenAIChatCompletionsRequest,
signal: AbortSignal,
): Promise<AudioGenerationResponse> {
const response = await authFetch("/api/inference/chat/completions", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ ...payload, stream: false }),
signal,
});
if (!response.ok) {
const body = await response.json().catch(() => null);
throw new Error(parseErrorText(response.status, body));
}
return (await response.json()) as AudioGenerationResponse;
}