Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

17.5 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

import { authFetch, getAuthToken } from "@/features/auth";
import { accountTransitionPending } from "@/lib/account-transition";
import { apiUrl } from "@/lib/api-base";
import {
  formatApiErrorBody,
  readFastApiError,
} from "@/lib/format-fastapi-error";
import type { AudioCatalogResponse } from "./audio-catalog-rows";
import {
  type AudioRunRequest,
  type AudioSourceRef,
  type AudioVoiceCreateRequest,
  buildAudioRunBody,
  buildVoiceCreateBody,
  transcribeUrl,
} from "./audio-run-request";

async function parseJson<T>(response: Response): Promise<T> {
  if (!response.ok) {
    throw await audioApiError(response);
  }
  return (await response.json()) as T;
}

export interface GeneratedAudio {
  data: string;
  format: string;
  sample_rate: number;
  seed?: number | null;
}

export interface GenerateAudioResponse {
  model: string;
  audio: GeneratedAudio;
  clip_id?: string | null;
  choices: { finish_reason: string }[];
  /** The seed the take used; null when the model takes none. */
  seed?: number | null;
  /** Option names the model doesn't take, so they weren't sent. */
  ignored_options?: string[];
}

export interface GenerateAudioOptions {
  temperature?: number;
  top_p?: number;
  max_tokens?: number;
  audio_instructions?: string;
  audio_language?: string;
  seed?: number;
  /** A GGUF audio model's own options by name, from its status `audio_options` schema. */
  audio_options?: Record<string, boolean | number | string>;
  signal?: AbortSignal;
}

export interface AudioDownloadPlan {
  entries: {
    repo_id: string;
    files: string[];
    bytes: number;
    gguf_filename: string | null;
    checkpoint?: boolean;
  }[];
  total_bytes: number;
  required_bytes?: number;
  checkpoint_bytes?: number;
}

export async function getAudioDownloadPlan(
  modelPath: string,
  hfToken?: string,
): Promise<AudioDownloadPlan> {
  const response = await authFetch("/api/inference/audio/download-plan", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ model_path: modelPath, hf_token: hfToken }),
  });
  return parseJson<AudioDownloadPlan>(response);
}

/** Every audio.cpp model the backend can place: umbrella folders plus Community repos. */
export async function fetchAudioCatalog(): Promise<AudioCatalogResponse> {
  const response = await authFetch("/api/inference/audio/catalog");
  return parseJson<AudioCatalogResponse>(response);
}

// The speech model kept beside the chat model (routing id, status name); null when it is the active one.
let audioSlotModel: string | null = null;
let audioSlotLabel: string | null = null;

export function setAudioSlotModel(
  model: string | null,
  label: string | null = model,
): void {
  audioSlotModel = model;
  audioSlotLabel = model ? label : null;
}

export function audioSlotRoutingId(
  model: string | null | undefined,
): string | null {
  return model && audioSlotModel && model === audioSlotLabel
    ? audioSlotModel
    : null;
}

export function isAudioSlotModel(model: string | null | undefined): boolean {
  return audioSlotRoutingId(model) !== null;
}

function withAudioSlotModel<T extends object>(body: T): T {
  return audioSlotModel ? { ...body, model: audioSlotModel } : body;
}

export async function generateAudio(
  text: string,
  options: GenerateAudioOptions = {},
): Promise<GenerateAudioResponse> {
  const { signal, ...sampling } = options;
  const response = await authFetch("/api/inference/audio/generate", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify(
      withAudioSlotModel({
        messages: [{ role: "user", content: text }],
        stream: false,
        ...sampling,
      }),
    ),
    signal,
  });
  return parseJson<GenerateAudioResponse>(response);
}

export interface AudioGalleryClip {
  id: string;
  url: string;
  prompt: string;
  model: string;
  audio_type: string;
  sample_rate: number;
  duration_s: number;
  created_at: string;
  pinned?: boolean;
  archived?: boolean;
  order_at?: number | null;
  /** The Audio workflow that made the clip. Older servers omit it; read it through clipWorkflow. */
  workflow?: string | null;
  reference_name?: string | null;
  source_clip_id?: string | null;
  source_input_id?: string | null;
  voice_id?: string | null;
  source_saved?: boolean;
  source_name?: string | null;
  target_builtin?: string | null;
  target_clip_id?: string | null;
  target_input_id?: string | null;
  group_id?: string | null;
  role?: string | null;
  settings?: Record<string, unknown> | null;
}

export interface AudioGalleryListResponse {
  audio: AudioGalleryClip[];
  has_more: boolean;
  next_before_mtime: number | null;
  next_before_id: string | null;
  next_before_pin?: number | null;
}

export interface AudioGalleryCursor {
  mtime: number;
  id: string;
  pin?: number | null;
}

export function audioGalleryCursor(
  page: AudioGalleryListResponse,
): AudioGalleryCursor | null {
  return page.next_before_mtime !== null && page.next_before_id !== null
    ? {
        mtime: page.next_before_mtime,
        id: page.next_before_id,
        pin: page.next_before_pin ?? null,
      }
    : null;
}

export async function listAudioGallery(
  offset: number,
  limit: number,
  before?: AudioGalleryCursor | null,
  archived = false,
): Promise<AudioGalleryListResponse> {
  const pin =
    before?.pin !== null && before?.pin !== undefined
      ? `&before_pin=${encodeURIComponent(before.pin)}`
      : "";
  const cursor = before
    ? `&before_mtime=${encodeURIComponent(before.mtime)}&before_id=${encodeURIComponent(before.id)}${pin}`
    : "";
  const response = await authFetch(
    `/api/inference/audio/gallery?offset=${offset}&limit=${limit}&archived=${archived}${cursor}`,
  );
  return parseJson<AudioGalleryListResponse>(response);
}

export async function setAudioClipFlags(
  id: string,
  flags: { pinned?: boolean; archived?: boolean },
): Promise<AudioGalleryClip> {
  const response = await authFetch(
    `/api/inference/audio/gallery/${encodeURIComponent(id)}`,
    {
      method: "PATCH",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify(flags),
    },
  );
  return parseJson<AudioGalleryClip>(response);
}

export async function moveAudioClip(
  id: string,
  afterId: string | null,
): Promise<AudioGalleryClip> {
  const response = await authFetch(
    `/api/inference/audio/gallery/${encodeURIComponent(id)}/move`,
    {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ after_id: afterId }),
    },
  );
  return parseJson<AudioGalleryClip>(response);
}

export async function addAudioClipToProject(
  id: string,
  projectId: string,
): Promise<{ path: string; already: boolean }> {
  const response = await authFetch(
    `/api/inference/audio/gallery/${encodeURIComponent(id)}/project`,
    {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ project_id: projectId }),
    },
  );
  return parseJson<{ path: string; already: boolean }>(response);
}

export async function deleteAudioClip(id: string): Promise<void> {
  const response = await authFetch(
    `/api/inference/audio/gallery/${encodeURIComponent(id)}`,
    { method: "DELETE" },
  );
  if (!response.ok) throw new Error(await readFastApiError(response));
}

export async function deleteAudioGroup(groupId: string): Promise<void> {
  const response = await authFetch(
    `/api/inference/audio/gallery/group/${encodeURIComponent(groupId)}`,
    { method: "DELETE" },
  );
  if (!response.ok) throw new Error(await readFastApiError(response));
}

export async function clearAudioGallery(
  workflow?: "speak" | "clone" | "edit" | "convert" | "music" | "separate",
): Promise<number> {
  const query = workflow ? `?workflow=${workflow}` : "";
  const response = await authFetch(`/api/inference/audio/gallery${query}`, {
    method: "DELETE",
  });
  const body = await parseJson<{ removed: number }>(response);
  return body.removed;
}

export async function fetchClipBlob(url: string): Promise<Blob> {
  const response = await authFetch(url);
  if (!response.ok) throw new Error(await readFastApiError(response));
  return response.blob();
}

/** Let the media element buffer and seek without downloading the entire WAV first. */
export async function fetchGalleryAudioSignedUrl(id: string): Promise<string> {
  const response = await authFetch(
    `/api/inference/audio/gallery/${encodeURIComponent(id)}/signed-url`,
  );
  const body = await parseJson<{ url?: string }>(response);
  if (!body.url) throw new Error("The server returned no audio link.");
  // Media elements bypass authFetch; desktop URLs need the backend origin.
  return apiUrl(body.url);
}

export async function listTranscripts(
  archived = false,
  before?: string | null,
): Promise<{
  transcripts: import("./transcript-stream").TranscriptRecord[];
  next_cursor: string | null;
}> {
  const params = new URLSearchParams({
    archived: String(archived),
    limit: "50",
  });
  if (before) params.set("before", before);
  return parseJson(
    await authFetch(`/api/inference/audio/transcripts?${params}`),
  );
}

export async function archiveTranscript(
  id: string,
  archived: boolean,
): Promise<void> {
  await parseJson(
    await authFetch(
      `/api/inference/audio/transcripts/${encodeURIComponent(id)}`,
      {
        method: "PATCH",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({ archived }),
      },
    ),
  );
}

export async function deleteTranscript(id?: string): Promise<void> {
  await parseJson(
    await authFetch(
      `/api/inference/audio/transcripts${id ? `/${encodeURIComponent(id)}` : ""}`,
      {
        method: "DELETE",
      },
    ),
  );
}

/** Carries the status so callers can tell an expired upload (404) from a refusal, and `detail` with its `code`. */
export class AudioApiError extends Error {
  readonly status: number;
  readonly code?: string;
  readonly detail?: unknown;
  constructor(message: string, status: number, detail?: unknown) {
    super(message);
    this.name = "AudioApiError";
    this.status = status;
    this.detail = detail;
    const code = (detail as { code?: unknown } | null | undefined)?.code;
    if (typeof code === "string") this.code = code;
  }
}

async function audioApiError(response: Response): Promise<AudioApiError> {
  const copy = response.clone();
  const message = await readFastApiError(response);
  const detail = await copy.json().then(
    (body: { detail?: unknown } | null) => body?.detail,
    () => undefined,
  );
  return new AudioApiError(message, response.status, detail);
}

async function parseAudioJson<T>(response: Response): Promise<T> {
  if (!response.ok) {
    throw await audioApiError(response);
  }
  return (await response.json()) as T;
}

interface AudioInputRecord {
  id: string;
  name: string;
  duration_s: number;
  sample_rate: number;
  channels: number;
  url: string;
  expires_at: string;
}

/** XHR is the only browser API with upload progress; the caller retries a 401 via authFetch to refresh the token. */
function uploadWithProgress(
  url: string,
  blob: Blob,
  onProgress: (fraction: number | null) => void,
  signal?: AbortSignal,
): Promise<{ status: number; body: unknown }> {
  return new Promise((resolve, reject) => {
    // Same fence as authFetch: mid-switch, this tab's file would be stored under the next account.
    if (accountTransitionPending()) {
      reject(new Error("Another tab is switching accounts; this tab will reload."));
      return;
    }
    const xhr = new XMLHttpRequest();
    xhr.open("POST", apiUrl(url));
    const token = getAuthToken();
    if (token) xhr.setRequestHeader("Authorization", `Bearer ${token}`);
    xhr.setRequestHeader(
      "Content-Type",
      blob.type || "application/octet-stream",
    );
    xhr.responseType = "json";
    xhr.upload.onprogress = (event) =>
      onProgress(event.lengthComputable ? event.loaded / event.total : null);
    xhr.onload = () => resolve({ status: xhr.status, body: xhr.response });
    xhr.onerror = () =>
      reject(
        new Error("The upload failed. Check the connection and try again."),
      );
    xhr.onabort = () => reject(new DOMException("Aborted", "AbortError"));
    if (signal) {
      if (signal.aborted) {
        reject(new DOMException("Aborted", "AbortError"));
        return;
      }
      signal.addEventListener("abort", () => xhr.abort(), { once: true });
    }
    xhr.send(blob);
  });
}

export async function uploadAudioInput(
  blob: Blob,
  name: string,
  options: {
    onProgress?: (fraction: number | null) => void;
    signal?: AbortSignal;
  } = {},
): Promise<AudioInputRecord> {
  const url = `/api/inference/audio/inputs?name=${encodeURIComponent(name.slice(0, 255))}`;
  if (options.onProgress && typeof XMLHttpRequest !== "undefined") {
    const { status, body } = await uploadWithProgress(
      url,
      blob,
      options.onProgress,
      options.signal,
    );
    if (status >= 200 && status < 300) return body as AudioInputRecord;
    if (status !== 401) {
      throw new AudioApiError(
        formatApiErrorBody(body) ?? `Upload failed (${status})`,
        status,
      );
    }
  }
  const response = await authFetch(url, {
    method: "POST",
    headers: { "Content-Type": blob.type || "application/octet-stream" },
    body: blob,
    signal: options.signal,
  });
  return parseAudioJson<AudioInputRecord>(response);
}

/** Upload alive unless 404. GET, not HEAD: FastAPI answers HEAD with 405. */
export async function audioInputAlive(inputId: string): Promise<boolean> {
  try {
    const response = await authFetch(
      `/api/inference/audio/inputs/${encodeURIComponent(inputId)}/file`,
      { headers: { Range: "bytes=0-0" } },
    );
    void response.body?.cancel().catch(() => undefined);
    return response.status !== 404;
  } catch {
    return true;
  }
}

export async function fetchAudioBlob(
  url: string,
  signal?: AbortSignal,
): Promise<Blob> {
  const response = await authFetch(url, { signal });
  if (!response.ok) {
    throw new AudioApiError(await readFastApiError(response), response.status);
  }
  return response.blob();
}

interface TranscribeInputResponse {
  text: string;
  language: string | null;
  model: string;
}

export async function transcribeAudioInput(
  ref: AudioSourceRef,
  body: {
    model: string;
    engine?: string;
    device?: string;
    context_length?: number;
    language?: string;
    purpose?: "reference" | "convert";
  },
  signal?: AbortSignal,
): Promise<TranscribeInputResponse> {
  const response = await authFetch(transcribeUrl(ref), {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify(body),
    signal,
  });
  return parseAudioJson<TranscribeInputResponse>(response);
}

export interface AudioRunResponse {
  clips: {
    id: string;
    role: string;
    url: string;
    sample_rate: number;
    duration_s: number;
    workflow: string;
    seed?: number | null;
  }[];
  group_id: string | null;
  model: string;
  audio: GeneratedAudio | null;
  ignored_options?: string[];
}

export async function runAudio(
  request: AudioRunRequest,
  signal?: AbortSignal,
): Promise<AudioRunResponse> {
  const response = await authFetch("/api/inference/audio/run", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify(withAudioSlotModel(buildAudioRunBody(request))),
    signal,
  });
  return parseAudioJson<AudioRunResponse>(response);
}

export async function runSpeechOnConnection(
  request: {
    text: string;
    providerId: string;
    model: string;
    voice: string;
    instructions: string | null;
    speed: number;
  },
  signal?: AbortSignal,
): Promise<AudioRunResponse> {
  const response = await authFetch("/api/inference/audio/run", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      workflow: "speak",
      text: request.text,
      provider_id: request.providerId,
      model: request.model,
      voice: request.voice,
      speed: request.speed,
      ...(request.instructions ? { instructions: request.instructions } : {}),
    }),
    signal,
  });
  return parseAudioJson<AudioRunResponse>(response);
}

export interface AudioVoice {
  id: string;
  name: string;
  transcript: string | null;
  language: string | null;
  duration_s: number;
  sample_rate: number;
  created_at: string;
  url: string;
}

export async function listVoices(): Promise<AudioVoice[]> {
  const body = await parseAudioJson<{ voices: AudioVoice[] }>(
    await authFetch("/api/inference/audio/voices"),
  );
  return body.voices;
}

export async function createVoice(
  request: AudioVoiceCreateRequest,
): Promise<AudioVoice> {
  return parseAudioJson<AudioVoice>(
    await authFetch("/api/inference/audio/voices", {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify(buildVoiceCreateBody(request)),
    }),
  );
}

export async function updateVoice(
  id: string,
  patch: {
    name?: string;
    transcript?: string | null;
    language?: string | null;
  },
): Promise<AudioVoice> {
  return parseAudioJson<AudioVoice>(
    await authFetch(`/api/inference/audio/voices/${encodeURIComponent(id)}`, {
      method: "PATCH",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify(patch),
    }),
  );
}

export async function deleteVoice(id: string): Promise<void> {
  await parseAudioJson(
    await authFetch(`/api/inference/audio/voices/${encodeURIComponent(id)}`, {
      method: "DELETE",
    }),
  );
}