Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

3.7 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.

import { authFetch } from "@/features/auth";
import { readFastApiError } from "@/lib/format-fastapi-error";

export type InferenceEngine = "auto" | "vllm" | "sglang";
export interface EngineStatus {
  engine: Exclude<InferenceEngine, "auto">;
  version: string;
  installed_version: string | null;
  installed: boolean;
  in_use: boolean;
  current: boolean;
  restored?: boolean;
  can_rollback: boolean;
  unsupported_reason: string | null;
  /** Load precisions this host's build of the engine supports; absent from older backends. */
  precisions?: string[];
  /** "rocm": the engine runs on an AMD GPU. */
  platform?: "cuda" | "rocm";
  /** "wsl": on Windows the engine runs inside Studio's own WSL2 distro. */
  host?: "local" | "wsl";
  wsl?: { state: string | null; distro: string | null };
  download_bytes?: number | null;
  job: {
    state: string;
    phase: string | null;
    message: string;
    activity?: string;
    log?: string[];
  };
}

export function isEngineReady(engine: EngineStatus | undefined): boolean {
  return (
    !!engine?.installed &&
    (engine.current || engine.restored === true) &&
    !engine.unsupported_reason &&
    engine.job.state !== "running"
  );
}

/** The backend says this host can run vLLM, installed or not. */
export function vllmHostSupported(engines: readonly EngineStatus[]): boolean {
  const vllm = engines.find((engine) => engine.engine === "vllm");
  return vllm !== undefined && vllm.unsupported_reason === null;
}

/** A retained precision the newly picked engine cannot load would fail only after unloading. */
export function precisionAfterEngineSwitch<P extends string>(
  precision: P,
  engine: EngineStatus | undefined,
): P | "auto" {
  return supportsPrecision(engine, precision) ? precision : "auto";
}

export function supportsPrecision(
  engine: EngineStatus | undefined,
  precision: string,
): boolean {
  if (
    (precision === "int4" || precision === "int8") &&
    !convertsToInteger(engine)
  ) {
    return false;
  }
  return !engine?.precisions || engine.precisions.includes(precision);
}

export function wslNoticeKey(
  engine: EngineStatus,
): "managedEngines.wslSetup" | "managedEngines.wslReady" | "managedEngines.wslRestart" | null {
  if (engine.host !== "wsl") return null;
  if (engine.wsl?.state === "restart_required") return "managedEngines.wslRestart";
  if (engine.wsl?.state === "ready" && engine.wsl?.distro) return "managedEngines.wslReady";
  return "managedEngines.wslSetup";
}

/** SGLang 0.5.18 dropped --torchao-config, its only load-time INT8 / 4-bit conversion. */
export function convertsToInteger(engine: EngineStatus | undefined): boolean {
  if (engine?.engine !== "sglang") return true;
  const parts = (engine.installed_version ?? engine.version)
    .split(".")
    .map(Number);
  const [major = 0, minor = 0, patch = 0] = parts;
  return major === 0 && (minor < 5 || (minor === 5 && patch < 18));
}

export async function listEngines(): Promise<EngineStatus[]> {
  const response = await authFetch("/api/engines");
  if (!response.ok) {
    throw new Error(
      await readFastApiError(response, "Could not read inference engines"),
    );
  }
  return response.json();
}

export async function changeEngine(
  engine: string,
  operation: "install" | "cancel" | "remove" | "rollback",
): Promise<void> {
  const response = await authFetch(
    `/api/engines/${engine}${operation === "remove" ? "" : `/${operation}`}`,
    {
      method: operation === "remove" ? "DELETE" : "POST",
    },
  );
  if (!response.ok) {
    throw new Error(
      await readFastApiError(response, "Could not change inference engine"),
    );
  }
  window.dispatchEvent(new Event("studio-engines-changed"));
}