Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

17.4 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

import { useEffect, useMemo, useState } from "react";
import { normalizeDenseQuantSchemes } from "@/lib/dense-quant-schemes";
import {
  type ReportedOffloadFitTier,
  normalizeReportedOffloadFitTiers,
} from "@/lib/offload-fit-tiers";
import {
  type GpuIndexKind,
  type PinnableGpuContext,
  type ReconciledGpuSelection,
  type SystemGpuDevice,
  pickLoadDevice,
  pinnableGpuContext,
  reconcileGpuSelection,
  resolveGpuSelectionContext,
} from "./gpu-selection";
import {
  gpuMemoryTotalsGb,
  gpuSharedHostMemoryGb,
  sharesHostMemory,
  systemRamAvailableOutsideSharedPoolGb,
} from "./gpu-vram";
import {
  type GpuDevice,
  type SystemInfoResponse,
  fetchSystemInfo,
  getCachedSystemInfo,
  subscribeSystemInfo,
} from "./use-system";

export {
  pinnableGpuContext,
  reconcileGpuSelection,
  type GpuIndexKind,
  type PinnableGpuContext,
  type ReconciledGpuSelection,
  type SystemGpuDevice,
} from "./gpu-selection";

export interface GpuInfo {
  available: boolean;
  budgetKnown: boolean;
  sharedMemory: boolean;
  /** A unified host pool (ROCm APU) is not a VRAM ceiling for fit verdicts. */
  unifiedMemory: boolean;
  /** Carried on the GPU-less path too. Empty until system info arrives. */
  backend: string;
  denseQuantSupported: boolean;
  /** Best first. Empty until system info arrives, and on older backends. */
  denseQuantSchemes: readonly string[];
  nvfp4Diffusion: boolean;
  /** Prequantized LLM checkpoint formats ("fp8", "nvfp4") this host runs natively. Empty until system info arrives. */
  checkpointQuantFormats: readonly string[];
  /** Group offload can stream torchao weights. Absent or false until resolved and on older backends. */
  quantisedStreaming?: boolean;
  extraOffloadFitTiers?: Readonly<
    Record<string, readonly ReportedOffloadFitTier[]>
  >;
  name: string;
  memoryTotalGb: number;
  memorySharedGb: number;
  /** Without shared-memory devices: VRAM beside system RAM, not a view of it. */
  dedicatedMemoryTotalGb: number;
  /** Image/video loads live on ONE device, so their fit math must not use the multi-GPU sum. */
  maxDeviceMemoryGb: number;
  /** Lowest visible ordinal, where a bare "cuda" diffusion load lands; NOT maxDeviceMemoryGb on a
   * heterogeneous host. */
  loadDeviceMemoryGb: number;
  loadDeviceSharedMemory: boolean;
  /** Includes Linux ROCm APUs, which report only unified_memory; offload frees nothing on either. */
  loadDeviceSharesHostMemory: boolean;
  /** How many GPUs memoryTotalGb sums over, for the loader's per-card VRAM reserve. */
  deviceCount: number;
  cpuCore: number;
  cpuThread: number;
  /** Host RAM free minus the host-backed shared GPU pool. */
  systemRamAvailableGb: number;
  systemRamAvailableHostGb: number;
  /** Whether host available memory was reported, including a real zero. */
  systemRamAvailableKnown?: boolean;
  systemRamTotalGb: number;
  /** Apple MLX only: what llama.cpp and MLX admit right now, `min(working set, free) * 0.85`
   *  (`_apple_metal_memory_budget_bytes`). Null on every other host and on older backends. */
  unifiedLoadBudgetGb?: number | null;
  /** `min(working set, free + AGX in use) * 0.85`: the budget once the loader has freed a resident model. */
  unifiedLoadCeilingGb?: number | null;
}

/** `_APPLE_UNIFIED_MEMORY_FRACTION` in llama_cpp.py and mlx_inference.py. */
const APPLE_UNIFIED_MEMORY_FRACTION = 0.85;

function finitePositive(value: number | null | undefined): number | null {
  return typeof value === "number" && Number.isFinite(value) && value > 0
    ? value
    : null;
}

// A real 0 GiB free reading is the most pressured case, so it must not read as unknown.
function finiteNonNegative(value: number | null | undefined): number | null {
  return typeof value === "number" && Number.isFinite(value) && value >= 0
    ? value
    : null;
}

function appleUnifiedLoadBudgetGb(
  data: SystemInfoResponse | null,
  devices: readonly GpuDevice[],
  afterUnload = false,
): number | null {
  if (data?.device_backend !== "mlx" || devices.length !== 1) return null;
  const device = devices[0];
  const workingSetGb = finitePositive(device.metal_working_set_gb);
  if (device.shared_memory !== true || workingSetGb === null) return null;
  let freeGb =
    finiteNonNegative(device.vram_free_gb) ??
    finiteNonNegative(data.memory?.available_gb);
  if (afterUnload) {
    // The AGX in-use figure holds the resident model's Metal buffers, which the loader frees first;
    // other apps' memory stays counted. Unknown usage leaves only the working-set ceiling.
    const usedGb = finiteNonNegative(device.vram_used_gb);
    freeGb = freeGb !== null && usedGb !== null ? freeGb + usedGb : null;
  }
  // Floored to 0.25 GiB: free memory drifts every poll, and each change re-renders every picker row.
  return (
    Math.floor(
      Math.min(workingSetGb, freeGb ?? workingSetGb) *
        APPLE_UNIFIED_MEMORY_FRACTION *
        4,
    ) / 4
  );
}

const DEFAULT_GPU: GpuInfo = {
  available: false,
  budgetKnown: false,
  sharedMemory: false,
  unifiedMemory: false,
  backend: "",
  denseQuantSupported: false,
  denseQuantSchemes: [],
  nvfp4Diffusion: false,
  checkpointQuantFormats: [],
  quantisedStreaming: false,
  name: "Unknown",
  memoryTotalGb: 0,
  memorySharedGb: 0,
  dedicatedMemoryTotalGb: 0,
  maxDeviceMemoryGb: 0,
  loadDeviceMemoryGb: 0,
  loadDeviceSharedMemory: false,
  loadDeviceSharesHostMemory: false,
  deviceCount: 0,
  cpuCore: 0,
  cpuThread: 0,
  systemRamAvailableGb: 0,
  systemRamAvailableHostGb: 0,
  systemRamAvailableKnown: false,
  systemRamTotalGb: 0,
  unifiedLoadBudgetGb: null,
  unifiedLoadCeilingGb: null,
};

function toGpuInfo(
  data: SystemInfoResponse | null,
  source: "gpu" | "inference_gpu" = "gpu",
): GpuInfo {
  // CPU/RAM exist on GPU-less hosts too, and unified-memory math needs a RAM budget.
  const base = {
    backend: data?.device_backend ?? "",
    denseQuantSupported: data?.dense_quant_supported === true,
    denseQuantSchemes: normalizeDenseQuantSchemes(data?.dense_quant_schemes),
    nvfp4Diffusion: data?.nvfp4_diffusion === true,
    checkpointQuantFormats: normalizeDenseQuantSchemes(
      data?.checkpoint_quant_formats,
    ),
    quantisedStreaming: data?.quantised_streaming === true,
    extraOffloadFitTiers: normalizeReportedOffloadFitTiers(
      data?.diffusers_offload_tiers,
    ),
    cpuCore: data?.cpu?.physical_count ?? 0,
    cpuThread: data?.cpu?.logical_count ?? 0,
    systemRamAvailableGb: data?.memory?.available_gb ?? 0,
    systemRamAvailableHostGb: data?.memory?.available_gb ?? 0,
    systemRamAvailableKnown:
      Number.isFinite(data?.memory?.available_gb) &&
      (data?.memory?.available_gb as number) >= 0,
    systemRamTotalGb: data?.memory?.total_gb ?? 0,
  };
  const gpuData =
    source === "inference_gpu" ? (data?.inference_gpu ?? data?.gpu) : data?.gpu;
  const devices = gpuData?.devices ?? [];
  if (!gpuData?.available || !devices.length) {
    return { ...DEFAULT_GPU, ...base, budgetKnown: data !== null };
  }
  const memoryTotals = gpuMemoryTotalsGb(devices);
  const loadDevice = pickLoadDevice(devices);
  return {
    ...base,
    // Raw: folding first collapsed multi-socket unified pools into one.
    systemRamAvailableGb: systemRamAvailableOutsideSharedPoolGb(
      base.systemRamAvailableGb,
      gpuSharedHostMemoryGb(devices),
    ),
    sharedMemory: memoryTotals.shared > 0 && memoryTotals.dedicated === 0,
    // some(), unlike sharedMemory: one unified part already stops the total being a VRAM ceiling.
    unifiedMemory: devices.some((device) => device.unified_memory === true),
    available: true,
    budgetKnown: true,
    name: devices[0]?.name ?? "Unknown",
    memoryTotalGb: memoryTotals.total,
    dedicatedMemoryTotalGb: memoryTotals.dedicated,
    memorySharedGb: memoryTotals.shared,
    maxDeviceMemoryGb: devices.reduce(
      (max, d) => Math.max(max, d.memory_total_gb ?? 0),
      0,
    ),
    // Lowest visible ordinal = torch's current device = where the pipeline lands.
    loadDeviceMemoryGb: loadDevice?.memory_total_gb ?? 0,
    loadDeviceSharedMemory: loadDevice?.shared_memory === true,
    loadDeviceSharesHostMemory: sharesHostMemory({
      sharedMemory: loadDevice?.shared_memory === true,
      unifiedMemory: loadDevice?.unified_memory === true,
    }),
    deviceCount: devices.length,
    unifiedLoadBudgetGb: appleUnifiedLoadBudgetGb(data, devices),
    unifiedLoadCeilingGb: appleUnifiedLoadBudgetGb(data, devices, true),
  };
}

function toGpuDevices(
  data: SystemInfoResponse | null,
  // Diffusion runs on torch, so it reads the torch inventory even on a Vulkan inference build.
  forDiffusion = false,
): SystemGpuDevice[] {
  // GGUF runs via llama-server, so on Vulkan the pickable set is its inventory and ggml ordinals.
  const inference = data?.inference_gpu;
  if (!forDiffusion && inference?.backend === "vulkan") {
    // Confirmed Vulkan: never fall through to torch IDs, which the backend rejects for Vulkan builds.
    if (!(inference.devices ?? []).length) return [];
    const picksAccepted = inference.gguf_gpu_ids_supported !== false;
    return (inference.devices ?? [])
      .filter((d) => typeof d.index === "number")
      .map((d) => ({
        index: d.index as number,
        indexKind: d.index_kind === "vulkan" ? ("vulkan" as const) : null,
        name: d.name ?? `GPU ${d.index}`,
        memoryTotalGb: d.memory_total_gb ?? 0,
        memoryFreeGb: d.vram_free_gb ?? 0,
        memoryFreeKnown:
          Number.isFinite(d.vram_free_gb) && (d.vram_free_gb as number) >= 0,
        sharedMemory: d.shared_memory === true,
        sharedMemoryHostBackedGb: d.shared_memory_host_backed_gb,
        unifiedMemory: d.unified_memory === true,
        pinnable: picksAccepted && d.index_kind === "vulkan",
        // The DiffusionGemma runner never speaks ggml ordinals.
        diffusionPinnable: false,
      }));
  }
  // Unpinnable configs must hide every pick surface; absent gguf_gpu_ids_supported means pinnable.
  const pinnableBackend = data?.gpu?.gguf_gpu_ids_supported !== false;
  // ROCm reuses torch.cuda.* and physical IDs; only the label differs.
  const diffusionBackend =
    data?.device_backend === "cuda" || data?.device_backend === "rocm";
  return (data?.gpu?.devices ?? [])
    .filter((d) => typeof d.index === "number")
    .map((d) => ({
      index: d.index as number,
      indexKind:
        d.index_kind === "physical" ||
        d.index_kind === "vulkan" ||
        d.index_kind === "cuda_device"
          ? d.index_kind
          : null,
      name: d.name ?? `GPU ${d.index}`,
      memoryTotalGb: d.memory_total_gb ?? 0,
      memoryFreeGb: d.vram_free_gb ?? 0,
      memoryFreeKnown:
        Number.isFinite(d.vram_free_gb) && (d.vram_free_gb as number) >= 0,
      sharedMemory: d.shared_memory === true,
      sharedMemoryHostBackedGb: d.shared_memory_host_backed_gb,
      unifiedMemory: d.unified_memory === true,
      // The XPU ban covers torch-xpu ordinals, not Vulkan ones.
      pinnable:
        pinnableBackend &&
        (d.index_kind === "vulkan" ||
          (data?.device_backend !== "xpu" &&
            (d.index_kind === "physical" || d.index_kind === "cuda_device"))),
      diffusionPinnable:
        diffusionBackend &&
        (d.index_kind === "physical" || d.index_kind === "cuda_device"),
    }));
}

/** Keep the previous array when unchanged: polls re-probe memory, and consumers memoise picker
 * options on this list. */
export function withStableSchemes(current: GpuInfo, next: GpuInfo): GpuInfo {
  const held = current.denseQuantSchemes;
  const fresh = next.denseQuantSchemes;
  if (held === fresh) return next;
  if (
    held.length === fresh.length &&
    held.every((scheme, index) => scheme === fresh[index])
  ) {
    return { ...next, denseQuantSchemes: held };
  }
  return next;
}

const RAM_DRIFT_TOLERANCE_GB = 0.25;

/** Keeps `current` when only free RAM drifted, so subscribed pages do not re-render on every poll. */
export function nextGpuInfo(current: GpuInfo, next: GpuInfo): GpuInfo {
  const ramDrift = Math.max(
    Math.abs(current.systemRamAvailableGb - next.systemRamAvailableGb),
    Math.abs(current.systemRamAvailableHostGb - next.systemRamAvailableHostGb),
  );
  const withoutLiveRam = (info: GpuInfo) =>
    JSON.stringify({
      ...info,
      systemRamAvailableGb: 0,
      systemRamAvailableHostGb: 0,
    });
  if (
    ramDrift < RAM_DRIFT_TOLERANCE_GB &&
    withoutLiveRam(current) === withoutLiveRam(next)
  ) {
    return current;
  }
  return withStableSchemes(current, next);
}

/** Aggregate GPU info from /api/system; shares one module-level fetch across all GPU hooks. */
function useGpuInfoSource(source: "gpu" | "inference_gpu"): GpuInfo {
  const cachedSystem = getCachedSystemInfo();
  const [gpu, setGpu] = useState<GpuInfo>(
    cachedSystem ? toGpuInfo(cachedSystem, source) : DEFAULT_GPU,
  );
  useEffect(() => {
    // No early return on cachedSystem: a consumer mounting as the cache fills would stay default.
    let cancelled = false;
    const sync = (data: SystemInfoResponse) => {
      if (cancelled) return;
      const next = toGpuInfo(data, source);
      setGpu((current) => nextGpuInfo(current, next));
    };
    const update = () => {
      fetchSystemInfo().then((d) => {
        if (cancelled) return;
        if (!d) return;
        // A cache hit publishes no snapshot, so sync here.
        sync(d);
      });
    };
    const unsubscribe = subscribeSystemInfo(sync, {
      retryUnavailableVulkan: source === "inference_gpu",
    });
    update();
    return () => {
      cancelled = true;
      unsubscribe();
    };
  }, [source]);
  return gpu;
}

export function useGpuInfo(): GpuInfo {
  return useGpuInfoSource("gpu");
}

/** Includes a separately installed Vulkan backend. */
export function useInferenceGpuInfo(): GpuInfo {
  return useGpuInfoSource("inference_gpu");
}

/** Optional engines launch on torch's physical CUDA/ROCm ids (`useGpuDevices(true)`), never a Vulkan ordinal. */
export function isEngineGpuDevice(device: SystemGpuDevice): boolean {
  return device.diffusionPinnable;
}

/** Where the backend puts an optional engine given no GPUs: the first one Studio sees. */
export function defaultEngineGpuIds(): number[] {
  const first = toGpuDevices(getCachedSystemInfo(), true).find(
    isEngineGpuDevice,
  );
  return first ? [first.index] : [0];
}

export function useGpuDevices(forDiffusion = false): SystemGpuDevice[] {
  const cachedSystem = getCachedSystemInfo();
  const [devices, setDevices] = useState<SystemGpuDevice[]>(
    cachedSystem ? toGpuDevices(cachedSystem, forDiffusion) : [],
  );
  useEffect(() => {
    // No early return on cachedSystem: a consumer mounting as the cache fills would stay default.
    let cancelled = false;
    let lastSerialized: string | null = null;
    const sync = (data: SystemInfoResponse | null) => {
      if (cancelled) return;
      const next = toGpuDevices(data, forDiffusion);
      // Compare by value, or the 3s Vulkan retry would re-render forever.
      const serialized = JSON.stringify(next);
      if (serialized === lastSerialized) return;
      lastSerialized = serialized;
      setDevices(next);
    };
    const unsubscribe = subscribeSystemInfo(sync, {
      retryUnavailableVulkan: true,
    });
    fetchSystemInfo().then(sync);
    return () => {
      cancelled = true;
      unsubscribe();
    };
  }, [forDiffusion]);
  return devices;
}

/** Empty when there is nothing to choose; diffusion checkpoints are never sharded. */
export function useDiffusionGpuChoices(): SystemGpuDevice[] {
  const devices = useGpuDevices(true);
  // Memoized: a fresh array per render cascaded into the GGUF picker re-POSTing download plans
  // on every status poll.
  return useMemo(() => {
    const context = pinnableGpuContext(devices, true);
    return (context.ids?.length ?? 0) > 1 ? (context.devices ?? []) : [];
  }, [devices]);
}

export function gpuDeviceCacheReady(): boolean {
  const cachedSystem = getCachedSystemInfo();
  if (cachedSystem === null) {
    return false;
  }
  const inferenceGpu = cachedSystem.inference_gpu;
  return !(inferenceGpu?.backend === "vulkan" && !inferenceGpu.available);
}

export async function ensureGpuDeviceCache(): Promise<void> {
  await fetchSystemInfo();
}

/** null before fetch, [] when pinning is unavailable. */
export function cachedPinnableGpuIndices(
  forDiffusion = false,
): number[] | null {
  return cachedPinnableGpuContext(forDiffusion).ids;
}

/** undefined before fetch, null when unavailable. */
export function cachedPinnableGpuIndexKind(
  forDiffusion = false,
): GpuIndexKind | null | undefined {
  return cachedPinnableGpuContext(forDiffusion).indexKind;
}

/** An unavailable Vulkan probe leaves membership unknown while the namespace stays authoritative. */
export function cachedPinnableGpuContext(
  forDiffusion = false,
  devices?: SystemGpuDevice[],
): PinnableGpuContext {
  const cachedSystem = getCachedSystemInfo();
  const unavailableVulkan =
    cachedSystem?.inference_gpu?.backend === "vulkan" &&
    !cachedSystem.inference_gpu.available;
  return resolveGpuSelectionContext(
    cachedSystem ? (devices ?? toGpuDevices(cachedSystem)) : null,
    forDiffusion,
    unavailableVulkan ? "vulkan" : undefined,
  );
}

export function reconcileCachedGpuSelection(
  ids: number[] | null,
  savedIndexKind?: GpuIndexKind | null,
  forDiffusion = false,
): ReconciledGpuSelection {
  const context = cachedPinnableGpuContext(forDiffusion);
  return reconcileGpuSelection(
    ids,
    savedIndexKind,
    context.indexKind,
    context.ids,
  );
}