Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

11.0 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

import { authFetch } from "@/features/auth";
import { consumeNativePathToken } from "@/features/native-intents/api";

/** Lives in the import-free memory-fit module so the node test runner can reach it. */
export { formatMemoryGb } from "../model-config/memory-fit";

export type MemoryEstimateReason =
  | "not_gguf"
  | "not_downloaded"
  | "unsupported_source"
  | "unsizable";

export interface MemoryEstimate {
  available: boolean;
  reason: MemoryEstimateReason | null;
  weightsBytes: number;
  /** Meaningless unless `kvEstimable`. */
  kvBytes: number;
  kvCheckpointBytes: number;
  computeBytes: number;
  drafterRuntimeBytes: number;
  /** Under MTP the target and draft halves can be placed differently. */
  drafterRuntimeGpuBytes: number;
  /** About 0.4x the projector file on top of it. */
  projectorRuntimeBytes: number;
  /** `--spec-draft-hf` names a remote repo, so its cache is unsized; the total is a floor. */
  drafterKvUnsized: boolean;
  /** The total is a floor. */
  adaptersUnsized: boolean;
  totalBytes: number;
  gpuBytes: number;
  /** False when the GGUF header lacks attention dims; the numbers are then a lower bound. */
  kvEstimable: boolean;
  kvOnGpu: boolean;
  nCtx: number;
  contextFitted: number | null;
  contextIsPinned: boolean;
  gpuFloorBytes: number | null;
  floorCanOffload: boolean;
  cacheTypeKv: string | null;
  nParallel: number;
  layerCount: number | null;
  gpuLayers: number | null;
  /** `--n-cpu-moe` is set, so the GPU figure ignores it and reads high. */
  moeOffloadUnmodelled: boolean;
  lookupTableBytes?: number;
  /** Tables streamed from disk, so excluded from `weightsBytes` and `totalBytes`. */
  lookupTableOnDisk?: boolean;
  /** MLX: share of the PLE tables / routed experts read from disk, priced in `weightsBytes`. */
  mlxPleOffload: number | null;
  mlxExpertOffload: number | null;
  mlxOffloadable: MlxOffloadablePart[];
}

export type MlxOffloadablePart = "ple" | "experts";

/** The load settings that move the estimate. Mirrors the fields /load takes. */
export interface MemoryEstimateRequest {
  modelPath: string;
  ggufVariant?: string | null;
  hfToken?: string | null;
  nativePathToken?: string | null;
  nCtx?: number | null;
  cacheTypeKv?: string | null;
  maxSeqLength?: number | null;
  mlxKvQuant?: string | null;
  mlxPleOffload?: "off" | number | null;
  mlxExpertOffload?: "off" | number | null;
  nParallel?: number | null;
  nBatch?: number | null;
  nUbatch?: number | null;
  ctxCheckpoints?: number | null;
  speculativeType?: string | null;
  specDraftNMax?: number | null;
  specDraftModel?: string | null;
  specDraftCacheType?: string | null;
  tensorParallel?: boolean;
  disableVision?: boolean;
  gpuMemoryMode?: string | null;
  gpuLayers?: number | null;
  nCpuMoe?: number | null;
  selectedGpuIds?: number[] | null;
  llamaExtraArgs?: string[] | null;
}

const UNAVAILABLE: MemoryEstimate = {
  available: false,
  reason: "unsizable",
  weightsBytes: 0,
  kvBytes: 0,
  kvCheckpointBytes: 0,
  computeBytes: 0,
  drafterRuntimeBytes: 0,
  drafterRuntimeGpuBytes: 0,
  projectorRuntimeBytes: 0,
  drafterKvUnsized: false,
  adaptersUnsized: false,
  totalBytes: 0,
  gpuBytes: 0,
  kvEstimable: false,
  kvOnGpu: true,
  nCtx: 0,
  contextFitted: null,
  contextIsPinned: true,
  gpuFloorBytes: null,
  floorCanOffload: false,
  cacheTypeKv: null,
  nParallel: 1,
  layerCount: null,
  gpuLayers: null,
  moeOffloadUnmodelled: false,
  mlxPleOffload: null,
  mlxExpertOffload: null,
  mlxOffloadable: [],
};

interface ApiEstimateResponse {
  available: boolean;
  reason: MemoryEstimateReason | null;
  weights_bytes: number;
  kv_bytes: number;
  kv_checkpoint_bytes?: number;
  compute_bytes: number;
  drafter_runtime_bytes: number;
  drafter_runtime_gpu_bytes: number;
  projector_runtime_bytes: number;
  drafter_kv_unsized: boolean;
  adapters_unsized?: boolean;
  total_bytes: number;
  gpu_bytes: number;
  kv_estimable: boolean;
  kv_on_gpu: boolean;
  n_ctx: number;
  context_fitted?: number | null;
  context_is_pinned?: boolean;
  gpu_floor_bytes?: number | null;
  floor_can_offload?: boolean;
  cache_type_kv: string | null;
  n_parallel: number;
  layer_count: number | null;
  gpu_layers: number | null;
  moe_offload_unmodelled: boolean;
  lookup_table_bytes?: number;
  lookup_table_on_disk?: boolean;
  mlx_offload?: { ple?: unknown; experts?: unknown; offloadable?: unknown } | null;
}

function estimateRequestBody(
  payload: MemoryEstimateRequest,
  nativePathLease: string | null,
): string {
  return JSON.stringify({
    model_path: payload.modelPath,
    gguf_variant: payload.ggufVariant ?? null,
    hf_token: payload.hfToken ?? null,
    native_path_lease: nativePathLease,
    n_ctx: payload.nCtx ?? null,
    cache_type_kv: payload.cacheTypeKv ?? null,
    max_seq_length: payload.maxSeqLength ?? null,
    mlx_kv_quant: payload.mlxKvQuant ?? null,
    mlx_ple_offload: payload.mlxPleOffload ?? null,
    mlx_expert_offload: payload.mlxExpertOffload ?? null,
    n_parallel: payload.nParallel ?? null,
    n_batch: payload.nBatch ?? null,
    n_ubatch: payload.nUbatch ?? null,
    ctx_checkpoints: payload.ctxCheckpoints ?? null,
    speculative_type: payload.speculativeType ?? null,
    spec_draft_n_max: payload.specDraftNMax ?? null,
    spec_draft_model: payload.specDraftModel ?? null,
    spec_draft_cache_type: payload.specDraftCacheType ?? null,
    tensor_parallel: payload.tensorParallel ?? false,
    disable_vision: payload.disableVision ?? false,
    gpu_memory_mode: payload.gpuMemoryMode ?? null,
    gpu_layers: payload.gpuLayers ?? null,
    n_cpu_moe: payload.nCpuMoe ?? null,
    selected_gpu_ids: payload.selectedGpuIds ?? null,
    llama_extra_args: payload.llamaExtraArgs ?? null,
  });
}

/** Guards Infinity, stringified and negative values; absent falls back, explicit 0 is real. */
function finiteBytes(value: unknown, fallback: number): number {
  return typeof value === "number" && Number.isFinite(value) && value >= 0
    ? value
    : fallback;
}

function finiteCount(value: unknown, fallback: number): number {
  return typeof value === "number" && Number.isFinite(value) && value >= 0
    ? Math.trunc(value)
    : fallback;
}

function nullableCount(value: unknown): number | null {
  return typeof value === "number" && Number.isFinite(value) && value >= 0
    ? Math.trunc(value)
    : null;
}

/** Non-booleans count as absent: `Boolean("false")` is true. */
function flag(value: unknown, fallback: boolean): boolean {
  return typeof value === "boolean" ? value : fallback;
}

function offloadFraction(value: unknown): number | null {
  return typeof value === "number" && Number.isFinite(value) && value > 0 && value <= 1
    ? value
    : null;
}

function offloadableParts(value: unknown): MlxOffloadablePart[] {
  return Array.isArray(value)
    ? value.filter((part): part is MlxOffloadablePart => part === "ple" || part === "experts")
    : [];
}

const ESTIMATE_REASONS: readonly MemoryEstimateReason[] = [
  "not_gguf",
  "not_downloaded",
  "unsupported_source",
  "unsizable",
];

function toMemoryEstimate(body: ApiEstimateResponse): MemoryEstimate {
  const drafterRuntimeBytes = finiteBytes(body.drafter_runtime_bytes, 0);
  const kvBytes = finiteBytes(body.kv_bytes, 0);
  return {
    available: flag(body.available, false),
    // An unknown reason would render nothing.
    reason: ESTIMATE_REASONS.includes(body.reason as MemoryEstimateReason)
      ? (body.reason as MemoryEstimateReason)
      : null,
    weightsBytes: finiteBytes(body.weights_bytes, 0),
    kvBytes,
    kvCheckpointBytes: Math.min(
      kvBytes,
      finiteBytes(body.kv_checkpoint_bytes, 0),
    ),
    computeBytes: finiteBytes(body.compute_bytes, 0),
    drafterRuntimeBytes,
    // Absent on an older backend: charge the whole term to the GPU rather than inventing a zero.
    drafterRuntimeGpuBytes: finiteBytes(
      body.drafter_runtime_gpu_bytes,
      drafterRuntimeBytes,
    ),
    projectorRuntimeBytes: finiteBytes(body.projector_runtime_bytes, 0),
    drafterKvUnsized: flag(body.drafter_kv_unsized, false),
    adaptersUnsized: flag(body.adapters_unsized, false),
    totalBytes: finiteBytes(body.total_bytes, 0),
    gpuBytes: finiteBytes(body.gpu_bytes, 0),
    // Absent on an older backend: treat KV as unverified, the safe direction.
    kvEstimable: flag(body.kv_estimable, false),
    kvOnGpu: flag(body.kv_on_gpu, true),
    nCtx: finiteCount(body.n_ctx, 0),
    contextFitted: nullableCount(body.context_fitted),
    contextIsPinned: flag(body.context_is_pinned, true),
    gpuFloorBytes: nullableCount(body.gpu_floor_bytes),
    floorCanOffload: flag(body.floor_can_offload, false),
    cacheTypeKv:
      typeof body.cache_type_kv === "string" ? body.cache_type_kv : null,
    nParallel: finiteCount(body.n_parallel, 1),
    layerCount: nullableCount(body.layer_count),
    gpuLayers: nullableCount(body.gpu_layers),
    moeOffloadUnmodelled: flag(body.moe_offload_unmodelled, false),
    lookupTableBytes: finiteBytes(body.lookup_table_bytes, 0),
    lookupTableOnDisk: flag(body.lookup_table_on_disk, false),
    mlxPleOffload: offloadFraction(body.mlx_offload?.ple),
    mlxExpertOffload: offloadFraction(body.mlx_offload?.experts),
    mlxOffloadable: offloadableParts(body.mlx_offload?.offloadable),
  };
}

/** 404/405/501 mean the route is absent; anything else is about this request. */
function routeAbsentStatus(status: number): boolean {
  return status === 404 || status === 405 || status === 501;
}

/** POSTed after every settings change; a TTL avoids a 404 per slider release yet re-probes
    after Studio replaces its backend in place. */
const ROUTE_ABSENT_TTL_MS = 5 * 60 * 1000;
let routeAbsentAt: number | null = null;

export function resetMemoryEstimateRouteMemo(): void {
  routeAbsentAt = null;
}

export async function fetchMemoryEstimate(
  payload: MemoryEstimateRequest,
  signal?: AbortSignal,
): Promise<MemoryEstimate> {
  if (
    routeAbsentAt !== null &&
    Date.now() - routeAbsentAt < ROUTE_ABSENT_TTL_MS
  ) {
    return UNAVAILABLE;
  }
  let nativePathLease: string | null = null;
  if (payload.nativePathToken) {
    try {
      nativePathLease = (
        await consumeNativePathToken(payload.nativePathToken, "validate-model")
      ).nativePathLease;
    } catch {
      return { ...UNAVAILABLE, reason: "unsupported_source" };
    }
  }
  const response = await authFetch("/api/inference/estimate-memory", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    signal,
    body: estimateRequestBody(payload, nativePathLease),
  });
  if (!response.ok) {
    // Only a structural miss latches; a transient 500 clears the memo.
    routeAbsentAt = routeAbsentStatus(response.status) ? Date.now() : null;
    return UNAVAILABLE;
  }
  routeAbsentAt = null;
  let body: unknown;
  try {
    body = await response.json();
  } catch {
    // A non-JSON 200 (captive portal, proxy HTML, truncated body) measured nothing.
    return UNAVAILABLE;
  }
  if (typeof body !== "object" || body === null) {
    return UNAVAILABLE;
  }
  return toMemoryEstimate(body as ApiEstimateResponse);
}