Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

17.9 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

import { authFetch } from "@/features/auth";
import {
  disposableTimeoutSignal,
  withAbort,
} from "@/features/hub/lib/abort-signals";
import { hubTokenHeader } from "@/features/hub/lib/hub-token-header";
import { readFastApiError } from "@/lib/format-fastapi-error";
import { localPathCacheKey } from "@/features/hub/lib/local-path";
import { isHuggingFaceOffline } from "@/features/hub/lib/network";
import { fingerprintToken } from "@/features/hub/lib/token-fingerprint";
import { bumpInventoryVersion } from "@/features/hub/stores/inventory-events";
import { notifyModelDeleted } from "@/lib/model-lifecycle-events";
import {
  discardDeletedInventoryHints,
  discardDeletedModelInventoryHints,
} from "../download-manager/download-manager-state";
import type { ScanFolderStatus } from "../lib/scan-folder-status";
import type { LocalSource } from "./constants";
import { bumpGgufVariantsCacheVersion } from "./gguf-variants-cache-events";

export type ModelInventoryFormat =
  | "gguf"
  | "safetensors"
  | "adapter"
  | "checkpoint"
  | "unknown";
export type ModelInventoryRuntime =
  | "llama_cpp"
  | "transformers"
  | "adapter"
  | "unknown";
export type BaseModelSource = "huggingface" | "local" | "unknown";

export interface BackendModelCapabilities {
  can_train?: boolean;
  can_chat?: boolean;
  can_delete?: boolean;
  can_download?: boolean;
  requires_variant?: boolean;
  supports_lora?: boolean;
  supports_vision?: boolean;
}

export interface CachedGgufRepo {
  repo_id: string;
  inventory_id?: string | null;
  load_id?: string | null;
  model_format?: ModelInventoryFormat | null;
  runtime?: ModelInventoryRuntime | null;
  format_variant?: string | null;
  capabilities?: BackendModelCapabilities | null;
  size_bytes: number;
  cache_path?: string;
  /** Epoch seconds; inventory view models normalize this to milliseconds. */
  last_modified?: number | null;
  partial?: boolean;
  partial_transport?: string | null;
  partial_resumable?: boolean;
  pipeline_tag?: string | null;
  task?: string | null;
  audio_type?: string | null;
  audio_workflows?: string[] | null;
  umbrella_folder?: boolean;
  tags?: string[];
  library_name?: string | null;
}

export interface CachedModelRepo {
  repo_id: string;
  inventory_id?: string | null;
  load_id?: string | null;
  model_format?: ModelInventoryFormat | null;
  artifact_kind?: LocalArtifactKind | null;
  runtime?: ModelInventoryRuntime | null;
  format_variant?: string | null;
  capabilities?: BackendModelCapabilities | null;
  size_bytes: number;
  cache_path?: string;
  /** Epoch seconds; inventory view models normalize this to milliseconds. */
  last_modified?: number | null;
  partial?: boolean;
  partial_transport?: string | null;
  partial_resumable?: boolean;
  pipeline_tag?: string | null;
  task?: string | null;
  audio_type?: string | null;
  tags?: string[];
  library_name?: string | null;
  quant_method?: string | null;
}

export interface CachedInventoryResponse<Repo> {
  cached: Repo[];
  scan_confirmed?: boolean;
}

export interface LocalModelInfo {
  id: string;
  inventory_id?: string | null;
  load_id?: string | null;
  display_name: string;
  path: string;
  size_bytes?: number;
  model_format?: ModelInventoryFormat | null;
  artifact_kind?: LocalArtifactKind | null;
  runtime?: ModelInventoryRuntime | null;
  format_variant?: string | null;
  capabilities?: BackendModelCapabilities | null;
  source: LocalSource;
  model_id?: string | null;
  active_cache?: boolean | null;
  base_model?: string | null;
  base_model_source?: BaseModelSource | null;
  adapter_type?: string | null;
  training_method?: string | null;
  updated_at?: number | null;
  partial?: boolean;
  partial_transport?: string | null;
  partial_resumable?: boolean;
  /** Pipeline repo holding only a GGUF load's VAE / text encoder: not a download to continue. */
  companion_prefetch?: boolean;
  pipeline_tag?: string | null;
  task?: string | null;
  audio_type?: string | null;
  tags?: string[];
  library_name?: string | null;
  quant_method?: string | null;
}

export type LocalArtifactKind =
  | "diffusers_pipeline"
  | "diffusers_modular_pipeline"
  | "diffusers_dual_pipeline"
  | "transformers_model"
  | "single_file_checkpoint"
  | "gguf"
  | "adapter"
  | "unknown";

export interface LocalModelListResponse {
  models_dir: string;
  hf_cache_dir?: string | null;
  lmstudio_dirs: string[];
  ollama_dirs?: string[];
  hermes_dirs?: string[];
  models: LocalModelInfo[];
}

export interface CachedDatasetRepo {
  repo_id: string;
  size_bytes: number;
  /** Epoch seconds; absent when no cache path has a readable mtime. */
  last_modified?: number | null;
  cache_path?: string;
  load_cache_path?: string;
  partial?: boolean;
  partial_transport?: string | null;
  partial_resumable?: boolean;
}

export type LocalDatasetInfo = {
  metadata?: {
    actual_num_records?: number | null;
    target_num_records?: number | null;
    total_num_batches?: number | null;
    num_completed_batches?: number | null;
    columns?: string[] | null;
  } | null;
  id: string;
  label: string;
  path: string;
  source: "recipe" | "upload";
  rows?: number | null;
  updated_at?: number | null;
};

export type LocalDatasetsResponse = {
  datasets: LocalDatasetInfo[];
};

export interface ScanFolderInfo {
  id: number;
  path: string;
  created_at: string;
  recursive?: boolean;
  /** Absent on older backends, which means "ok". */
  status?: ScanFolderStatus;
}

export interface GgufVariantDetail {
  context_length?: number | null;
  cache_path?: string | null;
  /** Opaque stand-in for `cache_path` under host-path redaction. */
  cache_ref?: string | null;
  filename: string;
  quant: string;
  display_label?: string | null;
  size_bytes: number;
  download_size_bytes?: number;
  estimated_resident_gb?: number | null;
  /** Streamed lookup-table bytes inside `size_bytes`; 0 when none or on an older backend. */
  lookup_table_bytes?: number;
  pending_drafter_filename?: string | null;
  pending_drafter_size_bytes?: number;
  download_remaining_bytes?: number | null;
  downloaded?: boolean;
  update_available?: boolean;
  partial?: boolean;
  partial_transport?: string | null;
  partial_resumable?: boolean;
  /** Variants sharing this key share one companion download footprint. */
  dependency_key?: string | null;
}

export interface GgufVariantsResponse {
  repo_id: string;
  variants: GgufVariantDetail[];
  has_vision: boolean;
  default_variant: string | null;
  dependencies_resolved?: boolean;
  /** "mlx" for precision folders of an MLX repo. */
  variant_format?: string | null;
}

async function parseJsonOrThrow<T>(
  response: Response,
  fallback?: string,
): Promise<T> {
  if (!response.ok) {
    throw new Error(await readFastApiError(response, fallback));
  }
  return response.json();
}

async function throwIfNotOk(
  response: Response,
  fallback?: string,
): Promise<void> {
  if (!response.ok) {
    throw new Error(await readFastApiError(response, fallback));
  }
}

const INVENTORY_TIMEOUT_MS = 30_000;

async function withHubTimeout<T>(
  ms: number,
  request: (signal: AbortSignal) => Promise<T>,
): Promise<T> {
  const timeout = disposableTimeoutSignal(ms);
  try {
    return await request(timeout.signal);
  } finally {
    timeout.dispose();
  }
}

export async function listLocalModels(): Promise<LocalModelListResponse> {
  const response = await withHubTimeout(INVENTORY_TIMEOUT_MS, (signal) =>
    authFetch("/api/hub/local", { signal }),
  );
  return parseJsonOrThrow<LocalModelListResponse>(response);
}

export async function listUnslothModels(): Promise<LocalModelListResponse> {
  const response = await withHubTimeout(INVENTORY_TIMEOUT_MS, (signal) =>
    authFetch("/api/hub/unsloth-models", { signal }),
  );
  return parseJsonOrThrow<LocalModelListResponse>(response);
}

export async function fetchCachedGgufInventory(
  hfToken?: string | null,
): Promise<CachedInventoryResponse<CachedGgufRepo>> {
  const response = await withHubTimeout(INVENTORY_TIMEOUT_MS, (signal) =>
    authFetch("/api/hub/cached-gguf", {
      headers: hubTokenHeader(hfToken),
      signal,
    }),
  );
  return await parseJsonOrThrow<CachedInventoryResponse<CachedGgufRepo>>(
    response,
  );
}

export async function listCachedGguf(
  hfToken?: string | null,
): Promise<CachedGgufRepo[]> {
  return (await fetchCachedGgufInventory(hfToken)).cached;
}

export async function fetchCachedModelsInventory(
  hfToken?: string | null,
): Promise<CachedInventoryResponse<CachedModelRepo>> {
  const response = await withHubTimeout(INVENTORY_TIMEOUT_MS, (signal) =>
    authFetch("/api/hub/cached-models", {
      headers: hubTokenHeader(hfToken),
      signal,
    }),
  );
  return await parseJsonOrThrow<CachedInventoryResponse<CachedModelRepo>>(
    response,
  );
}

export async function listCachedModels(
  hfToken?: string | null,
): Promise<CachedModelRepo[]> {
  return (await fetchCachedModelsInventory(hfToken)).cached;
}

export async function listLocalDatasets(): Promise<LocalDatasetsResponse> {
  const response = await withHubTimeout(INVENTORY_TIMEOUT_MS, (signal) =>
    authFetch("/api/hub/datasets/local", { signal }),
  );
  return parseJsonOrThrow<LocalDatasetsResponse>(response);
}

export async function listCachedDatasets(): Promise<CachedDatasetRepo[]> {
  const response = await withHubTimeout(INVENTORY_TIMEOUT_MS, (signal) =>
    authFetch("/api/hub/datasets/cached", { signal }),
  );
  const data = await parseJsonOrThrow<{ cached: CachedDatasetRepo[] }>(response);
  return data.cached;
}

export async function deleteCachedDataset(
  repoId: string,
  cachePath?: string | null,
): Promise<void> {
  const payload: Record<string, string> = { repo_id: repoId };
  if (cachePath) {
    payload.cache_path = cachePath;
  }
  const response = await authFetch("/api/hub/datasets/cached", {
    method: "DELETE",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify(payload),
  });
  await throwIfNotOk(response, `Failed to delete dataset (${response.status})`);
  discardDeletedInventoryHints(repoId, ["dataset"]);
  bumpInventoryVersion();
}

export interface CompanionAssetInfo {
  repo_id: string;
  size_bytes: number;
  needed_by: string[];
}

export interface DeleteImpact {
  cache_path?: string | null;
  repo_id: string;
  variant?: string | null;
  reclaimed_bytes: number;
  retained_companions: CompanionAssetInfo[];
  freeable_companions: CompanionAssetInfo[];
  blocked_by: string[];
}

/** return the delete preview or `null` so confirmation still opens when unavailable. */
export async function fetchDeleteImpact(
  repoId: string,
  variant?: string | null,
  cachePath?: string | null,
  hfToken?: string | null,
): Promise<DeleteImpact | null> {
  try {
    const response = await authFetch("/api/hub/delete-impact", {
      method: "POST",
      headers: { "Content-Type": "application/json", ...hubTokenHeader(hfToken) },
      body: JSON.stringify(
        variant
          ? { repo_id: repoId, variant, ...(cachePath ? { cache_path: cachePath } : {}) }
          : { repo_id: repoId, ...(cachePath ? { cache_path: cachePath } : {}) },
      ),
    });
    if (!response.ok) return null;
    return (await response.json()) as DeleteImpact;
  } catch {
    return null;
  }
}

export interface OrphanCompanion {
  repo_id: string;
  size_bytes: number;
  cache_path?: string | null;
}

export async function fetchOrphanCompanions(): Promise<{
  companions: OrphanCompanion[];
  total_bytes: number;
}> {
  const response = await authFetch("/api/hub/orphan-companions");
  return await parseJsonOrThrow<{
    companions: OrphanCompanion[];
    total_bytes: number;
  }>(response);
}

export async function deleteCachedModel(
  repoId: string,
  variant?: string,
  hfToken?: string | null,
  cachePath?: string | null,
  onlyIfOrphan?: boolean,
): Promise<void> {
  const payload: Record<string, string | boolean> = { repo_id: repoId };
  if (variant) {
    payload.variant = variant;
  }
  if (cachePath) {
    payload.cache_path = cachePath;
  }
  // The server re-checks the orphan condition before unlinking and 409s if it changed.
  if (onlyIfOrphan) {
    payload.only_if_orphan = true;
  }
  const response = await authFetch("/api/hub/delete-cached", {
    method: "DELETE",
    headers: { "Content-Type": "application/json", ...hubTokenHeader(hfToken) },
    body: JSON.stringify(payload),
  });
  try {
    await throwIfNotOk(response);
    discardDeletedModelInventoryHints(repoId, variant);
    bumpInventoryVersion();
    notifyModelDeleted(repoId);
  } finally {
    invalidateGgufVariantsCache(repoId);
  }
}

export const DELETABLE_LOCAL_SOURCES: ReadonlySet<string> = new Set([
  "models_dir",
  "lmstudio",
  "custom",
  "ollama",
  "hermes",
  "omlx",
]);

/** `id` (the row's load id) picks between Ollama tags that share one blob path. */
export async function revealLocalModel(path: string, id?: string): Promise<void> {
  const response = await authFetch("/api/hub/local/reveal", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ path, id }),
  });
  await throwIfNotOk(response);
}

export async function deleteLocalModel(path: string, id?: string): Promise<void> {
  const response = await authFetch("/api/hub/local/delete", {
    method: "DELETE",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ path, id }),
  });
  try {
    await throwIfNotOk(response);
    bumpInventoryVersion();
  } finally {
    invalidateGgufVariantsCache(path);
  }
}

export async function listScanFolders(): Promise<ScanFolderInfo[]> {
  const response = await authFetch("/api/hub/scan-folders");
  const data = await parseJsonOrThrow<{ folders?: ScanFolderInfo[] }>(
    response,
  );
  // Every surface iterates this list, so a body without it must not take the page down.
  return Array.isArray(data?.folders) ? data.folders : [];
}

export async function addScanFolder(
  path: string,
  recursive?: boolean,
): Promise<ScanFolderInfo> {
  const response = await authFetch("/api/hub/scan-folders", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ path, recursive }),
  });
  const folder = await parseJsonOrThrow<ScanFolderInfo>(response);
  bumpInventoryVersion();
  return folder;
}

export async function removeScanFolder(id: number): Promise<void> {
  const response = await authFetch(`/api/hub/scan-folders/${id}`, {
    method: "DELETE",
  });
  await throwIfNotOk(response);
  bumpInventoryVersion();
}

const GGUF_VARIANTS_TTL_MS = 30 * 1000;
const GGUF_VARIANTS_LOCAL_TTL_MS = 10 * 60 * 1000;
const GGUF_VARIANTS_TIMEOUT_MS = 30_000;
const GGUF_VARIANTS_CACHE_MAX_ENTRIES = 64;

interface GgufVariantsCacheEntry {
  expiresAt: number;
  promise: Promise<GgufVariantsResponse>;
}

const ggufVariantsCache = new Map<string, GgufVariantsCacheEntry>();

function pruneGgufVariantsCache(now = Date.now()): void {
  for (const [key, entry] of ggufVariantsCache) {
    if (entry.expiresAt <= now) {
      ggufVariantsCache.delete(key);
    }
  }
  while (ggufVariantsCache.size > GGUF_VARIANTS_CACHE_MAX_ENTRIES) {
    const oldest = ggufVariantsCache.keys().next().value;
    if (!oldest) {
      break;
    }
    ggufVariantsCache.delete(oldest);
  }
}

function ggufVariantsTtlMs(
  response: GgufVariantsResponse,
  preferLocalCache: boolean,
): number {
  if (
    preferLocalCache &&
    response.variants.some((variant) => variant.downloaded)
  ) {
    return GGUF_VARIANTS_LOCAL_TTL_MS;
  }
  return GGUF_VARIANTS_TTL_MS;
}

export async function listGgufVariants(
  repoId: string,
  hfToken?: string,
  options?: {
    localOnly?: boolean;
    preferLocalCache?: boolean;
    includeCacheLocations?: boolean;
    localPath?: string | null;
    signal?: AbortSignal;
  },
): Promise<GgufVariantsResponse> {
  const offline = options?.localOnly === true || isHuggingFaceOffline();
  const localPath = options?.localPath?.trim() || null;
  const preferLocalCache = !!options?.preferLocalCache || offline;
  const signal = options?.signal;
  const key = `${repoId}::${fingerprintToken(hfToken)}::${
    offline ? "offline" : preferLocalCache ? "local" : "remote"
  }::${localPathCacheKey(localPath)}::${!!options?.includeCacheLocations}`;
  const now = Date.now();
  const hit = ggufVariantsCache.get(key);
  if (hit && now < hit.expiresAt) {
    ggufVariantsCache.delete(key);
    ggufVariantsCache.set(key, hit);
    return withAbort(hit.promise, signal);
  }
  if (hit) {
    ggufVariantsCache.delete(key);
  }
  const params = new URLSearchParams({ repo_id: repoId });
  if (options?.includeCacheLocations) {
    params.set("include_cache_locations", "true");
  }
  if (preferLocalCache) {
    params.set("prefer_local_cache", "true");
  }
  if (localPath) {
    params.set("local_path", localPath);
  }
  if (offline) {
    params.set("offline", "true");
  }
  const requestPromise = withHubTimeout(
    GGUF_VARIANTS_TIMEOUT_MS,
    async (signal) => {
      const response = await authFetch(`/api/hub/gguf-variants?${params}`, {
        headers: hubTokenHeader(hfToken),
        signal,
      });
      return parseJsonOrThrow<GgufVariantsResponse>(response);
    },
  );
  const promise = requestPromise.then((result) => {
    const current = ggufVariantsCache.get(key);
    if (current?.promise === promise) {
      current.expiresAt =
        Date.now() + ggufVariantsTtlMs(result, preferLocalCache);
    }
    return result;
  });
  ggufVariantsCache.set(key, {
    expiresAt: now + GGUF_VARIANTS_TTL_MS,
    promise,
  });
  pruneGgufVariantsCache(now);
  promise.catch(() => {
    if (ggufVariantsCache.get(key)?.promise === promise) {
      ggufVariantsCache.delete(key);
    }
  });
  return withAbort(promise, signal);
}

export function invalidateGgufVariantsCache(repoId?: string): void {
  if (!repoId) {
    ggufVariantsCache.clear();
    bumpGgufVariantsCacheVersion();
    return;
  }
  const repoPrefix = `${repoId}::`.toLowerCase();
  for (const key of ggufVariantsCache.keys()) {
    if (key.toLowerCase().startsWith(repoPrefix)) {
      ggufVariantsCache.delete(key);
    }
  }
  bumpGgufVariantsCacheVersion(repoId);
}