Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Maps an OpenAI model string to a downloaded GGUF or non-GGUF model; unknown names fall through to
the loaded model; never triggers a download."""
from __future__ import annotations
import threading
import time
from dataclasses import dataclass
from typing import Optional
from core.inference.model_ids import public_model_id
from loggers import get_logger
from utils.scan_memo import in_scan_memo
from utils.account_context import (
AccountContext,
account_thread,
current_account,
current_account_id,
is_owner_context,
run_as,
)
logger = get_logger(__name__)
@dataclass(frozen = True)
class _LocalGgufEntry:
loader_id: str
load_path: str
variants: tuple[str, ...] # local quant labels; () for a standalone .gguf
is_gguf: bool = True # False routes the load to the inference orchestrator
repo_level_companions: bool = False
aliases: tuple[tuple[str, str], ...] = () # see _legacy_variant_aliases
withheld: tuple[str, ...] = () # listed build keys that cannot be served (torn split sets)
build_aliases: tuple[tuple[str, str], ...] = () # see _build_family_aliases
exact_builds: bool = False # custom folder: a build-shaped suffix names one file, never a tag
_CACHE_TTL_S = 5.0
# Negative stamp = additions-only invalidated at -stamp, kept inside the atomically published tuple.
_lock = threading.Lock()
_scan: tuple[float, dict[str, _LocalGgufEntry]] = (0.0, {})
_EMPTY_SCAN: tuple[float, dict[str, _LocalGgufEntry]] = (0.0, {})
# Per account: scanned roots are account private.
_managed_scans: dict[str, tuple[float, dict[str, _LocalGgufEntry]]] = {}
def _snapshot() -> tuple[float, dict[str, _LocalGgufEntry]]:
"""The acting account's published index snapshot."""
if is_owner_context():
return _scan
return _managed_scans.get(current_account_id(), _EMPTY_SCAN)
def _publish(snapshot: tuple[float, dict[str, _LocalGgufEntry]]) -> None:
global _scan
if is_owner_context():
_scan = snapshot
else:
_managed_scans[current_account_id()] = snapshot
# Not _lock: that is held for the whole scan, so the request path would wait on it.
_warm_lock = threading.Lock()
# Downloaded repos not yet in the published index; the request path must not report them absent.
_just_downloaded: set[str] = set()
_warming = False
# Invalidation during a warm asks for another pass so stale snapshots rebuild off the request path.
_warm_pending = False
_warm_accounts: dict[Optional[str], AccountContext] = {}
_warm_active_account: Optional[AccountContext] = None
_warm_retry_scopes: set[Optional[str]] = set()
_last_scan_s = 0.0
_managed_last_scan_s: dict[str, float] = {}
# rescan at most a tenth of the time: on the TTL alone a slow scan would run continuously
_WARM_DUTY = 10.0
def _scope_for_account(account: AccountContext) -> Optional[str]:
return None if account.is_owner else account.account_id
def _account_scope() -> Optional[str]:
return _scope_for_account(current_account())
def _account_duty_window(account_id: Optional[str]) -> float:
scan_s = _last_scan_s if account_id is None else _managed_last_scan_s.get(account_id, 0.0)
return max(_CACHE_TTL_S, scan_s * _WARM_DUTY)
def _duty_window() -> float:
"""Minimum snapshot age for a background rescan."""
return _account_duty_window(_account_scope())
def _record_scan_duration(duration: float) -> None:
global _last_scan_s
account_id = _account_scope()
if account_id is None:
_last_scan_s = duration
else:
_managed_last_scan_s[account_id] = duration
def _is_abs_path_id(value: str) -> bool:
"""Both POSIX and Windows forms count on any host; ids outlive the machine that wrote them."""
from pathlib import PurePosixPath, PureWindowsPath
try:
return PurePosixPath(value).is_absolute() or PureWindowsPath(value).is_absolute()
except Exception:
return False
def _advertised_loader_id(info) -> Optional[str]:
"""The id to advertise for a scanned model: prefer a client-facing alias over
an absolute filesystem path so /v1/models and the override key never expose a
host path (the ./models and LM Studio scanners report the path as info.id)."""
from hub.services.models.ollama import is_ollama_manifest_ref
raw_id = getattr(info, "id", None)
if isinstance(raw_id, str) and is_ollama_manifest_ref(raw_id):
return getattr(info, "model_id", None)
if not raw_id or not _is_abs_path_id(raw_id):
return raw_id
for alt in (getattr(info, "model_id", None), getattr(info, "display_name", None)):
if alt and not _is_abs_path_id(alt):
return alt
# No clean alias: strip to a path-free public id so a host path is never advertised.
return public_model_id(raw_id) or raw_id
def _resolve_load_dir(p, loader_id: Optional[str] = None):
from pathlib import Path
load_dir = p
try:
if (p / "snapshots").is_dir():
from routes.models import _resolve_hf_cache_realpath
real = _resolve_hf_cache_realpath(p)
if real:
load_dir = Path(real)
except Exception:
pass
try:
from utils.audio_tokens import detect_local_tts_audio_type
from utils.models.model_config import load_model_defaults
llm_dir = load_dir / "LLM"
if llm_dir.is_dir() and (
(loader_id and (load_model_defaults(loader_id) or {}).get("audio_type") == "bicodec")
or detect_local_tts_audio_type(llm_dir) == "bicodec"
):
return llm_dir
except Exception:
pass
return load_dir
def _resolve_gguf_load_snapshot(p):
"""Newest *complete* snapshot: a newer revision may hold only a companion, not weights."""
snapshots = p / "snapshots"
try:
if not snapshots.is_dir():
return p
except OSError:
return None
# Scoped to this exact repo dir so case-colliding repos cannot cross-load.
from hub.utils.gguf import select_gguf_cache_snapshot_for_repo_dir
selected = select_gguf_cache_snapshot_for_repo_dir(p)
if selected is None:
return None
return selected
def local_gguf_companion_roots(load_path: str, *, repo_level: bool = False) -> tuple[str, ...]:
from pathlib import Path
from hub.utils.hf_cache_state import snapshot_selection_key
if not repo_level:
return ()
selected = Path(load_path)
snapshots = selected.parent
repo = snapshots.parent
if snapshots.name != "snapshots" or not repo.name.startswith("models--"):
return ()
try:
if not selected.is_dir():
return ()
except OSError:
return ()
# Siblings must resolve inside this repo dir: is_dir() follows symlinks (follow_symlinks is 3.13+).
try:
repo_resolved = repo.resolve()
# By identity: a symlinked sibling would make one snapshot look like two.
selected_resolved = selected.resolve()
except OSError as exc:
logger.debug("Stopping at unresolvable repo dir %s: %s", repo, exc)
return ()
siblings = []
try:
for path in snapshots.iterdir():
if path == selected:
continue
try:
if not path.is_dir():
continue
path_resolved = path.resolve()
if path_resolved == selected_resolved:
logger.debug("Skipping companion snapshot aliasing %s: %s", selected, path)
continue
if repo_resolved not in path_resolved.parents:
logger.debug("Skipping companion snapshot outside %s: %s", repo_resolved, path)
continue
siblings.append(path)
except OSError as exc:
logger.debug("Skipping unreadable companion snapshot %s: %s", path, exc)
except OSError as exc:
logger.debug("Stopping at unreadable companion snapshots dir %s: %s", snapshots, exc)
siblings.sort(key = snapshot_selection_key, reverse = True)
return (str(selected), *(str(path) for path in siblings))
def local_path_gguf_companion_roots(load_path: str) -> tuple[str, ...]:
"""Siblings count as companions only when this snapshot is the one a repo-level selection picks."""
from pathlib import Path
from hub.utils.gguf import select_gguf_cache_snapshot_for_repo_dir
from hub.utils.hf_cache_state import same_existing_path
try:
selected = Path(load_path)
except (TypeError, ValueError):
return ()
snapshots = selected.parent
repo = snapshots.parent
# Same predicate as local_gguf_companion_roots, or the two disagree on what a snapshot is.
if snapshots.name != "snapshots" or not repo.name.startswith("models--"):
return ()
try:
if not selected.is_dir():
return ()
except OSError:
return ()
try:
chosen = select_gguf_cache_snapshot_for_repo_dir(repo)
except OSError as exc:
logger.debug("Stopping at unreadable repo dir %s: %s", repo, exc)
return ()
# samefile, not string equality: spellings differ by symlink, or by case on Windows.
if chosen is None or not same_existing_path(chosen[3], selected):
return ()
roots = local_gguf_companion_roots(load_path, repo_level = True)
# A lone root is not inert: callers read `roots is not None` as allow_disjoint_search_root.
return roots if len(roots) > 1 else ()
def local_gguf_companion_state(roots: tuple[str, ...]) -> tuple:
from pathlib import Path
state = []
for root in roots:
try:
for path in sorted(Path(root).rglob("*")):
if path.suffix.lower() != ".gguf":
continue
try:
stat = path.stat()
state.append((str(path), stat.st_size, stat.st_mtime_ns))
except OSError:
state.append((str(path), -1, -1))
except OSError:
state.append((root, -1, -1))
return tuple(state)
def _legacy_variant_aliases(variants) -> tuple[tuple[str, str], ...]:
"""Maps legacy /v1/models labels to current; ambiguous ones dropped; never raises, since one raise would drop the whole repo."""
try:
from utils.models.model_config import _extract_quant_label, _qualified_variant_name
# .lower() matches how _resolve_from_index folds the request
current = {str(v.quant).lower() for v in variants if getattr(v, "quant", None)}
seen: dict[str, Optional[str]] = {}
for variant in variants:
quant = getattr(variant, "quant", None)
filename = getattr(variant, "filename", None)
if not quant or not filename:
continue
legacy = _qualified_variant_name(filename, _extract_quant_label(filename))
key = str(legacy).lower()
if not key or key in current:
continue
# None = ambiguous: a second file claimed it, so it names neither.
seen[key] = None if key in seen else str(quant)
return tuple((legacy, quant) for legacy, quant in seen.items() if quant is not None)
except Exception:
return ()
def _build_family_aliases(variants) -> tuple[tuple[str, str], ...]:
"""``(file family, current key)`` per custom-folder build: a stem key a build was advertised
under keeps naming it after a sibling's removal hands it the bare quant key."""
try:
from hub.utils.gguf import gguf_variant_family
current = {str(v.quant).lower() for v in variants if getattr(v, "quant", None)}
seen: dict[str, Optional[str]] = {}
for variant in variants:
quant = getattr(variant, "quant", None)
filename = getattr(variant, "filename", None)
if not quant or not filename:
continue
relative = str(filename).replace("\\", "/")
# the family (beta-Q4_K_M) and the filename key a collision falls back to (F16.gguf)
for spelling in {gguf_variant_family(relative).lower(), relative.lower()}:
if not spelling or spelling in current:
continue
seen[spelling] = None if spelling in seen else str(quant)
return tuple((family, quant) for family, quant in seen.items() if quant is not None)
except Exception:
return ()
def _local_gguf_entry(
loader_id: str,
info,
*,
exact_snapshot: bool = False,
) -> Optional[_LocalGgufEntry]:
"""Build an entry only when GGUF quants are on disk (not Transformers/
safetensors), listing only on-disk quants. ``load_path`` is a concrete local
path so /load resolves the variant locally and never fetches a remote one."""
from pathlib import Path
# The lister every stored per-model setting is keyed by, so one file has one identity.
from hub.utils.gguf import list_local_gguf_variants
from utils.models.model_config import detect_gguf_model
path = getattr(info, "path", None)
if not isinstance(path, str):
return None
p = Path(path)
try:
if p.is_file():
# Reject mmproj projectors: the standalone pass skips the filter the dir scan applies, so a switch could load one and evict the model.
if p.suffix.lower() != ".gguf" or detect_gguf_model(str(p)) is None:
return None
return _LocalGgufEntry(loader_id, str(p), ())
cache_repo_dir = None
if getattr(info, "source", None) == "hf_cache":
try:
if p.name.startswith("models--") and (p / "snapshots").is_dir():
cache_repo_dir = p
except OSError:
return None
if (
cache_repo_dir is None
and p.parent.name == "snapshots"
and p.parent.parent.name.startswith("models--")
):
cache_repo_dir = p.parent.parent
if cache_repo_dir is not None:
selected = _resolve_gguf_load_snapshot(cache_repo_dir)
if selected is None:
load_dir = p
variants, _ = list_local_gguf_variants(str(load_dir), require_existing_files = True)
else:
selected_variants, _has_vision, complete, load_dir = selected
variants, _ = list_local_gguf_variants(str(load_dir), require_existing_files = True)
# Filter even when nothing is complete: the selector can fall back to a snapshot
# with no whole quant, and its torn split shards must not be offered.
complete_keys = {str(quant).casefold() for quant in complete}
complete_files = {
str(variant.filename).casefold()
for variant in selected_variants
if getattr(variant, "filename", None)
and getattr(variant, "quant", None)
and str(variant.quant).casefold() in complete_keys
}
variants = [
variant
for variant in variants
if getattr(variant, "filename", None)
and str(variant.filename).casefold() in complete_files
]
else:
load_dir = _resolve_load_dir(p) if p.name.startswith("models--") else p
variants, _ = list_local_gguf_variants(str(load_dir), require_existing_files = True)
if exact_snapshot:
from hub.utils.gguf import list_local_gguf_variants as inventory_variants
from hub.utils.inventory_scan import complete_snapshot_variants
complete = complete_snapshot_variants(str(load_dir))
offered, _ = inventory_variants(str(load_dir))
complete_files = {str(v.filename).casefold() for v in offered if v.quant in complete}
variants = [v for v in variants if str(v.filename).casefold() in complete_files]
withheld: tuple[str, ...] = ()
if cache_repo_dir is None:
listed = variants
variants = [
variant
for variant in variants
if getattr(variant, "filename", None)
and _split_set_is_loadable(Path(load_dir) / variant.filename)
]
kept = {str(v.quant).lower() for v in variants if getattr(v, "quant", None)}
withheld = tuple(
str(v.quant).lower()
for v in listed
if getattr(v, "quant", None) and str(v.quant).lower() not in kept
)
quants = tuple(v.quant for v in variants if getattr(v, "quant", None))
if not quants:
return None
# Head is the largest quant; a bare id must mean what a plain load takes, since the largest can evict a model and then OOM.
from core.inference.openai_auto_download import preferred_quant
# preferred_quant is order-sensitive: rank only default_gguf_scope, or a sibling checkpoint can win.
from hub.utils.gguf import default_gguf_scope
scope = default_gguf_scope(
quants,
getattr(info, "id", None),
getattr(info, "model_id", None),
loader_id,
str(cache_repo_dir) if cache_repo_dir is not None else None,
)
best = preferred_quant(scope)
from hub.utils.gguf import gguf_variant_key
legacy_groups: dict[str, int] = {}
for variant in variants:
legacy = gguf_variant_key(variant.filename).casefold()
legacy_groups[legacy] = legacy_groups.get(legacy, 0) + 1
if any(count > 1 for count in legacy_groups.values()):
try:
detected = detect_gguf_model(str(load_dir))
relative = (
Path(detected).relative_to(Path(load_dir)).as_posix() if detected else None
)
preferred = next((v for v in variants if v.quant == best), None)
preferred_group = (
gguf_variant_key(preferred.filename).casefold() if preferred else None
)
# detection only breaks ties between builds of the preferred quant
selected = next(
(
variant
for variant in variants
if variant.filename == relative
and gguf_variant_key(variant.filename).casefold() == preferred_group
and legacy_groups.get(preferred_group, 0) > 1
),
None,
)
if selected is not None and selected.quant in scope:
best = selected.quant
except (OSError, RuntimeError, ValueError):
pass
if best and quants[0] != best:
quants = (best, *(q for q in quants if q != best))
return _LocalGgufEntry(
loader_id,
str(load_dir),
quants,
repo_level_companions = cache_repo_dir is not None,
aliases = _legacy_variant_aliases(variants),
withheld = withheld,
build_aliases = _build_family_aliases(variants) if cache_repo_dir is None else (),
exact_builds = cache_repo_dir is None,
)
except Exception:
return None
# A LoRA dir with a copied config would resolve and download its base model.
_ADAPTER_MARKERS = ("adapter_config.json", "adapter_model.safetensors", "adapter_model.bin")
_SUPPORTED_CONDITIONAL_AUDIO_MODEL_TYPES = frozenset({"csm", "whisper"})
_MODALITY_KEY_WORDS = frozenset({"vision", "image", "img", "audio", "video", "projector"})
# Read by VALUE (may be null). video_token_index is the transformers 4 spelling.
_VISUAL_TOKEN_ID_KEYS = (
"image_token_id",
"video_token_id",
"video_token_index",
"vision_start_token_id",
"vision_end_token_id",
)
def _read_json(path):
"""Parsed JSON for *path*, or None when it is absent or unreadable."""
import json
try:
with path.open(encoding = "utf-8") as handle:
return json.load(handle)
except (OSError, ValueError):
return None
def _host_serves_mlx() -> bool:
"""Reads the detected device without triggering detection; an unknown device reads as not MLX."""
try:
from utils.hardware import hardware as hw
return hw.DEVICE == hw.DeviceType.MLX
except Exception:
return False
def _host_has_a_non_gguf_backend() -> bool:
"""Installed torch is not enough: Transformers needs NVIDIA, AMD or Intel, else the load fails after the swap unloads the GGUF."""
if _host_serves_mlx():
return True
try:
from utils.hardware import hardware as hw
if hw.DEVICE not in (hw.DeviceType.CUDA, hw.DeviceType.XPU):
return False
from importlib.util import find_spec
return find_spec("torch") is not None
except Exception:
return False
def _has_safetensors_weights(load_dir) -> bool:
"""Safetensors only: a .bin checkpoint is pickle-backed, and an API request must not execute one."""
import re
if (load_dir / "model.safetensors").is_file():
return True
if (load_dir / "model.safetensors.index.json").is_file():
return True
try:
return any(
re.fullmatch(r"model-\d+-of-\d+\.safetensors", f.name) for f in load_dir.iterdir()
)
except OSError:
return False
def _is_generative_chat_config(load_dir, config: dict) -> bool:
"""Whether a config.json describes a checkpoint the chat loader can generate with."""
architectures = config.get("architectures")
# model_type cannot stand in for the list: transformers' causal mapping lists bert and bart
if not isinstance(architectures, list) or not architectures:
return False
names = [name for name in architectures if isinstance(name, str)]
# GPT2LMHeadModel and friends are causal without the ForCausalLM suffix.
if any(name.endswith(("ForCausalLM", "LMHeadModel")) for name in names):
return True
if not any(name.endswith("ForConditionalGeneration") for name in names):
return False
# Audio models declare no modality, so the classifier refuses them; chat uses transcription.
if _model_type_is_audio(config.get("model_type")):
return not _host_serves_mlx()
# T5 and BART wear the suffix too, and udop-large shows modality cannot separate them.
if config.get("is_encoder_decoder") is True:
return False
if not _config_declares_multimodality(config):
return False
# The model picker's own classifier; its None means inconclusive, which must not qualify here.
from hub.services.models.common import _local_transformers_can_chat
return _local_transformers_can_chat(load_dir) is True
def _config_declares_multimodality(config: dict) -> bool:
"""Whether a config declares a non-text modality, by whole key words: substrings admit ``revision``."""
import re
for key in config:
if not isinstance(key, str):
continue
# Neither counts alone: text_config nests in text-only configs; audio_token_id skips audio allowlist.
if key in ("text_config", "audio_token_id"):
continue
if key in _VISUAL_TOKEN_ID_KEYS:
if _is_placeholder_token_id(config[key]):
return True
continue
if _MODALITY_KEY_WORDS & set(re.split(r"[^a-z0-9]+", key.lower())):
return True
return False
def _is_placeholder_token_id(value) -> bool:
"""A token id a tokenizer could emit: a non-negative int, or a non-empty list of them. ``bool`` is excluded by
hand, or ``"image_token_id": true`` reads as token 1; transformers 5 writes the list form for multi-slot models.
"""
if isinstance(value, bool):
return False
if isinstance(value, int):
return value >= 0
if isinstance(value, (list, tuple)):
return bool(value) and all(_is_placeholder_token_id(item) for item in value)
return False
def _model_type_is_audio(model_type) -> bool:
"""Whether *model_type* has a supported conditional audio serving path."""
return isinstance(model_type, str) and model_type in _SUPPORTED_CONDITIONAL_AUDIO_MODEL_TYPES
def _weights_are_servable(load_dir) -> bool:
"""On-disk shape only: safetensors weights, and no diffusers, bare LoRA or embedder markers."""
from routes.models import _local_pipeline_index
if _local_pipeline_index(load_dir) or not (load_dir / "config.json").is_file():
return False
if not _has_safetensors_weights(load_dir):
return False
if any((load_dir / name).is_file() for name in _ADAPTER_MARKERS):
return False
# Same marker as is_embedding_model, minus its memo, which would pin one scan's verdict.
return not (load_dir / "modules.json").is_file()
def _config_is_servable_here(approval_ids, load_dir, config: dict) -> bool:
"""Only the approval boundary and category are decided here; remote code is read as data only."""
from utils.security.remote_code_approvals import approved_local_fingerprint
from utils.security.remote_code_scan import REMOTE_CODE_CONFIG_FILES
auto_map = False
for name in REMOTE_CODE_CONFIG_FILES:
candidate = config if name == "config.json" else _read_json(load_dir / name)
if not isinstance(candidate, dict):
continue
# model_file runs repo code and bypasses trust_remote_code (MLX loaders exec_module it).
if candidate.get("model_file"):
return False
auto_map = auto_map or bool(candidate.get("auto_map"))
# A switch can only pass on an approval the account already gave for exactly this code.
if auto_map and not approved_local_fingerprint(
current_account().username, approval_ids, str(load_dir)
):
return False
return _is_generative_chat_config(load_dir, config)
def _host_can_serve_minimax_music3() -> bool:
import sys
from importlib.util import find_spec
from importlib.metadata import version
from packaging.version import Version
try:
from core.inference.audio_device import audio_device_forces_cpu
from utils.hardware import hardware as hw
return (
sys.version_info >= (3, 10)
and hw.DEVICE == hw.DeviceType.CUDA
and not hw.IS_ROCM
and not audio_device_forces_cpu(None)
and find_spec("diffusers") is not None
and Version(version("diffusers")) >= Version("0.40.0")
)
except Exception:
return False
def _native_audio_pipeline_is_servable_here(load_dir) -> bool:
from core.inference.native_audio import (
_MINIMAX_DOWNLOAD_COMPONENTS,
minimax_music3_local_components_complete,
native_audio_type_from_local_path,
)
from utils.security.remote_code_scan import REMOTE_CODE_CONFIG_FILES
if (
native_audio_type_from_local_path(str(load_dir)) != "minimax_music3"
or not _host_can_serve_minimax_music3()
or not minimax_music3_local_components_complete(load_dir)
):
return False
for directory in (
load_dir,
*(load_dir / component for component in _MINIMAX_DOWNLOAD_COMPONENTS),
):
for name in REMOTE_CODE_CONFIG_FILES:
config = _read_json(directory / name)
if isinstance(config, dict) and config.get("auto_map"):
return False
return not any((load_dir / name).is_file() for name in (*_ADAPTER_MARKERS, "modules.json"))
def _local_weights_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]:
"""Needs config.json beside safetensors; adapters, partial downloads and pipelines are rejected."""
from pathlib import Path
path = getattr(info, "path", None)
if not isinstance(path, str) or getattr(info, "partial", False):
return None
# model-independent, so first: with no backend every row is withheld anyway.
if not _host_has_a_non_gguf_backend():
return None
try:
p = Path(path)
if not p.is_dir():
return None
load_dir = _resolve_load_dir(p, loader_id)
if _native_audio_pipeline_is_servable_here(load_dir):
return _LocalGgufEntry(loader_id, str(load_dir), (), is_gguf = False)
if not _weights_are_servable(load_dir):
return None
config = _read_json(load_dir / "config.json")
# Listing passes the raw id, the index the alias: both must see a path-keyed approval.
approval_ids = (loader_id, getattr(info, "id", None), path, str(load_dir))
if not isinstance(config, dict) or not _config_is_servable_here(
approval_ids, load_dir, config
):
return None
# No quants: quantization is baked in, so there is no ":<quant>" to pin.
return _LocalGgufEntry(loader_id, str(load_dir), (), is_gguf = False)
except Exception:
return None
def _local_servable_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]:
"""Entry for whichever backend can serve *info* from disk, GGUF first."""
from hub.services.models.ollama import is_ollama_manifest_ref, ollama_model_ref_files
raw_id = getattr(info, "id", None)
if isinstance(raw_id, str) and is_ollama_manifest_ref(raw_id):
if getattr(info, "source", None) != "ollama":
return None
# Raises when the tag's layers are gone or unsupported, withholding rather than advertising.
try:
ollama_model_ref_files(raw_id)
except (OSError, ValueError):
return None
# No quants: an Ollama tag names one file, so there is no ":<quant>" to pin.
return _LocalGgufEntry(loader_id, raw_id, ())
return _local_gguf_entry(loader_id, info) or _local_weights_entry(loader_id, info)
def local_servable_model(info) -> Optional[tuple[bool, tuple[str, ...]]]:
"""Reads the files, not info.model_format: the HF-cache scanner leaves it unset for GGUF snapshots."""
from pathlib import Path
path = getattr(info, "path", None)
# A link an earlier load materialized, rescanned: the manifest row already has those weights.
if isinstance(path, str) and any(
seg in (".studio_links", "ollama_links") for seg in Path(path).parts
):
return None
entry = _local_servable_entry(getattr(info, "id", "") or "", info)
if entry is None:
return None
if not _advertises_this_ollama_row(info):
return None
return (entry.is_gguf, entry.variants)
def _advertises_this_ollama_row(info) -> bool:
"""Whether an Ollama row's catalog id is the one the resolver loads for it: two roots can hold
one tag. Asks the index, so it cannot be called from inside a scan."""
from hub.services.models.ollama import is_ollama_manifest_ref
raw_id = getattr(info, "id", None)
if not isinstance(raw_id, str) or not is_ollama_manifest_ref(raw_id):
return True
model_id = getattr(info, "model_id", None)
if not model_id:
return False
resolved = resolve_local_gguf(model_id)
return bool(resolved and resolved[0] == raw_id)
def local_load_dir(path: Optional[str]) -> Optional[str]:
"""Resolves an HF cache repo to its snapshot, the same string the orchestrator records as active."""
from pathlib import Path
if not isinstance(path, str) or not path:
return None
try:
return str(_resolve_load_dir(Path(path)))
except Exception:
return path
@in_scan_memo
def _build_index() -> dict[str, _LocalGgufEntry]:
"""Scans the model picker's roots; Ollama manifests are read only, since load materializes the .gguf."""
# Lazy import: routes.models imports core.inference.
from pathlib import Path
from routes.models import (
_scan_models_dir,
_scan_hf_cache,
_scan_lmstudio_dir,
_scan_ollama_dir,
_resolve_hf_cache_dir,
_is_hidden_model,
)
from hub.utils.gguf import dedupe_custom_gguf_rows, suppress_grouped_gguf_file_rows
from hub.utils.inventory_scan import scan_folder_hf_caches
from utils.paths import legacy_hf_cache_dir, hf_default_cache_dir, lmstudio_model_dirs
from utils.hf_cache_settings import known_hf_hub_caches
from core.inference.model_ids import public_model_id
index: dict[str, _LocalGgufEntry] = {}
seen_hf: set[str] = set()
try:
active_root = str(Path(_resolve_hf_cache_dir()).resolve())
except Exception:
active_root = None
def _scan_hf_once(directory) -> list:
if directory is None:
return []
try:
d = Path(directory)
if not d.is_dir():
return []
rp = str(d.resolve())
if rp in seen_hf:
return []
seen_hf.add(rp)
# Only the active cache loads by repo id, so an inactive repo must not be indexed under one; no format classification on this request path.
return _scan_hf_cache(directory, active_cache = rp == active_root, classify_format = False)
except Exception as exc: # a missing/malformed root must skip, never crash the index
logger.debug("auto-switch: skipping HF cache dir %r: %s", directory, exc)
return []
# Guard each source separately so one bad root drops only that source, not the whole index.
found: list = []
try:
found += _scan_models_dir(Path("./models").resolve())
except Exception as exc:
logger.debug("auto-switch: ./models scan failed: %s", exc)
try:
for hf_dir in (
*known_hf_hub_caches(),
_resolve_hf_cache_dir(),
legacy_hf_cache_dir(),
hf_default_cache_dir(),
):
found += _scan_hf_once(hf_dir)
except Exception as exc:
logger.debug("auto-switch: HF cache scan failed: %s", exc)
try:
for lm_dir in lmstudio_model_dirs():
found += _scan_lmstudio_dir(lm_dir)
except Exception as exc:
logger.debug("auto-switch: LM Studio scan failed: %s", exc)
try:
from utils.paths import omlx_model_dirs
for omlx_dir in omlx_model_dirs():
found += _scan_hf_once(omlx_dir) + _scan_lmstudio_dir(omlx_dir, source = "omlx")
except Exception as exc:
logger.debug("auto-switch: oMLX scan failed: %s", exc)
# Read-only like the LM Studio scan, so it is safe on the request path. This is the path Hermes
# itself takes: it downloads the GGUF, then asks Unsloth for it by name.
try:
from hub.services.models.hermes import scan_hermes_dir
from utils.paths import hermes_model_dirs
for hermes_dir in hermes_model_dirs():
found += scan_hermes_dir(hermes_dir)
except Exception as exc:
logger.debug("auto-switch: Hermes scan failed: %s", exc)
try:
from utils.paths import ollama_model_dirs
for ollama_dir in ollama_model_dirs():
found += _scan_ollama_dir(ollama_dir, materialize_links = False)
except Exception as exc:
logger.debug("auto-switch: Ollama scan failed: %s", exc)
try:
from storage.studio_db import list_scan_folders
custom_found = []
for folder in list_scan_folders():
try:
fp = Path(folder["path"])
rows = (
_scan_models_dir(fp, limit = 200)
+ [row for hub in scan_folder_hf_caches(fp) for row in _scan_hf_once(hub)]
+ _scan_lmstudio_dir(fp)
+ _scan_ollama_dir(fp, limit = 200, materialize_links = False)
)
if folder.get("recursive"):
from routes.models import _scan_nested_compat_rows
rows += _scan_nested_compat_rows(fp, rows, limit = 200)
custom_found += dedupe_custom_gguf_rows(rows)
except Exception as exc:
logger.debug("auto-switch: scan folder %r failed: %s", folder, exc)
found += suppress_grouped_gguf_file_rows(custom_found)
except Exception as exc:
logger.debug("auto-switch: scan folders enumerate failed: %s", exc)
for info in found:
raw_id = getattr(info, "id", None)
if not raw_id:
continue
# Skip models hidden from pickers (validation probe, RAG embed); never an auto-switch target.
if _is_hidden_model(
raw_id,
getattr(info, "model_id", None),
getattr(info, "path", None),
):
continue
# Advertise a client-facing alias, not an absolute filesystem path.
loader_id = _advertised_loader_id(info)
entry = _local_servable_entry(loader_id, info)
if entry is None:
continue
# Path-shaped ids name one exact revision, so index them only if that revision has complete weights.
path_alias_entry = entry
if entry.repo_level_companions and _is_abs_path_id(raw_id):
from types import SimpleNamespace
path_alias_entry = _local_gguf_entry(
loader_id, SimpleNamespace(path = raw_id), exact_snapshot = True
)
snapshot_name = (
Path(raw_id).name if entry.repo_level_companions and _is_abs_path_id(raw_id) else None
)
for key, alias_entry in (
(raw_id, path_alias_entry),
(snapshot_name, path_alias_entry),
(getattr(info, "model_id", None), entry),
(getattr(info, "display_name", None), entry),
(public_model_id(raw_id), path_alias_entry),
):
if key and alias_entry is not None:
index.setdefault(key.strip().lower(), alias_entry)
# Sibling revisions keep their own weights, so a pin survives a newer Hugging Face snapshot.
if entry.is_gguf:
for name, sibling_entry in _sibling_revision_entries(raw_id, loader_id):
index.setdefault(name.strip().lower(), sibling_entry)
return index
def _sibling_revision_entries(raw_id: str, loader_id: str):
"""Each revision gets its own entry, not an alias, so an old pin never lands on a newer partial."""
from pathlib import Path
from types import SimpleNamespace
snapshots = Path(raw_id).parent
if snapshots.name != "snapshots" or not snapshots.parent.name.startswith("models--"):
return
from routes.models import snapshot_variants_all_complete
try:
siblings = [p for p in snapshots.iterdir() if p.is_dir() and p.name != Path(raw_id).name]
except OSError:
return
for sibling in siblings:
if not snapshot_variants_all_complete(str(sibling)):
continue
entry = _local_gguf_entry(loader_id, SimpleNamespace(path = str(sibling)))
if entry is not None:
yield sibling.name, entry
def note_downloaded(repo_id: Optional[str]) -> None:
"""Record a repo as present ahead of the scan that will index it."""
if not repo_id:
return
with _lock:
_just_downloaded.add(repo_id.strip().lower())
def recently_downloaded(repo_id: str) -> bool:
"""Whether *repo_id* finished downloading since the last completed scan."""
if not isinstance(repo_id, str) or not repo_id.strip():
return False
return repo_id.strip().lower() in _just_downloaded
def local_gguf_torn_only(repo_id: str) -> bool:
"""Whether the HF cache holds GGUF builds of *repo_id* that the index left out because none is whole."""
if not isinstance(repo_id, str) or repo_id.strip().count("/") != 1:
return False
try:
from hub.utils.gguf import list_local_gguf_variants
from hub.utils.hf_cache_state import iter_repo_cache_dirs
for repo_dir in iter_repo_cache_dirs("model", repo_id.strip()):
snapshots = repo_dir / "snapshots"
if not snapshots.is_dir():
continue
for snapshot in snapshots.iterdir():
variants, _ = list_local_gguf_variants(str(snapshot))
if any(getattr(v, "quant", None) for v in variants):
return True
except Exception:
return False
return False
def _snapshot_is_trusted(
timestamp: float,
now: float,
*,
duty_window: Optional[float] = None,
) -> bool:
"""Negative stamps are trusted only while a rebuild may still run, so a failing rebuild cannot
linger."""
if timestamp > 0.0:
return now - timestamp < _CACHE_TTL_S
if timestamp < 0.0:
return now + timestamp < (duty_window if duty_window is not None else _duty_window())
return False
# Bumped by every invalidate_index(); dependent caches can key on it.
_generation = 0
# Confirmed misses keyed by account and name, valid for the recorded invalidation generation.
_misses: dict[tuple[Optional[str], str], int] = {}
_MAX_MISSES = 256
# Request bodies may be hundreds of MB; longer names are not retained.
_MAX_MISS_NAME = 4096
def index_generation() -> int:
"""How many times the local scan has been invalidated since this process started."""
return _generation
def invalidate_index(*, additions_only: bool = False) -> None:
"""Additions-only keeps known hits served during rebuild; any other invalidation revokes trust."""
global _scan, _warm_pending, _generation
if not additions_only:
from hub.utils import partial_verdict_cache
partial_verdict_cache.invalidate()
with _lock:
now = time.monotonic()
_generation += 1
def _invalidated(snapshot, account_id):
timestamp, retained = snapshot
# Entries and trust state publish together for lock-free readers.
stamp = (
-now
if additions_only
and _snapshot_is_trusted(
timestamp,
now,
duty_window = _account_duty_window(account_id),
)
else 0.0
)
return (stamp, retained)
# Every account's snapshot: what changed on disk is not scoped to whoever noticed.
_scan = _invalidated(_scan, None)
for account_id, snapshot in list(_managed_scans.items()):
_managed_scans[account_id] = _invalidated(snapshot, account_id)
# May have waited out a scan; the warmer's snapshot can be stale again. See _warm_pending.
with _warm_lock:
if _warm_active_account is not None:
_warm_retry_scopes.add(_scope_for_account(_warm_active_account))
_warm_pending = True
def _index() -> dict[str, _LocalGgufEntry]:
# Build under the lock so concurrent expired callers don't all scan.
with _lock:
now = time.monotonic()
ts, cached = _snapshot()
# `ts > 0`: within uptime < TTL an invalidated stamp would read as recent
if ts > 0.0 and now - ts < _CACHE_TTL_S:
return cached
# Request scans also set the background rescan interval.
try:
fresh = _build_index()
finally:
_record_scan_duration(time.monotonic() - now)
# Stamp AFTER the scan: a slow scan could exceed the TTL and store an expired cache.
_publish((time.monotonic(), fresh))
# The scan supersedes the notes: whatever landed is in the index now.
_just_downloaded.clear()
return fresh
def index_is_built() -> bool:
"""Lock-free because _lock is held for the whole scan; _scan is only ever rebound, never mutated."""
return _snapshot()[0] > 0.0
def resolve_trusted_cached_local_gguf(
requested: str, *, include_companion_scope: bool = False
) -> Optional[tuple]:
"""Positive hits from TTL expiry or a scan-root change must be rebuilt before they can switch models."""
snapshot = _snapshot()
resolved = _resolve_from_index(
requested,
snapshot[1],
include_companion_scope = include_companion_scope,
)
if resolved is None or _snapshot() is not snapshot:
return None
trusted = _snapshot_is_trusted(snapshot[0], time.monotonic())
return resolved if trusted and _snapshot() is snapshot else None
def warm_index_soon() -> None:
"""Rebuilds a stale index too, since new downloads have no invalidation hook; never takes _lock."""
global _warming, _warm_pending
stamp = _snapshot()[0]
if stamp > 0.0 and time.monotonic() - stamp < _duty_window():
return
account = current_account()
scope = _scope_for_account(account)
with _warm_lock:
if (
_warm_active_account is not None and scope == _scope_for_account(_warm_active_account)
) or scope in _warm_accounts:
return
_warm_accounts[scope] = account
if _warming:
return
_warming = True
_warm_pending = False
def _run() -> None:
global _warm_active_account, _warming, _warm_pending
released = False
try:
while True:
with _warm_lock:
if not _warm_accounts:
_warming, _warm_pending, released = False, False, True
return
scope, account = next(iter(_warm_accounts.items()))
del _warm_accounts[scope]
_warm_active_account = account
try:
run_as(account, _index)
except Exception:
pass
with _warm_lock:
_warm_active_account = None
if scope in _warm_retry_scopes:
_warm_retry_scopes.discard(scope)
_warm_accounts[scope] = account
_warm_pending = bool(_warm_retry_scopes)
finally:
# Only on BaseException: queued scopes without a worker would kill background warming.
if not released:
with _warm_lock:
_warm_accounts.clear()
_warm_retry_scopes.clear()
_warm_active_account = None
_warming = _warm_pending = False
# Capture a real account context for the thread; queued scans rebind to their own account above.
account_thread(target = _run, name = "local-model-index-warm", daemon = True).start()
def _miss_scope() -> Optional[str]:
return _account_scope()
def _scope_is_warming(scope: Optional[str]) -> bool:
with _warm_lock:
return scope in _warm_accounts or (
_warm_active_account is not None and _scope_for_account(_warm_active_account) == scope
)
def resolve_local_gguf_for_switch(
requested: str, *, include_companion_scope: bool = False
) -> Optional[tuple]:
"""Reuses known misses until invalidation or two duty windows; one window triggers a background scan."""
if not isinstance(requested, str) or not requested.strip():
return None
requested = requested.strip()
key = (_miss_scope(), requested)
generation = _generation
started = time.monotonic()
ts, index = _snapshot()
if (
_misses.get(key) == generation
and ts > 0.0
and (started - ts < 2 * _duty_window() or _scope_is_warming(key[0]))
and _resolve_from_index(requested, index) is None
):
warm_index_soon()
return None
resolved = resolve_local_gguf(requested, include_companion_scope = include_companion_scope)
if resolved is None:
ts, index = _snapshot()
# A failed rebuild leaves an older snapshot that cannot prove a miss.
if (
len(requested) <= _MAX_MISS_NAME
and ts > 0.0
and started - ts < _CACHE_TTL_S
and generation == _generation
and _resolve_from_index(requested, index) is None
):
if len(_misses) >= _MAX_MISSES:
_misses.clear()
_misses[key] = generation
return resolved
def resolve_local_gguf(
requested: str,
*,
allow_scan: bool = True,
include_companion_scope: bool = False,
) -> Optional[tuple]:
"""Exact id wins, then the last :VARIANT is split off; allow_scan=False never rebuilds; a suffix naming no quant means the repo."""
if not isinstance(requested, str) or not requested.strip():
return None
requested = requested.strip()
try:
index = _index() if allow_scan else _snapshot()[1]
return _resolve_from_index(
requested,
index,
include_companion_scope = include_companion_scope,
)
except Exception:
# Best-effort: a resolver failure falls through to the loaded model, never a 500.
return None
def _resolve_from_index(
requested: str,
index: dict[str, _LocalGgufEntry],
*,
include_companion_scope: bool = False,
) -> Optional[tuple]:
"""Resolve *requested* against one immutable published index mapping."""
try:
def _result(entry: _LocalGgufEntry, variant: Optional[str]) -> tuple:
result = (entry.load_path, variant, entry.loader_id)
return (*result, entry.repo_level_companions) if include_companion_scope else result
entry = index.get(requested.lower())
if entry is not None:
variant = entry.variants[0] if entry.variants else None
return _result(entry, variant)
base, sep, variant = requested.rpartition(":")
if not sep:
return None
entry = index.get(base.strip().lower())
if entry is None:
return None
wanted = variant.strip().lower()
for v in entry.variants:
if v.lower() == wanted:
return _result(entry, v)
# a listed build that cannot be served misses, never answered by a sibling or alias
if wanted in entry.withheld:
return None
# ahead of both branches below: one refuses a retired spelling, the other answers it
# with a different quant
for legacy, current in (*entry.aliases, *entry.build_aliases):
if legacy == wanted:
return _result(entry, current)
from core.inference.openai_auto_download import looks_like_quant
if looks_like_quant(variant) or _names_missing_build(entry, wanted):
return None
# A tag naming no file means the repo; a real missing quant still misses.
return _result(entry, entry.variants[0] if entry.variants else None)
except Exception:
return None
def _split_set_is_loadable(path) -> bool:
"""Whether the loader finds a complete shard set here or, like it, behind a shard-1 symlink."""
from utils.models.model_config import colocated_split_shards
if colocated_split_shards(path)[1]:
return True
try:
return path.is_symlink() and colocated_split_shards(path.resolve())[1]
except OSError:
return False
def local_gguf_pinned_variant(requested: str, allow_scan: bool = True) -> Optional[str]:
"""The on-disk quant ``repo:VARIANT`` pins, by current label or legacy alias, else None.
Unlike :func:`resolve_local_gguf`, a suffix naming no quant (``:latest``) never falls back to the
repo's preferred one. ``allow_scan=False`` answers from the last built index, never rebuilding it.
"""
base, sep, wanted = requested.rpartition(":")
if not sep:
return None
try:
entry = (_index() if allow_scan else _snapshot()[1]).get(base.strip().lower())
except Exception:
return None
if entry is None:
return None
wanted = wanted.strip().lower()
for variant in entry.variants:
if variant.lower() == wanted:
return variant
if wanted in entry.withheld:
return None
for legacy, current in (*entry.aliases, *entry.build_aliases):
if legacy == wanted:
return current
return None
def _names_missing_build(entry: _LocalGgufEntry, wanted: str) -> bool:
"""Whether *wanted* is a custom-folder build stem (``beta-Q4_K_M``) no indexed build answers."""
from core.inference.openai_auto_download import looks_like_quant
from hub.utils.gguf import extract_quant_token
return (
entry.exact_builds
and ("/" in wanted or not looks_like_quant(wanted))
and extract_quant_token(wanted) is not None
and all(v.lower() != wanted for v in entry.variants)
and all(legacy != wanted for legacy, _ in (*entry.aliases, *entry.build_aliases))
)
def local_gguf_same_build(base: Optional[str], resident: str, wanted: str) -> bool:
"""Whether *resident* names *wanted*, verbatim or through *base*'s index (a build loaded by
its stem, ``alpha-Q4_K_M``, is the build the bare key ``Q4_K_M`` now names)."""
resident = (resident or "").strip().lower()
wanted = (wanted or "").strip().lower()
if resident == wanted:
return True
if not base or not resident:
return False
canonical = local_gguf_pinned_variant(f"{base}:{resident}", allow_scan = False)
return canonical is not None and canonical.lower() == wanted
def local_gguf_withheld(requested: str) -> bool:
"""Whether ``repo:BUILD`` names a build the last index cannot serve: torn, or no longer there."""
base, sep, wanted = requested.rpartition(":")
if not sep:
return False
try:
entry = _snapshot()[1].get(base.strip().lower())
except Exception:
return False
if entry is None:
return False
wanted = wanted.strip().lower()
return wanted in entry.withheld or _names_missing_build(entry, wanted)
def local_target_is_gguf(load_path: Optional[str], loader_id: Optional[str] = None) -> bool:
"""Reads the weights the load opens so a rebuild cannot flip it; falls back to the indexed entry, then True; call off the event loop."""
from pathlib import Path
from types import SimpleNamespace
if isinstance(load_path, str) and load_path:
try:
if Path(load_path).exists():
return _local_gguf_entry("", SimpleNamespace(path = load_path)) is not None
except OSError:
pass
if not isinstance(loader_id, str) or not loader_id.strip():
return True
entry = _snapshot()[1].get(loader_id.strip().lower())
return entry.is_gguf if entry is not None else True
MISS_MODEL_NOT_FOUND = "model_not_found"
MISS_VARIANT_NOT_FOUND = "variant_not_found"
def describe_local_miss(requested: str) -> tuple[str, tuple[str, ...]]:
"""Says wrong quant when the repo is downloaded; a scan failure falls back to the generic miss."""
if not isinstance(requested, str) or not requested.strip():
return MISS_MODEL_NOT_FOUND, ()
base, sep, variant = requested.strip().rpartition(":")
from core.inference.openai_auto_download import looks_like_quant
# Split like the resolver: a tag naming no quant means the repo, so no missing quant is reported.
if not sep or not looks_like_quant(variant):
return MISS_MODEL_NOT_FOUND, ()
try:
entry = _index().get(base.strip().lower())
except Exception:
return MISS_MODEL_NOT_FOUND, ()
if entry is None or not entry.variants:
return MISS_MODEL_NOT_FOUND, ()
return MISS_VARIANT_NOT_FOUND, entry.variants