Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Model and LoRA configuration handling."""
from dataclasses import dataclass, replace as _dataclass_replace
from typing import Optional, Dict, Any
from utils import gguf_names as _gguf_names
from utils.paths.storage_roots import own_entry, within_account
from utils.paths import (
normalize_path,
is_local_path,
is_model_cached,
get_cache_path,
resolve_cached_repo_id_case,
outputs_root,
exports_root,
resolve_output_dir,
resolve_export_dir,
)
from contextlib import nullcontext
from hub.utils.hf_tokens import (
recording_a_request_token_fetch,
ANONYMOUS_CACHE_IDENTITY,
cached_read_refused,
qualify_cache_identity,
HfTokenArg,
apply_token_to_child_env,
cache_reads_authorized,
call_with_anonymous_retry,
collecting_hub_token_rejections,
note_saved_token_rejected,
saved_token_rejected,
HUB_TOKEN_REJECTED_ERROR,
hf_token_arg,
is_rejected_credential_error,
is_anonymous,
normalize_token,
)
from utils.utils import without_hf_auth
from utils.training_runs import (
base_model_from_run_dir_name as _base_model_from_dir_name,
)
from utils.audio_tokens import (
AUDIO_TOKENIZER_CONFIG_PATHS as _AUDIO_TOKENIZER_CONFIG_PATHS,
classify_audio_tokens as _check_token_patterns,
declares_audio_token as _declares_audio_token,
is_audio_input_type,
may_hold_audio_tokens as _may_hold_audio_tokens,
)
from utils.models.gguf_metadata import (
is_mmproj_by_metadata,
mmproj_accepts_image,
mmproj_functional_match,
pairing_score,
read_gguf_general_metadata,
read_gguf_nextn_predict_layers,
)
import structlog
from loggers import get_logger
import contextlib as _contextlib
from contextvars import ContextVar, copy_context
import functools
import os
import platform
import re
import subprocess
import sys
from pathlib import Path
from typing import Callable, List, Tuple
import hashlib
import json
import threading
import time
import yaml
from utils.native_path_leases import child_env_without_native_path_secret
from utils.child_stdio import utf8_child_env
from utils.hf_cache_settings import active_hf_hub_cache, get_hf_cache_paths
from utils.subprocess_compat import (
windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs,
)
logger = get_logger(__name__)
_OFFLINE_TRUE_VALUES = {"1", "true", "yes", "on"}
def _env_offline() -> bool:
"""Open force_hf_offline window wins over the env; raw requests.get ignores the hub constant."""
try:
from utils.utils import hf_env_offline
return hf_env_offline()
except Exception:
pass
return (
os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in _OFFLINE_TRUE_VALUES
or os.environ.get("TRANSFORMERS_OFFLINE", "").strip().lower() in _OFFLINE_TRUE_VALUES
)
@_contextlib.contextmanager
def _offline_while_reading(target: Optional[str]):
"""Forces offline to read a remote base of a local LoRA or GGUF, so retry backoff never resumes."""
try:
from core.inference.llama_cpp import _hf_offline_if_unreachable_for
except Exception:
yield # guard unavailable (worker context): behave as before
return
with _hf_offline_if_unreachable_for(target):
yield
# ── Model size extraction ────────────────────────────────────
import re as _re
_MODEL_SIZE_RE = _re.compile(r"(?:^|[-_/])(\d+\.?\d*)\s*([bm])(?:$|[-_/])", _re.IGNORECASE)
# MoE active-parameter pattern: "A3B", "A3.5B", etc.
_ACTIVE_SIZE_RE = _re.compile(r"(?:^|[-_/])a(\d+\.?\d*)\s*([bm])(?:$|[-_/])", _re.IGNORECASE)
# Gemma 3n/4 E2B/E4B: effective (runtime) size, which size-gated policies care about.
_EFFECTIVE_SIZE_RE = _re.compile(r"(?:^|[-_/])e(\d+\.?\d*)\s*([bm])(?:$|[-_/])", _re.IGNORECASE)
def extract_model_size_b(model_id: str) -> float | None:
"""Defaults trust_remote_code to False so probes never run a repo auto_map Python; local_files_only reads the local cache; revision pins remote reads."""
mid = (model_id or "").lower()
# First match wins, in priority order: active > effective > total.
for pattern in (_ACTIVE_SIZE_RE, _EFFECTIVE_SIZE_RE, _MODEL_SIZE_RE):
m = pattern.search(mid)
if m:
val = float(m.group(1))
return val / 1000.0 if m.group(2).lower() == "m" else val
return None
# Maps equivalent model names to canonical YAML; the file name derives from the first listed name.
MODEL_NAME_MAPPING = {
# ── Embedding models ──
"unsloth_all-MiniLM-L6-v2.yaml": [
"unsloth/all-MiniLM-L6-v2",
"sentence-transformers/all-MiniLM-L6-v2",
],
"unsloth_bge-m3.yaml": [
"unsloth/bge-m3",
"BAAI/bge-m3",
],
"unsloth_embeddinggemma-300m.yaml": [
"unsloth/embeddinggemma-300m",
"google/embeddinggemma-300m",
],
"unsloth_gte-modernbert-base.yaml": [
"unsloth/gte-modernbert-base",
"Alibaba-NLP/gte-modernbert-base",
],
"unsloth_Qwen3-Embedding-0.6B.yaml": [
"unsloth/Qwen3-Embedding-0.6B",
"Qwen/Qwen3-Embedding-0.6B",
"unsloth/Qwen3-Embedding-4B",
"Qwen/Qwen3-Embedding-4B",
],
# ── Other models ──
"unsloth_answerdotai_ModernBERT-large.yaml": [
"answerdotai/ModernBERT-large",
],
"unsloth_Qwen2.5-Coder-7B-Instruct-bnb-4bit.yaml": [
"unsloth/Qwen2.5-Coder-7B-Instruct-bnb-4bit",
"unsloth/Qwen2.5-Coder-7B-Instruct",
"Qwen/Qwen2.5-Coder-7B-Instruct",
],
"unsloth_codegemma-7b-bnb-4bit.yaml": [
"unsloth/codegemma-7b-bnb-4bit",
"unsloth/codegemma-7b",
"google/codegemma-7b",
],
"unsloth_ERNIE-4.5-21B-A3B-PT.yaml": [
"unsloth/ERNIE-4.5-21B-A3B-PT",
],
"unsloth_ERNIE-4.5-VL-28B-A3B-PT.yaml": [
"unsloth/ERNIE-4.5-VL-28B-A3B-PT",
],
"tiiuae_Falcon-H1-0.5B-Instruct.yaml": [
"tiiuae/Falcon-H1-0.5B-Instruct",
"unsloth/Falcon-H1-0.5B-Instruct",
],
"unsloth_functiongemma-270m-it.yaml": [
"unsloth/functiongemma-270m-it-unsloth-bnb-4bit",
"google/functiongemma-270m-it",
"unsloth/functiongemma-270m-it-unsloth-bnb-4bit",
],
"unsloth_gemma-2-2b.yaml": [
"unsloth/gemma-2-2b-bnb-4bit",
"google/gemma-2-2b",
],
"unsloth_gemma-2-27b-bnb-4bit.yaml": [
"unsloth/gemma-2-9b-bnb-4bit",
"unsloth/gemma-2-9b",
"google/gemma-2-9b",
"unsloth/gemma-2-27b",
"google/gemma-2-27b",
],
"unsloth_gemma-3-4b-pt.yaml": [
"unsloth/gemma-3-4b-pt-unsloth-bnb-4bit",
"google/gemma-3-4b-pt",
"unsloth/gemma-3-4b-pt-bnb-4bit",
],
"unsloth_gemma-3-4b-it.yaml": [
"unsloth/gemma-3-4b-it-unsloth-bnb-4bit",
"google/gemma-3-4b-it",
"unsloth/gemma-3-4b-it-bnb-4bit",
],
"unsloth_gemma-3-27b-it.yaml": [
"unsloth/gemma-3-27b-it-unsloth-bnb-4bit",
"google/gemma-3-27b-it",
"unsloth/gemma-3-27b-it-bnb-4bit",
],
"unsloth_gemma-3-270m-it.yaml": [
"unsloth/gemma-3-270m-it-unsloth-bnb-4bit",
"google/gemma-3-270m-it",
"unsloth/gemma-3-270m-it-bnb-4bit",
],
"unsloth_gemma-3n-E4B-it.yaml": [
"unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit",
"google/gemma-3n-E4B-it",
"unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit",
],
"unsloth_gemma-3n-E4B.yaml": [
"unsloth/gemma-3n-E4B-unsloth-bnb-4bit",
"google/gemma-3n-E4B",
],
"unsloth_gemma-4-31B-it.yaml": [
"unsloth/gemma-4-31B-it",
"google/gemma-4-31B-it",
"unsloth/gemma-4-31B-it-GGUF",
],
"unsloth_gemma-4-26B-A4B-it.yaml": [
"unsloth/gemma-4-26B-A4B-it",
"google/gemma-4-26B-A4B-it",
"unsloth/gemma-4-26B-A4B-it-GGUF",
],
"unsloth_gemma-4-E2B-it.yaml": [
"unsloth/gemma-4-E2B-it",
"google/gemma-4-E2B-it",
"unsloth/gemma-4-E2B-it-GGUF",
],
"unsloth_gemma-4-E4B-it.yaml": [
"unsloth/gemma-4-E4B-it",
"google/gemma-4-E4B-it",
"unsloth/gemma-4-E4B-it-GGUF",
],
"unsloth_gemma-4-31B.yaml": [
"unsloth/gemma-4-31B",
"google/gemma-4-31B",
],
"unsloth_gemma-4-26B-A4B.yaml": [
"unsloth/gemma-4-26B-A4B",
"google/gemma-4-26B-A4B",
],
"unsloth_gemma-4-E2B.yaml": [
"unsloth/gemma-4-E2B",
"google/gemma-4-E2B",
],
"unsloth_gemma-4-E4B.yaml": [
"unsloth/gemma-4-E4B",
"google/gemma-4-E4B",
],
"unsloth_gpt-oss-20b.yaml": [
"openai/gpt-oss-20b",
"unsloth/gpt-oss-20b-unsloth-bnb-4bit",
"unsloth/gpt-oss-20b-BF16",
],
"unsloth_gpt-oss-120b.yaml": [
"openai/gpt-oss-120b",
"unsloth/gpt-oss-120b-unsloth-bnb-4bit",
],
"unsloth_granite-4.0-350m-unsloth-bnb-4bit.yaml": [
"unsloth/granite-4.0-350m",
"ibm-granite/granite-4.0-350m",
"unsloth/granite-4.0-350m-bnb-4bit",
],
"unsloth_granite-4.0-h-micro.yaml": [
"ibm-granite/granite-4.0-h-micro",
"unsloth/granite-4.0-h-micro-bnb-4bit",
"unsloth/granite-4.0-h-micro-unsloth-bnb-4bit",
],
"unsloth_LFM2-1.2B.yaml": [
"unsloth/LFM2-1.2B",
"LiquidAI/LFM2-1.2B",
"unsloth/LFM2-1.2B-unsloth-bnb-4bit",
],
"LiquidAI_LFM3-Omni-2B.yaml": [
"LiquidAI/LFM3-Omni-2B",
"LiquidAI/LFM3-1.7B",
"LiquidAI/LFM3-1.7B-Base",
"LiquidAI/LFM3-VL-2B",
"LiquidAI/LFM3-Audio-2B",
],
"unsloth_llama-3-8b-bnb-4bit.yaml": [
"unsloth/llama-3-8b",
"meta-llama/Meta-Llama-3-8B",
],
"unsloth_llama-3-8b-Instruct-bnb-4bit.yaml": [
"unsloth/llama-3-8b-Instruct",
"meta-llama/Meta-Llama-3-8B-Instruct",
],
"unsloth_Meta-Llama-3.1-70B-bnb-4bit.yaml": [
"unsloth/Meta-Llama-3.1-8B-bnb-4bit",
"unsloth/Meta-Llama-3.1-8B-unsloth-bnb-4bit",
"meta-llama/Meta-Llama-3.1-8B",
"unsloth/Meta-Llama-3.1-70B-bnb-4bit",
"unsloth/Meta-Llama-3.1-8B",
"unsloth/Meta-Llama-3.1-70B",
"meta-llama/Meta-Llama-3.1-70B",
"unsloth/Meta-Llama-3.1-405B-bnb-4bit",
"meta-llama/Meta-Llama-3.1-405B",
],
"unsloth_Meta-Llama-3.1-8B-Instruct-bnb-4bit.yaml": [
"unsloth/Meta-Llama-3.1-8B-Instruct-unsloth-bnb-4bit",
"unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit",
"meta-llama/Meta-Llama-3.1-8B-Instruct",
"unsloth/Meta-Llama-3.1-8B-Instruct",
"RedHatAI/Llama-3.1-8B-Instruct-FP8",
"unsloth/Llama-3.1-8B-Instruct-FP8-Block",
"unsloth/Llama-3.1-8B-Instruct-FP8-Dynamic",
],
"unsloth_Llama-3.2-3B-Instruct.yaml": [
"unsloth/Llama-3.2-3B-Instruct-unsloth-bnb-4bit",
"meta-llama/Llama-3.2-3B-Instruct",
"unsloth/Llama-3.2-3B-Instruct-bnb-4bit",
"RedHatAI/Llama-3.2-3B-Instruct-FP8",
"unsloth/Llama-3.2-3B-Instruct-FP8-Block",
"unsloth/Llama-3.2-3B-Instruct-FP8-Dynamic",
],
"unsloth_Llama-3.2-1B-Instruct.yaml": [
"unsloth/Llama-3.2-1B-Instruct-unsloth-bnb-4bit",
"meta-llama/Llama-3.2-1B-Instruct",
"unsloth/Llama-3.2-1B-Instruct-bnb-4bit",
"RedHatAI/Llama-3.2-1B-Instruct-FP8",
"unsloth/Llama-3.2-1B-Instruct-FP8-Block",
"unsloth/Llama-3.2-1B-Instruct-FP8-Dynamic",
],
"unsloth_Llama-3.2-11B-Vision-Instruct.yaml": [
"unsloth/Llama-3.2-11B-Vision-Instruct-unsloth-bnb-4bit",
"meta-llama/Llama-3.2-11B-Vision-Instruct",
"unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit",
],
"unsloth_Llama-3.3-70B-Instruct.yaml": [
"unsloth/Llama-3.3-70B-Instruct-unsloth-bnb-4bit",
"meta-llama/Llama-3.3-70B-Instruct",
"unsloth/Llama-3.3-70B-Instruct-bnb-4bit",
"RedHatAI/Llama-3.3-70B-Instruct-FP8",
"unsloth/Llama-3.3-70B-Instruct-FP8-Block",
"unsloth/Llama-3.3-70B-Instruct-FP8-Dynamic",
],
"unsloth_Llasa-3B.yaml": [
"HKUSTAudio/Llasa-1B",
"unsloth/Llasa-3B",
],
"unsloth_Magistral-Small-2509-unsloth-bnb-4bit.yaml": [
"unsloth/Magistral-Small-2509",
"mistralai/Magistral-Small-2509",
"unsloth/Magistral-Small-2509-bnb-4bit",
],
"unsloth_Ministral-3-3B-Instruct-2512.yaml": [
"unsloth/Ministral-3-3B-Instruct-2512",
],
"unsloth_mistral-7b-v0.3-bnb-4bit.yaml": [
"unsloth/mistral-7b-v0.3-bnb-4bit",
"unsloth/mistral-7b-v0.3",
"mistralai/Mistral-7B-v0.3",
],
"unsloth_Mistral-Nemo-Base-2407-bnb-4bit.yaml": [
"unsloth/Mistral-Nemo-Base-2407-bnb-4bit",
"unsloth/Mistral-Nemo-Base-2407",
"mistralai/Mistral-Nemo-Base-2407",
"unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit",
"unsloth/Mistral-Nemo-Instruct-2407",
"mistralai/Mistral-Nemo-Instruct-2407",
],
"unsloth_Mistral-Small-Instruct-2409.yaml": [
"unsloth/Mistral-Small-Instruct-2409-bnb-4bit",
"mistralai/Mistral-Small-Instruct-2409",
],
"unsloth_mistral-7b-instruct-v0.3-bnb-4bit.yaml": [
"unsloth/mistral-7b-instruct-v0.3-bnb-4bit",
"unsloth/mistral-7b-instruct-v0.3",
"mistralai/Mistral-7B-Instruct-v0.3",
],
"unsloth_Qwen2.5-1.5B-Instruct.yaml": [
"unsloth/Qwen2.5-1.5B-Instruct-unsloth-bnb-4bit",
"Qwen/Qwen2.5-1.5B-Instruct",
"unsloth/Qwen2.5-1.5B-Instruct-bnb-4bit",
],
"unsloth_Nemotron-3-Nano-30B-A3B.yaml": [
"unsloth/Nemotron-3-Nano-30B-A3B",
],
"unsloth_orpheus-3b-0.1-ft.yaml": [
"unsloth/orpheus-3b-0.1-ft",
"unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit",
"canopylabs/orpheus-3b-0.1-ft",
"unsloth/orpheus-3b-0.1-ft-bnb-4bit",
],
"OuteAI_Llama-OuteTTS-1.0-1B.yaml": [
"OuteAI/Llama-OuteTTS-1.0-1B",
"unsloth/Llama-OuteTTS-1.0-1B",
"unsloth/llama-outetts-1.0-1b",
"OuteAI/OuteTTS-1.0-0.6B",
"unsloth/OuteTTS-1.0-0.6B",
"unsloth/outetts-1.0-0.6b",
],
"unsloth_PaddleOCR-VL.yaml": [
"unsloth/PaddleOCR-VL",
],
"unsloth_Phi-3-medium-4k-instruct.yaml": [
"unsloth/Phi-3-medium-4k-instruct-bnb-4bit",
"microsoft/Phi-3-medium-4k-instruct",
],
"unsloth_Phi-3.5-mini-instruct.yaml": [
"unsloth/Phi-3.5-mini-instruct-bnb-4bit",
"microsoft/Phi-3.5-mini-instruct",
],
"unsloth_Phi-4.yaml": [
"unsloth/phi-4-unsloth-bnb-4bit",
"microsoft/phi-4",
"unsloth/phi-4-bnb-4bit",
],
"unsloth_Pixtral-12B-2409.yaml": [
"unsloth/Pixtral-12B-2409-unsloth-bnb-4bit",
"mistralai/Pixtral-12B-2409",
"unsloth/Pixtral-12B-2409-bnb-4bit",
],
"unsloth_Qwen2-7B.yaml": [
"unsloth/Qwen2-7B-bnb-4bit",
"Qwen/Qwen2-7B",
],
"unsloth_Qwen2-VL-7B-Instruct.yaml": [
"unsloth/Qwen2-VL-7B-Instruct-unsloth-bnb-4bit",
"Qwen/Qwen2-VL-7B-Instruct",
"unsloth/Qwen2-VL-7B-Instruct-bnb-4bit",
],
"unsloth_Qwen2.5-7B.yaml": [
"unsloth/Qwen2.5-7B-unsloth-bnb-4bit",
"Qwen/Qwen2.5-7B",
"unsloth/Qwen2.5-7B-bnb-4bit",
],
"unsloth_Qwen2.5-Coder-1.5B-Instruct.yaml": [
"unsloth/Qwen2.5-Coder-1.5B-Instruct-bnb-4bit",
"Qwen/Qwen2.5-Coder-1.5B-Instruct",
],
"unsloth_Qwen2.5-Coder-14B-Instruct.yaml": [
"unsloth/Qwen2.5-Coder-14B-Instruct-bnb-4bit",
"Qwen/Qwen2.5-Coder-14B-Instruct",
],
"unsloth_Qwen2.5-VL-7B-Instruct-bnb-4bit.yaml": [
"unsloth/Qwen2.5-VL-7B-Instruct",
"Qwen/Qwen2.5-VL-7B-Instruct",
"unsloth/Qwen2.5-VL-7B-Instruct-unsloth-bnb-4bit",
],
"unsloth_Qwen3-0.6B.yaml": [
"unsloth/Qwen3-0.6B-unsloth-bnb-4bit",
"Qwen/Qwen3-0.6B",
"unsloth/Qwen3-0.6B-bnb-4bit",
"Qwen/Qwen3-0.6B-FP8",
"unsloth/Qwen3-0.6B-FP8",
],
"unsloth_Qwen3-4B-Instruct-2507.yaml": [
"unsloth/Qwen3-4B-Instruct-2507-unsloth-bnb-4bit",
"Qwen/Qwen3-4B-Instruct-2507",
"unsloth/Qwen3-4B-Instruct-2507-bnb-4bit",
"Qwen/Qwen3-4B-Instruct-2507-FP8",
"unsloth/Qwen3-4B-Instruct-2507-FP8",
],
"unsloth_Qwen3-30B-A3B-Instruct-2507.yaml": [
"Qwen/Qwen3-30B-A3B-Instruct-2507",
"unsloth/Qwen3-30B-A3B-Instruct-2507-bnb-4bit",
],
"unsloth_Qwen3-4B-Thinking-2507.yaml": [
"unsloth/Qwen3-4B-Thinking-2507-unsloth-bnb-4bit",
"Qwen/Qwen3-4B-Thinking-2507",
"unsloth/Qwen3-4B-Thinking-2507-bnb-4bit",
"Qwen/Qwen3-4B-Thinking-2507-FP8",
"unsloth/Qwen3-4B-Thinking-2507-FP8",
],
"unsloth_Qwen3-14B-Base-unsloth-bnb-4bit.yaml": [
"unsloth/Qwen3-14B-Base",
"Qwen/Qwen3-14B-Base",
"unsloth/Qwen3-14B-Base-bnb-4bit",
],
"unsloth_Qwen3-14B.yaml": [
"unsloth/Qwen3-14B-unsloth-bnb-4bit",
"Qwen/Qwen3-14B",
"unsloth/Qwen3-14B-bnb-4bit",
"Qwen/Qwen3-14B-FP8",
"unsloth/Qwen3-14B-FP8",
],
"unsloth_Qwen3-32B.yaml": [
"unsloth/Qwen3-32B-unsloth-bnb-4bit",
"Qwen/Qwen3-32B",
"unsloth/Qwen3-32B-bnb-4bit",
"Qwen/Qwen3-32B-FP8",
"unsloth/Qwen3-32B-FP8",
],
"unsloth_Qwen3-VL-8B-Instruct-unsloth-bnb-4bit.yaml": [
"Qwen/Qwen3-VL-8B-Instruct-FP8",
"unsloth/Qwen3-VL-8B-Instruct-FP8",
"unsloth/Qwen3-VL-8B-Instruct",
"Qwen/Qwen3-VL-8B-Instruct",
"unsloth/Qwen3-VL-8B-Instruct-bnb-4bit",
],
"unsloth_Qwen3.5.yaml": [
f"{org}/Qwen3.5-{size}{suffix}"
for size in ("0.8B", "2B", "4B", "9B", "27B", "35B-A3B")
for org in ("unsloth", "Qwen")
for suffix in (("",) if size == "27B" else ("", "-Base"))
],
"sesame_csm-1b.yaml": [
"sesame/csm-1b",
"unsloth/csm-1b",
],
"Spark-TTS-0.5B_LLM.yaml": [
"Spark-TTS-0.5B/LLM",
"unsloth/Spark-TTS-0.5B",
],
"unsloth_tinyllama-bnb-4bit.yaml": [
"unsloth/tinyllama",
"TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
],
"unsloth_GLM-4.7-Flash.yaml": [
"unsloth/GLM-4.7-Flash-unsloth-bnb-4bit",
"unsloth/GLM-4.7-Flash-bnb-4bit",
"THUDM/GLM-4.7-Flash",
],
"unsloth_whisper-large-v3.yaml": [
"unsloth/whisper-large-v3",
"openai/whisper-large-v3",
],
}
# Reverse lookup: model_name -> canonical_filename
_REVERSE_MODEL_MAPPING = {}
for canonical_file, model_names in MODEL_NAME_MAPPING.items():
for model_name in model_names:
_REVERSE_MODEL_MAPPING[model_name.lower()] = canonical_file
def load_model_config(
model_name: str,
use_auth: bool = False,
token: Optional[str] = None,
trust_remote_code: bool = False,
local_files_only: bool = False,
revision: Optional[str] = None,
):
"""Defaults trust_remote_code to False: capability probes must never run a repo's auto_map Python."""
from transformers import AutoConfig
revision_kwargs = {"revision": revision} if revision is not None else {}
if is_anonymous(token):
# token=False must be checked explicitly (falsy). Not without_hf_auth(): it mutates HF_TOKEN
# process-wide. AutoConfig serves cached config.json regardless of token, so gate here.
if not is_local_path(model_name) and cached_read_refused(
token,
repo_id = model_name,
is_cached = lambda: _config_json_already_cached(model_name, revision),
# The caller's own cache-only contract, as in the explicit-token gate below.
offline = bool(local_files_only),
):
raise OSError(
f"config.json for {model_name} is not available to an unauthorized caller"
)
return AutoConfig.from_pretrained(
model_name,
trust_remote_code = trust_remote_code,
token = False,
local_files_only = local_files_only,
cache_dir = active_hf_hub_cache(),
**revision_kwargs,
)
if (
isinstance(token, str)
and token
and not is_local_path(model_name)
and cached_read_refused(
token,
repo_id = model_name,
is_cached = lambda: _config_json_already_cached(model_name, revision),
# Record before the fetch, since a half fetch still fills the cache; skip when already cached, which would withhold the repo from a tokenless offline caller.
offline = bool(local_files_only),
)
):
raise OSError(f"config.json for {model_name} is not available to an unauthorized caller")
if token:
# Record before the fetch (a half fetch still fills the cache), only when this call can fetch.
recording = (
recording_a_request_token_fetch(token, model_name, "model")
if not local_files_only and not _config_json_already_cached(model_name, revision)
else nullcontext()
)
with recording:
return AutoConfig.from_pretrained(
model_name,
trust_remote_code = trust_remote_code,
token = token,
local_files_only = local_files_only,
cache_dir = active_hf_hub_cache(),
**revision_kwargs,
)
if not use_auth:
# No auth, for public model checks
with without_hf_auth():
return AutoConfig.from_pretrained(
model_name,
trust_remote_code = trust_remote_code,
token = None,
local_files_only = local_files_only,
cache_dir = active_hf_hub_cache(),
**revision_kwargs,
)
# Default auth (cached tokens)
return AutoConfig.from_pretrained(
model_name,
trust_remote_code = trust_remote_code,
local_files_only = local_files_only,
cache_dir = active_hf_hub_cache(),
**revision_kwargs,
)
# Returns (vlm_model_types, vlm_class_names, audio_model_types) from static registry dicts plus curated set; falls back to curated values without transformers.
_VLM_ARCH_SUFFIXES = ("ForVisionText2Text",)
_CURATED_REMOTE_VLM_TYPES = frozenset(
{
"phi3_v",
"phi4mm",
"llava",
"llava_next",
"llava_onevision",
"internvl_chat",
"cogvlm2",
"minicpmv",
"gemma4",
"deepseek_vl_v2",
"kimi_k25",
}
)
# Fallbacks used only if the transformers registry import fails.
_FALLBACK_AUDIO_MODEL_TYPES = frozenset({"csm", "whisper"})
_VLM_MAPPING_ATTRS = (
"MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES",
"MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES",
)
_AUDIO_MAPPING_ATTRS = (
"MODEL_FOR_CTC_MAPPING_NAMES",
"MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES",
"MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES",
"MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES",
"MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES",
"MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES",
)
_READ_ONLY_DICT_METHODS = frozenset({"items", "keys", "values", "get", "copy"})
def _modeling_auto_source_path() -> Optional[str]:
"""modeling_auto.py of the transformers an import would load now, found without importing it."""
import importlib.util
package = sys.modules.get("transformers")
try:
if package is not None:
locations = list(getattr(package, "__path__", None) or [])
else:
spec = importlib.util.find_spec("transformers")
locations = list(spec.submodule_search_locations or []) if spec is not None else []
except (ImportError, ValueError, AttributeError):
return None
if len(locations) != 1:
return None
path = os.path.join(locations[0], "models", "auto", "modeling_auto.py")
return path if os.path.isfile(path) else None
def _transformers_enables_torch() -> bool:
"""True only when transformers surely enables torch (installed, USE_TORCH/USE_TF, declared
floor); False keeps the import path, so no-torch installs get the curated fallback."""
import importlib.metadata
import importlib.util
# transformers 4.x honours these; `import unsloth` sets USE_TF=0, which leaves torch on.
true_values = {"1", "ON", "YES", "TRUE"}
if os.environ.get("USE_TORCH", "AUTO").upper() not in true_values | {"AUTO"}:
return False
if os.environ.get("USE_TF", "AUTO").upper() in true_values:
return False
try:
from packaging.requirements import Requirement
from packaging.version import Version
if importlib.util.find_spec("torch") is None:
return False
torch_version = Version(importlib.metadata.version("torch"))
requirements = [Requirement(r) for r in importlib.metadata.requires("transformers") or []]
except Exception: # noqa: BLE001
return False
# Floors only: ceilings are not runtime gates (4.52 declares torch<2.7, runs on newer torch).
floors = [
spec
for r in requirements
if r.name == "torch"
for spec in r.specifier
if spec.operator in (">=", ">")
]
if not floors:
return False
return all(spec.contains(torch_version, prereleases = True) for spec in floors)
def _mapping_names_from_source(path: str) -> Optional[Dict[str, dict]]:
"""The wanted *_MAPPING_NAMES from modeling_auto.py's source via ast, without importing torch.
None (caller imports) unless each is bound once at module level to a literal and never
rebound or mutated."""
import ast
wanted = frozenset(_VLM_MAPPING_ATTRS + _AUDIO_MAPPING_ATTRS)
try:
with open(path, encoding = "utf-8") as fh:
tree = ast.parse(fh.read())
except (OSError, SyntaxError, ValueError, UnicodeDecodeError, MemoryError, RecursionError):
return None
found: Dict[str, dict] = {}
for node in tree.body:
if not (
isinstance(node, ast.Assign)
and len(node.targets) == 1
and isinstance(node.targets[0], ast.Name)
and node.targets[0].id in wanted
):
continue
value = node.value
if (
isinstance(value, ast.Call)
and isinstance(value.func, ast.Name)
and value.func.id == "OrderedDict"
and len(value.args) == 1
and not value.keywords
):
value = value.args[0]
try:
literal = ast.literal_eval(value)
mapping = dict(literal)
except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError):
return None
if node.targets[0].id in found or not all(isinstance(k, str) for k in mapping):
return None
found[node.targets[0].id] = mapping
# Unmentioned names read as {} like getattr(..., None) (VISION_2_SEQ is gone in 5.x), unless
# __getattr__, a star import or globals() could create them.
mentioned = {n.id for n in ast.walk(tree) if isinstance(n, ast.Name) and n.id in wanted}
for node in ast.walk(tree):
if (
(isinstance(node, ast.FunctionDef) and node.name == "__getattr__")
or (isinstance(node, ast.ImportFrom) and any(a.name == "*" for a in node.names))
or (isinstance(node, ast.Name) and node.id in ("globals", "vars", "setattr", "exec"))
):
if wanted - set(found):
return None
if not found:
return None
for name in wanted - set(found):
if name in mentioned:
return None
found[name] = {}
stores: Dict[str, int] = {}
for node in ast.walk(tree):
if isinstance(node, ast.Name) and node.id in wanted and not isinstance(node.ctx, ast.Load):
stores[node.id] = stores.get(node.id, 0) + 1
elif isinstance(node, (ast.Attribute, ast.Subscript)):
base = node.value
if (
isinstance(base, ast.Name)
and base.id in wanted
and not (
isinstance(node.ctx, ast.Load)
and (isinstance(node, ast.Subscript) or node.attr in _READ_ONLY_DICT_METHODS)
)
):
return None
elif isinstance(node, (ast.Global, ast.Nonlocal)) and wanted.intersection(node.names):
return None
elif isinstance(node, (ast.Import, ast.ImportFrom)) and any(
(alias.asname or alias.name) in wanted for alias in node.names
):
return None
if any(stores.get(name, 0) != (1 if name in mentioned else 0) for name in wanted):
return None
return found
def _build_detection_sets():
"""Reads static name dicts from the transformers registry only; no model loads and no code runs."""
try:
source_path = (
None
if "transformers.models.auto.modeling_auto" in sys.modules
or not _transformers_enables_torch()
else _modeling_auto_source_path()
)
from_source = _mapping_names_from_source(source_path) if source_path else None
if from_source is not None:
def _names(attr):
d = from_source[attr]
return dict(d) if d else {}
else:
from transformers.models.auto import modeling_auto as _ma
def _names(attr):
d = getattr(_ma, attr, None)
return dict(d) if d else {}
itt = _names("MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES")
v2s = _names("MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES")
vlm_types = set(itt) | set(v2s) | set(_CURATED_REMOTE_VLM_TYPES)
vlm_classes = set(itt.values()) | set(v2s.values())
audio_types: set = set()
for attr in _AUDIO_MAPPING_ATTRS:
audio_types |= set(_names(attr))
audio_types |= set(_FALLBACK_AUDIO_MODEL_TYPES)
return frozenset(vlm_types), frozenset(vlm_classes), frozenset(audio_types)
except Exception as exc: # pragma: no cover - defensive
logger.warning("Could not build detection sets from transformers: %s", exc)
return (
frozenset(_CURATED_REMOTE_VLM_TYPES),
frozenset(),
frozenset(_FALLBACK_AUDIO_MODEL_TYPES),
)
# (snapshot, filenames) for the current commit; None when offline, anonymous, a local path, no such snapshot, or the document names no files.
_DETECTION_SETS: Optional[Tuple[frozenset, frozenset, frozenset]] = None
_DETECTION_SETS_LOCK = threading.Lock()
def _detection_sets() -> Tuple[frozenset, frozenset, frozenset]:
"""(vlm_model_types, vlm_class_names, audio_model_types), built once."""
global _DETECTION_SETS
sets = _DETECTION_SETS
if sets is None:
with _DETECTION_SETS_LOCK:
sets = _DETECTION_SETS
if sets is None:
sets = _DETECTION_SETS = _build_detection_sets()
return sets
# Precomputed .venv_t5 paths for subprocess switching; Gemma 4 vision checks use the 5.5 sidecar.
from utils.paths.storage_roots import studio_root as _studio_root # noqa: E402
_VENV_T5_DIR = str(_studio_root() / ".venv_t5_550")
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent.parent)
def _is_vlm(config) -> bool:
vlm_types, vlm_classes, audio_types = _detection_sets()
architectures = getattr(config, "architectures", None) or []
model_type = getattr(config, "model_type", None)
explicit_vision = (
hasattr(config, "vision_config")
or hasattr(config, "img_processor")
or hasattr(config, "image_token_index")
or hasattr(config, "projector_config")
)
# Audio-only models are vision only if they carry an explicit vision sub-config.
if model_type in audio_types and not explicit_vision:
return False
return (
explicit_vision
or any(x in vlm_classes for x in architectures)
or any(isinstance(x, str) and x.endswith(_VLM_ARCH_SUFFIXES) for x in architectures)
or model_type in vlm_types
)
def _current_cached_snapshot(
model_name: str,
hf_token: HfTokenArg = None,
local_files_only: bool = False,
):
"""(snapshot, filenames) for the repo's current commit; None when offline, anonymous, or local path."""
if local_files_only or is_anonymous(hf_token) or is_local_path(model_name):
return None
try:
repo_dir = get_cache_path(model_name)
# Cheap check first: with nothing cached, the document below buys only that answer.
if repo_dir is None or not (Path(repo_dir) / "snapshots").is_dir():
return None
info = _hub_model_info(model_name, hf_token)
sha = getattr(info, "sha", None)
if not sha:
return None
snapshot = Path(repo_dir) / "snapshots" / str(sha)
if not snapshot.is_dir():
return None
listed = {
getattr(sibling, "rfilename", None)
for sibling in (getattr(info, "siblings", None) or ())
}
# siblings is optional: an empty listing cannot prove a file absent.
if not listed:
return None
return snapshot, listed
except Exception as exc:
logger.debug("No current cached snapshot for '%s': %s", model_name, exc)
return None
def _read_raw_config_json(
model_name: str,
hf_token: Optional[str] = None,
local_files_only: bool = False,
revision: Optional[str] = None,
) -> Optional[dict]:
"""config.json as a dict, read without executing any repo code; None when the repo publishes none.
Raises on read errors: callers decide what an unreadable config means."""
current = (
None
if revision is not None
else _current_cached_snapshot(model_name, hf_token, local_files_only)
)
if is_local_path(model_name):
config_path = Path(normalize_path(model_name)).expanduser() / "config.json"
elif current is not None and (current[0] / "config.json").is_file():
# The current commit's own copy: skips the absence probe and the freshness check.
config_path = current[0] / "config.json"
elif current is not None and "config.json" not in current[1]:
# The document just said the repo publishes none; the probe below re-asks.
logger.debug("'%s' has no config.json on the Hub", model_name)
return None
else:
from huggingface_hub import hf_hub_download
from utils.hf_probe import hf_file_definitely_absent
# Avoid caching the expected 404 for GGUF repos without config.json.
if not local_files_only and hf_file_definitely_absent(
model_name,
"config.json",
revision = revision,
token = hf_token,
):
logger.debug("'%s' has no config.json on the Hub", model_name)
return None
download_kwargs = {
"repo_id": model_name,
"filename": "config.json",
"token": hf_token,
"local_files_only": local_files_only,
"cache_dir": active_hf_hub_cache(),
}
if revision is not None:
download_kwargs["revision"] = revision
# Measured: hf_hub_download falls back to the cache even with
# local_files_only=False, never consulting the credential, so a planted entry
# plus a dead endpoint returns a private config.json to a token that cannot read it.
if cached_read_refused(
hf_token,
repo_id = model_name,
is_cached = lambda: _config_json_already_cached(model_name, revision),
):
return None
config_path = Path(
call_with_anonymous_retry(
lambda token: hf_hub_download(**{**download_kwargs, "token": token}),
hf_token,
)
)
return json.loads(config_path.read_text(encoding = "utf-8-sig"))
def _raw_config_has_vision_config(
model_name: str,
hf_token: Optional[str] = None,
local_files_only: bool = False,
revision: Optional[str] = None,
) -> Optional[bool]:
try:
config = _read_raw_config_json(model_name, hf_token, local_files_only, revision)
if config is None:
return None
architectures = config.get("architectures") or []
model_type = config.get("model_type")
explicit_vision = (
"vision_config" in config
or "img_processor" in config
or "image_token_index" in config
or "projector_config" in config
)
vlm_types, vlm_classes, audio_types = _detection_sets()
# Audio-only models are vision only if they carry an explicit vision sub-config.
if model_type in audio_types and not explicit_vision:
return False
return (
explicit_vision
or any(isinstance(x, str) and x in vlm_classes for x in architectures)
or any(isinstance(x, str) and x.endswith(_VLM_ARCH_SUFFIXES) for x in architectures)
or model_type in vlm_types
)
except Exception as exc:
logger.warning("Could not read config.json for '%s': %s", model_name, exc)
return None
# Refuses offline cache reads to unentitled callers, not local paths; offline must be forwarded, not only tested, or local_files_only probes anyway.
def _offline_cache_read_refused(hf_token, model_name: str, repo_id: str, offline: bool) -> bool:
"""Refuses offline cache reads to unentitled callers; ``offline`` must be forwarded, not only tested."""
return (
offline
and not is_local_path(model_name)
and not cache_reads_authorized(hf_token, repo_id = repo_id, offline = offline)
)
def _config_json_already_cached(model_name: str, revision: Optional[str] = None) -> bool:
"""True if this repo's config.json is on disk, so an unauthorized read could be served it."""
try:
from huggingface_hub import try_to_load_from_cache
hit = try_to_load_from_cache(
repo_id = model_name,
filename = "config.json",
revision = revision,
cache_dir = active_hf_hub_cache(),
)
# Also returns a sentinel object recording a known-absent file; only a str is a real hit.
return isinstance(hit, str)
except Exception as exc:
# Never let the guard's own failure open the path it guards.
logger.debug("Could not check cached config.json for '%s': %s", model_name, exc)
return True
def _build_vision_check_inline_helpers() -> str:
vlm_types, vlm_classes, audio_types = _detection_sets()
return (
"_VLM_ARCH_SUFFIXES = " + repr(tuple(_VLM_ARCH_SUFFIXES)) + "\n"
"_VLM_MODEL_TYPES = " + repr(set(vlm_types)) + "\n"
"_VLM_CLASS_NAMES = " + repr(set(vlm_classes)) + "\n"
"_AUDIO_ONLY_MODEL_TYPES = " + repr(set(audio_types)) + "\n"
"def _is_vlm(config):\n"
" architectures = getattr(config, 'architectures', None) or []\n"
" model_type = getattr(config, 'model_type', None)\n"
" explicit_vision = (\n"
" hasattr(config, 'vision_config')\n"
" or hasattr(config, 'img_processor')\n"
" or hasattr(config, 'image_token_index')\n"
" or hasattr(config, 'projector_config')\n"
" )\n"
" if model_type in _AUDIO_ONLY_MODEL_TYPES and not explicit_vision:\n"
" return False\n"
" return (\n"
" explicit_vision\n"
" or any(x in _VLM_CLASS_NAMES for x in architectures)\n"
" or any(isinstance(x, str) and x.endswith(_VLM_ARCH_SUFFIXES) for x in architectures)\n"
" or model_type in _VLM_MODEL_TYPES\n"
" )\n"
)
# Subprocess script run with transformers 5.x active: model_name via argv, token via env, JSON out.
def _build_vision_check_script() -> str:
return (
r"""
import sys, os, json
os.environ["TOKENIZERS_PARALLELISM"] = "false"
# Activate transformers 5.x
venv_t5 = sys.argv[1]
backend_dir = sys.argv[2]
model_name = sys.argv[3]
token = sys.argv[4] if len(sys.argv) > 4 and sys.argv[4] != "" else None
revision = sys.argv[5] if len(sys.argv) > 5 else None
sys.path.insert(0, venv_t5)
if backend_dir not in sys.path:
sys.path.insert(0, backend_dir)
"""
+ _build_vision_check_inline_helpers()
+ r"""
# Fresh interpreter, and AutoConfig.from_pretrained below may hit the Hub.
try:
from utils.native_tls import activate_native_tls
activate_native_tls()
except Exception:
pass
try:
from utils.happy_eyeballs import activate_happy_eyeballs
activate_happy_eyeballs()
except Exception:
pass
try:
from transformers import AutoConfig
# Union the ACTIVE sidecar's registry into the inlined parent-process sets
# so architectures only the sidecar knows still classify correctly.
try:
from transformers.models.auto import modeling_auto as _ma
for _attr in ("MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES",
"MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES"):
_d = dict(getattr(_ma, _attr, None) or {})
_VLM_MODEL_TYPES |= set(_d)
_VLM_CLASS_NAMES |= set(_d.values())
for _attr in ("MODEL_FOR_CTC_MAPPING_NAMES",
"MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES",
"MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES",
"MODEL_FOR_TEXT_TO_WAVEFORM_MAPPING_NAMES",
"MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING_NAMES",
"MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES"):
_AUDIO_ONLY_MODEL_TYPES |= set(dict(getattr(_ma, _attr, None) or {}))
except Exception:
pass
# Capability detection never executes model repo code.
kwargs = {"trust_remote_code": False}
if token:
kwargs["token"] = token
if revision is not None:
kwargs["revision"] = revision
config = AutoConfig.from_pretrained(model_name, **kwargs)
is_vlm = _is_vlm(config)
model_type = getattr(config, "model_type", None)
archs = getattr(config, "architectures", [])
print(json.dumps({"is_vision": is_vlm, "model_type": model_type,
"architectures": archs}))
except Exception as exc:
print(json.dumps({"error": str(exc)}))
sys.exit(1)
"""
)
# PEP 562 keeps the two script strings and three detection sets reachable under their
# original module-level names. Memoised so access doesn't rebuild the ~40 KB script.
_LAZY_MODULE_ATTRS = {
"_VLM_MODEL_TYPES": lambda: _detection_sets()[0],
"_VLM_CLASS_NAMES": lambda: _detection_sets()[1],
"_AUDIO_ONLY_MODEL_TYPES": lambda: _detection_sets()[2],
"_VISION_CHECK_INLINE_HELPERS": _build_vision_check_inline_helpers,
"_VISION_CHECK_SCRIPT": _build_vision_check_script,
}
_LAZY_MODULE_CACHE: Dict[str, Any] = {}
_LAZY_MODULE_LOCK = threading.Lock()
def _lazy_module_attr(name: str) -> Any:
"""Build-once accessor for in-module callers too; a bare global read would skip PEP 562."""
with _LAZY_MODULE_LOCK:
if name not in _LAZY_MODULE_CACHE:
_LAZY_MODULE_CACHE[name] = _LAZY_MODULE_ATTRS[name]()
return _LAZY_MODULE_CACHE[name]
def __getattr__(name: str) -> Any:
if name not in _LAZY_MODULE_ATTRS:
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
return _lazy_module_attr(name)
def _vision_check_child_env(hf_token: HfTokenArg) -> Dict[str, str]:
"""Child environment for the vision-check probe, carrying the caller's credential."""
env = utf8_child_env(get_hf_cache_paths().child_env(child_env_without_native_path_secret()))
apply_token_to_child_env(env, hf_token)
return env
def _is_vision_model_subprocess(
model_name: str,
hf_token: Optional[str] = None,
revision: Optional[str] = None,
) -> Optional[bool]:
"""Run is_vision_model in a subprocess with transformers 5.x.
Spawns a clean subprocess with .venv_t5/ on sys.path so AutoConfig
recognizes newer architectures. Returns True/False for definitive results,
or None for transient failures (timeouts, subprocess errors), which are not
cached so they can be retried.
"""
# Token only via the child env (argv is world-readable); the empty slot keeps the argv layout.
token_arg = ""
# Latest-only architectures need the latest sidecar for AutoConfig; other tiers keep 5.5.
sidecar_dir = _VENV_T5_DIR
try:
from utils.transformers_version import _VENV_T5_LATEST_DIR, get_transformers_tier
if get_transformers_tier(model_name, hf_token, probe = False) == "latest":
sidecar_dir = _VENV_T5_LATEST_DIR
except Exception:
pass
try:
command = [
sys.executable,
"-c",
_lazy_module_attr("_VISION_CHECK_SCRIPT"),
sidecar_dir,
_BACKEND_DIR,
model_name,
token_arg,
]
if revision is not None:
command.append(revision)
result = subprocess.run(
command,
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 60,
env = _vision_check_child_env(hf_token),
**_windows_hidden_subprocess_kwargs(),
)
if result.returncode != 0:
stderr = result.stderr.strip()
logger.warning(
"Vision check subprocess failed for '%s': %s",
model_name,
stderr or result.stdout.strip(),
)
return None
data = json.loads(result.stdout.strip())
if "error" in data:
logger.warning(
"Vision check subprocess error for '%s': %s",
model_name,
data["error"],
)
return None
is_vlm = data["is_vision"]
logger.info(
"Vision check (subprocess, transformers 5.x) for '%s': "
"model_type=%s, architectures=%s, is_vision=%s",
model_name,
data.get("model_type"),
data.get("architectures"),
is_vlm,
)
return is_vlm
except subprocess.TimeoutExpired:
logger.warning("Vision check subprocess timed out for '%s'", model_name)
return None
except Exception as exc:
logger.warning("Vision check subprocess failed for '%s': %s", model_name, exc)
return None
def _token_fingerprint(token: HfTokenArg) -> Optional[str]:
"""SHA256 digest of the token as cache key; the anonymous sentinel gets its own slot, not None's."""
if is_anonymous(token):
return ANONYMOUS_CACHE_IDENTITY
if token is None:
return None
return qualify_cache_identity(token, hashlib.sha256(token.encode("utf-8")).hexdigest())
# Scoped to a request, not cached: a repo gated or deleted between requests is seen on the next.
_HubModelInfoScope = Dict[Tuple[str, Optional[str]], Any]
_hub_model_info_scope: ContextVar[Optional[_HubModelInfoScope]] = ContextVar(
"hub_model_info_scope", default = None
)
# Bound the Hub lookup so a DNS-dead session fails fast to the cache instead of hanging on retries.
_HUB_MODEL_INFO_TIMEOUT = 15.0
# Unbounded before #10230; three identical 15s reads never pass on a slow link (#11551).
_GGUF_LISTING_TIMEOUTS = (_HUB_MODEL_INFO_TIMEOUT, 30.0, 60.0)
@_contextlib.contextmanager
def shared_hub_model_info():
"""Share a ``model_info`` response between the Hub probes here, per repo and credential. A nested block joins the open one."""
open_scope = _hub_model_info_scope.get()
token = _hub_model_info_scope.set({} if open_scope is None else open_scope)
try:
yield
finally:
_hub_model_info_scope.reset(token)
def _hub_model_info(
repo_id: str,
hf_token: HfTokenArg = None,
*,
files_metadata: bool = False,
timeout: Optional[float] = None,
):
from huggingface_hub import model_info as hf_model_info
# from_identifier's remote-LoRA probe needs this to raise; _offline_while_reading can force offline.
scope = None if _env_offline() else _hub_model_info_scope.get()
# The forced-anonymous sentinel is a credential of its own, so key on its fingerprint.
key = (repo_id, _token_fingerprint(hf_token))
if scope is not None:
if key in scope:
info, rejected = scope[key]
if rejected:
# The read may have run before the caller's rejection scope opened.
note_saved_token_rejected(hf_token)
return info
# Asked unconditionally so one response serves every probe; the listing needs sizes.
files_metadata = True
kwargs: Dict[str, Any] = {
"token": hf_token,
"files_metadata": files_metadata,
# Shared, so whichever probe reads first fixes the bound the rest inherit.
"timeout": _HUB_MODEL_INFO_TIMEOUT if timeout is None else timeout,
}
with collecting_hub_token_rejections():
info = call_with_anonymous_retry(
lambda token: hf_model_info(repo_id, **{**kwargs, "token": token}), hf_token
)
rejected = saved_token_rejected(hf_token)
if scope is not None:
scope[key] = (info, rejected)
return info
def read_hub_model_info_ahead(repo_id: str, hf_token: HfTokenArg) -> Optional[threading.Thread]:
"""Fill the open scope from a thread. Join it before the first probe; a failed read leaves the answer to that probe."""
if is_local_path(repo_id):
return None
def read():
try:
_hub_model_info(repo_id, hf_token)
except Exception as exc:
logger.debug("model_info read ahead failed for %s: %s", repo_id, exc)
thread = threading.Thread(target = copy_context().run, args = (read,), daemon = True)
thread.start()
return thread
def _is_timeout_error(exc: BaseException) -> bool:
if isinstance(exc, TimeoutError):
return True
return any("Timeout" in cls.__name__ for cls in type(exc).__mro__)
def _hub_model_info_slow_link(
repo_id: str,
hf_token: HfTokenArg = None,
*,
files_metadata: bool = False,
):
"""Re-read with a longer bound only after a timeout; a refusal fails on the first attempt."""
for attempt, timeout in enumerate(_GGUF_LISTING_TIMEOUTS):
try:
return _hub_model_info(
repo_id, hf_token, files_metadata = files_metadata, timeout = timeout
)
except Exception as e:
if attempt == len(_GGUF_LISTING_TIMEOUTS) - 1 or not _is_timeout_error(e):
raise
# Revision-less entries keep the historical 3-part key; pinned entries append revision.
_CapabilityCacheKey = Tuple[Any, ...]
_vision_detection_cache: Dict[_CapabilityCacheKey, bool] = {}
_vision_cache_lock = threading.Lock()
def _is_vision_model_or_none(
model_name: str,
hf_token: Optional[str] = None,
local_files_only: bool = False,
revision: Optional[str] = None,
gguf_variant: Optional[str] = None,
require_image: bool = True,
gguf_companion_roots: Optional[Tuple[str, ...]] = None,
) -> Optional[bool]:
"""Detect VLMs via the config architecture (works for fine-tunes); transformers-5.x
models are checked in a .venv_t5/ subprocess. Cached per (model_name, token,
local_files_only, revision) minus transient failures; local_files_only is in the
key so an offline probe never shares an online entry.
``gguf_variant`` picks the quant out of a directory, so the probe reads the weight
file the load will open. ``require_image=False`` asks only for a companion projector,
which is what an audio request needs: audio input rides the same file."""
# Local GGUF models are served by llama-server, so multimodal support comes from a companion
# mmproj, not a Transformers config; do not cache, as a projector may be added later.
if is_local_path(model_name):
local_path = normalize_path(model_name)
# Same rule as ModelConfig.from_identifier: variant lookup is directory-only, and
# detect_gguf_model reads only the directory root, which holds no weights in a
# repo that files every quant under a per-quant subdir.
if gguf_variant and Path(local_path).is_dir():
gguf_file = _find_local_gguf_by_variant(local_path, gguf_variant)
else:
gguf_file = detect_gguf_model(local_path)
if gguf_file:
companion_root = _local_gguf_companion_search_root(local_path, gguf_file)
companion_roots = gguf_companion_roots or (companion_root,)
mmproj_file = next(
(
found
for root in companion_roots
if (
found := detect_mmproj_file(
gguf_file,
search_root = root,
allow_disjoint_search_root = gguf_companion_roots is not None,
)
)
),
None,
)
# An audio-only projector serves this model's audio input, never an image.
is_vision = mmproj_file is not None and (
not require_image or mmproj_accepts_image(mmproj_file)
)
logger.debug(
"Local GGUF vision check for '%s': mmproj=%s, is_vision=%s",
gguf_file,
mmproj_file,
is_vision,
)
return is_vision
# Normalize model name so different casings of the same repo share a key
try:
if is_local_path(model_name):
resolved_name = normalize_path(model_name)
else:
resolved_name = resolve_cached_repo_id_case(model_name)
except Exception as exc:
logger.debug(
"Could not normalize model name '%s' for cache key: %s",
model_name,
exc,
)
resolved_name = model_name
# Key on effective offline (kwarg OR env) so an offline probe can't poison a later lookup.
effective_offline = bool(local_files_only or _env_offline())
# Uncached, so use is_vision_model(); None marks a transient failure the caller must not cache, and the code-free raw-config reader runs first.
if _offline_cache_read_refused(hf_token, model_name, resolved_name, effective_offline):
return False
cache_key: _CapabilityCacheKey = (
resolved_name,
_token_fingerprint(hf_token),
effective_offline,
)
if revision is not None:
cache_key += (revision,)
# Lock-free fast path. Sentinel separates "key not found" from "value is False".
_MISS = object()
cached = _vision_detection_cache.get(cache_key, _MISS)
if cached is not _MISS:
return cached
# Compute outside the lock; two concurrent calls may both run but agree.
uncached_kwargs = {"local_files_only": effective_offline}
if revision is not None:
uncached_kwargs["revision"] = revision
result = _is_vision_model_uncached(resolved_name, hf_token, **uncached_kwargs)
# Only cache definitive results; None is a transient failure, retry later.
if result is not None:
with _vision_cache_lock:
_vision_detection_cache[cache_key] = result
return result
return None
def is_vision_model(
model_name: str,
hf_token: Optional[str] = None,
local_files_only: bool = False,
revision: Optional[str] = None,
gguf_variant: Optional[str] = None,
require_image: bool = True,
gguf_companion_roots: Optional[Tuple[str, ...]] = None,
) -> bool:
"""Whether model_name takes images; a transient probe failure reads as False (see _is_vision_model_or_none)."""
return bool(
_is_vision_model_or_none(
model_name,
hf_token,
local_files_only,
revision,
gguf_variant,
require_image,
gguf_companion_roots,
)
)
def audio_vlm_multimodal_lm_checked(
model_name: str,
hf_token: Optional[str] = None,
local_files_only: bool = False,
revision: Optional[str] = None,
) -> Tuple[bool, bool]:
"""(loads through AutoModelForMultimodalLM, config.json readable): no vision tower and a class registered there,
natively or by auto_map. AudioFlamingo3 has none and keeps FastModel's resolver. Raw config: no repo code runs."""
vision = _raw_config_has_vision_config(model_name, hf_token, local_files_only, revision)
if vision is None:
return False, False
if vision:
return False, True
try:
config = _read_raw_config_json(model_name, hf_token, local_files_only, revision) or {}
except Exception:
return False, False
auto_map = config.get("auto_map")
if isinstance(auto_map, dict) and "AutoModelForMultimodalLM" in auto_map:
return True, True
try:
from transformers.models.auto.modeling_auto import MODEL_FOR_MULTIMODAL_LM_MAPPING_NAMES
except ImportError:
return False, True
return config.get("model_type") in MODEL_FOR_MULTIMODAL_LM_MAPPING_NAMES, True
def is_vision_model_checked(
model_name: str,
hf_token: Optional[str] = None,
local_files_only: bool = False,
revision: Optional[str] = None,
) -> Tuple[bool, bool]:
"""is_vision_model, plus whether the answer is definitive (False after a transient probe failure)."""
result = _is_vision_model_or_none(model_name, hf_token, local_files_only, revision)
return bool(result), result is not None
def _is_vision_model_uncached(
model_name: str,
hf_token: Optional[str] = None,
local_files_only: bool = False,
revision: Optional[str] = None,
) -> Optional[bool]:
"""Uncached vision detection; use is_vision_model() instead.
Returns True/False for definitive results, or None on transient errors
(network, timeout, subprocess failure) so the caller knows not to cache.
"""
# Try the raw-config reader FIRST (code-free, version-independent): it classifies repo-code
# VLMs like DeepSeek-OCR via declarative vision_config, with no remote code or subprocess.
raw_kwargs = {
"hf_token": hf_token,
"local_files_only": local_files_only,
}
if revision is not None:
raw_kwargs["revision"] = revision
raw = _raw_config_has_vision_config(model_name, **raw_kwargs)
if raw is not None:
if raw is False and not local_files_only:
# Latest-tier archs: trust the sidecar AutoConfig probe over heuristic False; an inconclusive probe returns None so the False is not cached.
try:
from utils.transformers_version import get_transformers_tier
if get_transformers_tier(model_name, hf_token, probe = False) == "latest":
subprocess_kwargs = {"hf_token": hf_token}
if revision is not None:
subprocess_kwargs["revision"] = revision
return _is_vision_model_subprocess(model_name, **subprocess_kwargs)
except Exception:
pass
return raw
# Fallback AutoConfig read runs with remote code DISABLED; skip the 5.x subprocess offline, since it probes the network.
from utils.transformers_version import needs_transformers_5
if not local_files_only and needs_transformers_5(model_name):
logger.info(
"Model '%s' needs transformers 5.x -- checking vision via subprocess",
model_name,
)
subprocess_kwargs = {"hf_token": hf_token}
if revision is not None:
subprocess_kwargs["revision"] = revision
return _is_vision_model_subprocess(model_name, **subprocess_kwargs)
try:
config_kwargs = {
"use_auth": True,
"token": hf_token,
"local_files_only": local_files_only,
}
if revision is not None:
config_kwargs["revision"] = revision
config = load_model_config(model_name, **config_kwargs)
if _is_vlm(config):
model_type = getattr(config, "model_type", None)
archs = getattr(config, "architectures", None) or []
logger.info(
"Model %s detected as VLM (model_type=%s, architectures=%s)",
model_name,
model_type,
archs,
)
return True
return False
except Exception as e:
logger.warning(f"Could not determine if {model_name} is vision model: {e}")
# Permanent failures (not found, gated, bad config) cache as False; transient ones don't.
try:
from huggingface_hub.errors import RepositoryNotFoundError, GatedRepoError
except ImportError:
try:
from huggingface_hub.utils import (
RepositoryNotFoundError,
GatedRepoError,
)
except ImportError:
RepositoryNotFoundError = GatedRepoError = None
if RepositoryNotFoundError is not None and isinstance(
e, (RepositoryNotFoundError, GatedRepoError)
):
return False
if isinstance(e, (ValueError, json.JSONDecodeError)):
return False
return None
# Keyed like the vision cache so an unauthenticated/offline/other-revision miss can't poison.
_audio_detection_cache: Dict[_CapabilityCacheKey, Optional[str]] = {}
# Short-lived cache of offline misses (/loras repeats them); non-definitive answers are never cached; answers can appear later.
_AUDIO_OFFLINE_MISS_TTL_S = 60.0
_audio_offline_miss_cache: Dict[_CapabilityCacheKey, float] = {}
def _local_audio_metadata_fingerprint(model_name: str) -> tuple:
try:
root = Path(normalize_path(model_name)).expanduser()
if root.is_file():
root = root.parent
identities = []
for relative in (
"config.json",
"modular_model_index.json",
*_AUDIO_TOKENIZER_CONFIG_PATHS,
):
candidate = root / relative
try:
stat = candidate.stat()
identity = (
stat.st_mtime_ns,
stat.st_size,
getattr(stat, "st_ctime_ns", None),
getattr(stat, "st_dev", None),
getattr(stat, "st_ino", None),
)
except OSError:
identity = None
identities.append((relative, identity))
return tuple(identities)
except Exception:
# An unreadable path must not turn cache-key construction into a request failure.
return (("unreadable", None),)
def detect_audio_type(
model_name: str,
hf_token: Optional[str] = None,
local_files_only: bool = False,
revision: Optional[str] = None,
) -> Optional[str]:
"""None is ambiguous (not audio or unreadable repo); gating callers want detect_audio_type_checked."""
return detect_audio_type_checked(
model_name,
hf_token = hf_token,
local_files_only = local_files_only,
revision = revision,
)[0]
def _audio_cpp_repo_audio_type(
model_name: str, hf_token: Optional[str], offline: bool
) -> Optional[str]:
"""audiocpp_tts or audiocpp_music for an audio.cpp GGUF repo, empty for speech-to-text, None if not audio.cpp or not inspected; reads only GGUF-shaped names."""
try:
from core.inference import audio_cpp_models
except Exception: # noqa: BLE001 - no audio.cpp support
return None
name = model_name.strip()
lowered = name.lower()
if not (
audio_cpp_models.is_umbrella_id(name)
or lowered.startswith("audio-cpp/")
or "gguf" in lowered.rsplit("/", 1)[-1]
or "audiocpp" in lowered
or "audio.cpp" in lowered
):
return None
if audio_cpp_models.parse_identifier(name) is None:
return None
try:
model = audio_cpp_models.resolve(name, None, hf_token, network = not offline)
except Exception: # noqa: BLE001 - unknown, not "not audio.cpp"
return None
if model is None:
return None
return model.audio_type or ""
def detect_audio_type_checked(
model_name: str,
hf_token: Optional[str] = None,
local_files_only: bool = False,
revision: Optional[str] = None,
) -> Tuple[Optional[str], bool]:
"""definitive is False on a 401, 5xx, or timeout, so None then means unknown rather than not audio."""
# Empty name: no model is definitively not audio, and the name would land in the Hub URL.
if not model_name:
return None, True
try:
from core.inference.native_audio import NATIVE_AUDIO_MODEL_IDS, audio_cpp_audio_type
curated_type = NATIVE_AUDIO_MODEL_IDS.get(
str(model_name).strip().lower()
) or audio_cpp_audio_type(str(model_name))
except Exception:
curated_type = None
if curated_type:
return curated_type, True
# Key on effective offline (kwarg OR env) so an offline negative can't poison a later probe.
effective_offline = bool(local_files_only or _env_offline())
audio_cpp_type = _audio_cpp_repo_audio_type(str(model_name), hf_token, effective_offline)
if audio_cpp_type is not None:
return audio_cpp_type or None, True
if _offline_cache_read_refused(hf_token, model_name, model_name, effective_offline):
return None, False
local_fingerprint = (
_local_audio_metadata_fingerprint(model_name) if is_local_path(model_name) else None
)
# Checked on the raw name: casing resolution walks the whole cache, the cost this avoids.
miss_key: _CapabilityCacheKey = (
model_name,
_token_fingerprint(hf_token),
effective_offline,
)
if revision is not None:
miss_key += (revision,)
if local_fingerprint is not None:
miss_key += (local_fingerprint,)
if effective_offline:
seen_at = _audio_offline_miss_cache.get(miss_key)
if seen_at is not None and time.monotonic() - seen_at < _AUDIO_OFFLINE_MISS_TTL_S:
return None, False
# Normalize casing + include the token fingerprint (mirrors is_vision_model).
try:
if is_local_path(model_name):
resolved_name = normalize_path(model_name)
else:
resolved_name = resolve_cached_repo_id_case(model_name)
except Exception:
resolved_name = model_name
cache_key: _CapabilityCacheKey = (
resolved_name,
_token_fingerprint(hf_token),
effective_offline,
)
if revision is not None:
cache_key += (revision,)
if local_fingerprint is not None:
cache_key += (local_fingerprint,)
if cache_key in _audio_detection_cache:
# Only definitive results are cached, so a hit is definitive by construction.
return _audio_detection_cache[cache_key], True
config_kwargs = {"local_files_only": effective_offline}
if revision is not None:
config_kwargs["revision"] = revision
result = _detect_audio_from_config(model_name, hf_token, **config_kwargs)
if result:
_audio_detection_cache[cache_key] = result
_audio_offline_miss_cache.pop(miss_key, None)
logger.info(f"Model {model_name} detected as audio model: audio_type={result}")
return result, True
tokenizer_kwargs = {"local_files_only": effective_offline}
if revision is not None:
tokenizer_kwargs["revision"] = revision
result, definitive = _detect_audio_from_tokenizer(model_name, hf_token, **tokenizer_kwargs)
# Cache only definitive results; a transient read failure stays None and retries.
if definitive:
_audio_detection_cache[cache_key] = result
_audio_offline_miss_cache.pop(miss_key, None)
elif effective_offline:
_audio_offline_miss_cache[miss_key] = time.monotonic()
if result:
logger.info(f"Model {model_name} detected as audio model: audio_type={result}")
elif not definitive:
# Debug offline: /loras asks per checkpoint per poll; online it means gated or upstream error.
(logger.debug if effective_offline else logger.info)(
f"Could not determine whether {model_name} is an audio model: "
f"tokenizer_config.json was unreadable (gated, offline or upstream error)"
)
return result, definitive
def _detect_audio_from_config(
model_name: str,
hf_token: Optional[str] = None,
local_files_only: bool = False,
revision: Optional[str] = None,
) -> Optional[str]:
"""Reads only local config metadata, never the Hub; a failed read falls back to the tokenizer path."""
try:
del hf_token, local_files_only, revision
if not is_local_path(model_name):
return None
local_path = Path(normalize_path(model_name)).expanduser()
from core.inference.native_audio import native_audio_type_from_local_path
return native_audio_type_from_local_path(str(local_path))
except Exception as exc:
logger.debug("Could not read audio model_type for '%s': %s", model_name, exc)
return None
def _read_local_model_config(root: Path, missing_is_absent: bool) -> Optional[dict]:
"""None when config.json cannot be read. A cache snapshot can hold the tokenizer without
it (a tokenizer-only download), so there a missing file is unknown, not absent, unless the
snapshot is an adapter repo, which never ships one."""
config_file = root / "config.json"
if not config_file.is_file():
return {} if missing_is_absent or (root / "adapter_config.json").is_file() else None
try:
config = json.loads(config_file.read_text(encoding = "utf-8-sig"))
except Exception:
return None
return config if isinstance(config, dict) else {}
def _classify_audio_tokenizer(tok_config: dict, read_model_config) -> Tuple[Optional[str], bool]:
"""(audio_type, known). A declared audio_token is audio input only with an audio encoder:
Gemma 4 26B/31B carry the token and none, so config.json decides, read only for that case."""
result = _check_token_patterns(tok_config)
if result or not _declares_audio_token(tok_config):
return result, True
model_config = read_model_config()
if model_config is None:
return None, False
return ("audio_vlm" if model_config.get("audio_config") else None), True
def _detect_audio_from_tokenizer(
model_name: str,
hf_token: Optional[str] = None,
local_files_only: bool = False,
revision: Optional[str] = None,
) -> Tuple[Optional[str], bool]:
"""Returns definitive False only on a transient read failure, so callers retry instead of caching."""
read_any = False # parsed at least one tokenizer_config -> a None is definitive
# A declared audio_token whose encoder config could not be read keeps every negative unknown.
unresolved = False
# 1) Selected local directory or local HF cache first (works for gated/offline models)
try:
roots: list[Path] = []
if is_local_path(model_name):
local_path = Path(normalize_path(model_name)).expanduser()
if local_path.is_file():
local_path = local_path.parent
if local_path.is_dir():
roots.append(local_path)
else:
# Read before any network branch; never authorizes, so it would serve cached private repo tokens online as well as offline.
repo_dir = (
get_cache_path(model_name)
if cache_reads_authorized(hf_token, repo_id = model_name)
else None
)
if repo_dir is not None and repo_dir.is_dir():
snapshots_dir = repo_dir / "snapshots"
if snapshots_dir.is_dir() and revision is None:
roots.extend(
snapshot for snapshot in snapshots_dir.iterdir() if snapshot.is_dir()
)
elif snapshots_dir.is_dir():
snapshots_root = snapshots_dir.resolve(strict = False)
snapshot = (snapshots_dir / revision).resolve(strict = False)
try:
snapshot.relative_to(snapshots_root)
except ValueError:
snapshot = None
if snapshot is not None and snapshot.is_dir():
roots.append(snapshot)
current: list = [] # resolved lazily: only a negative answer needs it
# Only a read standing in for the Hub copy has to prove the file whole.
local_model = is_local_path(model_name)
may_answer_for_hub = not local_files_only and not local_model
for root in roots:
root_read: set = set()
for tok_path in _AUDIO_TOKENIZER_CONFIG_PATHS:
tok_file = root / tok_path
try:
if not tok_file.is_file():
continue
raw = tok_file.read_text(encoding = "utf-8-sig")
if not _may_hold_audio_tokens(raw):
# Answering for the Hub copy must parse: a half-written file ending in } would cache 'not audio'.
if may_answer_for_hub:
try:
decoded = json.loads(raw)
except Exception:
continue
# Raw scan misses escaped markers (Go's encoding/json writes them), so decode first.
result, known = _classify_audio_tokenizer(
decoded, lambda: _read_local_model_config(root, local_model)
)
if result:
return result, True
if not known:
unresolved = True
continue
elif not raw.rstrip().endswith("}"):
continue
read_any = True
root_read.add(tok_path)
continue
result, known = _classify_audio_tokenizer(
json.loads(raw), lambda: _read_local_model_config(root, local_model)
)
if result:
return result, True
if not known:
unresolved = True
continue
read_any = True
root_read.add(tok_path)
except Exception as e:
logger.debug(f"Could not read {tok_file} for {model_name}: {e}")
# Negatives are cached for the process, so only answer negatively after reading every path.
if root_read and not is_local_path(model_name):
if not current:
current.append(_current_cached_snapshot(model_name, hf_token, local_files_only))
snapshot = current[0]
if (
snapshot is not None
and root == snapshot[0]
and all(
path in root_read
for path in _AUDIO_TOKENIZER_CONFIG_PATHS
if path in snapshot[1]
)
):
return None, True
except Exception as e:
logger.debug(f"Could not check local cache for {model_name}: {e}")
# This raw requests.get ignores HF offline flags, so gate it explicitly.
if local_files_only or _env_offline():
return None, read_any and not unresolved
# A filesystem path is not a repo id, so no Hub URL; the local read is the whole answer, or /loras blocks the event loop on timeouts.
if is_local_path(model_name):
return None, read_any and not unresolved
try:
import requests
import os
except Exception:
return None, read_any and not unresolved
# `False` is falsy, so `or` would reach past it to the ambient credential.
token = None if is_anonymous(hf_token) else (hf_token or os.environ.get("HF_TOKEN"))
headers = {"Authorization": f"Bearer {token}"} if token else {}
transient = False # a fetch failed for a non-404 reason (network/5xx)
from urllib.parse import quote
revision_path = "main" if revision is None else quote(revision, safe = "")
from utils.hf_endpoint import get_hf_endpoint
hf_endpoint = get_hf_endpoint()
def _fetch_model_config() -> Optional[dict]:
url = f"{hf_endpoint}/{model_name}/resolve/{revision_path}/config.json"
try:
resp = requests.get(url, headers = headers, timeout = 15)
if resp.status_code == 404:
return {}
if not resp.ok:
return None
config = resp.json()
except Exception as e:
logger.debug(f"Could not fetch config.json for {model_name}: {e}")
return None
return config if isinstance(config, dict) else {}
for tok_path in _AUDIO_TOKENIZER_CONFIG_PATHS:
url = f"{hf_endpoint}/{model_name}/resolve/{revision_path}/{tok_path}"
try:
resp = requests.get(url, headers = headers, timeout = 15)
except Exception as e:
logger.debug(f"Could not fetch {tok_path} for {model_name}: {e}")
transient = True
continue
if resp.status_code == 404:
continue # genuinely absent on this path
if not resp.ok:
transient = True # 5xx/403/etc -- can't tell, don't cache
continue
try:
tok_config = resp.json()
except Exception as e:
logger.debug(f"Bad tokenizer_config for {model_name}/{tok_path}: {e}")
transient = True
continue
result, known = _classify_audio_tokenizer(tok_config, _fetch_model_config)
if result:
return result, True
if not known:
unresolved = True
continue
read_any = True
# No audio tokens: definitive unless every attempt failed transiently.
return None, (read_any or not transient) and not unresolved
def _is_mmproj(filename: str) -> bool:
"""Check if a GGUF filename is a vision projection (mmproj) file."""
return _gguf_names.is_mmproj_filename(filename)
_IMATRIX_TOKEN_RE = _gguf_names.IMATRIX_TOKEN_RE
def _is_imatrix_path(path: str) -> bool:
return _gguf_names.is_imatrix_filename(path)
_DRAFTER_KINDS = _gguf_names.DRAFTER_KINDS
_DRAFTER_DIR_KINDS = _gguf_names.DRAFTER_DIR_KINDS
def _is_mtp_drafter(path: str) -> bool:
return _gguf_names.is_mtp_drafter_path(path)
# Family tokens for #5347's filename fallback. Lowercase; order irrelevant.
_MODEL_FAMILY_TOKENS: tuple[str, ...] = (
"qwen",
"gemma",
"llama",
"mistral",
"ministral",
"magistral",
"devstral",
"phi",
"deepseek",
"internvl",
"minicpm",
"llava",
"glm",
"yi",
"command-r",
"molmo",
"pixtral",
"smolvlm",
"moondream",
"granite",
"ovis",
"nemotron",
"kimi",
"nanonets",
"cosmos",
"mimo",
"apriel",
"lfm",
)
# Word-bounded: stops ``phi`` matching ``sapphire`` or ``yi`` matching ``tiny``.
_FAMILY_TOKEN_RE_CACHE: Dict[str, "_re.Pattern[str]"] = {}
def _family_token_re(token: str) -> "_re.Pattern[str]":
pat = _FAMILY_TOKEN_RE_CACHE.get(token)
if pat is None:
pat = _re.compile(rf"(?:^|[^a-z])({_re.escape(token)})(?:[^a-z]|$)")
_FAMILY_TOKEN_RE_CACHE[token] = pat
return pat
def _detect_family_token(filename: str) -> Optional[str]:
"""Leftmost-position match; ties prefer the longer token."""
name = filename.lower()
best: Optional[tuple[int, int, str]] = None # (start, -len, token)
for token in _MODEL_FAMILY_TOKENS:
m = _family_token_re(token).search(name)
if m is None:
continue
key = (m.start(1), -len(token), token)
if best is None or key < best:
best = key
return None if best is None else best[2]
def mmproj_matches_model_family(model_path: str, mmproj_path: str) -> bool:
"""Prefer known functional metadata, falling back to filename family hints."""
functional_match, _ = mmproj_functional_match(model_path, mmproj_path)
if functional_match is not None:
return functional_match
model_fam = _detect_family_token(Path(model_path).name)
mmproj_fam = _detect_family_token(Path(mmproj_path).name)
if model_fam is None or mmproj_fam is None:
return True
return model_fam == mmproj_fam
# MiMo-VL's F16 projector overflows to NaN on GPU: every image, then the rest of the chat, answers "!!!!".
_F16_UNSAFE_MMPROJ_RE = _re.compile(r"mimo[-_ ]?vl", _re.IGNORECASE)
_MMPROJ_PRECISION_RE = _re.compile(r"[-_.](bf16|f16|f32)(?:[-_.]mmproj)?\.gguf$", _re.IGNORECASE)
_NAME_DELIMS = "-_. "
_MMPROJ_PRECISION_TAIL_RE = _re.compile(r"(?:[-_.](?:bf16|f16|f32))?(?:[-_.]mmproj)?$")
# None = a projector name with no precision tag.
MMPROJ_ORDER_BF16 = ("bf16", "f16", "f32", None)
MMPROJ_ORDER_NO_F16 = ("bf16", "f32", None, "f16")
def mmproj_f16_unsafe(name: Optional[str]) -> bool:
return bool(name) and _F16_UNSAFE_MMPROJ_RE.search(str(name)) is not None
@functools.lru_cache(maxsize = 1)
def device_mmproj_precision() -> str:
"""BF16 on ROCm and pre-Ampere NVIDIA, else F16 (Mac, Ampere and newer, CPU, Vulkan, unknown)."""
try:
if platform.system() == "Darwin":
return "f16"
from utils.hardware.hardware import _installed_llama_backend
backend = _installed_llama_backend()
if backend == "rocm":
return "bf16"
if backend == "cuda":
from core.inference.llama_cpp import LlamaCppBackend
caps = LlamaCppBackend._cuda_compute_caps()
if caps and min(caps.values()) < 80:
return "bf16"
except Exception as e:
logger.debug(f"device_mmproj_precision probe failed: {e}")
return "f16"
def mmproj_order(*model_names: Optional[str]) -> Optional[tuple]:
"""None keeps the F16-first default."""
if any(mmproj_f16_unsafe(n) for n in model_names):
return MMPROJ_ORDER_NO_F16
return MMPROJ_ORDER_BF16 if device_mmproj_precision() == "bf16" else None
def mmproj_precision(mmproj_name: str) -> Optional[str]:
m = _MMPROJ_PRECISION_RE.search(Path(mmproj_name).name)
return m.group(1).lower() if m else None
def _pairing_stem(name: str) -> str:
stem = _re.sub(r"^mmproj[-_]", "", Path(name).stem.lower())
return _MMPROJ_PRECISION_TAIL_RE.sub("", stem)
def mmproj_weight_match(weight: Optional[str], mmproj_name: str) -> int:
"""Whole-token prefix shared past ``mmproj-`` and before the precision tag (an F16 weight must
not pull the F16 projector); an exact checkpoint name beats a longer one (``Model-7B-v2``)."""
if not weight:
return 0
a, b = _pairing_stem(weight), _pairing_stem(mmproj_name)
k = _shared_prefix_len(a, b)
while k and not (
(k == len(a) or a[k] in _NAME_DELIMS) and (k == len(b) or b[k] in _NAME_DELIMS)
):
k -= 1
return 2 * k + (k == len(b))
def mmproj_order_rank(mmproj_name: str, order: tuple) -> int:
precision = mmproj_precision(mmproj_name)
return -order.index(precision) if precision in order else -len(order)
def _shared_prefix_len(a: str, b: str) -> int:
n = min(len(a), len(b))
for i in range(n):
if a[i] != b[i]:
return i
return n
def _is_gguf_filename(filename: str) -> bool:
return filename.lower().endswith(".gguf")
def _iter_gguf_files(directory: Path, recursive: bool = False):
if not directory.is_dir():
return
iterator = directory.rglob("*") if recursive else directory.iterdir()
for f in iterator:
if f.is_file() and _is_gguf_filename(f.name) and not is_appledouble_metadata(f):
yield f
_GGUF_SPLIT_FILE_RE = re.compile(
r"^(?P<prefix>.+)-(?P<index>\d{5})-of-(?P<total>\d{5})\.gguf$",
re.IGNORECASE,
)
def _colocated_first_split_shard(path: Path) -> tuple[Optional[Path], bool]:
"""Return shard 1 and whether every shard is beside *path*."""
match = _GGUF_SPLIT_FILE_RE.match(path.name)
if match is None:
return None, False
prefix = match.group("prefix").casefold()
total_text = match.group("total")
total = int(total_text)
if total < 1:
return None, False
first: Optional[Path] = None
indices: set[int] = set()
try:
siblings = path.parent.iterdir()
for sibling in siblings:
sibling_match = _GGUF_SPLIT_FILE_RE.match(sibling.name)
if (
sibling_match is None
or sibling_match.group("prefix").casefold() != prefix
or sibling_match.group("total") != total_text
):
continue
try:
if not sibling.is_file():
continue
except OSError:
continue
index = int(sibling_match.group("index"))
if not 1 <= index <= total:
continue
indices.add(index)
if index == 1:
first = sibling
except OSError:
return None, False
return first, first is not None and len(indices) == total
def colocated_split_shards(path: Path) -> tuple[list[Path], bool]:
"""Every sibling shard and whether the set is complete; llama-server opens siblings implicitly, so an incomplete set fails at startup; validate every shard."""
match = _GGUF_SPLIT_FILE_RE.match(path.name)
if match is None:
return [path], True
prefix = match.group("prefix").casefold()
total_text = match.group("total")
total = int(total_text)
if total < 1:
return [], False
found: dict[int, Path] = {}
try:
for sibling in path.parent.iterdir():
sibling_match = _GGUF_SPLIT_FILE_RE.match(sibling.name)
if (
sibling_match is None
or sibling_match.group("prefix").casefold() != prefix
or sibling_match.group("total") != total_text
):
continue
try:
if not sibling.is_file():
continue
except OSError:
continue
index = int(sibling_match.group("index"))
if 1 <= index <= total:
found[index] = sibling
except OSError:
return [], False
return [found[i] for i in sorted(found)], len(found) == total
def _local_gguf_load_path(path: Path) -> Path:
"""Choose a loadable local path while preserving complete symlink sets."""
if _GGUF_SPLIT_FILE_RE.match(path.name) is None:
return path.absolute()
first, complete = _colocated_first_split_shard(path)
if complete and first is not None:
return first.absolute()
try:
is_symlink = path.is_symlink()
except OSError:
is_symlink = False
if is_symlink:
try:
target = path.resolve()
except OSError:
return (first or path).absolute()
target_first, _ = _colocated_first_split_shard(target)
return (target_first or target).absolute()
return (first or path).absolute()
def detect_mmproj_file(
path: str,
search_root: Optional[str] = None,
allow_disjoint_search_root: bool = False,
accept: Optional[Callable[[str], bool]] = None,
) -> Optional[str]:
"""Finds the mmproj GGUF; accept runs before projector metadata is read, so authorization comes first; allow_disjoint_search_root is trusted-cache only."""
p = Path(path)
start_dir = p.parent if p.is_file() else p
if not start_dir.is_dir():
return None
# Walk incrementally so a sibling subdir's mmproj cannot leak in.
seen: set[Path] = set()
scan_order: list[Path] = []
recursive_root: Optional[Path] = None
def _add(d: Path) -> None:
try:
resolved = d.resolve()
except OSError:
return
if resolved in seen or not resolved.is_dir():
return
seen.add(resolved)
scan_order.append(resolved)
_add(start_dir)
# Hermes stages projectors under models/assets/; the weight sits one level up.
_add(start_dir / "assets")
# Ollama's .studio_links/foo.gguf -> blobs/sha256-...: also scan target dir.
try:
if p.is_symlink() and p.is_file():
target_parent = p.resolve().parent
if target_parent.is_dir():
_add(target_parent)
except OSError:
pass
if search_root is not None:
try:
root_resolved = Path(search_root).resolve()
start_resolved = start_dir.resolve()
root_contains_start = root_resolved == start_resolved or (
start_resolved.is_relative_to(root_resolved)
if hasattr(start_resolved, "is_relative_to")
else str(start_resolved).startswith(str(root_resolved) + "/")
)
if root_contains_start:
cur = start_resolved
while cur != root_resolved and cur.parent != cur:
cur = cur.parent
_add(cur)
if cur == root_resolved:
break
elif allow_disjoint_search_root:
_add(root_resolved)
# Only for a root not holding the weights: rglob there would pick a sibling quant's projector.
if allow_disjoint_search_root and not root_contains_start:
recursive_root = root_resolved
except OSError:
pass
candidates: list[Path] = []
seen_resolved: set[Path] = set()
for d in scan_order:
try:
files = list(_iter_gguf_files(d, recursive = d == recursive_root))
except OSError:
continue
for f in files:
if accept is not None and not accept(str(f)):
continue
try:
resolved = f.resolve()
# Interrupted download: llama-server can't open it and it must not shadow a real projector.
if resolved.stat().st_size <= 0:
continue
except OSError:
continue
if resolved in seen_resolved:
continue
# Prefer ``general.type=='mmproj'``, else filename.
meta = read_gguf_general_metadata(str(resolved))
by_meta = is_mmproj_by_metadata(meta)
if by_meta is True or (by_meta is None and _is_mmproj(f.name)):
seen_resolved.add(resolved)
candidates.append(resolved)
if not candidates:
return None
# Directory path: no model name to compare against; legacy behaviour.
if not p.is_file():
return str(candidates[0])
# Stage 1: GGUF metadata. Stage 2: filename family token (#5347).
model_stem = p.stem.lower()
model_family = _detect_family_token(p.name)
weight_meta = read_gguf_general_metadata(str(p))
scored: list[tuple[int, Path]] = []
for c in candidates:
cand_meta = read_gguf_general_metadata(str(c))
meta_score = pairing_score(weight_meta, cand_meta)
functional_match, mismatch = mmproj_functional_match(str(p), str(c))
if functional_match is False:
logger.info(f"detect_mmproj_file: dropped {c.name} ({mismatch})")
continue
if functional_match is True:
meta_score = max(meta_score, 50)
if meta_score == -1:
differences = "; ".join(
f"{key} {weight_meta.get(key)!r} != {cand_meta.get(key)!r}"
for key in (
"general.base_model.0.repo_url",
"general.basename",
"general.base_model.0.organization",
"general.organization",
)
if weight_meta.get(key) != cand_meta.get(key)
)
logger.info(f"detect_mmproj_file: dropped {c.name} (metadata mismatch: {differences})")
continue
if meta_score == 0 and model_family is not None:
# Unrecognised candidate family is a wildcard (``mmproj-F16.gguf``).
cand_family = _detect_family_token(c.name)
if cand_family is not None and cand_family != model_family:
logger.info(
f"detect_mmproj_file: dropped {c.name} "
f"(filename family {cand_family!r} vs model {model_family!r})"
)
continue
scored.append((meta_score, c))
if not scored:
return None
# Score first, then longest shared prefix, then shorter stem. The prefix is read past
# the ``mmproj-`` marker, or every projector in a shared pool ties at zero.
order = mmproj_order(str(p))
best = max(
scored,
key = lambda sc: (
sc[0],
mmproj_weight_match(p.name, sc[1].name)
if order
else _shared_prefix_len(model_stem, _re.sub(r"^mmproj[-_]", "", sc[1].stem.lower())),
mmproj_order_rank(sc[1].name, order) if order else 0,
-len(sc[1].stem),
),
)
return str(best[1])
# Imported by name, not re-exported: verify_import_hoist.py blocks re-exports outside __init__.
from utils.models.drafters import ( # noqa: E402
_drafter_launch_path,
_drafter_matches_weight,
_drafter_split_is_complete,
_drafter_stem_rank,
_drafter_total_size,
detect_dflash_file,
dspark_precision_rank,
is_published_drafter_filename,
)
from utils.models.drafters import ( # noqa: E402
dspark_preference_key as _drafters_dspark_preference_key,
)
from utils.paths.path_utils import is_appledouble_metadata
def dspark_preference_key(name: str) -> Tuple[int, str]:
"""Delegates instead of re-exporting: repointing routes/inference.py's import trips the lint rule."""
return _drafters_dspark_preference_key(name)
def detect_mtp_file(
path: str,
search_root: Optional[str] = None,
skip_root: bool = False,
accept: Optional[Callable[[str], bool]] = None,
) -> Optional[str]:
"""Pairs by name so a multi-model folder cannot attach a foreign drafter; MTP/ is a fallback; skip_root scans only MTP/; Qwen-style baked heads give None."""
if Path(path).is_file() and (read_gguf_nextn_predict_layers(path) or 0) > 0:
# A root mtp-*.gguf may mirror an embedded head; -md would replace it.
return None
def _matches_weight(candidate: Path) -> bool:
return _drafter_matches_weight(candidate.name, weight_name, kind = "mtp")
def _launchable(candidate: Path) -> bool:
return _drafter_split_is_complete(candidate)
def _smallest_first(candidate: Path) -> tuple[int, int, int, int, str]:
# Same order as the hub picker (mtp_preference_key). Borrowing heads sit after precision: --fit cannot measure them, so the MTP context OOMs.
from utils.models.drafters.preference import mtp_precision_rank
name = candidate.name.lower()
borrows = 1 if "-shared-" in name else 0
# Family first: base-family and release sidecars both prefix-match; the longer one wins.
return (
_drafter_stem_rank(candidate.name, kind = "mtp"),
mtp_precision_rank(name),
borrows,
_drafter_total_size(candidate),
name,
)
p = Path(path)
weight_name = p.name.lower() if p.suffix.lower() == ".gguf" else None
start_dir = p.parent if p.is_file() else p
# Hermes stages a download's drafter under models/assets/, like its projector.
dirs = [start_dir, start_dir / "assets"]
if search_root is not None:
dirs.append(Path(search_root))
# Collect both tiers before choosing: several sidecars can prefix-match one weight.
root_candidates: list[Path] = []
if not skip_root:
for d in dirs:
try:
entries = sorted(d.iterdir())
except OSError:
continue
for f in entries:
if not is_published_drafter_filename(f.name, kind = "mtp", allow_legacy_suffix = False):
continue
if not _matches_weight(f):
continue
# Filter unlaunchable candidates before ranking, or one could win and then be skipped.
try:
if not (f.is_file() and _launchable(f)):
continue
except OSError:
continue
root_candidates.append(f)
subdir_candidates: list[Path] = []
for d in dirs:
try:
parent_entries = sorted(d.iterdir())
except OSError:
continue
mtp_dirs: list[Path] = []
for entry in parent_entries:
if entry.name.casefold() != "mtp":
continue
try:
if entry.is_dir():
mtp_dirs.append(entry)
except OSError:
continue
for mtp_dir in mtp_dirs:
try:
entries = sorted(mtp_dir.iterdir())
except OSError:
continue
for f in entries:
# Require a published drafter name: a weight copy under MTP/ would launch as --model-draft.
if not is_published_drafter_filename(f.name, kind = "mtp"):
continue
if not _matches_weight(f):
continue
try:
if f.is_file() and _launchable(f):
# Collapse a split copy to shard 1 before ranking.
subdir_candidates.append(_local_gguf_load_path(f))
except OSError:
continue
def _first_pick(candidates: list[Path]) -> Optional[tuple[int, str]]:
"""A tier is represented by its first candidate that survives every filter, not its best name."""
for c in candidates:
try:
launch = _drafter_launch_path(c)
except OSError:
continue
if accept is not None and not accept(launch):
continue
return _drafter_stem_rank(c.name, kind = "mtp"), launch
return None
# Ranking is stable, so files of equal specificity keep the scan order.
root_pick = _first_pick(
sorted(root_candidates, key = lambda c: _drafter_stem_rank(c.name, kind = "mtp"))
)
subdir_pick = _first_pick(sorted(dict.fromkeys(subdir_candidates), key = _smallest_first))
# Root wins ties; loses only to a strictly more specific subdir copy.
if subdir_pick is not None and (root_pick is None or subdir_pick[0] < root_pick[0]):
logger.info(f"Detected MTP subdirectory drafter: {subdir_pick[1]}")
return subdir_pick[1]
if root_pick is not None:
return root_pick[1]
return None
def detect_dspark_file(
path: str,
search_root: Optional[str] = None,
accept: Optional[Callable[[str], bool]] = None,
) -> Optional[str]:
"""Requires a published drafter name even in dspark/, so a parked weight never loads as --model-draft."""
def _rank(candidate: Path) -> tuple[int, int, int, str]:
# Most specific family, then precision, then total size, then name.
return (
_drafter_stem_rank(candidate.name, kind = "dspark"),
dspark_precision_rank(candidate.name),
_drafter_total_size(candidate),
candidate.name.lower(),
)
p = Path(path)
weight_name = p.name if p.suffix.lower() == ".gguf" else None
start_dir = p.parent if p.is_file() else p
# Hermes stages a download's drafter under models/assets/, like its projector.
dirs = [start_dir, start_dir / "assets"]
if search_root is not None:
dirs.append(Path(search_root))
candidates: list[Path] = []
seen: set[Path] = set()
# dict.fromkeys dedupes roots; a flat layout's search root is the weight's parent, scanned once.
for root in dict.fromkeys(dirs):
try:
root_entries = list(root.iterdir())
except OSError:
continue
# One listing per directory, partitioned into the dspark/ subdir and the sidecars beside the weight.
scan: list[tuple[Path, list[Path]]] = [(root, root_entries)]
for entry in root_entries:
if entry.name.casefold() != "dspark":
continue
try:
if entry.is_dir():
scan.append((entry, list(entry.iterdir())))
except OSError:
continue
for _directory, entries in scan:
for candidate in entries:
lower = candidate.name.lower()
if not lower.endswith(".gguf"):
continue
# Drop the shard suffix first: old split names do not end in -dspark.
stem = re.sub(r"-[0-9]{5}-of-[0-9]{5}$", "", Path(lower).stem)
if not (lower.startswith("dspark-") or stem.endswith("-dspark")):
continue
if not _drafter_matches_weight(candidate.name, weight_name, kind = "dspark"):
continue
try:
# Collapse a split copy to shard 1 before ranking.
launch = _local_gguf_load_path(candidate)
# is_file() follows links: drops dangling symlinks and dirs that would fail the whole load.
if not (launch.is_file() and _drafter_split_is_complete(launch)):
continue
resolved = launch.resolve()
except OSError:
continue
if resolved in seen:
continue
seen.add(resolved)
candidates.append(launch)
for candidate in sorted(candidates, key = _rank):
try:
launch = _drafter_launch_path(candidate)
except OSError:
continue
if accept is not None and not accept(launch):
continue
logger.info("Detected DSpark drafter: %s", launch)
return launch
return None
def _registered_custom_model_root(path: str) -> Optional[Path]:
try:
from storage.studio_db import list_scan_folders
from utils.paths import studio_db_path
from utils.scan_memo import memoized
folders = memoized(("scan_folders", str(studio_db_path())), list_scan_folders)
except Exception:
return None
candidate = Path(os.path.abspath(Path(path).expanduser()))
matches = []
for folder in folders:
raw = folder.get("path")
if not isinstance(raw, str) or not raw:
continue
root = Path(os.path.abspath(Path(normalize_path(raw)).expanduser()))
try:
candidate.relative_to(root)
except ValueError:
continue
matches.append(root)
return max(matches, key = lambda root: len(root.parts), default = None)
def _local_mtp_context(path: Path, model_root: Optional[Path], fallback: str) -> str:
if model_root is not None:
try:
return Path(os.path.abspath(path)).relative_to(model_root).as_posix()
except ValueError:
pass
return fallback
def _is_terminal_gemma_mtp(name: str) -> bool:
stem = re.sub(r"-\d{3,}-of-\d{3,}$", "", Path(name).stem.lower())
return stem.endswith("-mtp") and bool(re.search(r"(?:^|[-_])gemma[-_]?4(?:[-_]|$)", stem))
def _is_local_mtp_drafter(path: Path, model_root: Optional[Path], fallback: str) -> bool:
context = _local_mtp_context(path, model_root, fallback)
if _is_mtp_drafter(context):
return True
return (
model_root is not None
and model_root.name.lower() == "mtp"
and "/" not in context
and _is_terminal_gemma_mtp(path.name)
)
def detect_gguf_model(path: str, model_root: Optional[str] = None) -> Optional[str]:
"""Skips mmproj projectors (see detect_mmproj_file); for HF repo ids use detect_gguf_model_remote()."""
p = Path(path)
root = (
Path(os.path.abspath(Path(model_root).expanduser()))
if model_root is not None
else _registered_custom_model_root(path)
)
# Case 1: direct .gguf file
if p.suffix.lower() == ".gguf":
# Include the parent dir so MTP/ subdir copies are caught; basename misses the mtp- prefix.
rel = f"{p.parent.name}/{p.name}"
quant = _extract_quant_label(rel)
if (
_is_mmproj(p.name)
or _is_local_mtp_drafter(p, root, rel)
or _is_imatrix_path(p.name)
or _is_big_endian_gguf_path(rel, quant)
# Identified positively so an unreadable file still reaches the lock window below.
or is_appledouble_metadata(p)
):
return None
# Extension is authoritative: is_file()/exists() can fail in the Windows lock window.
try:
is_dir = p.is_dir()
except OSError:
is_dir = False # stat() unavailable in the lock window
if not is_dir:
return str(_local_gguf_load_path(p))
# A dir named *.gguf falls through to the dir scan, which skips mmproj and MTP drafter .gguf files.
if p.is_dir():
gguf_files = []
for f in _iter_gguf_files(p):
context_rel = f"{f.parent.name}/{f.name}"
quant = _extract_quant_label(context_rel)
if (
_is_mmproj(f.name)
or _is_local_mtp_drafter(f, root, context_rel)
or _is_imatrix_path(f.name)
or _is_big_endian_gguf_path(context_rel, quant)
):
continue
gguf_files.append(f)
gguf_files.sort(key = lambda f: f.stat().st_size, reverse = True)
# A torn split's lone shard can sort largest but cannot load; prefer complete ones.
def _split_is_whole(f: Path) -> bool:
if _GGUF_SPLIT_FILE_RE.match(f.name) is None:
return True
if _colocated_first_split_shard(f)[1]:
return True
# Follow a shard symlink to the target's set; the alias's dir proves nothing.
try:
if not f.is_symlink():
return False
target = f.resolve()
except OSError:
return True
return (
_GGUF_SPLIT_FILE_RE.match(target.name) is None
or _colocated_first_split_shard(target)[1]
)
whole = [f for f in gguf_files if _split_is_whole(f)]
if whole:
gguf_files = whole
if gguf_files:
return str(_local_gguf_load_path(gguf_files[0]))
return None
def _pick_best_gguf(filenames: list[str]) -> Optional[str]:
"""Pick the best GGUF file by the model picker's ranking, so a bare repo id loads its default."""
from hub.utils.gguf import pick_best_gguf
return pick_best_gguf(filenames)
@dataclass
class GgufVariantInfo:
"""A single GGUF quantization variant from a HuggingFace repo."""
filename: str # e.g., "gemma-3-4b-it-Q4_K_M.gguf"
quant: str # e.g., "Q4_K_M" (extracted from filename)
size_bytes: int # file size
def _extract_quant_label(filename: str) -> str:
"""Quant label such as Q4_K_M or UD-IQ1_S from a GGUF filename, keeping any trailing bpw modifier."""
import re
basename = filename.rsplit("/", 1)[-1]
# Strip .gguf and any shard suffix (-00001-of-00010)
stem = re.sub(r"-\d{3,}-of-\d{3,}", "", basename.rsplit(".", 1)[0])
quant_re = (
r"(UD-)?" # Optional UD- prefix (Ultra Discrete)
r"(MXFP[0-9]+(?:_[A-Z0-9]+)*" # MXFP variants: MXFP4, MXFP4_MOE
r"|NVFP[0-9]+(?:_[A-Z0-9]+)*" # NVFP4, last in the name only
r"(?=(?:-[0-9]+(?:[.][0-9]+)?bpw)?(?:-[0-9]{3,}-of-[0-9]{3,})?(?:[.]gguf)?$)"
r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?" # IQ variants: IQ4_XS, IQ4_NL, IQ1_S
r"|P?TQ[0-9]+_[0-9]+" # Ternary quant: TQ1_0, TQ2_0
r"|Q[0-9]+_K_[A-Z]+" # K-quant: Q4_K_M, Q3_K_S
r"|P?Q[0-9]+_(?:[0-9]+(?:_G[0-9]+)?|G[0-9]+)" # Standard/Packed/Grouped: Q8_0, PQ2_0, Q2_0_g64, Q2_g64
r"|Q[0-9]+_K" # Short K-quant: Q6_K
r"|BF16|F16|F32)" # Full precision
# Optional bpw modifier so several builds of one base quant stay distinct.
r"(-[0-9]+(?:\.[0-9]+)?bpw)?"
)
match = re.search(quant_re, stem, re.IGNORECASE)
# Subdir layouts like BF16/foo.gguf keep the quant in the directory.
if not match and "/" in filename:
parents = filename.rsplit("/", 1)[0]
for segment in reversed(parents.split("/")):
m = re.search(quant_re, segment, re.IGNORECASE)
if m:
match = m
break
if match:
prefix = match.group(1) or ""
bpw = match.group(3) or ""
return f"{prefix}{match.group(2)}{bpw}"
# Fallback: last hyphen-separated segment
return stem.split("-")[-1]
_BIG_ENDIAN_GGUF_FILENAME_RE = re.compile(r"(^|[-_])be(?:[._-]|$)", re.IGNORECASE)
_GGUF_KNOWN_QUANT_RE = _gguf_names.GGUF_QUANT_RE
_DSPARK_BEFORE_QUANT_RE = _gguf_names.DSPARK_BEFORE_QUANT_RE
_FLOAT_PRECISION_QUANTS = frozenset({"BF16", "F16", "F32"})
_GGUF_SPLIT_SUFFIX_RE = re.compile(r"-\d{3,}-of-(\d{3,})", re.IGNORECASE)
def _select_known_quant_match(text: str):
fallback = None
for match in _GGUF_KNOWN_QUANT_RE.finditer(text):
if match.group(2).upper() in _FLOAT_PRECISION_QUANTS:
if fallback is None:
fallback = match
continue
return match
return fallback
def _gguf_variant_stem(filename: str) -> str:
basename = filename.rsplit("/", 1)[-1]
return _GGUF_SPLIT_SUFFIX_RE.sub("", basename.rsplit(".", 1)[0]).strip()
def _quant_search_stem(filename: str) -> str:
"""Mirrors hub.utils.gguf; keeps text after the last dot, so IQ4_XS-3.53bpw is not cut to IQ4_XS-3."""
return _GGUF_SPLIT_SUFFIX_RE.sub("", filename.rsplit("/", 1)[-1]).strip()
def _locate_quant_match(filename: str):
"""Mirrors hub.utils.gguf; the basename decides, and parent dirs count only if it names no quant."""
stem = _quant_search_stem(filename)
match = _select_known_quant_match(stem)
if match:
return match, stem
if "/" in filename:
for segment in reversed(filename.rsplit("/", 1)[0].split("/")):
parent_match = _select_known_quant_match(segment)
if parent_match:
return parent_match, segment
return None, ""
def _gguf_variant_token(filename: str) -> Optional[str]:
match, _ = _locate_quant_match(filename)
return f"{match.group(1) or ''}{match.group(2)}" if match else None
def _gguf_variant_family(filename: str) -> str:
stem = _gguf_variant_stem(filename)
if "/" not in filename:
return stem or "gguf"
parents = filename.rsplit("/", 1)[0].strip("/")
return f"{parents}/{stem}" if parents and stem else stem or "gguf"
# MIRROR of ``hub.utils.gguf.gguf_shard_set``.
def _gguf_shard_set(filename: str) -> tuple[str, int]:
split = _GGUF_SPLIT_SUFFIX_RE.search(filename.rsplit("/", 1)[-1])
return _gguf_variant_family(filename), int(split.group(1)) if split else 0
# MIRROR of hub.utils.gguf._GGUF_BPW_SUFFIX_RE. Use match, not search: only a modifier right
# after the quant qualifies it.
_GGUF_BPW_SUFFIX_RE = re.compile(r"-[0-9]+(?:\.[0-9]+)?bpw", re.IGNORECASE)
# MIRROR of hub.utils.gguf._GGUF_BPW_TRAILING_RE, for basenames under a quant directory.
_GGUF_BPW_TRAILING_RE = re.compile(r"-[0-9]+(?:\.[0-9]+)?bpw(?=\.[A-Za-z0-9]+$|$)", re.IGNORECASE)
def _quant_token_with_bpw(filename: str) -> Optional[str]:
"""Mirrors hub.utils.gguf.quant_token_with_bpw; the bpw modifier is searched where the token was."""
path = filename.replace("\\", "/")
match, text = _locate_quant_match(path)
if match is None:
return None
token = f"{match.group(1) or ''}{match.group(2)}"
adjacent = _GGUF_BPW_SUFFIX_RE.match(text[match.end() :])
if adjacent:
return f"{token}{adjacent.group(0)}"
stem = _quant_search_stem(path)
if text != stem:
trailing = _GGUF_BPW_TRAILING_RE.search(stem)
if trailing:
return f"{token}{trailing.group(0)}"
return token
def _gguf_variant_key(filename: str) -> str:
"""Mirrors hub.utils.gguf.gguf_variant_key; must agree with it, as test_gguf_variant_rows.py checks."""
path = filename.replace("\\", "/")
# The bpw modifier stays on, so two builds of one base quant keep two identities.
quant = _quant_token_with_bpw(path)
if quant is None:
return _gguf_variant_family(path)
if path.rsplit("/", 1)[-1].lower().startswith(("minimax_h3_fl2va", "minimax_h3_ref2va")):
return _gguf_variant_family(path)
for segment in path.rpartition("/")[0].split("/"):
# A directory naming something other than a quant is a different checkpoint and qualifies.
if segment and _select_known_quant_match(segment) is None:
return _gguf_variant_family(path)
return quant
def _qualified_variant_name(filename: str, label: str) -> str:
"""Bare label unless a non-quant folder qualifies it, since the label alone can name several checkpoints; changing the label would rename rows."""
if _gguf_variant_token(filename) is None:
return label
key = _gguf_variant_key(filename)
# Compare against the plain key, not a slash: H3 root partitions qualify by full stem.
plain = _quant_token_with_bpw(filename)
return key if plain is not None and key.lower() != plain.lower() else label
def _is_big_endian_gguf_path(path: str, quant: str = "") -> bool:
normalized = path.replace("\\", "/")
name = normalized.rsplit("/", 1)[-1]
stem = name.rsplit(".", 1)[0].lower()
quant_key = quant.strip().lower()
quant_index = stem.find(quant_key) if quant_key else -1
parent = normalized.rsplit("/", 1)[0].lower() if "/" in normalized else ""
quant_in_parent_only = (
bool(parent)
and quant_index < 0
and (
(quant_key and quant_key in parent)
or (not quant_key and _GGUF_KNOWN_QUANT_RE.search(parent) is not None)
)
)
for match in _BIG_ENDIAN_GGUF_FILENAME_RE.finditer(stem):
if quant_index >= 0 and quant_index < match.start():
return True
tail = stem[match.end() :].lstrip("._-")
if not tail or _GGUF_KNOWN_QUANT_RE.search(tail) is None:
return not quant_in_parent_only
return False
def _local_gguf_companion_search_root(selected_path: str, gguf_file: str) -> str:
"""Directory to scan upward from for local GGUF companion files."""
import re
selected = Path(selected_path)
gguf_path = Path(gguf_file)
# One shared quant vocabulary: a local copy fell behind on bpw, hiding IQ4_XS-3.53bpw dirs.
quant_dir_re = rf"{_GGUF_KNOWN_QUANT_RE.pattern}(-[0-9]+(?:\.[0-9]+)?bpw)?"
search_dir = gguf_path.parent if selected.suffix.lower() == ".gguf" else selected
if not search_dir.name:
return str(search_dir)
if re.fullmatch(quant_dir_re, search_dir.name, re.IGNORECASE):
return str(search_dir.parent)
return str(search_dir)
def _hf_cache_repo_dir(weight_path: str) -> Optional[str]:
"""Cache repo dir for *weight_path*, never a sibling repo's; name matching is case-insensitive."""
for directory in Path(weight_path).parents:
parent = directory.parent
if parent.name.casefold() == "snapshots" and parent.parent.name.casefold().startswith(
"models--"
):
return str(parent.parent)
return None
def _hf_cached_local_mmproj(weight_path: str) -> Optional[str]:
"""A hand-added projector in *weight_path*'s snapshot, ``snapshots/`` or
``models--<repo>/``, or None (#9286). Metadata pairing decides between them."""
return detect_mmproj_file(weight_path, search_root = _hf_cache_repo_dir(weight_path))
def _snapshot_selection_key(snapshot: Path) -> tuple[float, str]:
"""Mirrors hub.utils.hf_cache_state.snapshot_selection_key and must change in lockstep; mtime alone is not total, so ties break on resolved path."""
try:
mtime = snapshot.stat().st_mtime
except OSError:
mtime = 0.0
try:
return mtime, str(snapshot.resolve())
except (OSError, RuntimeError, ValueError):
return mtime, str(snapshot)
def _iter_hf_cache_snapshots(repo_id: str, cache_dir: Optional[str | Path] = None):
"""Yields snapshot dirs newest first; the repo name matches case-insensitively, since casing drifts."""
if cache_dir is None:
try:
from utils.hf_cache_settings import get_hf_cache_paths
cache_dir = get_hf_cache_paths().hub_cache
except Exception:
return
cache_dir = Path(cache_dir)
target = f"models--{repo_id.replace('/', '--')}".lower()
repo_dirs: list[Path] = []
try:
if not cache_dir.is_dir():
return
for entry in cache_dir.iterdir():
if entry.is_dir() and entry.name.lower() == target:
repo_dirs.append(entry)
except OSError:
return
if not repo_dirs:
return
snap_dirs: list[Path] = []
for repo_dir in repo_dirs:
snapshots = repo_dir / "snapshots"
try:
if snapshots.is_dir():
for snap_dir in snapshots.iterdir():
try:
if snap_dir.is_dir():
snap_dirs.append(snap_dir)
except OSError:
continue
except OSError:
continue
if not snap_dirs:
return
ordered = []
for snap_dir in snap_dirs:
try:
snap_dir.stat()
except OSError:
continue
ordered.append((_snapshot_selection_key(snap_dir), snap_dir))
ordered.sort(key = lambda item: item[0], reverse = True)
yield from (snap_dir for _key, snap_dir in ordered)
def _list_gguf_variants_from_hf_cache(repo_id: str) -> Optional[tuple[list[GgufVariantInfo], bool]]:
"""Keeps scanning older snapshots when the newest has only a projector, so quants are not hidden."""
any_vision = False
for snap in _iter_hf_cache_snapshots(repo_id):
variants, has_vision = list_local_gguf_variants(str(snap))
any_vision = any_vision or has_vision
if variants:
return variants, any_vision
if any_vision:
return [], True
return None
def list_gguf_variants(
repo_id: str, hf_token: Optional[str] = None
) -> tuple[list[GgufVariantInfo], bool]:
"""Returns (variants, has_vision); the mmproj projector in the repo is what flags vision."""
if _env_offline():
cached = _list_gguf_variants_from_hf_cache(repo_id)
return cached if cached is not None else ([], False)
try:
info = _hub_model_info_slow_link(repo_id, hf_token, files_metadata = True)
except Exception as e:
# Permanent errors must surface; a stale cache would mask them. Matches detect_gguf_model_remote.
if type(e).__name__ in (
"RepositoryNotFoundError",
"GatedRepoError",
"RevisionNotFoundError",
"EntryNotFoundError",
):
raise
# API failed transiently; fall back to local snapshot if fully downloaded.
cached = _list_gguf_variants_from_hf_cache(repo_id)
if cached is not None:
logger.warning(
"HF API unreachable for %s (%s); using local cache snapshot.",
repo_id,
e.__class__.__name__,
)
return cached
raise
variants: list[GgufVariantInfo] = []
has_vision = False
# (name, quant, size); grouped per shard FAMILY below, not summed across copies.
main_files: list[tuple[str, str, int]] = []
from hub.utils.gguf import drop_shadowed_appledouble_siblings
# Sidecars in the listing would otherwise be advertised as the variant, at their own size.
for sibling in drop_shadowed_appledouble_siblings(list(info.siblings)):
fname = sibling.rfilename
if not fname.lower().endswith(".gguf"):
continue
size = sibling.size or 0
# mmproj files are vision projections, not main model files
if "mmproj" in fname.lower():
has_vision = True
continue
# MTP drafters are speculative-decoding companions, not quants.
if _is_mtp_drafter(fname):
continue
# An imatrix is calibration data, not a quant.
if _is_imatrix_path(fname):
continue
label = _extract_quant_label(fname)
if _is_big_endian_gguf_path(fname, label):
continue
main_files.append((fname, _qualified_variant_name(fname, label), size))
for quant, (first_file, total_size) in _group_gguf_variant_files(main_files).items():
variants.append(
GgufVariantInfo(
filename = first_file,
quant = quant,
size_bytes = total_size,
)
)
# largest first; pinning and OOM demotion happen client-side with VRAM data.
variants.sort(key = lambda v: -v.size_bytes)
return variants, has_vision
def _group_gguf_variant_files(entries: list[tuple[str, str, int]]) -> dict[str, tuple[str, int]]:
"""mirror ``hub.utils.gguf.group_gguf_variant_files`` for consistent VRAM accounting."""
families: dict[str, dict[tuple[str, int], list[tuple[str, str, int]]]] = {}
for name, quant, size in entries:
families.setdefault(quant.casefold(), {}).setdefault(_gguf_shard_set(name), []).append(
(name, quant, int(size or 0))
)
grouped: dict[str, tuple[str, int]] = {}
for by_family in families.values():
chosen = min(by_family.values(), key = lambda members: min(n for n, _q, _s in members))
first_name, first_quant, _ = min(chosen, key = lambda member: member[0])
grouped[first_quant] = (first_name, sum(size for _name, _quant, size in chosen))
return grouped
def _resolve_gguf_dir(p: Path) -> Optional[Path]:
"""resolve a GGUF directory; reject loose files without metadata to avoid mixing models."""
if p.is_dir():
return p
if p.is_file() and p.suffix.lower() == ".gguf":
parent = p.parent
if (
(parent / "config.json").exists()
or (parent / "adapter_config.json").exists()
or (parent / "export_metadata.json").exists()
):
return parent
return None
def list_local_gguf_variants(
directory: str, model_root: Optional[str] = None
) -> tuple[list[GgufVariantInfo], bool]:
"""Returns (variants, has_vision); shard sizes sum per quant label, so a split GGUF is one variant."""
p = _resolve_gguf_dir(Path(directory))
if p is None:
return [], False
root = (
Path(os.path.abspath(Path(model_root).expanduser()))
if model_root is not None
else _registered_custom_model_root(directory)
)
main_files: list[tuple[str, str, int]] = []
has_vision = False
# Keep the relative subpath so _find_local_gguf_by_variant can locate the file again.
for f in sorted(_iter_gguf_files(p, recursive = True)):
if _is_imatrix_path(f.name):
continue
if _is_mmproj(f.name):
has_vision = True
continue
try:
size = f.stat().st_size
except OSError:
size = 0
# Relative path so ``BF16/foo.gguf`` and ``Q4_K_M/foo.gguf`` get distinct quant labels.
rel = f.relative_to(p).as_posix()
if _is_local_mtp_drafter(f, root, rel):
continue
label = _extract_quant_label(rel)
if _is_big_endian_gguf_path(rel, label):
continue
main_files.append((rel, _qualified_variant_name(rel, label), size))
variants = [
GgufVariantInfo(
filename = first_file,
quant = quant,
size_bytes = total_size,
)
for quant, (first_file, total_size) in _group_gguf_variant_files(main_files).items()
]
variants.sort(key = lambda v: -v.size_bytes)
return variants, has_vision
def _variant_matches(variant: str, *labels: str) -> bool:
"""Case-insensitive like llama.cpp's _gguf_files_for_variant, so local and hub paths resolve alike."""
wanted = str(variant).casefold()
return any(wanted == str(label).casefold() for label in labels)
def _direct_gguf_for_variant(
path: str,
variant: str,
model_root: Optional[str] = None,
) -> Optional[str]:
"""resolve a ``.gguf`` by case-insensitive filename or quant labels to match load gating."""
f = Path(path).expanduser()
if f.suffix.lower() != ".gguf" or not f.is_file():
return None
context = f"{f.parent.name}/{f.name}"
quant = _extract_quant_label(context)
fallback_variant = re.sub(r"-\d{3,}-of-\d{3,}$", "", f.name.rsplit(".", 1)[0])
# the hub extractor drops the -bpw modifier, so accept its shorter advertised label.
stripped = re.sub(r"-[0-9]+(?:\.[0-9]+)?bpw$", "", quant, flags = re.IGNORECASE)
# accept any token because the hub extractor prefers K-quants in composite filenames.
stem_tokens = [
f"{m.group(1) or ''}{m.group(2)}"
for m in _GGUF_KNOWN_QUANT_RE.finditer(fallback_variant.rsplit("/", 1)[-1])
]
if not _variant_matches(variant, f.name, quant, fallback_variant, stripped, *stem_tokens):
return None
return detect_gguf_model(str(f), model_root)
def _find_local_gguf_by_variant(
directory: str,
variant: str,
model_root: Optional[str] = None,
) -> Optional[str]:
"""match a GGUF quant, stem, or relative filename and return llama-server's first shard."""
p = _resolve_gguf_dir(Path(directory))
if p is None:
return _direct_gguf_for_variant(directory, variant, model_root)
root = (
Path(os.path.abspath(Path(model_root).expanduser()))
if model_root is not None
else _registered_custom_model_root(directory)
)
# preserve Audio variant paths during recursive search to prevent basename cross-matches
literal_selector = str(variant)
path_selector = literal_selector.replace("\\", "/")
selectors = (
(literal_selector,)
if path_selector == literal_selector
else (literal_selector, path_selector)
)
matches = []
literal_exact = []
literal_casefold = []
path_exact = []
owned = []
for f in _iter_gguf_files(p, recursive = True):
rel = f.relative_to(p).as_posix()
if _is_mmproj(f.name) or _is_local_mtp_drafter(f, root, rel) or _is_imatrix_path(f.name):
continue
quant = _extract_quant_label(rel)
fallback_variant = re.sub(r"-\d{3,}-of-\d{3,}$", "", rel.rsplit(".", 1)[0])
# bpw-stripped labels and basename quant tokens identify the file, never a parent quant.
stripped = re.sub(r"-[0-9]+(?:\.[0-9]+)?bpw$", "", quant, flags = re.IGNORECASE)
basename_stem = fallback_variant.rsplit("/", 1)[-1]
stem_tokens = [
f"{m.group(1) or ''}{m.group(2)}" for m in _GGUF_KNOWN_QUANT_RE.finditer(basename_stem)
]
labels = (rel, quant, fallback_variant, stripped, *stem_tokens)
if not any(
_variant_matches(selector, *labels) for selector in selectors
) or _is_big_endian_gguf_path(rel, quant):
continue
matches.append(f)
# exact ownership prevents a repo-root quant from selecting an earlier subdirectory copy.
if literal_selector == rel:
literal_exact.append(f)
elif _variant_matches(literal_selector, rel):
literal_casefold.append(f)
elif path_selector == rel:
path_exact.append(f)
elif _variant_matches(path_selector, rel, _gguf_variant_key(rel)):
owned.append(f)
matches.sort()
literal_exact.sort()
literal_casefold.sort()
path_exact.sort()
owned.sort()
if literal_exact:
return str(_local_gguf_load_path(literal_exact[0]))
if literal_casefold:
return str(_local_gguf_load_path(literal_casefold[0]))
if path_exact:
return str(_local_gguf_load_path(path_exact[0]))
if owned:
return str(_local_gguf_load_path(owned[0]))
if matches:
return str(_local_gguf_load_path(matches[0]))
return None
def _detect_gguf_from_hf_cache(repo_id: str) -> Optional[str]:
"""Excludes mmproj files, so a cache holding only the projector cannot route it as the main model."""
rel_files = _hf_cache_main_gguf_files(repo_id)
return _pick_best_gguf(rel_files) if rel_files else None
def _hf_cache_main_gguf_files(repo_id: str, *, every_snapshot: bool = False) -> list[str]:
"""Main GGUF paths of the first cached snapshot of *repo_id* that holds any, or with
*every_snapshot* of all of them (newest first, each path once), remembered caches included
as the strict load lookup includes them."""
found: list[str] = []
if every_snapshot:
from core.inference.llama_cpp import _cached_gguf_snapshots
snapshots = _cached_gguf_snapshots(repo_id)
else:
snapshots = _iter_hf_cache_snapshots(repo_id)
for snap in snapshots:
rel_files = []
for f in _iter_gguf_files(snap, recursive = True):
rel = f.relative_to(snap).as_posix()
quant = _extract_quant_label(rel)
if (
_is_mmproj(f.name)
or _is_mtp_drafter(rel)
or _is_imatrix_path(f.name)
or _is_big_endian_gguf_path(rel, quant)
):
continue
rel_files.append(rel)
if rel_files and not every_snapshot:
return rel_files
found += [rel for rel in rel_files if rel not in found]
return found
class GgufRepoUnreadableError(ValueError):
"""A ValueError so /load and /validate answer 400 with its text; also the from_identifier sink for the Hub error behind a detect_gguf_model_remote None."""
_gguf_remote_detect_failure: ContextVar[Optional[List[Exception]]] = ContextVar(
"gguf_remote_detect_failure", default = None
)
def _note_gguf_remote_detect_failure(error: Exception) -> None:
sink = _gguf_remote_detect_failure.get()
if sink is not None:
sink.append(error)
_GGUF_REPO_NAME_RE = _re.compile(r"(?:^|[-_.])gguf(?:$|[-_.])", _re.IGNORECASE)
def _looks_like_gguf_repo(repo_id: str, gguf_variant: Optional[str] = None) -> bool:
from hub.utils.mlx_variants import is_precision_folder
# A cached Transformers, diffusers, adapter or MLX precision-folder checkpoint can still load.
names = ["config.json", "model_index.json", "adapter_config.json"]
if is_precision_folder(gguf_variant):
names.append(f"{gguf_variant.strip()}/config.json")
if any((snap / name).is_file() for snap in _iter_hf_cache_snapshots(repo_id) for name in names):
return False
return bool(gguf_variant) or bool(
_GGUF_REPO_NAME_RE.search(repo_id.rstrip("/").rsplit("/", 1)[-1])
)
def _gguf_repo_unreadable_message(
repo_id: str,
error: Optional[Exception],
*,
token_rejected: bool = False,
) -> str:
if error is None:
return (
f"Could not load the GGUF repo '{repo_id}': Unsloth is offline and the repo is not "
"in the local cache. Connect to the internet once to download it, then try again."
)
status = getattr(getattr(error, "response", None), "status_code", None)
first_line = (str(error).strip().splitlines() or [""])[0][:200]
cause = type(error).__name__
if status is not None:
cause += f", HTTP {status}"
if first_line:
cause += f": {first_line}"
if token_rejected:
# Anonymous access could not answer either: private, gated or missing without it.
return (
f"Could not read the GGUF repo '{repo_id}' from Hugging Face ({cause}). "
f"{HUB_TOKEN_REJECTED_ERROR} If Unsloth reaches the Hub through a proxy, mirror "
"or HF_ENDPOINT, check that as well."
)
return (
f"Could not read the GGUF repo '{repo_id}' from Hugging Face ({cause}). "
"Unsloth needs the repo's file list to pick a GGUF file. Check the Hugging Face "
"token in Settings (clear it if it is expired or revoked), your network, proxy "
"or HF_ENDPOINT, then try again."
)
def _is_hub_refusal(error: Exception) -> bool:
"""The Hub answered no (401/403/404, gated), as opposed to not answering at all."""
from hub.utils.hf_tokens import _is_probe_timeout
if _is_probe_timeout(error):
return False
if type(error).__name__ in ("RepositoryNotFoundError", "GatedRepoError"):
return True
status = getattr(getattr(error, "response", None), "status_code", None)
return status in (401, 403, 404)
def _refused_repo_cache_token(hf_token: HfTokenArg, owner_session: bool) -> HfTokenArg:
"""Owner session may use ambient auth for its own cache; others never borrow the installation token."""
if is_anonymous(hf_token):
return False
token = hf_token.strip() if isinstance(hf_token, str) else ""
return hf_token_arg(token, allow_ambient_token = owner_session)
def _remembered_companions_present(repo_id: str, variant: str, local_file: str) -> bool:
"""Whether every file an earlier live listing said *variant* needs is beside *local_file*.
A manifest-less download is otherwise judged by its shards alone."""
from core.inference.llama_cpp import _snapshot_dir_of
from hub.services.models import gguf_variants
requirement = gguf_variants._variant_requirement_last_known((repo_id.lower(), variant.lower()))
if requirement is None:
return not gguf_variants._variant_requirement_may_be_forgotten()
snapshot = _snapshot_dir_of(local_file)
if snapshot is None:
return False
try:
for expected in requirement.expected_files:
target = Path(snapshot) / expected.path
if not target.is_file() or (expected.size and target.stat().st_size != expected.size):
return False
except OSError:
return False
return True
def _refused_repo_cached_gguf(
repo_id: str, gguf_variant: Optional[str], hf_token: HfTokenArg, *, owner_session: bool
) -> Optional[tuple[str, str]]:
"""Owner session only, when the Hub refuses the repo: a complete cached copy, else None; an earlier auth-check verdict or probe never outranks the refusal."""
if not owner_session:
return None
if not cache_reads_authorized(
_refused_repo_cache_token(hf_token, owner_session), repo_id = repo_id
):
return None
from core.inference.llama_cpp import cached_gguf_for_load
if gguf_variant:
local_file = cached_gguf_for_load(repo_id, gguf_variant, strict = True)
if local_file and _remembered_companions_present(repo_id, gguf_variant, local_file):
return local_file, gguf_variant
return None
# Auto: preferred variant first, falling through incomplete ones, across every snapshot.
remaining = sorted(_hf_cache_main_gguf_files(repo_id, every_snapshot = True))
while remaining:
from hub.utils.gguf import default_gguf_scope
best = _pick_best_gguf(
default_gguf_scope(
remaining,
repo_id,
key = lambda f: _qualified_variant_name(f, _extract_quant_label(f)),
)
)
if not best:
return None
label = _extract_quant_label(best)
variant = _qualified_variant_name(best, label) if label else None
local_file = cached_gguf_for_load(repo_id, variant, strict = True) if variant else None
if local_file and _remembered_companions_present(repo_id, variant, local_file):
return local_file, variant
remaining = [
f
for f in remaining
if f != best
and not (label and _qualified_variant_name(f, _extract_quant_label(f)) == variant)
]
return None
def detect_gguf_model_remote(repo_id: str, hf_token: Optional[str] = None) -> Optional[str]:
"""Retries transient Hub failures, since a silent None would send a GGUF-only repo down the MLX path."""
if _env_offline():
return _detect_gguf_from_hf_cache(repo_id)
last_err: Optional[Exception] = None
for attempt, timeout in enumerate(_GGUF_LISTING_TIMEOUTS):
try:
info = _hub_model_info(repo_id, hf_token, timeout = timeout)
repo_files = []
for sibling in info.siblings:
fname = sibling.rfilename
if not fname.lower().endswith(".gguf"):
continue
quant = _extract_quant_label(fname)
if (
_is_mmproj(fname)
or _is_mtp_drafter(fname)
or _is_imatrix_path(fname)
or _is_big_endian_gguf_path(fname, quant)
):
continue
repo_files.append(fname)
return _pick_best_gguf(repo_files)
except Exception as e:
last_err = e
# 404 / RepoNotFound is permanent -- don't retry
err_name = type(e).__name__
# Status-only refusals too (DisabledRepoError, bare 403), or the refused-repo policy is skipped.
if err_name in (
"RepositoryNotFoundError",
"GatedRepoError",
"RevisionNotFoundError",
"EntryNotFoundError",
) or _is_hub_refusal(e):
logger.debug(f"Could not check GGUF files for '{repo_id}': {e}")
_note_gguf_remote_detect_failure(e)
return None
if attempt < 2:
time.sleep(2**attempt)
# All attempts failed; fall back to local cache for offline users.
cached = _detect_gguf_from_hf_cache(repo_id)
if cached is not None:
logger.warning(
"HF API unreachable for '%s' (%s); using local cache to detect GGUF.",
repo_id,
type(last_err).__name__ if last_err else "unknown",
)
return cached
logger.warning(f"Could not check GGUF files for '{repo_id}' after 3 attempts: {last_err}")
if last_err is not None:
_note_gguf_remote_detect_failure(last_err)
return None
def download_gguf_file(
repo_id: str,
filename: str,
hf_token: Optional[str] = None,
) -> str:
"""Download a specific GGUF file from a HF repo; returns the local path."""
from huggingface_hub import hf_hub_download
local_path = hf_hub_download(
repo_id = repo_id,
filename = filename,
token = hf_token,
cache_dir = active_hf_hub_cache(),
)
return local_path
# Cache embedding detection per session to avoid repeated HF API calls
_embedding_detection_cache: Dict[tuple, bool] = {}
def _embedding_marker_in_hf_cache(model_name: str) -> bool:
"""True when model_name's cached snapshot carries a modules.json (the ST marker).
Cache-only, no network; used offline and as a fallback when the Hub lookup times out."""
from utils.utils import hf_cache_snapshot_dir
snapshot = hf_cache_snapshot_dir(model_name)
if snapshot is None:
return False
try:
return (snapshot / "modules.json").is_file()
except OSError:
return False
def is_embedding_model(model_name: str, hf_token: Optional[str] = None) -> bool:
"""Embedding check from HF tags and pipeline_tag; False for local paths and on any error."""
from utils.utils import hf_env_offline
# Offline: reclassify from local cache before the memo; online memo can be stale either way.
if not is_local_path(model_name) and hf_env_offline():
# The marker never authorizes, so offline an unverified token reports the default.
if not cache_reads_authorized(hf_token, repo_id = model_name):
return False
return _embedding_marker_in_hf_cache(model_name)
# Fingerprint, not raw token: the marker str subclass equals a plain token, so a UI-computed classification was served to API callers.
cache_key = (model_name, _token_fingerprint(hf_token))
if cache_key in _embedding_detection_cache:
return _embedding_detection_cache[cache_key]
# Local paths: check for sentence-transformer marker (modules.json)
if is_local_path(model_name):
local_dir = normalize_path(model_name)
is_emb = os.path.isfile(os.path.join(local_dir, "modules.json"))
_embedding_detection_cache[cache_key] = is_emb
return is_emb
try:
info = _hub_model_info(model_name, hf_token)
tags = set(info.tags or [])
pipeline_tag = info.pipeline_tag or ""
is_emb = (
"sentence-transformers" in tags
or "feature-extraction" in tags
or pipeline_tag in ("sentence-similarity", "feature-extraction")
)
_embedding_detection_cache[cache_key] = is_emb
if is_emb:
logger.info(
f"Model {model_name} detected as embedding model: "
f"pipeline_tag={pipeline_tag}, "
f"sentence-transformers in tags={('sentence-transformers' in tags)}, "
f"feature-extraction in tags={('feature-extraction' in tags)}"
)
return is_emb
except Exception as e:
# Timeout or transient network error: fall back to the local cache marker, don't hard-fail.
logger.warning(f"Could not determine if {model_name} is embedding model: {e}")
if not cache_reads_authorized(hf_token, repo_id = model_name):
# The anonymous 404 lands here too, and the marker read never authorizes.
return False
is_emb = _embedding_marker_in_hf_cache(model_name)
_embedding_detection_cache[cache_key] = is_emb
return is_emb
_LAYA_MARKER = "rl_agent_config.json"
# Clef: a backbone (merged, or LoRA adapters over the base LLM) plus the joint schema head.
CLEF_HEAD_MARKERS = ("joint_head.safetensors", "joint_head_config.json")
CLEF_MARKERS = ("config.json", *CLEF_HEAD_MARKERS)
CLEF_ADAPTER_MARKERS = ("adapter_config.json", *CLEF_HEAD_MARKERS)
def clef_files_kind(has) -> Optional[str]:
""" "merged", "adapter" or None, from has(name) -> bool over a folder's files."""
if all(has(name) for name in CLEF_MARKERS):
return "merged"
if all(has(name) for name in CLEF_ADAPTER_MARKERS):
return "adapter"
return None
def is_d1_folder(folder: Path) -> bool:
try:
config = json.loads((folder / "config.json").read_text(encoding = "utf-8"))
except (OSError, ValueError):
return False
if not isinstance(config, dict):
return False
# Markers unsloth-zoo's d1 loader reads: d1-3B only in auto_map, d1-omni by model_type.
automodel = str((config.get("auto_map") or {}).get("AutoModel", ""))
return (
config.get("model_type") == "d1_omni"
or "D1Model" in (config.get("architectures") or ())
or automodel.endswith("D1Model")
)
def clef_folder_kind(folder: Path) -> Optional[str]:
kind = clef_files_kind(lambda name: (folder / name).is_file())
return kind or ("merged" if is_d1_folder(folder) else None)
def _folder_decision_layout(folder: Path) -> Optional[str]:
if clef_folder_kind(folder) is not None:
return "clef"
if all((folder / name).is_file() for name in (_LAYA_MARKER, "model.safetensors")) and all(
(folder / name).is_dir() for name in ("encoder", "tokenizer")
):
return "laya"
return None
def decision_layout(
model_name: str,
hf_token: Optional[str] = None,
local_files_only: bool = False,
subfolder: Optional[str] = None,
) -> Optional[str]:
""" "laya", "clef" or None for a model that is not a decision model."""
if is_local_path(model_name):
folder = Path(normalize_path(model_name))
# The subfolder first, as on the Hub; an escaping one is refused later by request validation.
nested = None
if subfolder and not Path(subfolder).is_absolute() and ".." not in Path(subfolder).parts:
nested = _folder_decision_layout(folder / subfolder)
return nested or _folder_decision_layout(folder)
from utils.utils import hf_cache_snapshot_dir, hf_env_offline
prefix = f"{subfolder}/" if subfolder else ""
if not (local_files_only or hf_env_offline()):
try:
info = _hub_model_info(model_name, hf_token)
files = {getattr(sibling, "rfilename", None) for sibling in info.siblings or ()}
if clef_files_kind(lambda name: prefix + name in files) is not None:
return "clef"
return "laya" if prefix + _LAYA_MARKER in files else None
except Exception as e:
logger.warning(f"Could not determine if {model_name} is a decision model: {e}")
if not cache_reads_authorized(hf_token, repo_id = model_name):
return None
snapshot = hf_cache_snapshot_dir(model_name)
if snapshot is None:
return None
if clef_folder_kind(snapshot / prefix) is not None:
return "clef"
# The Decision API caches only the checkpoint subfolder it serves.
laya = (snapshot / _LAYA_MARKER, *snapshot.glob(f"*/{_LAYA_MARKER}"))
return "laya" if any(path.is_file() for path in laya) else None
def is_decision_model(
model_name: str,
hf_token: Optional[str] = None,
local_files_only: bool = False,
subfolder: Optional[str] = None,
) -> bool:
return decision_layout(model_name, hf_token, local_files_only, subfolder) is not None
LLM_DECISION_DEFAULTS = (
Path(__file__).parent.parent.parent
/ "assets"
/ "configs"
/ "model_defaults"
/ "decision"
/ "llm_decision_defaults.yaml"
)
def load_llm_decision_defaults() -> Dict[str, Any]:
"""The recipe for training a text or vision LLM as a decision model (a new Clef head)."""
with open(LLM_DECISION_DEFAULTS, "r", encoding = "utf-8") as f:
return yaml.safe_load(f) or {}
def _has_model_weight_files(model_dir: Path) -> bool:
"""Return True when a directory contains loadable model weights."""
for item in model_dir.iterdir():
if not item.is_file():
continue
if is_appledouble_metadata(item):
continue
suffix = item.suffix.lower()
if suffix == ".safetensors":
return True
if suffix == ".gguf":
return "mmproj" not in item.name.lower()
if suffix == ".bin":
name = item.name.lower()
if (
name.startswith("pytorch_model")
or name.startswith("model")
or name.startswith("adapter_model")
or name.startswith("consolidated")
):
return True
return False
def _detect_training_output_type(model_dir: Path) -> Optional[str]:
"""Classify an Unsloth training output as LoRA or full finetune."""
adapter_config = model_dir / "adapter_config.json"
adapter_model = model_dir / "adapter_model.safetensors"
if adapter_config.exists() or adapter_model.exists():
return "lora"
config_file = model_dir / "config.json"
if config_file.exists() and _has_model_weight_files(model_dir):
return "merged"
return None
def _looks_like_lora_adapter(model_dir: Path) -> bool:
return model_dir.is_dir() and (
(model_dir / "adapter_config.json").exists()
or any(model_dir.glob("adapter_model*.safetensors"))
or any(model_dir.glob("adapter_model*.bin"))
)
def scan_trained_models(outputs_dir: Optional[str] = None) -> List[Tuple[str, str, str]]:
"""Returns (display_name, model_path, model_type) tuples, where model_type is lora or merged."""
if outputs_dir is None:
outputs_dir = str(outputs_root())
trained_models = []
outputs_path = resolve_output_dir(outputs_dir)
if not outputs_path.exists():
logger.warning(f"Outputs directory not found: {outputs_path}")
return trained_models
try:
for item in outputs_path.iterdir():
if item.is_dir() and within_account(item):
model_type = _detect_training_output_type(item)
if model_type is None:
continue
display_name = item.name
model_path = str(item)
trained_models.append((display_name, model_path, model_type))
logger.debug("Found trained model: %s (%s)", display_name, model_type)
trained_models.sort(key = lambda x: Path(x[1]).stat().st_mtime, reverse = True)
logger.info(
"Found %s trained models in %s",
len(trained_models),
outputs_dir,
)
return trained_models
except Exception as e:
logger.error(f"Error scanning outputs folder: {e}")
return []
def scan_exported_models(
exports_dir: Optional[str] = None,
) -> List[Tuple[str, str, str, Optional[str]]]:
"""Scans run/checkpoint and flat *-finetune-gguf layouts; export_type is lora, merged, or gguf."""
if exports_dir is None:
exports_dir = str(exports_root())
from core.inference.diffusion_lora import is_image_lora_file
results = []
exports_path = resolve_export_dir(exports_dir)
if not exports_path.exists():
return results
try:
for run_dir in exports_path.iterdir():
if not run_dir.is_dir() or not within_account(run_dir):
continue
# mmproj and imatrix files beside the weights are not main models.
gguf_files = [
f
for f in _iter_gguf_files(run_dir)
if not _is_mmproj(f.name)
and not _is_imatrix_path(f.name)
and not is_image_lora_file(f)
]
if gguf_files:
base_model = None
export_meta = run_dir / "export_metadata.json"
try:
if export_meta.exists():
meta = json.loads(export_meta.read_text(encoding = "utf-8-sig"))
base_model = meta.get("base_model")
except Exception:
pass
display_name = run_dir.name
model_path = str(gguf_files[0])
results.append((display_name, model_path, "gguf", base_model))
logger.debug(f"Found GGUF export: {display_name}")
continue
# Two-level: {run}/{checkpoint}/
for checkpoint_dir in run_dir.iterdir():
if not checkpoint_dir.is_dir() or not within_account(checkpoint_dir):
continue
adapter_config = checkpoint_dir / "adapter_config.json"
config_file = checkpoint_dir / "config.json"
has_weights = any(
not is_appledouble_metadata(f)
for f in (*checkpoint_dir.glob("*.safetensors"), *checkpoint_dir.glob("*.bin"))
)
# Same filter as the flat layout: mmproj and imatrix files are not main models.
gguf_list = [
f
for f in _iter_gguf_files(checkpoint_dir)
if not _is_mmproj(f.name)
and not _is_imatrix_path(f.name)
and not is_image_lora_file(f)
]
has_gguf = bool(gguf_list)
base_model = None
export_type = None
if own_entry(adapter_config):
export_type = "lora"
try:
cfg = json.loads(adapter_config.read_text(encoding = "utf-8-sig"))
base_model = cfg.get("base_model_name_or_path")
except Exception:
pass
elif own_entry(config_file) and has_weights:
export_type = "merged"
export_meta = checkpoint_dir / "export_metadata.json"
try:
if own_entry(export_meta):
meta = json.loads(export_meta.read_text(encoding = "utf-8-sig"))
base_model = meta.get("base_model")
except Exception:
pass
elif has_gguf:
export_type = "gguf"
# checkpoint_dir first, then run_dir (export.py writes metadata to the top-level dir)
for meta_dir in (checkpoint_dir, run_dir):
export_meta = meta_dir / "export_metadata.json"
try:
if own_entry(export_meta):
meta = json.loads(export_meta.read_text(encoding = "utf-8-sig"))
base_model = meta.get("base_model")
if base_model:
break
except Exception:
pass
display_name = f"{run_dir.name} / {checkpoint_dir.name}"
model_path = str(gguf_list[0])
results.append((display_name, model_path, export_type, base_model))
logger.debug(f"Found GGUF export: {display_name}")
continue
else:
continue
# Fallback: base model from ./outputs/{run_name}/adapter_config.json
if not base_model:
outputs_adapter_cfg = resolve_output_dir(run_dir.name) / "adapter_config.json"
try:
if own_entry(outputs_adapter_cfg):
cfg = json.loads(outputs_adapter_cfg.read_text(encoding = "utf-8-sig"))
base_model = cfg.get("base_model_name_or_path")
except Exception:
pass
display_name = f"{run_dir.name} / {checkpoint_dir.name}"
model_path = str(checkpoint_dir)
results.append((display_name, model_path, export_type, base_model))
logger.debug(f"Found exported model: {display_name} ({export_type})")
results.sort(key = lambda x: Path(x[1]).stat().st_mtime, reverse = True)
logger.info(f"Found {len(results)} exported models in {exports_dir}")
return results
except Exception as e:
logger.error(f"Error scanning exports folder: {e}")
return []
def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]:
"""Read the base model name from a local training or checkpoint directory."""
try:
checkpoint_path_obj = Path(checkpoint_path)
adapter_config_path = checkpoint_path_obj / "adapter_config.json"
if adapter_config_path.exists():
with open(adapter_config_path, "r", encoding = "utf-8-sig") as f:
config = json.load(f)
base_model = config.get("base_model_name_or_path")
if base_model:
logger.info("Detected base model from adapter_config.json: %s", base_model)
return base_model
config_path = checkpoint_path_obj / "config.json"
if config_path.exists():
with open(config_path, "r", encoding = "utf-8-sig") as f:
config = json.load(f)
for key in ("model_name", "_name_or_path"):
base_model = config.get(key)
if base_model and str(base_model) != str(checkpoint_path_obj):
logger.info(
"Detected base model from config.json (%s): %s",
key,
base_model,
)
return base_model
# TODO: training_args.bin read disabled: torch>=2.6 weights_only rejects TrainingArguments.
base_model = _base_model_from_dir_name(checkpoint_path_obj.name)
if base_model:
logger.info("Detected base model from directory name: %s", base_model)
return base_model
logger.warning(f"Could not detect base model for checkpoint: {checkpoint_path}")
return None
except Exception as e:
logger.error(f"Error reading base model from checkpoint config: {e}")
return None
def get_base_model_from_lora(lora_path: str) -> Optional[str]:
"""Read the base model name from a LoRA adapter's config, or None."""
try:
lora_path_obj = Path(lora_path)
if not _looks_like_lora_adapter(lora_path_obj):
return None
# adapter_config.json first
adapter_config_path = lora_path_obj / "adapter_config.json"
if adapter_config_path.exists():
with open(adapter_config_path, "r", encoding = "utf-8-sig") as f:
config = json.load(f)
base_model = config.get("base_model_name_or_path")
if base_model:
logger.info(f"Detected base model from adapter_config.json: {base_model}")
return base_model
# training_args.bin read disabled: weights_only rejects it, and it is a code-exec sink; needs a trust check.
base_model = _base_model_from_dir_name(lora_path_obj.name)
if base_model:
logger.info(f"Detected base model from directory name: {base_model}")
return base_model
logger.warning(f"Could not detect base model for LoRA: {lora_path}")
return None
except Exception as e:
logger.error(f"Error reading base model from LoRA config: {e}")
return None
def load_mlx_adapter_tokenizer(
tokenizer,
lora_path: str,
hf_token: HfTokenArg = None,
):
# FastMLXModel hands back the base repo's tokenizer, not the one trained and saved with the adapter.
if getattr(tokenizer, "chat_template", None):
return tokenizer
adapter_dir = Path(lora_path)
if not adapter_dir.is_dir():
try:
from huggingface_hub import snapshot_download
adapter_dir = Path(
call_with_anonymous_retry(
lambda token: snapshot_download(
lora_path,
allow_patterns = [
"*.json",
"*.jinja",
"*.txt",
"tokenizer.model",
"*.tiktoken",
],
token = token,
cache_dir = active_hf_hub_cache(),
),
hf_token,
)
)
except Exception as e:
logger.debug(f"Could not fetch the tokenizer saved with the adapter {lora_path}: {e}")
return tokenizer
if not (adapter_dir / "tokenizer_config.json").is_file():
return tokenizer
try:
from mlx_lm.utils import load_tokenizer
adapter_tokenizer = load_tokenizer(
adapter_dir, eos_token_ids = getattr(tokenizer, "eos_token_ids", None)
)
except Exception as e:
logger.warning(f"Could not load the tokenizer saved with the adapter at {lora_path}: {e}")
return tokenizer
return adapter_tokenizer if adapter_tokenizer.chat_template else tokenizer
def load_mlx_adapter_processor(
processor,
lora_path: str,
hf_token: HfTokenArg = None,
):
"""The processor counterpart of load_mlx_adapter_tokenizer, for multimodal bases such as Qwen3.5 Base."""
own = getattr(processor, "tokenizer", None)
if (
own is None
or getattr(processor, "chat_template", None)
or getattr(own, "chat_template", None)
):
return processor
adapter = load_mlx_adapter_tokenizer(own, lora_path, hf_token)
trained = (
getattr(adapter, "_tokenizer", adapter)
if type(adapter).__name__ == "TokenizerWrapper"
else adapter
)
template = getattr(trained, "chat_template", None)
if adapter is own or not template:
return processor
# mlx-vlm (stopping_criteria) and the MLX loader (_unsloth_* tags) hang state off the processor's tokenizer.
for name, value in vars(own).items():
if not hasattr(trained, name):
setattr(trained, name, value)
processor.tokenizer = trained
processor.chat_template = template
return processor
def get_base_model_from_lora_identifier(
identifier: str, hf_token: Optional[str] = None
) -> Optional[str]:
"""Base model for a local or remote LoRA, fetching only adapter_config.json; gates must scan that base, since an adapter can point it at poisoned code."""
try:
if is_local_path(identifier):
return get_base_model_from_lora(identifier)
except Exception:
return get_base_model_from_lora(identifier)
# Remote repo id: read base_model_name_or_path from adapter_config.json only.
from huggingface_hub import hf_hub_download
from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
from utils.hf_probe import hf_file_definitely_absent
# Plain model repos normally lack adapter_config.json, so probe without caching its 404.
if hf_file_definitely_absent(
identifier,
"adapter_config.json",
token = normalize_token(hf_token),
):
return None
last_exc = None
for _attempt in range(2): # one retry: a transient blip must not skip the base
try:
cfg_path = call_with_anonymous_retry(
lambda token: hf_hub_download(
identifier,
"adapter_config.json",
token = token,
cache_dir = active_hf_hub_cache(),
),
normalize_token(hf_token),
)
except (EntryNotFoundError, RepositoryNotFoundError):
# No adapter_config.json -> not a resolvable LoRA; caller scans the identifier.
return None
except Exception as exc: # transient / auth / network -> retry once
last_exc = exc
continue
try:
with open(cfg_path, "r", encoding = "utf-8-sig") as f:
base_model = json.load(f).get("base_model_name_or_path")
except Exception as exc:
logger.warning("Could not parse adapter_config.json for '%s': %s", identifier, exc)
return None
if base_model:
logger.info(
"Detected base model from remote adapter_config.json (%s): %s",
identifier,
base_model,
)
return base_model # may be None if the key is absent (still a valid answer)
# Both attempts failed transiently: log loudly -- a missed base is gated by neither gate.
logger.warning(
"Could not resolve remote LoRA base for '%s' after retry (%s); its base, if "
"any, will not be added to the security scan targets.",
identifier,
type(last_exc).__name__ if last_exc else "unknown",
)
return None
# Status indicators that appear in UI dropdowns
UI_STATUS_INDICATORS = [" (Ready)", " (Loading...)", " (Active)", "↓ "]
# Status is polled every 5s; log each (model, path) once at info, repeats at debug. Bounded.
_ANNOUNCED_MODEL_DEFAULTS: set = set()
_ANNOUNCED_MODEL_DEFAULTS_MAX = 4096
# Two threads can resolve one unseen model at once, so the check-then-add must run under the lock.
_ANNOUNCED_MODEL_DEFAULTS_LOCK = threading.Lock()
def _log_model_defaults(
message: str,
key: str,
level: str = "info",
) -> None:
"""Log `message` at `level` the first time `key` is seen, then at debug."""
with _ANNOUNCED_MODEL_DEFAULTS_LOCK:
if key in _ANNOUNCED_MODEL_DEFAULTS:
logger.debug(message)
return
if len(_ANNOUNCED_MODEL_DEFAULTS) >= _ANNOUNCED_MODEL_DEFAULTS_MAX:
_ANNOUNCED_MODEL_DEFAULTS.clear()
_ANNOUNCED_MODEL_DEFAULTS.add(key)
getattr(logger, level)(message)
def defaults_lookup_names(model_name: str) -> List[str]:
"""Local paths also yield their last two and last one parts to match the repo id config; drive and root dropped (rglob fails on Windows)."""
if not is_local_path(model_name):
return [model_name]
# Normalize Windows backslashes: pathlib treats them as literals on POSIX/WSL.
path = Path(normalize_path(model_name))
parts = path.parts[1:] if path.anchor else path.parts
return ["/".join(parts[-depth:]) for depth in (2, 1) if len(parts) >= depth]
def load_model_defaults(model_name: str) -> Dict[str, Any]:
"""Looks up model_defaults YAML by model name or MODEL_NAME_MAPPING alias, else default.yaml; a non-string id returns {} before .lower() can raise."""
if not isinstance(model_name, str) or not model_name:
return {}
try:
script_dir = Path(__file__).parent.parent.parent
defaults_dir = script_dir / "assets" / "configs" / "model_defaults"
lookup_names = defaults_lookup_names(model_name)
for name in lookup_names:
canonical_file = _REVERSE_MODEL_MAPPING.get(name.lower())
if canonical_file is None:
continue
for config_path in defaults_dir.rglob(canonical_file):
if config_path.is_file():
with open(config_path, "r", encoding = "utf-8") as f:
config = yaml.safe_load(f) or {}
_log_model_defaults(
f"Loaded model defaults from {config_path} (via mapping '{name}')",
f"{model_name}|{config_path}|mapping",
)
return config
# Runs after the registry so an alias never loses to a filename.
for name in lookup_names:
for config_path in defaults_dir.rglob(name.replace("/", "_") + ".yaml"):
if config_path.is_file():
with open(config_path, "r", encoding = "utf-8") as f:
config = yaml.safe_load(f) or {}
_log_model_defaults(
f"Loaded model defaults from {config_path} (via name '{name}')",
f"{model_name}|{config_path}|exact",
)
return config
default_config_path = defaults_dir / "default.yaml"
if default_config_path.exists():
with open(default_config_path, "r", encoding = "utf-8") as f:
config = yaml.safe_load(f) or {}
_log_model_defaults(
f"Loaded default model defaults from {default_config_path}",
f"{model_name}|{default_config_path}|fallback",
)
return config
_log_model_defaults(
f"No default config found for model {model_name}",
f"{model_name}|<missing>",
level = "warning",
)
return {}
except Exception as e:
logger.error(f"Error loading model defaults for {model_name}: {e}")
return {}
@dataclass
class ModelConfig:
"""Configuration for a model to load."""
identifier: str # Clean model identifier (org/name or path)
display_name: str # Original UI display name
path: str # Normalized filesystem path
is_local: bool # Local file vs HF model?
is_cached: bool # Already in HF cache?
is_vision: bool # Vision model?
is_lora: bool # LoRA adapter?
is_gguf: bool = False # GGUF model?
is_audio: bool = False # TTS audio model?
audio_type: Optional[str] = None # Codec or native audio generation architecture.
has_audio_input: bool = False # Accepts audio input (ASR/speech understanding)
gguf_file: Optional[str] = None # Full path to the .gguf file (local mode)
# (repo, variant, path, sizes); sizes covers every shard.
gguf_verified: Optional[tuple[str, str, str, tuple[tuple[str, int], ...]]] = None
gguf_mmproj_file: Optional[str] = None # Full path to the mmproj .gguf file (vision projection)
# Remote (-hf) only: hand-added projector for VRAM accounting, never passed to llama-server.
gguf_local_mmproj_file: Optional[str] = None
gguf_mtp_file: Optional[str] = None # Full path to the separate MTP drafter (local mode)
gguf_dspark_file: Optional[str] = None # Full path to a DSpark sidecar (local mode)
gguf_dflash_file: Optional[str] = None # Full path to a DFlash sidecar (local mode)
gguf_hf_repo: Optional[str] = (
None # HF repo ID for -hf mode (e.g. "unsloth/gemma-3-4b-it-GGUF")
)
# Repo a refused-Hub cached GGUF came from, for the download interlock.
gguf_cache_repo: Optional[str] = None
gguf_variant: Optional[str] = None # Quantization variant (e.g. "Q4_K_M")
# A picked MLX precision folder's config, for route checks only; sizing needs the folder's weights.
config_dir: Optional[str] = None
base_model: Optional[str] = None # Base model (for LoRAs)
# Set for a GGUF only audio.cpp runs; loads via the native-audio worker, never llama-server.
audio_cpp: Optional[Any] = None
@classmethod
def from_lora_path(
cls,
lora_path: str,
hf_token: Optional[str] = None,
) -> Optional["ModelConfig"]:
"""ModelConfig from a LoRA adapter dir, auto-detecting the base model; hf_token for vision detection."""
try:
lora_path_obj = Path(lora_path)
if not lora_path_obj.exists():
logger.error(f"LoRA path does not exist: {lora_path}")
return None
base_model = get_base_model_from_lora(lora_path)
if not base_model:
logger.error(f"Could not determine base model for LoRA: {lora_path}")
return None
is_vision = is_vision_model(base_model, hf_token = hf_token)
audio_type = detect_audio_type(base_model, hf_token = hf_token)
display_name = lora_path_obj.name
identifier = lora_path # path is the identifier for local LoRAs
return cls(
identifier = identifier,
display_name = display_name,
path = lora_path,
is_local = True,
is_cached = True, # local LoRAs are always cached
is_vision = is_vision,
is_lora = True,
is_audio = audio_type is not None and audio_type != "audio_vlm",
audio_type = audio_type,
has_audio_input = is_audio_input_type(audio_type),
base_model = base_model,
)
except Exception as e:
logger.error(f"Error creating ModelConfig from LoRA path: {e}")
return None
@classmethod
def from_identifier(
cls,
model_id: str,
hf_token: Optional[str] = None,
is_lora: bool = False,
gguf_variant: Optional[str] = None,
drafter_accept: Optional[Callable[[str, str, str, str], bool]] = None,
gguf_companion_roots: Optional[Tuple[str, ...]] = None,
mmproj_accept: Optional[Callable[[str, str], bool]] = None,
owner_session: bool = False,
) -> Optional["ModelConfig"]:
"""Create ModelConfig from a clean model identifier (HF repo or local
path), for FastAPI routes that send sanitized paths.
Args:
model_id: Clean model identifier (HF repo name or local path)
hf_token: Optional HF token for vision detection on gated models
is_lora: Whether this is a LoRA adapter
gguf_variant: Optional GGUF quant variant (e.g. "Q4_K_M") to load
via -hf for remote repos; None auto-selects via _pick_best_gguf().
drafter_accept: ``(candidate, gguf_file, kind, search_root) -> bool``,
the caller's extra admission rule for a discovered drafter. A
native-grant load passes the lease boundary here so it is applied
BEFORE this scan inspects a candidate: detect_dflash_file reads
the header of the file it is about to accept, and a
``dflash-*.gguf`` symlink in a granted directory can point at a
target outside the lease, which the validated rescan on the load
route rejects only after the read already happened. Left None by
every caller that has no boundary to impose, which sees the same
candidates in the same order as before.
gguf_companion_roots: Trusted snapshot directories belonging to the
resolver-selected local cache entry. Used only to locate a
compatible mmproj without changing the selected main weights.
mmproj_accept: ``(candidate, gguf_file) -> bool`` admission rule
applied before reading projector metadata for native loads.
owner_session: the caller is the machine owner's own UI session, which may
run a GGUF it already downloaded when the Hub refuses the repo.
Returns:
ModelConfig or None if it cannot be created.
"""
if not model_id or not model_id.strip():
return None
identifier = model_id.strip()
audio_cpp_config = cls._from_audio_cpp_identifier(identifier, gguf_variant, hf_token)
if audio_cpp_config is not None:
return audio_cpp_config
is_local = is_local_path(identifier)
path = normalize_path(identifier) if is_local else identifier
# Add unsloth/ prefix for shorthand HF models
if not is_local and "/" not in identifier:
identifier = f"unsloth/{identifier}"
path = identifier
# Reuse a cached case-variant's exact repo_id spelling to avoid re-downloads (#2592).
if not is_local:
resolved_identifier = resolve_cached_repo_id_case(identifier)
if resolved_identifier != identifier:
logger.info(
"Using cached repo_id casing '%s' for requested '%s'",
resolved_identifier,
identifier,
)
identifier = resolved_identifier
path = resolved_identifier
# Auto-detect GGUF models (check before LoRA/vision detection)
if is_local:
# Variant lookup is directory-only: a loose GGUF echoing its quant would fall to Transformers.
if gguf_variant and Path(path).is_dir():
gguf_file = _find_local_gguf_by_variant(path, gguf_variant)
else:
gguf_file = detect_gguf_model(path)
if gguf_file:
audio_cpp_config = cls._from_audio_cpp_identifier(
identifier, gguf_variant, hf_token, gguf_file = gguf_file
)
if audio_cpp_config is not None:
return audio_cpp_config
display_name = Path(gguf_file).stem
logger.info(f"Detected local GGUF model: {gguf_file}")
# Vision: check base model, then look for mmproj
mmproj_file = None
gguf_is_vision = False
gguf_dir = Path(gguf_file).parent
# Is this a vision model, per export metadata?
base_is_vision = False
meta_path = gguf_dir / "export_metadata.json"
if meta_path.exists():
try:
meta = json.loads(meta_path.read_text(encoding = "utf-8-sig"))
base = meta.get("base_model")
# Exporter writes null base for non-LoRA; skipping avoids the reachability probe.
if base:
with _offline_while_reading(base):
base_is_vision = bool(is_vision_model(base, hf_token = hf_token))
if base_is_vision:
logger.info(f"GGUF base model '{base}' is a vision model")
except Exception as e:
logger.debug(f"Could not read export metadata: {e}")
# Direct file selections may point into a quant subdir while mmproj-*.gguf sits at the root.
companion_root = _local_gguf_companion_search_root(path, gguf_file)
# Per-kind accept: each kind admits a different companion dir, so no shared closure.
def _drafter_accept_for(kind: str) -> Optional[Callable[[str], bool]]:
if drafter_accept is None:
return None
return lambda candidate: drafter_accept(
candidate, gguf_file, kind, companion_root
)
mmproj_file = next(
(
found
for root in (gguf_companion_roots or (companion_root,))
if (
found := detect_mmproj_file(
gguf_file,
search_root = root,
allow_disjoint_search_root = gguf_companion_roots is not None,
accept = (
(lambda candidate: mmproj_accept(candidate, gguf_file))
if mmproj_accept is not None
else None
),
)
)
),
None,
)
if mmproj_file:
gguf_is_vision = True
logger.info(f"Detected mmproj for vision: {mmproj_file}")
elif base_is_vision:
logger.warning(f"Base model is vision but no mmproj file found in {gguf_dir}")
companion_roots = gguf_companion_roots or (companion_root,)
def _find_drafter(detector, kind: str) -> Optional[str]:
return next(
(
found
for root in companion_roots
if (
found := detector(
gguf_file,
search_root = root,
accept = _drafter_accept_for(kind),
)
)
),
None,
)
mtp_file = _find_drafter(detect_mtp_file, "mtp")
if mtp_file:
logger.info(f"Detected MTP drafter: {mtp_file}")
# DSpark and DFlash take the boundary too; skipping it hands load a sidecar it must reject again.
dspark_file = _find_drafter(detect_dspark_file, "dspark")
dflash_file = _find_drafter(detect_dflash_file, "dflash")
return cls(
identifier = identifier,
display_name = display_name,
path = path,
is_local = True,
is_cached = True,
is_vision = gguf_is_vision,
is_lora = False,
is_gguf = True,
gguf_file = gguf_file,
# Carry the caller's variant through, or /status and the deletion guard see the bare label.
gguf_variant = gguf_variant,
gguf_mmproj_file = mmproj_file,
gguf_mtp_file = mtp_file,
gguf_dspark_file = dspark_file,
gguf_dflash_file = dflash_file,
)
else:
# Does the HF repo contain GGUF files?
detect_failures: List[Exception] = []
failure_token = _gguf_remote_detect_failure.set(detect_failures)
try:
with collecting_hub_token_rejections() as token_rejections:
gguf_filename = detect_gguf_model_remote(identifier, hf_token = hf_token)
finally:
_gguf_remote_detect_failure.reset(failure_token)
# A failed listing is not 'no GGUF': a refused repo's cache goes only to an allowed reader.
if not gguf_filename and detect_failures and _is_hub_refusal(detect_failures[-1]):
cached = _refused_repo_cached_gguf(
identifier, gguf_variant, hf_token, owner_session = owner_session
)
if cached is not None:
local_file, cached_variant = cached
logger.warning(
"Hugging Face refused '%s' (%s); loading the downloaded copy.",
identifier,
type(detect_failures[-1]).__name__,
)
from core.inference.llama_cpp import _snapshot_dir_of
# Snapshot root, so a repo-root mmproj or drafter is found.
snapshot = _snapshot_dir_of(local_file)
local_config = cls.from_identifier(
model_id = local_file,
hf_token = hf_token,
gguf_variant = cached_variant,
drafter_accept = drafter_accept,
gguf_companion_roots = gguf_companion_roots
or ((str(snapshot),) if snapshot is not None else None),
mmproj_accept = mmproj_accept,
)
if local_config is not None and local_config.is_gguf:
# Preflight, so /load never unloads the resident model for nothing.
from core.inference.llama_cpp import (
LLAMA_SERVER_NOT_FOUND_DETAIL,
LlamaCppBackend,
LlamaServerNotFoundError,
)
if not LlamaCppBackend._find_llama_server_binary(include_denied = True):
raise LlamaServerNotFoundError(LLAMA_SERVER_NOT_FOUND_DETAIL)
token_rejections.served_from_cache.append(identifier)
return _dataclass_replace(
local_config,
identifier = identifier,
display_name = f"{identifier.split('/')[-1]} ({cached_variant})",
gguf_cache_repo = identifier,
is_local = False,
)
if not gguf_filename and _looks_like_gguf_repo(identifier, gguf_variant):
if detect_failures:
raise GgufRepoUnreadableError(
_gguf_repo_unreadable_message(
identifier,
detect_failures[-1],
token_rejected = token_rejections.refused
and is_rejected_credential_error(detect_failures[-1], hf_token),
)
) from None
if _env_offline():
raise GgufRepoUnreadableError(_gguf_repo_unreadable_message(identifier, None))
if gguf_filename:
# A GGUF only audio.cpp reads never reaches llama-server; its header reveals this before download.
audio_cpp_config = cls._from_audio_cpp_identifier(
identifier, gguf_variant, hf_token, gguf_hint = gguf_filename
)
if audio_cpp_config is not None:
return audio_cpp_config
# Verify the llama-server binary exists before a multi-GB download. include_denied: a locked binary still exists.
from core.inference.llama_cpp import (
LLAMA_SERVER_NOT_FOUND_DETAIL,
LlamaCppBackend,
LlamaServerNotFoundError,
)
if not LlamaCppBackend._find_llama_server_binary(include_denied = True):
raise LlamaServerNotFoundError(LLAMA_SERVER_NOT_FOUND_DETAIL)
# list_gguf_variants() detects vision & resolves the variant
variants, has_vision = list_gguf_variants(identifier, hf_token = hf_token)
variant = gguf_variant
verified_file: Optional[str] = None
if variant:
from core.inference.llama_cpp import (
_gguf_files_for_variant,
cached_gguf_for_load,
)
# A verified cached file also proves that the requested variant exists.
verified_file = cached_gguf_for_load(
identifier, variant, verify_sizes = True, hf_token = hf_token
)
if verified_file is None:
# Reject before the load unloads the resident model; only a live, complete listing proves
# absence.
try:
from huggingface_hub import list_repo_files
repo_files = call_with_anonymous_retry(
lambda token: list_repo_files(identifier, token = token), hf_token
)
except Exception:
repo_files = None
if repo_files and not _gguf_files_for_variant(repo_files, variant):
available = ", ".join(v.quant for v in variants)
raise ValueError(
f"GGUF variant '{variant}' not found in {identifier}. "
f"Available variants: {available}"
)
if not variant: # auto-select best quant
# _pick_best_gguf is order-sensitive: use the scope every resolver uses.
from hub.utils.gguf import default_gguf_scope
variant_filenames = [
v.filename
for v in default_gguf_scope(
variants,
identifier,
key = lambda v: _qualified_variant_name(v.filename, v.quant),
)
]
best = _pick_best_gguf(variant_filenames)
if best:
# Use the same qualified identity the lister advertised; a bare label is ambiguous.
variant = _qualified_variant_name(best, _extract_quant_label(best))
else:
variant = "Q4_K_M" # Fallback — llama-server's own default
from core.inference.llama_cpp import cached_gguf_for_load
verified_file = cached_gguf_for_load(
identifier, variant, verify_sizes = True, hf_token = hf_token
)
verified_gguf = None
if verified_file:
from core.inference.llama_cpp import (
_cached_variant_sizes,
_resolve_repo_id_casing,
)
# Every shard's byte count, so the load can spot damage from here on without asking the Hub again.
sizes = _cached_variant_sizes(
_resolve_repo_id_casing(identifier), variant, verified_file
)
if sizes:
verified_gguf = (identifier, variant, verified_file, sizes)
# A projector hand-added beside the cached weight (#9286).
local_mmproj: Optional[str] = None
if not has_vision and verified_file:
local_mmproj = _hf_cached_local_mmproj(verified_file)
has_vision = local_mmproj is not None
display_name = f"{identifier.split('/')[-1]} ({variant})"
# Debug: re-resolved on every validate/estimate/load; the load path announces the model.
logger.debug(
f"Detected remote GGUF repo '{identifier}', "
f"variant={variant}, vision={has_vision}"
)
return cls(
identifier = identifier,
display_name = display_name,
path = identifier,
is_local = False,
is_cached = False,
is_vision = has_vision,
is_lora = False,
is_gguf = True,
gguf_file = None,
gguf_verified = verified_gguf,
gguf_local_mmproj_file = local_mmproj,
gguf_hf_repo = identifier,
gguf_variant = variant,
)
# Auto-detect LoRA for local paths (adapter_config.json on disk)
if not is_lora and is_local:
detected_base = (
get_base_model_from_lora(path) if _looks_like_lora_adapter(Path(path)) else None
)
if detected_base:
is_lora = True
logger.info(f"Auto-detected local LoRA adapter at '{path}' (base: {detected_base})")
# Remote HF models: when offline, huggingface_hub raises OfflineModeIsEnabled in ~0ms.
if not is_lora and not is_local:
try:
info = _hub_model_info(identifier, hf_token)
repo_files = [s.rfilename for s in info.siblings]
if "adapter_config.json" in repo_files:
is_lora = True
logger.info(f"Auto-detected remote LoRA adapter: '{identifier}'")
except Exception as e:
logger.debug(f"Could not check remote LoRA status for '{identifier}': {e}")
# API may have failed; adapter_config.json could still be cached.
if not is_lora:
for snap in _iter_hf_cache_snapshots(identifier):
if (snap / "adapter_config.json").is_file():
is_lora = True
logger.info(f"Auto-detected cached LoRA adapter: '{identifier}'")
break
base_model = None
if is_lora:
if is_local:
# Local LoRA: read adapter_config.json from disk
base_model = get_base_model_from_lora(path)
else:
# Remote LoRA: fetch adapter_config.json from HF
try:
from huggingface_hub import hf_hub_download
config_path = call_with_anonymous_retry(
lambda token: hf_hub_download(
identifier,
"adapter_config.json",
token = token,
cache_dir = active_hf_hub_cache(),
),
hf_token,
)
with open(config_path, "r", encoding = "utf-8-sig") as f:
adapter_config = json.load(f)
base_model = adapter_config.get("base_model_name_or_path")
if base_model:
logger.info(f"Resolved remote LoRA base model: '{base_model}'")
except Exception as e:
logger.warning(
f"Could not download adapter_config.json for '{identifier}': {e}"
)
if not base_model:
logger.warning(f"Could not determine base model for LoRA '{path}'")
return None
check_model = base_model
else:
check_model = identifier
precision_folder = None
if gguf_variant and not is_local and not is_lora:
from hub.utils.mlx_variants import precision_folder_config_dir
folder_config = precision_folder_config_dir(identifier, gguf_variant.strip(), hf_token)
if folder_config is not None:
precision_folder = gguf_variant.strip()
check_model = str(folder_config)
with _offline_while_reading(check_model):
vision = is_vision_model(check_model, hf_token = hf_token)
audio_type_val = detect_audio_type(check_model, hf_token = hf_token)
has_audio_in = is_audio_input_type(audio_type_val)
display_name = Path(path).name if is_local else identifier.split("/")[-1]
return cls(
gguf_variant = precision_folder,
config_dir = check_model if precision_folder else None,
identifier = identifier,
display_name = display_name,
path = path,
is_local = is_local,
is_cached = is_model_cached(identifier) if not is_local else True,
is_vision = vision,
is_lora = is_lora,
is_audio = audio_type_val is not None and audio_type_val != "audio_vlm",
audio_type = audio_type_val,
has_audio_input = has_audio_in,
base_model = base_model,
)
@classmethod
def _from_audio_cpp_identifier(
cls,
identifier: str,
gguf_variant: Optional[str],
hf_token: Optional[str],
*,
gguf_hint: Optional[str] = None,
gguf_file: Optional[str] = None,
) -> Optional["ModelConfig"]:
"""resolves umbrella or header-checked audio.cpp GGUFs; raises ValueError if unsupported."""
try:
from core.inference import audio_cpp_models
except Exception: # noqa: BLE001 - no audio.cpp support, no audio.cpp id
return None
if gguf_file is not None:
header = audio_cpp_models.read_local_header(gguf_file)
if header is None or not header.is_audio_cpp:
return None
target = gguf_file
resolved_variant = Path(gguf_file).name
elif gguf_hint is None and not audio_cpp_models.is_umbrella_id(identifier):
return None
else:
target = identifier
resolved_variant = gguf_variant
model = audio_cpp_models.resolve(
target,
resolved_variant,
hf_token,
network = not _env_offline(),
gguf_hint = gguf_hint,
)
if model is None:
return None
audio_cpp_models.require_runnable(model, "tts")
from core.inference import audio_cpp_files
return cls(
identifier = model.id,
display_name = model.display_name,
path = model.local_path or model.id,
is_local = bool(model.local_path),
is_cached = audio_cpp_files.is_downloaded(model),
is_vision = False,
is_lora = False,
is_audio = True,
audio_type = model.audio_type,
has_audio_input = False,
gguf_variant = model.variant.key,
base_model = None,
audio_cpp = model,
)
@classmethod
def from_ui_selection(
cls,
dropdown_value: Optional[str],
search_value: Optional[str],
local_models: list = None,
hf_token: Optional[str] = None,
is_lora: bool = False,
) -> Optional["ModelConfig"]:
"""Create a ModelConfig from UI dropdown/search selections (base models and LoRAs)."""
selected = None
if search_value and search_value.strip():
selected = search_value.strip()
elif dropdown_value:
selected = dropdown_value
if not selected:
return None
display_name = selected
# Resolve display names via the 'local_models' parameter
if " (Active)" in selected or " (Ready)" in selected:
clean_display_name = selected.replace(" (Active)", "").replace(" (Ready)", "")
if local_models:
for local_display, local_path in local_models:
if local_display == clean_display_name:
selected = local_path
break
# Strip all UI status indicators to get the final identifier
identifier = selected
for status in UI_STATUS_INDICATORS:
identifier = identifier.replace(status, "")
identifier = identifier.strip()
is_local = is_local_path(identifier)
path = normalize_path(identifier) if is_local else identifier
# Add unsloth/ prefix for shorthand HF models
if not is_local and "/" not in identifier:
identifier = f"unsloth/{identifier}"
path = identifier
if not is_local:
resolved_identifier = resolve_cached_repo_id_case(identifier)
if resolved_identifier != identifier:
identifier = resolved_identifier
path = resolved_identifier
# Legacy constructor: keep local GGUFs on the llama-server path, not FastVisionModel.
if is_local and not is_lora and detect_gguf_model(path):
gguf_config = cls.from_identifier(path, hf_token = hf_token)
if gguf_config is not None:
gguf_config.display_name = display_name
return gguf_config
# --- Base Model and Vision Detection ---
base_model = None
is_vision = False
if is_lora:
# A LoRA MUST have a base model.
base_model = get_base_model_from_lora(path)
if not base_model:
logger.warning(
f"Could not determine base model for LoRA '{path}'. Cannot create config."
)
return None # cannot proceed without a base model
# A LoRA's vision capability comes from its base model.
is_vision = is_vision_model(base_model, hf_token = hf_token)
else:
# Base model: check its own vision status.
is_vision = is_vision_model(identifier, hf_token = hf_token)
from utils.paths import is_model_cached
is_cached = is_model_cached(identifier) if not is_local else True
return cls(
identifier = identifier,
display_name = display_name,
path = path,
is_local = is_local,
is_cached = is_cached,
is_vision = is_vision,
is_lora = is_lora,
base_model = base_model, # None for base models, set for LoRAs
)