Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: chat with a cached upstream model instead of downloading its Unsloth copy (#11307)

oobaboogaa4d77b47d1f4
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index c9b95fe987e..39cbe3e1009 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -77,6 +77,66 @@ def _hf_token_for_loader(hf_token: Optional[str] | bool) -> Optional[str] | bool
return hf_token.strip() if isinstance(hf_token, str) and hf_token.strip() else None
+def _exact_model_name_for_load(config: ModelConfig, load_in_4bit: bool) -> Optional[str]:
+ """The repo id to hand the loader verbatim, or None to let Unsloth's mapper choose.
+
+ ``config.path`` when the mapped repo is not on disk and the user's own weights are.
+ The mapped repo's cached spelling when it IS on disk under another case: the mapper
+ emits one lowercased id and huggingface_hub keys the cache directory on the id
+ verbatim, so asking for any other spelling re-downloads it (huggingface_hub#3838).
+ """
+ if config.is_local or config.is_lora or not config.path:
+ return None
+ try:
+ from unsloth.models import loader, loader_utils
+
+ # ModelScope downloads every repo id to its own cache, so the HF cache says nothing here.
+ if loader.USE_MODELSCOPE:
+ return None
+ # Resolve the repo the loader will fetch on this host, after its bitsandbytes fallbacks.
+ if not loader.ALLOW_BITSANDBYTES:
+ load_in_4bit = False
+ name = config.path.lower()
+ if name in loader_utils.BAD_MAPPINGS:
+ return None
+ table = (
+ loader_utils.FLOAT_TO_INT_MAPPER if load_in_4bit else loader_utils.MAP_TO_UNSLOTH_16bit
+ )
+ # Avoid fetching the remote mapper for unknown names.
+ if name not in table:
+ return None
+ target = loader_utils.get_model_name(config.path, load_in_4bit = load_in_4bit)
+ if target and not loader.ALLOW_PREQUANTIZED_MODELS:
+ target = loader._strip_unsloth_bnb_4bit_suffix(target)
+ except Exception as e:
+ logger.debug(f"Could not resolve the Unsloth mapping for {config.path}: {e}")
+ return None
+ if not target or target.lower() == name:
+ return None
+ from utils.utils import (
+ active_hf_cache_loadable_snapshot,
+ active_hf_cache_repo_spelling,
+ )
+
+ cached_target = active_hf_cache_repo_spelling(target)
+ if cached_target == target:
+ return None # on disk under the name the mapper will ask for: nothing to do
+ if cached_target is not None:
+ logger.info(f"Loading cached {cached_target} instead of downloading {target} again")
+ return cached_target
+ snapshot = active_hf_cache_loadable_snapshot(config.path)
+ if snapshot is None:
+ return None
+ try:
+ checkpoint = json.loads((snapshot / "config.json").read_text(encoding = "utf-8"))
+ except (OSError, ValueError):
+ return None
+ if not isinstance(checkpoint, dict) or checkpoint.get("quantization_config") is not None:
+ return None
+ logger.info(f"Loading cached {config.path} as named instead of downloading {target}")
+ return config.path
+
+
class HarmonyTextStreamer:
"""Streaming text decoder for the gpt-oss harmony channel protocol.
@@ -809,15 +869,19 @@ class InferenceBackend:
logger.info(f"Loading {model_type} model{adapter_info}: {model_name}")
log_gpu_memory(f"Before loading {model_name}")
+ exact_model_name = _exact_model_name_for_load(config, load_in_4bit)
+ use_exact_model_name = exact_model_name is not None
+ load_path = exact_model_name or config.path
if config.is_vision:
model, processor = FastVisionModel.from_pretrained(
- model_name = config.path,
+ model_name = load_path,
max_seq_length = max_seq_length,
dtype = dtype,
load_in_4bit = load_in_4bit,
device_map = device_map,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
+ use_exact_model_name = use_exact_model_name,
)
FastVisionModel.for_inference(model)
@@ -859,13 +923,14 @@ class InferenceBackend:
else:
model, tokenizer = FastLanguageModel.from_pretrained(
- model_name = config.path,
+ model_name = load_path,
max_seq_length = max_seq_length,
dtype = dtype,
load_in_4bit = load_in_4bit,
device_map = device_map,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
+ use_exact_model_name = use_exact_model_name,
)
FastLanguageModel.for_inference(model)
diff --git a/studio/backend/tests/test_inference_cached_repo_exact_load.py b/studio/backend/tests/test_inference_cached_repo_exact_load.py
new file mode 100644
index 00000000000..919f03b1f59
--- /dev/null
+++ b/studio/backend/tests/test_inference_cached_repo_exact_load.py
@@ -0,0 +1,329 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Test which repo id a chat load is handed, and when Unsloth's mapper picks it."""
+
+from __future__ import annotations
+
+import importlib
+import importlib.machinery
+import json
+import sys
+import types
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parent.parent
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+_STUBBED: list[str] = []
+
+
+def _stub_if_missing(
+ name,
+ attrs = (),
+ named_spec = False,
+):
+ """Stub dependencies unavailable in backend CI so inference can be imported."""
+ if name in sys.modules:
+ return
+ try:
+ importlib.import_module(name)
+ return
+ except Exception: # noqa: BLE001 - unusable here either way, so stub it
+ pass
+ _STUBBED.append(name)
+ module = types.ModuleType(name)
+ module.__spec__ = importlib.machinery.ModuleSpec(name, None) if named_spec else None
+ module.__version__ = "0.0.0"
+ module.__getattr__ = lambda _attr: MagicMock()
+ for attr in attrs:
+ setattr(module, attr, MagicMock())
+ sys.modules[name] = module
+ parent, _, child = name.rpartition(".")
+ if parent and parent in sys.modules:
+ setattr(sys.modules[parent], child, module)
+
+
+for _torchao in (
+ "torchao",
+ "torchao.prototype",
+ "torchao.prototype.safetensors",
+ "torchao.prototype.safetensors.safetensors_support",
+ "torchao.prototype.safetensors.safetensors_utils",
+ "torchao.quantization",
+ "torchao.dtypes",
+ "torchao.float8",
+ "torchao.utils",
+):
+ _stub_if_missing(_torchao, named_spec = True)
+
+_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
+_stub_if_missing("unsloth_zoo")
+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))
+
+from core.inference.inference import _exact_model_name_for_load # noqa: E402
+
+for _name in reversed(_STUBBED):
+ sys.modules.pop(_name, None)
+
+
+UPSTREAM = "Qwen/Qwen3-VL-4B-Instruct"
+PREQUANT = "unsloth/qwen3-vl-4b-instruct-unsloth-bnb-4bit"
+UNSLOTH_16BIT = "unsloth/Qwen3-VL-4B-Instruct"
+
+
+def mapper(monkeypatch):
+ """Stub the mapper with one model."""
+ loader_utils = types.ModuleType("unsloth.models.loader_utils")
+ loader_utils.FLOAT_TO_INT_MAPPER = {UPSTREAM.lower(): PREQUANT, UNSLOTH_16BIT.lower(): PREQUANT}
+ loader_utils.MAP_TO_UNSLOTH_16bit = {UPSTREAM.lower(): UNSLOTH_16BIT}
+ loader_utils.BAD_MAPPINGS = {}
+ loader_utils.calls = []
+
+ def get_model_name(model_name, load_in_4bit = True):
+ loader_utils.calls.append(model_name)
+ table = (
+ loader_utils.FLOAT_TO_INT_MAPPER if load_in_4bit else loader_utils.MAP_TO_UNSLOTH_16bit
+ )
+ return table.get(model_name.lower(), model_name)
+
+ loader_utils.get_model_name = get_model_name
+ loader = types.ModuleType("unsloth.models.loader")
+ loader.ALLOW_BITSANDBYTES = True
+ loader.ALLOW_PREQUANTIZED_MODELS = True
+ loader.USE_MODELSCOPE = False
+ loader._strip_unsloth_bnb_4bit_suffix = lambda name: name.removesuffix("-unsloth-bnb-4bit")
+ loader_utils.loader = loader
+ models = types.ModuleType("unsloth.models")
+ models.loader_utils = loader_utils
+ models.loader = loader
+ unsloth = types.ModuleType("unsloth")
+ unsloth.models = models
+ monkeypatch.setitem(sys.modules, "unsloth", unsloth)
+ monkeypatch.setitem(sys.modules, "unsloth.models", models)
+ monkeypatch.setitem(sys.modules, "unsloth.models.loader_utils", loader_utils)
+ monkeypatch.setitem(sys.modules, "unsloth.models.loader", loader)
+ return loader_utils
+
+
+def hub_cache(tmp_path, monkeypatch):
+ import utils.hf_cache_settings as hf_cache_settings
+ monkeypatch.setattr(hf_cache_settings, "active_hf_hub_cache", lambda: str(tmp_path))
+ return tmp_path
+
+
+def _cache_repo(
+ hub_cache: Path,
+ repo_id: str,
+ *,
+ missing_shard: bool = False,
+ config: dict | None = None,
+) -> None:
+ repo_dir = hub_cache / ("models--" + repo_id.replace("/", "--"))
+ snapshot = repo_dir / "snapshots" / "0123abcd"
+ snapshot.mkdir(parents = True)
+ (repo_dir / "refs").mkdir()
+ (repo_dir / "refs" / "main").write_text("0123abcd")
+ (snapshot / "config.json").write_text(json.dumps(config or {}))
+ if not missing_shard:
+ (snapshot / "model.safetensors").write_bytes(b"\0" * 64)
+ return
+ shards = [f"model-0000{i}-of-00002.safetensors" for i in (1, 2)]
+ (snapshot / "model.safetensors.index.json").write_text(
+ json.dumps({"weight_map": {"a": shards[0], "b": shards[1]}})
+ )
+ (snapshot / shards[0]).write_bytes(b"\0" * 64)
+
+
+def _config(path = UPSTREAM, **overrides):
+ return types.SimpleNamespace(**{"is_local": False, "is_lora": False, "path": path, **overrides})
+
+
[email protected]("load_in_4bit", [True, False])
+def test_cached_repo_loads_as_named_when_the_swap_target_is_missing(
+ mapper, hub_cache, load_in_4bit
+):
+ _cache_repo(hub_cache, UPSTREAM)
+
+ assert _exact_model_name_for_load(_config(), load_in_4bit) == UPSTREAM
+
+
[email protected]("load_in_4bit", [True, False])
+def test_cached_swap_target_keeps_the_swap(mapper, hub_cache, load_in_4bit):
+ _cache_repo(hub_cache, UPSTREAM)
+ _cache_repo(hub_cache, PREQUANT if load_in_4bit else UNSLOTH_16BIT)
+
+ assert _exact_model_name_for_load(_config(), load_in_4bit) is None
+
+
[email protected]("load_in_4bit", [True, False])
+def test_a_differently_cased_swap_target_is_loaded_under_its_cached_spelling(
+ mapper, hub_cache, load_in_4bit
+):
+ """The mapper lowercases its targets; the Hub, and Studio's own downloads, do not.
+
+ Answering "cached, let the mapper have it" is not enough: the mapper hands the
+ loader its own lowercase spelling, huggingface_hub keys the cache directory on the
+ id verbatim (huggingface/huggingface_hub#3838), and the copy on disk is fetched a
+ second time. Measured at 1356 MiB on Qwen3-1.7B with Xet dedup off.
+ """
+ target = (PREQUANT if load_in_4bit else UNSLOTH_16BIT).upper()
+ _cache_repo(hub_cache, UPSTREAM)
+ _cache_repo(hub_cache, target)
+
+ assert _exact_model_name_for_load(_config(), load_in_4bit) == target
+
+
+def test_a_differently_cased_target_short_a_shard_still_loads_as_named(mapper, hub_cache):
+ # Matching the case is not enough on its own: the copy still has to be loadable.
+ _cache_repo(hub_cache, UPSTREAM)
+ _cache_repo(hub_cache, PREQUANT.upper(), missing_shard = True)
+
+ assert _exact_model_name_for_load(_config(), True) == UPSTREAM
+
+
+def test_without_bitsandbytes_the_16bit_target_decides(mapper, hub_cache):
+ # The loader drops 4-bit here, so an unusable cached prequant does not count.
+ mapper.loader.ALLOW_BITSANDBYTES = False
+ _cache_repo(hub_cache, UPSTREAM)
+ _cache_repo(hub_cache, PREQUANT)
+ assert _exact_model_name_for_load(_config(), True) == UPSTREAM
+
+ _cache_repo(hub_cache, UNSLOTH_16BIT)
+ assert _exact_model_name_for_load(_config(), True) is None
+
+
+def test_without_prequantized_models_the_stripped_target_decides(mapper, hub_cache):
+ mapper.loader.ALLOW_PREQUANTIZED_MODELS = False
+ _cache_repo(hub_cache, UPSTREAM)
+ _cache_repo(hub_cache, PREQUANT)
+ assert _exact_model_name_for_load(_config(), True) == UPSTREAM
+
+ _cache_repo(hub_cache, PREQUANT.removesuffix("-unsloth-bnb-4bit"))
+ assert _exact_model_name_for_load(_config(), True) is None
+
+
+def test_modelscope_keeps_the_swap(mapper, hub_cache):
+ mapper.loader.USE_MODELSCOPE = True
+ _cache_repo(hub_cache, UPSTREAM)
+
+ assert _exact_model_name_for_load(_config(), True) is None
+
+
+def test_named_repo_short_a_shard_keeps_the_swap(mapper, hub_cache):
+ _cache_repo(hub_cache, UPSTREAM, missing_shard = True)
+
+ assert _exact_model_name_for_load(_config(), True) is None
+
+
+def test_uncached_named_repo_keeps_the_swap(mapper, hub_cache):
+ assert _exact_model_name_for_load(_config(), True) is None
+
+
+def test_checkpoint_in_a_previous_cache_root_keeps_the_swap(
+ mapper, hub_cache, tmp_path_factory, monkeypatch
+):
+ # The loader cannot reuse weights from a previous cache root.
+ from utils.utils import hf_cache_snapshot_is_loadable
+
+ previous = tmp_path_factory.mktemp("previous-hub-cache")
+ _cache_repo(previous, UPSTREAM)
+ monkeypatch.setenv("HF_HUB_CACHE", str(previous))
+
+ assert hf_cache_snapshot_is_loadable(UPSTREAM) is True
+ assert _exact_model_name_for_load(_config(), True) is None
+
+
+def test_known_bad_repo_keeps_the_swap(mapper, hub_cache):
+ _cache_repo(hub_cache, UPSTREAM)
+ mapper.BAD_MAPPINGS[UPSTREAM.lower()] = UNSLOTH_16BIT
+
+ assert _exact_model_name_for_load(_config(), True) is None
+
+
[email protected]("load_in_4bit", [True, False])
+def test_quantized_checkpoint_keeps_the_swap(mapper, hub_cache, load_in_4bit):
+ # MXFP4 weights cannot satisfy the requested 4-bit or 16-bit load as-is.
+ _cache_repo(hub_cache, UPSTREAM, config = {"quantization_config": {"quant_method": "mxfp4"}})
+
+ assert _exact_model_name_for_load(_config(), load_in_4bit) is None
+
+
+def test_name_the_installed_tables_do_not_know_skips_the_mapper(mapper, hub_cache):
+ # Leave remote mapper lookups to the loader.
+ _cache_repo(hub_cache, "someone/custom-model")
+
+ assert _exact_model_name_for_load(_config("someone/custom-model"), True) is None
+ assert mapper.calls == []
+
+
[email protected]("overrides", [{"is_local": True}, {"is_lora": True}])
+def test_local_and_adapter_loads_are_untouched(mapper, hub_cache, overrides):
+ _cache_repo(hub_cache, UPSTREAM)
+
+ assert _exact_model_name_for_load(_config(**overrides), True) is None
+ assert mapper.calls == []
+
+
+def test_load_model_hands_the_verdict_to_both_loaders():
+ """Dropping either keyword argument restores the download and leaves every other test
+ here passing, so the wiring needs its own assertion. ``model_name`` matters as much as
+ the flag: the verdict can name the swap target's cached spelling, and passing
+ ``config.path`` there would load the wrong repo. A real ``load_model`` call cannot run
+ here (no weights, no network, no unsloth), so read the call sites.
+ """
+ import ast
+
+ source = (_BACKEND / "core/inference/inference.py").read_text(encoding = "utf-8")
+ load_model = next(
+ node
+ for node in ast.walk(ast.parse(source))
+ if isinstance(node, ast.FunctionDef) and node.name == "load_model"
+ )
+
+ def assigned_from(predicate):
+ return {
+ target.id
+ for node in ast.walk(load_model)
+ if isinstance(node, ast.Assign) and predicate(node.value)
+ for target in node.targets
+ if isinstance(target, ast.Name)
+ }
+
+ verdicts = assigned_from(
+ lambda v: isinstance(v, ast.Call)
+ and isinstance(v.func, ast.Name)
+ and v.func.id == "_exact_model_name_for_load"
+ )
+ assert verdicts, "load_model never calls _exact_model_name_for_load"
+ # Whatever the flag and the path are called, they have to be computed FROM the
+ # verdict, so a rename stays green and a hardcoded True or config.path does not.
+ derived = verdicts | assigned_from(
+ lambda v: any(isinstance(n, ast.Name) and n.id in verdicts for n in ast.walk(v))
+ )
+
+ def receives(kwarg):
+ return {
+ node.func.value.id
+ for node in ast.walk(load_model)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "from_pretrained"
+ and isinstance(node.func.value, ast.Name)
+ and node.func.value.id in ("FastLanguageModel", "FastVisionModel")
+ and any(
+ kw.arg == kwarg and isinstance(kw.value, ast.Name) and kw.value.id in derived
+ for kw in node.keywords
+ )
+ }
+
+ both = {"FastLanguageModel", "FastVisionModel"}
+ for kwarg in ("use_exact_model_name", "model_name"):
+ assert receives(kwarg) == both, f"only {sorted(receives(kwarg))} receive {kwarg}"
diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py
index 2f20a8758c4..a753f2a782e 100644
--- a/studio/backend/utils/utils.py
+++ b/studio/backend/utils/utils.py
@@ -575,18 +575,18 @@ def snapshot_has_st_weights(model_name: str) -> bool:
return cached_st_source(model_name) is not None
-def _snapshot_in_root(cache_root: Path, repo_id: str) -> Optional[Path]:
- """``repo_id``'s main-revision snapshot under exactly ``cache_root``, or None."""
+def _repo_folder_name(repo_id: str) -> str:
+ """The cache directory name huggingface_hub gives ``repo_id``."""
try:
from huggingface_hub.file_download import repo_folder_name
except Exception:
- repo_folder_name = None
+ return "models--" + repo_id.replace("/", "--")
+ return repo_folder_name(repo_id = repo_id, repo_type = "model")
+
+
+def _snapshot_in_repo_dir(repo_dir: Path) -> Optional[Path]:
+ """``repo_dir``'s main-revision snapshot, or None. ``repo_dir`` is a cache entry the caller already picked, so no repo id is re-derived from its name."""
try:
- if repo_folder_name is not None:
- folder = repo_folder_name(repo_id = repo_id, repo_type = "model")
- else:
- folder = "models--" + repo_id.replace("/", "--")
- repo_dir = cache_root / folder
ref = repo_dir / "refs" / "main"
if not ref.is_file():
return None
@@ -600,6 +600,14 @@ def _snapshot_in_root(cache_root: Path, repo_id: str) -> Optional[Path]:
return None
+def _snapshot_in_root(cache_root: Path, repo_id: str) -> Optional[Path]:
+ """``repo_id``'s main-revision snapshot under exactly ``cache_root``, or None."""
+ try:
+ return _snapshot_in_repo_dir(cache_root / _repo_folder_name(repo_id))
+ except (OSError, UnicodeDecodeError):
+ return None
+
+
def hf_cache_snapshot_dir_for_repo(repo_id: str) -> Optional[Path]:
"""Snapshot dir for exactly ``repo_id``, with no alias expansion. ``hf_cache_snapshot_dir`` answers "is this model cached anywhere", trying the ST alias, so asking it about a literal slashless name can return the namespaced snapshot. A caller that has to report WHICH repo supplied the weights needs this one instead, or it pairs the alias's directory with the literal id and sends verification at a repo that does not exist."""
for cache_root in _hf_cache_roots():
@@ -671,6 +679,43 @@ def hf_cache_snapshot_is_loadable(model_name: str) -> bool:
return snapshot_is_loadable(snapshot, model_name)
+def active_hf_cache_loadable_snapshot(repo_id: str) -> Optional[Path]:
+ """Return a loadable snapshot from the active cache only, or None."""
+ try:
+ from utils.hf_cache_settings import active_hf_hub_cache
+ snapshot = _snapshot_in_root(_expand_path(active_hf_hub_cache()), repo_id)
+ except Exception:
+ return None
+ return snapshot if snapshot is not None and snapshot_is_loadable(snapshot, repo_id) else None
+
+
+def active_hf_cache_repo_spelling(repo_id: str) -> Optional[str]:
+ """The spelling ``repo_id`` is actually cached under in the active cache, or None.
+
+ The Hub is case insensitive, the cache directories it creates are not, and huggingface_hub keys the directory on the id verbatim, so one checkpoint sits under whichever spelling first asked for it (huggingface/huggingface_hub#3838). Returning the spelling rather than a bool is what lets a caller LOAD the copy it found: asking for any other casing downloads it again.
+ """
+ if active_hf_cache_loadable_snapshot(repo_id) is not None:
+ return repo_id
+ try:
+ from utils.hf_cache_settings import active_hf_hub_cache
+
+ exact = _repo_folder_name(repo_id)
+ wanted = exact.lower()
+ prefix = len("models--")
+ for repo_dir in _expand_path(active_hf_hub_cache()).iterdir():
+ if repo_dir.name.lower() != wanted or repo_dir.name == exact:
+ continue
+ # Repo ids cannot contain '--' (huggingface_hub.utils.validate_repo_id), so
+ # the directory name maps back to exactly one id.
+ variant = repo_dir.name[prefix:].replace("--", "/")
+ snapshot = _snapshot_in_repo_dir(repo_dir)
+ if snapshot is not None and snapshot_is_loadable(snapshot, variant):
+ return variant
+ except Exception:
+ return None
+ return None
+
+
def snapshot_is_loadable(snapshot, model_name: str) -> bool:
"""``hf_cache_snapshot_is_loadable`` for a snapshot the caller already has. A caller that picked a specific directory has to have THAT one judged: the lookup above expands the ST alias within each cache root while an exact per-repo lookup walks the roots for one id, so with several roots configured the two can land on different snapshots and the verdict would belong to a directory nobody is going to load."""
try: