Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: chat with a cached upstream model instead of downloading its Unsloth copy (#11307)
oobaboogaa4d77b47d1f4
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex c9b95fe987e..39cbe3e1009 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -77,6 +77,66 @@ def _hf_token_for_loader(hf_token: Optional[str] | bool) -> Optional[str] | boolreturn hf_token.strip() if isinstance(hf_token, str) and hf_token.strip() else None+def _exact_model_name_for_load(config: ModelConfig, load_in_4bit: bool) -> Optional[str]:+ """The repo id to hand the loader verbatim, or None to let Unsloth's mapper choose.++ ``config.path`` when the mapped repo is not on disk and the user's own weights are.+ The mapped repo's cached spelling when it IS on disk under another case: the mapper+ emits one lowercased id and huggingface_hub keys the cache directory on the id+ verbatim, so asking for any other spelling re-downloads it (huggingface_hub#3838).+ """+ if config.is_local or config.is_lora or not config.path:+ return None+ try:+ from unsloth.models import loader, loader_utils++ # ModelScope downloads every repo id to its own cache, so the HF cache says nothing here.+ if loader.USE_MODELSCOPE:+ return None+ # Resolve the repo the loader will fetch on this host, after its bitsandbytes fallbacks.+ if not loader.ALLOW_BITSANDBYTES:+ load_in_4bit = False+ name = config.path.lower()+ if name in loader_utils.BAD_MAPPINGS:+ return None+ table = (+ loader_utils.FLOAT_TO_INT_MAPPER if load_in_4bit else loader_utils.MAP_TO_UNSLOTH_16bit+ )+ # Avoid fetching the remote mapper for unknown names.+ if name not in table:+ return None+ target = loader_utils.get_model_name(config.path, load_in_4bit = load_in_4bit)+ if target and not loader.ALLOW_PREQUANTIZED_MODELS:+ target = loader._strip_unsloth_bnb_4bit_suffix(target)+ except Exception as e:+ logger.debug(f"Could not resolve the Unsloth mapping for {config.path}: {e}")+ return None+ if not target or target.lower() == name:+ return None+ from utils.utils import (+ active_hf_cache_loadable_snapshot,+ active_hf_cache_repo_spelling,+ )++ cached_target = active_hf_cache_repo_spelling(target)+ if cached_target == target:+ return None # on disk under the name the mapper will ask for: nothing to do+ if cached_target is not None:+ logger.info(f"Loading cached {cached_target} instead of downloading {target} again")+ return cached_target+ snapshot = active_hf_cache_loadable_snapshot(config.path)+ if snapshot is None:+ return None+ try:+ checkpoint = json.loads((snapshot / "config.json").read_text(encoding = "utf-8"))+ except (OSError, ValueError):+ return None+ if not isinstance(checkpoint, dict) or checkpoint.get("quantization_config") is not None:+ return None+ logger.info(f"Loading cached {config.path} as named instead of downloading {target}")+ return config.path++class HarmonyTextStreamer:"""Streaming text decoder for the gpt-oss harmony channel protocol.@@ -809,15 +869,19 @@ class InferenceBackend:logger.info(f"Loading {model_type} model{adapter_info}: {model_name}")log_gpu_memory(f"Before loading {model_name}")+ exact_model_name = _exact_model_name_for_load(config, load_in_4bit)+ use_exact_model_name = exact_model_name is not None+ load_path = exact_model_name or config.pathif config.is_vision:model, processor = FastVisionModel.from_pretrained(- model_name = config.path,+ model_name = load_path,max_seq_length = max_seq_length,dtype = dtype,load_in_4bit = load_in_4bit,device_map = device_map,token = _hf_token_for_loader(hf_token),trust_remote_code = trust_remote_code,+ use_exact_model_name = use_exact_model_name,)FastVisionModel.for_inference(model)@@ -859,13 +923,14 @@ class InferenceBackend:else:model, tokenizer = FastLanguageModel.from_pretrained(- model_name = config.path,+ model_name = load_path,max_seq_length = max_seq_length,dtype = dtype,load_in_4bit = load_in_4bit,device_map = device_map,token = _hf_token_for_loader(hf_token),trust_remote_code = trust_remote_code,+ use_exact_model_name = use_exact_model_name,)FastLanguageModel.for_inference(model)diff --git a/studio/backend/tests/test_inference_cached_repo_exact_load.py b/studio/backend/tests/test_inference_cached_repo_exact_load.pynew file mode 100644index 00000000000..919f03b1f59--- /dev/null+++ b/studio/backend/tests/test_inference_cached_repo_exact_load.py@@ -0,0 +1,329 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Test which repo id a chat load is handed, and when Unsloth's mapper picks it."""++from __future__ import annotations++import importlib+import importlib.machinery+import json+import sys+import types+from pathlib import Path+from unittest.mock import MagicMock++import pytest++_BACKEND = Path(__file__).resolve().parent.parent+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++_STUBBED: list[str] = []+++def _stub_if_missing(+ name,+ attrs = (),+ named_spec = False,+):+ """Stub dependencies unavailable in backend CI so inference can be imported."""+ if name in sys.modules:+ return+ try:+ importlib.import_module(name)+ return+ except Exception: # noqa: BLE001 - unusable here either way, so stub it+ pass+ _STUBBED.append(name)+ module = types.ModuleType(name)+ module.__spec__ = importlib.machinery.ModuleSpec(name, None) if named_spec else None+ module.__version__ = "0.0.0"+ module.__getattr__ = lambda _attr: MagicMock()+ for attr in attrs:+ setattr(module, attr, MagicMock())+ sys.modules[name] = module+ parent, _, child = name.rpartition(".")+ if parent and parent in sys.modules:+ setattr(sys.modules[parent], child, module)+++for _torchao in (+ "torchao",+ "torchao.prototype",+ "torchao.prototype.safetensors",+ "torchao.prototype.safetensors.safetensors_support",+ "torchao.prototype.safetensors.safetensors_utils",+ "torchao.quantization",+ "torchao.dtypes",+ "torchao.float8",+ "torchao.utils",+):+ _stub_if_missing(_torchao, named_spec = True)++_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))+_stub_if_missing("unsloth_zoo")+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))++from core.inference.inference import _exact_model_name_for_load # noqa: E402++for _name in reversed(_STUBBED):+ sys.modules.pop(_name, None)+++UPSTREAM = "Qwen/Qwen3-VL-4B-Instruct"+PREQUANT = "unsloth/qwen3-vl-4b-instruct-unsloth-bnb-4bit"+UNSLOTH_16BIT = "unsloth/Qwen3-VL-4B-Instruct"+++def mapper(monkeypatch):+ """Stub the mapper with one model."""+ loader_utils = types.ModuleType("unsloth.models.loader_utils")+ loader_utils.FLOAT_TO_INT_MAPPER = {UPSTREAM.lower(): PREQUANT, UNSLOTH_16BIT.lower(): PREQUANT}+ loader_utils.MAP_TO_UNSLOTH_16bit = {UPSTREAM.lower(): UNSLOTH_16BIT}+ loader_utils.BAD_MAPPINGS = {}+ loader_utils.calls = []++ def get_model_name(model_name, load_in_4bit = True):+ loader_utils.calls.append(model_name)+ table = (+ loader_utils.FLOAT_TO_INT_MAPPER if load_in_4bit else loader_utils.MAP_TO_UNSLOTH_16bit+ )+ return table.get(model_name.lower(), model_name)++ loader_utils.get_model_name = get_model_name+ loader = types.ModuleType("unsloth.models.loader")+ loader.ALLOW_BITSANDBYTES = True+ loader.ALLOW_PREQUANTIZED_MODELS = True+ loader.USE_MODELSCOPE = False+ loader._strip_unsloth_bnb_4bit_suffix = lambda name: name.removesuffix("-unsloth-bnb-4bit")+ loader_utils.loader = loader+ models = types.ModuleType("unsloth.models")+ models.loader_utils = loader_utils+ models.loader = loader+ unsloth = types.ModuleType("unsloth")+ unsloth.models = models+ monkeypatch.setitem(sys.modules, "unsloth", unsloth)+ monkeypatch.setitem(sys.modules, "unsloth.models", models)+ monkeypatch.setitem(sys.modules, "unsloth.models.loader_utils", loader_utils)+ monkeypatch.setitem(sys.modules, "unsloth.models.loader", loader)+ return loader_utils+++def hub_cache(tmp_path, monkeypatch):+ import utils.hf_cache_settings as hf_cache_settings+ monkeypatch.setattr(hf_cache_settings, "active_hf_hub_cache", lambda: str(tmp_path))+ return tmp_path+++def _cache_repo(+ hub_cache: Path,+ repo_id: str,+ *,+ missing_shard: bool = False,+ config: dict | None = None,+) -> None:+ repo_dir = hub_cache / ("models--" + repo_id.replace("/", "--"))+ snapshot = repo_dir / "snapshots" / "0123abcd"+ snapshot.mkdir(parents = True)+ (repo_dir / "refs").mkdir()+ (repo_dir / "refs" / "main").write_text("0123abcd")+ (snapshot / "config.json").write_text(json.dumps(config or {}))+ if not missing_shard:+ (snapshot / "model.safetensors").write_bytes(b"\0" * 64)+ return+ shards = [f"model-0000{i}-of-00002.safetensors" for i in (1, 2)]+ (snapshot / "model.safetensors.index.json").write_text(+ json.dumps({"weight_map": {"a": shards[0], "b": shards[1]}})+ )+ (snapshot / shards[0]).write_bytes(b"\0" * 64)+++def _config(path = UPSTREAM, **overrides):+ return types.SimpleNamespace(**{"is_local": False, "is_lora": False, "path": path, **overrides})++[email protected]("load_in_4bit", [True, False])+def test_cached_repo_loads_as_named_when_the_swap_target_is_missing(+ mapper, hub_cache, load_in_4bit+):+ _cache_repo(hub_cache, UPSTREAM)++ assert _exact_model_name_for_load(_config(), load_in_4bit) == UPSTREAM++[email protected]("load_in_4bit", [True, False])+def test_cached_swap_target_keeps_the_swap(mapper, hub_cache, load_in_4bit):+ _cache_repo(hub_cache, UPSTREAM)+ _cache_repo(hub_cache, PREQUANT if load_in_4bit else UNSLOTH_16BIT)++ assert _exact_model_name_for_load(_config(), load_in_4bit) is None++[email protected]("load_in_4bit", [True, False])+def test_a_differently_cased_swap_target_is_loaded_under_its_cached_spelling(+ mapper, hub_cache, load_in_4bit+):+ """The mapper lowercases its targets; the Hub, and Studio's own downloads, do not.++ Answering "cached, let the mapper have it" is not enough: the mapper hands the+ loader its own lowercase spelling, huggingface_hub keys the cache directory on the+ id verbatim (huggingface/huggingface_hub#3838), and the copy on disk is fetched a+ second time. Measured at 1356 MiB on Qwen3-1.7B with Xet dedup off.+ """+ target = (PREQUANT if load_in_4bit else UNSLOTH_16BIT).upper()+ _cache_repo(hub_cache, UPSTREAM)+ _cache_repo(hub_cache, target)++ assert _exact_model_name_for_load(_config(), load_in_4bit) == target+++def test_a_differently_cased_target_short_a_shard_still_loads_as_named(mapper, hub_cache):+ # Matching the case is not enough on its own: the copy still has to be loadable.+ _cache_repo(hub_cache, UPSTREAM)+ _cache_repo(hub_cache, PREQUANT.upper(), missing_shard = True)++ assert _exact_model_name_for_load(_config(), True) == UPSTREAM+++def test_without_bitsandbytes_the_16bit_target_decides(mapper, hub_cache):+ # The loader drops 4-bit here, so an unusable cached prequant does not count.+ mapper.loader.ALLOW_BITSANDBYTES = False+ _cache_repo(hub_cache, UPSTREAM)+ _cache_repo(hub_cache, PREQUANT)+ assert _exact_model_name_for_load(_config(), True) == UPSTREAM++ _cache_repo(hub_cache, UNSLOTH_16BIT)+ assert _exact_model_name_for_load(_config(), True) is None+++def test_without_prequantized_models_the_stripped_target_decides(mapper, hub_cache):+ mapper.loader.ALLOW_PREQUANTIZED_MODELS = False+ _cache_repo(hub_cache, UPSTREAM)+ _cache_repo(hub_cache, PREQUANT)+ assert _exact_model_name_for_load(_config(), True) == UPSTREAM++ _cache_repo(hub_cache, PREQUANT.removesuffix("-unsloth-bnb-4bit"))+ assert _exact_model_name_for_load(_config(), True) is None+++def test_modelscope_keeps_the_swap(mapper, hub_cache):+ mapper.loader.USE_MODELSCOPE = True+ _cache_repo(hub_cache, UPSTREAM)++ assert _exact_model_name_for_load(_config(), True) is None+++def test_named_repo_short_a_shard_keeps_the_swap(mapper, hub_cache):+ _cache_repo(hub_cache, UPSTREAM, missing_shard = True)++ assert _exact_model_name_for_load(_config(), True) is None+++def test_uncached_named_repo_keeps_the_swap(mapper, hub_cache):+ assert _exact_model_name_for_load(_config(), True) is None+++def test_checkpoint_in_a_previous_cache_root_keeps_the_swap(+ mapper, hub_cache, tmp_path_factory, monkeypatch+):+ # The loader cannot reuse weights from a previous cache root.+ from utils.utils import hf_cache_snapshot_is_loadable++ previous = tmp_path_factory.mktemp("previous-hub-cache")+ _cache_repo(previous, UPSTREAM)+ monkeypatch.setenv("HF_HUB_CACHE", str(previous))++ assert hf_cache_snapshot_is_loadable(UPSTREAM) is True+ assert _exact_model_name_for_load(_config(), True) is None+++def test_known_bad_repo_keeps_the_swap(mapper, hub_cache):+ _cache_repo(hub_cache, UPSTREAM)+ mapper.BAD_MAPPINGS[UPSTREAM.lower()] = UNSLOTH_16BIT++ assert _exact_model_name_for_load(_config(), True) is None++[email protected]("load_in_4bit", [True, False])+def test_quantized_checkpoint_keeps_the_swap(mapper, hub_cache, load_in_4bit):+ # MXFP4 weights cannot satisfy the requested 4-bit or 16-bit load as-is.+ _cache_repo(hub_cache, UPSTREAM, config = {"quantization_config": {"quant_method": "mxfp4"}})++ assert _exact_model_name_for_load(_config(), load_in_4bit) is None+++def test_name_the_installed_tables_do_not_know_skips_the_mapper(mapper, hub_cache):+ # Leave remote mapper lookups to the loader.+ _cache_repo(hub_cache, "someone/custom-model")++ assert _exact_model_name_for_load(_config("someone/custom-model"), True) is None+ assert mapper.calls == []++[email protected]("overrides", [{"is_local": True}, {"is_lora": True}])+def test_local_and_adapter_loads_are_untouched(mapper, hub_cache, overrides):+ _cache_repo(hub_cache, UPSTREAM)++ assert _exact_model_name_for_load(_config(**overrides), True) is None+ assert mapper.calls == []+++def test_load_model_hands_the_verdict_to_both_loaders():+ """Dropping either keyword argument restores the download and leaves every other test+ here passing, so the wiring needs its own assertion. ``model_name`` matters as much as+ the flag: the verdict can name the swap target's cached spelling, and passing+ ``config.path`` there would load the wrong repo. A real ``load_model`` call cannot run+ here (no weights, no network, no unsloth), so read the call sites.+ """+ import ast++ source = (_BACKEND / "core/inference/inference.py").read_text(encoding = "utf-8")+ load_model = next(+ node+ for node in ast.walk(ast.parse(source))+ if isinstance(node, ast.FunctionDef) and node.name == "load_model"+ )++ def assigned_from(predicate):+ return {+ target.id+ for node in ast.walk(load_model)+ if isinstance(node, ast.Assign) and predicate(node.value)+ for target in node.targets+ if isinstance(target, ast.Name)+ }++ verdicts = assigned_from(+ lambda v: isinstance(v, ast.Call)+ and isinstance(v.func, ast.Name)+ and v.func.id == "_exact_model_name_for_load"+ )+ assert verdicts, "load_model never calls _exact_model_name_for_load"+ # Whatever the flag and the path are called, they have to be computed FROM the+ # verdict, so a rename stays green and a hardcoded True or config.path does not.+ derived = verdicts | assigned_from(+ lambda v: any(isinstance(n, ast.Name) and n.id in verdicts for n in ast.walk(v))+ )++ def receives(kwarg):+ return {+ node.func.value.id+ for node in ast.walk(load_model)+ if isinstance(node, ast.Call)+ and isinstance(node.func, ast.Attribute)+ and node.func.attr == "from_pretrained"+ and isinstance(node.func.value, ast.Name)+ and node.func.value.id in ("FastLanguageModel", "FastVisionModel")+ and any(+ kw.arg == kwarg and isinstance(kw.value, ast.Name) and kw.value.id in derived+ for kw in node.keywords+ )+ }++ both = {"FastLanguageModel", "FastVisionModel"}+ for kwarg in ("use_exact_model_name", "model_name"):+ assert receives(kwarg) == both, f"only {sorted(receives(kwarg))} receive {kwarg}"diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.pyindex 2f20a8758c4..a753f2a782e 100644--- a/studio/backend/utils/utils.py+++ b/studio/backend/utils/utils.py@@ -575,18 +575,18 @@ def snapshot_has_st_weights(model_name: str) -> bool:return cached_st_source(model_name) is not None-def _snapshot_in_root(cache_root: Path, repo_id: str) -> Optional[Path]:- """``repo_id``'s main-revision snapshot under exactly ``cache_root``, or None."""+def _repo_folder_name(repo_id: str) -> str:+ """The cache directory name huggingface_hub gives ``repo_id``."""try:from huggingface_hub.file_download import repo_folder_nameexcept Exception:- repo_folder_name = None+ return "models--" + repo_id.replace("/", "--")+ return repo_folder_name(repo_id = repo_id, repo_type = "model")+++def _snapshot_in_repo_dir(repo_dir: Path) -> Optional[Path]:+ """``repo_dir``'s main-revision snapshot, or None. ``repo_dir`` is a cache entry the caller already picked, so no repo id is re-derived from its name."""try:- if repo_folder_name is not None:- folder = repo_folder_name(repo_id = repo_id, repo_type = "model")- else:- folder = "models--" + repo_id.replace("/", "--")- repo_dir = cache_root / folderref = repo_dir / "refs" / "main"if not ref.is_file():return None@@ -600,6 +600,14 @@ def _snapshot_in_root(cache_root: Path, repo_id: str) -> Optional[Path]:return None+def _snapshot_in_root(cache_root: Path, repo_id: str) -> Optional[Path]:+ """``repo_id``'s main-revision snapshot under exactly ``cache_root``, or None."""+ try:+ return _snapshot_in_repo_dir(cache_root / _repo_folder_name(repo_id))+ except (OSError, UnicodeDecodeError):+ return None++def hf_cache_snapshot_dir_for_repo(repo_id: str) -> Optional[Path]:"""Snapshot dir for exactly ``repo_id``, with no alias expansion. ``hf_cache_snapshot_dir`` answers "is this model cached anywhere", trying the ST alias, so asking it about a literal slashless name can return the namespaced snapshot. A caller that has to report WHICH repo supplied the weights needs this one instead, or it pairs the alias's directory with the literal id and sends verification at a repo that does not exist."""for cache_root in _hf_cache_roots():@@ -671,6 +679,43 @@ def hf_cache_snapshot_is_loadable(model_name: str) -> bool:return snapshot_is_loadable(snapshot, model_name)+def active_hf_cache_loadable_snapshot(repo_id: str) -> Optional[Path]:+ """Return a loadable snapshot from the active cache only, or None."""+ try:+ from utils.hf_cache_settings import active_hf_hub_cache+ snapshot = _snapshot_in_root(_expand_path(active_hf_hub_cache()), repo_id)+ except Exception:+ return None+ return snapshot if snapshot is not None and snapshot_is_loadable(snapshot, repo_id) else None+++def active_hf_cache_repo_spelling(repo_id: str) -> Optional[str]:+ """The spelling ``repo_id`` is actually cached under in the active cache, or None.++ The Hub is case insensitive, the cache directories it creates are not, and huggingface_hub keys the directory on the id verbatim, so one checkpoint sits under whichever spelling first asked for it (huggingface/huggingface_hub#3838). Returning the spelling rather than a bool is what lets a caller LOAD the copy it found: asking for any other casing downloads it again.+ """+ if active_hf_cache_loadable_snapshot(repo_id) is not None:+ return repo_id+ try:+ from utils.hf_cache_settings import active_hf_hub_cache++ exact = _repo_folder_name(repo_id)+ wanted = exact.lower()+ prefix = len("models--")+ for repo_dir in _expand_path(active_hf_hub_cache()).iterdir():+ if repo_dir.name.lower() != wanted or repo_dir.name == exact:+ continue+ # Repo ids cannot contain '--' (huggingface_hub.utils.validate_repo_id), so+ # the directory name maps back to exactly one id.+ variant = repo_dir.name[prefix:].replace("--", "/")+ snapshot = _snapshot_in_repo_dir(repo_dir)+ if snapshot is not None and snapshot_is_loadable(snapshot, variant):+ return variant+ except Exception:+ return None+ return None++def snapshot_is_loadable(snapshot, model_name: str) -> bool:"""``hf_cache_snapshot_is_loadable`` for a snapshot the caller already has. A caller that picked a specific directory has to have THAT one judged: the lookup above expands the ST alias within each cache root while an exact per-repo lookup walks the roots for one id, so with several roots configured the two can land on different snapshots and the verdict would belong to a directory nobody is going to load."""try: