Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop suggesting bnb-4bit models MLX cannot load on Mac (#9722)
Nilayb37bddb43978
diff --git a/studio/backend/core/inference/defaults.py b/studio/backend/core/inference/defaults.pyindex a1d03c03e09..665895a75c4 100644--- a/studio/backend/core/inference/defaults.py+++ b/studio/backend/core/inference/defaults.py@@ -3,7 +3,10 @@"""Default model lists for inference, split by platform."""+from typing import Iterable+import utils.hardware.hardware as hw+from core.inference.model_ids import mlx_bnb_base_repoDEFAULT_MODELS_GGUF = ["unsloth/Qwen3.6-27B-MTP-GGUF",@@ -50,8 +53,22 @@ DEFAULT_MODELS_STANDARD = []+def suggestions_for_host(models: Iterable[str], device) -> list[str]:+ """Name MLX suggestions by their loaded repositories, preserving order."""+ if device != hw.DeviceType.MLX:+ return list(models)+ from core.inference.diffusion_families import detect_family++ def _named(model: str) -> str:+ if detect_family(model) is not None:+ return model+ return mlx_bnb_base_repo(model) or model++ return list(dict.fromkeys(_named(model) for model in models))++def get_default_models() -> list[str]:- hw.get_device() # ensures detect_hardware() has run+ device = hw.get_device() # ensures detect_hardware() has runif hw.CHAT_ONLY:return list(DEFAULT_MODELS_GGUF)- return list(DEFAULT_MODELS_STANDARD)+ return suggestions_for_host(DEFAULT_MODELS_STANDARD, device)diff --git a/studio/backend/core/inference/model_ids.py b/studio/backend/core/inference/model_ids.pyindex fb19b95bafe..0d86d16b18c 100644--- a/studio/backend/core/inference/model_ids.py+++ b/studio/backend/core/inference/model_ids.py@@ -14,7 +14,7 @@ and already-clean names untouched.from __future__ import annotationsimport os-from typing import Optional+from typing import Iterable, Optional_GGUF_SUFFIX = ".gguf"@@ -119,3 +119,40 @@ def model_id_matches(requested: Optional[str], internal: Optional[str]) -> bool:if requested == internal:return Truereturn public_model_id(internal) == requested+++# Mirror Zoo’s MLX repository substitution without importing the ML stack.+_BNB_SUFFIXES = ("-unsloth-bnb-4bit", "-bnb-4bit")+++def mlx_bnb_base_repo(model_name: Optional[str]) -> Optional[str]:+ """Return the replacement base repository, or None."""+ if not isinstance(model_name, str) or not model_name.startswith("unsloth/"):+ return None+ if os.path.exists(model_name):+ return None+ for suffix in _BNB_SUFFIXES:+ if model_name.endswith(suffix):+ return model_name[: -len(suffix)]+ return None+++def mlx_host_bnb_base_repo(model_name: Optional[str]) -> Optional[str]:+ """Return the MLX replacement, excluding diffusion models."""+ import utils.hardware.hardware as hw+ from core.inference.diffusion_families import detect_family++ if hw.get_device() != hw.DeviceType.MLX:+ return None+ if not isinstance(model_name, str) or detect_family(model_name) is not None:+ return None+ return mlx_bnb_base_repo(model_name)+++def mlx_bnb_substitutions(repos: Iterable[str]) -> list[tuple[str, str]]:+ swaps = []+ for repo in repos:+ base = mlx_bnb_base_repo(repo)+ if base:+ swaps.append((repo, base))+ return swapsdiff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 08725f71972..64ae394fe37 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -482,11 +482,19 @@ class InferenceOrchestrator:self._start_top_models_fetch()top_gguf = self._top_gguf_cache or []top_hub = self._top_hub_cache or []- # Never wait for the remote Hugging Face ranking during startup. Chat's first /api/models/list needs curated- # defaults immediately; the background fetch backfills extra choices on later calls.+ # Use detected hardware here: discovery runs on the event loop.+ from core.inference.defaults import suggestions_for_host+ import utils.hardware.hardware as _hw_mod++ # A chat-only Mac never reaches the MLX loader, so its ranking is left as fetched.+ device = None if _hw_mod.CHAT_ONLY else _hw_mod.DEVICE+ fetched = suggestions_for_host(top_gguf + top_hub, device)+ # Never wait for the remote Hugging Face ranking during startup. Chat's+ # first /api/models/list needs curated defaults immediately; the+ # background fetch backfills extra choices on later calls.result: list[str] = []seen: set[str] = set()- for m in self._static_models + top_gguf + top_hub:+ for m in self._static_models + fetched:if m not in seen:result.append(m)seen.add(m)diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex 644f892d0a5..3e8ccb0989e 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -451,6 +451,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:# loads; a no-progress Xet download is reported as a stall so the parent# can respawn over HTTP. Watch model + base repos (base is the LoRA# download bottleneck).+ from core.inference.model_ids import mlx_bnb_substitutionsfrom utils.hf_xet_fallback import start_watchdogwatch_repos = [mc.identifier]@@ -458,6 +459,23 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:if base and str(base) != mc.identifier:watch_repos.append(str(base))+ # Watch the repositories Zoo downloads after substitution.+ if getattr(backend, "device", None) == "mlx":+ substitutions = mlx_bnb_substitutions(watch_repos)+ replacements = dict(substitutions)+ watch_repos = list(dict.fromkeys(replacements.get(repo, repo) for repo in watch_repos))+ for requested, mlx_base in substitutions:+ _send_response(+ resp_queue,+ {+ "type": "status",+ "message": (+ f"MLX cannot read bitsandbytes 4-bit weights; "+ f"downloading {mlx_base} instead of {requested}"+ ),+ },+ )+heartbeat_stop = start_watchdog(repo_ids = watch_repos,on_stall = lambda msg: _send_response(resp_queue, {"type": "stall", "message": msg}),diff --git a/studio/backend/hub/services/models/cache_inventory.py b/studio/backend/hub/services/models/cache_inventory.pyindex 79d7e402711..e6c2022ebb9 100644--- a/studio/backend/hub/services/models/cache_inventory.py+++ b/studio/backend/hub/services/models/cache_inventory.py@@ -136,6 +136,72 @@ def get_repo_snapshot_metadata_cached(return total, blob_hashes+_mlx_plan_cache: OrderedDict = OrderedDict()+++def _cached_mlx_siblings(repo_id):+ from huggingface_hub.hf_api import RepoSibling++ # The raw tree retains LFS SHA256s, not just Git pointer IDs.+ from huggingface_hub._tree_cache import read_tree_cache+ from hub.utils.hf_cache_state import preferred_repo_cache_dirs++ for entry in preferred_repo_cache_dirs("model", repo_id):+ snapshot = hf_cache_scan.default_ref_snapshot(entry)+ tree = read_tree_cache(str(entry), snapshot.name) if snapshot is not None else None+ if tree:+ return [+ RepoSibling(rfilename = path, size = item.size, blob_id = item.lfs_sha256 or item.blob_id)+ for path, item in tree.items()+ ]+ return []+++def get_mlx_load_plan_cached(repo_id: str, hf_token: Optional[str] = None):+ from hub.utils.snapshot_filters import blob_hashes_for_siblings, mlx_load_siblings+ from huggingface_hub import HfApi++ key = (repo_id, hf_cache_scan.token_fingerprint(hf_token))+ with _repo_size_cache_lock:+ cached = _mlx_plan_cache.get(key)+ if cached is not None and time.monotonic() - cached[1] < _REPO_SIZE_POS_TTL:+ _mlx_plan_cache.move_to_end(key)+ return cached[0]+ try:+ siblings = (+ HfApi(token = hf_token)+ .model_info(+ repo_id,+ files_metadata = True,+ timeout = _MODEL_METADATA_TIMEOUT_SECONDS,+ )+ .siblings+ )+ except Exception:+ try:+ siblings = _cached_mlx_siblings(repo_id)+ except Exception:+ siblings = []+ siblings = mlx_load_siblings(siblings)+ files = tuple(+ download_manifest.ExpectedFile(+ path = item.rfilename,+ size = int(item.size or 0),+ sha256 = getattr(getattr(item, "lfs", None), "sha256", None),+ )+ for item in siblings+ )+ plan = (sum(file.size for file in files), blob_hashes_for_siblings(siblings), files)+ if not siblings and cached is not None:+ plan = cached[0]+ with _repo_size_cache_lock:+ _mlx_plan_cache[key] = (plan, time.monotonic())+ _mlx_plan_cache.move_to_end(key)+ while len(_mlx_plan_cache) > _REPO_SIZE_CACHE_MAX:+ _mlx_plan_cache.popitem(last = False)+ return plan++def all_hf_cache_scans():return hf_cache_scan.all_hf_cache_scans()diff --git a/studio/backend/hub/services/models/downloads.py b/studio/backend/hub/services/models/downloads.pyindex 4bc4d98978f..997e0a113c6 100644--- a/studio/backend/hub/services/models/downloads.py+++ b/studio/backend/hub/services/models/downloads.py@@ -714,6 +714,7 @@ async def get_download_progress_response(repo_id: str,expected_bytes: int = 0,hf_token: Optional[str] = None,+ mlx_load: bool = False,) -> dict:"""Return download progress for any HuggingFace model repo.@@ -727,6 +728,26 @@ async def get_download_progress_response(if account_access.managed_account():await asyncio.to_thread(account_access.require_download_progress_access, _registry, repo_id)hf_token = account_access.account_hf_token(hf_token)+ if mlx_load:++ def _metadata(repo, token):+ total, hashes, _files = cache_inventory.get_mlx_load_plan_cached(repo, token)+ return total, hashes++ def _files(repo, token):+ return cache_inventory.get_mlx_load_plan_cached(repo, token)[2]++ return await snapshot_progress.snapshot_progress_response(+ repo_type = "model",+ repo_id = repo_id,+ job_key = _download_job_key(repo_id, "@mlx"),+ expected_bytes = 0,+ hf_token = hf_token,+ registry = _registry,+ metadata_resolver = _metadata,+ expected_files_resolver = _files,+ variant = "@mlx",+ )return await snapshot_progress.snapshot_progress_response(repo_type = "model",repo_id = repo_id,diff --git a/studio/backend/hub/utils/snapshot_filters.py b/studio/backend/hub/utils/snapshot_filters.pyindex 39cedef6bcd..a0ad4952eb4 100644--- a/studio/backend/hub/utils/snapshot_filters.py+++ b/studio/backend/hub/utils/snapshot_filters.py@@ -197,3 +197,28 @@ def blob_hashes_for_siblings(siblings: Iterable) -> frozenset[str]:def snapshot_download_blob_hashes(siblings: Iterable) -> frozenset[str]:return blob_hashes_for_siblings(snapshot_download_siblings(siblings))+++# mlx-lm's download filter, plus the weights of an exported adapter.+_MLX_LOAD_PATTERNS = (+ "*.json",+ "model*.safetensors",+ "*.py",+ "tokenizer.model",+ "*.tiktoken",+ "tiktoken.model",+ "*.txt",+ "*.jsonl",+ "*.jinja",+ "adapters.safetensors",+ "adapter_model.safetensors",+ "adapter_model.bin",+)+++def mlx_load_siblings(siblings):+ return [+ sibling+ for sibling in siblings+ if any(fnmatchcase(sibling.rfilename, pattern) for pattern in _MLX_LOAD_PATTERNS)+ ]diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 599f1442179..9d3ab93ffc9 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -820,6 +820,12 @@ class ValidateModelResponse(BaseModel):description = "Details for the transformers-upgrade dialog; set only when ""requires_transformers_upgrade is true.",)+ mlx_loads_base_model: Optional[str] = Field(+ None,+ description = "On an MLX host, the full-precision repo that will be downloaded and "+ "loaded in place of the requested unsloth bnb-4bit repo (or of a LoRA's bnb base), "+ "because MLX cannot read bitsandbytes weights. None when the pick loads as asked.",+ )class EstimateMemoryRequest(BaseModel):diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 1af41809bf4..9987d58778b 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -16988,6 +16988,15 @@ def _requires_security_review_for_model(return False+def _mlx_base_for_config(config) -> Optional[str]:+ from core.inference.model_ids import mlx_host_bnb_base_repo+ for candidate in (getattr(config, "identifier", None), getattr(config, "base_model", None)):+ base = mlx_host_bnb_base_repo(candidate)+ if base:+ return base+ return None++@router.post("/validate", response_model = ValidateModelResponse)async def validate_model(request: ValidateModelRequest,@@ -17368,6 +17377,7 @@ async def validate_model(chat_template = chat_template,requires_transformers_upgrade = transformers_upgrade is not None,transformers_upgrade = transformers_upgrade,+ mlx_loads_base_model = await asyncio.to_thread(_mlx_base_for_config, config),))diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.pyindex 5e9f6a090c2..c4ffda8abe1 100644--- a/studio/backend/routes/models.py+++ b/studio/backend/routes/models.py@@ -4591,6 +4591,7 @@ def _resolve_hf_cache_realpath(repo_dir: Path) -> Optional[str]:@router.get("/download-progress")async def get_download_progress(repo_id: str = Query(..., description = "HuggingFace repo ID"),+ mlx_load: bool = Query(False),hf_token: HfTokenArg = Depends(get_request_hf_token),current_subject: str = Depends(get_current_subject),via_api_key: bool = Depends(authenticated_via_api_key),@@ -4599,7 +4600,9 @@ async def get_download_progress(the cache directory it measured, so it takes the caller class like its ``/api/hub`` twin."""from hub.services.models import downloadsreturn redact_host_paths(- await downloads.get_download_progress_response(repo_id, hf_token = hf_token),+ await downloads.get_download_progress_response(+ repo_id, hf_token = hf_token, mlx_load = mlx_load+ ),via_api_key = via_api_key,)diff --git a/studio/backend/tests/test_cached_gguf_routes.py b/studio/backend/tests/test_cached_gguf_routes.pyindex db20fec6362..f4b9c211f0c 100644--- a/studio/backend/tests/test_cached_gguf_routes.py+++ b/studio/backend/tests/test_cached_gguf_routes.py@@ -2602,11 +2602,12 @@ def test_legacy_gguf_progress_delegates_to_shared_service(monkeypatch):assert calls == [("org/repo", "Q4_K_M", 20, "token")]-def test_legacy_model_progress_delegates_to_shared_service(monkeypatch):[email protected]("mlx_load", [False, True])+def test_legacy_model_progress_delegates_to_shared_service(monkeypatch, mlx_load):calls = []- async def shared(repo_id, *, hf_token):- calls.append((repo_id, hf_token))+ async def shared(repo_id, *, hf_token, mlx_load):+ calls.append((repo_id, hf_token, mlx_load))return {"downloaded_bytes": 10, "expected_bytes": 20, "progress": 0.5}monkeypatch.setattr(@@ -2619,11 +2620,12 @@ def test_legacy_model_progress_delegates_to_shared_service(monkeypatch):repo_id = "org/repo",hf_token = "token",current_subject = "test-user",+ mlx_load = mlx_load,))assert result["progress"] == 0.5- assert calls == [("org/repo", "token")]+ assert calls == [("org/repo", "token", mlx_load)]def test_legacy_delete_delegates_to_shared_service(monkeypatch):diff --git a/studio/backend/tests/test_mlx_bnb_substitution.py b/studio/backend/tests/test_mlx_bnb_substitution.pynew file mode 100644index 00000000000..3aa76df19a3--- /dev/null+++ b/studio/backend/tests/test_mlx_bnb_substitution.py@@ -0,0 +1,370 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+++from __future__ import annotations++import sys+from pathlib import Path++import pytest++_BACKEND = Path(__file__).resolve().parent.parent+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++import core.inference.defaults as defaults_mod # noqa: E402+import utils.hardware.hardware as hw # noqa: E402+from core.inference.model_ids import mlx_bnb_base_repo, mlx_bnb_substitutions # noqa: E402+++def test_unsloth_bnb_repos_resolve_to_their_base():+ assert (+ mlx_bnb_base_repo("unsloth/Qwen2-VL-2B-Instruct-bnb-4bit") == "unsloth/Qwen2-VL-2B-Instruct"+ )+ assert mlx_bnb_base_repo("unsloth/gemma-3-4b-it-unsloth-bnb-4bit") == "unsloth/gemma-3-4b-it"+++def test_repos_mlx_loads_as_given_have_no_base():+ assert mlx_bnb_base_repo("unsloth/Qwen3-4B-Instruct-2507") is None+ assert mlx_bnb_base_repo("unsloth/Llama-3.2-1B-Instruct-GGUF") is None+ assert mlx_bnb_base_repo("someone-else/model-bnb-4bit") is None+ assert mlx_bnb_base_repo(None) is None+++def test_a_local_directory_is_never_remapped(monkeypatch, tmp_path):+ (tmp_path / "unsloth" / "model-bnb-4bit").mkdir(parents = True)+ monkeypatch.chdir(tmp_path)++ assert mlx_bnb_base_repo("unsloth/model-bnb-4bit") is None+++def test_substitutions_cover_a_loras_base():+ swaps = mlx_bnb_substitutions(["me/my-lora", "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit"])+ assert swaps == [+ ("unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit", "unsloth/Meta-Llama-3.1-8B-Instruct")+ ]++ already_watched = ["unsloth/gemma-3-4b-it-bnb-4bit", "unsloth/gemma-3-4b-it"]+ assert mlx_bnb_substitutions(already_watched) == [+ ("unsloth/gemma-3-4b-it-bnb-4bit", "unsloth/gemma-3-4b-it")+ ]+++def _standard_host(monkeypatch, device):+ monkeypatch.setattr(hw, "CHAT_ONLY", False)+ monkeypatch.setattr(hw, "get_device", lambda: device)+++def test_mlx_defaults_recommend_the_repos_mlx_actually_loads(monkeypatch):+ _standard_host(monkeypatch, hw.DeviceType.MLX)++ models = defaults_mod.get_default_models()++ assert [model for model in models if model.endswith("bnb-4bit")] == []+ assert "unsloth/Qwen2-VL-2B-Instruct" in models+ assert len(models) == len(set(models))+++def test_cuda_defaults_keep_the_bnb_repos(monkeypatch):+ _standard_host(monkeypatch, hw.DeviceType.CUDA)++ assert defaults_mod.get_default_models() == defaults_mod.DEFAULT_MODELS_STANDARD+++def test_chat_only_hosts_are_untouched(monkeypatch):+ monkeypatch.setattr(hw, "CHAT_ONLY", True)+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.MLX)++ assert defaults_mod.get_default_models() == defaults_mod.DEFAULT_MODELS_GGUF+++ "identifier,base_model,is_lora,expected",+ [+ (+ "unsloth/Qwen2-VL-2B-Instruct-bnb-4bit",+ None,+ False,+ ["unsloth/Qwen2-VL-2B-Instruct"],+ ),+ (+ "me/my-lora",+ "unsloth/Qwen2-VL-2B-Instruct-bnb-4bit",+ True,+ ["me/my-lora", "unsloth/Qwen2-VL-2B-Instruct"],+ ),+ (+ "unsloth/Qwen2-VL-2B-Instruct-bnb-4bit",+ "unsloth/Qwen2-VL-2B-Instruct",+ True,+ ["unsloth/Qwen2-VL-2B-Instruct"],+ ),+ ],+)+def test_worker_only_watches_the_repositories_mlx_downloads(+ monkeypatch, identifier, base_model, is_lora, expected+):+ from types import SimpleNamespace++ import core.inference.worker as worker_mod+ import utils.hf_xet_fallback as fallback_mod++ model_config = SimpleNamespace(+ identifier = identifier,+ base_model = base_model,+ is_lora = is_lora,+ )+ captured = {}++ class StopEvent:+ def set(self):+ pass++ class ResponseQueue:+ def put(self, response):+ pass++ class Backend:+ device = "mlx"++ def load_model(self, **kwargs):+ return False++ def start_watchdog(**kwargs):+ captured.update(kwargs)+ return StopEvent()++ monkeypatch.setattr(worker_mod, "_build_model_config", lambda config: model_config)+ monkeypatch.setattr(worker_mod, "_resolve_lora_4bit", lambda model, requested: False)+ monkeypatch.setattr(worker_mod, "_run_security_gates", lambda *args, **kwargs: True)+ monkeypatch.setattr(fallback_mod, "start_watchdog", start_watchdog)++ worker_mod._handle_load(Backend(), {"model_name": identifier}, ResponseQueue())++ assert captured["repo_ids"] == expected+++def test_the_host_rule_only_fires_on_mlx(monkeypatch):+ from core.inference.model_ids import mlx_host_bnb_base_repo++ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)+ assert mlx_host_bnb_base_repo("unsloth/Qwen2-VL-2B-Instruct-bnb-4bit") is None++ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.MLX)+ assert (+ mlx_host_bnb_base_repo("unsloth/Qwen2-VL-2B-Instruct-bnb-4bit")+ == "unsloth/Qwen2-VL-2B-Instruct"+ )+++def test_diffusion_bnb_repos_are_loaded_as_named(monkeypatch):+ from core.inference.model_ids import mlx_host_bnb_base_repo++ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.MLX)+ assert mlx_host_bnb_base_repo("unsloth/Qwen-Image-2512-unsloth-bnb-4bit") is None+ assert mlx_host_bnb_base_repo("unsloth/Z-Image-Turbo-unsloth-bnb-4bit") is None+++def test_validate_reports_the_repo_mlx_will_load(monkeypatch):+ from types import SimpleNamespace++ from routes.inference import _mlx_base_for_config++ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.MLX)+ pick = SimpleNamespace(identifier = "unsloth/Qwen2-VL-2B-Instruct-bnb-4bit", base_model = None)+ assert _mlx_base_for_config(pick) == "unsloth/Qwen2-VL-2B-Instruct"++ adapter = SimpleNamespace(+ identifier = "me/my-lora",+ base_model = "unsloth/gemma-3-4b-it-bnb-4bit",+ )+ assert _mlx_base_for_config(adapter) == "unsloth/gemma-3-4b-it"++ plain = SimpleNamespace(identifier = "unsloth/Qwen3-4B-Instruct-2507", base_model = None)+ assert _mlx_base_for_config(plain) is None+++def test_the_fetched_ranking_is_mapped_on_a_mac_too(monkeypatch):+ import core.inference.orchestrator as orch_mod++ monkeypatch.setattr(orch_mod.InferenceOrchestrator, "_fetch_top_models", lambda self: None)+ monkeypatch.setattr(defaults_mod, "get_default_models", lambda: ["unsloth/curated"])+ monkeypatch.setattr(hw, "DETECTION_GENERATION", 1)+ monkeypatch.setattr(hw, "CHAT_ONLY", False)+ monkeypatch.setattr(hw, "DEVICE", hw.DeviceType.MLX)++ orch = orch_mod.InferenceOrchestrator()+ orch._top_gguf_cache = ["unsloth/Qwen3-4B-GGUF"]+ orch._top_hub_cache = ["unsloth/Qwen3-8B-unsloth-bnb-4bit", "unsloth/Qwen3-4B-Instruct-2507"]++ assert orch.default_models == [+ "unsloth/curated",+ "unsloth/Qwen3-4B-GGUF",+ "unsloth/Qwen3-8B",+ "unsloth/Qwen3-4B-Instruct-2507",+ ]+++def test_a_diffusion_bnb_repo_in_the_ranking_keeps_its_name(monkeypatch):+ import core.inference.orchestrator as orch_mod++ monkeypatch.setattr(orch_mod.InferenceOrchestrator, "_fetch_top_models", lambda self: None)+ monkeypatch.setattr(defaults_mod, "get_default_models", lambda: ["unsloth/curated"])+ monkeypatch.setattr(hw, "DETECTION_GENERATION", 1)+ monkeypatch.setattr(hw, "CHAT_ONLY", False)+ monkeypatch.setattr(hw, "DEVICE", hw.DeviceType.MLX)++ orch = orch_mod.InferenceOrchestrator()+ orch._top_hub_cache = [+ "unsloth/Qwen-Image-2512-unsloth-bnb-4bit",+ "unsloth/Z-Image-Turbo-unsloth-bnb-4bit",+ "unsloth/Qwen3-8B-unsloth-bnb-4bit",+ ]++ assert orch.default_models == [+ "unsloth/curated",+ # diffusion: read as named by diffusers/MPS+ "unsloth/Qwen-Image-2512-unsloth-bnb-4bit",+ "unsloth/Z-Image-Turbo-unsloth-bnb-4bit",+ # text: mlx-lm cannot read it, so name the repo it really loads+ "unsloth/Qwen3-8B",+ ]+++def test_the_fetched_ranking_is_untouched_off_mlx(monkeypatch):+ import core.inference.orchestrator as orch_mod++ monkeypatch.setattr(orch_mod.InferenceOrchestrator, "_fetch_top_models", lambda self: None)+ monkeypatch.setattr(defaults_mod, "get_default_models", lambda: ["unsloth/curated"])+ monkeypatch.setattr(hw, "DETECTION_GENERATION", 1)+ monkeypatch.setattr(hw, "CHAT_ONLY", False)+ monkeypatch.setattr(hw, "DEVICE", hw.DeviceType.CUDA)++ orch = orch_mod.InferenceOrchestrator()+ orch._top_hub_cache = ["unsloth/Qwen3-8B-unsloth-bnb-4bit"]++ assert orch.default_models == ["unsloth/curated", "unsloth/Qwen3-8B-unsloth-bnb-4bit"]+++def test_the_mirror_still_matches_the_loader_it_mirrors():+ loader = pytest.importorskip("unsloth_zoo.mlx.loader")++ for name in (+ "unsloth/Qwen2-VL-2B-Instruct-bnb-4bit",+ "unsloth/gemma-3-4b-it-unsloth-bnb-4bit",+ "unsloth/Qwen3-4B-Instruct-2507",+ "unsloth/Llama-3.2-1B-Instruct-GGUF",+ "someone-else/model-bnb-4bit",+ ):+ remapped, _revision, swapped_from = loader._remap_unsloth_bnb_hub_id_for_mlx(+ name, "some-revision"+ )+ expected = remapped if swapped_from is not None else None+ assert mlx_bnb_base_repo(name) == expected, name+++ "weight_names",+ [+ ("model-00001-of-00002.safetensors", "model-00002-of-00002.safetensors"),+ ("adapter_model.safetensors",),+ ("adapters.safetensors",),+ ],+)[email protected]("offline", [False, True])+def test_mlx_progress_verifies_loader_files_without_a_manifest(+ monkeypatch, tmp_path, weight_names, offline+):+ import asyncio+ from collections import OrderedDict+ from types import SimpleNamespace+ from huggingface_hub import HfApi+ from hub.services import snapshot_progress+ from hub.services.models import cache_inventory, downloads++ repo = "unsloth/progress-model"+ entry = tmp_path / "models--unsloth--progress-model"+ snap = entry / "snapshots" / ("a" * 40)+ snap.mkdir(parents = True)+ blobs = entry / "blobs"+ blobs.mkdir()+ names = ("config.json", *weight_names, "README.md")+ siblings = [+ SimpleNamespace(rfilename = name, size = 4, blob_id = f"blob{i}", lfs = None)+ for i, name in enumerate(names)+ ]+ monkeypatch.setattr(HfApi, "model_info", lambda *_a, **_k: SimpleNamespace(siblings = siblings))+ monkeypatch.setattr(cache_inventory, "_mlx_plan_cache", OrderedDict())+ if offline:+ from huggingface_hub._tree_cache import TreeCacheEntry, write_tree_cache+ import hub.utils.hf_cache_state as cache_state++ (entry / "refs").mkdir()+ (entry / "refs" / "main").write_text(snap.name)+ write_tree_cache(+ str(entry),+ snap.name,+ {+ item.rfilename: TreeCacheEntry(+ size = item.size, blob_id = "pointer", lfs_sha256 = item.blob_id+ )+ for item in siblings+ },+ )++ def offline_info(*_a, **_k):+ raise OSError("offline")++ monkeypatch.setattr(HfApi, "model_info", offline_info)+ monkeypatch.setattr(cache_state, "preferred_repo_cache_dirs", lambda *_a, **_k: [entry])+ monkeypatch.setattr(snapshot_progress, "preferred_repo_cache_dirs", lambda *_a, **_k: [entry])+ monkeypatch.setattr(+ downloads, "_registry", SimpleNamespace(get_job = lambda _key: SimpleNamespace(state = "idle"))+ )++ def progress():+ return asyncio.run(downloads.get_download_progress_response(repo, mlx_load = True))++ for i, name in enumerate(names[:-1]):+ (blobs / f"blob{i}").write_bytes(b"data")+ (snap / name).symlink_to(blobs / f"blob{i}")+ reading = progress()+ assert reading["expected_bytes"] == 4 * (len(names) - 1)+ assert reading["complete_on_disk"] is (i == len(names) - 2)+ assert reading["progress"] == 1+ generic = asyncio.run(downloads.get_download_progress_response(repo))+ assert generic["complete_on_disk"] is False+ assert generic["progress"] < 1+ for name in weight_names:+ (snap / name).unlink()+ assert progress()["complete_on_disk"] is False+ i = names.index(name)+ (snap / name).symlink_to(blobs / f"blob{i}")+ assert progress()["progress"] == 1++[email protected]("warm", [False, True])+def test_mlx_metadata_outage_reuses_plan_and_bounds_retries(monkeypatch, warm):+ from collections import OrderedDict+ from huggingface_hub import HfApi+ from hub.services.models import cache_inventory++ key = ("unsloth/cached-base", cache_inventory.hf_cache_scan.token_fingerprint(None))+ plan = (4, frozenset({"weight"}), ())+ expired = cache_inventory.time.monotonic() - cache_inventory._REPO_SIZE_POS_TTL - 1+ monkeypatch.setattr(+ cache_inventory, "_mlx_plan_cache", OrderedDict({key: (plan, expired)} if warm else {})+ )+ monkeypatch.setattr(cache_inventory, "_cached_mlx_siblings", lambda _repo: [])+ calls = []++ def unavailable(*_a, **_k):+ calls.append(True)+ raise OSError("unreachable")++ monkeypatch.setattr(HfApi, "model_info", unavailable)+ expected = plan if warm else (0, frozenset(), ())+ for _ in range(2):+ assert cache_inventory.get_mlx_load_plan_cached(key[0]) == expected+ assert len(calls) == 1diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.tsindex 581c9f847a2..4ef2e9c4f88 100644--- a/studio/frontend/src/features/chat/api/chat-api.ts+++ b/studio/frontend/src/features/chat/api/chat-api.ts@@ -538,8 +538,10 @@ export interface DownloadProgressResponse {export async function getDownloadProgress(repoId: string,hfToken?: string | null,+ mlxLoad = false,): Promise<DownloadProgressResponse> {const params = new URLSearchParams({ repo_id: repoId });+ if (mlxLoad) params.set("mlx_load", "true");const response = await authFetch(`/api/models/download-progress?${params}`, {headers: hubTokenHeader(hfToken),});diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.tsindex c462da12b46..bbb399b8fab 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts@@ -1186,7 +1186,7 @@ export function useChatModelRuntime() {nativePathToken != null ||model?.isGguf === true);const loraIsAdapter = lora?.exportType === "lora";- const isLora =+ let isLora =explicitIsLora ?? model?.isLora ?? loraIsAdapter ?? false;const displayName = model?.name || lora?.name || modelId;const toastDisplayName = shortModelLabel(displayName);@@ -1208,7 +1208,7 @@ export function useChatModelRuntime() {previousModel?.isLora ?? (previousLora?.exportType === "lora");const isLocal = isLocalModelPath(modelId);const isCachedLora = isLora && isLocal;- const loadingDescription = [+ let loadingDescription = [currentCheckpoint ? "Switching models." : null,extraLoadingDescription ?? null,isDownloaded ? "Loading cached model into memory." : null,@@ -1239,6 +1239,9 @@ export function useChatModelRuntime() {loadAbortRef.current = abortCtrl;let hfToken = useChatRuntimeStore.getState().hfToken || null;const postLoadRefresh = { needed: false };+ let progressModelIds = [modelId];+ let mlxLoadProgress = false;+ let downloadComplete = isDownloaded || isCachedLora;let cpuFallbackReason: CpuFallbackReason | null = null;let mmprojFallbackReason: MmprojFallbackReason | null = null;try {@@ -1600,6 +1603,32 @@ export function useChatModelRuntime() {}: {}),});+ isLora = validation.is_lora ?? isLora;+ if (validation.mlx_loads_base_model) {+ mlxLoadProgress = true;+ const mlxBaseDescription = isLora+ ? `Loading the adapter with ${validation.mlx_loads_base_model} in place of its bitsandbytes base, downloading it first if needed.`+ : `Loading ${validation.mlx_loads_base_model} instead, downloading it first if needed.`;+ progressModelIds = isLora && !isLocal+ ? [modelId, validation.mlx_loads_base_model]+ : [validation.mlx_loads_base_model];+ downloadComplete = false;+ loadingDescription = [+ currentCheckpoint ? "Switching models." : null,+ extraLoadingDescription ?? null,+ mlxBaseDescription,+ ]+ .filter(Boolean)+ .join(" ");+ setLoadProgress({+ percent: 0,+ label: "Preparing download",+ phase: "downloading",+ });+ toast.info("MLX cannot use 4-bit bitsandbytes weights", {+ description: mlxBaseDescription,+ });+ }// Upgrade consent runs before the security dialogs; Accept installs and the load continues.if (validation.requires_transformers_upgrade) {const upgraded = await confirmTransformersUpgradeIfNeeded({@@ -2316,7 +2345,7 @@ export function useChatModelRuntime() {}}- const isCachedLoad = isDownloaded || isCachedLora;+ const isCachedLoad = downloadComplete;const toastTitle = isCachedLoad ? "Starting model…" : "Downloading model…";const modelLoadToastOptions = (description: ReturnType<typeof renderLoadDescription>) => ({description,@@ -2398,8 +2427,6 @@ export function useChatModelRuntime() {: `${base} • ${rateStr}`;}- let downloadComplete = isDownloaded || isCachedLora;-// A load that believes the weights are cached can still turn into a download (#9094): the// backend re-fetches a blob it judged unsafe to resume. MOVEMENT is the only proof accepted,// since bytes below the expected total is the ordinary state of a partial revision.@@ -2422,16 +2449,49 @@ export function useChatModelRuntime() {return;}try {- const prog =+ const progressModelIdsAtRequest = [...progressModelIds];+ const progressResponses =ggufVariant && expectedBytes > 0- ? await getGgufDownloadProgress(- modelId,- ggufVariant,- expectedBytes,- hfToken,- )- : await getDownloadProgress(modelId, hfToken);+ ? [+ await getGgufDownloadProgress(+ modelId,+ ggufVariant,+ expectedBytes,+ hfToken,+ ),+ ]+ : await Promise.all(+ progressModelIdsAtRequest.map((progressModelId) =>+ getDownloadProgress(progressModelId, hfToken, mlxLoadProgress),+ ),+ );if (!loadingModelRef.current) return;+ if (+ progressModelIdsAtRequest.length !== progressModelIds.length ||+ progressModelIdsAtRequest.some(+ (progressModelId, index) =>+ progressModelId !== progressModelIds[index],+ )+ ) {+ return;+ }+ const allDownloadsComplete = progressResponses.every(+ ({ progress }) => progress >= 1,+ );+ const firstProgress = progressResponses[0];+ if (!firstProgress) return;+ const prog =+ progressResponses.find(+ ({ progress }) => progress > 0 && progress < 1,+ ) ??+ progressResponses.find(+ ({ downloaded_bytes, expected_bytes, progress }) =>+ downloaded_bytes > 0 &&+ expected_bytes === 0 &&+ progress === 0,+ ) ??+ progressResponses.find(({ progress }) => progress < 1) ??+ firstProgress;if (prog.progress > 0 && prog.progress < 1) {hasShownProgress = true;@@ -2499,7 +2559,13 @@ export function useChatModelRuntime() {),});}- } else if (prog.progress >= 1 && hasShownProgress) {+ } else if (+ allDownloadsComplete &&+ (hasShownProgress ||+ progressModelIds.some(+ (progressModelId) => progressModelId !== modelId,+ ))+ ) {downloadComplete = true;if (loadToastDismissedRef.current) {setLoadProgress({@@ -2513,9 +2579,11 @@ export function useChatModelRuntime() {...modelLoadToastOptions(renderLoadDescription("Starting model…",- "Download complete. Loading the model into memory.",- 100,- "Download complete",+ hasShownProgress+ ? "Download complete. Loading the model into memory."+ : loadingDescription,+ hasShownProgress ? 100 : null,+ hasShownProgress ? "Download complete" : null,),),});diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex 61ad5e3ac85..25b26c98186 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -155,6 +155,8 @@ export interface ValidateModelResponse {/** Architecture only shipped by a newer transformers; UI pauses on the upgrade dialog. */requires_transformers_upgrade?: boolean;transformers_upgrade?: TransformersUpgradeInfo | null;+ /** Replacement repository for an MLX BNB model or adapter base. */+ mlx_loads_base_model?: string | null;}export interface GgufVariantDetail {diff --git a/studio/frontend/tests/chat-cache-miss-download-label.test.ts b/studio/frontend/tests/chat-cache-miss-download-label.test.tsindex fb55f0de274..3657f702514 100644--- a/studio/frontend/tests/chat-cache-miss-download-label.test.ts+++ b/studio/frontend/tests/chat-cache-miss-download-label.test.ts@@ -119,7 +119,7 @@ test("an unknown total still updates the toast the user is looking at", () => {const hook = readText("../src/features/chat/hooks/use-chat-model-runtime.ts");const start = hook.indexOf("prog.expected_bytes === 0 &&");assert.ok(start > 0, "the unknown-total branch moved");- const end = hook.indexOf("} else if (prog.progress >= 1 && hasShownProgress)", start);+ const end = hook.indexOf("allDownloadsComplete &&", start);assert.ok(end > start, "the branch after the unknown-total one moved");const branch = hook.slice(start, end);diff --git a/studio/frontend/tests/mlx-bnb-load-progress.test.ts b/studio/frontend/tests/mlx-bnb-load-progress.test.tsnew file mode 100644index 00000000000..76bc1fed8b2--- /dev/null+++ b/studio/frontend/tests/mlx-bnb-load-progress.test.ts@@ -0,0 +1,87 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import test from "node:test";++const SOURCE = readFileSync(+ new URL(+ "../src/features/chat/hooks/use-chat-model-runtime.ts",+ import.meta.url,+ ),+ "utf8",+);++const VALIDATION = SOURCE.slice(+ SOURCE.indexOf("const validation = await validateModel"),+ SOURCE.indexOf("// Upgrade consent runs before the security dialogs"),+);+const DOWNLOAD_POLL = SOURCE.slice(+ SOURCE.indexOf("const pollDownload = async () =>"),+ SOURCE.indexOf("const pollLoad = async () =>"),+);++test("MLX base substitution tracks every downloaded repository", () => {+ assert.match(+ VALIDATION,+ /progressModelIds = isLora/,+ );+ assert.match(+ VALIDATION,+ /\[modelId, validation\.mlx_loads_base_model\]/,+ );+ assert.match(VALIDATION, /\[validation\.mlx_loads_base_model\]/);+ assert.match(VALIDATION, /downloadComplete = false;/);+ assert.match(VALIDATION, /mlxLoadProgress = true;/);+ assert.match(+ DOWNLOAD_POLL,+ /progressModelIdsAtRequest\.map\(\(progressModelId\) =>/,+ );+ assert.match(DOWNLOAD_POLL, /getDownloadProgress\(progressModelId, hfToken, mlxLoadProgress\)/);+});++test("validated LoRA status controls download tracking", () => {+ assert.match(SOURCE, /let isLora =/);+ assert.match(VALIDATION, /isLora = validation\.is_lora \?\? isLora;/);+ assert.ok(+ VALIDATION.indexOf("isLora = validation.is_lora ?? isLora;") <+ VALIDATION.indexOf("if (validation.mlx_loads_base_model)"),+ );+});++test("local LoRA tracks only the substituted base download", () => {+ assert.match(VALIDATION, /progressModelIds = isLora && !isLocal/);+});++test("a pre-substitution progress response cannot complete the base download", () => {+ assert.match(+ DOWNLOAD_POLL,+ /const progressModelIdsAtRequest = \[\.\.\.progressModelIds\];/,+ );+ assert.match(+ DOWNLOAD_POLL,+ /progressModelIdsAtRequest\.length !== progressModelIds\.length/,+ );+ assert.match(+ DOWNLOAD_POLL,+ /progressModelIdsAtRequest\.some\(/,+ );+});++test("all substituted downloads must finish before model startup", () => {+ assert.match(+ DOWNLOAD_POLL,+ /const allDownloadsComplete = progressResponses\.every\(/,+ );+ assert.match(DOWNLOAD_POLL, /allDownloadsComplete &&/);+ assert.match(+ DOWNLOAD_POLL,+ /progressModelIds\.some\(/,+ );+});++test("LoRA substitution says the adapter still loads", () => {+ assert.match(VALIDATION, /Loading the adapter with/);+ assert.match(VALIDATION, /in place of its bitsandbytes base/);+});