Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop suggesting bnb-4bit models MLX cannot load on Mac (#9722)

Nilayb37bddb43978
diff --git a/studio/backend/core/inference/defaults.py b/studio/backend/core/inference/defaults.py
index a1d03c03e09..665895a75c4 100644
--- a/studio/backend/core/inference/defaults.py
+++ b/studio/backend/core/inference/defaults.py
@@ -3,7 +3,10 @@
"""Default model lists for inference, split by platform."""
+from typing import Iterable
+
import utils.hardware.hardware as hw
+from core.inference.model_ids import mlx_bnb_base_repo
DEFAULT_MODELS_GGUF = [
"unsloth/Qwen3.6-27B-MTP-GGUF",
@@ -50,8 +53,22 @@ DEFAULT_MODELS_STANDARD = [
]
+def suggestions_for_host(models: Iterable[str], device) -> list[str]:
+ """Name MLX suggestions by their loaded repositories, preserving order."""
+ if device != hw.DeviceType.MLX:
+ return list(models)
+ from core.inference.diffusion_families import detect_family
+
+ def _named(model: str) -> str:
+ if detect_family(model) is not None:
+ return model
+ return mlx_bnb_base_repo(model) or model
+
+ return list(dict.fromkeys(_named(model) for model in models))
+
+
def get_default_models() -> list[str]:
- hw.get_device() # ensures detect_hardware() has run
+ device = hw.get_device() # ensures detect_hardware() has run
if hw.CHAT_ONLY:
return list(DEFAULT_MODELS_GGUF)
- return list(DEFAULT_MODELS_STANDARD)
+ return suggestions_for_host(DEFAULT_MODELS_STANDARD, device)
diff --git a/studio/backend/core/inference/model_ids.py b/studio/backend/core/inference/model_ids.py
index fb19b95bafe..0d86d16b18c 100644
--- a/studio/backend/core/inference/model_ids.py
+++ b/studio/backend/core/inference/model_ids.py
@@ -14,7 +14,7 @@ and already-clean names untouched.
from __future__ import annotations
import os
-from typing import Optional
+from typing import Iterable, Optional
_GGUF_SUFFIX = ".gguf"
@@ -119,3 +119,40 @@ def model_id_matches(requested: Optional[str], internal: Optional[str]) -> bool:
if requested == internal:
return True
return public_model_id(internal) == requested
+
+
+# Mirror Zoo’s MLX repository substitution without importing the ML stack.
+_BNB_SUFFIXES = ("-unsloth-bnb-4bit", "-bnb-4bit")
+
+
+def mlx_bnb_base_repo(model_name: Optional[str]) -> Optional[str]:
+ """Return the replacement base repository, or None."""
+ if not isinstance(model_name, str) or not model_name.startswith("unsloth/"):
+ return None
+ if os.path.exists(model_name):
+ return None
+ for suffix in _BNB_SUFFIXES:
+ if model_name.endswith(suffix):
+ return model_name[: -len(suffix)]
+ return None
+
+
+def mlx_host_bnb_base_repo(model_name: Optional[str]) -> Optional[str]:
+ """Return the MLX replacement, excluding diffusion models."""
+ import utils.hardware.hardware as hw
+ from core.inference.diffusion_families import detect_family
+
+ if hw.get_device() != hw.DeviceType.MLX:
+ return None
+ if not isinstance(model_name, str) or detect_family(model_name) is not None:
+ return None
+ return mlx_bnb_base_repo(model_name)
+
+
+def mlx_bnb_substitutions(repos: Iterable[str]) -> list[tuple[str, str]]:
+ swaps = []
+ for repo in repos:
+ base = mlx_bnb_base_repo(repo)
+ if base:
+ swaps.append((repo, base))
+ return swaps
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 08725f71972..64ae394fe37 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -482,11 +482,19 @@ class InferenceOrchestrator:
self._start_top_models_fetch()
top_gguf = self._top_gguf_cache or []
top_hub = self._top_hub_cache or []
- # Never wait for the remote Hugging Face ranking during startup. Chat's first /api/models/list needs curated
- # defaults immediately; the background fetch backfills extra choices on later calls.
+ # Use detected hardware here: discovery runs on the event loop.
+ from core.inference.defaults import suggestions_for_host
+ import utils.hardware.hardware as _hw_mod
+
+ # A chat-only Mac never reaches the MLX loader, so its ranking is left as fetched.
+ device = None if _hw_mod.CHAT_ONLY else _hw_mod.DEVICE
+ fetched = suggestions_for_host(top_gguf + top_hub, device)
+ # Never wait for the remote Hugging Face ranking during startup. Chat's
+ # first /api/models/list needs curated defaults immediately; the
+ # background fetch backfills extra choices on later calls.
result: list[str] = []
seen: set[str] = set()
- for m in self._static_models + top_gguf + top_hub:
+ for m in self._static_models + fetched:
if m not in seen:
result.append(m)
seen.add(m)
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 644f892d0a5..3e8ccb0989e 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -451,6 +451,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
# loads; a no-progress Xet download is reported as a stall so the parent
# can respawn over HTTP. Watch model + base repos (base is the LoRA
# download bottleneck).
+ from core.inference.model_ids import mlx_bnb_substitutions
from utils.hf_xet_fallback import start_watchdog
watch_repos = [mc.identifier]
@@ -458,6 +459,23 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
if base and str(base) != mc.identifier:
watch_repos.append(str(base))
+ # Watch the repositories Zoo downloads after substitution.
+ if getattr(backend, "device", None) == "mlx":
+ substitutions = mlx_bnb_substitutions(watch_repos)
+ replacements = dict(substitutions)
+ watch_repos = list(dict.fromkeys(replacements.get(repo, repo) for repo in watch_repos))
+ for requested, mlx_base in substitutions:
+ _send_response(
+ resp_queue,
+ {
+ "type": "status",
+ "message": (
+ f"MLX cannot read bitsandbytes 4-bit weights; "
+ f"downloading {mlx_base} instead of {requested}"
+ ),
+ },
+ )
+
heartbeat_stop = start_watchdog(
repo_ids = watch_repos,
on_stall = lambda msg: _send_response(resp_queue, {"type": "stall", "message": msg}),
diff --git a/studio/backend/hub/services/models/cache_inventory.py b/studio/backend/hub/services/models/cache_inventory.py
index 79d7e402711..e6c2022ebb9 100644
--- a/studio/backend/hub/services/models/cache_inventory.py
+++ b/studio/backend/hub/services/models/cache_inventory.py
@@ -136,6 +136,72 @@ def get_repo_snapshot_metadata_cached(
return total, blob_hashes
+_mlx_plan_cache: OrderedDict = OrderedDict()
+
+
+def _cached_mlx_siblings(repo_id):
+ from huggingface_hub.hf_api import RepoSibling
+
+ # The raw tree retains LFS SHA256s, not just Git pointer IDs.
+ from huggingface_hub._tree_cache import read_tree_cache
+ from hub.utils.hf_cache_state import preferred_repo_cache_dirs
+
+ for entry in preferred_repo_cache_dirs("model", repo_id):
+ snapshot = hf_cache_scan.default_ref_snapshot(entry)
+ tree = read_tree_cache(str(entry), snapshot.name) if snapshot is not None else None
+ if tree:
+ return [
+ RepoSibling(rfilename = path, size = item.size, blob_id = item.lfs_sha256 or item.blob_id)
+ for path, item in tree.items()
+ ]
+ return []
+
+
+def get_mlx_load_plan_cached(repo_id: str, hf_token: Optional[str] = None):
+ from hub.utils.snapshot_filters import blob_hashes_for_siblings, mlx_load_siblings
+ from huggingface_hub import HfApi
+
+ key = (repo_id, hf_cache_scan.token_fingerprint(hf_token))
+ with _repo_size_cache_lock:
+ cached = _mlx_plan_cache.get(key)
+ if cached is not None and time.monotonic() - cached[1] < _REPO_SIZE_POS_TTL:
+ _mlx_plan_cache.move_to_end(key)
+ return cached[0]
+ try:
+ siblings = (
+ HfApi(token = hf_token)
+ .model_info(
+ repo_id,
+ files_metadata = True,
+ timeout = _MODEL_METADATA_TIMEOUT_SECONDS,
+ )
+ .siblings
+ )
+ except Exception:
+ try:
+ siblings = _cached_mlx_siblings(repo_id)
+ except Exception:
+ siblings = []
+ siblings = mlx_load_siblings(siblings)
+ files = tuple(
+ download_manifest.ExpectedFile(
+ path = item.rfilename,
+ size = int(item.size or 0),
+ sha256 = getattr(getattr(item, "lfs", None), "sha256", None),
+ )
+ for item in siblings
+ )
+ plan = (sum(file.size for file in files), blob_hashes_for_siblings(siblings), files)
+ if not siblings and cached is not None:
+ plan = cached[0]
+ with _repo_size_cache_lock:
+ _mlx_plan_cache[key] = (plan, time.monotonic())
+ _mlx_plan_cache.move_to_end(key)
+ while len(_mlx_plan_cache) > _REPO_SIZE_CACHE_MAX:
+ _mlx_plan_cache.popitem(last = False)
+ return plan
+
+
def all_hf_cache_scans():
return hf_cache_scan.all_hf_cache_scans()
diff --git a/studio/backend/hub/services/models/downloads.py b/studio/backend/hub/services/models/downloads.py
index 4bc4d98978f..997e0a113c6 100644
--- a/studio/backend/hub/services/models/downloads.py
+++ b/studio/backend/hub/services/models/downloads.py
@@ -714,6 +714,7 @@ async def get_download_progress_response(
repo_id: str,
expected_bytes: int = 0,
hf_token: Optional[str] = None,
+ mlx_load: bool = False,
) -> dict:
"""Return download progress for any HuggingFace model repo.
@@ -727,6 +728,26 @@ async def get_download_progress_response(
if account_access.managed_account():
await asyncio.to_thread(account_access.require_download_progress_access, _registry, repo_id)
hf_token = account_access.account_hf_token(hf_token)
+ if mlx_load:
+
+ def _metadata(repo, token):
+ total, hashes, _files = cache_inventory.get_mlx_load_plan_cached(repo, token)
+ return total, hashes
+
+ def _files(repo, token):
+ return cache_inventory.get_mlx_load_plan_cached(repo, token)[2]
+
+ return await snapshot_progress.snapshot_progress_response(
+ repo_type = "model",
+ repo_id = repo_id,
+ job_key = _download_job_key(repo_id, "@mlx"),
+ expected_bytes = 0,
+ hf_token = hf_token,
+ registry = _registry,
+ metadata_resolver = _metadata,
+ expected_files_resolver = _files,
+ variant = "@mlx",
+ )
return await snapshot_progress.snapshot_progress_response(
repo_type = "model",
repo_id = repo_id,
diff --git a/studio/backend/hub/utils/snapshot_filters.py b/studio/backend/hub/utils/snapshot_filters.py
index 39cedef6bcd..a0ad4952eb4 100644
--- a/studio/backend/hub/utils/snapshot_filters.py
+++ b/studio/backend/hub/utils/snapshot_filters.py
@@ -197,3 +197,28 @@ def blob_hashes_for_siblings(siblings: Iterable) -> frozenset[str]:
def snapshot_download_blob_hashes(siblings: Iterable) -> frozenset[str]:
return blob_hashes_for_siblings(snapshot_download_siblings(siblings))
+
+
+# mlx-lm's download filter, plus the weights of an exported adapter.
+_MLX_LOAD_PATTERNS = (
+ "*.json",
+ "model*.safetensors",
+ "*.py",
+ "tokenizer.model",
+ "*.tiktoken",
+ "tiktoken.model",
+ "*.txt",
+ "*.jsonl",
+ "*.jinja",
+ "adapters.safetensors",
+ "adapter_model.safetensors",
+ "adapter_model.bin",
+)
+
+
+def mlx_load_siblings(siblings):
+ return [
+ sibling
+ for sibling in siblings
+ if any(fnmatchcase(sibling.rfilename, pattern) for pattern in _MLX_LOAD_PATTERNS)
+ ]
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 599f1442179..9d3ab93ffc9 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -820,6 +820,12 @@ class ValidateModelResponse(BaseModel):
description = "Details for the transformers-upgrade dialog; set only when "
"requires_transformers_upgrade is true.",
)
+ mlx_loads_base_model: Optional[str] = Field(
+ None,
+ description = "On an MLX host, the full-precision repo that will be downloaded and "
+ "loaded in place of the requested unsloth bnb-4bit repo (or of a LoRA's bnb base), "
+ "because MLX cannot read bitsandbytes weights. None when the pick loads as asked.",
+ )
class EstimateMemoryRequest(BaseModel):
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 1af41809bf4..9987d58778b 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -16988,6 +16988,15 @@ def _requires_security_review_for_model(
return False
+def _mlx_base_for_config(config) -> Optional[str]:
+ from core.inference.model_ids import mlx_host_bnb_base_repo
+ for candidate in (getattr(config, "identifier", None), getattr(config, "base_model", None)):
+ base = mlx_host_bnb_base_repo(candidate)
+ if base:
+ return base
+ return None
+
+
@router.post("/validate", response_model = ValidateModelResponse)
async def validate_model(
request: ValidateModelRequest,
@@ -17368,6 +17377,7 @@ async def validate_model(
chat_template = chat_template,
requires_transformers_upgrade = transformers_upgrade is not None,
transformers_upgrade = transformers_upgrade,
+ mlx_loads_base_model = await asyncio.to_thread(_mlx_base_for_config, config),
)
)
diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py
index 5e9f6a090c2..c4ffda8abe1 100644
--- a/studio/backend/routes/models.py
+++ b/studio/backend/routes/models.py
@@ -4591,6 +4591,7 @@ def _resolve_hf_cache_realpath(repo_dir: Path) -> Optional[str]:
@router.get("/download-progress")
async def get_download_progress(
repo_id: str = Query(..., description = "HuggingFace repo ID"),
+ mlx_load: bool = Query(False),
hf_token: HfTokenArg = Depends(get_request_hf_token),
current_subject: str = Depends(get_current_subject),
via_api_key: bool = Depends(authenticated_via_api_key),
@@ -4599,7 +4600,9 @@ async def get_download_progress(
the cache directory it measured, so it takes the caller class like its ``/api/hub`` twin."""
from hub.services.models import downloads
return redact_host_paths(
- await downloads.get_download_progress_response(repo_id, hf_token = hf_token),
+ await downloads.get_download_progress_response(
+ repo_id, hf_token = hf_token, mlx_load = mlx_load
+ ),
via_api_key = via_api_key,
)
diff --git a/studio/backend/tests/test_cached_gguf_routes.py b/studio/backend/tests/test_cached_gguf_routes.py
index db20fec6362..f4b9c211f0c 100644
--- a/studio/backend/tests/test_cached_gguf_routes.py
+++ b/studio/backend/tests/test_cached_gguf_routes.py
@@ -2602,11 +2602,12 @@ def test_legacy_gguf_progress_delegates_to_shared_service(monkeypatch):
assert calls == [("org/repo", "Q4_K_M", 20, "token")]
-def test_legacy_model_progress_delegates_to_shared_service(monkeypatch):
[email protected]("mlx_load", [False, True])
+def test_legacy_model_progress_delegates_to_shared_service(monkeypatch, mlx_load):
calls = []
- async def shared(repo_id, *, hf_token):
- calls.append((repo_id, hf_token))
+ async def shared(repo_id, *, hf_token, mlx_load):
+ calls.append((repo_id, hf_token, mlx_load))
return {"downloaded_bytes": 10, "expected_bytes": 20, "progress": 0.5}
monkeypatch.setattr(
@@ -2619,11 +2620,12 @@ def test_legacy_model_progress_delegates_to_shared_service(monkeypatch):
repo_id = "org/repo",
hf_token = "token",
current_subject = "test-user",
+ mlx_load = mlx_load,
)
)
assert result["progress"] == 0.5
- assert calls == [("org/repo", "token")]
+ assert calls == [("org/repo", "token", mlx_load)]
def test_legacy_delete_delegates_to_shared_service(monkeypatch):
diff --git a/studio/backend/tests/test_mlx_bnb_substitution.py b/studio/backend/tests/test_mlx_bnb_substitution.py
new file mode 100644
index 00000000000..3aa76df19a3
--- /dev/null
+++ b/studio/backend/tests/test_mlx_bnb_substitution.py
@@ -0,0 +1,370 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parent.parent
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+import core.inference.defaults as defaults_mod # noqa: E402
+import utils.hardware.hardware as hw # noqa: E402
+from core.inference.model_ids import mlx_bnb_base_repo, mlx_bnb_substitutions # noqa: E402
+
+
+def test_unsloth_bnb_repos_resolve_to_their_base():
+ assert (
+ mlx_bnb_base_repo("unsloth/Qwen2-VL-2B-Instruct-bnb-4bit") == "unsloth/Qwen2-VL-2B-Instruct"
+ )
+ assert mlx_bnb_base_repo("unsloth/gemma-3-4b-it-unsloth-bnb-4bit") == "unsloth/gemma-3-4b-it"
+
+
+def test_repos_mlx_loads_as_given_have_no_base():
+ assert mlx_bnb_base_repo("unsloth/Qwen3-4B-Instruct-2507") is None
+ assert mlx_bnb_base_repo("unsloth/Llama-3.2-1B-Instruct-GGUF") is None
+ assert mlx_bnb_base_repo("someone-else/model-bnb-4bit") is None
+ assert mlx_bnb_base_repo(None) is None
+
+
+def test_a_local_directory_is_never_remapped(monkeypatch, tmp_path):
+ (tmp_path / "unsloth" / "model-bnb-4bit").mkdir(parents = True)
+ monkeypatch.chdir(tmp_path)
+
+ assert mlx_bnb_base_repo("unsloth/model-bnb-4bit") is None
+
+
+def test_substitutions_cover_a_loras_base():
+ swaps = mlx_bnb_substitutions(["me/my-lora", "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit"])
+ assert swaps == [
+ ("unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit", "unsloth/Meta-Llama-3.1-8B-Instruct")
+ ]
+
+ already_watched = ["unsloth/gemma-3-4b-it-bnb-4bit", "unsloth/gemma-3-4b-it"]
+ assert mlx_bnb_substitutions(already_watched) == [
+ ("unsloth/gemma-3-4b-it-bnb-4bit", "unsloth/gemma-3-4b-it")
+ ]
+
+
+def _standard_host(monkeypatch, device):
+ monkeypatch.setattr(hw, "CHAT_ONLY", False)
+ monkeypatch.setattr(hw, "get_device", lambda: device)
+
+
+def test_mlx_defaults_recommend_the_repos_mlx_actually_loads(monkeypatch):
+ _standard_host(monkeypatch, hw.DeviceType.MLX)
+
+ models = defaults_mod.get_default_models()
+
+ assert [model for model in models if model.endswith("bnb-4bit")] == []
+ assert "unsloth/Qwen2-VL-2B-Instruct" in models
+ assert len(models) == len(set(models))
+
+
+def test_cuda_defaults_keep_the_bnb_repos(monkeypatch):
+ _standard_host(monkeypatch, hw.DeviceType.CUDA)
+
+ assert defaults_mod.get_default_models() == defaults_mod.DEFAULT_MODELS_STANDARD
+
+
+def test_chat_only_hosts_are_untouched(monkeypatch):
+ monkeypatch.setattr(hw, "CHAT_ONLY", True)
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.MLX)
+
+ assert defaults_mod.get_default_models() == defaults_mod.DEFAULT_MODELS_GGUF
+
+
+ "identifier,base_model,is_lora,expected",
+ [
+ (
+ "unsloth/Qwen2-VL-2B-Instruct-bnb-4bit",
+ None,
+ False,
+ ["unsloth/Qwen2-VL-2B-Instruct"],
+ ),
+ (
+ "me/my-lora",
+ "unsloth/Qwen2-VL-2B-Instruct-bnb-4bit",
+ True,
+ ["me/my-lora", "unsloth/Qwen2-VL-2B-Instruct"],
+ ),
+ (
+ "unsloth/Qwen2-VL-2B-Instruct-bnb-4bit",
+ "unsloth/Qwen2-VL-2B-Instruct",
+ True,
+ ["unsloth/Qwen2-VL-2B-Instruct"],
+ ),
+ ],
+)
+def test_worker_only_watches_the_repositories_mlx_downloads(
+ monkeypatch, identifier, base_model, is_lora, expected
+):
+ from types import SimpleNamespace
+
+ import core.inference.worker as worker_mod
+ import utils.hf_xet_fallback as fallback_mod
+
+ model_config = SimpleNamespace(
+ identifier = identifier,
+ base_model = base_model,
+ is_lora = is_lora,
+ )
+ captured = {}
+
+ class StopEvent:
+ def set(self):
+ pass
+
+ class ResponseQueue:
+ def put(self, response):
+ pass
+
+ class Backend:
+ device = "mlx"
+
+ def load_model(self, **kwargs):
+ return False
+
+ def start_watchdog(**kwargs):
+ captured.update(kwargs)
+ return StopEvent()
+
+ monkeypatch.setattr(worker_mod, "_build_model_config", lambda config: model_config)
+ monkeypatch.setattr(worker_mod, "_resolve_lora_4bit", lambda model, requested: False)
+ monkeypatch.setattr(worker_mod, "_run_security_gates", lambda *args, **kwargs: True)
+ monkeypatch.setattr(fallback_mod, "start_watchdog", start_watchdog)
+
+ worker_mod._handle_load(Backend(), {"model_name": identifier}, ResponseQueue())
+
+ assert captured["repo_ids"] == expected
+
+
+def test_the_host_rule_only_fires_on_mlx(monkeypatch):
+ from core.inference.model_ids import mlx_host_bnb_base_repo
+
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA)
+ assert mlx_host_bnb_base_repo("unsloth/Qwen2-VL-2B-Instruct-bnb-4bit") is None
+
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.MLX)
+ assert (
+ mlx_host_bnb_base_repo("unsloth/Qwen2-VL-2B-Instruct-bnb-4bit")
+ == "unsloth/Qwen2-VL-2B-Instruct"
+ )
+
+
+def test_diffusion_bnb_repos_are_loaded_as_named(monkeypatch):
+ from core.inference.model_ids import mlx_host_bnb_base_repo
+
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.MLX)
+ assert mlx_host_bnb_base_repo("unsloth/Qwen-Image-2512-unsloth-bnb-4bit") is None
+ assert mlx_host_bnb_base_repo("unsloth/Z-Image-Turbo-unsloth-bnb-4bit") is None
+
+
+def test_validate_reports_the_repo_mlx_will_load(monkeypatch):
+ from types import SimpleNamespace
+
+ from routes.inference import _mlx_base_for_config
+
+ monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.MLX)
+ pick = SimpleNamespace(identifier = "unsloth/Qwen2-VL-2B-Instruct-bnb-4bit", base_model = None)
+ assert _mlx_base_for_config(pick) == "unsloth/Qwen2-VL-2B-Instruct"
+
+ adapter = SimpleNamespace(
+ identifier = "me/my-lora",
+ base_model = "unsloth/gemma-3-4b-it-bnb-4bit",
+ )
+ assert _mlx_base_for_config(adapter) == "unsloth/gemma-3-4b-it"
+
+ plain = SimpleNamespace(identifier = "unsloth/Qwen3-4B-Instruct-2507", base_model = None)
+ assert _mlx_base_for_config(plain) is None
+
+
+def test_the_fetched_ranking_is_mapped_on_a_mac_too(monkeypatch):
+ import core.inference.orchestrator as orch_mod
+
+ monkeypatch.setattr(orch_mod.InferenceOrchestrator, "_fetch_top_models", lambda self: None)
+ monkeypatch.setattr(defaults_mod, "get_default_models", lambda: ["unsloth/curated"])
+ monkeypatch.setattr(hw, "DETECTION_GENERATION", 1)
+ monkeypatch.setattr(hw, "CHAT_ONLY", False)
+ monkeypatch.setattr(hw, "DEVICE", hw.DeviceType.MLX)
+
+ orch = orch_mod.InferenceOrchestrator()
+ orch._top_gguf_cache = ["unsloth/Qwen3-4B-GGUF"]
+ orch._top_hub_cache = ["unsloth/Qwen3-8B-unsloth-bnb-4bit", "unsloth/Qwen3-4B-Instruct-2507"]
+
+ assert orch.default_models == [
+ "unsloth/curated",
+ "unsloth/Qwen3-4B-GGUF",
+ "unsloth/Qwen3-8B",
+ "unsloth/Qwen3-4B-Instruct-2507",
+ ]
+
+
+def test_a_diffusion_bnb_repo_in_the_ranking_keeps_its_name(monkeypatch):
+ import core.inference.orchestrator as orch_mod
+
+ monkeypatch.setattr(orch_mod.InferenceOrchestrator, "_fetch_top_models", lambda self: None)
+ monkeypatch.setattr(defaults_mod, "get_default_models", lambda: ["unsloth/curated"])
+ monkeypatch.setattr(hw, "DETECTION_GENERATION", 1)
+ monkeypatch.setattr(hw, "CHAT_ONLY", False)
+ monkeypatch.setattr(hw, "DEVICE", hw.DeviceType.MLX)
+
+ orch = orch_mod.InferenceOrchestrator()
+ orch._top_hub_cache = [
+ "unsloth/Qwen-Image-2512-unsloth-bnb-4bit",
+ "unsloth/Z-Image-Turbo-unsloth-bnb-4bit",
+ "unsloth/Qwen3-8B-unsloth-bnb-4bit",
+ ]
+
+ assert orch.default_models == [
+ "unsloth/curated",
+ # diffusion: read as named by diffusers/MPS
+ "unsloth/Qwen-Image-2512-unsloth-bnb-4bit",
+ "unsloth/Z-Image-Turbo-unsloth-bnb-4bit",
+ # text: mlx-lm cannot read it, so name the repo it really loads
+ "unsloth/Qwen3-8B",
+ ]
+
+
+def test_the_fetched_ranking_is_untouched_off_mlx(monkeypatch):
+ import core.inference.orchestrator as orch_mod
+
+ monkeypatch.setattr(orch_mod.InferenceOrchestrator, "_fetch_top_models", lambda self: None)
+ monkeypatch.setattr(defaults_mod, "get_default_models", lambda: ["unsloth/curated"])
+ monkeypatch.setattr(hw, "DETECTION_GENERATION", 1)
+ monkeypatch.setattr(hw, "CHAT_ONLY", False)
+ monkeypatch.setattr(hw, "DEVICE", hw.DeviceType.CUDA)
+
+ orch = orch_mod.InferenceOrchestrator()
+ orch._top_hub_cache = ["unsloth/Qwen3-8B-unsloth-bnb-4bit"]
+
+ assert orch.default_models == ["unsloth/curated", "unsloth/Qwen3-8B-unsloth-bnb-4bit"]
+
+
+def test_the_mirror_still_matches_the_loader_it_mirrors():
+ loader = pytest.importorskip("unsloth_zoo.mlx.loader")
+
+ for name in (
+ "unsloth/Qwen2-VL-2B-Instruct-bnb-4bit",
+ "unsloth/gemma-3-4b-it-unsloth-bnb-4bit",
+ "unsloth/Qwen3-4B-Instruct-2507",
+ "unsloth/Llama-3.2-1B-Instruct-GGUF",
+ "someone-else/model-bnb-4bit",
+ ):
+ remapped, _revision, swapped_from = loader._remap_unsloth_bnb_hub_id_for_mlx(
+ name, "some-revision"
+ )
+ expected = remapped if swapped_from is not None else None
+ assert mlx_bnb_base_repo(name) == expected, name
+
+
+ "weight_names",
+ [
+ ("model-00001-of-00002.safetensors", "model-00002-of-00002.safetensors"),
+ ("adapter_model.safetensors",),
+ ("adapters.safetensors",),
+ ],
+)
[email protected]("offline", [False, True])
+def test_mlx_progress_verifies_loader_files_without_a_manifest(
+ monkeypatch, tmp_path, weight_names, offline
+):
+ import asyncio
+ from collections import OrderedDict
+ from types import SimpleNamespace
+ from huggingface_hub import HfApi
+ from hub.services import snapshot_progress
+ from hub.services.models import cache_inventory, downloads
+
+ repo = "unsloth/progress-model"
+ entry = tmp_path / "models--unsloth--progress-model"
+ snap = entry / "snapshots" / ("a" * 40)
+ snap.mkdir(parents = True)
+ blobs = entry / "blobs"
+ blobs.mkdir()
+ names = ("config.json", *weight_names, "README.md")
+ siblings = [
+ SimpleNamespace(rfilename = name, size = 4, blob_id = f"blob{i}", lfs = None)
+ for i, name in enumerate(names)
+ ]
+ monkeypatch.setattr(HfApi, "model_info", lambda *_a, **_k: SimpleNamespace(siblings = siblings))
+ monkeypatch.setattr(cache_inventory, "_mlx_plan_cache", OrderedDict())
+ if offline:
+ from huggingface_hub._tree_cache import TreeCacheEntry, write_tree_cache
+ import hub.utils.hf_cache_state as cache_state
+
+ (entry / "refs").mkdir()
+ (entry / "refs" / "main").write_text(snap.name)
+ write_tree_cache(
+ str(entry),
+ snap.name,
+ {
+ item.rfilename: TreeCacheEntry(
+ size = item.size, blob_id = "pointer", lfs_sha256 = item.blob_id
+ )
+ for item in siblings
+ },
+ )
+
+ def offline_info(*_a, **_k):
+ raise OSError("offline")
+
+ monkeypatch.setattr(HfApi, "model_info", offline_info)
+ monkeypatch.setattr(cache_state, "preferred_repo_cache_dirs", lambda *_a, **_k: [entry])
+ monkeypatch.setattr(snapshot_progress, "preferred_repo_cache_dirs", lambda *_a, **_k: [entry])
+ monkeypatch.setattr(
+ downloads, "_registry", SimpleNamespace(get_job = lambda _key: SimpleNamespace(state = "idle"))
+ )
+
+ def progress():
+ return asyncio.run(downloads.get_download_progress_response(repo, mlx_load = True))
+
+ for i, name in enumerate(names[:-1]):
+ (blobs / f"blob{i}").write_bytes(b"data")
+ (snap / name).symlink_to(blobs / f"blob{i}")
+ reading = progress()
+ assert reading["expected_bytes"] == 4 * (len(names) - 1)
+ assert reading["complete_on_disk"] is (i == len(names) - 2)
+ assert reading["progress"] == 1
+ generic = asyncio.run(downloads.get_download_progress_response(repo))
+ assert generic["complete_on_disk"] is False
+ assert generic["progress"] < 1
+ for name in weight_names:
+ (snap / name).unlink()
+ assert progress()["complete_on_disk"] is False
+ i = names.index(name)
+ (snap / name).symlink_to(blobs / f"blob{i}")
+ assert progress()["progress"] == 1
+
+
[email protected]("warm", [False, True])
+def test_mlx_metadata_outage_reuses_plan_and_bounds_retries(monkeypatch, warm):
+ from collections import OrderedDict
+ from huggingface_hub import HfApi
+ from hub.services.models import cache_inventory
+
+ key = ("unsloth/cached-base", cache_inventory.hf_cache_scan.token_fingerprint(None))
+ plan = (4, frozenset({"weight"}), ())
+ expired = cache_inventory.time.monotonic() - cache_inventory._REPO_SIZE_POS_TTL - 1
+ monkeypatch.setattr(
+ cache_inventory, "_mlx_plan_cache", OrderedDict({key: (plan, expired)} if warm else {})
+ )
+ monkeypatch.setattr(cache_inventory, "_cached_mlx_siblings", lambda _repo: [])
+ calls = []
+
+ def unavailable(*_a, **_k):
+ calls.append(True)
+ raise OSError("unreachable")
+
+ monkeypatch.setattr(HfApi, "model_info", unavailable)
+ expected = plan if warm else (0, frozenset(), ())
+ for _ in range(2):
+ assert cache_inventory.get_mlx_load_plan_cached(key[0]) == expected
+ assert len(calls) == 1
diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts
index 581c9f847a2..4ef2e9c4f88 100644
--- a/studio/frontend/src/features/chat/api/chat-api.ts
+++ b/studio/frontend/src/features/chat/api/chat-api.ts
@@ -538,8 +538,10 @@ export interface DownloadProgressResponse {
export async function getDownloadProgress(
repoId: string,
hfToken?: string | null,
+ mlxLoad = false,
): Promise<DownloadProgressResponse> {
const params = new URLSearchParams({ repo_id: repoId });
+ if (mlxLoad) params.set("mlx_load", "true");
const response = await authFetch(`/api/models/download-progress?${params}`, {
headers: hubTokenHeader(hfToken),
});
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index c462da12b46..bbb399b8fab 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -1186,7 +1186,7 @@ export function useChatModelRuntime() {
nativePathToken != null ||
model?.isGguf === true);
const loraIsAdapter = lora?.exportType === "lora";
- const isLora =
+ let isLora =
explicitIsLora ?? model?.isLora ?? loraIsAdapter ?? false;
const displayName = model?.name || lora?.name || modelId;
const toastDisplayName = shortModelLabel(displayName);
@@ -1208,7 +1208,7 @@ export function useChatModelRuntime() {
previousModel?.isLora ?? (previousLora?.exportType === "lora");
const isLocal = isLocalModelPath(modelId);
const isCachedLora = isLora && isLocal;
- const loadingDescription = [
+ let loadingDescription = [
currentCheckpoint ? "Switching models." : null,
extraLoadingDescription ?? null,
isDownloaded ? "Loading cached model into memory." : null,
@@ -1239,6 +1239,9 @@ export function useChatModelRuntime() {
loadAbortRef.current = abortCtrl;
let hfToken = useChatRuntimeStore.getState().hfToken || null;
const postLoadRefresh = { needed: false };
+ let progressModelIds = [modelId];
+ let mlxLoadProgress = false;
+ let downloadComplete = isDownloaded || isCachedLora;
let cpuFallbackReason: CpuFallbackReason | null = null;
let mmprojFallbackReason: MmprojFallbackReason | null = null;
try {
@@ -1600,6 +1603,32 @@ export function useChatModelRuntime() {
}
: {}),
});
+ isLora = validation.is_lora ?? isLora;
+ if (validation.mlx_loads_base_model) {
+ mlxLoadProgress = true;
+ const mlxBaseDescription = isLora
+ ? `Loading the adapter with ${validation.mlx_loads_base_model} in place of its bitsandbytes base, downloading it first if needed.`
+ : `Loading ${validation.mlx_loads_base_model} instead, downloading it first if needed.`;
+ progressModelIds = isLora && !isLocal
+ ? [modelId, validation.mlx_loads_base_model]
+ : [validation.mlx_loads_base_model];
+ downloadComplete = false;
+ loadingDescription = [
+ currentCheckpoint ? "Switching models." : null,
+ extraLoadingDescription ?? null,
+ mlxBaseDescription,
+ ]
+ .filter(Boolean)
+ .join(" ");
+ setLoadProgress({
+ percent: 0,
+ label: "Preparing download",
+ phase: "downloading",
+ });
+ toast.info("MLX cannot use 4-bit bitsandbytes weights", {
+ description: mlxBaseDescription,
+ });
+ }
// Upgrade consent runs before the security dialogs; Accept installs and the load continues.
if (validation.requires_transformers_upgrade) {
const upgraded = await confirmTransformersUpgradeIfNeeded({
@@ -2316,7 +2345,7 @@ export function useChatModelRuntime() {
}
}
- const isCachedLoad = isDownloaded || isCachedLora;
+ const isCachedLoad = downloadComplete;
const toastTitle = isCachedLoad ? "Starting model…" : "Downloading model…";
const modelLoadToastOptions = (description: ReturnType<typeof renderLoadDescription>) => ({
description,
@@ -2398,8 +2427,6 @@ export function useChatModelRuntime() {
: `${base} • ${rateStr}`;
}
- let downloadComplete = isDownloaded || isCachedLora;
-
// A load that believes the weights are cached can still turn into a download (#9094): the
// backend re-fetches a blob it judged unsafe to resume. MOVEMENT is the only proof accepted,
// since bytes below the expected total is the ordinary state of a partial revision.
@@ -2422,16 +2449,49 @@ export function useChatModelRuntime() {
return;
}
try {
- const prog =
+ const progressModelIdsAtRequest = [...progressModelIds];
+ const progressResponses =
ggufVariant && expectedBytes > 0
- ? await getGgufDownloadProgress(
- modelId,
- ggufVariant,
- expectedBytes,
- hfToken,
- )
- : await getDownloadProgress(modelId, hfToken);
+ ? [
+ await getGgufDownloadProgress(
+ modelId,
+ ggufVariant,
+ expectedBytes,
+ hfToken,
+ ),
+ ]
+ : await Promise.all(
+ progressModelIdsAtRequest.map((progressModelId) =>
+ getDownloadProgress(progressModelId, hfToken, mlxLoadProgress),
+ ),
+ );
if (!loadingModelRef.current) return;
+ if (
+ progressModelIdsAtRequest.length !== progressModelIds.length ||
+ progressModelIdsAtRequest.some(
+ (progressModelId, index) =>
+ progressModelId !== progressModelIds[index],
+ )
+ ) {
+ return;
+ }
+ const allDownloadsComplete = progressResponses.every(
+ ({ progress }) => progress >= 1,
+ );
+ const firstProgress = progressResponses[0];
+ if (!firstProgress) return;
+ const prog =
+ progressResponses.find(
+ ({ progress }) => progress > 0 && progress < 1,
+ ) ??
+ progressResponses.find(
+ ({ downloaded_bytes, expected_bytes, progress }) =>
+ downloaded_bytes > 0 &&
+ expected_bytes === 0 &&
+ progress === 0,
+ ) ??
+ progressResponses.find(({ progress }) => progress < 1) ??
+ firstProgress;
if (prog.progress > 0 && prog.progress < 1) {
hasShownProgress = true;
@@ -2499,7 +2559,13 @@ export function useChatModelRuntime() {
),
});
}
- } else if (prog.progress >= 1 && hasShownProgress) {
+ } else if (
+ allDownloadsComplete &&
+ (hasShownProgress ||
+ progressModelIds.some(
+ (progressModelId) => progressModelId !== modelId,
+ ))
+ ) {
downloadComplete = true;
if (loadToastDismissedRef.current) {
setLoadProgress({
@@ -2513,9 +2579,11 @@ export function useChatModelRuntime() {
...modelLoadToastOptions(
renderLoadDescription(
"Starting model…",
- "Download complete. Loading the model into memory.",
- 100,
- "Download complete",
+ hasShownProgress
+ ? "Download complete. Loading the model into memory."
+ : loadingDescription,
+ hasShownProgress ? 100 : null,
+ hasShownProgress ? "Download complete" : null,
),
),
});
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 61ad5e3ac85..25b26c98186 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -155,6 +155,8 @@ export interface ValidateModelResponse {
/** Architecture only shipped by a newer transformers; UI pauses on the upgrade dialog. */
requires_transformers_upgrade?: boolean;
transformers_upgrade?: TransformersUpgradeInfo | null;
+ /** Replacement repository for an MLX BNB model or adapter base. */
+ mlx_loads_base_model?: string | null;
}
export interface GgufVariantDetail {
diff --git a/studio/frontend/tests/chat-cache-miss-download-label.test.ts b/studio/frontend/tests/chat-cache-miss-download-label.test.ts
index fb55f0de274..3657f702514 100644
--- a/studio/frontend/tests/chat-cache-miss-download-label.test.ts
+++ b/studio/frontend/tests/chat-cache-miss-download-label.test.ts
@@ -119,7 +119,7 @@ test("an unknown total still updates the toast the user is looking at", () => {
const hook = readText("../src/features/chat/hooks/use-chat-model-runtime.ts");
const start = hook.indexOf("prog.expected_bytes === 0 &&");
assert.ok(start > 0, "the unknown-total branch moved");
- const end = hook.indexOf("} else if (prog.progress >= 1 && hasShownProgress)", start);
+ const end = hook.indexOf("allDownloadsComplete &&", start);
assert.ok(end > start, "the branch after the unknown-total one moved");
const branch = hook.slice(start, end);
diff --git a/studio/frontend/tests/mlx-bnb-load-progress.test.ts b/studio/frontend/tests/mlx-bnb-load-progress.test.ts
new file mode 100644
index 00000000000..76bc1fed8b2
--- /dev/null
+++ b/studio/frontend/tests/mlx-bnb-load-progress.test.ts
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+
+const SOURCE = readFileSync(
+ new URL(
+ "../src/features/chat/hooks/use-chat-model-runtime.ts",
+ import.meta.url,
+ ),
+ "utf8",
+);
+
+const VALIDATION = SOURCE.slice(
+ SOURCE.indexOf("const validation = await validateModel"),
+ SOURCE.indexOf("// Upgrade consent runs before the security dialogs"),
+);
+const DOWNLOAD_POLL = SOURCE.slice(
+ SOURCE.indexOf("const pollDownload = async () =>"),
+ SOURCE.indexOf("const pollLoad = async () =>"),
+);
+
+test("MLX base substitution tracks every downloaded repository", () => {
+ assert.match(
+ VALIDATION,
+ /progressModelIds = isLora/,
+ );
+ assert.match(
+ VALIDATION,
+ /\[modelId, validation\.mlx_loads_base_model\]/,
+ );
+ assert.match(VALIDATION, /\[validation\.mlx_loads_base_model\]/);
+ assert.match(VALIDATION, /downloadComplete = false;/);
+ assert.match(VALIDATION, /mlxLoadProgress = true;/);
+ assert.match(
+ DOWNLOAD_POLL,
+ /progressModelIdsAtRequest\.map\(\(progressModelId\) =>/,
+ );
+ assert.match(DOWNLOAD_POLL, /getDownloadProgress\(progressModelId, hfToken, mlxLoadProgress\)/);
+});
+
+test("validated LoRA status controls download tracking", () => {
+ assert.match(SOURCE, /let isLora =/);
+ assert.match(VALIDATION, /isLora = validation\.is_lora \?\? isLora;/);
+ assert.ok(
+ VALIDATION.indexOf("isLora = validation.is_lora ?? isLora;") <
+ VALIDATION.indexOf("if (validation.mlx_loads_base_model)"),
+ );
+});
+
+test("local LoRA tracks only the substituted base download", () => {
+ assert.match(VALIDATION, /progressModelIds = isLora && !isLocal/);
+});
+
+test("a pre-substitution progress response cannot complete the base download", () => {
+ assert.match(
+ DOWNLOAD_POLL,
+ /const progressModelIdsAtRequest = \[\.\.\.progressModelIds\];/,
+ );
+ assert.match(
+ DOWNLOAD_POLL,
+ /progressModelIdsAtRequest\.length !== progressModelIds\.length/,
+ );
+ assert.match(
+ DOWNLOAD_POLL,
+ /progressModelIdsAtRequest\.some\(/,
+ );
+});
+
+test("all substituted downloads must finish before model startup", () => {
+ assert.match(
+ DOWNLOAD_POLL,
+ /const allDownloadsComplete = progressResponses\.every\(/,
+ );
+ assert.match(DOWNLOAD_POLL, /allDownloadsComplete &&/);
+ assert.match(
+ DOWNLOAD_POLL,
+ /progressModelIds\.some\(/,
+ );
+});
+
+test("LoRA substitution says the adapter still loads", () => {
+ assert.match(VALIDATION, /Loading the adapter with/);
+ assert.match(VALIDATION, /in place of its bitsandbytes base/);
+});