Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: offer the precision folders of an MLX repo like GGUF quants (#13565)

Daniel Han4e712c18fc9d
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 5abc6e5557f..bec90fb116f 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -4775,13 +4775,21 @@ class MLXInferenceBackend:
else:
load_kwargs["tensor_group"] = distributed_group
+ precision_folder = (
+ None if getattr(config, "is_gguf", False) else getattr(config, "gguf_variant", None)
+ )
+ load_source = model_name
+ if precision_folder:
+ from hub.utils.mlx_variants import download_precision_folder
+ load_source = download_precision_folder(model_name, precision_folder, hf_token)
+
# Freed before the replacement weights are allocated, for headroom.
self._model_fusion.close()
self._clear_prompt_cache()
model = None
try:
model, tokenizer_or_processor = FastMLXModel.from_pretrained(
- model_name,
+ load_source,
**load_kwargs,
)
except Exception as exc:
@@ -4824,7 +4832,7 @@ class MLXInferenceBackend:
use_vlm = False
load_kwargs["text_only"] = True
model, tokenizer_or_processor = FastMLXModel.from_pretrained(
- model_name,
+ load_source,
**load_kwargs,
)
@@ -4995,6 +5003,8 @@ class MLXInferenceBackend:
self.models[model_name] = {
# Per-model token for the native-template fallback (matches transformers).
"hf_token": hf_token,
+ "gguf_variant": precision_folder,
+ "precision_folder": precision_folder,
# Per-model trust_remote_code reused by the native-template reload (matches transformers).
"trust_remote_code": trust_remote_code,
"model": self._model,
diff --git a/studio/backend/hub/schemas/inventory.py b/studio/backend/hub/schemas/inventory.py
index c2229707d37..dd75ddcbb2e 100644
--- a/studio/backend/hub/schemas/inventory.py
+++ b/studio/backend/hub/schemas/inventory.py
@@ -146,6 +146,13 @@ class GgufVariantsResponse(BaseModel):
None,
description = "Whether a variantless load resolves GGUF weights; None when unanswered",
)
+ variant_format: Optional[str] = Field(
+ None,
+ description = (
+ "'mlx' when the variants are precision folders of an MLX repo rather than GGUF "
+ "quants; None for GGUF"
+ ),
+ )
class LocalModelCapabilities(BaseModel):
diff --git a/studio/backend/hub/services/download_lifecycle.py b/studio/backend/hub/services/download_lifecycle.py
index b4f71a23d8f..816a739eac0 100644
--- a/studio/backend/hub/services/download_lifecycle.py
+++ b/studio/backend/hub/services/download_lifecycle.py
@@ -146,17 +146,13 @@ def largest_download_file_bytes(
)
if not siblings:
return None
- from hub.utils.gguf_plan import (
- build_gguf_variant_plans,
- expected_file_from_sibling,
- plan_for_variant,
- )
+ from hub.utils.gguf_plan import expected_file_from_sibling, resolve_variant_plan
wanted = {f for f in (files or []) if f}
if wanted:
chosen = [s for s in siblings if getattr(s, "rfilename", None) in wanted]
elif variant:
- plan = plan_for_variant(build_gguf_variant_plans(siblings), variant)
+ plan = resolve_variant_plan(siblings, variant)
if plan is None:
# An unresolvable variant is the worker's error to report, not a transport verdict.
return None
diff --git a/studio/backend/hub/services/models/gguf_variants.py b/studio/backend/hub/services/models/gguf_variants.py
index c8551f019ea..36dd30ba042 100644
--- a/studio/backend/hub/services/models/gguf_variants.py
+++ b/studio/backend/hub/services/models/gguf_variants.py
@@ -9,6 +9,7 @@ from hub.services.models import account_access
import asyncio
import hashlib
+import json
import re
import threading
import time
@@ -66,6 +67,8 @@ from hub.utils.gguf_plan import (
build_gguf_variant_plans,
is_main_gguf_variant_path,
plan_from_expected_files,
+ precision_folder_plans,
+ sibling_sha256,
)
from utils.paths.path_utils import is_appledouble_metadata
@@ -217,7 +220,7 @@ def _variant_requirement_may_be_forgotten() -> bool:
def _build_gguf_variant_requirements(siblings: list) -> dict[str, _GgufVariantRequirement]:
- return build_gguf_variant_plans(siblings)
+ return build_gguf_variant_plans(siblings) or precision_folder_plans(siblings)
def gguf_variant_requirements(
@@ -1214,6 +1217,121 @@ async def _audio_cpp_variants_answer(
)
+_VISION_PIPELINE_TAGS = frozenset(
+ {"image-text-to-text", "image-to-text", "visual-question-answering"}
+)
+
+
+async def _mlx_precision_variants_answer(
+ repo_id: str,
+ hf_token: Optional[str],
+ offline: bool,
+ cache_readable: bool = True,
+) -> Optional[VariantsAnswer]:
+ """The precision folders of an MLX repo (``4-bit/``, ``6-bit/``, ``8bit/``) as its variants,
+ else None. Each folder is a whole checkpoint, so a pick downloads and loads that folder alone."""
+ if is_local_path(repo_id) or not _is_valid_repo_id(repo_id):
+ return None
+ from hub.utils.mlx_variants import cached_precision_folder
+ from hub.utils.snapshot_filters import (
+ precision_variant_subfolders,
+ repo_ships_root_safetensors,
+ )
+ from utils.utils import hf_cache_snapshot_dir_for_repo
+
+ def _listing() -> Optional[GgufVariantsResponse]:
+ sizes: dict[str, int] = {}
+ hashes: dict[str, Optional[str]] = {}
+ has_vision = False
+ if not offline:
+ from huggingface_hub import HfApi
+ try:
+ info = call_with_anonymous_retry(
+ lambda token: HfApi(token = token).model_info(
+ repo_id, files_metadata = True, timeout = 5.0
+ ),
+ hf_token,
+ )
+ except Exception as exc: # noqa: BLE001 - fall back to the cached copy below
+ logger.debug("MLX precision folders for %s unavailable: %s", repo_id, exc)
+ else:
+ tags = {str(tag).lower() for tag in (info.tags or [])}
+ if "mlx" not in tags and str(info.library_name or "").lower() != "mlx":
+ return None
+ sizes = {s.rfilename: int(s.size or 0) for s in (info.siblings or [])}
+ hashes = {s.rfilename: sibling_sha256(s) for s in (info.siblings or [])}
+ has_vision = str(info.pipeline_tag or "").lower() in _VISION_PIPELINE_TAGS
+ if not sizes:
+ # A caller refused this repo's cache learns nothing from it.
+ if not cache_readable:
+ return None
+ snapshot = hf_cache_snapshot_dir_for_repo(repo_id)
+ if snapshot is None:
+ return None
+ sizes = {
+ path.relative_to(snapshot).as_posix(): path.stat().st_size
+ for path in snapshot.rglob("*")
+ if path.is_file()
+ }
+ for folder in precision_variant_subfolders(sizes):
+ try:
+ config = json.loads((snapshot / folder / "config.json").read_text("utf-8"))
+ except (OSError, ValueError):
+ continue
+ has_vision = has_vision or isinstance(config.get("vision_config"), dict)
+ folders = precision_variant_subfolders(sizes)
+ if not folders:
+ return None
+
+ def _weights(names) -> Optional[tuple[str, ...]]:
+ # Content hashes: two folders of one architecture and dtype can match in size alone.
+ found = [hashes.get(name) for name in names if name.lower().endswith(".safetensors")]
+ return tuple(sorted(found)) if found and all(found) else None
+
+ root_weights = (
+ _weights([name for name in sizes if "/" not in name])
+ if repo_ships_root_safetensors(sizes)
+ else None
+ )
+ rows = sorted(
+ (
+ GgufVariantDetail(
+ filename = f"{folder}/",
+ quant = folder,
+ size_bytes = sum(sizes[name] for name in names),
+ download_size_bytes = sum(sizes[name] for name in names),
+ downloaded = cache_readable
+ and cached_precision_folder(repo_id, folder) is not None,
+ )
+ for folder, names in folders.items()
+ ),
+ key = lambda row: row.size_bytes,
+ )
+ default = next(
+ (
+ folder
+ for folder, names in folders.items()
+ if root_weights and _weights(names) == root_weights
+ ),
+ None,
+ )
+ loadable = [row.quant for row in rows if row.downloaded]
+ return GgufVariantsResponse(
+ repo_id = repo_id,
+ variants = rows,
+ has_vision = has_vision,
+ default_variant = default,
+ dependencies_resolved = True,
+ loadable_variants = loadable,
+ variant_format = "mlx",
+ )
+
+ response = await asyncio.to_thread(_listing)
+ if response is None:
+ return None
+ return VariantsAnswer(response, None, cache_authorized = False)
+
+
async def get_gguf_variants_answer(
repo_id: str,
prefer_local_cache: bool = False,
@@ -2370,8 +2488,28 @@ async def get_gguf_variants_answer(
configured_cache_key(),
)
try:
- return await _shared_variants_scan(inflight_key, _compute_with_cleanables)
- except HTTPException:
+ answer = await _shared_variants_scan(inflight_key, _compute_with_cleanables)
+ # Only after an empty GGUF listing, so GGUF answers never change.
+ if not answer.response.variants and not local_path:
+ mlx_answer = await _mlx_precision_variants_answer(
+ repo_id, hf_token, offline, cache_readable = answer.cache_authorized
+ )
+ if mlx_answer is not None:
+ return mlx_answer
+ return answer
+ except HTTPException as exc:
+ # Offline, a repo with no cached GGUF raises here, before the MLX fallback above.
+ if (
+ offline
+ and exc.status_code == 404
+ and not local_path
+ and not hub_cached_read_refused(
+ hf_token, repo_id = repo_id, is_cached = lambda: True, offline = True
+ )
+ ):
+ mlx_answer = await _mlx_precision_variants_answer(repo_id, hf_token, True)
+ if mlx_answer is not None:
+ return mlx_answer
raise
except Exception as e:
scrubbed = modelscope_missing(e) or download_registry.scrub_secrets(
diff --git a/studio/backend/hub/utils/gguf_plan.py b/studio/backend/hub/utils/gguf_plan.py
index a7d2e3a0e42..51d21a034af 100644
--- a/studio/backend/hub/utils/gguf_plan.py
+++ b/studio/backend/hub/utils/gguf_plan.py
@@ -451,6 +451,45 @@ def plan_for_variant(plans: dict[str, GgufVariantPlan], variant: str) -> Optiona
return plans[matches[0]] if len(matches) == 1 else None
+def precision_folder_plans(siblings: Sequence) -> dict[str, GgufVariantPlan]:
+ """Each MLX precision folder (``6-bit/``) as a variant plan over every file in the folder."""
+ from hub.utils.snapshot_filters import precision_variant_subfolders
+
+ by_name = {getattr(s, "rfilename", ""): s for s in siblings}
+ return {
+ folder.lower(): _precision_folder_plan(
+ tuple(expected_file_from_sibling(by_name[name]) for name in sorted(names))
+ )
+ for folder, names in precision_variant_subfolders(by_name).items()
+ }
+
+
+def _precision_folder_plan(expected: tuple[ExpectedFile, ...]) -> GgufVariantPlan:
+ hashes = frozenset(file.sha256 for file in expected if file.sha256)
+ size = sum(max(0, int(file.size or 0)) for file in expected)
+ return GgufVariantPlan(
+ main_filenames = frozenset(file.path for file in expected),
+ target_filenames = tuple(file.path for file in expected),
+ main_hashes = hashes,
+ required_hashes = hashes,
+ companion_hashes = frozenset(),
+ mmproj_filenames = frozenset(),
+ mmproj_hashes = frozenset(),
+ expected_files = expected,
+ main_size_bytes = size,
+ download_size_bytes = size,
+ )
+
+
+def resolve_variant_plan(siblings: Sequence, variant: str) -> Optional[GgufVariantPlan]:
+ """A GGUF variant's plan, else the MLX precision folder of that name."""
+ siblings = list(siblings)
+ plan = plan_for_variant(build_gguf_variant_plans(siblings), variant)
+ if plan is not None:
+ return plan
+ return precision_folder_plans(siblings).get((variant or "").strip().lower())
+
+
def _one_shard_family(main_files: Sequence[ExpectedFile]) -> tuple[ExpectedFile, ...]:
"""Narrow a variant's weight files to the one shard set the loader opens (see ``group_gguf_variant_files``); a genuine split keeps every shard."""
if len(main_files) < 2:
@@ -473,6 +512,16 @@ def plan_from_expected_files(
) -> GgufVariantPlan:
expected = tuple(expected_files)
all_main = tuple(file for file in expected if is_main_gguf_variant_path(file.path, variant))
+ from hub.utils.snapshot_filters import PRECISION_FOLDER_RE
+
+ folder = (variant or "").strip()
+ if (
+ not all_main
+ and expected
+ and PRECISION_FOLDER_RE.fullmatch(folder)
+ and all(file.path.lower().startswith(f"{folder.lower()}/") for file in expected)
+ ):
+ return _precision_folder_plan(expected)
main_files = _one_shard_family(all_main)
# A discarded family has to leave the plan ENTIRELY: target_filenames, required_hashes and download_size_bytes are what the worker fetches, so leaving the copy there downloaded it, then reclaim deleted it as not-ours (absent from main_hashes) and the job fetched it again.
kept = {file.path for file in main_files}
diff --git a/studio/backend/hub/utils/mlx_variants.py b/studio/backend/hub/utils/mlx_variants.py
new file mode 100644
index 00000000000..3ae9dbbf52e
--- /dev/null
+++ b/studio/backend/hub/utils/mlx_variants.py
@@ -0,0 +1,92 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Precision folders of one Hub repo (``2-bit/``, ``6-bit/``, ``8bit/``), each a whole MLX
+checkpoint, offered and loaded as variants of that repo the way GGUF quants are."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Optional
+
+from hub.utils.snapshot_filters import PRECISION_FOLDER_RE
+
+
+def is_precision_folder(variant: Optional[str]) -> bool:
+ return bool(variant) and PRECISION_FOLDER_RE.fullmatch(str(variant).strip()) is not None
+
+
+def cached_precision_folder(repo_id: str, folder: str) -> Optional[Path]:
+ """The cached ``<snapshot>/<folder>`` when it holds a complete checkpoint, else None. No network."""
+ if not is_precision_folder(folder):
+ return None
+ from utils.utils import checkpoint_directory_is_complete, hf_cache_snapshot_dir_for_repo
+
+ snapshot = hf_cache_snapshot_dir_for_repo(repo_id)
+ if snapshot is None:
+ return None
+ path = snapshot / folder
+ if (path / "config.json").is_file() and checkpoint_directory_is_complete(path):
+ return path
+ return None
+
+
+def precision_folder_config_dir(
+ repo_id: str, folder: str, hf_token: Optional[str]
+) -> Optional[Path]:
+ """The local directory holding ``<folder>/config.json`` (fetching only that file), or None
+ when the repo has no such folder, so a GGUF quant name never takes this path."""
+ if not is_precision_folder(folder):
+ return None
+ cached = cached_precision_folder(repo_id, folder)
+ if cached is not None:
+ return cached
+ from huggingface_hub import hf_hub_download
+
+ from hub.utils.hf_tokens import call_with_anonymous_retry
+ from utils.hf_cache_settings import active_hf_hub_cache
+
+ from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError
+
+ try:
+ config_path = call_with_anonymous_retry(
+ lambda token: hf_hub_download(
+ repo_id,
+ f"{folder}/config.json",
+ token = token,
+ cache_dir = active_hf_hub_cache(),
+ ),
+ hf_token,
+ )
+ except EntryNotFoundError as exc:
+ # Only "no such file" means no folder: any other failure must not load the root instead.
+ if isinstance(exc, LocalEntryNotFoundError):
+ raise
+ return None
+ return Path(config_path).parent
+
+
+def download_precision_folder(repo_id: str, folder: str, hf_token: Optional[str]) -> str:
+ """Fetch only ``<folder>/*`` and return its local directory; files already cached are reused."""
+ cached = cached_precision_folder(repo_id, folder)
+ if cached is not None:
+ return str(cached)
+ from huggingface_hub import snapshot_download
+
+ from hub.utils.hf_tokens import call_with_anonymous_retry
+ from utils.hf_cache_settings import active_hf_hub_cache
+
+ snapshot = call_with_anonymous_retry(
+ lambda token: snapshot_download(
+ repo_id,
+ allow_patterns = [f"{folder}/*"],
+ token = token,
+ cache_dir = active_hf_hub_cache(),
+ ),
+ hf_token,
+ )
+ path = os.path.join(snapshot, folder)
+ if not os.path.isfile(os.path.join(path, "config.json")):
+ raise FileNotFoundError(f"{repo_id} has no checkpoint in its '{folder}' folder.")
+ return path
diff --git a/studio/backend/hub/utils/snapshot_filters.py b/studio/backend/hub/utils/snapshot_filters.py
index a0ad4952eb4..d834626c2e1 100644
--- a/studio/backend/hub/utils/snapshot_filters.py
+++ b/studio/backend/hub/utils/snapshot_filters.py
@@ -18,6 +18,12 @@ SNAPSHOT_IGNORE_PATTERNS: tuple[str, ...] = (
"*.bin.index.json.bak",
)
CONSOLIDATED_PATTERN = "consolidated*"
+# No q<N>: GGUF quant names (Q4_K_M) must never match. [0-9], not \d: mirrored in dataset-size.ts.
+PRECISION_FOLDER_SOURCE = (
+ "(?:mlx[-_]?)?(?:[0-9]{1,2}[-_]?bits?(?:[-_][a-z0-9]+)*"
+ "|bf16|fp16|f16|fp32|f32|fp8|mxfp4|nvfp4)"
+)
+PRECISION_FOLDER_RE = re.compile(PRECISION_FOLDER_SOURCE, re.IGNORECASE)
DUPLICATE_WEIGHT_FORMAT_PATTERNS: tuple[str, ...] = (
"original/*",
"metal/*",
@@ -140,6 +146,23 @@ def redundant_torch_bin_files(filenames: Iterable[str]) -> list[str]:
return redundant
+def precision_variant_subfolders(filenames: Iterable[str]) -> dict[str, list[str]]:
+ """Top-level folders that each hold a whole checkpoint at one precision (``4-bit/``, ``8bit/``,
+ ``mlx-4bit/``, ``bf16/``), mapped to their files. A folder needs a precision-like name, its own
+ ``config.json`` and safetensors weights, so component folders (``vae/``, ``mtp/``) never count."""
+ by_folder: dict[str, list[str]] = {}
+ for name in filenames:
+ folder, sep, rest = name.partition("/")
+ if sep and rest and "/" not in rest and PRECISION_FOLDER_RE.fullmatch(folder):
+ by_folder.setdefault(folder, []).append(name)
+ return {
+ folder: names
+ for folder, names in by_folder.items()
+ if f"{folder}/config.json" in names
+ and any(name.lower().endswith(".safetensors") for name in names)
+ }
+
+
def resolve_snapshot_ignore_patterns_for_files(filenames: Iterable[str]) -> list[str]:
names = list(filenames)
ignore = list(SNAPSHOT_IGNORE_PATTERNS)
@@ -149,6 +172,9 @@ def resolve_snapshot_ignore_patterns_for_files(filenames: Iterable[str]) -> list
ignore.extend(DUPLICATE_WEIGHT_FORMAT_PATTERNS)
# Exact names, not globs: fnmatch treats "." literally, so each entry matches only itself.
ignore.extend(redundant_torch_bin_files(names))
+ # Only when the root is itself a checkpoint, or the folders are the only runnable copies.
+ if "config.json" in names:
+ ignore.extend(f"{folder}/*" for folder in precision_variant_subfolders(names))
return ignore
diff --git a/studio/backend/hub/workers/hf_download.py b/studio/backend/hub/workers/hf_download.py
index 06f82f03abf..04858f112a5 100644
--- a/studio/backend/hub/workers/hf_download.py
+++ b/studio/backend/hub/workers/hf_download.py
@@ -36,9 +36,8 @@ from hub.utils.snapshot_filters import (
)
from hub.utils.gguf_plan import (
GgufVariantPlan,
- build_gguf_variant_plans,
- plan_for_variant,
plan_from_expected_files,
+ resolve_variant_plan,
sibling_sha256,
)
from hub.utils.state_dir import RepoType
@@ -713,8 +712,8 @@ def _gguf_variant_target_plan(
raise RuntimeError(
f"Metadata unavailable while resolving GGUF variant '{variant}' " f"for {repo_id}"
) from e
- # plan_for_variant, not .get: a repo filing every variant under one shared container qualifies every key, so a stored pin or an explicit repo:Q4_K_M missed the map and the worker exited with "No GGUF shards matching variant".
- return plan_for_variant(build_gguf_variant_plans(list(info.siblings)), variant)
+ # plan_for_variant (inside resolve_variant_plan), not .get: a repo filing every variant under one shared container qualifies every key, so a stored pin or an explicit repo:Q4_K_M missed the map and the worker exited with "No GGUF shards matching variant".
+ return resolve_variant_plan(info.siblings, variant)
def _download_gguf_variant(
diff --git a/studio/backend/models/models.py b/studio/backend/models/models.py
index 49026d3bf9d..caa895a48d6 100644
--- a/studio/backend/models/models.py
+++ b/studio/backend/models/models.py
@@ -257,6 +257,13 @@ class GgufVariantsResponse(BaseModel):
None,
description = "Whether a variantless load resolves GGUF weights; None when unanswered",
)
+ variant_format: Optional[str] = Field(
+ None,
+ description = (
+ "'mlx' when the variants are precision folders of an MLX repo rather than GGUF "
+ "quants; None for GGUF"
+ ),
+ )
LocalModelSource = Literal[
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index cb95436faa9..a482822bd89 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -15210,9 +15210,15 @@ def _local_mlx_model_dir(config: ModelConfig) -> Optional[str]:
return False
return _complete(directory, shards)
+ folder = None if getattr(config, "is_gguf", False) else getattr(config, "gguf_variant", None)
+
+ def _pick(path) -> Optional[str]:
+ target = Path(path) / folder if folder else Path(path)
+ return str(target) if _usable(target) else None
+
candidate = getattr(config, "path", None)
- if candidate and _usable(candidate):
- return str(candidate)
+ if candidate and (found := _pick(candidate)):
+ return found
identifier = candidate or getattr(config, "identifier", None)
if not identifier or getattr(config, "is_local", False):
return None
@@ -15231,8 +15237,8 @@ def _local_mlx_model_dir(config: ModelConfig) -> Optional[str]:
except Exception:
continue
for snapshot in _the_revision_that_loads(list(snapshots)):
- if _usable(snapshot):
- return str(snapshot)
+ if found := _pick(snapshot):
+ return found
return None
@@ -18706,6 +18712,10 @@ async def _load_model_impl(
_same_loaded_identifier(backend.active_model_name, model_identifier)
and (backend.models.get(backend.active_model_name) or {}).get("engine", "auto")
== request.engine
+ # A resident precision folder (6-bit/) is not the root checkpoint a variantless request names.
+ and not (backend.models.get(backend.active_model_name) or {}).get(
+ "precision_folder"
+ )
and _mlx_runtime_settings_match(backend, request)
and _non_gguf_runtime_settings_match(backend, request)
and _resident_context_satisfies(
diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py
index 8940c8c9491..947659a43ca 100644
--- a/studio/backend/routes/models.py
+++ b/studio/backend/routes/models.py
@@ -4843,6 +4843,7 @@ async def get_gguf_variants(
dependencies_resolved = bool(getattr(response, "dependencies_resolved", False)),
loadable_variants = getattr(response, "loadable_variants", None),
loadable = getattr(response, "loadable", None),
+ variant_format = getattr(response, "variant_format", None),
),
via_api_key = via_api_key,
)
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 1f9407d9e80..ced42257845 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -9647,3 +9647,39 @@ def test_a_turboquant_load_turns_a_pin_into_a_request_budget(monkeypatch):
assert policy(8192, 8192) == (3.5, None, False, 8192)
assert policy(0, 262144) == (3.5, None, False, None)
assert policy(8192, 8192, "refused") == (None, 8192, True, None)
+
+
+def test_mlx_precision_folder_loads_from_its_own_download(monkeypatch, tmp_path):
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ _install_fake_mlx(monkeypatch)
+ calls = []
+ _install_fake_fast_mlx(monkeypatch, calls)
+ fetched = []
+
+ def _snapshot_download(
+ repo_id,
+ allow_patterns = None,
+ **_,
+ ):
+ fetched.append((repo_id, allow_patterns))
+ (tmp_path / "6-bit").mkdir(exist_ok = True)
+ (tmp_path / "6-bit" / "config.json").write_text("{}")
+ return str(tmp_path)
+
+ monkeypatch.setattr("huggingface_hub.snapshot_download", _snapshot_download)
+ monkeypatch.setattr("utils.utils.hf_cache_snapshot_dir_for_repo", lambda repo: None)
+
+ backend = MLXInferenceBackend()
+ repo = "orcarouter/Qwen3.8-27B-Uncensored-MLX"
+ config = SimpleNamespace(identifier = repo, is_vision = False, gguf_variant = "6-bit")
+ assert backend.load_model(config)
+ assert calls[-1][0] == (str(tmp_path / "6-bit"),)
+ assert fetched == [(repo, ["6-bit/*"])]
+ assert backend.active_model_name == repo
+ assert backend.models[repo]["gguf_variant"] == "6-bit"
+
+ assert backend.load_model(SimpleNamespace(identifier = repo, is_vision = False))
+ assert calls[-1][0] == (repo,)
+ assert backend.models[repo]["gguf_variant"] is None
+ assert len(fetched) == 1
diff --git a/studio/backend/tests/test_mlx_precision_variants.py b/studio/backend/tests/test_mlx_precision_variants.py
new file mode 100644
index 00000000000..6086d5cc7f1
--- /dev/null
+++ b/studio/backend/tests/test_mlx_precision_variants.py
@@ -0,0 +1,557 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MLX repos that keep each precision in its own folder (#13418): listed like GGUF quants, the
+picked folder fetched and loaded alone, and a download of the repo root skipping the others."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from hub.services.models import gguf_variants
+from hub.utils.snapshot_filters import (
+ resolve_snapshot_ignore_patterns_for_files,
+ snapshot_download_size,
+)
+
+GB = 1_000_000_000
+_REPO = "orcarouter/Qwen3.8-27B-Uncensored-MLX"
+
+
+def _folder(folder: str, shards: list[int]) -> dict[str, int]:
+ prefix = f"{folder}/" if folder else ""
+ files = {
+ f"{prefix}config.json": 4_000,
+ f"{prefix}tokenizer_config.json": 9_000,
+ f"{prefix}model.safetensors.index.json": 90_000,
+ }
+ for index, size in enumerate(shards, 1):
+ files[f"{prefix}model-{index:05d}-of-{len(shards):05d}.safetensors"] = size
+ return files
+
+
+_SIZES = {
+ **_folder("", [5 * GB, 5 * GB, 6 * GB]),
+ **_folder("2-bit", [5 * GB, 4 * GB]),
+ **_folder("4-bit", [5 * GB, 5 * GB, 6 * GB]),
+ **_folder("6-bit", [5 * GB, 5 * GB, 5 * GB, 5 * GB, 3 * GB]),
+ **_folder("8-bit", [5 * GB] * 6),
+ "mtp/config.json": 2_000,
+ "mtp/model.safetensors": GB,
+ "README.md": 10_000,
+}
+
+
+def _siblings(sizes = _SIZES, content = None):
+ """Hub siblings whose LFS hash stands for the file's bytes: by default the root and 4-bit/ hold
+ the same files, as in the issue's repo."""
+ content = content or (lambda name: name.replace("4-bit/", "", 1))
+ return [
+ SimpleNamespace(rfilename = name, size = size, lfs = {"sha256": f"sha-{content(name)}"})
+ for name, size in sizes.items()
+ ]
+
+
+def test_precision_folders_are_whole_checkpoints_with_a_precision_name():
+ from hub.utils.snapshot_filters import precision_variant_subfolders
+
+ folders = precision_variant_subfolders(_SIZES)
+ assert sorted(folders) == ["2-bit", "4-bit", "6-bit", "8-bit"]
+ for name in ("4bit", "8bit", "mlx-4bit", "2bit-lite", "bf16", "MLX-8BIT", "mxfp4"):
+ assert precision_variant_subfolders(
+ {f"{name}/config.json": 1, f"{name}/model.safetensors": 1}
+ )
+ for name in ("mtp", "vae", "text_encoder", "transformer", "dflash", "layers-0-24", "merged-hf"):
+ assert not precision_variant_subfolders(
+ {f"{name}/config.json": 1, f"{name}/model.safetensors": 1}
+ )
+ assert not precision_variant_subfolders({"4-bit/model.safetensors": 1})
+ assert not precision_variant_subfolders({"4-bit/config.json": 1, "4-bit/model.Q4_K_M.gguf": 1})
+ assert not precision_variant_subfolders(
+ {"4-bit/x/config.json": 1, "4-bit/x/model.safetensors": 1}
+ )
+
+
+def test_root_download_skips_the_other_precisions():
+ ignore = resolve_snapshot_ignore_patterns_for_files(_SIZES)
+ assert {"2-bit/*", "4-bit/*", "6-bit/*", "8-bit/*"} <= set(ignore)
+ assert "mtp/*" not in ignore
+ assert snapshot_download_size(_siblings()) < 18 * GB
+ no_root = {k: v for k, v in _SIZES.items() if "/" in k}
+ assert not any(p.endswith("bit/*") for p in resolve_snapshot_ignore_patterns_for_files(no_root))
+ stray = {**no_root, "model.safetensors": GB}
+ assert not any(p.endswith("bit/*") for p in resolve_snapshot_ignore_patterns_for_files(stray))
+
+
+def test_frontend_size_estimate_mirrors_the_precision_folder_rule():
+ from hub.utils.snapshot_filters import PRECISION_FOLDER_SOURCE
+ source = (
+ Path(__file__).resolve().parents[2]
+ / "frontend"
+ / "src"
+ / "features"
+ / "hub"
+ / "lib"
+ / "dataset-size.ts"
+ ).read_text(encoding = "utf-8")
+ assert f'"{PRECISION_FOLDER_SOURCE}"' in source
+
+
+def _fake_hub(
+ monkeypatch,
+ *,
+ tags = ("mlx", "safetensors"),
+ library = "mlx",
+ sizes = _SIZES,
+ content = None,
+):
+ calls = []
+
+ class _Api:
+ def __init__(self, token = None):
+ pass
+
+ def model_info(
+ self,
+ repo_id,
+ files_metadata = False,
+ timeout = None,
+ ):
+ calls.append(repo_id)
+ return SimpleNamespace(
+ tags = list(tags),
+ library_name = library,
+ pipeline_tag = "image-text-to-text",
+ siblings = _siblings(sizes, content),
+ )
+
+ monkeypatch.setattr("huggingface_hub.HfApi", _Api)
+ monkeypatch.setattr("utils.utils.hf_cache_snapshot_dir_for_repo", lambda repo: None)
+ return calls
+
+
+def _no_gguf_rows(monkeypatch, cache_authorized = True):
+ empty = gguf_variants.VariantsAnswer(
+ gguf_variants.GgufVariantsResponse(repo_id = _REPO), None, cache_authorized
+ )
+
+ async def _scan(key, compute):
+ return empty
+
+ monkeypatch.setattr(gguf_variants, "_shared_variants_scan", _scan)
+ return empty
+
+
+def test_mlx_repo_lists_its_precision_folders_as_variants(monkeypatch):
+ _no_gguf_rows(monkeypatch)
+ _fake_hub(monkeypatch)
+ answer = asyncio.run(gguf_variants.get_gguf_variants_answer(_REPO))
+ response = answer.response
+ assert [(v.quant, v.filename) for v in response.variants] == [
+ ("2-bit", "2-bit/"),
+ ("4-bit", "4-bit/"),
+ ("6-bit", "6-bit/"),
+ ("8-bit", "8-bit/"),
+ ]
+ sizes = {v.quant: v.size_bytes for v in response.variants}
+ assert sizes["6-bit"] == 23 * GB + 103_000
+ assert sizes["8-bit"] == 30 * GB + 103_000
+ assert response.default_variant == "4-bit"
+ assert response.has_vision is True
+ assert not any(v.downloaded for v in response.variants)
+ assert response.variant_format == "mlx"
+ assert answer.cache_authorized is False
+
+
+def test_non_mlx_repo_and_repo_without_folders_get_no_mlx_listing(monkeypatch):
+ empty = _no_gguf_rows(monkeypatch)
+ _fake_hub(monkeypatch, tags = ("safetensors",), library = "transformers")
+ assert asyncio.run(gguf_variants.get_gguf_variants_answer(_REPO)) is empty
+ _fake_hub(monkeypatch, sizes = _folder("", [GB]))
+ assert asyncio.run(gguf_variants.get_gguf_variants_answer(_REPO)) is empty
+
+
+def test_gguf_listing_is_answered_without_asking_for_mlx_folders(monkeypatch):
+ row = gguf_variants.GgufVariantDetail(filename = "m-Q4_K_M.gguf", quant = "Q4_K_M", size_bytes = 1)
+ gguf = gguf_variants.VariantsAnswer(
+ gguf_variants.GgufVariantsResponse(repo_id = "a/b-GGUF", variants = [row]), None
+ )
+
+ async def _scan(key, compute):
+ return gguf
+
+ class _NoHub:
+ def __init__(self, token = None):
+ raise AssertionError("a GGUF repo must not be listed as MLX")
+
+ monkeypatch.setattr(gguf_variants, "_shared_variants_scan", _scan)
+ monkeypatch.setattr("huggingface_hub.HfApi", _NoHub)
+ assert asyncio.run(gguf_variants.get_gguf_variants_answer("a/b-GGUF")) is gguf
+
+
+def test_download_fetches_only_the_picked_folder(monkeypatch, tmp_path):
+ from hub.utils import mlx_variants
+
+ snapshot = tmp_path / "snapshots" / "abc"
+ calls = []
+
+ def _snapshot_download(
+ repo_id,
+ allow_patterns = None,
+ token = None,
+ cache_dir = None,
+ ):
+ calls.append((repo_id, allow_patterns))
+ (snapshot / "6-bit").mkdir(parents = True, exist_ok = True)
+ (snapshot / "6-bit" / "config.json").write_text("{}")
+ return str(snapshot)
+
+ monkeypatch.setattr(mlx_variants, "cached_precision_folder", lambda repo, folder: None)
+ monkeypatch.setattr("huggingface_hub.snapshot_download", _snapshot_download)
+ path = mlx_variants.download_precision_folder(_REPO, "6-bit", None)
+ assert path == str(snapshot / "6-bit")
+ assert calls == [(_REPO, ["6-bit/*"])]
+
+ monkeypatch.setattr(
+ mlx_variants, "cached_precision_folder", lambda repo, folder: snapshot / folder
+ )
+ assert mlx_variants.download_precision_folder(_REPO, "6-bit", None) == str(snapshot / "6-bit")
+ assert len(calls) == 1
+
+
+def test_cached_precision_folder_needs_every_shard(monkeypatch, tmp_path):
+ from hub.utils import mlx_variants
+
+ folder = tmp_path / "8-bit"
+ folder.mkdir()
+ (folder / "config.json").write_text("{}")
+ index = {
+ "weight_map": {
+ "a": "model-00001-of-00002.safetensors",
+ "b": "model-00002-of-00002.safetensors",
+ }
+ }
+ (folder / "model.safetensors.index.json").write_text(json.dumps(index))
+ (folder / "model-00001-of-00002.safetensors").write_bytes(b"x")
+ monkeypatch.setattr("utils.utils.hf_cache_snapshot_dir_for_repo", lambda repo: tmp_path)
+ assert mlx_variants.cached_precision_folder(_REPO, "8-bit") is None
+ (folder / "model-00002-of-00002.safetensors").write_bytes(b"x")
+ assert mlx_variants.cached_precision_folder(_REPO, "8-bit") == folder
+ assert mlx_variants.cached_precision_folder(_REPO, "../8-bit") is None
+
+
[email protected]("variant", ["Q4_K_M", "UD-Q4_K_XL", "Q8_0", "../6-bit", "6-bit/../x"])
+def test_gguf_quant_names_never_resolve_as_folders(monkeypatch, variant):
+ from hub.utils import mlx_variants
+
+ reads = []
+ monkeypatch.setattr("huggingface_hub.hf_hub_download", lambda *a, **k: reads.append(a))
+ assert mlx_variants.precision_folder_config_dir(_REPO, variant, None) is None
+ assert reads == []
+
+
+def test_model_config_carries_the_folder_and_classifies_it(monkeypatch, tmp_path):
+ from utils.models import model_config
+
+ folder = tmp_path / "6-bit"
+ folder.mkdir()
+ (folder / "config.json").write_text(json.dumps({"model_type": "qwen3_5", "vision_config": {}}))
+ checked = []
+ fetched = []
+
+ def _hf_hub_download(
+ repo_id,
+ filename,
+ token = None,
+ cache_dir = None,
+ **_,
+ ):
+ fetched.append(filename)
+ if filename != "6-bit/config.json":
+ raise FileNotFoundError(filename)
+ return str(folder / "config.json")
+
+ monkeypatch.setattr("huggingface_hub.hf_hub_download", _hf_hub_download)
+ monkeypatch.setattr("utils.utils.hf_cache_snapshot_dir_for_repo", lambda repo: None)
+ monkeypatch.setattr(model_config, "detect_gguf_model_remote", lambda *a, **k: None)
+ monkeypatch.setattr(
+ model_config, "is_vision_model", lambda name, hf_token = None: checked.append(name) or True
+ )
+ monkeypatch.setattr(model_config, "detect_audio_type", lambda *a, **k: None)
+ monkeypatch.setattr(model_config, "is_model_cached", lambda *a, **k: False)
+
+ config = model_config.ModelConfig.from_identifier(_REPO, gguf_variant = "6-bit")
+ assert config.identifier == _REPO
+ assert config.gguf_variant == "6-bit"
+ assert config.is_gguf is False
+ assert checked == [str(folder)]
+ assert fetched == ["6-bit/config.json"]
+
+ checked.clear()
+ plain = model_config.ModelConfig.from_identifier(_REPO)
+ assert plain.gguf_variant is None
+ assert checked == [_REPO]
+
+
+def _cached_repo(monkeypatch, tmp_path, repo, folder):
+ """``repo`` in a private Hub cache with only ``folder/`` downloaded, as a precision pick leaves it."""
+ from utils import hf_cache_settings
+
+ hub = tmp_path / "hub"
+ repo_dir = hub / f"models--{repo.replace('/', '--')}"
+ snapshot = repo_dir / "snapshots" / ("a" * 40)
+ (snapshot / folder).mkdir(parents = True)
+ (repo_dir / "refs").mkdir()
+ (repo_dir / "refs" / "main").write_text("a" * 40)
+ monkeypatch.setattr(
+ hf_cache_settings,
+ "get_hf_cache_paths",
+ lambda: hf_cache_settings.HuggingFaceCachePaths(tmp_path, hub, tmp_path / "xet", "studio"),
+ )
+ return snapshot / folder
+
+
+def test_a_downloaded_precision_folder_loads_offline(monkeypatch, tmp_path):
+ from utils.models import model_config
+
+ folder = _cached_repo(monkeypatch, tmp_path, _REPO, "6-bit")
+ (folder / "config.json").write_text(
+ json.dumps({"model_type": "qwen3", "quantization": {"bits": 6}})
+ )
+ (folder / "model.safetensors").write_bytes(b"x")
+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")
+
+ def _no_hub(*args, **kwargs):
+ raise AssertionError("an offline load of a downloaded folder must not reach the Hub")
+
+ monkeypatch.setattr("huggingface_hub.hf_hub_download", _no_hub)
+ monkeypatch.setattr("huggingface_hub.snapshot_download", _no_hub)
+ monkeypatch.setattr(model_config, "is_model_cached", lambda *a, **k: True)
+
+ config = model_config.ModelConfig.from_identifier(_REPO, gguf_variant = "6-bit")
+ assert config.identifier == _REPO
+ assert config.gguf_variant == "6-bit"
+ assert config.is_gguf is False
+
+ from hub.utils.mlx_variants import download_precision_folder
+
+ assert download_precision_folder(_REPO, "6-bit", None) == str(folder)
+
+
+def test_the_download_manager_plans_a_precision_folder(monkeypatch):
+ # Was: "No GGUF shards matching variant '6-bit'" when the picker staged an uncached folder.
+ from hub.workers import hf_download
+
+ monkeypatch.setattr(
+ hf_download,
+ "_model_info_with_retry",
+ lambda repo_id, token: SimpleNamespace(siblings = _siblings()),
+ )
+ plan = hf_download._gguf_variant_target_plan(_REPO, "6-bit", None)
+ assert plan is not None
+ assert all(name.startswith("6-bit/") for name in plan.target_filenames)
+ assert plan.download_size_bytes == 23 * GB + 103_000
+ assert hf_download._gguf_variant_target_plan(_REPO, "mtp", None) is None
+
+
+def test_a_gguf_variant_keeps_its_own_plan():
+ from hub.utils.gguf_plan import resolve_variant_plan
+
+ siblings = _siblings({"m-Q4_K_M.gguf": 5 * GB, "m-Q8_0.gguf": 9 * GB, **_folder("4-bit", [GB])})
+ plan = resolve_variant_plan(siblings, "Q4_K_M")
+ assert plan.target_filenames == ("m-Q4_K_M.gguf",)
+
+
+def test_a_failed_folder_probe_never_falls_back_to_the_root(monkeypatch):
+ from huggingface_hub.errors import LocalEntryNotFoundError
+
+ from hub.utils import mlx_variants
+
+ monkeypatch.setattr("utils.utils.hf_cache_snapshot_dir_for_repo", lambda repo: None)
+
+ def _raise(error):
+ def _download(*args, **kwargs):
+ raise error
+
+ return _download
+
+ from huggingface_hub.errors import EntryNotFoundError
+
+ monkeypatch.setattr("huggingface_hub.hf_hub_download", _raise(EntryNotFoundError("404")))
+ assert mlx_variants.precision_folder_config_dir(_REPO, "6-bit", None) is None
+ for error in (LocalEntryNotFoundError("offline"), ConnectionError("reset")):
+ monkeypatch.setattr("huggingface_hub.hf_hub_download", _raise(error))
+ with pytest.raises(type(error)):
+ mlx_variants.precision_folder_config_dir(_REPO, "6-bit", None)
+
+
+def test_a_rejected_token_still_downloads_a_public_folder(monkeypatch, tmp_path):
+ from hub.utils import mlx_variants
+
+ tokens = []
+
+ def _snapshot_download(
+ repo_id,
+ allow_patterns = None,
+ token = None,
+ cache_dir = None,
+ ):
+ tokens.append(token)
+ if token == "expired":
+ raise PermissionError("401 Client Error: Unauthorized")
+ (tmp_path / "6-bit").mkdir(exist_ok = True)
+ (tmp_path / "6-bit" / "config.json").write_text("{}")
+ return str(tmp_path)
+
+ def _retry(read, hf_token):
+ try:
+ return read(hf_token)
+ except PermissionError:
+ return read(False)
+
+ monkeypatch.setattr("utils.utils.hf_cache_snapshot_dir_for_repo", lambda repo: None)
+ monkeypatch.setattr("huggingface_hub.snapshot_download", _snapshot_download)
+ monkeypatch.setattr("hub.utils.hf_tokens.call_with_anonymous_retry", _retry)
+ assert mlx_variants.download_precision_folder(_REPO, "6-bit", "expired") == str(
+ tmp_path / "6-bit"
+ )
+ assert tokens == ["expired", False]
+
+
+def test_the_root_duplicate_is_found_by_content_not_size(monkeypatch):
+ sizes = {**_SIZES, **_folder("4bit-dwq", [5 * GB, 5 * GB, 6 * GB])}
+ _no_gguf_rows(monkeypatch)
+ _fake_hub(monkeypatch, sizes = sizes)
+ response = asyncio.run(gguf_variants.get_gguf_variants_answer(_REPO)).response
+ assert response.default_variant == "4-bit"
+ _fake_hub(monkeypatch, sizes = sizes, content = lambda name: name)
+ response = asyncio.run(gguf_variants.get_gguf_variants_answer(_REPO)).response
+ assert response.default_variant is None
+
+
+def test_a_resumed_folder_download_rebuilds_its_plan_from_the_manifest():
+ from hub.utils.gguf_plan import plan_from_expected_files, resolve_variant_plan
+
+ siblings = _siblings({"m-Q4_K_M.gguf": 5 * GB, **_SIZES})
+ folder = resolve_variant_plan(siblings, "6-bit")
+ resumed = plan_from_expected_files("6-bit", folder.expected_files)
+ assert resumed.main_hashes == folder.main_hashes != frozenset()
+ assert resumed.target_filenames == folder.target_filenames
+ gguf = resolve_variant_plan(siblings, "Q4_K_M")
+ assert plan_from_expected_files("Q4_K_M", gguf.expected_files) == gguf
+
+
+def test_memory_estimate_prices_the_picked_folder(monkeypatch, tmp_path):
+ from routes import inference as ri
+
+ folder = _cached_repo(monkeypatch, tmp_path, _REPO, "6-bit")
+ (folder / "config.json").write_text(json.dumps({"model_type": "qwen3"}))
+ (folder / "model.safetensors").write_bytes(b"x")
+ monkeypatch.setattr(ri, "_estimate_hf_cache_roots", lambda: [str(tmp_path / "hub")])
+ config = SimpleNamespace(
+ path = _REPO, identifier = _REPO, is_local = False, is_lora = False, is_gguf = False
+ )
+ assert ri._local_mlx_model_dir(SimpleNamespace(**vars(config), gguf_variant = None)) is None
+ assert ri._local_mlx_model_dir(SimpleNamespace(**vars(config), gguf_variant = "6-bit")) == str(
+ folder
+ )
+ (folder.parent / "config.json").write_text(json.dumps({"model_type": "qwen3"}))
+ (folder.parent / "model.safetensors").write_bytes(b"x")
+ assert ri._local_mlx_model_dir(SimpleNamespace(**vars(config), gguf_variant = "6-bit")) == str(
+ folder
+ )
+
+
[email protected]("resident_folder, expected", [(None, "reuse"), ("6-bit", "load")])
+def test_a_root_request_never_reuses_a_resident_precision_folder(
+ monkeypatch, resident_folder, expected
+):
+ from hub.services.models import account_access
+ from models.inference import LoadRequest
+ from routes import inference
+
+ calls = []
+
+ def _record(step):
+ def _stop(*args, **kwargs):
+ calls.append(step)
+ raise RuntimeError(step)
+
+ return _stop
+
+ backend = SimpleNamespace(
+ active_model_name = _REPO,
+ models = {
+ _REPO: {
+ "engine": "auto",
+ "gguf_variant": resident_folder,
+ "precision_folder": resident_folder,
+ }
+ },
+ get_loading_model = lambda: None,
+ loading_models = (),
+ set_parallel_slots = _record("reuse"),
+ load_model = _record("load"),
+ )
+ llama = SimpleNamespace(is_loaded = False, is_active = False, model_identifier = None)
+ monkeypatch.setattr(inference, "get_llama_cpp_backend", lambda: llama)
+ monkeypatch.setattr(inference, "get_inference_backend", lambda: backend)
+ monkeypatch.setattr(inference, "_peek_inference_backend", lambda: backend)
+ monkeypatch.setattr(
+ inference, "_resolve_model_identifier_for_request", lambda *a, **k: (_REPO, _REPO, False)
+ )
+ monkeypatch.setattr(account_access, "require_model_access", lambda *a, **k: None)
+ for name in (
+ "_mlx_runtime_settings_match",
+ "_non_gguf_runtime_settings_match",
+ "_resident_audio_placement_matches",
+ "_resident_context_satisfies",
+ ):
+ monkeypatch.setattr(inference, name, lambda *a, **k: True)
+ fastapi_request = SimpleNamespace(
+ app = SimpleNamespace(state = SimpleNamespace(llama_parallel_slots = 1))
+ )
+ with pytest.raises(Exception):
+ asyncio.run(
+ inference._load_model_impl(
+ LoadRequest(model_path = _REPO), fastapi_request, current_subject = "user"
+ )
+ )
+ assert calls[:1] == [expected]
+
+
+def test_a_caller_refused_the_cache_learns_nothing_from_it(monkeypatch, tmp_path):
+ folder = _cached_repo(monkeypatch, tmp_path, _REPO, "6-bit")
+ (folder / "config.json").write_text(json.dumps({"model_type": "qwen3"}))
+ (folder / "model.safetensors").write_bytes(b"x")
+ monkeypatch.setattr("utils.utils.hf_cache_snapshot_dir_for_repo", lambda repo: folder.parent)
+ _fake_hub(monkeypatch)
+ monkeypatch.setattr("hub.utils.mlx_variants.cached_precision_folder", lambda repo, name: folder)
+ _no_gguf_rows(monkeypatch)
+ rows = asyncio.run(gguf_variants.get_gguf_variants_answer(_REPO)).response.variants
+ assert [v.quant for v in rows if v.downloaded] == ["2-bit", "4-bit", "6-bit", "8-bit"]
+ _no_gguf_rows(monkeypatch, cache_authorized = False)
+ rows = asyncio.run(gguf_variants.get_gguf_variants_answer(_REPO)).response.variants
+ assert rows and not any(v.downloaded for v in rows)
+ answer = asyncio.run(gguf_variants.get_gguf_variants_answer(_REPO, offline = True))
+ assert not answer.response.variants
+
+
+def test_offline_lists_a_cached_precision_folder(monkeypatch, tmp_path):
+ from fastapi import HTTPException
+
+ folder = _cached_repo(monkeypatch, tmp_path, _REPO, "6-bit")
+ (folder / "config.json").write_text(json.dumps({"model_type": "qwen3"}))
+ (folder / "model.safetensors").write_bytes(b"x")
+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")
+ answer = asyncio.run(gguf_variants.get_gguf_variants_answer(_REPO, offline = True))
+ assert [(v.quant, v.downloaded) for v in answer.response.variants] == [("6-bit", True)]
+ assert answer.response.variant_format == "mlx"
+ with pytest.raises(HTTPException) as missing:
+ asyncio.run(gguf_variants.get_gguf_variants_answer("org/absent", offline = True))
+ assert missing.value.status_code == 404
diff --git a/studio/backend/tests/test_mlx_precision_variants_apple.py b/studio/backend/tests/test_mlx_precision_variants_apple.py
new file mode 100644
index 00000000000..eceda7e3119
--- /dev/null
+++ b/studio/backend/tests/test_mlx_precision_variants_apple.py
@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Real MLX load of one precision folder of a repo (#13418), on Apple Silicon only."""
+
+from __future__ import annotations
+
+import platform
+import shutil
+import sys
+
+import pytest
+
+pytestmark = [
+ pytest.mark.skipif(
+ sys.platform != "darwin" or platform.machine() != "arm64",
+ reason = "MLX runs on Apple Silicon only",
+ ),
+ pytest.mark.allow_network,
+]
+
+_TINY = "mlx-community/SmolLM-135M-Instruct-4bit"
+_REPO = "example/SmolLM-135M-Instruct-MLX"
+
+
+def test_precision_folder_loads_and_generates(monkeypatch, tmp_path):
+ pytest.importorskip("mlx_lm")
+ pytest.importorskip("unsloth_zoo.mlx.loader")
+ from huggingface_hub import snapshot_download
+
+ from core.inference.mlx_inference import MLXInferenceBackend
+ from utils import hf_cache_settings
+ from utils.models.model_config import ModelConfig
+
+ tiny = snapshot_download(_TINY)
+ hub = tmp_path / "hub"
+ repo_dir = hub / f"models--{_REPO.replace('/', '--')}"
+ snapshot = repo_dir / "snapshots" / ("a" * 40)
+ shutil.copytree(tiny, snapshot / "4bit")
+ (repo_dir / "refs").mkdir()
+ (repo_dir / "refs" / "main").write_text("a" * 40)
+ monkeypatch.setattr(
+ hf_cache_settings,
+ "get_hf_cache_paths",
+ lambda: hf_cache_settings.HuggingFaceCachePaths(tmp_path, hub, tmp_path / "xet", "studio"),
+ )
+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")
+
+ config = ModelConfig.from_identifier(_REPO, gguf_variant = "4bit")
+ assert config.identifier == _REPO
+ assert config.gguf_variant == "4bit"
+ assert config.is_gguf is False
+
+ backend = MLXInferenceBackend()
+ assert backend.load_model(config, max_seq_length = 512)
+ assert backend.active_model_name == _REPO
+ assert backend.models[_REPO]["gguf_variant"] == "4bit"
+ assert str(getattr(backend._model, "_src_path", "")).endswith("4bit")
+
+ from mlx_lm import generate
+
+ text = generate(
+ backend._model, backend._tokenizer, prompt = "The capital of France is", max_tokens = 8
+ )
+ assert isinstance(text, str) and text.strip()
diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py
index 5e8a9be340b..82d51710f4d 100644
--- a/studio/backend/utils/models/model_config.py
+++ b/studio/backend/utils/models/model_config.py
@@ -3452,12 +3452,13 @@ _GGUF_REPO_NAME_RE = _re.compile(r"(?:^|[-_.])gguf(?:$|[-_.])", _re.IGNORECASE)
def _looks_like_gguf_repo(repo_id: str, gguf_variant: Optional[str] = None) -> bool:
- # A cached Transformers, diffusers or adapter checkpoint can still load from that cache.
- if any(
- (snap / name).is_file()
- for snap in _iter_hf_cache_snapshots(repo_id)
- for name in ("config.json", "model_index.json", "adapter_config.json")
- ):
+ from hub.utils.mlx_variants import is_precision_folder
+
+ # A cached Transformers, diffusers, adapter or MLX precision-folder checkpoint can still load.
+ names = ["config.json", "model_index.json", "adapter_config.json"]
+ if is_precision_folder(gguf_variant):
+ names.append(f"{gguf_variant.strip()}/config.json")
+ if any((snap / name).is_file() for snap in _iter_hf_cache_snapshots(repo_id) for name in names):
return False
return bool(gguf_variant) or bool(
_GGUF_REPO_NAME_RE.search(repo_id.rstrip("/").rsplit("/", 1)[-1])
@@ -4944,6 +4945,14 @@ class ModelConfig:
else:
check_model = identifier
+ precision_folder = None
+ if gguf_variant and not is_local and not is_lora:
+ from hub.utils.mlx_variants import precision_folder_config_dir
+ folder_config = precision_folder_config_dir(identifier, gguf_variant.strip(), hf_token)
+ if folder_config is not None:
+ precision_folder = gguf_variant.strip()
+ check_model = str(folder_config)
+
with _offline_while_reading(check_model):
vision = is_vision_model(check_model, hf_token = hf_token)
audio_type_val = detect_audio_type(check_model, hf_token = hf_token)
@@ -4952,6 +4961,7 @@ class ModelConfig:
display_name = Path(path).name if is_local else identifier.split("/")[-1]
return cls(
+ gguf_variant = precision_folder,
identifier = identifier,
display_name = display_name,
path = path,
diff --git a/studio/frontend/src/features/api-monitor/reload-last-model.ts b/studio/frontend/src/features/api-monitor/reload-last-model.ts
index abf5126f753..da8652a665c 100644
--- a/studio/frontend/src/features/api-monitor/reload-last-model.ts
+++ b/studio/frontend/src/features/api-monitor/reload-last-model.ts
@@ -32,9 +32,8 @@ export function resolveReloadTarget<C>(
if (!(lastLoad && id)) {
return null;
}
- // A null variant on a GGUF record is a direct .gguf file path, which the loader accepts as is.
- const ggufVariant =
- lastLoad.kind === "gguf" ? lastLoad.ggufVariant?.trim() || null : null;
+ // A GGUF record's null variant is a direct .gguf path; a model record's is an MLX precision folder.
+ const ggufVariant = lastLoad.ggufVariant?.trim() || null;
return {
id,
kind: lastLoad.kind,
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index 5cd360bdd7c..b0eb4fa2d1f 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -3077,8 +3077,15 @@ async function resolveAutoLoadCandidate(
: `Loaded ${source.id}`,
});
if (!source.listVariants) {
- const candidate = build(null);
- return isSkipped(candidate) ? null : candidate;
+ const variants =
+ source.kind === "model" && rememberedVariant
+ ? [rememberedVariant, null]
+ : [null];
+ for (const variant of variants) {
+ const candidate = build(variant);
+ if (!isSkipped(candidate)) return candidate;
+ }
+ return null;
}
const downloaded = (await source.listVariants())
.filter(
diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx
index 8ece87a0913..e34f51b4e5e 100644
--- a/studio/frontend/src/features/chat/chat-page.tsx
+++ b/studio/frontend/src/features/chat/chat-page.tsx
@@ -2796,6 +2796,7 @@ export function ChatPage({
const activeGgufVariant = useChatRuntimeStore(
(state) => state.activeGgufVariant,
);
+ const loadedIsMlx = useChatRuntimeStore((state) => state.loadedIsMlx);
const residentCheckpoint = useChatRuntimeStore(
(state) => state.residentCheckpoint,
);
@@ -4713,6 +4714,7 @@ export function ChatPage({
residentCheckpoint,
})}
activeGgufVariant={activeGgufVariant}
+ activeVariantIsMlx={loadedIsMlx === true}
activeModelConfig={activeModelConfig}
activeLoadedContextLength={loadedContextLength}
configRequest={modelConfigRequest}
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index d13db614292..c1048de6dbd 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -3120,7 +3120,9 @@ export function useChatModelRuntime() {
recordLastLocalModelLoad({
id: modelId,
kind:
- loadResponse.is_gguf || isGguf || ggufVariant
+ loadResponse.is_gguf ||
+ isGguf ||
+ (ggufVariant && loadResponse.is_mlx !== true)
? "gguf"
: "model",
ggufVariant: ggufVariant ?? null,
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 6ac40557d95..548cc0e1efb 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -207,6 +207,8 @@ export interface GgufVariantsResponse {
dependencies_resolved?: boolean;
/** Native max context from GGUF metadata; present once a variant is downloaded. */
context_length?: number | null;
+ /** "mlx" for precision folders of an MLX repo. */
+ variant_format?: string | null;
}
export function isMultimodalResponse(
diff --git a/studio/frontend/src/features/hub/catalog/download-section.tsx b/studio/frontend/src/features/hub/catalog/download-section.tsx
index 200d90fd80a..72e3c52a8b7 100644
--- a/studio/frontend/src/features/hub/catalog/download-section.tsx
+++ b/studio/frontend/src/features/hub/catalog/download-section.tsx
@@ -6,6 +6,7 @@ import type { MediaStudioPage } from "../hooks/use-media-companion-bytes";
import type { ModelInventoryFormat } from "../inventory";
import type { HubModelRunSelection } from "../lib/model-run-selection";
import { GgufDownloadCard } from "./gguf-download-card";
+import { MlxPrecisionCard } from "./mlx-precision-card";
import { SafetensorsDownloadCard } from "./safetensors-download-card";
export function DownloadSection({
@@ -34,6 +35,8 @@ export function DownloadSection({
showMemoryBar = true,
mediaPage,
assetRuntime,
+ isMlx = false,
+ onRunPrecision,
}: {
repoId: string;
isGguf: boolean;
@@ -62,6 +65,8 @@ export function DownloadSection({
showMemoryBar?: boolean;
mediaPage?: MediaStudioPage;
assetRuntime?: AssetRuntime;
+ isMlx?: boolean;
+ onRunPrecision?: (selection: HubModelRunSelection) => void;
}) {
if (isGguf || preferredGgufFile) {
return (
@@ -85,7 +90,7 @@ export function DownloadSection({
/>
);
}
- return (
+ const repoCard = (
<SafetensorsDownloadCard
assetRuntime={assetRuntime}
repoId={repoId}
@@ -104,4 +109,15 @@ export function DownloadSection({
onChange={onChange}
/>
);
+ if (!isMlx) return repoCard;
+ return (
+ <div className="flex w-full flex-col gap-2">
+ <MlxPrecisionCard
+ repoId={repoId}
+ onRun={onRunPrecision}
+ runPending={runPending}
+ />
+ {repoCard}
+ </div>
+ );
}
diff --git a/studio/frontend/src/features/hub/catalog/mlx-precision-card.tsx b/studio/frontend/src/features/hub/catalog/mlx-precision-card.tsx
new file mode 100644
index 00000000000..b38bb8dd25c
--- /dev/null
+++ b/studio/frontend/src/features/hub/catalog/mlx-precision-card.tsx
@@ -0,0 +1,106 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { useEffect, useState } from "react";
+import { cn } from "@/lib/utils";
+import { type GgufVariantsResponse, listGgufVariants } from "../inventory/api";
+import type { HubModelRunSelection } from "../lib/model-run-selection";
+import { formatBytes } from "../lib/format";
+import { useHfTokenStore } from "../stores/hf-token-store";
+import { DotTag } from "./dot-tag";
+import { CardDivider, ModelRunActionButton } from "./download-card";
+
+export function MlxPrecisionCard({
+ repoId,
+ onRun,
+ runPending = false,
+}: {
+ repoId: string;
+ onRun?: (selection: HubModelRunSelection) => void;
+ runPending?: boolean;
+}) {
+ const hfToken = useHfTokenStore((s) => s.token);
+ const [listing, setListing] = useState<GgufVariantsResponse | null>(null);
+ const [picked, setPicked] = useState<string | null>(null);
+
+ useEffect(() => {
+ const controller = new AbortController();
+ setListing(null);
+ setPicked(null);
+ listGgufVariants(repoId, hfToken || undefined, {
+ signal: controller.signal,
+ })
+ .then((res) => {
+ if (controller.signal.aborted) return;
+ if (res.variant_format === "mlx" && res.variants.length > 0) {
+ setListing(res);
+ setPicked(res.default_variant ?? res.variants[0].quant);
+ }
+ })
+ .catch(() => {
+ });
+ return () => controller.abort();
+ }, [repoId, hfToken]);
+
+ if (!listing) return null;
+ const chosen =
+ listing.variants.find((v) => v.quant === picked) ?? listing.variants[0];
+
+ return (
+ <div className="hub-download-card" data-mlx-precision-card={true}>
+ <div className="px-3 pt-2 text-ui-12 text-muted-foreground">
+ Precision
+ </div>
+ <div
+ role="radiogroup"
+ aria-label="Precision"
+ className="flex flex-col px-1.5 pb-1.5 pt-1"
+ >
+ {listing.variants.map((v) => (
+ <button
+ key={v.quant}
+ type="button"
+ role="radio"
+ aria-checked={v.quant === chosen.quant}
+ onClick={() => setPicked(v.quant)}
+ className={cn(
+ "flex h-8 items-center gap-2 rounded-md px-1.5 text-left text-ui-12",
+ v.quant === chosen.quant
+ ? "bg-accent text-foreground"
+ : "text-muted-foreground hover:bg-accent/60",
+ )}
+ >
+ <span className="font-medium text-foreground">{v.quant}</span>
+ {listing.default_variant === v.quant && (
+ <span>same as the repo root</span>
+ )}
+ <span className="ml-auto flex items-center gap-2">
+ {v.downloaded && <DotTag tone="success" label="On device" />}
+ <span className="tabular-nums">{formatBytes(v.size_bytes)}</span>
+ </span>
+ </button>
+ ))}
+ </div>
+ {onRun && (
+ <>
+ <CardDivider />
+ <div className="flex items-center justify-end">
+ <ModelRunActionButton
+ label={`Configure and run ${repoId} ${chosen.quant}`}
+ loading={runPending}
+ onClick={() =>
+ onRun({
+ ggufVariant: chosen.quant,
+ ggufFilename: chosen.filename,
+ expectedBytes: chosen.size_bytes,
+ precisionFolder: true,
+ downloaded: chosen.downloaded === true,
+ })
+ }
+ />
+ </div>
+ </>
+ )}
+ </div>
+ );
+}
diff --git a/studio/frontend/src/features/hub/catalog/model-inspector.tsx b/studio/frontend/src/features/hub/catalog/model-inspector.tsx
index 3c983712f6c..de07b86a626 100644
--- a/studio/frontend/src/features/hub/catalog/model-inspector.tsx
+++ b/studio/frontend/src/features/hub/catalog/model-inspector.tsx
@@ -49,6 +49,7 @@ import { useDatasetSize } from "../hooks/use-dataset-size";
import {
type HubModelRunSelection,
hubModelRunsOnAudioPage,
+ isEmbeddingOnly,
isHubModelRunEligible,
} from "../lib/model-run-selection";
import { studioPageForTask } from "../lib/unsloth-support";
@@ -603,6 +604,21 @@ export const ModelInspector = memo(function ModelInspector({
? (selection: HubModelRunSelection) =>
onRun(selection, audioPage ? "audio" : mediaPage)
: undefined;
+ // A precision folder downloads as it loads, so it need not be on device.
+ const isMlx =
+ !model.isGguf &&
+ (model.libraryName?.toLowerCase() === "mlx" ||
+ Boolean(model.tags?.some((tag) => tag.toLowerCase() === "mlx")));
+ const precisionRunAction =
+ isMlx &&
+ onRun &&
+ !isDataset &&
+ !isEmbeddingOnly(model) &&
+ !runsOnMediaRuntime &&
+ !chatOnlyMeasured &&
+ unslothSupport.status !== "unsupported"
+ ? (selection: HubModelRunSelection) => onRun(selection, mediaPage)
+ : undefined;
const languages = parseLanguageTags(model.tags);
const datasetSizeBytes =
@@ -758,6 +774,8 @@ export const ModelInspector = memo(function ModelInspector({
onRun={runAction}
runPending={runConfigPending}
onChange={onInventoryChange}
+ isMlx={isMlx}
+ onRunPrecision={precisionRunAction}
/>
)}
</InspectorDownloadSlot>
diff --git a/studio/frontend/src/features/hub/hub-page.tsx b/studio/frontend/src/features/hub/hub-page.tsx
index ba6fd647f31..3f7f9f9986c 100644
--- a/studio/frontend/src/features/hub/hub-page.tsx
+++ b/studio/frontend/src/features/hub/hub-page.tsx
@@ -395,6 +395,7 @@ function selectedRepoMatchesRuntime(
selectedModel: SelectedModelView | null,
runtimeId: string | null,
ggufVariant: string | null,
+ runtimeIsMlx = false,
): boolean {
if (!selectedModel || !runtimeId) return false;
if (
@@ -413,7 +414,8 @@ function selectedRepoMatchesRuntime(
const localPath = selectedModel.path ?? "";
return ggufVariant !== null || localPath.toLowerCase().endsWith(".gguf");
}
- return ggufVariant === null;
+ // An MLX load of a precision folder (6-bit/) reports the folder as its variant.
+ return ggufVariant === null || runtimeIsMlx;
}
export function ModelsPage() {
@@ -463,6 +465,7 @@ export function ModelsPage() {
? checkpoint
: null;
const activeGgufVariant = useChatRuntimeStore((s) => s.activeGgufVariant);
+ const loadedIsMlx = useChatRuntimeStore((s) => s.loadedIsMlx);
const loadingModel = useChatRuntimeStore((s) => s.loadingModelPick);
const [initialResidentStatusSettled, setInitialResidentStatusSettled] =
useState(false);
@@ -1502,8 +1505,9 @@ export function ModelsPage() {
selectedModel,
activeCheckpoint,
activeGgufVariant,
+ loadedIsMlx === true,
),
- [activeCheckpoint, activeGgufVariant, selectedModel],
+ [activeCheckpoint, activeGgufVariant, loadedIsMlx, selectedModel],
);
const isLoadingThisModel = useMemo(() => {
diff --git a/studio/frontend/src/features/hub/inventory/api.ts b/studio/frontend/src/features/hub/inventory/api.ts
index e048dc7aaef..54c6939bb7d 100644
--- a/studio/frontend/src/features/hub/inventory/api.ts
+++ b/studio/frontend/src/features/hub/inventory/api.ts
@@ -229,6 +229,8 @@ export interface GgufVariantsResponse {
default_variant: string | null;
/** True when Hub metadata or a complete cached download plan proves companion readiness. */
dependencies_resolved?: boolean;
+ /** "mlx" for precision folders of an MLX repo. */
+ variant_format?: string | null;
}
async function parseJsonOrThrow<T>(
diff --git a/studio/frontend/src/features/hub/lib/dataset-size.ts b/studio/frontend/src/features/hub/lib/dataset-size.ts
index 8e2c6b550da..76bdbff7e36 100644
--- a/studio/frontend/src/features/hub/lib/dataset-size.ts
+++ b/studio/frontend/src/features/hub/lib/dataset-size.ts
@@ -327,6 +327,33 @@ function redundantTorchBinFiles(names: string[]): Set<string> {
return out;
}
+// Mirrors PRECISION_FOLDER_SOURCE and precision_variant_subfolders in snapshot_filters.py.
+const PRECISION_FOLDER_SOURCE =
+ "(?:mlx[-_]?)?(?:[0-9]{1,2}[-_]?bits?(?:[-_][a-z0-9]+)*|bf16|fp16|f16|fp32|f32|fp8|mxfp4|nvfp4)";
+const PRECISION_FOLDER_RE = new RegExp(`^(?:${PRECISION_FOLDER_SOURCE})$`, "i");
+
+function precisionVariantFolders(names: string[]): Set<string> {
+ const files = new Map<string, string[]>();
+ for (const name of names) {
+ const slash = name.indexOf("/");
+ const folder = name.slice(0, slash);
+ const rest = name.slice(slash + 1);
+ if (slash <= 0 || !rest || rest.includes("/")) continue;
+ if (!PRECISION_FOLDER_RE.test(folder)) continue;
+ files.set(folder, [...(files.get(folder) ?? []), rest]);
+ }
+ const out = new Set<string>();
+ for (const [folder, rest] of files) {
+ if (
+ rest.includes("config.json") &&
+ rest.some((n) => n.toLowerCase().endsWith(".safetensors"))
+ ) {
+ out.add(folder);
+ }
+ }
+ return out;
+}
+
function basename(path: string): string {
return path.split("/").pop() ?? path;
}
@@ -359,8 +386,10 @@ function isSnapshotIgnored(
skipConsolidated: boolean,
skipDuplicateFormats: boolean,
redundantBins: Set<string>,
+ precisionFolders: Set<string> = new Set(),
): boolean {
const lower = filename.toLowerCase();
+ const folder = filename.includes("/") ? filename.slice(0, filename.indexOf("/")) : "";
return (
lower.endsWith(".gguf") ||
lower.endsWith(".onnx") ||
@@ -370,7 +399,8 @@ function isSnapshotIgnored(
lower.endsWith(".bin.index.json.bak") ||
(skipConsolidated && lower.startsWith("consolidated")) ||
(skipDuplicateFormats &&
- (DUPLICATE_WEIGHT_FORMAT_RE.test(filename) || redundantBins.has(filename)))
+ (DUPLICATE_WEIGHT_FORMAT_RE.test(filename) || redundantBins.has(filename) ||
+ precisionFolders.has(folder)))
);
}
@@ -405,9 +435,11 @@ export function fetchModelSize(
const siblings = data.siblings ?? [];
const skipConsolidated = shipsTransformersWeights(siblings);
const skipDuplicateFormats = shipsRootSafetensors(siblings);
- const redundantBins = redundantTorchBinFiles(
- siblings.map((s) => s.rfilename ?? ""),
- );
+ const names = siblings.map((s) => s.rfilename ?? "");
+ const redundantBins = redundantTorchBinFiles(names);
+ const precisionFolders = names.includes("config.json")
+ ? precisionVariantFolders(names)
+ : new Set<string>();
let total = 0;
let weights = 0;
for (const s of siblings) {
@@ -420,6 +452,7 @@ export function fetchModelSize(
skipConsolidated,
skipDuplicateFormats,
redundantBins,
+ precisionFolders,
)
) {
continue;
diff --git a/studio/frontend/src/features/hub/lib/model-run-selection.ts b/studio/frontend/src/features/hub/lib/model-run-selection.ts
index 0e0f6779c62..dfe0b6ae479 100644
--- a/studio/frontend/src/features/hub/lib/model-run-selection.ts
+++ b/studio/frontend/src/features/hub/lib/model-run-selection.ts
@@ -20,6 +20,9 @@ export interface HubModelRunSelection {
ggufVariant?: string;
ggufFilename?: string;
expectedBytes?: number;
+ /** ggufVariant is an MLX precision folder, fetched by the load itself. */
+ precisionFolder?: boolean;
+ downloaded?: boolean;
}
function isPresent(value: string | null | undefined): value is string {
@@ -48,7 +51,7 @@ const GENERATIVE_CAPABILITIES = new Set([
"diffusion",
]);
-function isEmbeddingOnly(model: SelectedModelView): boolean {
+export function isEmbeddingOnly(model: SelectedModelView): boolean {
if (
model.isGguf ||
!model.capabilities.some((capability) => capability.key === "embedding")
@@ -117,6 +120,40 @@ function createHandoffMeta(
return meta;
}
+function createPrecisionFolderHandoff(
+ requestId: string,
+ model: SelectedModelView,
+ selection: HubModelRunSelection,
+): ModelConfigHandoffRequest | null {
+ const id = model.hubRepoId ?? model.id;
+ if (
+ !isPresent(id) ||
+ !isPresent(selection.ggufVariant) ||
+ isEmbeddingOnly(model)
+ ) {
+ return null;
+ }
+ const meta: ModelConfigHandoffRequest["meta"] = {
+ source: "hub",
+ isLora: false,
+ isDownloaded: selection.downloaded === true,
+ isGguf: false,
+ ggufVariant: selection.ggufVariant,
+ pipelineTag: model.task ?? model.pipelineTag ?? null,
+ };
+ if (isPresent(selection.ggufFilename)) {
+ meta.ggufFilename = selection.ggufFilename;
+ }
+ if (
+ selection.expectedBytes != null &&
+ Number.isFinite(selection.expectedBytes) &&
+ selection.expectedBytes > 0
+ ) {
+ meta.expectedBytes = selection.expectedBytes;
+ }
+ return { requestId, id, displayName: model.title, meta };
+}
+
/** The task the Audio handoff runs on: offline, a cached curated audio GGUF carries only the
* backend's generic text-generation task, which the picker maps back from the catalog too. */
export function hubAudioTask(
@@ -204,6 +241,9 @@ export function createHubModelConfigHandoff({
if (!isPresent(requestId)) {
return null;
}
+ if (selection.precisionFolder) {
+ return createPrecisionFolderHandoff(requestId, model, selection);
+ }
if (!hasRunnableChatModel(model)) {
return null;
}
diff --git a/studio/frontend/src/features/model-picker/components/model-selector.tsx b/studio/frontend/src/features/model-picker/components/model-selector.tsx
index 43573399eab..f9a4107cd91 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-selector.tsx
@@ -110,6 +110,7 @@ interface ModelSelectorProps {
* every caller assumed until an image or video load started evicting the chat model. */
loaded?: boolean;
activeGgufVariant?: string | null;
+ activeVariantIsMlx?: boolean;
activeModelConfig?: PerModelConfig | null;
activeLoadedContextLength?: number | null;
selectedConfig?: PerModelConfig | null;
@@ -733,6 +734,7 @@ export function ModelSelector({
value,
defaultValue,
activeGgufVariant,
+ activeVariantIsMlx = false,
activeModelConfig,
activeLoadedContextLength,
selectedConfig,
@@ -855,7 +857,9 @@ export function ModelSelector({
const fallbackName = missingExternal?.modelName ?? modelDisplayName(selected);
if (triggerGgufVariant) {
// The variant is the quant, so it goes in the suffix.
- const desc = `GGUF · ${triggerGgufVariant}`;
+ const isMlx =
+ activeVariantIsMlx && loadedGgufVariantOverride === undefined;
+ const desc = `${isMlx ? "MLX" : "GGUF"} · ${triggerGgufVariant}`;
return found
? { ...found, description: undefined, descriptionSuffix: desc }
: { id: selected, name: fallbackName, descriptionSuffix: desc };
@@ -881,6 +885,8 @@ export function ModelSelector({
optionById,
catalog,
triggerGgufVariant,
+ activeVariantIsMlx,
+ loadedGgufVariantOverride,
externalModels,
externalConnections,
t,
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
index 641e9895aef..d2bc85f2a01 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx
@@ -1925,13 +1925,14 @@ function GgufVariantExpander({
}) {
const pinnedKeys = usePinnedModelsStore((s) => s.pinned);
const togglePinnedQuant = usePinnedModelsStore((s) => s.togglePinned);
- const onUpdateVariant = variantActions?.onUpdate;
+ const [variantsAreGguf, setVariantsAreGguf] = useState(true);
+ const onUpdateVariant = variantsAreGguf ? variantActions?.onUpdate : undefined;
const updateVariantTitle =
variantActions?.updateTitle ?? "Update cached model?";
const renderUpdateVariantDescription =
variantActions?.renderUpdateDescription;
const updateDisabled = variantActions?.updateDisabled ?? false;
- const onDeleteVariant = variantActions?.onDelete;
+ const onDeleteVariant = variantsAreGguf ? variantActions?.onDelete : undefined;
const deleteVariantTitle =
variantActions?.deleteTitle ?? "Delete cached model?";
const renderDeleteVariantDescription =
@@ -1989,6 +1990,7 @@ function GgufVariantExpander({
}
setNativeContext(normalized.contextLength);
setResolvedLocally(normalized.resolvedLocally);
+ setVariantsAreGguf(res?.variant_format !== "mlx");
};
loadPickerGgufVariants(
@@ -2070,7 +2072,7 @@ function GgufVariantExpander({
expectedBytes: sizeBytes,
downloadPresentation,
contextLength: isAvailable ? contextLength : undefined,
- isGguf: true,
+ isGguf: variantsAreGguf,
isVision: variantVisionHint,
pipelineTag,
});
@@ -2084,6 +2086,7 @@ function GgufVariantExpander({
nativeContext,
pipelineTag,
variantVisionHint,
+ variantsAreGguf,
],
);
@@ -2256,7 +2259,7 @@ function GgufVariantExpander({
setCompanionBytesByKey(new Map());
// A local path is resolved too: only the CHECKPOINT is on disk, and its remote base is the
// larger half, so suppressing the request understated a local row by many gigabytes.
- if (!resolveDownloadFootprint) {
+ if (!resolveDownloadFootprint || !variantsAreGguf) {
return () => {
cancelled = true;
};
@@ -2310,6 +2313,7 @@ function GgufVariantExpander({
repoId,
resolveDownloadFootprint,
sourceOverride,
+ variantsAreGguf,
]);
const variantOptionKeys = useMemo(
@@ -2542,7 +2546,7 @@ function GgufVariantExpander({
isDownloaded: true,
expectedBytes,
contextLength: variantContext,
- isGguf: true,
+ isGguf: variantsAreGguf,
isVision: variantVisionHint,
})
}
@@ -3285,6 +3289,32 @@ export function HubModelPicker({
[modelGgufIds, resultGgufIds],
);
+ const resultMlxIds = useMemo(() => {
+ const ids = new Set<string>();
+ for (const result of [
+ ...results,
+ ...recommendedSearch.results,
+ ...communityQuerySearch.results,
+ ...communityBrowse.results,
+ ]) {
+ if (
+ result.libraryName?.toLowerCase() === "mlx" ||
+ result.tags?.some((tag) => tag.toLowerCase() === "mlx")
+ ) {
+ ids.add(result.id.toLowerCase());
+ }
+ }
+ return ids;
+ }, [
+ results,
+ recommendedSearch.results,
+ communityQuerySearch.results,
+ communityBrowse.results,
+ ]);
+ const [mlxVariantRepoIds, setMlxVariantRepoIds] = useState<
+ ReadonlySet<string>
+ >(() => new Set());
+
const [expandedGguf, setExpandedGguf] = useState<string | null>(null);
// GGUF vision support per repo, reported by the expander once it has read the metadata, so On
// Device rows can badge the name.
@@ -5958,12 +5988,15 @@ export function HubModelPicker({
scrollRef,
]);
- /** Handle clicking a model row: GGUF repos expand, others load directly. */
+ /** Handle clicking a model row: GGUF repos and MLX precision-folder repos expand, others load. */
const handleModelClick = useCallback(
(id: string) => {
- if (isKnownGgufRepo(id)) {
+ const key = id.toLowerCase();
+ if (isKnownGgufRepo(id) || mlxVariantRepoIds.has(key)) {
setExpandedGguf((prev) => (prev === id ? null : id));
- } else {
+ return;
+ }
+ const select = () => {
// Cached repos load now (a cached vendor copy stands in for its mirror); others download.
const cached = cachedIdFor(id);
onSelect(cached ?? id, {
@@ -5972,9 +6005,31 @@ export function HubModelPicker({
isDownloaded: cached !== null,
pipelineTag: pipelineTagById.get(id) ?? null,
});
+ };
+ if (!resultMlxIds.has(key) || isHuggingFaceOffline()) {
+ select();
+ return;
}
+ void listGgufVariants(id, hfToken || undefined)
+ .then((res) => {
+ if (res.variant_format === "mlx" && res.variants.length > 0) {
+ setMlxVariantRepoIds((prev) => new Set(prev).add(key));
+ setExpandedGguf(id);
+ } else {
+ select();
+ }
+ })
+ .catch(select);
},
- [onSelect, isKnownGgufRepo, cachedIdFor, pipelineTagById],
+ [
+ onSelect,
+ isKnownGgufRepo,
+ mlxVariantRepoIds,
+ resultMlxIds,
+ hfToken,
+ cachedIdFor,
+ pipelineTagById,
+ ],
);
// On Device owns the downloaded and custom-folder models; the Unsloth tab searches the HF
diff --git a/studio/frontend/tests/auto-load-mlx-precision.test.ts b/studio/frontend/tests/auto-load-mlx-precision.test.ts
new file mode 100644
index 00000000000..9ab1d5b1a06
--- /dev/null
+++ b/studio/frontend/tests/auto-load-mlx-precision.test.ts
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+import ts from "typescript";
+
+const adapterPath = fileURLToPath(
+ new URL("../src/features/chat/api/chat-adapter.ts", import.meta.url),
+);
+const source = readFileSync(adapterPath, "utf8");
+const start = source.indexOf("async function resolveAutoLoadCandidate(");
+assert.ok(start >= 0, "resolveAutoLoadCandidate is no longer defined in chat-adapter.ts");
+const declaration = source.slice(start, source.indexOf("\n}", start) + 2);
+
+type Candidate = { kind: string; ggufVariant: string | null };
+const resolve = new Function(
+ `${ts.transpileModule(declaration, {
+ compilerOptions: { target: ts.ScriptTarget.ES2020 },
+ }).outputText}; return resolveAutoLoadCandidate;`,
+)() as (
+ source: object,
+ remembered: string | null,
+ isSkipped: (candidate: Candidate) => boolean,
+) => Promise<Candidate | null>;
+
+const mlxRepo = {
+ id: "org/Model-MLX",
+ loadId: "org/Model-MLX",
+ kind: "model",
+ maxSeqLength: 4096,
+ listVariants: null,
+};
+
+test("startup restores a remembered MLX precision folder", async () => {
+ const candidate = await resolve(mlxRepo, "6-bit", () => false);
+ assert.equal(candidate?.ggufVariant, "6-bit");
+ assert.equal(candidate?.kind, "model");
+});
+
+test("a failed precision falls back to the repo root, then stops", async () => {
+ const tried = new Set(["6-bit"]);
+ const skip = (candidate: Candidate) => tried.has(String(candidate.ggufVariant));
+ assert.equal((await resolve(mlxRepo, "6-bit", skip))?.ggufVariant, null);
+ tried.add("null");
+ assert.equal(await resolve(mlxRepo, "6-bit", skip), null);
+});
+
+test("a model without a remembered precision loads its root", async () => {
+ assert.equal((await resolve(mlxRepo, null, () => false))?.ggufVariant, null);
+});
diff --git a/studio/frontend/tests/hub-model-run-selection.test.ts b/studio/frontend/tests/hub-model-run-selection.test.ts
index c8b9fe47775..d04b4ed6659 100644
--- a/studio/frontend/tests/hub-model-run-selection.test.ts
+++ b/studio/frontend/tests/hub-model-run-selection.test.ts
@@ -479,3 +479,38 @@ test("invalid format and variant combinations fail closed", () => {
null,
);
});
+
+test("an MLX precision folder runs before the repo is on device, through MLX", () => {
+ const notDownloaded = selectedModel({
+ id: "orcarouter/Qwen3.8-27B-Uncensored-MLX",
+ hubRepoId: "orcarouter/Qwen3.8-27B-Uncensored-MLX",
+ loadId: null,
+ isDownloaded: false,
+ runtimeCanChat: false,
+ });
+ const request = createHubModelConfigHandoff({
+ requestId: "request-1",
+ model: notDownloaded,
+ selection: {
+ ggufVariant: "6-bit",
+ ggufFilename: "6-bit/",
+ expectedBytes: 22_800_000_000,
+ precisionFolder: true,
+ downloaded: false,
+ },
+ });
+ assert.equal(request?.id, "orcarouter/Qwen3.8-27B-Uncensored-MLX");
+ assert.equal(request?.meta.ggufVariant, "6-bit");
+ assert.equal(request?.meta.isGguf, false);
+ assert.equal(request?.meta.isDownloaded, false);
+ assert.equal(request?.meta.expectedBytes, 22_800_000_000);
+
+ assert.equal(
+ createHubModelConfigHandoff({
+ requestId: "request-2",
+ model: notDownloaded,
+ selection: { ggufVariant: "6-bit" },
+ }),
+ null,
+ );
+});
diff --git a/studio/frontend/tests/model-selector-trigger-label.test.ts b/studio/frontend/tests/model-selector-trigger-label.test.ts
index c2bd7f5e8a1..e300275dc63 100644
--- a/studio/frontend/tests/model-selector-trigger-label.test.ts
+++ b/studio/frontend/tests/model-selector-trigger-label.test.ts
@@ -118,5 +118,9 @@ test("an audio page's loaded quant names the trigger only for the model it belon
selector,
/const triggerGgufVariant =\s*loadedGgufVariantOverride !== undefined\s*\? selected &&\s*loadedModelIdOverride &&\s*selected\.toLowerCase\(\) === loadedModelIdOverride\.toLowerCase\(\)\s*\? loadedGgufVariantOverride\s*: null\s*: activeGgufVariant;/,
);
- assert.match(selector, /const desc = `GGUF · \$\{triggerGgufVariant\}`;/);
+ // An MLX precision label belongs to the chat's own load, never to a task-owned quant.
+ assert.match(
+ selector,
+ /const isMlx =\s*activeVariantIsMlx && loadedGgufVariantOverride === undefined;\s*const desc = `\$\{isMlx \? "MLX" : "GGUF"\} · \$\{triggerGgufVariant\}`;/,
+ );
});