Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: faster MLX load and reload, ready to serve when loaded (#16536)

Long Yixing81dc73ac43b3
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 84d61793145..48d442eedfe 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -6515,6 +6515,8 @@ class MLXInferenceBackend:
_mlx_warm_inference(
model, self._int8_prefill["active"], self._fast_verify_decision()
)
+ with mlx_rng_preserved():
+ self._warm_resident_batch()
if self._ane_prefill["active"]:
self._ane_prefill = _ane_prefill_status(True, model)
self.models[model_name].update(
@@ -6524,6 +6526,87 @@ class MLXInferenceBackend:
logger.info("Model %s loaded successfully", model_name)
return True
+ def narrow_context(self, max_seq_length):
+ """Serve a smaller context from the held weights; None where a fresh load could decide differently."""
+ entry = self.models.get(self.active_model_name)
+ pinned = _positive_int(max_seq_length)
+ if entry is None or pinned is None or not self._served_context:
+ return None
+ offload = entry.get("mlx_offload") or {}
+ if (
+ any(part.get("applied") for part in offload.values() if isinstance(part, dict))
+ or self._kv_quant["requested_kv_bits"] is not None
+ or self._turboquant
+ or self._prompt_cache_room is not None
+ or entry.get("spec_fallback_reason")
+ ):
+ return None
+ try:
+ if not self._resume_files or self._resume_files != self._resume_weight_files():
+ return None
+ except OSError:
+ return None
+ # The served length reads this attribute before the argument.
+ previous = self._model.max_seq_length
+ self._model.max_seq_length = pinned
+ served = self._resolve_context_lengths(self._model, pinned)[0]
+ if not served or served > self._served_context:
+ self._model.max_seq_length = previous
+ return None
+ self._clear_prompt_cache()
+ self._served_context = served
+ self._kv_quant, self._kv_cache_window, enforced, self._kv_context_budget = (
+ self._resolve_kv_policy(self._is_vlm, None, pinned, served)
+ )
+ fields = {
+ "context_length": served,
+ "requested_context_length": pinned,
+ "context_length_enforced": enforced,
+ "context_length_fitted": None,
+ "mlx_context_budget": self._kv_context_budget,
+ }
+ entry.update(fields)
+ return fields
+
+ def _warm_resident_batch(self):
+ """Warm the request batch Zoo's warm-up misses: twice greedy (Qwen4 scopes build kernels only after a reply), then sampled."""
+ if self.resident_unavailable_reason({}) is not None:
+ return
+ messages = [{"role": "user", "content": "Count from one to ten."}]
+ # Cold replies would teach the drafter's controller that drafting does not pay.
+ draft = self._speculative_draft
+ controller = getattr(draft, "controller", None)
+ if controller is not None:
+ draft.controller = copy.deepcopy(controller)
+ try:
+ for temperature in (0.0, 0.0, 0.7):
+ session = self.open_resident_batch(width = 1)
+ try:
+ session.admit(
+ {"messages": messages, "temperature": temperature, "max_new_tokens": 16},
+ "warm",
+ )
+ while session.rows_in_flight:
+ for _, snapshot in session.step():
+ if isinstance(snapshot, BaseException):
+ raise snapshot
+ finally:
+ session.close()
+ except Exception as error:
+ if is_metal_queue_dead(error):
+ raise
+ logger.warning(
+ "MLX batch warm-up failed, so the first request runs it: %s: %s",
+ type(error).__name__,
+ error,
+ )
+ finally:
+ if controller is not None:
+ draft.controller = controller
+ room = self._prompt_cache_room
+ self._clear_prompt_cache()
+ self._prompt_cache_room = room
+
def _render_template_probe(self, is_vision: bool) -> str:
"""Renders through the target generation uses, since the recovery renderer returns None off-list."""
from core.inference.chat_template_helpers import (
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index d31b2d582d1..4b0e38bf7d8 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -414,6 +414,7 @@ def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:
"is_vision": model_info.get("is_vision", False),
"is_lora": model_info.get("is_lora", False),
"is_mlx": model_info.get("is_mlx", False),
+ "loaded_auto_map": model_info.get("loaded_auto_map"),
"display_name": model_info.get("display_name", model_name),
"is_audio": model_info.get("is_audio", False),
"audio_type": model_info.get("audio_type"),
@@ -451,6 +452,15 @@ def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:
}
+def _load_settings(sub_config: dict) -> dict:
+ """What decides the weights and runtime a worker holds; the rest changes with each request."""
+ return {
+ key: value
+ for key, value in sub_config.items()
+ if key not in ("max_seq_length", "gpu_selection", "disable_xet")
+ }
+
+
class InferenceOrchestrator:
"""Inference backend orchestrator, subprocess-based. Same API surface as InferenceBackend (so
routes/inference.py needs minimal changes); all heavy ML work happens in a persistent
@@ -2257,8 +2267,9 @@ class InferenceOrchestrator:
engine_options = None,
n_parallel: Optional[int] = None,
) -> bool:
- """Load a model for inference. Always spawns a fresh subprocess per load for a clean
- interpreter (no stale unsloth patches, torch.compile caches, or getsource failures)."""
+ """Load a model for inference. Spawns a fresh subprocess per load for a clean interpreter
+ (no stale unsloth patches, torch.compile caches, or getsource failures); only a smaller
+ context on the MLX load already held is applied in place."""
if engine != "auto":
return self._load_managed_engine(
engine,
@@ -2378,7 +2389,15 @@ class InferenceOrchestrator:
"retry when it completes."
)
- # Always spawn fresh: unsloth patches torch internals, breaking getsource on reuse.
+ if self._narrow_context_in_place(
+ sub_config, model_name, parallel_slots, load_cancel_event
+ ):
+ self.loading_models.discard(model_name)
+ if on_prior_worker_released is not None:
+ on_prior_worker_released()
+ return True
+
+ # Otherwise spawn fresh: unsloth patches torch internals, breaking getsource on reuse.
had_worker_handle = self._proc is not None
worker_shutdown_at = 0.0
before_kill_free_mib = None
@@ -2417,7 +2436,10 @@ class InferenceOrchestrator:
required_gb = gpu_selection.get("required_gb")
if required_gb is not None:
expected_free_gb = {int(resolved_gpu_ids[0]): float(required_gb)}
- settled = self._wait_for_worker_vram_settle(
+ # An MLX worker first reads memory seconds after its imports, past the old worker's reclaim.
+ settled = (
+ expected_free_gb is None and sub_config["device_backend"] == "mlx"
+ ) or self._wait_for_worker_vram_settle(
worker_shutdown_at or time.monotonic(),
expected_free_gb = expected_free_gb,
before_kill_free_mib = before_kill_free_mib,
@@ -2553,6 +2575,7 @@ class InferenceOrchestrator:
self.models[self.active_model_name] = _mirrored_model_entry(
model_info, model_name
)
+ self._worker_load_settings = (self._proc, _load_settings(sub_config))
self.models[self.active_model_name]["parallel_slots"] = parallel_slots
self.models[self.active_model_name]["can_batch"] = model_info.get(
"can_batch"
@@ -2619,6 +2642,77 @@ class InferenceOrchestrator:
finally:
self._release_load_downloads()
+ def _narrow_context_in_place(
+ self,
+ sub_config: dict,
+ model_name: str,
+ parallel_slots,
+ cancel_event = None,
+ ) -> bool:
+ """Whether the idle MLX worker holding this load took a smaller context in place."""
+ proc = self._proc
+ entry = self.models.get(model_name)
+ held = getattr(self, "_worker_load_settings", None)
+ if (
+ sub_config["device_backend"] != "mlx"
+ or sub_config["mlx_distributed"]
+ or entry is None
+ or self.active_model_name != model_name
+ or entry.get("parallel_slots") != parallel_slots
+ or not 0 < (sub_config["max_seq_length"] or 0) < (entry.get("context_length") or 0)
+ or held != (proc, _load_settings(sub_config))
+ or not proc.is_alive()
+ or self._replies_in_flight()
+ ):
+ return False
+ with self._dispatcher_lifecycle_lock:
+ if self._worker_reserved_for is not None:
+ return False
+ self._worker_reserved_for = "the context is being narrowed"
+ acquired = False
+ try:
+ # A running dispatcher would drop the reply.
+ if not self._wait_worker_idle():
+ return False
+ acquired = self._gen_lock.acquire(timeout = 5.0)
+ if not acquired or self._proc is not proc:
+ return False
+ with self._active_cancel_lock:
+ if self._active_cancel_events:
+ return False
+ with self._send_order_lock:
+ self._send_cmd(
+ {"type": "narrow_context", "max_seq_length": sub_config["max_seq_length"]}
+ )
+ fields = self._wait_response("context_narrowed", timeout = 60.0).get("fields")
+ if not fields:
+ return False
+ from utils.process_lifetime import is_process_shutting_down
+
+ with self._subprocess_shutdown_lock:
+ if (
+ is_process_shutting_down()
+ or self._proc is not proc
+ or model_name not in self.loading_models
+ or (cancel_event is not None and cancel_event.is_set())
+ ):
+ return False
+ entry.update(fields)
+ self.load_generation += 1
+ self._loaded_by_user_action = False
+ logger.info(
+ "Model '%s' now serves a context of %s", model_name, fields["context_length"]
+ )
+ return True
+ except Exception as exc:
+ logger.warning("Could not narrow the context in place: %s", exc)
+ return False
+ finally:
+ if acquired:
+ self._gen_lock.release()
+ with self._dispatcher_lifecycle_lock:
+ self._worker_reserved_for = None
+
def _claim_load_downloads(self, resp: dict) -> None:
from hub.services.load_downloads import claim_load_downloads
self._release_load_downloads()
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 243bc14d91a..3d4128827fd 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -12,6 +12,7 @@ mp.Queue, and exits on shutdown or unload. Pattern follows core/training/worker.
from __future__ import annotations
+import contextlib
import functools
import base64
import inspect
@@ -674,6 +675,24 @@ def _in_token_rejection_scope(handler):
return scoped
+def _loaded_auto_map(backend, mc) -> bool | None:
+ """Whether the opened MLX snapshot declares remote code; None when it is not the requested repo (adapter, precision folder, bnb substitute)."""
+ if getattr(backend, "device", None) != "mlx" or mc.is_lora or getattr(mc, "gguf_variant", None):
+ return None
+ try:
+ from core.inference.mlx_inference import _snapshot_dir
+ from core.inference.model_ids import mlx_host_bnb_base_repo
+ from utils.security.consent import _config_has_auto_map
+
+ if mlx_host_bnb_base_repo(mc.identifier):
+ return None
+ snapshot = _snapshot_dir(getattr(backend, "_model", None), mc.identifier)
+ return _config_has_auto_map(snapshot) if snapshot else None
+ except Exception as exc:
+ logger.warning("Could not read remote-code declarations from the loaded files: %s", exc)
+ return None
+
+
@_in_token_rejection_scope
def _handle_load(backend, config: dict, resp_queue: Any) -> None:
_restore_token_environment()
@@ -821,6 +840,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
"is_gguf": False,
# MLX backend sets device="mlx"; lets the UI tag MLX models.
"is_mlx": getattr(backend, "device", None) == "mlx",
+ "loaded_auto_map": _loaded_auto_map(backend, mc),
"is_audio": getattr(mc, "is_audio", False),
"audio_type": getattr(mc, "audio_type", None),
"has_audio_input": getattr(mc, "has_audio_input", False),
@@ -979,6 +999,15 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
)
+def _handle_narrow_context(backend, cmd: dict, resp_queue: Any) -> None:
+ try:
+ fields = backend.narrow_context(cmd.get("max_seq_length"))
+ except Exception as exc:
+ logger.warning("Could not narrow the context in place: %s", exc)
+ fields = None
+ _send_response(resp_queue, {"type": "context_narrowed", "fields": fields})
+
+
def _drain_skip_generate(
cmd: dict,
resp_queue: Any,
@@ -2100,6 +2129,7 @@ def run_inference_process(
# Offline auto-detect, run before any HF import: child_env scrubs the parent's scoped guard, so probe here; skipped for filesystem-only loads.
_probe_model = config["model_name"]
_probe_base, _probe_needs_hub = _recorded_local_base(_probe_model)
+ _probed_offline = None
if "HF_HUB_OFFLINE" not in os.environ and (
_probe_needs_hub or not _hub_targets_are_local(_probe_model, _probe_base)
):
@@ -2110,6 +2140,9 @@ def run_inference_process(
_offline = hf_env_offline()
if not _offline:
_offline = hf_dns_dead()
+ if not _offline and is_apple_silicon():
+ from utils.transformers_version import read_activation_files_ahead
+ read_activation_files_ahead(_probe_model, _config_hf_token(config))
if not _offline and not hf_probe_disabled():
from utils.transformers_version import hf_endpoint_unreachable
@@ -2118,6 +2151,7 @@ def run_inference_process(
gateway_errors_offline = False,
proxy_timeouts_offline = False,
)
+ _probed_offline = _offline
if _offline:
os.environ["HF_HUB_OFFLINE"] = "1"
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
@@ -2177,13 +2211,34 @@ def run_inference_process(
)
from utils.hardware import hardware as _hw
+ from utils.utils import hf_env_offline
+
+ hub_answers = contextlib.ExitStack()
+ model_info_ahead = None
+ if (
+ is_apple_silicon()
+ and not _native_audio_worker
+ and not hf_env_offline()
+ and (_probe_needs_hub or not _hub_targets_are_local(_probe_model, _probe_base))
+ ):
+ from utils.models.model_config import read_hub_model_info_ahead, shared_hub_model_info
+ hub_answers.enter_context(shared_hub_model_info())
+ model_info_ahead = read_hub_model_info_ahead(model_name, _config_hf_token(config))
_hw.detect_hardware()
+ if _hw.DEVICE != _hw.DeviceType.MLX or _native_audio_worker:
+ hub_answers.close()
if _hw.DEVICE == _hw.DeviceType.MLX and not _native_audio_worker:
try:
from core.inference.mlx_inference import MLXInferenceBackend, _init_mlx_distributed
from utils.models.diffusion_lm import mlx_refusal
+ from utils.models.model_config import shared_hub_model_info
+ from utils.utils import pinned_hf_reachability
+ hub_answers.enter_context(pinned_hf_reachability(_probed_offline))
+ hub_answers.enter_context(shared_hub_model_info())
+ if model_info_ahead is not None:
+ model_info_ahead.join()
refusal = mlx_refusal(model_name, _config_hf_token(config))
if refusal:
raise RuntimeError(refusal)
@@ -2220,10 +2275,13 @@ def run_inference_process(
},
)
return
+ finally:
+ hub_answers.close()
logger.info("MLX inference subprocess ready, entering command loop")
batch = _ResidentBatch(backend, resp_queue)
warmth = _MLXIdleWarmth()
+ warmth.active()
deferred: list[dict] = []
stops = _Stops(stop_ledger, resp_queue, batch, deferred)
waiting = lambda: stops.unread() or not cmd_queue.empty()
@@ -2368,6 +2426,11 @@ def run_inference_process(
if backend.active_model_name:
backend.unload_model(backend.active_model_name)
_handle_load(backend, cmd, resp_queue)
+ elif cmd_type == "narrow_context":
+ batch.cancel_all()
+ batch.close()
+ _abandon_held_commands(deferred, resp_queue)
+ _handle_narrow_context(backend, cmd, resp_queue)
elif cmd_type == "unload":
batch.cancel_all()
batch.close()
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 210358ed04d..4d8d9eb37b8 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -19825,6 +19825,7 @@ async def _load_model_impl(
inference_config,
request.hf_token,
trust_remote_code_used = bool(getattr(request, "trust_remote_code", False)),
+ loaded_auto_map = _model_info.get("loaded_auto_map"),
)
try:
backend.models.setdefault(config.identifier, {})["requires_trust_remote_code"] = (
@@ -20105,11 +20106,15 @@ def _resolve_loaded_trust_remote_code(
inference_config,
hf_token = None,
trust_remote_code_used = False,
+ loaded_auto_map = None,
) -> bool:
- """Resolves a loaded model's TRC: stored value, value used, YAML default, then raw auto_map."""
+ """Resolves a loaded model's TRC: stored value, value used, YAML default, then raw auto_map,
+ which `loaded_auto_map` answers without the Hub when the worker read it from the loaded files."""
known = _stored_trust_remote_code(model_info, inference_config, trust_remote_code_used)
if known is not None:
return known
+ if loaded_auto_map is not None:
+ return bool(loaded_auto_map)
return _auto_map_trust_remote_code(model_id, hf_token)
diff --git a/studio/backend/tests/test_audio_device_guards.py b/studio/backend/tests/test_audio_device_guards.py
index 2d439dfc8c7..b1abd0ca491 100644
--- a/studio/backend/tests/test_audio_device_guards.py
+++ b/studio/backend/tests/test_audio_device_guards.py
@@ -494,7 +494,8 @@ def test_the_chat_claim_outlives_the_worker_that_earned_it():
assert "on_prior_worker_released" in sig.parameters
load_src = inspect.getsource(InferenceOrchestrator.load_model)
- hook = load_src.index("on_prior_worker_released()")
+ # The fresh path's call; a context narrowed in the held worker has no teardown to wait for.
+ hook = load_src.rindex("on_prior_worker_released()")
assert load_src.index("did not exit and still holds GPU") < hook
assert load_src.index("if expected_free_gb is not None and not settled:") < hook
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 103efc5246b..4fc3857a7c3 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -414,6 +414,8 @@ def test_a_load_warms_zoo_scopes_under_its_int8_decision(monkeypatch, mlx_moe, o
("warm", True, active),
"/int8",
"/rng",
+ # The resident batch warms in its own block, unless the first warm-up lost the queue.
+ *["rng", "/rng"] * (outcome != "dead_queue"),
]
)
assert events == ["gate_up", *warmed]
diff --git a/studio/backend/tests/test_mlx_load_hub_reads_ahead.py b/studio/backend/tests/test_mlx_load_hub_reads_ahead.py
new file mode 100644
index 00000000000..893fca5f1b7
--- /dev/null
+++ b/studio/backend/tests/test_mlx_load_hub_reads_ahead.py
@@ -0,0 +1,116 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Hub reads an MLX load starts early: the same requests, the same answers, less waiting."""
+
+from __future__ import annotations
+
+import threading
+import urllib.error
+from types import SimpleNamespace
+
+import pytest
+
+from utils import transformers_version as tv
+from utils.models import model_config as mc
+
+
+def hub(monkeypatch):
+ monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
+ monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False)
+ monkeypatch.setattr(tv, "_hf_json_ahead", {})
+ reads = []
+ gate = threading.Barrier(3, timeout = 10)
+
+ def read(url, token):
+ reads.append((url, token))
+ gate.wait()
+ if url.endswith("adapter_config.json"):
+ raise urllib.error.HTTPError(url, 404, "missing", None, None)
+ return {"read": len(reads), "url": url}
+
+ monkeypatch.setattr(tv, "_read_hf_json", read)
+ return reads
+
+
+def test_reads_run_together_and_each_answer_serves_one_read(hub, monkeypatch):
+ tv.read_activation_files_ahead("org/model", "tok")
+
+ config = tv._hf_raw_url("org/model", "config.json")
+ tokenizer = tv._hf_raw_url("org/model", "tokenizer_config.json")
+ assert tv._hf_json(config, "tok")["url"] == config
+ assert tv._hf_json(tokenizer, "tok")["url"] == tokenizer
+ assert sorted(token for _, token in hub) == ["tok"] * 3
+
+ monkeypatch.setattr(tv, "_read_hf_json", lambda url, token: "read again")
+ assert tv._hf_json(config, "tok") == "read again"
+
+
+def test_a_404_reaches_the_caller_as_it_would_have(hub):
+ tv.read_activation_files_ahead("org/model", None)
+ url = tv._hf_raw_url("org/model", "adapter_config.json")
+ with pytest.raises(urllib.error.HTTPError) as caught:
+ tv._hf_json(url, None)
+ assert caught.value.code == 404
+
+
+def test_another_credential_does_not_read_this_answer(hub, monkeypatch):
+ tv.read_activation_files_ahead("org/model", "tok")
+ monkeypatch.setattr(tv, "_read_hf_json", lambda url, token: f"own read {token}")
+ url = tv._hf_raw_url("org/model", "config.json")
+ assert tv._hf_json(url, False) == "own read False"
+
+
+ ("repo", "offline"),
+ [("org/model", True), ("/local/model", False)],
+)
+def test_nothing_starts_offline_or_for_a_local_path(hub, monkeypatch, repo, offline):
+ if offline:
+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")
+ tv.read_activation_files_ahead(repo, None)
+ assert hub == [] and tv._hf_json_ahead == {}
+
+
+def test_an_existing_relative_checkpoint_is_not_read_from_the_hub(hub, monkeypatch, tmp_path):
+ (tmp_path / "outputs" / "private-model").mkdir(parents = True)
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(mc, "_env_offline", lambda: False)
+ calls = []
+ monkeypatch.setattr("huggingface_hub.model_info", lambda repo, **kwargs: calls.append(repo))
+ tv.read_activation_files_ahead("outputs/private-model", "tok")
+ with mc.shared_hub_model_info():
+ assert mc.read_hub_model_info_ahead("outputs/private-model", None) is None
+ assert hub == [] and tv._hf_json_ahead == {} and calls == []
+
+
+def test_model_info_ahead_fills_the_open_scope(monkeypatch):
+ calls = []
+ monkeypatch.delenv("HF_HUB_OFFLINE", raising = False)
+ monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False)
+ monkeypatch.setattr(mc, "_env_offline", lambda: False)
+ monkeypatch.setattr(
+ "huggingface_hub.model_info",
+ lambda repo, **kwargs: calls.append(repo) or SimpleNamespace(id = repo),
+ )
+ with mc.shared_hub_model_info():
+ mc.read_hub_model_info_ahead("org/model", None).join()
+ assert mc._hub_model_info("org/model", None).id == "org/model"
+ assert calls == ["org/model"]
+
+
+def test_a_failed_read_ahead_leaves_the_answer_to_the_probe(monkeypatch):
+ outcomes = [OSError("reset"), SimpleNamespace(id = "second")]
+
+ def model_info(repo, **kwargs):
+ outcome = outcomes.pop(0)
+ if isinstance(outcome, Exception):
+ raise outcome
+ return outcome
+
+ monkeypatch.setattr(mc, "_env_offline", lambda: False)
+ monkeypatch.setattr("huggingface_hub.model_info", model_info)
+ with mc.shared_hub_model_info():
+ mc.read_hub_model_info_ahead("org/model", None).join()
+ assert mc._hub_model_info("org/model", None).id == "second"
diff --git a/studio/backend/tests/test_mlx_load_trust_from_snapshot.py b/studio/backend/tests/test_mlx_load_trust_from_snapshot.py
new file mode 100644
index 00000000000..e21cdaa07da
--- /dev/null
+++ b/studio/backend/tests/test_mlx_load_trust_from_snapshot.py
@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""An MLX load's response answers trust_remote_code from the files the worker opened, not the Hub."""
+
+from __future__ import annotations
+
+import json
+from types import SimpleNamespace
+
+import pytest
+
+import routes.inference as routes
+from core.inference import model_ids, worker
+from core.inference.orchestrator import _mirrored_model_entry
+
+resolve = routes._resolve_loaded_trust_remote_code
+
+
+def test_the_loaded_answer_replaces_the_hub_read_only(monkeypatch):
+ monkeypatch.setattr(routes, "_auto_map_trust_remote_code", lambda *_a: 1 / 0)
+ assert resolve("org/m", None, {}, loaded_auto_map = True) is True
+ assert resolve("org/m", None, {}, loaded_auto_map = False) is False
+ assert resolve("org/m", None, {}, trust_remote_code_used = True, loaded_auto_map = False) is True
+ stored = {"requires_trust_remote_code": False}
+ assert resolve("org/m", stored, {}, loaded_auto_map = True) is False
+ monkeypatch.setattr(routes, "_auto_map_trust_remote_code", lambda *_a: True)
+ assert resolve("org/m", None, {}) is True
+ assert _mirrored_model_entry({"loaded_auto_map": False}, "org/m")["loaded_auto_map"] is False
+ assert hasattr(worker._handle_load, "__wrapped__")
+
+
+ "config, device, fields, base, answer",
+ [
+ ('{"auto_map": {"AutoModel": "m.M"}}', "mlx", {}, None, True),
+ ('{"model_type": "qwen3"}', "mlx", {}, None, False),
+ ('{"model_type": "qwen3"}', "mlx", {"is_lora": True}, None, None),
+ ('{"model_type": "qwen3"}', "mlx", {"gguf_variant": "4bit"}, None, None),
+ ('{"model_type": "qwen3"}', "mlx", {}, "org/base", None),
+ ],
+)
+def test_the_worker_answers_only_for_its_own_files(
+ tmp_path, monkeypatch, config, device, fields, base, answer
+):
+ monkeypatch.setattr(model_ids, "mlx_host_bnb_base_repo", lambda _name: base)
+ (tmp_path / "config.json").write_text(config)
+ backend = SimpleNamespace(device = device, _model = SimpleNamespace(_src_path = str(tmp_path)))
+ mc = SimpleNamespace(identifier = "org/m", **{"is_lora": False, **fields})
+ assert worker._loaded_auto_map(backend, mc) is answer
diff --git a/studio/backend/tests/test_mlx_load_warm_and_narrow.py b/studio/backend/tests/test_mlx_load_warm_and_narrow.py
new file mode 100644
index 00000000000..fd21a7da46e
--- /dev/null
+++ b/studio/backend/tests/test_mlx_load_warm_and_narrow.py
@@ -0,0 +1,226 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A load serves through the batch before it reports loaded, and a smaller context reuses the held weights."""
+
+from __future__ import annotations
+
+import inspect
+import threading
+from types import SimpleNamespace
+
+import pytest
+
+from core.inference import mlx_inference, orchestrator
+from core.inference.mlx_inference import MLXInferenceBackend
+from core.inference.runtime_context import runtime_context_length
+
+
+class _Session:
+ def __init__(self, backend, fail):
+ self.backend, self.fail, self.rows_in_flight = backend, fail, 0
+
+ def admit(self, request, _handle):
+ self.backend.temperatures.append(request["temperature"])
+ if self.backend._speculative_draft is not None:
+ self.backend._speculative_draft.controller.replies += 1
+ self.rows_in_flight = 1
+
+ def step(self):
+ self.rows_in_flight = 0
+ yield "warm", self.fail
+
+ def close(self):
+ self.backend.closed += 1
+
+
+def _warmed(
+ fail = None,
+ room = None,
+ draft = None,
+):
+ backend = MLXInferenceBackend.__new__(MLXInferenceBackend)
+ vars(backend).update(
+ temperatures = [], closed = 0, cleared = 0, _prompt_cache_room = room, _speculative_draft = draft
+ )
+
+ def clear():
+ backend.cleared += 1
+ backend._prompt_cache_room = None
+
+ backend.resident_unavailable_reason = lambda _request: None
+ backend.open_resident_batch = lambda *, width: _Session(backend, fail)
+ backend._clear_prompt_cache = clear
+ backend._warm_resident_batch()
+ return backend
+
+
+def test_warm_serves_greedy_twice_then_sampled_and_clears_the_cache():
+ backend = _warmed(room = 123)
+ assert backend.temperatures == [0.0, 0.0, 0.7] and backend.closed == 3
+ assert backend.cleared == 1 and backend._prompt_cache_room == 123
+
+
+def test_warm_replies_leave_the_drafter_controller_untaught():
+ controller = SimpleNamespace(replies = 0)
+ draft = SimpleNamespace(controller = controller)
+ _warmed(draft = draft)
+ assert draft.controller is controller and controller.replies == 0
+
+
+def test_the_load_warms_the_batch_with_the_rng_preserved():
+ source = inspect.getsource(MLXInferenceBackend.load_model)
+ assert "with mlx_rng_preserved():\n self._warm_resident_batch()" in source
+
+
+def test_a_failed_warm_leaves_the_load_standing_and_the_cache_clear():
+ backend = _warmed(fail = ValueError("row"))
+ assert backend.temperatures == [0.0] and backend.closed == 1 and backend.cleared == 1
+
+
+def test_a_dead_metal_queue_still_fails_the_load(monkeypatch):
+ monkeypatch.setattr(mlx_inference, "is_metal_queue_dead", lambda _error: True)
+ with pytest.raises(ValueError):
+ _warmed(fail = ValueError("queue"))
+
+
+def _narrowing(**entry):
+ backend = MLXInferenceBackend.__new__(MLXInferenceBackend)
+ vars(backend).update(
+ active_model_name = "org/m",
+ models = {"org/m": {"requested_context_length": 8192, **entry}},
+ _model = SimpleNamespace(max_seq_length = 8192),
+ _is_vlm = False,
+ _served_context = 8192,
+ _kv_quant = {"requested_kv_bits": None},
+ _turboquant = False,
+ _prompt_cache_room = None,
+ _speculative_draft = None,
+ _kv_cache_window = 8192,
+ _kv_context_budget = None,
+ _resume_files = (("w.safetensors", 1, 1),),
+ _resume_weight_files = lambda: (("w.safetensors", 1, 1),),
+ cleared = 0,
+ )
+ backend._clear_prompt_cache = lambda: setattr(backend, "cleared", backend.cleared + 1)
+ backend._resolve_context_lengths = lambda model, pinned: (
+ runtime_context_length(model, pinned) or 4096,
+ )
+ backend._resolve_kv_policy = lambda _vlm, _bits, _pinned, served: ({}, served, True, None)
+ return backend
+
+
+def test_a_smaller_context_is_applied_to_the_held_model():
+ backend = _narrowing(context_length_fitted = 8192)
+ fields = backend.narrow_context(4096)
+ assert fields["requested_context_length"] == fields["context_length"] == 4096
+ assert backend.models["org/m"]["context_length_fitted"] is None and backend.cleared == 1
+ assert backend._served_context == backend._kv_cache_window == 4096
+
+
+ "size, entry, attrs",
+ [
+ (8193, {}, {}),
+ (0, {}, {}),
+ (4096, {"mlx_offload": {"ple": {"applied": True}}}, {}),
+ (4096, {}, {"_kv_quant": {"requested_kv_bits": 8}}),
+ (4096, {}, {"_turboquant": True}),
+ (4096, {}, {"_prompt_cache_room": 1}),
+ (4096, {"spec_fallback_reason": "does not fit"}, {}),
+ (4096, {"spec_fallback_reason": 1}, {"_speculative_draft": SimpleNamespace(drafter = None)}),
+ (
+ 4096,
+ {"spec_fallback_reason": 1},
+ {"_speculative_draft": SimpleNamespace(drafter = object())},
+ ),
+ (4096, {}, {"_resume_weight_files": lambda: (("w.safetensors", 2, 2),)}),
+ (4096, {}, {"_resume_files": False}),
+ (4096, {}, {"_resume_files": None}),
+ ],
+)
+def test_a_load_a_fresh_one_could_plan_differently_is_not_narrowed(size, entry, attrs):
+ backend = _narrowing(**entry)
+ vars(backend).update(attrs)
+ assert backend.narrow_context(size) is None
+ assert backend._served_context == backend._model.max_seq_length == 8192 and backend.cleared == 0
+
+
+def _held(fields = {"context_length": 4096}, **settings):
+ proc, sent = SimpleNamespace(is_alive = lambda: True), []
+ config = dict(
+ device_backend = "mlx", mlx_distributed = False, max_seq_length = 4096, gpu_selection = {"gb": 3}
+ )
+ config.update(settings)
+ backend = orchestrator.InferenceOrchestrator.__new__(orchestrator.InferenceOrchestrator)
+ vars(backend).update(
+ _proc = proc,
+ active_model_name = "org/m",
+ models = {"org/m": {"parallel_slots": 1, "context_length": 8192}},
+ load_generation = 0,
+ _loaded_by_user_action = True,
+ _worker_reserved_for = None,
+ _active_cancel_events = set(),
+ loading_models = {"org/m"},
+ _replies_in_flight = lambda: 0,
+ _wait_worker_idle = lambda: sent.append("idle") or True,
+ _send_cmd = sent.append,
+ _wait_response = lambda kind, timeout: {"type": kind, "fields": fields},
+ _worker_load_settings = (proc, orchestrator._load_settings({**config, "gpu_selection": {}})),
+ )
+ for lock in (
+ "_dispatcher_lifecycle",
+ "_gen",
+ "_active_cancel",
+ "_send_order",
+ "_subprocess_shutdown",
+ ):
+ setattr(backend, f"{lock}_lock", threading.Lock())
+ return backend, config, sent
+
+
+def test_the_held_worker_takes_a_smaller_context_and_the_parent_mirrors_it():
+ backend, config, sent = _held()
+ assert backend._narrow_context_in_place(config, "org/m", 1) is True
+ assert sent == ["idle", {"type": "narrow_context", "max_seq_length": 4096}]
+ assert backend.models["org/m"]["context_length"] == 4096 and backend.load_generation == 1
+ assert backend._worker_reserved_for is None and not backend._gen_lock.locked()
+
+
[email protected]("outcome", ["declined", "event", "withdrawn"])
+def test_a_load_declined_or_cancelled_meanwhile_is_not_published(outcome):
+ backend, config, _sent = _held(fields = None if outcome == "declined" else {"context_length": 1})
+ event, answer = threading.Event(), backend._wait_response
+
+ def answered(kind, timeout):
+ event.set() if outcome == "event" else None
+ backend.loading_models.clear() if outcome == "withdrawn" else None
+ return answer(kind, timeout)
+
+ backend._wait_response = answered
+ assert backend._narrow_context_in_place(config, "org/m", 1, event) is False
+ assert backend.models["org/m"]["context_length"] == 8192 and backend.load_generation == 0
+
+
+ "change",
+ [
+ lambda b, c: c.update(mlx_kv_quant = "8"),
+ lambda b, c: c.update(mlx_distributed = True),
+ lambda b, c: c.update(max_seq_length = 8192),
+ lambda b, c: setattr(b, "_wait_worker_idle", lambda: False),
+ lambda b, c: setattr(b, "_replies_in_flight", lambda: 1),
+ lambda b, c: b.models["org/m"].update(parallel_slots = 4),
+ ],
+)
+def test_any_other_change_spawns_fresh(change):
+ backend, config, sent = _held()
+ change(backend, config)
+ assert backend._narrow_context_in_place(config, "org/m", 1) is False
+ assert not [cmd for cmd in sent if cmd != "idle"]
+
+
[email protected]("settings", [{"device_backend": "cuda"}, {"mlx_distributed": True}])
+def test_only_a_single_mlx_worker_narrows(settings):
+ backend, config, sent = _held(**settings)
+ assert backend._narrow_context_in_place(config, "org/m", 1) is False and sent == []
diff --git a/studio/backend/tests/test_mlx_reload_skips_redundant_waits.py b/studio/backend/tests/test_mlx_reload_skips_redundant_waits.py
new file mode 100644
index 00000000000..e66211f5215
--- /dev/null
+++ b/studio/backend/tests/test_mlx_reload_skips_redundant_waits.py
@@ -0,0 +1,128 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""One MLX load asks the Hub each question once, and a reload does not wait for memory nothing reads yet."""
+
+from __future__ import annotations
+
+import inspect
+import urllib.error
+from types import SimpleNamespace
+
+import pytest
+
+from core.inference import orchestrator, worker
+from utils import utils as uu
+from utils.models import model_config as mc
+
+
+class _PastTheSettle(BaseException):
+ pass
+
+
+def _settle_calls(
+ monkeypatch,
+ device,
+ expected = None,
+ settled = True,
+):
+ calls = []
+ monkeypatch.setattr(orchestrator, "get_device", lambda: SimpleNamespace(value = device))
+ monkeypatch.setattr(orchestrator, "prepare_gpu_selection", lambda *a, **k: (None, {}))
+ monkeypatch.setattr("utils.transformers_version.needs_transformers_5", lambda *_a: False)
+ monkeypatch.setattr("utils.transformers_version.sidecar_swap_kind", lambda: None)
+ monkeypatch.setattr(orchestrator.time, "sleep", lambda _s: None)
+
+ def released():
+ raise _PastTheSettle
+
+ backend = orchestrator.InferenceOrchestrator.__new__(orchestrator.InferenceOrchestrator)
+ vars(backend).update(
+ loading_models = set(),
+ models = {},
+ active_model_name = None,
+ _managed_engine = None,
+ _proc = object(),
+ _ensure_subprocess_alive = lambda: True,
+ _cancel_generation = lambda: None,
+ _shutdown_subprocess = lambda *a, **k: True,
+ _wait_for_worker_vram_settle = lambda _since, *, expected_free_gb = None, **_: (
+ calls.append(expected_free_gb) or True
+ ),
+ )
+ with pytest.raises(_PastTheSettle):
+ backend.load_model(
+ SimpleNamespace(identifier = "org/m", gguf_variant = None),
+ on_prior_worker_released = released,
+ post_handoff_expected_free_gb = expected,
+ )
+ return calls
+
+
+ "device, expected, calls",
+ [("mlx", None, []), ("mlx", {0: 8.0}, [{0: 8.0}]), ("cuda", None, [None])],
+)
+def test_only_mlx_with_no_figure_skips_the_settle(monkeypatch, device, expected, calls):
+ assert _settle_calls(monkeypatch, device, expected) == calls
+
+
+def test_a_supplied_verdict_answers_without_a_probe(monkeypatch):
+ monkeypatch.setattr("utils.transformers_version.hf_endpoint_unreachable", lambda *a, **k: 1 / 0)
+ uu.reset_hf_reachability_cache()
+ with uu.pinned_hf_reachability(True):
+ assert uu.hf_unreachable() is True
+
+
+def test_nested_scopes_join_and_the_scope_ends_with_the_outermost(monkeypatch):
+ reads = []
+ monkeypatch.setattr("huggingface_hub.model_info", lambda repo, **_k: reads.append(repo) or repo)
+ monkeypatch.setattr(mc, "_env_offline", lambda: False)
+ with mc.shared_hub_model_info():
+ mc._hub_model_info("org/m")
+ with mc.shared_hub_model_info():
+ mc._hub_model_info("org/m")
+ mc._hub_model_info("org/m")
+ assert reads == ["org/m"]
+ assert mc._hub_model_info_scope.get() is None
+
+
+def test_a_shared_answer_replays_its_token_rejection_into_a_later_scope(monkeypatch):
+ from hub.utils.hf_tokens import saved_token_rejected, token_rejection_scope
+
+ def model_info(repo, *, token, **_k):
+ if token:
+ raise urllib.error.HTTPError(repo, 401, "expired", None, None)
+ return repo
+
+ monkeypatch.setattr("huggingface_hub.model_info", model_info)
+ monkeypatch.setattr(mc, "_env_offline", lambda: False)
+ with mc.shared_hub_model_info():
+ mc._hub_model_info("org/m", "hf_expired")
+ with token_rejection_scope():
+ assert not saved_token_rejected("hf_expired")
+ assert mc._hub_model_info("org/m", "hf_expired") == "org/m"
+ assert saved_token_rejected("hf_expired")
+
+
+def test_only_the_mlx_load_holds_the_answers():
+ source = inspect.getsource(worker.run_inference_process)
+ scope = source.index("hub_answers.enter_context(pinned_hf_reachability(_probed_offline))")
+ assert (
+ scope
+ < source.index("mlx_refusal(model_name")
+ < source.index("_handle_load(backend, config")
+ )
+ closed = source.rindex("hub_answers.close()")
+ assert (
+ source.index("read_hub_model_info_ahead(model_name")
+ < source.index("_hw.detect_hardware()")
+ < source.index("hub_answers.close()")
+ < source.index("model_info_ahead.join()")
+ < source.index("mlx_refusal(model_name")
+ )
+ assert (
+ source.index("_handle_load(backend, config")
+ < closed
+ < source.index("batch = _ResidentBatch")
+ )
diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py
index 77a0a52c32c..79358f2f703 100644
--- a/studio/backend/utils/models/model_config.py
+++ b/studio/backend/utils/models/model_config.py
@@ -30,6 +30,8 @@ from hub.utils.hf_tokens import (
cache_reads_authorized,
call_with_anonymous_retry,
collecting_hub_token_rejections,
+ note_saved_token_rejected,
+ saved_token_rejected,
HUB_TOKEN_REJECTED_ERROR,
hf_token_arg,
is_rejected_credential_error,
@@ -57,7 +59,7 @@ from utils.models.gguf_metadata import (
import structlog
from loggers import get_logger
import contextlib as _contextlib
-from contextvars import ContextVar
+from contextvars import ContextVar, copy_context
import functools
import os
import platform
@@ -1295,8 +1297,9 @@ _GGUF_LISTING_TIMEOUTS = (_HUB_MODEL_INFO_TIMEOUT, 30.0, 60.0)
@_contextlib.contextmanager
def shared_hub_model_info():
- """Share a ``model_info`` response between the Hub probes here, per repo and credential."""
- token = _hub_model_info_scope.set({})
+ """Share a ``model_info`` response between the Hub probes here, per repo and credential. A nested block joins the open one."""
+ open_scope = _hub_model_info_scope.get()
+ token = _hub_model_info_scope.set({} if open_scope is None else open_scope)
try:
yield
finally:
@@ -1318,7 +1321,11 @@ def _hub_model_info(
key = (repo_id, _token_fingerprint(hf_token))
if scope is not None:
if key in scope:
- return scope[key]
+ info, rejected = scope[key]
+ if rejected:
+ # The read may have run before the caller's rejection scope opened.
+ note_saved_token_rejected(hf_token)
+ return info
# Asked unconditionally so one response serves every probe; the listing needs sizes.
files_metadata = True
@@ -1328,15 +1335,33 @@ def _hub_model_info(
# Shared, so whichever probe reads first fixes the bound the rest inherit.
"timeout": _HUB_MODEL_INFO_TIMEOUT if timeout is None else timeout,
}
- info = call_with_anonymous_retry(
- lambda token: hf_model_info(repo_id, **{**kwargs, "token": token}), hf_token
- )
+ with collecting_hub_token_rejections():
+ info = call_with_anonymous_retry(
+ lambda token: hf_model_info(repo_id, **{**kwargs, "token": token}), hf_token
+ )
+ rejected = saved_token_rejected(hf_token)
if scope is not None:
- scope[key] = info
+ scope[key] = (info, rejected)
return info
+def read_hub_model_info_ahead(repo_id: str, hf_token: HfTokenArg) -> Optional[threading.Thread]:
+ """Fill the open scope from a thread. Join it before the first probe; a failed read leaves the answer to that probe."""
+ if is_local_path(repo_id):
+ return None
+
+ def read():
+ try:
+ _hub_model_info(repo_id, hf_token)
+ except Exception as exc:
+ logger.debug("model_info read ahead failed for %s: %s", repo_id, exc)
+
+ thread = threading.Thread(target = copy_context().run, args = (read,), daemon = True)
+ thread.start()
+ return thread
+
+
def _is_timeout_error(exc: BaseException) -> bool:
if isinstance(exc, TimeoutError):
return True
diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py
index 5d767a1bf3f..529443d8143 100644
--- a/studio/backend/utils/transformers_version.py
+++ b/studio/backend/utils/transformers_version.py
@@ -38,6 +38,7 @@ from loggers import get_logger
import errno
import contextlib
import os
+from concurrent.futures import Future
import re
import shutil
import stat
@@ -129,7 +130,37 @@ def _hf_proxy_opener(url: str):
return None
+_hf_json_ahead: dict[tuple[str, str | None], Future] = {}
+
+
+def read_activation_files_ahead(model_name: str, hf_token: str | None) -> None:
+ """Start activation's raw Hub reads together so TLS handshakes overlap; each answer serves one read."""
+ if not _is_canonical_repo_id(model_name) or _env_offline():
+ return
+ from utils.paths import is_local_path
+
+ if is_local_path(model_name):
+ return
+
+ def fill(future: Future, url: str) -> None:
+ try:
+ future.set_result(_read_hf_json(url, hf_token))
+ except BaseException as exc:
+ future.set_exception(exc)
+
+ for filename in ("adapter_config.json", "tokenizer_config.json", "config.json"):
+ url = _hf_raw_url(model_name, filename)
+ future = Future()
+ _hf_json_ahead[(url, hf_token)] = future
+ threading.Thread(target = fill, args = (future, url), daemon = True).start()
+
+
def _hf_json(url: str, hf_token: str | None):
+ ahead = _hf_json_ahead.pop((url, hf_token), None)
+ return _read_hf_json(url, hf_token) if ahead is None else ahead.result()
+
+
+def _read_hf_json(url: str, hf_token: str | None):
"""GET a JSON file from the Hub, once more without the token if the Hub refuses it (#11551)."""
import urllib.request
diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py
index cd361ad7f13..ea93595da77 100644
--- a/studio/backend/utils/utils.py
+++ b/studio/backend/utils/utils.py
@@ -268,9 +268,9 @@ _hf_reachability_pin: "ContextVar[Optional[list]]" = ContextVar("hf_reachability
@contextmanager
-def pinned_hf_reachability():
- """Hold one reachability verdict for this block, however long it runs. Every verdict this module reaches fills it (probed, already memoised, or opted out of) and the rest read it. Nothing is pinned until something asks, so a block that never touches the Hub pays nothing."""
- token = _hf_reachability_pin.set([])
+def pinned_hf_reachability(verdict: Optional[bool] = None):
+ """Hold one Hub reachability verdict for this block, pinned lazily on first ask; pass `verdict` if already probed."""
+ token = _hf_reachability_pin.set([] if verdict is None else [verdict])
try:
yield
finally: