Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
feat(studio): run fp16 Laya checkpoints in fp16 on MLX (#12256)
Long Yixing90db6a8267e4
diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.pyindex f6cb68dcfa6..b222dd45f1d 100644--- a/studio/backend/core/systemone/laya_runtime.py+++ b/studio/backend/core/systemone/laya_runtime.py@@ -259,9 +259,15 @@ class _MLXAgent:device = "mlx"- def __init__(self, folder: Path):+ def __init__(+ self,+ folder: Path,+ fp16_checkpoint: bool = False,+ ):+ import inspectimport json+ import mlx.core as mxfrom transformers import AutoTokenizerfrom unsloth_zoo.mlx.decision import load_decision_model@@ -277,7 +283,11 @@ class _MLXAgent:k: clamp_temperature(v) for k, v in self.cfg.get("temperature_by_options", {}).items()}self._to_internal = laya.agent.Agent._to_internal- self.model = load_decision_model(folder)+ # As _precision does for fp16 checkpoints on CUDA: exact fp16 weights, fp16 matmuls, fp32 norms.+ # An unsloth-zoo without compute_dtype casts the whole model instead, so it stays fp32.+ mixed = "compute_dtype" in inspect.signature(load_decision_model).parameters+ self.dtype = mx.float16 if fp16_checkpoint and mixed and not _fp32_forced() else mx.float32+ self.model = load_decision_model(folder, **({"compute_dtype": self.dtype} if mixed else {}))def _load_checkpoint(checkpoint: Checkpoint):@@ -288,14 +298,14 @@ def _load_checkpoint(checkpoint: Checkpoint):_evict()device = _device()folder = root / checkpoint.subfolder if checkpoint.subfolder else root+ fp16_checkpoint = checkpoint.name in CHECKPOINTS or _stored_fp16(folder)if device == "mlx":- return _MLXAgent(folder), device+ return _MLXAgent(folder, fp16_checkpoint), deviceif device not in ("cuda", "cpu"):return _load_laya(str(root), subfolder = checkpoint.subfolder, device = device), deviceimport torch# Built on CPU and cast before the move, so the device never holds laya's fp32 copy.- fp16_checkpoint = checkpoint.name in CHECKPOINTS or _stored_fp16(folder)weights, _ = _precision(torch.device(device), fp16_checkpoint)agent = _load_laya(str(root), subfolder = checkpoint.subfolder, device = "cpu", embedding_dtype = weights@@ -410,18 +420,22 @@ def _stored_fp16(folder: Path) -> bool:return bool(sizes) and max(sizes, key = sizes.get) == "F16"+def _fp32_forced() -> bool:+ import os+ return os.environ.get("UNSLOTH_SYSTEMONE_FP32", "") == "1"++def _precision(device, fp16_checkpoint: bool):"""(weight dtype, compute dtype) for ``device``; ``(None, None)`` keeps laya's fp32 weights and compute.An fp16 checkpoint held in fp16 is exact (fp16 -> fp32 is lossless), and fp16 compute tracked fp32 about10x closer than bf16 on both GPU and CPU. Anything else follows the device: bf16 where it is native, else fp16."""- import osimport platformimport torch- if os.environ.get("UNSLOTH_SYSTEMONE_FP32", "") == "1":+ if _fp32_forced():return None, Noneif device.type == "cuda":if fp16_checkpoint:@@ -698,12 +712,21 @@ def _predict(agent, state, questions: dict[str, dict[str, Any]]) -> tuple[dict[sdef _forward(agent, items: list[dict[str, Any]]):global _agent, _loaded, _device_name+ import numpy as npimport torchbatch = _collate(items, agent.tok.pad_token_id)if agent.device == "mlx":try:- return agent.model.logits(batch), int(batch["attention_mask"].sum())+ logits = agent.model.logits(batch)+ if not np.isfinite(logits).all():+ import mlx.core as mx+ if agent.dtype != mx.float32:+ logger.warning("Laya overflowed in float16; continuing in float32")+ agent.model.set_dtype(mx.float32)+ agent.dtype = mx.float32+ logits = agent.model.logits(batch)+ return logits, int(batch["attention_mask"].sum())except RuntimeError as exc:# MLX reports exhausted memory as "[malloc] Unable to allocate ..." or "Insufficient Memory".reason = str(exc).lower()diff --git a/studio/backend/tests/test_systemone.py b/studio/backend/tests/test_systemone.pyindex a56c0229694..1ebefeb16b6 100644--- a/studio/backend/tests/test_systemone.py+++ b/studio/backend/tests/test_systemone.py@@ -1757,6 +1757,66 @@ def test_rocm_never_builds_cuda_graphs(tmp_path, monkeypatch):assert "_unsloth_graphs" not in agent.__dict__+def _fake_mlx(monkeypatch, **zoo):+ mx = SimpleNamespace(float16 = "float16", float32 = "float32")+ monkeypatch.setitem(sys.modules, "mlx", SimpleNamespace(core = mx))+ monkeypatch.setitem(sys.modules, "mlx.core", mx)+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.decision", SimpleNamespace(**zoo))+++def test_mlx_runs_fp16_checkpoints_in_fp16(monkeypatch, tmp_path):+ import transformers++ _fake_mlx(+ monkeypatch, load_decision_model = lambda folder, compute_dtype = "float32": compute_dtype+ )+ fake_laya = SimpleNamespace(+ common = SimpleNamespace(clamp_temperature = float),+ agent = SimpleNamespace(Agent = SimpleNamespace(_to_internal = None)),+ )+ monkeypatch.setitem(sys.modules, "laya", fake_laya)+ monkeypatch.setattr(transformers.AutoTokenizer, "from_pretrained", lambda path: None)+ (tmp_path / "rl_agent_config.json").write_text("{}")+ assert laya_runtime._MLXAgent(tmp_path, True).model == "float16"+ assert laya_runtime._MLXAgent(tmp_path, False).model == "float32"+ monkeypatch.setenv("UNSLOTH_SYSTEMONE_FP32", "1")+ assert laya_runtime._MLXAgent(tmp_path, True).model == "float32"+ monkeypatch.delenv("UNSLOTH_SYSTEMONE_FP32")+ # An unsloth-zoo whose loader predates compute_dtype.+ _fake_mlx(monkeypatch, load_decision_model = lambda folder, dtype = "float32": dtype)+ old = laya_runtime._MLXAgent(tmp_path, True)+ assert old.dtype == old.model == "float32"++ monkeypatch.setattr(laya_runtime, "_checkpoint_dir", lambda checkpoint: tmp_path)+ monkeypatch.setattr(laya_runtime, "_device", lambda: "mlx")+ monkeypatch.setattr(laya_runtime, "_MLXAgent", lambda folder, fp16: fp16)+ assert _REAL_LOAD(catalog.CHECKPOINTS["laya-english"]) == (True, "mlx")+++def test_mlx_fp16_overflow_reruns_in_fp32(monkeypatch, gpu_agent):+ import numpy as np++ _fake_mlx(monkeypatch)++ class Model:+ dtype = "float16"++ def set_dtype(self, dtype):+ self.dtype = dtype++ def logits(self, batch):+ return np.array([[np.inf if self.dtype == "float16" else 1.0, 0.5]])++ agent = SimpleNamespace(device = "mlx", dtype = "float16", model = Model(), tok = gpu_agent.tok)+ logits, _ = laya_runtime._forward(agent, _items())+ assert logits.tolist() == [[1.0, 0.5]] and agent.dtype == agent.model.dtype == "float32"+ # An fp32 agent has nothing wider to retry in: its non-finite logits come back without a set_dtype call.+ agent.model.dtype = "float16" # only makes the fake emit inf+ agent.model.set_dtype = None+ logits, _ = laya_runtime._forward(agent, _items())+ assert np.isinf(logits[0, 0])++def _mcp_decide(arguments):from fastmcp import Clientasync def call():