Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio MLX estimate: price attention scores where mlx really holds them, at the width it holds them (#16021)
Daniel Han3ec3350edcc3
diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.pyindex 45cdbfd8bc1..b02b3565459 100644--- a/studio/backend/core/inference/mlx_memory.py+++ b/studio/backend/core/inference/mlx_memory.py@@ -51,9 +51,8 @@ _COMPUTE_BASE_BYTES = int(0.64 * 1024**3)_RECURRENT_DTYPE_SIZE = 4-_QUANT_SCORE_DTYPE_SIZE = 4--_QUANT_SCORE_LIVE = 1.0+# Score copies an unfused call holds at the queries' width; Metal measured 1.03 to 1.15 (mlx 0.32.4).+_SCORE_LIVE = 1.25_WIDTH_NAMES = {"float64": "f64", "float32": "f32", "bfloat16": "bf16", "float16": "f16"}@@ -995,15 +994,104 @@ def _compute_bytes(recurrent_width = sum(int(entry["conv_width"]) for entry in plan) * _RECURRENT_DTYPE_SIZEtotal = _COMPUTE_BASE_BYTES + prefill_chunk * (attention_width + recurrent_width)if quant_boundary is not None:- score = heads * _QUANT_SCORE_DTYPE_SIZE * _QUANT_SCORE_LIVE+ score = heads * dtype_size * _SCORE_LIVErow = _dequantized_row_bytes(widths, dtype_size, kv_bits, kv_group_size, prefill_chunk)if row is None:total += _widest_quantized_scores(n_ctx, quant_boundary, prefill_chunk) * scoreelse:total += max(row * n_ctx, (n_ctx + 1) * score)+ elif not any(entry.get("bound_spec") for entry in plan):+ # Windowed models are skipped: the config's head_dim need not be the full-context layers' width.+ score = heads * dtype_size * _SCORE_LIVE+ total += _widest_unfused_scores(widths, dtype_size, n_ctx, prefill_chunk) * scorereturn int(total)+def _widest_unfused_scores(widths, dtype_size: int, n_ctx: int, prefill_chunk: int) -> int:+ """Query x key cells of the widest call mlx leaves unfused over every prompt the context admits."""+ heads, kv_heads, head_dim = (*widths[2:], 0, 0, 0)[:3]+ route = _zoo_route() if heads and kv_heads and head_dim else None+ if route is None:+ return 0+ widths, chunk = tuple(widths), prefill_chunk++ def unfused(queries, keys):+ return not _attention_fuses(route, widths, dtype_size, queries, keys)++ def longest_tail(whole, limit):+ # The tail of a prompt holding no whole chunk is its first call, so it is square.+ short, first_fused = _unfused_tails(route, widths, dtype_size, chunk if whole else 0)+ if limit > 8 and first_fused > 9:+ return min(limit, first_fused - 1)+ return max((r for r in short if r <= limit), default = 0)++ whole, rest = divmod(max(n_ctx - 1, 0), chunk)+ cells = [n_ctx + 1 if unfused(1, 2) or unfused(1, n_ctx + 1) else 0]+ if whole and unfused(chunk, chunk):+ cells.append(chunk * chunk)+ if whole >= 2 and unfused(chunk, whole * chunk):+ cells.append(chunk * whole * chunk)+ r = longest_tail(whole, rest)+ cells.append(r * (whole * chunk + r))+ if whole:+ r = longest_tail(whole - 1, chunk - 1)+ cells.append(r * ((whole - 1) * chunk + r))+ return max(cells)+++# Past any context: a shape still unfused here never fuses.+_TAIL_MAX = 1 << 24++[email protected]_cache(maxsize = 64)+def _unfused_tails(route, widths, dtype_size: int, later: int):+ """Unfused tails of up to 8 queries, and the first longer one that fuses.++ Past 8 queries mlx's kernels only switch on as queries grow, so one boundary splits the two."""++ def unfused(r):+ return not _attention_fuses(route, widths, dtype_size, r, r + later)++ short = tuple(r for r in range(1, 9) if unfused(r))+ lo, hi = 8, 9+ while hi < _TAIL_MAX and unfused(hi):+ lo, hi = hi, min(hi * 2, _TAIL_MAX)+ while hi - lo > 1:+ mid = (lo + hi) // 2+ lo, hi = (mid, hi) if unfused(mid) else (lo, mid)+ return short, hi+++def _zoo_route():+ try:+ import mlx.core # noqa: F401++ from unsloth_zoo.mlx.attention import fused_kernel_exists+ except Exception:+ return None+ return fused_kernel_exists+++def _attention_fuses(route, widths, dtype_size: int, queries: int, keys: int) -> bool:+ """Whether mlx fuses a full-width causal call; elsewhere it materializes the scores."""+ import mlx.core as mx++ _heads, kv_heads, head_dim, value_dim = (*widths[2:], 0, 0, 0, 0)[:4]+ try:+ dtype = mx.float32 if dtype_size == 4 else mx.bfloat16++ # zoo reads only shapes and dtypes; a group of one makes the scales the cache's width.+ def cache(width):+ scales = mx.zeros((1, kv_heads, keys, width), dtype = dtype)+ return scales, scales, scales++ q = mx.zeros((1, widths[2], queries, head_dim), dtype = dtype)+ return bool(route(q, cache(head_dim), cache(value_dim or head_dim), 1, "causal"))+ except Exception as exc:+ logger.debug("MLX estimate cannot route the attention: %s", exc)+ return True++def _load_is_refused(model_dir: str, config: dict, load_in_4bit: bool) -> Optional[str]:try:from unsloth_zoo.mlx.loader import (diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.pyindex 4518979433d..5f041882416 100644--- a/studio/backend/tests/test_memory_estimate.py+++ b/studio/backend/tests/test_memory_estimate.py@@ -4409,6 +4409,86 @@ class TestComputeBytes:)+def _unfused_route(monkeypatch, fuses):+ """mlx and zoo stubs, so the unquantized route runs where MLX does not."""+ asked = []+ mx = SimpleNamespace(+ float32 = SimpleNamespace(size = 4),+ bfloat16 = SimpleNamespace(size = 2),+ zeros = lambda shape, dtype: SimpleNamespace(shape = shape, dtype = dtype),+ )++ def fused_kernel_exists(+ queries,+ q_keys,+ q_values,+ group_size,+ mask = None,+ ):+ L, S = queries.shape[-2], q_keys[0].shape[-2]+ assert q_keys[1].shape[-1] * group_size == q_values[1].shape[-1] * group_size == 256+ assert queries.shape[1] == 24 and q_keys[0].shape[1] == 4 and mask == "causal"+ asked.append((L, S))+ return fuses(L, S)++ monkeypatch.setitem(sys.modules, "mlx", SimpleNamespace(core = mx))+ monkeypatch.setitem(sys.modules, "mlx.core", mx)+ monkeypatch.setitem(+ sys.modules,+ "unsloth_zoo.mlx.attention",+ SimpleNamespace(fused_kernel_exists = fused_kernel_exists),+ )+ return asked+++_BONSAI = (5120, 17408, 24, 4, 256, 0)+_SCORE = 24 * 2 * mm._SCORE_LIVE+++ "fuses, cells",+ [+ (lambda L, S: True, 0),+ (lambda L, S: L <= 8, 2048 * 32768),+ # NAX: the split kernel needs 1024 queries, so only the 1023-token tail is unfused.+ (lambda L, S: L <= 8 or L >= 1024, 1023 * 33791),+ ],+)+def test_a_full_width_cache_pays_for_the_widest_call_mlx_does_not_fuse(monkeypatch, fuses, cells):+ n_ctx = 33792+ floor = mm._compute_bytes(_BONSAI[:3], 2, 2048, [_GROWING], n_ctx, None)+ asked = _unfused_route(monkeypatch, fuses)+ priced = mm._compute_bytes(_BONSAI, 2, 2048, [_GROWING], n_ctx, None)+ assert priced - floor == cells * _SCORE+ assert (2048, 16 * 2048) in asked+++def test_a_tail_unfused_past_any_fixed_scan_is_still_priced(monkeypatch):+ # NAX d512 with few heads: tails fuse only past 8160 queries, longer than a fixed scan reaches.+ _unfused_route(monkeypatch, lambda L, S: L <= 8 or L > 8160)+ floor = mm._compute_bytes(_BONSAI[:3], 2, 16384, [_GROWING], 32768, None)+ priced = mm._compute_bytes(_BONSAI, 2, 16384, [_GROWING], 32768, None)+ assert priced - floor == 8160 * (16384 + 8160) * _SCORE+++def test_the_scores_charged_never_fall_as_the_context_grows(monkeypatch):+ _unfused_route(monkeypatch, lambda L, S: L <= 8 or L == S >= 2048)+ steps = range(256, 12_289, 256)+ priced = [mm._compute_bytes(_BONSAI, 2, 2048, [_GROWING], n, None) for n in steps]+ assert priced == sorted(priced)+++def test_a_windowed_model_and_an_unreachable_zoo_keep_the_old_price(monkeypatch):+ widths = _BONSAI+ floor = mm._compute_bytes(widths[:3], 2, 2048, [_GROWING], 262144, None)+ asked = _unfused_route(monkeypatch, lambda L, S: False)+ windowed = [_GROWING, _plan_entry(bound_spec = (object, "max_size", 1024))]+ assert mm._compute_bytes(widths, 2, 2048, windowed, 262144, None) == floor+ assert asked == []+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.attention", None)+ assert mm._compute_bytes(widths, 2, 2048, [_GROWING], 262144, None) == floor++@_NEEDS_MLXclass TestTheQuantizedAttentionRouteIsTheOneZooTakes:"""unsloth-zoo routes each call between a score matrix and a dequantized copy of the cache."""@@ -4445,17 +4525,30 @@ class TestTheQuantizedAttentionRouteIsTheOneZooTakes:row = mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048)floor = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, None)priced = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, 2048, 4, 64)- scores = mm._widest_quantized_scores(32768, 2048, 2048) * 32 * mm._QUANT_SCORE_DTYPE_SIZE+ scores = mm._widest_quantized_scores(32768, 2048, 2048) * 32 * 2 * mm._SCORE_LIVEassert priced - floor == row * 32768 < scoresdef test_the_decode_step_still_scores_where_that_is_the_wider_of_the_two(self):- widths = (3584, 18944, 128, 1, 64, 0)+ widths = (3584, 18944, 256, 1, 64, 0)row = mm._dequantized_row_bytes(widths, 2, 8, 64, 2048)- floor = mm._compute_bytes(widths, 2, 2048, [], 32768, None)+ floor = mm._compute_bytes(widths[:3], 2, 2048, [], 32768, None)charged = mm._compute_bytes(widths, 2, 2048, [], 32768, 2048, 8, 64) - floor- assert row is not None and charged == (32769) * 128 * mm._QUANT_SCORE_DTYPE_SIZE+ assert row is not None and charged == (32769) * 256 * 2 * mm._SCORE_LIVEassert charged > row * 32768+ def test_a_full_width_cache_is_charged_the_scores_mlx_builds_below_nax(self):+ from unsloth_zoo.mlx.nax import nax_available++ if nax_available():+ pytest.skip("NAX fuses head_dim 256 prefill")+ plan = [_GROWING] * 4+ unrouted = mm._compute_bytes(self.GQA[:3], 2, 2048, plan, 32768, None)+ assert mm._compute_bytes(self.GQA, 2, 2048, plan, 32768, None) == unrouted+ wide = mm._compute_bytes(self.WIDE, 2, 2048, plan, 32768, None)+ assert (+ wide - unrouted == mm._widest_quantized_scores(32768, 0, 2048) * 16 * 2 * mm._SCORE_LIVE+ )+def test_a_zoo_that_cannot_route_it_keeps_the_scores_whole(self, monkeypatch):import builtins@@ -4471,7 +4564,7 @@ class TestTheQuantizedAttentionRouteIsTheOneZooTakes:floor = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, None)priced = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, 2048, 4, 64)assert priced - floor == (- mm._widest_quantized_scores(32768, 2048, 2048) * 32 * mm._QUANT_SCORE_DTYPE_SIZE+ mm._widest_quantized_scores(32768, 2048, 2048) * 32 * 2 * mm._SCORE_LIVE)@pytest.mark.parametrize(@@ -5140,6 +5233,12 @@ class TestShardsTheLoaderReads:mm.mlx_shard_files(str(tmp_path), self._VISION)+# mlx 0.32.4 leaves head_dim 256 prefill unfused below NAX, and tails under 1024 queries on NAX.+_UNFUSED_HEAD_DIM_256_SCORES = {+ "unsloth/Qwen3.6-35B-A3B-UD-MLX-4bit": {True: 1_298_923_560, False: 2_682_961_960}+}++@_NEEDS_MLX@pytest.mark.parametrize("repo, n_ctx, kv_bits, weights, kv, compute, layers, caption",@@ -5231,7 +5330,7 @@ class TestShardsTheLoaderReads:None,5_298_715_127,249_561_088,- 1_281_737_359,+ 1_282_888_719,28,"bf16",),@@ -5241,7 +5340,7 @@ class TestShardsTheLoaderReads:None,2_061_600,98_304,- 688_341_647,+ 690_952_857,2,"bf16",),@@ -5256,7 +5355,10 @@ def test_a_real_checkpoint_is_priced_to_the_byte()assert breakdown is not Noneassert (breakdown.weights_bytes, breakdown.kv_bytes) == (weights, kv)- assert breakdown.compute_bytes == compute+ from unsloth_zoo.mlx.nax import nax_available++ unfused = _UNFUSED_HEAD_DIM_256_SCORES.get(repo, {}).get(nax_available(), 0)+ assert breakdown.compute_bytes == compute + unfusedassert breakdown.layer_count == layersassert breakdown.cache_type_kv == caption