Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio MLX estimate: price attention scores where mlx really holds them, at the width it holds them (#16021)

Daniel Han3ec3350edcc3
diff --git a/studio/backend/core/inference/mlx_memory.py b/studio/backend/core/inference/mlx_memory.py
index 45cdbfd8bc1..b02b3565459 100644
--- a/studio/backend/core/inference/mlx_memory.py
+++ b/studio/backend/core/inference/mlx_memory.py
@@ -51,9 +51,8 @@ _COMPUTE_BASE_BYTES = int(0.64 * 1024**3)
_RECURRENT_DTYPE_SIZE = 4
-_QUANT_SCORE_DTYPE_SIZE = 4
-
-_QUANT_SCORE_LIVE = 1.0
+# Score copies an unfused call holds at the queries' width; Metal measured 1.03 to 1.15 (mlx 0.32.4).
+_SCORE_LIVE = 1.25
_WIDTH_NAMES = {"float64": "f64", "float32": "f32", "bfloat16": "bf16", "float16": "f16"}
@@ -995,15 +994,104 @@ def _compute_bytes(
recurrent_width = sum(int(entry["conv_width"]) for entry in plan) * _RECURRENT_DTYPE_SIZE
total = _COMPUTE_BASE_BYTES + prefill_chunk * (attention_width + recurrent_width)
if quant_boundary is not None:
- score = heads * _QUANT_SCORE_DTYPE_SIZE * _QUANT_SCORE_LIVE
+ score = heads * dtype_size * _SCORE_LIVE
row = _dequantized_row_bytes(widths, dtype_size, kv_bits, kv_group_size, prefill_chunk)
if row is None:
total += _widest_quantized_scores(n_ctx, quant_boundary, prefill_chunk) * score
else:
total += max(row * n_ctx, (n_ctx + 1) * score)
+ elif not any(entry.get("bound_spec") for entry in plan):
+ # Windowed models are skipped: the config's head_dim need not be the full-context layers' width.
+ score = heads * dtype_size * _SCORE_LIVE
+ total += _widest_unfused_scores(widths, dtype_size, n_ctx, prefill_chunk) * score
return int(total)
+def _widest_unfused_scores(widths, dtype_size: int, n_ctx: int, prefill_chunk: int) -> int:
+ """Query x key cells of the widest call mlx leaves unfused over every prompt the context admits."""
+ heads, kv_heads, head_dim = (*widths[2:], 0, 0, 0)[:3]
+ route = _zoo_route() if heads and kv_heads and head_dim else None
+ if route is None:
+ return 0
+ widths, chunk = tuple(widths), prefill_chunk
+
+ def unfused(queries, keys):
+ return not _attention_fuses(route, widths, dtype_size, queries, keys)
+
+ def longest_tail(whole, limit):
+ # The tail of a prompt holding no whole chunk is its first call, so it is square.
+ short, first_fused = _unfused_tails(route, widths, dtype_size, chunk if whole else 0)
+ if limit > 8 and first_fused > 9:
+ return min(limit, first_fused - 1)
+ return max((r for r in short if r <= limit), default = 0)
+
+ whole, rest = divmod(max(n_ctx - 1, 0), chunk)
+ cells = [n_ctx + 1 if unfused(1, 2) or unfused(1, n_ctx + 1) else 0]
+ if whole and unfused(chunk, chunk):
+ cells.append(chunk * chunk)
+ if whole >= 2 and unfused(chunk, whole * chunk):
+ cells.append(chunk * whole * chunk)
+ r = longest_tail(whole, rest)
+ cells.append(r * (whole * chunk + r))
+ if whole:
+ r = longest_tail(whole - 1, chunk - 1)
+ cells.append(r * ((whole - 1) * chunk + r))
+ return max(cells)
+
+
+# Past any context: a shape still unfused here never fuses.
+_TAIL_MAX = 1 << 24
+
+
[email protected]_cache(maxsize = 64)
+def _unfused_tails(route, widths, dtype_size: int, later: int):
+ """Unfused tails of up to 8 queries, and the first longer one that fuses.
+
+ Past 8 queries mlx's kernels only switch on as queries grow, so one boundary splits the two."""
+
+ def unfused(r):
+ return not _attention_fuses(route, widths, dtype_size, r, r + later)
+
+ short = tuple(r for r in range(1, 9) if unfused(r))
+ lo, hi = 8, 9
+ while hi < _TAIL_MAX and unfused(hi):
+ lo, hi = hi, min(hi * 2, _TAIL_MAX)
+ while hi - lo > 1:
+ mid = (lo + hi) // 2
+ lo, hi = (mid, hi) if unfused(mid) else (lo, mid)
+ return short, hi
+
+
+def _zoo_route():
+ try:
+ import mlx.core # noqa: F401
+
+ from unsloth_zoo.mlx.attention import fused_kernel_exists
+ except Exception:
+ return None
+ return fused_kernel_exists
+
+
+def _attention_fuses(route, widths, dtype_size: int, queries: int, keys: int) -> bool:
+ """Whether mlx fuses a full-width causal call; elsewhere it materializes the scores."""
+ import mlx.core as mx
+
+ _heads, kv_heads, head_dim, value_dim = (*widths[2:], 0, 0, 0, 0)[:4]
+ try:
+ dtype = mx.float32 if dtype_size == 4 else mx.bfloat16
+
+ # zoo reads only shapes and dtypes; a group of one makes the scales the cache's width.
+ def cache(width):
+ scales = mx.zeros((1, kv_heads, keys, width), dtype = dtype)
+ return scales, scales, scales
+
+ q = mx.zeros((1, widths[2], queries, head_dim), dtype = dtype)
+ return bool(route(q, cache(head_dim), cache(value_dim or head_dim), 1, "causal"))
+ except Exception as exc:
+ logger.debug("MLX estimate cannot route the attention: %s", exc)
+ return True
+
+
def _load_is_refused(model_dir: str, config: dict, load_in_4bit: bool) -> Optional[str]:
try:
from unsloth_zoo.mlx.loader import (
diff --git a/studio/backend/tests/test_memory_estimate.py b/studio/backend/tests/test_memory_estimate.py
index 4518979433d..5f041882416 100644
--- a/studio/backend/tests/test_memory_estimate.py
+++ b/studio/backend/tests/test_memory_estimate.py
@@ -4409,6 +4409,86 @@ class TestComputeBytes:
)
+def _unfused_route(monkeypatch, fuses):
+ """mlx and zoo stubs, so the unquantized route runs where MLX does not."""
+ asked = []
+ mx = SimpleNamespace(
+ float32 = SimpleNamespace(size = 4),
+ bfloat16 = SimpleNamespace(size = 2),
+ zeros = lambda shape, dtype: SimpleNamespace(shape = shape, dtype = dtype),
+ )
+
+ def fused_kernel_exists(
+ queries,
+ q_keys,
+ q_values,
+ group_size,
+ mask = None,
+ ):
+ L, S = queries.shape[-2], q_keys[0].shape[-2]
+ assert q_keys[1].shape[-1] * group_size == q_values[1].shape[-1] * group_size == 256
+ assert queries.shape[1] == 24 and q_keys[0].shape[1] == 4 and mask == "causal"
+ asked.append((L, S))
+ return fuses(L, S)
+
+ monkeypatch.setitem(sys.modules, "mlx", SimpleNamespace(core = mx))
+ monkeypatch.setitem(sys.modules, "mlx.core", mx)
+ monkeypatch.setitem(
+ sys.modules,
+ "unsloth_zoo.mlx.attention",
+ SimpleNamespace(fused_kernel_exists = fused_kernel_exists),
+ )
+ return asked
+
+
+_BONSAI = (5120, 17408, 24, 4, 256, 0)
+_SCORE = 24 * 2 * mm._SCORE_LIVE
+
+
+ "fuses, cells",
+ [
+ (lambda L, S: True, 0),
+ (lambda L, S: L <= 8, 2048 * 32768),
+ # NAX: the split kernel needs 1024 queries, so only the 1023-token tail is unfused.
+ (lambda L, S: L <= 8 or L >= 1024, 1023 * 33791),
+ ],
+)
+def test_a_full_width_cache_pays_for_the_widest_call_mlx_does_not_fuse(monkeypatch, fuses, cells):
+ n_ctx = 33792
+ floor = mm._compute_bytes(_BONSAI[:3], 2, 2048, [_GROWING], n_ctx, None)
+ asked = _unfused_route(monkeypatch, fuses)
+ priced = mm._compute_bytes(_BONSAI, 2, 2048, [_GROWING], n_ctx, None)
+ assert priced - floor == cells * _SCORE
+ assert (2048, 16 * 2048) in asked
+
+
+def test_a_tail_unfused_past_any_fixed_scan_is_still_priced(monkeypatch):
+ # NAX d512 with few heads: tails fuse only past 8160 queries, longer than a fixed scan reaches.
+ _unfused_route(monkeypatch, lambda L, S: L <= 8 or L > 8160)
+ floor = mm._compute_bytes(_BONSAI[:3], 2, 16384, [_GROWING], 32768, None)
+ priced = mm._compute_bytes(_BONSAI, 2, 16384, [_GROWING], 32768, None)
+ assert priced - floor == 8160 * (16384 + 8160) * _SCORE
+
+
+def test_the_scores_charged_never_fall_as_the_context_grows(monkeypatch):
+ _unfused_route(monkeypatch, lambda L, S: L <= 8 or L == S >= 2048)
+ steps = range(256, 12_289, 256)
+ priced = [mm._compute_bytes(_BONSAI, 2, 2048, [_GROWING], n, None) for n in steps]
+ assert priced == sorted(priced)
+
+
+def test_a_windowed_model_and_an_unreachable_zoo_keep_the_old_price(monkeypatch):
+ widths = _BONSAI
+ floor = mm._compute_bytes(widths[:3], 2, 2048, [_GROWING], 262144, None)
+ asked = _unfused_route(monkeypatch, lambda L, S: False)
+ windowed = [_GROWING, _plan_entry(bound_spec = (object, "max_size", 1024))]
+ assert mm._compute_bytes(widths, 2, 2048, windowed, 262144, None) == floor
+ assert asked == []
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.attention", None)
+ assert mm._compute_bytes(widths, 2, 2048, [_GROWING], 262144, None) == floor
+
+
@_NEEDS_MLX
class TestTheQuantizedAttentionRouteIsTheOneZooTakes:
"""unsloth-zoo routes each call between a score matrix and a dequantized copy of the cache."""
@@ -4445,17 +4525,30 @@ class TestTheQuantizedAttentionRouteIsTheOneZooTakes:
row = mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048)
floor = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, None)
priced = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, 2048, 4, 64)
- scores = mm._widest_quantized_scores(32768, 2048, 2048) * 32 * mm._QUANT_SCORE_DTYPE_SIZE
+ scores = mm._widest_quantized_scores(32768, 2048, 2048) * 32 * 2 * mm._SCORE_LIVE
assert priced - floor == row * 32768 < scores
def test_the_decode_step_still_scores_where_that_is_the_wider_of_the_two(self):
- widths = (3584, 18944, 128, 1, 64, 0)
+ widths = (3584, 18944, 256, 1, 64, 0)
row = mm._dequantized_row_bytes(widths, 2, 8, 64, 2048)
- floor = mm._compute_bytes(widths, 2, 2048, [], 32768, None)
+ floor = mm._compute_bytes(widths[:3], 2, 2048, [], 32768, None)
charged = mm._compute_bytes(widths, 2, 2048, [], 32768, 2048, 8, 64) - floor
- assert row is not None and charged == (32769) * 128 * mm._QUANT_SCORE_DTYPE_SIZE
+ assert row is not None and charged == (32769) * 256 * 2 * mm._SCORE_LIVE
assert charged > row * 32768
+ def test_a_full_width_cache_is_charged_the_scores_mlx_builds_below_nax(self):
+ from unsloth_zoo.mlx.nax import nax_available
+
+ if nax_available():
+ pytest.skip("NAX fuses head_dim 256 prefill")
+ plan = [_GROWING] * 4
+ unrouted = mm._compute_bytes(self.GQA[:3], 2, 2048, plan, 32768, None)
+ assert mm._compute_bytes(self.GQA, 2, 2048, plan, 32768, None) == unrouted
+ wide = mm._compute_bytes(self.WIDE, 2, 2048, plan, 32768, None)
+ assert (
+ wide - unrouted == mm._widest_quantized_scores(32768, 0, 2048) * 16 * 2 * mm._SCORE_LIVE
+ )
+
def test_a_zoo_that_cannot_route_it_keeps_the_scores_whole(self, monkeypatch):
import builtins
@@ -4471,7 +4564,7 @@ class TestTheQuantizedAttentionRouteIsTheOneZooTakes:
floor = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, None)
priced = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, 2048, 4, 64)
assert priced - floor == (
- mm._widest_quantized_scores(32768, 2048, 2048) * 32 * mm._QUANT_SCORE_DTYPE_SIZE
+ mm._widest_quantized_scores(32768, 2048, 2048) * 32 * 2 * mm._SCORE_LIVE
)
@pytest.mark.parametrize(
@@ -5140,6 +5233,12 @@ class TestShardsTheLoaderReads:
mm.mlx_shard_files(str(tmp_path), self._VISION)
+# mlx 0.32.4 leaves head_dim 256 prefill unfused below NAX, and tails under 1024 queries on NAX.
+_UNFUSED_HEAD_DIM_256_SCORES = {
+ "unsloth/Qwen3.6-35B-A3B-UD-MLX-4bit": {True: 1_298_923_560, False: 2_682_961_960}
+}
+
+
@_NEEDS_MLX
@pytest.mark.parametrize(
"repo, n_ctx, kv_bits, weights, kv, compute, layers, caption",
@@ -5231,7 +5330,7 @@ class TestShardsTheLoaderReads:
None,
5_298_715_127,
249_561_088,
- 1_281_737_359,
+ 1_282_888_719,
28,
"bf16",
),
@@ -5241,7 +5340,7 @@ class TestShardsTheLoaderReads:
None,
2_061_600,
98_304,
- 688_341_647,
+ 690_952_857,
2,
"bf16",
),
@@ -5256,7 +5355,10 @@ def test_a_real_checkpoint_is_priced_to_the_byte(
)
assert breakdown is not None
assert (breakdown.weights_bytes, breakdown.kv_bytes) == (weights, kv)
- assert breakdown.compute_bytes == compute
+ from unsloth_zoo.mlx.nax import nax_available
+
+ unfused = _UNFUSED_HEAD_DIM_256_SCORES.get(repo, {}).get(nax_available(), 0)
+ assert breakdown.compute_bytes == compute + unfused
assert breakdown.layer_count == layers
assert breakdown.cache_type_kv == caption