Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Needle 3 MLX forward pass (#149)

Fernando Olléef3cf7543204
diff --git a/needle/model/architecture_mlx.py b/needle/model/architecture_mlx.py
new file mode 100644
index 0000000..c174689
--- /dev/null
+++ b/needle/model/architecture_mlx.py
@@ -0,0 +1,308 @@
+import functools
+import math
+
+import numpy as np
+import mlx.core as mx
+
+from .architecture import (
+ ENGRAM_CONV_TAPS, HEAD_KEYS, _ENGRAM_PRIME, _ENGRAM_SEED, _hada_perms,
+ engram_geometry, head_dims, ladder_layer_ranks,
+)
+from .checkpoints import flatten
+from .quantize_mlx import Numerics
+
+MX_DTYPES = {"float32": mx.float32, "bfloat16": mx.bfloat16, "float16": mx.float16}
+BLOCK = "stack/layers/block/"
+
+
+def to_mlx(params):
+ """A checkpoint param tree as the flat {"a/b/c": array} dict the MLX forward reads."""
+ return {k: mx.array(np.asarray(v, np.float32)) for k, v in flatten(params).items()
+ if k.split("/", 1)[0] not in HEAD_KEYS}
+
+
+def make_causal_mask(seq_len):
+ return mx.tril(mx.ones((seq_len, seq_len), dtype=mx.bool_))[None, None]
+
+
+def _shift_right(x, offset):
+ if offset == 0:
+ return x
+ pad = [(0, 0)] * x.ndim
+ pad[1] = (offset, 0)
+ return mx.pad(x, pad)[:, : x.shape[1]]
+
+
+def _mask_diag(mask, offset):
+ m = mask[:, 0]
+ T = m.shape[-1]
+ if offset >= T:
+ return mx.zeros(m.shape[:-2] + (T,), m.dtype)
+ d = mx.diagonal(m, offset=-offset, axis1=-2, axis2=-1)
+ if offset == 0:
+ return d
+ return mx.pad(d, [(0, 0), (offset, 0)])
+
+
+def _rms_unit(x, epsilon=1e-6):
+ xf = x.astype(mx.float32)
+ return xf * mx.rsqrt(mx.mean(xf ** 2, axis=-1, keepdims=True) + epsilon)
+
+
+def zc_rms_norm(x, scale, dtype, epsilon=1e-6):
+ rms = mx.sqrt(mx.mean(x.astype(mx.float32) ** 2, axis=-1, keepdims=True) + epsilon)
+ return ((1 + scale) * x / rms).astype(dtype)
+
+
+def _sinkhorn(logits, iters=20):
+ log_K = logits
+ for _ in range(iters):
+ log_K = log_K - mx.logsumexp(log_K, axis=-1, keepdims=True)
+ log_K = log_K - mx.logsumexp(log_K, axis=-2, keepdims=True)
+ return mx.exp(log_K)
+
+
+def _dense(x, kernel, dtype):
+ return x.astype(dtype) @ kernel.astype(dtype)
+
+
+def precompute_rope_freqs(head_dim, seq_len, theta=10000.0):
+ freqs = 1.0 / (theta ** (mx.arange(0, head_dim, 2).astype(mx.float32) / head_dim))
+ angles = mx.arange(seq_len).astype(mx.float32)[:, None] * freqs[None, :]
+ return mx.cos(angles), mx.sin(angles)
+
+
+def apply_rope(x, cos, sin):
+ T = x.shape[2]
+ half = x.shape[-1] // 2
+ cos = cos[:T][None, None, :, :]
+ sin = sin[:T][None, None, :, :]
+ x1, x2 = x[..., :half], x[..., half:]
+ return mx.concatenate([x1 * cos - x2 * sin, x2 * cos + x1 * sin], axis=-1).astype(x.dtype)
+
+
+def engram_indices(tokens, orders, heads, slots, seed_heads=0):
+ u = tokens.astype(mx.uint32)
+ stride = seed_heads or heads
+ prime = mx.array(_ENGRAM_PRIME, mx.uint32)
+ idx = []
+ for oi, order in enumerate(orders):
+ for h in range(heads):
+ seed = (_ENGRAM_SEED * (oi * stride + h + 1)) & 0xFFFFFFFF
+ acc = mx.full(u.shape, seed, mx.uint32)
+ for j in range(order):
+ acc = (acc ^ _shift_right(u, j)) * prime
+ acc = acc ^ (acc >> mx.array(15, mx.uint32))
+ idx.append((acc % mx.array(slots, mx.uint32)).astype(mx.int32))
+ return mx.stack(idx, axis=-1)
+
+
+def engram_kv(p, cfg, tokens, mask, num, dtype):
+ if not cfg.engram_layers:
+ return None
+ orders, heads, _ = engram_geometry(cfg)
+ dilation = max(orders)
+ indices = engram_indices(tokens, orders, heads, cfg.engram_slots,
+ getattr(cfg, "engram_seed_heads", 0))
+ ngram_ok = mx.stack([_mask_diag(mask, o - 1) for o in orders for _ in range(heads)], axis=-1)
+ tap_ok = [_mask_diag(mask, j * dilation) for j in range(ENGRAM_CONV_TAPS)]
+ ks, vs = [], []
+ for site in range(len(cfg.engram_layers)):
+ name = f"engrams_{site}/"
+ tables = p[name + "embedding"]
+ num_tables, slots, sub_dim = tables.shape
+ flat = indices + mx.arange(num_tables, dtype=mx.int32) * slots
+ fetched = tables.reshape(num_tables * slots, sub_dim)[flat] * ngram_ok[..., None]
+ e = num.act(fetched.reshape(*indices.shape[:2], num_tables * sub_dim).astype(dtype))
+ k = _dense(e, p[name + "key_proj/kernel"], dtype)
+ v = _dense(e, p[name + "value_proj/kernel"], dtype)
+ taps = p[name + "taps"].astype(dtype)
+ v = sum(taps[j] * _shift_right(v, j * dilation) * tap_ok[j][..., None]
+ for j in range(ENGRAM_CONV_TAPS))
+ ks.append(k)
+ vs.append(v)
+ return ks, vs
+
+
+def attention(lp, cfg, x, mask, rope, num, dtype):
+ qk_hd, v_hd = head_dims(cfg)
+ H, KV = cfg.num_heads, cfg.num_kv_heads
+ B, T, _ = x.shape
+ x = num.act(x)
+ q = _dense(x, lp["self_attn/q_proj/kernel"], dtype)
+ k = _dense(x, lp["self_attn/k_proj/kernel"], dtype)
+ v = _dense(x, lp["self_attn/v_proj/kernel"], dtype)
+
+ n_taps = getattr(cfg, "qkv_conv_taps", 0)
+ if n_taps:
+ tap_ok = [None] + [_mask_diag(mask, j)[..., None].astype(dtype) for j in range(1, n_taps)]
+
+ def conv(z, taps):
+ taps = taps.astype(dtype)
+ return sum(taps[j] * (z if j == 0 else _shift_right(z, j) * tap_ok[j])
+ for j in range(n_taps))
+
+ q = conv(q, lp["self_attn/q_taps"])
+ k = conv(k, lp["self_attn/k_taps"])
+ v = conv(v, lp["self_attn/v_taps"])
+
+ q = q.reshape(B, T, H, qk_hd).transpose(0, 2, 1, 3)
+ k = k.reshape(B, T, KV, qk_hd).transpose(0, 2, 1, 3)
+ v = v.reshape(B, T, KV, v_hd).transpose(0, 2, 1, 3)
+ q = zc_rms_norm(q, lp["self_attn/q_norm/scale"], dtype)
+ k = zc_rms_norm(k, lp["self_attn/k_norm/scale"], dtype)
+ cos, sin = rope
+ q, k = apply_rope(q, cos, sin), apply_rope(k, cos, sin)
+ q, k, v = num.query(q), num.kv(k), num.kv(v)
+
+ out = mx.fast.scaled_dot_product_attention(q, k, v, scale=1.0 / math.sqrt(qk_hd), mask=mask)
+ out = out.transpose(0, 2, 1, 3).reshape(B, T, H * v_hd)
+ out = out * mx.sigmoid(_dense(x, lp["self_attn/gate_proj/kernel"], dtype))
+ return _dense(num.act(out), lp["self_attn/out_proj/kernel"], dtype)
+
+
+def _kron_apply(z, a, b):
+ lead = z.shape[:-1]
+ z = z.reshape(*lead, a.shape[0], b.shape[0])
+ return (a.T @ z @ b).reshape(*lead, a.shape[0] * b.shape[0])
+
+
[email protected]_cache(maxsize=None)
+def _perms(n, split):
+ return tuple(mx.array(np.asarray(p)) for p in _hada_perms(n, split))
+
+
+def _silu(x):
+ return x * mx.sigmoid(x)
+
+
+def hadamard_mlp(lp, cfg, x, dtype):
+ d_model = cfg.d_model
+ n = 1 << (d_model - 1).bit_length()
+ p1, p2 = _perms(n, bool(getattr(cfg, "ladder_widths", ())))
+ w = {k: lp["hadamard_mlp/" + k].astype(dtype)
+ for k in ("w1a", "w1b", "w2a", "w2b", "w3a", "w3b",
+ "d1", "d2", "b2", "d3", "d4", "cond_v", "cond_u")}
+ cond = 1 + mx.softmax(x @ w["cond_v"], axis=-1) @ w["cond_u"]
+ pad = n - d_model
+ z = mx.pad(x, [(0, 0), (0, 0), (0, pad)]) if pad else x
+ z = _kron_apply(w["d1"] * z, w["w1a"], w["w1b"])[..., p1]
+ z = _kron_apply(_silu(w["d2"] * cond * z + w["b2"]), w["w2a"], w["w2b"])[..., p2]
+ z = _kron_apply(w["d3"] * z, w["w3a"], w["w3b"])
+ return (w["d4"] * z)[..., :d_model]
+
+
+def block(lp, cfg, x, mask, rope, num, dtype, site_kv):
+ if site_kv is not None:
+ ek, ev = site_kv
+ alpha = mx.sigmoid(mx.sum(_rms_unit(x) * _rms_unit(ek), axis=-1) / math.sqrt(cfg.d_model))
+ x = x + (alpha[..., None] * ev.astype(mx.float32)).astype(x.dtype)
+
+ skip = x
+ x = zc_rms_norm(x, lp["ZCRMSNorm_0/scale"], dtype)
+ x = attention(lp, cfg, x, mask, rope, num, dtype)
+ x = zc_rms_norm(x, lp["post_attn_norm/scale"], dtype)
+ x = skip + mx.sigmoid(lp["attn_gate"]).astype(dtype) * x
+
+ skip = x
+ x = zc_rms_norm(x, lp["pre_hada_norm/scale"], dtype)
+ return skip + hadamard_mlp(lp, cfg, x, dtype)
+
+
+def _advance(cfg, num, dtype, stream, lp, pre_off, post_off, mask, rope, site_kv):
+ B, T, n, C = stream.shape
+ xf = stream.astype(mx.float32)
+ nx = num.act(_rms_unit(stream.reshape(B, T, n * C)))
+ hpre = mx.sigmoid(lp["a_pre"] * (nx @ lp["phi_pre"]) + lp["b_pre"] + pre_off)
+ u = mx.sum(hpre[..., None] * xf, axis=2).astype(dtype)
+ y = block(lp, cfg, u, mask, rope, num, dtype, site_kv) - u
+ hpost = 2 * mx.sigmoid(lp["a_post"] * (nx @ lp["phi_post"]) + lp["b_post"] + post_off)
+ hres = _sinkhorn(lp["a_res"] * (nx @ lp["phi_res"]).reshape(B, T, n, n) + lp["b_res"])
+ return (hres @ xf + hpost[..., None] * y.astype(mx.float32)[:, :, None, :]).astype(dtype)
+
+
+def _layer_params(p, layer):
+ lp = {k[len(BLOCK):]: v[layer] for k, v in p.items() if k.startswith(BLOCK)}
+ for name in ("phi_pre", "phi_post", "phi_res", "b_pre", "b_post", "b_res",
+ "a_pre", "a_post", "a_res"):
+ lp[name] = p["stack/mhc_" + name][layer]
+ return lp
+
+
+def _lane_offsets(positions, n):
+ lane = mx.array(np.eye(n, dtype=np.float32)[np.asarray(positions) % n])
+ return 8 * lane - 4, -4 * (1 - lane)
+
+
+def stack(p, cfg, x, mask, rope, ekv, num, dtype, exit_depth=None, subnetwork_only=False):
+ if subnetwork_only and exit_depth is None:
+ raise ValueError("subnetwork_only requires exit_depth")
+ L, n = cfg.num_layers, cfg.mhc_lanes
+ x = x.astype(dtype)
+ local_mask = None
+ if cfg.sliding_window:
+ pos = mx.arange(x.shape[1])
+ local_mask = mask & ((pos[:, None] - pos[None, :]) < cfg.sliding_window)[None, None]
+ sites = {layer: s for s, layer in enumerate(cfg.engram_layers)}
+
+ ranks = ladder_layer_ranks(cfg)
+ pre_off, post_off = _lane_offsets(np.arange(L), n)
+ if exit_depth is not None:
+ active = np.asarray(ranks) < exit_depth
+ sub_pre_off, sub_post_off = _lane_offsets(np.cumsum(active) - 1, n)
+ capture = exit_depth is not None and not subnetwork_only
+
+ x = mx.broadcast_to(x[:, :, None, :], (*x.shape[:2], n, x.shape[-1]))
+ sub_x = x
+ for layer in range(L):
+ lp = _layer_params(p, layer)
+ layer_mask = mask if local_mask is None or layer in cfg.global_layers else local_mask
+ s = sites.get(layer)
+ site_kv = (ekv[0][s], ekv[1][s]) if ekv is not None and s is not None else None
+ step = functools.partial(_advance, cfg, num, dtype)
+ if cfg.remat:
+ step = mx.checkpoint(step)
+ in_sub = exit_depth is not None and ranks[layer] < exit_depth
+ if subnetwork_only:
+ if in_sub:
+ x = step(x, lp, sub_pre_off[layer], sub_post_off[layer], layer_mask, rope, site_kv)
+ continue
+ x = step(x, lp, pre_off[layer], post_off[layer], layer_mask, rope, site_kv)
+ if capture and in_sub:
+ sub_x = step(sub_x, lp, sub_pre_off[layer], sub_post_off[layer], layer_mask, rope,
+ site_kv)
+
+ final = p["stack/final_norm/scale"]
+ x = zc_rms_norm(mx.mean(x, axis=2), final, dtype)
+ if capture:
+ return x, zc_rms_norm(mx.mean(sub_x, axis=2), final, dtype)
+ return x, None
+
+
+def forward(p, cfg, tokens, mask=None, quant=False, exit_depth=None, exit_only=False,
+ subnetwork_only=False):
+ """SimpleAttentionNetwork.__call__ on MLX: logits, or (logits, exit_logits) with exit_depth.
+
+ `p` comes from to_mlx; under quant it must already hold the CQ weights, as the
+ JAX path passes cq_ste_params(...) to model.apply.
+ """
+ dtype = MX_DTYPES[cfg.dtype]
+ num = Numerics(cfg, quant)
+ if mask is None:
+ mask = make_causal_mask(tokens.shape[1])
+ embedding = p["embedding/embedding"]
+ x = embedding[tokens] * math.sqrt(cfg.d_model)
+ rope = precompute_rope_freqs(head_dims(cfg)[0], tokens.shape[1], cfg.rope_theta)
+ ekv = engram_kv(p, cfg, tokens, mask, num, dtype)
+ x, exit_x = stack(p, cfg, x, mask, rope, ekv, num, dtype, exit_depth=exit_depth,
+ subnetwork_only=subnetwork_only)
+ head = embedding[: cfg.out_vocab] if cfg.out_vocab else embedding
+
+ def logits(h):
+ return num.act(h).astype(mx.float32) @ head.T
+
+ if exit_x is None:
+ return logits(x)
+ if exit_only:
+ return logits(exit_x)
+ return logits(x), logits(exit_x)
diff --git a/needle/model/quantize_mlx.py b/needle/model/quantize_mlx.py
new file mode 100644
index 0000000..190497f
--- /dev/null
+++ b/needle/model/quantize_mlx.py
@@ -0,0 +1,91 @@
+import mlx.core as mx
+
+from . import quantize as _quantize
+
+KV_GROUP = 64
+
+
+def fake_quant(w, group_size=128, bits=4):
+ qmax = 2 ** (bits - 1) - 1
+ D = w.shape[-1]
+ pad = (-D) % group_size
+ wp = mx.pad(w, [(0, 0)] * (w.ndim - 1) + [(0, pad)]) if pad else w
+ g = wp.reshape(*wp.shape[:-1], -1, group_size).astype(mx.float32)
+ absmax = mx.max(mx.abs(g), axis=-1, keepdims=True)
+ scale = mx.where(absmax > 0, absmax / qmax, 1.0)
+ q = mx.clip(mx.round(g / scale), -qmax - 1, qmax) * scale
+ q = q.reshape(wp.shape).astype(w.dtype)
+ if pad:
+ q = q[..., :D]
+ return w + mx.stop_gradient(q - w)
+
+
+def _cq_nearest(x, cb):
+ # Counts the codebook steps x is strictly closer to, which is the left-on-tie
+ # choice quantize._cq_nearest makes with searchsorted.
+ idx = mx.zeros(x.shape, mx.int32)
+ for k in range(cb.shape[0] - 1):
+ idx = idx + (mx.abs(x - cb[k + 1]) < mx.abs(x - cb[k])).astype(mx.int32)
+ return cb[idx]
+
+
+def cq_quantize(w, bits, group_size=128):
+ cb = mx.array(_quantize._cq_codebook_np(bits, group_size))
+ D, g = w.shape[-1], group_size
+ pad = (-D) % g
+ wp = mx.pad(w, [(0, 0)] * (w.ndim - 1) + [(0, pad)]) if pad else w
+ groups = wp.reshape(*wp.shape[:-1], -1, g).astype(mx.float32)
+ H = mx.array(_quantize._cq_hadamard_np(g))
+ rot = groups @ H
+ norm = mx.sqrt(mx.sum(rot ** 2, axis=-1, keepdims=True))
+ unit = rot / mx.maximum(norm, 1e-12)
+ norm = norm.astype(mx.float16).astype(mx.float32)
+ deq = (_cq_nearest(unit, cb) * norm) @ H
+ deq = deq.reshape(wp.shape).astype(w.dtype)
+ return deq[..., :D] if pad else deq
+
+
+def cq_ste(w, bits, group_size=_quantize.CQ_GROUP_SIZE):
+ return w + mx.stop_gradient(cq_quantize(w, bits, group_size) - w)
+
+
+def is_quant_leaf(name, leaf):
+ key = name.rsplit("/", 1)[-1]
+ return (key in ("kernel", "embedding") or key.startswith("mhc_phi")) and leaf.ndim >= 2
+
+
+def cq_ste_leaf(name, w, bits):
+ """quantize._map_quant_leaves for one leaf: kernels and mhc_phi reduce along their input axis."""
+ key = name.rsplit("/", 1)[-1]
+ if key == "kernel" or key.startswith("mhc_phi"):
+ return mx.swapaxes(cq_ste(mx.swapaxes(w, -1, -2), bits), -1, -2)
+ return cq_ste(w, bits)
+
+
+def cq_ste_params(params, bits):
+ """quantize.cq_ste_params over the flat dict architecture_mlx.to_mlx returns."""
+ if any(k.split("/", 1)[0] == _quantize.AB_KEY for k in params):
+ raise NotImplementedError("the MLX backend does not support ab_scales checkpoints")
+ return {k: cq_ste_leaf(k, v, bits) if is_quant_leaf(k, v) else v for k, v in params.items()}
+
+
+class Numerics:
+ """The fake-quant a forward applies: none, or the export's A8 activations and KV."""
+
+ def __init__(self, config, quant):
+ self.quant = bool(quant)
+ self.act_bits = int(getattr(config, "act_bits", 8))
+ self.kv_bits = int(getattr(config, "kv_bits", 8))
+
+ def act(self, x):
+ return fake_quant(x, x.shape[-1], self.act_bits) if self.quant else x
+
+ def query(self, x):
+ return fake_quant(x, x.shape[-1], 8) if self.quant and self.act_bits else x
+
+ def kv(self, x):
+ if not (self.quant and self.kv_bits):
+ return x
+ if self.kv_bits >= 8:
+ return fake_quant(x, x.shape[-1], 8)
+ return x + mx.stop_gradient(cq_quantize(x, self.kv_bits, KV_GROUP) - x)
diff --git a/tests/conftest.py b/tests/conftest.py
index 8d8bfad..67165e7 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -124,3 +124,63 @@ def _fake_cact(path, heads):
out += struct.pack(record_fmt, *rec)
out += body
path.write_bytes(bytes(out))
+
+
[email protected](scope="session")
+def needle3_checkpoint():
+ path = os.environ.get("NEEDLE_MLX_PARITY_CHECKPOINT", "checkpoints/needle3.safetensors")
+ if not os.path.exists(path):
+ pytest.skip(f"no Needle 3 checkpoint at {path} (set NEEDLE_MLX_PARITY_CHECKPOINT)")
+ return path
+
+
+def mlx_cpu():
+ """Exact parity with JAX holds on the MLX CPU device; Metal is checked on its own."""
+ import mlx.core as mx
+
+ with mx.stream(mx.cpu):
+ yield
+
+
+def perturbed_model():
+ return _perturbed_model
+
+
+def _perturbed_model(**overrides):
+ """A tiny model with every parameter moved off its init, so identity-initialised
+ paths (conv taps, cond_u, b2, Sinkhorn bias) are exercised."""
+ import numpy as np
+ import jax
+ import jax.numpy as jnp
+ from needle.model.architecture import SimpleAttentionNetwork, TransformerConfig
+ from needle.model.checkpoints import flatten, unflatten
+
+ fields = dict(
+ vocab_size=512, out_vocab=500, d_model=48, num_heads=4, num_kv_heads=2,
+ num_layers=5, qk_head_dim=8, v_head_dim=12, max_seq_len=64, engram_layers=(1, 4),
+ engram_slots=64, global_layers=(2, 4), sliding_window=8, mhc_lanes=4,
+ qkv_conv_taps=3, dtype="float32", flash=False)
+ fields.update(overrides)
+ config = TransformerConfig(**fields)
+ model = SimpleAttentionNetwork(config)
+ params = model.init(jax.random.PRNGKey(0), jnp.ones((1, 8), jnp.int32))["params"]
+ rng = np.random.default_rng(1)
+ params = unflatten({k: (v + 0.05 * rng.standard_normal(v.shape)).astype(np.float32)
+ for k, v in flatten(jax.device_get(params)).items()})
+ tokens = rng.integers(1, config.out_vocab, size=(2, 24)).astype(np.int32)
+ return model, config, params, tokens
+
+
+def deploy_numerics():
+ from needle.model import quantize
+
+ saved = quantize.ACT_BITS, quantize.KV_BITS, quantize._KV_GROUP
+
+ def configure(config):
+ quantize.configure_deploy(act_bits=config.act_bits, kv_bits=config.kv_bits)
+
+ yield configure
+ quantize.configure_deploy(*saved)
diff --git a/tests/test_mlx.py b/tests/test_mlx.py
new file mode 100644
index 0000000..f185787
--- /dev/null
+++ b/tests/test_mlx.py
@@ -0,0 +1,91 @@
+import numpy as np
+import pytest
+
+mx = pytest.importorskip("mlx.core")
+jax = pytest.importorskip("jax")
+
+pytestmark = [pytest.mark.slow, pytest.mark.usefixtures("mlx_cpu")]
+
+
+def _jax_logits(model, params, tokens, **kw):
+ return model.apply({"params": params}, jax.numpy.asarray(tokens), **kw)
+
+
+def _mlx_logits(config, params, tokens, **kw):
+ from needle.model.architecture_mlx import forward, to_mlx
+
+ return forward(to_mlx(params), config, mx.array(tokens), **kw)
+
+
[email protected]("flash", [False, True])
+def test_forward_matches_jax(flash, perturbed_model):
+ model, config, params, tokens = perturbed_model(flash=flash)
+ ref = np.asarray(_jax_logits(model, params, tokens))
+ got = np.asarray(_mlx_logits(config, params, tokens))
+ assert got.shape == ref.shape == (2, 24, 500)
+ np.testing.assert_allclose(got, ref, atol=5e-5)
+
+
+def test_bfloat16_forward_is_as_close_to_float32_as_jax(perturbed_model):
+ model, config, params, tokens = perturbed_model()
+ ref32 = np.asarray(_jax_logits(model, params, tokens))
+ bf16_model, bf16_config, _, _ = perturbed_model(dtype="bfloat16")
+ jax_err = np.abs(np.asarray(_jax_logits(bf16_model, params, tokens)) - ref32).mean()
+ mlx_err = np.abs(np.asarray(_mlx_logits(bf16_config, params, tokens)) - ref32).mean()
+ assert 0 < mlx_err < 2 * jax_err
+
+
+def test_split_hadamard_perms_match_jax(perturbed_model):
+ model, config, params, tokens = perturbed_model(ladder_widths=(24,))
+ np.testing.assert_allclose(np.asarray(_mlx_logits(config, params, tokens)),
+ np.asarray(_jax_logits(model, params, tokens)), atol=5e-5)
+
+
+def test_exit_depth_matches_jax(perturbed_model):
+ model, config, params, tokens = perturbed_model()
+ ref_full, ref_exit = _jax_logits(model, params, tokens, exit_depth=3)
+ got_full, got_exit = _mlx_logits(config, params, tokens, exit_depth=3)
+ np.testing.assert_allclose(np.asarray(got_full), np.asarray(ref_full), atol=5e-5)
+ np.testing.assert_allclose(np.asarray(got_exit), np.asarray(ref_exit), atol=5e-5)
+
+ ref_sub = _jax_logits(model, params, tokens, exit_depth=3, subnetwork_only=True)
+ got_sub = _mlx_logits(config, params, tokens, exit_depth=3, subnetwork_only=True)
+ np.testing.assert_allclose(np.asarray(got_sub), np.asarray(ref_sub), atol=5e-5)
+
+
+def test_padding_mask_matches_jax(perturbed_model):
+ from needle.model.architecture import make_causal_mask, make_padding_mask
+
+ model, config, params, tokens = perturbed_model()
+ tokens[:, -5:] = config.pad_token_id
+ mask = make_causal_mask(tokens.shape[1]) & make_padding_mask(tokens, config.pad_token_id)
+ ref = _jax_logits(model, params, tokens, mask=mask)
+ got = _mlx_logits(config, params, tokens, mask=mx.array(np.asarray(mask)))
+ np.testing.assert_allclose(np.asarray(got), np.asarray(ref), atol=5e-5)
+
+
+def test_published_checkpoint_matches_jax(needle3_checkpoint):
+ from needle.model.architecture import SimpleAttentionNetwork
+ from needle.model.run import load_checkpoint
+
+ params, config = load_checkpoint(needle3_checkpoint)
+ config.dtype = "float32"
+ params = jax.tree.map(lambda a: np.asarray(a, np.float32), params)
+ tokens = np.random.default_rng(0).integers(14, config.vocab_size, (2, 96)).astype(np.int32)
+
+ ref = np.asarray(_jax_logits(SimpleAttentionNetwork(config), params, tokens))
+ got = np.asarray(_mlx_logits(config, params, tokens))
+ np.testing.assert_allclose(got, ref, atol=5e-4)
+ assert (got.argmax(-1) == ref.argmax(-1)).all()
+
+
[email protected](not mx.metal.is_available(), reason="no Metal device")
+def test_metal_forward_agrees_with_jax(perturbed_model):
+ """Metal is not bit-for-bit float32 on every Apple GPU, so the default device gets
+ argmax agreement and a 5e-3 band instead of the CPU tolerance."""
+ model, config, params, tokens = perturbed_model()
+ ref = np.asarray(_jax_logits(model, params, tokens))
+ with mx.stream(mx.gpu):
+ got = np.asarray(_mlx_logits(config, params, tokens))
+ np.testing.assert_allclose(got, ref, atol=5e-3)
+ assert (got.argmax(-1) == ref.argmax(-1)).all()
diff --git a/tests/test_quantize_mlx.py b/tests/test_quantize_mlx.py
new file mode 100644
index 0000000..674c22b
--- /dev/null
+++ b/tests/test_quantize_mlx.py
@@ -0,0 +1,78 @@
+import numpy as np
+import pytest
+
+mx = pytest.importorskip("mlx.core")
+jax = pytest.importorskip("jax")
+
+pytestmark = [pytest.mark.slow, pytest.mark.usefixtures("mlx_cpu")]
+
+
+def _jax_logits(model, params, tokens):
+ from needle.model.quantize import WEIGHT_BITS, cq_ste_params
+
+ return model.apply({"params": cq_ste_params(params, WEIGHT_BITS)},
+ jax.numpy.asarray(tokens), quant=True)
+
+
+def _mlx_logits(config, params, tokens):
+ from needle.model.architecture_mlx import forward, to_mlx
+ from needle.model.quantize import WEIGHT_BITS
+ from needle.model.quantize_mlx import cq_ste_params
+
+ return forward(cq_ste_params(to_mlx(params), WEIGHT_BITS), config, mx.array(tokens),
+ quant=True)
+
+
[email protected]("bits", [2, 4])
+def test_cq_quantize_matches_jax(bits):
+ from needle.model import quantize, quantize_mlx
+
+ w = np.random.default_rng(bits).standard_normal((3, 200, 96)).astype(np.float32)
+ ref = np.asarray(quantize.cq_quantize(jax.numpy.asarray(w), bits))
+ got = np.asarray(quantize_mlx.cq_quantize(mx.array(w), bits))
+ assert np.mean(np.abs(got - ref) > 1e-5) < 1e-3
+ np.testing.assert_allclose(np.median(np.abs(got - ref)), 0, atol=1e-6)
+
+
[email protected]("flash", [False, True])
[email protected]("kv_bits", [8, 4])
+def test_quantized_forward_matches_jax(flash, kv_bits, deploy_numerics, perturbed_model):
+ """An ulp-level difference can move a value across a rounding boundary; that shows
+ up as a few whole positions off, never as a drift across all of them."""
+ model, config, params, tokens = perturbed_model(flash=flash, kv_bits=kv_bits)
+ deploy_numerics(config)
+ ref = np.asarray(_jax_logits(model, params, tokens))
+ got = np.asarray(_mlx_logits(config, params, tokens))
+ exact = np.all(np.abs(got - ref) <= 5e-5, axis=-1)
+ assert exact.mean() >= 0.9
+ assert np.abs(got - ref).max() < 1e-2
+
+
+def test_published_checkpoint_quantized_loss_matches_jax(needle3_checkpoint, deploy_numerics):
+ """Under QAT the published model flips on ulp-level differences (JAX against itself
+ with 1e-7 weight noise agrees on ~96% of argmaxes), so it is compared by loss."""
+ import optax
+ from needle.model.architecture import SimpleAttentionNetwork
+ from needle.model.run import load_checkpoint
+
+ params, config = load_checkpoint(needle3_checkpoint)
+ config.dtype = "float32"
+ params = jax.tree.map(lambda a: np.asarray(a, np.float32), params)
+ deploy_numerics(config)
+ tokens = np.random.default_rng(0).integers(14, config.vocab_size, (2, 96)).astype(np.int32)
+
+ def loss(logits):
+ return float(optax.softmax_cross_entropy_with_integer_labels(
+ np.asarray(logits)[:, :-1], tokens[:, 1:]).mean())
+
+ ref = _jax_logits(SimpleAttentionNetwork(config), params, tokens)
+ got = _mlx_logits(config, params, tokens)
+ assert abs(loss(got) - loss(ref)) < 1e-2 * loss(ref)
+ assert (np.asarray(got).argmax(-1) == np.asarray(ref).argmax(-1)).mean() > 0.9
+
+
+def test_ab_scales_checkpoints_are_rejected():
+ from needle.model.quantize_mlx import cq_ste_params
+
+ with pytest.raises(NotImplementedError):
+ cq_ste_params({"ab_scales/stack/a": mx.zeros((2, 2))}, 4)