Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Needle 3 MLX forward pass (#149)
Fernando Olléef3cf7543204
diff --git a/needle/model/architecture_mlx.py b/needle/model/architecture_mlx.pynew file mode 100644index 0000000..c174689--- /dev/null+++ b/needle/model/architecture_mlx.py@@ -0,0 +1,308 @@+import functools+import math++import numpy as np+import mlx.core as mx++from .architecture import (+ ENGRAM_CONV_TAPS, HEAD_KEYS, _ENGRAM_PRIME, _ENGRAM_SEED, _hada_perms,+ engram_geometry, head_dims, ladder_layer_ranks,+)+from .checkpoints import flatten+from .quantize_mlx import Numerics++MX_DTYPES = {"float32": mx.float32, "bfloat16": mx.bfloat16, "float16": mx.float16}+BLOCK = "stack/layers/block/"+++def to_mlx(params):+ """A checkpoint param tree as the flat {"a/b/c": array} dict the MLX forward reads."""+ return {k: mx.array(np.asarray(v, np.float32)) for k, v in flatten(params).items()+ if k.split("/", 1)[0] not in HEAD_KEYS}+++def make_causal_mask(seq_len):+ return mx.tril(mx.ones((seq_len, seq_len), dtype=mx.bool_))[None, None]+++def _shift_right(x, offset):+ if offset == 0:+ return x+ pad = [(0, 0)] * x.ndim+ pad[1] = (offset, 0)+ return mx.pad(x, pad)[:, : x.shape[1]]+++def _mask_diag(mask, offset):+ m = mask[:, 0]+ T = m.shape[-1]+ if offset >= T:+ return mx.zeros(m.shape[:-2] + (T,), m.dtype)+ d = mx.diagonal(m, offset=-offset, axis1=-2, axis2=-1)+ if offset == 0:+ return d+ return mx.pad(d, [(0, 0), (offset, 0)])+++def _rms_unit(x, epsilon=1e-6):+ xf = x.astype(mx.float32)+ return xf * mx.rsqrt(mx.mean(xf ** 2, axis=-1, keepdims=True) + epsilon)+++def zc_rms_norm(x, scale, dtype, epsilon=1e-6):+ rms = mx.sqrt(mx.mean(x.astype(mx.float32) ** 2, axis=-1, keepdims=True) + epsilon)+ return ((1 + scale) * x / rms).astype(dtype)+++def _sinkhorn(logits, iters=20):+ log_K = logits+ for _ in range(iters):+ log_K = log_K - mx.logsumexp(log_K, axis=-1, keepdims=True)+ log_K = log_K - mx.logsumexp(log_K, axis=-2, keepdims=True)+ return mx.exp(log_K)+++def _dense(x, kernel, dtype):+ return x.astype(dtype) @ kernel.astype(dtype)+++def precompute_rope_freqs(head_dim, seq_len, theta=10000.0):+ freqs = 1.0 / (theta ** (mx.arange(0, head_dim, 2).astype(mx.float32) / head_dim))+ angles = mx.arange(seq_len).astype(mx.float32)[:, None] * freqs[None, :]+ return mx.cos(angles), mx.sin(angles)+++def apply_rope(x, cos, sin):+ T = x.shape[2]+ half = x.shape[-1] // 2+ cos = cos[:T][None, None, :, :]+ sin = sin[:T][None, None, :, :]+ x1, x2 = x[..., :half], x[..., half:]+ return mx.concatenate([x1 * cos - x2 * sin, x2 * cos + x1 * sin], axis=-1).astype(x.dtype)+++def engram_indices(tokens, orders, heads, slots, seed_heads=0):+ u = tokens.astype(mx.uint32)+ stride = seed_heads or heads+ prime = mx.array(_ENGRAM_PRIME, mx.uint32)+ idx = []+ for oi, order in enumerate(orders):+ for h in range(heads):+ seed = (_ENGRAM_SEED * (oi * stride + h + 1)) & 0xFFFFFFFF+ acc = mx.full(u.shape, seed, mx.uint32)+ for j in range(order):+ acc = (acc ^ _shift_right(u, j)) * prime+ acc = acc ^ (acc >> mx.array(15, mx.uint32))+ idx.append((acc % mx.array(slots, mx.uint32)).astype(mx.int32))+ return mx.stack(idx, axis=-1)+++def engram_kv(p, cfg, tokens, mask, num, dtype):+ if not cfg.engram_layers:+ return None+ orders, heads, _ = engram_geometry(cfg)+ dilation = max(orders)+ indices = engram_indices(tokens, orders, heads, cfg.engram_slots,+ getattr(cfg, "engram_seed_heads", 0))+ ngram_ok = mx.stack([_mask_diag(mask, o - 1) for o in orders for _ in range(heads)], axis=-1)+ tap_ok = [_mask_diag(mask, j * dilation) for j in range(ENGRAM_CONV_TAPS)]+ ks, vs = [], []+ for site in range(len(cfg.engram_layers)):+ name = f"engrams_{site}/"+ tables = p[name + "embedding"]+ num_tables, slots, sub_dim = tables.shape+ flat = indices + mx.arange(num_tables, dtype=mx.int32) * slots+ fetched = tables.reshape(num_tables * slots, sub_dim)[flat] * ngram_ok[..., None]+ e = num.act(fetched.reshape(*indices.shape[:2], num_tables * sub_dim).astype(dtype))+ k = _dense(e, p[name + "key_proj/kernel"], dtype)+ v = _dense(e, p[name + "value_proj/kernel"], dtype)+ taps = p[name + "taps"].astype(dtype)+ v = sum(taps[j] * _shift_right(v, j * dilation) * tap_ok[j][..., None]+ for j in range(ENGRAM_CONV_TAPS))+ ks.append(k)+ vs.append(v)+ return ks, vs+++def attention(lp, cfg, x, mask, rope, num, dtype):+ qk_hd, v_hd = head_dims(cfg)+ H, KV = cfg.num_heads, cfg.num_kv_heads+ B, T, _ = x.shape+ x = num.act(x)+ q = _dense(x, lp["self_attn/q_proj/kernel"], dtype)+ k = _dense(x, lp["self_attn/k_proj/kernel"], dtype)+ v = _dense(x, lp["self_attn/v_proj/kernel"], dtype)++ n_taps = getattr(cfg, "qkv_conv_taps", 0)+ if n_taps:+ tap_ok = [None] + [_mask_diag(mask, j)[..., None].astype(dtype) for j in range(1, n_taps)]++ def conv(z, taps):+ taps = taps.astype(dtype)+ return sum(taps[j] * (z if j == 0 else _shift_right(z, j) * tap_ok[j])+ for j in range(n_taps))++ q = conv(q, lp["self_attn/q_taps"])+ k = conv(k, lp["self_attn/k_taps"])+ v = conv(v, lp["self_attn/v_taps"])++ q = q.reshape(B, T, H, qk_hd).transpose(0, 2, 1, 3)+ k = k.reshape(B, T, KV, qk_hd).transpose(0, 2, 1, 3)+ v = v.reshape(B, T, KV, v_hd).transpose(0, 2, 1, 3)+ q = zc_rms_norm(q, lp["self_attn/q_norm/scale"], dtype)+ k = zc_rms_norm(k, lp["self_attn/k_norm/scale"], dtype)+ cos, sin = rope+ q, k = apply_rope(q, cos, sin), apply_rope(k, cos, sin)+ q, k, v = num.query(q), num.kv(k), num.kv(v)++ out = mx.fast.scaled_dot_product_attention(q, k, v, scale=1.0 / math.sqrt(qk_hd), mask=mask)+ out = out.transpose(0, 2, 1, 3).reshape(B, T, H * v_hd)+ out = out * mx.sigmoid(_dense(x, lp["self_attn/gate_proj/kernel"], dtype))+ return _dense(num.act(out), lp["self_attn/out_proj/kernel"], dtype)+++def _kron_apply(z, a, b):+ lead = z.shape[:-1]+ z = z.reshape(*lead, a.shape[0], b.shape[0])+ return (a.T @ z @ b).reshape(*lead, a.shape[0] * b.shape[0])++[email protected]_cache(maxsize=None)+def _perms(n, split):+ return tuple(mx.array(np.asarray(p)) for p in _hada_perms(n, split))+++def _silu(x):+ return x * mx.sigmoid(x)+++def hadamard_mlp(lp, cfg, x, dtype):+ d_model = cfg.d_model+ n = 1 << (d_model - 1).bit_length()+ p1, p2 = _perms(n, bool(getattr(cfg, "ladder_widths", ())))+ w = {k: lp["hadamard_mlp/" + k].astype(dtype)+ for k in ("w1a", "w1b", "w2a", "w2b", "w3a", "w3b",+ "d1", "d2", "b2", "d3", "d4", "cond_v", "cond_u")}+ cond = 1 + mx.softmax(x @ w["cond_v"], axis=-1) @ w["cond_u"]+ pad = n - d_model+ z = mx.pad(x, [(0, 0), (0, 0), (0, pad)]) if pad else x+ z = _kron_apply(w["d1"] * z, w["w1a"], w["w1b"])[..., p1]+ z = _kron_apply(_silu(w["d2"] * cond * z + w["b2"]), w["w2a"], w["w2b"])[..., p2]+ z = _kron_apply(w["d3"] * z, w["w3a"], w["w3b"])+ return (w["d4"] * z)[..., :d_model]+++def block(lp, cfg, x, mask, rope, num, dtype, site_kv):+ if site_kv is not None:+ ek, ev = site_kv+ alpha = mx.sigmoid(mx.sum(_rms_unit(x) * _rms_unit(ek), axis=-1) / math.sqrt(cfg.d_model))+ x = x + (alpha[..., None] * ev.astype(mx.float32)).astype(x.dtype)++ skip = x+ x = zc_rms_norm(x, lp["ZCRMSNorm_0/scale"], dtype)+ x = attention(lp, cfg, x, mask, rope, num, dtype)+ x = zc_rms_norm(x, lp["post_attn_norm/scale"], dtype)+ x = skip + mx.sigmoid(lp["attn_gate"]).astype(dtype) * x++ skip = x+ x = zc_rms_norm(x, lp["pre_hada_norm/scale"], dtype)+ return skip + hadamard_mlp(lp, cfg, x, dtype)+++def _advance(cfg, num, dtype, stream, lp, pre_off, post_off, mask, rope, site_kv):+ B, T, n, C = stream.shape+ xf = stream.astype(mx.float32)+ nx = num.act(_rms_unit(stream.reshape(B, T, n * C)))+ hpre = mx.sigmoid(lp["a_pre"] * (nx @ lp["phi_pre"]) + lp["b_pre"] + pre_off)+ u = mx.sum(hpre[..., None] * xf, axis=2).astype(dtype)+ y = block(lp, cfg, u, mask, rope, num, dtype, site_kv) - u+ hpost = 2 * mx.sigmoid(lp["a_post"] * (nx @ lp["phi_post"]) + lp["b_post"] + post_off)+ hres = _sinkhorn(lp["a_res"] * (nx @ lp["phi_res"]).reshape(B, T, n, n) + lp["b_res"])+ return (hres @ xf + hpost[..., None] * y.astype(mx.float32)[:, :, None, :]).astype(dtype)+++def _layer_params(p, layer):+ lp = {k[len(BLOCK):]: v[layer] for k, v in p.items() if k.startswith(BLOCK)}+ for name in ("phi_pre", "phi_post", "phi_res", "b_pre", "b_post", "b_res",+ "a_pre", "a_post", "a_res"):+ lp[name] = p["stack/mhc_" + name][layer]+ return lp+++def _lane_offsets(positions, n):+ lane = mx.array(np.eye(n, dtype=np.float32)[np.asarray(positions) % n])+ return 8 * lane - 4, -4 * (1 - lane)+++def stack(p, cfg, x, mask, rope, ekv, num, dtype, exit_depth=None, subnetwork_only=False):+ if subnetwork_only and exit_depth is None:+ raise ValueError("subnetwork_only requires exit_depth")+ L, n = cfg.num_layers, cfg.mhc_lanes+ x = x.astype(dtype)+ local_mask = None+ if cfg.sliding_window:+ pos = mx.arange(x.shape[1])+ local_mask = mask & ((pos[:, None] - pos[None, :]) < cfg.sliding_window)[None, None]+ sites = {layer: s for s, layer in enumerate(cfg.engram_layers)}++ ranks = ladder_layer_ranks(cfg)+ pre_off, post_off = _lane_offsets(np.arange(L), n)+ if exit_depth is not None:+ active = np.asarray(ranks) < exit_depth+ sub_pre_off, sub_post_off = _lane_offsets(np.cumsum(active) - 1, n)+ capture = exit_depth is not None and not subnetwork_only++ x = mx.broadcast_to(x[:, :, None, :], (*x.shape[:2], n, x.shape[-1]))+ sub_x = x+ for layer in range(L):+ lp = _layer_params(p, layer)+ layer_mask = mask if local_mask is None or layer in cfg.global_layers else local_mask+ s = sites.get(layer)+ site_kv = (ekv[0][s], ekv[1][s]) if ekv is not None and s is not None else None+ step = functools.partial(_advance, cfg, num, dtype)+ if cfg.remat:+ step = mx.checkpoint(step)+ in_sub = exit_depth is not None and ranks[layer] < exit_depth+ if subnetwork_only:+ if in_sub:+ x = step(x, lp, sub_pre_off[layer], sub_post_off[layer], layer_mask, rope, site_kv)+ continue+ x = step(x, lp, pre_off[layer], post_off[layer], layer_mask, rope, site_kv)+ if capture and in_sub:+ sub_x = step(sub_x, lp, sub_pre_off[layer], sub_post_off[layer], layer_mask, rope,+ site_kv)++ final = p["stack/final_norm/scale"]+ x = zc_rms_norm(mx.mean(x, axis=2), final, dtype)+ if capture:+ return x, zc_rms_norm(mx.mean(sub_x, axis=2), final, dtype)+ return x, None+++def forward(p, cfg, tokens, mask=None, quant=False, exit_depth=None, exit_only=False,+ subnetwork_only=False):+ """SimpleAttentionNetwork.__call__ on MLX: logits, or (logits, exit_logits) with exit_depth.++ `p` comes from to_mlx; under quant it must already hold the CQ weights, as the+ JAX path passes cq_ste_params(...) to model.apply.+ """+ dtype = MX_DTYPES[cfg.dtype]+ num = Numerics(cfg, quant)+ if mask is None:+ mask = make_causal_mask(tokens.shape[1])+ embedding = p["embedding/embedding"]+ x = embedding[tokens] * math.sqrt(cfg.d_model)+ rope = precompute_rope_freqs(head_dims(cfg)[0], tokens.shape[1], cfg.rope_theta)+ ekv = engram_kv(p, cfg, tokens, mask, num, dtype)+ x, exit_x = stack(p, cfg, x, mask, rope, ekv, num, dtype, exit_depth=exit_depth,+ subnetwork_only=subnetwork_only)+ head = embedding[: cfg.out_vocab] if cfg.out_vocab else embedding++ def logits(h):+ return num.act(h).astype(mx.float32) @ head.T++ if exit_x is None:+ return logits(x)+ if exit_only:+ return logits(exit_x)+ return logits(x), logits(exit_x)diff --git a/needle/model/quantize_mlx.py b/needle/model/quantize_mlx.pynew file mode 100644index 0000000..190497f--- /dev/null+++ b/needle/model/quantize_mlx.py@@ -0,0 +1,91 @@+import mlx.core as mx++from . import quantize as _quantize++KV_GROUP = 64+++def fake_quant(w, group_size=128, bits=4):+ qmax = 2 ** (bits - 1) - 1+ D = w.shape[-1]+ pad = (-D) % group_size+ wp = mx.pad(w, [(0, 0)] * (w.ndim - 1) + [(0, pad)]) if pad else w+ g = wp.reshape(*wp.shape[:-1], -1, group_size).astype(mx.float32)+ absmax = mx.max(mx.abs(g), axis=-1, keepdims=True)+ scale = mx.where(absmax > 0, absmax / qmax, 1.0)+ q = mx.clip(mx.round(g / scale), -qmax - 1, qmax) * scale+ q = q.reshape(wp.shape).astype(w.dtype)+ if pad:+ q = q[..., :D]+ return w + mx.stop_gradient(q - w)+++def _cq_nearest(x, cb):+ # Counts the codebook steps x is strictly closer to, which is the left-on-tie+ # choice quantize._cq_nearest makes with searchsorted.+ idx = mx.zeros(x.shape, mx.int32)+ for k in range(cb.shape[0] - 1):+ idx = idx + (mx.abs(x - cb[k + 1]) < mx.abs(x - cb[k])).astype(mx.int32)+ return cb[idx]+++def cq_quantize(w, bits, group_size=128):+ cb = mx.array(_quantize._cq_codebook_np(bits, group_size))+ D, g = w.shape[-1], group_size+ pad = (-D) % g+ wp = mx.pad(w, [(0, 0)] * (w.ndim - 1) + [(0, pad)]) if pad else w+ groups = wp.reshape(*wp.shape[:-1], -1, g).astype(mx.float32)+ H = mx.array(_quantize._cq_hadamard_np(g))+ rot = groups @ H+ norm = mx.sqrt(mx.sum(rot ** 2, axis=-1, keepdims=True))+ unit = rot / mx.maximum(norm, 1e-12)+ norm = norm.astype(mx.float16).astype(mx.float32)+ deq = (_cq_nearest(unit, cb) * norm) @ H+ deq = deq.reshape(wp.shape).astype(w.dtype)+ return deq[..., :D] if pad else deq+++def cq_ste(w, bits, group_size=_quantize.CQ_GROUP_SIZE):+ return w + mx.stop_gradient(cq_quantize(w, bits, group_size) - w)+++def is_quant_leaf(name, leaf):+ key = name.rsplit("/", 1)[-1]+ return (key in ("kernel", "embedding") or key.startswith("mhc_phi")) and leaf.ndim >= 2+++def cq_ste_leaf(name, w, bits):+ """quantize._map_quant_leaves for one leaf: kernels and mhc_phi reduce along their input axis."""+ key = name.rsplit("/", 1)[-1]+ if key == "kernel" or key.startswith("mhc_phi"):+ return mx.swapaxes(cq_ste(mx.swapaxes(w, -1, -2), bits), -1, -2)+ return cq_ste(w, bits)+++def cq_ste_params(params, bits):+ """quantize.cq_ste_params over the flat dict architecture_mlx.to_mlx returns."""+ if any(k.split("/", 1)[0] == _quantize.AB_KEY for k in params):+ raise NotImplementedError("the MLX backend does not support ab_scales checkpoints")+ return {k: cq_ste_leaf(k, v, bits) if is_quant_leaf(k, v) else v for k, v in params.items()}+++class Numerics:+ """The fake-quant a forward applies: none, or the export's A8 activations and KV."""++ def __init__(self, config, quant):+ self.quant = bool(quant)+ self.act_bits = int(getattr(config, "act_bits", 8))+ self.kv_bits = int(getattr(config, "kv_bits", 8))++ def act(self, x):+ return fake_quant(x, x.shape[-1], self.act_bits) if self.quant else x++ def query(self, x):+ return fake_quant(x, x.shape[-1], 8) if self.quant and self.act_bits else x++ def kv(self, x):+ if not (self.quant and self.kv_bits):+ return x+ if self.kv_bits >= 8:+ return fake_quant(x, x.shape[-1], 8)+ return x + mx.stop_gradient(cq_quantize(x, self.kv_bits, KV_GROUP) - x)diff --git a/tests/conftest.py b/tests/conftest.pyindex 8d8bfad..67165e7 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -124,3 +124,63 @@ def _fake_cact(path, heads):out += struct.pack(record_fmt, *rec)out += bodypath.write_bytes(bytes(out))++[email protected](scope="session")+def needle3_checkpoint():+ path = os.environ.get("NEEDLE_MLX_PARITY_CHECKPOINT", "checkpoints/needle3.safetensors")+ if not os.path.exists(path):+ pytest.skip(f"no Needle 3 checkpoint at {path} (set NEEDLE_MLX_PARITY_CHECKPOINT)")+ return path+++def mlx_cpu():+ """Exact parity with JAX holds on the MLX CPU device; Metal is checked on its own."""+ import mlx.core as mx++ with mx.stream(mx.cpu):+ yield+++def perturbed_model():+ return _perturbed_model+++def _perturbed_model(**overrides):+ """A tiny model with every parameter moved off its init, so identity-initialised+ paths (conv taps, cond_u, b2, Sinkhorn bias) are exercised."""+ import numpy as np+ import jax+ import jax.numpy as jnp+ from needle.model.architecture import SimpleAttentionNetwork, TransformerConfig+ from needle.model.checkpoints import flatten, unflatten++ fields = dict(+ vocab_size=512, out_vocab=500, d_model=48, num_heads=4, num_kv_heads=2,+ num_layers=5, qk_head_dim=8, v_head_dim=12, max_seq_len=64, engram_layers=(1, 4),+ engram_slots=64, global_layers=(2, 4), sliding_window=8, mhc_lanes=4,+ qkv_conv_taps=3, dtype="float32", flash=False)+ fields.update(overrides)+ config = TransformerConfig(**fields)+ model = SimpleAttentionNetwork(config)+ params = model.init(jax.random.PRNGKey(0), jnp.ones((1, 8), jnp.int32))["params"]+ rng = np.random.default_rng(1)+ params = unflatten({k: (v + 0.05 * rng.standard_normal(v.shape)).astype(np.float32)+ for k, v in flatten(jax.device_get(params)).items()})+ tokens = rng.integers(1, config.out_vocab, size=(2, 24)).astype(np.int32)+ return model, config, params, tokens+++def deploy_numerics():+ from needle.model import quantize++ saved = quantize.ACT_BITS, quantize.KV_BITS, quantize._KV_GROUP++ def configure(config):+ quantize.configure_deploy(act_bits=config.act_bits, kv_bits=config.kv_bits)++ yield configure+ quantize.configure_deploy(*saved)diff --git a/tests/test_mlx.py b/tests/test_mlx.pynew file mode 100644index 0000000..f185787--- /dev/null+++ b/tests/test_mlx.py@@ -0,0 +1,91 @@+import numpy as np+import pytest++mx = pytest.importorskip("mlx.core")+jax = pytest.importorskip("jax")++pytestmark = [pytest.mark.slow, pytest.mark.usefixtures("mlx_cpu")]+++def _jax_logits(model, params, tokens, **kw):+ return model.apply({"params": params}, jax.numpy.asarray(tokens), **kw)+++def _mlx_logits(config, params, tokens, **kw):+ from needle.model.architecture_mlx import forward, to_mlx++ return forward(to_mlx(params), config, mx.array(tokens), **kw)++[email protected]("flash", [False, True])+def test_forward_matches_jax(flash, perturbed_model):+ model, config, params, tokens = perturbed_model(flash=flash)+ ref = np.asarray(_jax_logits(model, params, tokens))+ got = np.asarray(_mlx_logits(config, params, tokens))+ assert got.shape == ref.shape == (2, 24, 500)+ np.testing.assert_allclose(got, ref, atol=5e-5)+++def test_bfloat16_forward_is_as_close_to_float32_as_jax(perturbed_model):+ model, config, params, tokens = perturbed_model()+ ref32 = np.asarray(_jax_logits(model, params, tokens))+ bf16_model, bf16_config, _, _ = perturbed_model(dtype="bfloat16")+ jax_err = np.abs(np.asarray(_jax_logits(bf16_model, params, tokens)) - ref32).mean()+ mlx_err = np.abs(np.asarray(_mlx_logits(bf16_config, params, tokens)) - ref32).mean()+ assert 0 < mlx_err < 2 * jax_err+++def test_split_hadamard_perms_match_jax(perturbed_model):+ model, config, params, tokens = perturbed_model(ladder_widths=(24,))+ np.testing.assert_allclose(np.asarray(_mlx_logits(config, params, tokens)),+ np.asarray(_jax_logits(model, params, tokens)), atol=5e-5)+++def test_exit_depth_matches_jax(perturbed_model):+ model, config, params, tokens = perturbed_model()+ ref_full, ref_exit = _jax_logits(model, params, tokens, exit_depth=3)+ got_full, got_exit = _mlx_logits(config, params, tokens, exit_depth=3)+ np.testing.assert_allclose(np.asarray(got_full), np.asarray(ref_full), atol=5e-5)+ np.testing.assert_allclose(np.asarray(got_exit), np.asarray(ref_exit), atol=5e-5)++ ref_sub = _jax_logits(model, params, tokens, exit_depth=3, subnetwork_only=True)+ got_sub = _mlx_logits(config, params, tokens, exit_depth=3, subnetwork_only=True)+ np.testing.assert_allclose(np.asarray(got_sub), np.asarray(ref_sub), atol=5e-5)+++def test_padding_mask_matches_jax(perturbed_model):+ from needle.model.architecture import make_causal_mask, make_padding_mask++ model, config, params, tokens = perturbed_model()+ tokens[:, -5:] = config.pad_token_id+ mask = make_causal_mask(tokens.shape[1]) & make_padding_mask(tokens, config.pad_token_id)+ ref = _jax_logits(model, params, tokens, mask=mask)+ got = _mlx_logits(config, params, tokens, mask=mx.array(np.asarray(mask)))+ np.testing.assert_allclose(np.asarray(got), np.asarray(ref), atol=5e-5)+++def test_published_checkpoint_matches_jax(needle3_checkpoint):+ from needle.model.architecture import SimpleAttentionNetwork+ from needle.model.run import load_checkpoint++ params, config = load_checkpoint(needle3_checkpoint)+ config.dtype = "float32"+ params = jax.tree.map(lambda a: np.asarray(a, np.float32), params)+ tokens = np.random.default_rng(0).integers(14, config.vocab_size, (2, 96)).astype(np.int32)++ ref = np.asarray(_jax_logits(SimpleAttentionNetwork(config), params, tokens))+ got = np.asarray(_mlx_logits(config, params, tokens))+ np.testing.assert_allclose(got, ref, atol=5e-4)+ assert (got.argmax(-1) == ref.argmax(-1)).all()++[email protected](not mx.metal.is_available(), reason="no Metal device")+def test_metal_forward_agrees_with_jax(perturbed_model):+ """Metal is not bit-for-bit float32 on every Apple GPU, so the default device gets+ argmax agreement and a 5e-3 band instead of the CPU tolerance."""+ model, config, params, tokens = perturbed_model()+ ref = np.asarray(_jax_logits(model, params, tokens))+ with mx.stream(mx.gpu):+ got = np.asarray(_mlx_logits(config, params, tokens))+ np.testing.assert_allclose(got, ref, atol=5e-3)+ assert (got.argmax(-1) == ref.argmax(-1)).all()diff --git a/tests/test_quantize_mlx.py b/tests/test_quantize_mlx.pynew file mode 100644index 0000000..674c22b--- /dev/null+++ b/tests/test_quantize_mlx.py@@ -0,0 +1,78 @@+import numpy as np+import pytest++mx = pytest.importorskip("mlx.core")+jax = pytest.importorskip("jax")++pytestmark = [pytest.mark.slow, pytest.mark.usefixtures("mlx_cpu")]+++def _jax_logits(model, params, tokens):+ from needle.model.quantize import WEIGHT_BITS, cq_ste_params++ return model.apply({"params": cq_ste_params(params, WEIGHT_BITS)},+ jax.numpy.asarray(tokens), quant=True)+++def _mlx_logits(config, params, tokens):+ from needle.model.architecture_mlx import forward, to_mlx+ from needle.model.quantize import WEIGHT_BITS+ from needle.model.quantize_mlx import cq_ste_params++ return forward(cq_ste_params(to_mlx(params), WEIGHT_BITS), config, mx.array(tokens),+ quant=True)++[email protected]("bits", [2, 4])+def test_cq_quantize_matches_jax(bits):+ from needle.model import quantize, quantize_mlx++ w = np.random.default_rng(bits).standard_normal((3, 200, 96)).astype(np.float32)+ ref = np.asarray(quantize.cq_quantize(jax.numpy.asarray(w), bits))+ got = np.asarray(quantize_mlx.cq_quantize(mx.array(w), bits))+ assert np.mean(np.abs(got - ref) > 1e-5) < 1e-3+ np.testing.assert_allclose(np.median(np.abs(got - ref)), 0, atol=1e-6)++[email protected]("flash", [False, True])[email protected]("kv_bits", [8, 4])+def test_quantized_forward_matches_jax(flash, kv_bits, deploy_numerics, perturbed_model):+ """An ulp-level difference can move a value across a rounding boundary; that shows+ up as a few whole positions off, never as a drift across all of them."""+ model, config, params, tokens = perturbed_model(flash=flash, kv_bits=kv_bits)+ deploy_numerics(config)+ ref = np.asarray(_jax_logits(model, params, tokens))+ got = np.asarray(_mlx_logits(config, params, tokens))+ exact = np.all(np.abs(got - ref) <= 5e-5, axis=-1)+ assert exact.mean() >= 0.9+ assert np.abs(got - ref).max() < 1e-2+++def test_published_checkpoint_quantized_loss_matches_jax(needle3_checkpoint, deploy_numerics):+ """Under QAT the published model flips on ulp-level differences (JAX against itself+ with 1e-7 weight noise agrees on ~96% of argmaxes), so it is compared by loss."""+ import optax+ from needle.model.architecture import SimpleAttentionNetwork+ from needle.model.run import load_checkpoint++ params, config = load_checkpoint(needle3_checkpoint)+ config.dtype = "float32"+ params = jax.tree.map(lambda a: np.asarray(a, np.float32), params)+ deploy_numerics(config)+ tokens = np.random.default_rng(0).integers(14, config.vocab_size, (2, 96)).astype(np.int32)++ def loss(logits):+ return float(optax.softmax_cross_entropy_with_integer_labels(+ np.asarray(logits)[:, :-1], tokens[:, 1:]).mean())++ ref = _jax_logits(SimpleAttentionNetwork(config), params, tokens)+ got = _mlx_logits(config, params, tokens)+ assert abs(loss(got) - loss(ref)) < 1e-2 * loss(ref)+ assert (np.asarray(got).argmax(-1) == np.asarray(ref).argmax(-1)).mean() > 0.9+++def test_ab_scales_checkpoints_are_rejected():+ from needle.model.quantize_mlx import cq_ste_params++ with pytest.raises(NotImplementedError):+ cq_ste_params({"ab_scales/stack/a": mx.zeros((2, 2))}, 4)