Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

proper matryoshka FFN splitting

Noah Cylich58cc69ea2c65
diff --git a/src/export.py b/src/export.py
index d088c7d..ba82d97 100644
--- a/src/export.py
+++ b/src/export.py
@@ -1,4 +1,8 @@
-"""Export MRL sub-models by slicing a full checkpoint to a target dimension."""
+"""Export MRL sub-models by slicing FFN weights to a target dimension.
+
+With FFN interior matryoshka, d_model stays constant — only FFN intermediate
+dimensions (gate_proj, up_proj, down_proj) are sliced proportionally.
+"""
import os
import pickle
@@ -11,74 +15,67 @@ import numpy as np
from .model import TransformerConfig
+# FFN kernel names that should be sliced in the d_ff dimension
+_FFN_KERNEL_NAMES = {"gate_proj", "up_proj", "down_proj"}
+
def export_submodel(checkpoint_path, d_prime, output_path):
- """Slice a full MRL checkpoint to a self-contained sub-model at dimension d_prime."""
+ """Slice a full MRL checkpoint to a sub-model at MRL dimension d_prime.
+
+ Only FFN weights are reduced: d_ff' = d_ff * d_prime // d_model.
+ Attention, embeddings, norms, and mixer weights are unchanged.
+ """
- # Load checkpoint
with open(checkpoint_path, "rb") as f:
data = pickle.load(f)
params = data["params"]
config = TransformerConfig(**data["config"])
- head_dim = config.d_model // config.num_heads
- if d_prime % head_dim != 0:
- raise ValueError(
- f"d_prime={d_prime} must be divisible by head_dim={head_dim} "
- f"(d_model={config.d_model} // num_heads={config.num_heads})"
- )
-
- # Derived dims
- n_h = d_prime // head_dim
- n_kv = min(config.num_kv_heads, n_h)
d_ff_prime = config.d_ff * d_prime // config.d_model
- kv_dim = config.num_kv_heads * head_dim
- kv_dim_prime = n_kv * head_dim
+ if d_ff_prime == 0:
+ raise ValueError(f"d_prime={d_prime} too small: would give d_ff=0")
- d_model = config.d_model
d_ff = config.d_ff
def slice_leaf(key_path, leaf):
arr = np.asarray(leaf)
-
- if arr.ndim == 1:
- # Norm scales: shape (d_model,)
- return arr[:d_prime]
-
- if arr.ndim == 3:
- # Memory slots: shape (1, M, d_model)
- return arr[:, :, :d_prime]
-
- if arr.ndim == 2:
- rows, cols = arr.shape
- # Determine slice for each axis based on original size
- def _target(size):
- if size == d_model:
- return d_prime
- if size == d_ff:
- return d_ff_prime
- if size == kv_dim:
- return kv_dim_prime
- return size # vocab, num_memory_slots, etc.
-
- r = _target(rows)
- c = _target(cols)
- return arr[:r, :c]
+ if arr.ndim != 2:
+ return arr
+
+ # Check if this is an FFN kernel by looking at parent module name
+ parent_name = None
+ for part in key_path:
+ name = part.key if hasattr(part, "key") else str(part)
+ if name in _FFN_KERNEL_NAMES:
+ parent_name = name
+ break
+
+ if parent_name is None:
+ return arr
+
+ # Also skip mixer FFN kernels (token_mix_*, channel_mix_*)
+ path_str = "/".join(
+ p.key if hasattr(p, "key") else str(p) for p in key_path
+ )
+ if "mixer" in path_str:
+ return arr
+
+ rows, cols = arr.shape
+ if parent_name in ("gate_proj", "up_proj"):
+ # (d_model, d_ff) → (d_model, d_ff')
+ if cols == d_ff:
+ return arr[:, :d_ff_prime]
+ elif parent_name == "down_proj":
+ # (d_ff, d_model) → (d_ff', d_model)
+ if rows == d_ff:
+ return arr[:d_ff_prime, :]
return arr
sliced = jax.tree_util.tree_map_with_path(slice_leaf, params)
- # Build new config
- new_config = replace(
- config,
- d_model=d_prime,
- d_ff=d_ff_prime,
- num_heads=n_h,
- num_kv_heads=n_kv,
- )
+ new_config = replace(config, d_ff=d_ff_prime)
- # Convert any JAX arrays to numpy for pickling
sliced_np = jax.tree.map(
lambda x: np.asarray(x) if isinstance(x, jnp.ndarray) else x, sliced
)
@@ -87,7 +84,6 @@ def export_submodel(checkpoint_path, d_prime, output_path):
with open(output_path, "wb") as f:
pickle.dump({"params": sliced_np, "config": new_config.__dict__}, f)
- # Print summary
orig_count = sum(x.size for x in jax.tree.leaves(params))
new_count = sum(x.size for x in jax.tree.leaves(sliced_np))
orig_bytes = sum(x.nbytes for x in jax.tree.leaves(params))
@@ -96,10 +92,10 @@ def export_submodel(checkpoint_path, d_prime, output_path):
print(f"\n Export complete: {output_path}")
print(f" ─────────────────────────────────────")
print(f" {'':>20s} {'Original':>12s} {'Exported':>12s}")
- print(f" {'d_model':>20s} {config.d_model:>12d} {d_prime:>12d}")
+ print(f" {'d_model':>20s} {config.d_model:>12d} {config.d_model:>12d}")
print(f" {'d_ff':>20s} {config.d_ff:>12d} {d_ff_prime:>12d}")
- print(f" {'num_heads':>20s} {config.num_heads:>12d} {n_h:>12d}")
- print(f" {'num_kv_heads':>20s} {config.num_kv_heads:>12d} {n_kv:>12d}")
+ print(f" {'num_heads':>20s} {config.num_heads:>12d} {config.num_heads:>12d}")
+ print(f" {'num_kv_heads':>20s} {config.num_kv_heads:>12d} {config.num_kv_heads:>12d}")
print(f" {'params':>20s} {orig_count:>12,d} {new_count:>12,d}")
print(f" {'size (MB)':>20s} {orig_bytes / 1e6:>12.1f} {new_bytes / 1e6:>12.1f}")
print()
diff --git a/src/model.py b/src/model.py
index 15bd222..38b0b2a 100644
--- a/src/model.py
+++ b/src/model.py
@@ -129,16 +129,32 @@ class FeedForward(nn.Module):
activation: str = "drelu"
@nn.compact
- def __call__(self, x):
- gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="gate_proj")(x)
- up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="up_proj")(x)
+ def __call__(self, x, d_ff_slice=None):
+ sliced = d_ff_slice is not None and d_ff_slice < self.d_ff
+
+ if sliced:
+ # Sliced path: access existing kernels and slice for efficiency.
+ # Params must already exist from a prior full-width call (e.g. the
+ # non-MRL forward in forward_with_aux).
+ gk = self.variables['params']['gate_proj']['kernel'][:, :d_ff_slice]
+ uk = self.variables['params']['up_proj']['kernel'][:, :d_ff_slice]
+ gate = x.astype(self.dtype) @ gk.astype(self.dtype)
+ up = x.astype(self.dtype) @ uk.astype(self.dtype)
+ else:
+ gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="gate_proj")(x)
+ up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="up_proj")(x)
+
if self.activation == "swiglu":
- x = nn.silu(gate) * up
+ h = nn.silu(gate) * up
elif self.activation == "geglu":
- x = nn.gelu(gate) * up
+ h = nn.gelu(gate) * up
else: # drelu
- x = nn.relu(gate) * nn.relu(up)
- return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)
+ h = nn.relu(gate) * nn.relu(up)
+
+ if sliced:
+ dk = self.variables['params']['down_proj']['kernel'][:d_ff_slice]
+ return (h @ dk.astype(self.dtype))
+ return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(h)
class MLPMixer(nn.Module):
@@ -194,7 +210,7 @@ class MemoryMixerBlock(nn.Module):
activation: str = "drelu"
@nn.compact
- def __call__(self, x, s, mask=None, rope=None):
+ def __call__(self, x, s, mask=None, rope=None, d_ff_slice=None):
s_norm = ZCRMSNorm(dtype=self.dtype, name="pack_s_norm")(s)
x_norm = ZCRMSNorm(dtype=self.dtype, name="pack_x_norm")(x)
s = s + MultiHeadAttention(
@@ -208,7 +224,7 @@ class MemoryMixerBlock(nn.Module):
residual = x
x = ZCRMSNorm(dtype=self.dtype, name="local_norm")(x)
- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="local_ffn")(x)
+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="local_ffn")(x, d_ff_slice=d_ff_slice)
x = x + residual
return x, s
@@ -219,7 +235,7 @@ class MemoryMixerEncoder(nn.Module):
config: TransformerConfig
@nn.compact
- def __call__(self, x, mask=None, rope=None):
+ def __call__(self, x, mask=None, rope=None, d_ff_slice=None):
cfg = self.config
dt = cfg.jax_dtype
x = x.astype(dt)
@@ -246,7 +262,7 @@ class MemoryMixerEncoder(nn.Module):
x, s = nn.remat(MemoryMixerBlock)(
cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,
cfg.num_memory_slots, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"
- )(x, s, mask=mask, rope=rope)
+ )(x, s, mask=mask, rope=rope, d_ff_slice=d_ff_slice)
s = ZCRMSNorm(dtype=dt, name="final_norm")(s)
return s
@@ -262,7 +278,7 @@ class DecoderBlock(nn.Module):
activation: str = "drelu"
@nn.compact
- def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None):
+ def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None, d_ff_slice=None):
residual = x
x = ZCRMSNorm(dtype=self.dtype)(x)
x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(
@@ -279,7 +295,7 @@ class DecoderBlock(nn.Module):
residual = x
x = ZCRMSNorm(dtype=self.dtype)(x)
- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x)
+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x, d_ff_slice=d_ff_slice)
x = x + residual
return x
@@ -290,7 +306,7 @@ class Decoder(nn.Module):
config: TransformerConfig
@nn.compact
- def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None):
+ def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None, d_ff_slice=None):
cfg = self.config
dt = cfg.jax_dtype
x = x.astype(dt)
@@ -298,7 +314,7 @@ class Decoder(nn.Module):
for i in range(cfg.num_decoder_layers):
x = nn.remat(DecoderBlock)(
cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"
- )(x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope)
+ )(x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope, d_ff_slice=d_ff_slice)
x = ZCRMSNorm(dtype=dt)(x)
return x
@@ -330,19 +346,19 @@ class EncoderDecoderTransformer(nn.Module):
head_dim = self.config.d_model // self.config.num_heads
return precompute_rope_freqs(head_dim, seq_len, self.config.rope_theta)
- def encode_text(self, src, src_mask=None):
+ def encode_text(self, src, src_mask=None, d_ff_slice=None):
x = self.embedding(src) * self.embed_scale
rope = self._rope(src.shape[1])
- return self.encoder(x, mask=src_mask, rope=rope)
+ return self.encoder(x, mask=src_mask, rope=rope, d_ff_slice=d_ff_slice)
def encode(self, src, src_mask=None):
"""Backward-compatible alias for encode_text."""
return self.encode_text(src, src_mask=src_mask)
- def encode_speech(self, mel, src_mask=None):
+ def encode_speech(self, mel, src_mask=None, d_ff_slice=None):
x = self.mel_proj(mel) * self.embed_scale
rope = self._rope(mel.shape[1])
- return self.encoder(x, mask=src_mask, rope=rope)
+ return self.encoder(x, mask=src_mask, rope=rope, d_ff_slice=d_ff_slice)
def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None):
x = self.embedding(tgt) * self.embed_scale
@@ -360,34 +376,59 @@ class EncoderDecoderTransformer(nn.Module):
)
return logits
- def _decode_with_aux(self, encoder_out, tgt, tgt_mask=None, mrl_dims=None):
- """Shared decode logic returning logits, slot_div, mrl_logits."""
+ def _run_decoder(self, encoder_out, tgt, tgt_mask=None, d_ff_slice=None):
+ """Run decoder and return float32 hidden states."""
x = self.embedding(tgt) * self.embed_scale
rope = self._rope(tgt.shape[1])
- x = self.decoder(x, encoder_out, self_mask=tgt_mask, cross_mask=None, rope=rope)
- x_f32 = x.astype(jnp.float32)
+ x = self.decoder(x, encoder_out, self_mask=tgt_mask, cross_mask=None, rope=rope, d_ff_slice=d_ff_slice)
+ return x.astype(jnp.float32)
+
+ def _slot_diversity(self, encoder_out):
+ s = encoder_out.astype(jnp.float32)
+ gram = jnp.matmul(s, s.transpose(0, 2, 1))
+ diag_sq = jnp.sum(jnp.diagonal(gram, axis1=1, axis2=2) ** 2)
+ return (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]
+
+ def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mrl_dims=None):
emb = self.embedding.embedding
+
+ # Full forward (no FFN slicing)
+ encoder_out = self.encode_text(src, src_mask=src_mask)
+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask)
logits = x_f32 @ emb.T
+ slot_div = self._slot_diversity(encoder_out)
mrl_logits = []
if mrl_dims is not None:
for d in mrl_dims:
if d < self.config.d_model:
- mrl_logits.append(x_f32[..., :d] @ emb[:, :d].T)
+ d_ff_slice = self.config.d_ff * d // self.config.d_model
+ enc_m = self.encode_text(src, src_mask=src_mask, d_ff_slice=d_ff_slice)
+ x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, d_ff_slice=d_ff_slice)
+ mrl_logits.append(x_m @ emb.T)
- s = encoder_out.astype(jnp.float32)
- gram = jnp.matmul(s, s.transpose(0, 2, 1))
- diag_sq = jnp.sum(jnp.diagonal(gram, axis1=1, axis2=2) ** 2)
- slot_div = (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]
return logits, slot_div, mrl_logits
- def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mrl_dims=None):
- encoder_out = self.encode_text(src, src_mask=src_mask)
- return self._decode_with_aux(encoder_out, tgt, tgt_mask=tgt_mask, mrl_dims=mrl_dims)
-
def forward_speech_with_aux(self, mel, tgt, src_mask=None, tgt_mask=None, mrl_dims=None):
+ emb = self.embedding.embedding
+
+ # Full forward (no FFN slicing)
encoder_out = self.encode_speech(mel, src_mask=src_mask)
- return self._decode_with_aux(encoder_out, tgt, tgt_mask=tgt_mask, mrl_dims=mrl_dims)
+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask)
+ logits = x_f32 @ emb.T
+ slot_div = self._slot_diversity(encoder_out)
+
+ # MRL sub-model forwards with sliced FFN
+ mrl_logits = []
+ if mrl_dims is not None:
+ for d in mrl_dims:
+ if d < self.config.d_model:
+ d_ff_slice = self.config.d_ff * d // self.config.d_model
+ enc_m = self.encode_speech(mel, src_mask=src_mask, d_ff_slice=d_ff_slice)
+ x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, d_ff_slice=d_ff_slice)
+ mrl_logits.append(x_m @ emb.T)
+
+ return logits, slot_div, mrl_logits
def init_all(self, src, tgt, mel):
"""Dummy forward through both text and speech pathways to initialize all params."""
@@ -396,8 +437,8 @@ class EncoderDecoderTransformer(nn.Module):
text_out = self.encode_text(src, src_mask=src_mask)
mel_mask = make_mel_padding_mask(mel)
speech_out = self.encode_speech(mel, src_mask=mel_mask)
- _ = self._decode_with_aux(text_out, tgt, tgt_mask=tgt_mask)
- _ = self._decode_with_aux(speech_out, tgt, tgt_mask=tgt_mask)
+ _ = self._run_decoder(text_out, tgt, tgt_mask=tgt_mask)
+ _ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask)
return jnp.zeros(())
diff --git a/src/train.py b/src/train.py
index 45903e1..e15821c 100644
--- a/src/train.py
+++ b/src/train.py
@@ -378,7 +378,7 @@ def _make_val_loss_fn(apply_fn):
def _make_mrl_val_loss_fn(apply_fn, mrl_dim):
- """Val loss using truncated decoder hidden states and embedding at dimension mrl_dim."""
+ """Val loss for MRL sub-model at dimension mrl_dim (FFN interior slicing)."""
@jax.jit
def val_loss_batch(params, src, tgt_in, tgt_out, causal_mask):
pad_id = 0
@@ -415,31 +415,28 @@ def _make_speech_val_loss_fn(apply_fn):
def _estimate_mrl_params(config, d_prime):
- """Estimate parameter count of an exported sub-model at dimension d_prime.
-
- Slicing rules (matching what an actual MRL export would produce):
- - Embedding: [vocab, d_prime] (first d_prime cols)
- - Heads: same head_dim, fewer heads → n_h = d_prime // head_dim
- - KV heads: min(original, n_h)
- - FFN: proportional → d_ff' = d_ff * d_prime // d_model
- - Memory: slots unchanged, but projections shrink with d / d_ff
+ """Estimate parameter count of an exported sub-model at MRL dimension d_prime.
+
+ FFN interior slicing: only FFN d_ff is reduced proportionally.
+ d_model, attention, embedding, and mixer are all unchanged.
+ - Embedding: [vocab, d_model] (unchanged)
+ - Attention: full d_model heads/projections (unchanged)
+ - FFN: d_ff' = d_ff * d_prime // d_model (sliced)
+ - Mixer: full d_ff (unchanged, not masked)
"""
- d = d_prime
+ d = config.d_model
v = config.vocab_size
n_enc = config.num_encoder_layers
n_dec = config.num_decoder_layers
- head_dim = config.d_model // config.num_heads
- n_h = max(1, d // head_dim)
- n_kv = min(config.num_kv_heads, n_h)
- d_ff = config.d_ff * d // config.d_model
+ kv_dim = config.num_kv_heads * (d // config.num_heads)
+ d_ff_prime = config.d_ff * d_prime // d
m = config.num_memory_slots
emb = v * d
- kv_dim = n_kv * head_dim
attn = d * d + d * kv_dim * 2 + d * d
- ffn = d * d_ff * 3
- mixer_token = d * d_ff * 2 + d_ff * m
- mixer_channel = d * d_ff * 3
+ ffn = d * d_ff_prime * 3
+ mixer_token = d * config.d_ff * 2 + config.d_ff * m
+ mixer_channel = d * config.d_ff * 3
enc_block = attn + ffn + mixer_token + mixer_channel
dec_block = attn * 2 + ffn
total = emb + n_enc * enc_block + n_dec * dec_block
@@ -881,14 +878,13 @@ def train(args):
print(f" Sparsity {sparsity:>11.2f}% ({near_zero:,}/{total_params:,})")
if mrl_results:
print(f" ─────────────────────────────────────")
- print(f" MRL exportable sub-models:")
- print(f" {'dim':>6} {'val ppl':>10} {'params':>12} {'heads':>5}")
- head_dim = config.d_model // config.num_heads
- print(f" {config.d_model:>6} {last_val_ppl:>10.2f} {total_params:>12,} {config.num_heads:>5} (full)")
+ print(f" MRL exportable sub-models (FFN interior slicing):")
+ print(f" {'dim':>6} {'d_ff':>6} {'val ppl':>10} {'params':>12}")
+ print(f" {config.d_model:>6} {config.d_ff:>6} {last_val_ppl:>10.2f} {total_params:>12,} (full)")
for d_prime in sorted(mrl_results.keys(), reverse=True):
mrl_ppl, mrl_params = mrl_results[d_prime]
- n_h = max(1, d_prime // head_dim)
- print(f" {d_prime:>6} {mrl_ppl:>10.2f} {mrl_params:>12,} {n_h:>5}")
+ d_ff_prime = config.d_ff * d_prime // config.d_model
+ print(f" {d_prime:>6} {d_ff_prime:>6} {mrl_ppl:>10.2f} {mrl_params:>12,}")
print(f" ─────────────────────────────────────")
print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")
print(f" Latency {tp['avg_latency_s']:>11.3f}s")