Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
proper matryoshka FFN splitting
Noah Cylich58cc69ea2c65
diff --git a/src/export.py b/src/export.pyindex d088c7d..ba82d97 100644--- a/src/export.py+++ b/src/export.py@@ -1,4 +1,8 @@-"""Export MRL sub-models by slicing a full checkpoint to a target dimension."""+"""Export MRL sub-models by slicing FFN weights to a target dimension.++With FFN interior matryoshka, d_model stays constant — only FFN intermediate+dimensions (gate_proj, up_proj, down_proj) are sliced proportionally.+"""import osimport pickle@@ -11,74 +15,67 @@ import numpy as npfrom .model import TransformerConfig+# FFN kernel names that should be sliced in the d_ff dimension+_FFN_KERNEL_NAMES = {"gate_proj", "up_proj", "down_proj"}+def export_submodel(checkpoint_path, d_prime, output_path):- """Slice a full MRL checkpoint to a self-contained sub-model at dimension d_prime."""+ """Slice a full MRL checkpoint to a sub-model at MRL dimension d_prime.++ Only FFN weights are reduced: d_ff' = d_ff * d_prime // d_model.+ Attention, embeddings, norms, and mixer weights are unchanged.+ """- # Load checkpointwith open(checkpoint_path, "rb") as f:data = pickle.load(f)params = data["params"]config = TransformerConfig(**data["config"])- head_dim = config.d_model // config.num_heads- if d_prime % head_dim != 0:- raise ValueError(- f"d_prime={d_prime} must be divisible by head_dim={head_dim} "- f"(d_model={config.d_model} // num_heads={config.num_heads})"- )-- # Derived dims- n_h = d_prime // head_dim- n_kv = min(config.num_kv_heads, n_h)d_ff_prime = config.d_ff * d_prime // config.d_model- kv_dim = config.num_kv_heads * head_dim- kv_dim_prime = n_kv * head_dim+ if d_ff_prime == 0:+ raise ValueError(f"d_prime={d_prime} too small: would give d_ff=0")- d_model = config.d_modeld_ff = config.d_ffdef slice_leaf(key_path, leaf):arr = np.asarray(leaf)-- if arr.ndim == 1:- # Norm scales: shape (d_model,)- return arr[:d_prime]-- if arr.ndim == 3:- # Memory slots: shape (1, M, d_model)- return arr[:, :, :d_prime]-- if arr.ndim == 2:- rows, cols = arr.shape- # Determine slice for each axis based on original size- def _target(size):- if size == d_model:- return d_prime- if size == d_ff:- return d_ff_prime- if size == kv_dim:- return kv_dim_prime- return size # vocab, num_memory_slots, etc.-- r = _target(rows)- c = _target(cols)- return arr[:r, :c]+ if arr.ndim != 2:+ return arr++ # Check if this is an FFN kernel by looking at parent module name+ parent_name = None+ for part in key_path:+ name = part.key if hasattr(part, "key") else str(part)+ if name in _FFN_KERNEL_NAMES:+ parent_name = name+ break++ if parent_name is None:+ return arr++ # Also skip mixer FFN kernels (token_mix_*, channel_mix_*)+ path_str = "/".join(+ p.key if hasattr(p, "key") else str(p) for p in key_path+ )+ if "mixer" in path_str:+ return arr++ rows, cols = arr.shape+ if parent_name in ("gate_proj", "up_proj"):+ # (d_model, d_ff) → (d_model, d_ff')+ if cols == d_ff:+ return arr[:, :d_ff_prime]+ elif parent_name == "down_proj":+ # (d_ff, d_model) → (d_ff', d_model)+ if rows == d_ff:+ return arr[:d_ff_prime, :]return arrsliced = jax.tree_util.tree_map_with_path(slice_leaf, params)- # Build new config- new_config = replace(- config,- d_model=d_prime,- d_ff=d_ff_prime,- num_heads=n_h,- num_kv_heads=n_kv,- )+ new_config = replace(config, d_ff=d_ff_prime)- # Convert any JAX arrays to numpy for picklingsliced_np = jax.tree.map(lambda x: np.asarray(x) if isinstance(x, jnp.ndarray) else x, sliced)@@ -87,7 +84,6 @@ def export_submodel(checkpoint_path, d_prime, output_path):with open(output_path, "wb") as f:pickle.dump({"params": sliced_np, "config": new_config.__dict__}, f)- # Print summaryorig_count = sum(x.size for x in jax.tree.leaves(params))new_count = sum(x.size for x in jax.tree.leaves(sliced_np))orig_bytes = sum(x.nbytes for x in jax.tree.leaves(params))@@ -96,10 +92,10 @@ def export_submodel(checkpoint_path, d_prime, output_path):print(f"\n Export complete: {output_path}")print(f" ─────────────────────────────────────")print(f" {'':>20s} {'Original':>12s} {'Exported':>12s}")- print(f" {'d_model':>20s} {config.d_model:>12d} {d_prime:>12d}")+ print(f" {'d_model':>20s} {config.d_model:>12d} {config.d_model:>12d}")print(f" {'d_ff':>20s} {config.d_ff:>12d} {d_ff_prime:>12d}")- print(f" {'num_heads':>20s} {config.num_heads:>12d} {n_h:>12d}")- print(f" {'num_kv_heads':>20s} {config.num_kv_heads:>12d} {n_kv:>12d}")+ print(f" {'num_heads':>20s} {config.num_heads:>12d} {config.num_heads:>12d}")+ print(f" {'num_kv_heads':>20s} {config.num_kv_heads:>12d} {config.num_kv_heads:>12d}")print(f" {'params':>20s} {orig_count:>12,d} {new_count:>12,d}")print(f" {'size (MB)':>20s} {orig_bytes / 1e6:>12.1f} {new_bytes / 1e6:>12.1f}")print()diff --git a/src/model.py b/src/model.pyindex 15bd222..38b0b2a 100644--- a/src/model.py+++ b/src/model.py@@ -129,16 +129,32 @@ class FeedForward(nn.Module):activation: str = "drelu"@nn.compact- def __call__(self, x):- gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="gate_proj")(x)- up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="up_proj")(x)+ def __call__(self, x, d_ff_slice=None):+ sliced = d_ff_slice is not None and d_ff_slice < self.d_ff++ if sliced:+ # Sliced path: access existing kernels and slice for efficiency.+ # Params must already exist from a prior full-width call (e.g. the+ # non-MRL forward in forward_with_aux).+ gk = self.variables['params']['gate_proj']['kernel'][:, :d_ff_slice]+ uk = self.variables['params']['up_proj']['kernel'][:, :d_ff_slice]+ gate = x.astype(self.dtype) @ gk.astype(self.dtype)+ up = x.astype(self.dtype) @ uk.astype(self.dtype)+ else:+ gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="gate_proj")(x)+ up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="up_proj")(x)+if self.activation == "swiglu":- x = nn.silu(gate) * up+ h = nn.silu(gate) * upelif self.activation == "geglu":- x = nn.gelu(gate) * up+ h = nn.gelu(gate) * upelse: # drelu- x = nn.relu(gate) * nn.relu(up)- return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)+ h = nn.relu(gate) * nn.relu(up)++ if sliced:+ dk = self.variables['params']['down_proj']['kernel'][:d_ff_slice]+ return (h @ dk.astype(self.dtype))+ return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(h)class MLPMixer(nn.Module):@@ -194,7 +210,7 @@ class MemoryMixerBlock(nn.Module):activation: str = "drelu"@nn.compact- def __call__(self, x, s, mask=None, rope=None):+ def __call__(self, x, s, mask=None, rope=None, d_ff_slice=None):s_norm = ZCRMSNorm(dtype=self.dtype, name="pack_s_norm")(s)x_norm = ZCRMSNorm(dtype=self.dtype, name="pack_x_norm")(x)s = s + MultiHeadAttention(@@ -208,7 +224,7 @@ class MemoryMixerBlock(nn.Module):residual = xx = ZCRMSNorm(dtype=self.dtype, name="local_norm")(x)- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="local_ffn")(x)+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="local_ffn")(x, d_ff_slice=d_ff_slice)x = x + residualreturn x, s@@ -219,7 +235,7 @@ class MemoryMixerEncoder(nn.Module):config: TransformerConfig@nn.compact- def __call__(self, x, mask=None, rope=None):+ def __call__(self, x, mask=None, rope=None, d_ff_slice=None):cfg = self.configdt = cfg.jax_dtypex = x.astype(dt)@@ -246,7 +262,7 @@ class MemoryMixerEncoder(nn.Module):x, s = nn.remat(MemoryMixerBlock)(cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,cfg.num_memory_slots, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"- )(x, s, mask=mask, rope=rope)+ )(x, s, mask=mask, rope=rope, d_ff_slice=d_ff_slice)s = ZCRMSNorm(dtype=dt, name="final_norm")(s)return s@@ -262,7 +278,7 @@ class DecoderBlock(nn.Module):activation: str = "drelu"@nn.compact- def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None):+ def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None, d_ff_slice=None):residual = xx = ZCRMSNorm(dtype=self.dtype)(x)x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(@@ -279,7 +295,7 @@ class DecoderBlock(nn.Module):residual = xx = ZCRMSNorm(dtype=self.dtype)(x)- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x)+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x, d_ff_slice=d_ff_slice)x = x + residualreturn x@@ -290,7 +306,7 @@ class Decoder(nn.Module):config: TransformerConfig@nn.compact- def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None):+ def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None, d_ff_slice=None):cfg = self.configdt = cfg.jax_dtypex = x.astype(dt)@@ -298,7 +314,7 @@ class Decoder(nn.Module):for i in range(cfg.num_decoder_layers):x = nn.remat(DecoderBlock)(cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"- )(x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope)+ )(x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope, d_ff_slice=d_ff_slice)x = ZCRMSNorm(dtype=dt)(x)return x@@ -330,19 +346,19 @@ class EncoderDecoderTransformer(nn.Module):head_dim = self.config.d_model // self.config.num_headsreturn precompute_rope_freqs(head_dim, seq_len, self.config.rope_theta)- def encode_text(self, src, src_mask=None):+ def encode_text(self, src, src_mask=None, d_ff_slice=None):x = self.embedding(src) * self.embed_scalerope = self._rope(src.shape[1])- return self.encoder(x, mask=src_mask, rope=rope)+ return self.encoder(x, mask=src_mask, rope=rope, d_ff_slice=d_ff_slice)def encode(self, src, src_mask=None):"""Backward-compatible alias for encode_text."""return self.encode_text(src, src_mask=src_mask)- def encode_speech(self, mel, src_mask=None):+ def encode_speech(self, mel, src_mask=None, d_ff_slice=None):x = self.mel_proj(mel) * self.embed_scalerope = self._rope(mel.shape[1])- return self.encoder(x, mask=src_mask, rope=rope)+ return self.encoder(x, mask=src_mask, rope=rope, d_ff_slice=d_ff_slice)def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None):x = self.embedding(tgt) * self.embed_scale@@ -360,34 +376,59 @@ class EncoderDecoderTransformer(nn.Module):)return logits- def _decode_with_aux(self, encoder_out, tgt, tgt_mask=None, mrl_dims=None):- """Shared decode logic returning logits, slot_div, mrl_logits."""+ def _run_decoder(self, encoder_out, tgt, tgt_mask=None, d_ff_slice=None):+ """Run decoder and return float32 hidden states."""x = self.embedding(tgt) * self.embed_scalerope = self._rope(tgt.shape[1])- x = self.decoder(x, encoder_out, self_mask=tgt_mask, cross_mask=None, rope=rope)- x_f32 = x.astype(jnp.float32)+ x = self.decoder(x, encoder_out, self_mask=tgt_mask, cross_mask=None, rope=rope, d_ff_slice=d_ff_slice)+ return x.astype(jnp.float32)++ def _slot_diversity(self, encoder_out):+ s = encoder_out.astype(jnp.float32)+ gram = jnp.matmul(s, s.transpose(0, 2, 1))+ diag_sq = jnp.sum(jnp.diagonal(gram, axis1=1, axis2=2) ** 2)+ return (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]++ def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mrl_dims=None):emb = self.embedding.embedding++ # Full forward (no FFN slicing)+ encoder_out = self.encode_text(src, src_mask=src_mask)+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask)logits = x_f32 @ emb.T+ slot_div = self._slot_diversity(encoder_out)mrl_logits = []if mrl_dims is not None:for d in mrl_dims:if d < self.config.d_model:- mrl_logits.append(x_f32[..., :d] @ emb[:, :d].T)+ d_ff_slice = self.config.d_ff * d // self.config.d_model+ enc_m = self.encode_text(src, src_mask=src_mask, d_ff_slice=d_ff_slice)+ x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, d_ff_slice=d_ff_slice)+ mrl_logits.append(x_m @ emb.T)- s = encoder_out.astype(jnp.float32)- gram = jnp.matmul(s, s.transpose(0, 2, 1))- diag_sq = jnp.sum(jnp.diagonal(gram, axis1=1, axis2=2) ** 2)- slot_div = (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]return logits, slot_div, mrl_logits- def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mrl_dims=None):- encoder_out = self.encode_text(src, src_mask=src_mask)- return self._decode_with_aux(encoder_out, tgt, tgt_mask=tgt_mask, mrl_dims=mrl_dims)-def forward_speech_with_aux(self, mel, tgt, src_mask=None, tgt_mask=None, mrl_dims=None):+ emb = self.embedding.embedding++ # Full forward (no FFN slicing)encoder_out = self.encode_speech(mel, src_mask=src_mask)- return self._decode_with_aux(encoder_out, tgt, tgt_mask=tgt_mask, mrl_dims=mrl_dims)+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask)+ logits = x_f32 @ emb.T+ slot_div = self._slot_diversity(encoder_out)++ # MRL sub-model forwards with sliced FFN+ mrl_logits = []+ if mrl_dims is not None:+ for d in mrl_dims:+ if d < self.config.d_model:+ d_ff_slice = self.config.d_ff * d // self.config.d_model+ enc_m = self.encode_speech(mel, src_mask=src_mask, d_ff_slice=d_ff_slice)+ x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, d_ff_slice=d_ff_slice)+ mrl_logits.append(x_m @ emb.T)++ return logits, slot_div, mrl_logitsdef init_all(self, src, tgt, mel):"""Dummy forward through both text and speech pathways to initialize all params."""@@ -396,8 +437,8 @@ class EncoderDecoderTransformer(nn.Module):text_out = self.encode_text(src, src_mask=src_mask)mel_mask = make_mel_padding_mask(mel)speech_out = self.encode_speech(mel, src_mask=mel_mask)- _ = self._decode_with_aux(text_out, tgt, tgt_mask=tgt_mask)- _ = self._decode_with_aux(speech_out, tgt, tgt_mask=tgt_mask)+ _ = self._run_decoder(text_out, tgt, tgt_mask=tgt_mask)+ _ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask)return jnp.zeros(())diff --git a/src/train.py b/src/train.pyindex 45903e1..e15821c 100644--- a/src/train.py+++ b/src/train.py@@ -378,7 +378,7 @@ def _make_val_loss_fn(apply_fn):def _make_mrl_val_loss_fn(apply_fn, mrl_dim):- """Val loss using truncated decoder hidden states and embedding at dimension mrl_dim."""+ """Val loss for MRL sub-model at dimension mrl_dim (FFN interior slicing)."""@jax.jitdef val_loss_batch(params, src, tgt_in, tgt_out, causal_mask):pad_id = 0@@ -415,31 +415,28 @@ def _make_speech_val_loss_fn(apply_fn):def _estimate_mrl_params(config, d_prime):- """Estimate parameter count of an exported sub-model at dimension d_prime.-- Slicing rules (matching what an actual MRL export would produce):- - Embedding: [vocab, d_prime] (first d_prime cols)- - Heads: same head_dim, fewer heads → n_h = d_prime // head_dim- - KV heads: min(original, n_h)- - FFN: proportional → d_ff' = d_ff * d_prime // d_model- - Memory: slots unchanged, but projections shrink with d / d_ff+ """Estimate parameter count of an exported sub-model at MRL dimension d_prime.++ FFN interior slicing: only FFN d_ff is reduced proportionally.+ d_model, attention, embedding, and mixer are all unchanged.+ - Embedding: [vocab, d_model] (unchanged)+ - Attention: full d_model heads/projections (unchanged)+ - FFN: d_ff' = d_ff * d_prime // d_model (sliced)+ - Mixer: full d_ff (unchanged, not masked)"""- d = d_prime+ d = config.d_modelv = config.vocab_sizen_enc = config.num_encoder_layersn_dec = config.num_decoder_layers- head_dim = config.d_model // config.num_heads- n_h = max(1, d // head_dim)- n_kv = min(config.num_kv_heads, n_h)- d_ff = config.d_ff * d // config.d_model+ kv_dim = config.num_kv_heads * (d // config.num_heads)+ d_ff_prime = config.d_ff * d_prime // dm = config.num_memory_slotsemb = v * d- kv_dim = n_kv * head_dimattn = d * d + d * kv_dim * 2 + d * d- ffn = d * d_ff * 3- mixer_token = d * d_ff * 2 + d_ff * m- mixer_channel = d * d_ff * 3+ ffn = d * d_ff_prime * 3+ mixer_token = d * config.d_ff * 2 + config.d_ff * m+ mixer_channel = d * config.d_ff * 3enc_block = attn + ffn + mixer_token + mixer_channeldec_block = attn * 2 + ffntotal = emb + n_enc * enc_block + n_dec * dec_block@@ -881,14 +878,13 @@ def train(args):print(f" Sparsity {sparsity:>11.2f}% ({near_zero:,}/{total_params:,})")if mrl_results:print(f" ─────────────────────────────────────")- print(f" MRL exportable sub-models:")- print(f" {'dim':>6} {'val ppl':>10} {'params':>12} {'heads':>5}")- head_dim = config.d_model // config.num_heads- print(f" {config.d_model:>6} {last_val_ppl:>10.2f} {total_params:>12,} {config.num_heads:>5} (full)")+ print(f" MRL exportable sub-models (FFN interior slicing):")+ print(f" {'dim':>6} {'d_ff':>6} {'val ppl':>10} {'params':>12}")+ print(f" {config.d_model:>6} {config.d_ff:>6} {last_val_ppl:>10.2f} {total_params:>12,} (full)")for d_prime in sorted(mrl_results.keys(), reverse=True):mrl_ppl, mrl_params = mrl_results[d_prime]- n_h = max(1, d_prime // head_dim)- print(f" {d_prime:>6} {mrl_ppl:>10.2f} {mrl_params:>12,} {n_h:>5}")+ d_ff_prime = config.d_ff * d_prime // config.d_model+ print(f" {d_prime:>6} {d_ff_prime:>6} {mrl_ppl:>10.2f} {mrl_params:>12,}")print(f" ─────────────────────────────────────")print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")print(f" Latency {tp['avg_latency_s']:>11.3f}s")