Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
memoryMixer
HenryNdubuakue880863f96bd
diff --git a/src/cli.py b/src/cli.pyindex 86eba54..6478924 100644--- a/src/cli.py+++ b/src/cli.py@@ -137,6 +137,7 @@ def main():p.add_argument("--layer-prune-ratio", type=float, default=0.33)p.add_argument("--group-size", type=int, default=32)p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])+ p.add_argument("--num-memory-slots", type=int, default=64)p = sub.add_parser("run", add_help=False)p.add_argument("--checkpoint", type=str, required=True)diff --git a/src/model.py b/src/model.pyindex f4523c6..4f8d44c 100644--- a/src/model.py+++ b/src/model.py@@ -43,6 +43,7 @@ class TransformerConfig:rope_theta: float = 10000.0dtype: str = "bfloat16"activation: str = "drelu"+ num_memory_slots: int = 64@propertydef jax_dtype(self):@@ -94,7 +95,6 @@ class MultiHeadAttention(nn.Module):q = ZCRMSNorm(dtype=self.dtype, name="q_norm")(q)k = ZCRMSNorm(dtype=self.dtype, name="k_norm")(k)- # Repeat KV heads to match Q headsrepeats = self.num_heads // self.num_kv_headsif repeats > 1:k = jnp.repeat(k, repeats, axis=1)@@ -138,30 +138,100 @@ class FeedForward(nn.Module):return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)-class EncoderBlock(nn.Module):+class MLPMixer(nn.Module):+ """MLP-Mixer operating on fixed-size memory slots (B, M, d)."""+ num_slots: int+ d_model: int+ d_ff: int+ dtype: jnp.dtype = jnp.bfloat16+ activation: str = "drelu"++ @nn.compact+ def __call__(self, s):+ residual = s+ s = ZCRMSNorm(dtype=self.dtype, name="token_mix_norm")(s)+ s = s.transpose(0, 2, 1)+ gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_gate")(s)+ up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_up")(s)+ if self.activation == "swiglu":+ s = nn.silu(gate) * up+ elif self.activation == "geglu":+ s = nn.gelu(gate) * up+ else:+ s = nn.relu(gate) * nn.relu(up)+ s = nn.Dense(self.num_slots, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_down")(s)+ s = s.transpose(0, 2, 1)+ s = s + residual++ residual = s+ s = ZCRMSNorm(dtype=self.dtype, name="channel_mix_norm")(s)+ gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_gate")(s)+ up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_up")(s)+ if self.activation == "swiglu":+ s = nn.silu(gate) * up+ elif self.activation == "geglu":+ s = nn.gelu(gate) * up+ else:+ s = nn.relu(gate) * nn.relu(up)+ s = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_down")(s)+ s = s + residual++ return s+++class MemoryMixerBlock(nn.Module):+ """Block 4a: Pack (cross-attn) → Mix (MLP-Mixer) → Local Update (MLP)."""num_heads: intnum_kv_heads: intd_model: intd_ff: int+ num_slots: intnum_layers: intdtype: jnp.dtype = jnp.bfloat16activation: str = "drelu"@nn.compact- def __call__(self, x, mask=None, rope=None):- residual = x- x = ZCRMSNorm(dtype=self.dtype)(x)- x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype)(- x, x, mask=mask, rope=rope+ def __call__(self, x, s, mask=None):+ s_norm = ZCRMSNorm(dtype=self.dtype, name="pack_s_norm")(s)+ x_norm = ZCRMSNorm(dtype=self.dtype, name="pack_x_norm")(x)+ s = s + MultiHeadAttention(+ self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="pack_attn"+ )(s_norm, x_norm, mask=mask)++ s = s + MLPMixer(self.num_slots, self.d_model, self.d_ff, self.dtype, self.activation, name="mixer")(+ ZCRMSNorm(dtype=self.dtype, name="mix_norm")(s))- x = x + residualresidual = x- x = ZCRMSNorm(dtype=self.dtype)(x)- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x)+ x = ZCRMSNorm(dtype=self.dtype, name="local_norm")(x)+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="local_ffn")(x)x = x + residual- return x+ return x, s+++class MemoryMixerEncoder(nn.Module):+ """Encoder using MemoryMixer blocks. Output is the final memory slots S."""+ config: TransformerConfig++ @nn.compact+ def __call__(self, x, mask=None):+ cfg = self.config+ dt = cfg.jax_dtype+ x = x.astype(dt)++ s = self.param("memory_slots", jinit.normal(stddev=0.02), (1, cfg.num_memory_slots, cfg.d_model))+ s = jnp.broadcast_to(s.astype(dt), (x.shape[0], cfg.num_memory_slots, cfg.d_model))++ for i in range(cfg.num_encoder_layers):+ x, s = MemoryMixerBlock(+ cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,+ cfg.num_memory_slots, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"+ )(x, s, mask=mask)++ s = ZCRMSNorm(dtype=dt, name="final_norm")(s)+ return s+class DecoderBlock(nn.Module):@@ -175,7 +245,6 @@ class DecoderBlock(nn.Module):@nn.compactdef __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None):- # Self-attention with RoPEresidual = xx = ZCRMSNorm(dtype=self.dtype)(x)x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(@@ -183,7 +252,6 @@ class DecoderBlock(nn.Module):)x = x + residual- # Cross-attention (no RoPE)residual = xx = ZCRMSNorm(dtype=self.dtype)(x)x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="cross_attn")(@@ -199,23 +267,6 @@ class DecoderBlock(nn.Module):return x-class Encoder(nn.Module):- config: TransformerConfig-- @nn.compact- def __call__(self, x, mask=None, rope=None):- cfg = self.config- dt = cfg.jax_dtype- x = x.astype(dt)-- for i in range(cfg.num_encoder_layers):- x = EncoderBlock(- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"- )(x, mask=mask, rope=rope)-- x = ZCRMSNorm(dtype=dt)(x)- return x-class Decoder(nn.Module):config: TransformerConfig@@ -242,7 +293,7 @@ class EncoderDecoderTransformer(nn.Module):def setup(self):self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))self.embed_scale = math.sqrt(self.config.d_model)- self.encoder = Encoder(self.config)+ self.encoder = MemoryMixerEncoder(self.config)self.decoder = Decoder(self.config)def _rope(self, seq_len):@@ -251,14 +302,13 @@ class EncoderDecoderTransformer(nn.Module):def encode(self, src, src_mask=None):x = self.embedding(src) * self.embed_scale- rope = self._rope(src.shape[1])- return self.encoder(x, mask=src_mask, rope=rope)+ return self.encoder(x, mask=src_mask)def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None):x = self.embedding(tgt) * self.embed_scalerope = self._rope(tgt.shape[1])x = self.decoder(- x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope+ x, encoder_out, self_mask=self_mask, cross_mask=None, rope=rope)logits = x.astype(jnp.float32) @ self.embedding.embedding.Treturn logits@@ -266,7 +316,7 @@ class EncoderDecoderTransformer(nn.Module):def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):encoder_out = self.encode(src, src_mask=src_mask)logits = self.decode(- tgt, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask+ tgt, encoder_out, self_mask=tgt_mask)return logitsdiff --git a/src/train.py b/src/train.pyindex 3f2d7c5..76d7bba 100644--- a/src/train.py+++ b/src/train.py@@ -388,6 +388,7 @@ def train(args):max_seq_len=max(args.max_enc_len, args.max_dec_len),dtype=args.dtype,activation=getattr(args, "activation", "drelu"),+ num_memory_slots=getattr(args, "num_memory_slots", 64),)# Set group size for QAT and sparsification (captured by _train_step closure)