Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

memoryMixer

HenryNdubuakue880863f96bd
diff --git a/src/cli.py b/src/cli.py
index 86eba54..6478924 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -137,6 +137,7 @@ def main():
p.add_argument("--layer-prune-ratio", type=float, default=0.33)
p.add_argument("--group-size", type=int, default=32)
p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])
+ p.add_argument("--num-memory-slots", type=int, default=64)
p = sub.add_parser("run", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
diff --git a/src/model.py b/src/model.py
index f4523c6..4f8d44c 100644
--- a/src/model.py
+++ b/src/model.py
@@ -43,6 +43,7 @@ class TransformerConfig:
rope_theta: float = 10000.0
dtype: str = "bfloat16"
activation: str = "drelu"
+ num_memory_slots: int = 64
@property
def jax_dtype(self):
@@ -94,7 +95,6 @@ class MultiHeadAttention(nn.Module):
q = ZCRMSNorm(dtype=self.dtype, name="q_norm")(q)
k = ZCRMSNorm(dtype=self.dtype, name="k_norm")(k)
- # Repeat KV heads to match Q heads
repeats = self.num_heads // self.num_kv_heads
if repeats > 1:
k = jnp.repeat(k, repeats, axis=1)
@@ -138,30 +138,100 @@ class FeedForward(nn.Module):
return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)
-class EncoderBlock(nn.Module):
+class MLPMixer(nn.Module):
+ """MLP-Mixer operating on fixed-size memory slots (B, M, d)."""
+ num_slots: int
+ d_model: int
+ d_ff: int
+ dtype: jnp.dtype = jnp.bfloat16
+ activation: str = "drelu"
+
+ @nn.compact
+ def __call__(self, s):
+ residual = s
+ s = ZCRMSNorm(dtype=self.dtype, name="token_mix_norm")(s)
+ s = s.transpose(0, 2, 1)
+ gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_gate")(s)
+ up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_up")(s)
+ if self.activation == "swiglu":
+ s = nn.silu(gate) * up
+ elif self.activation == "geglu":
+ s = nn.gelu(gate) * up
+ else:
+ s = nn.relu(gate) * nn.relu(up)
+ s = nn.Dense(self.num_slots, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_down")(s)
+ s = s.transpose(0, 2, 1)
+ s = s + residual
+
+ residual = s
+ s = ZCRMSNorm(dtype=self.dtype, name="channel_mix_norm")(s)
+ gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_gate")(s)
+ up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_up")(s)
+ if self.activation == "swiglu":
+ s = nn.silu(gate) * up
+ elif self.activation == "geglu":
+ s = nn.gelu(gate) * up
+ else:
+ s = nn.relu(gate) * nn.relu(up)
+ s = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_down")(s)
+ s = s + residual
+
+ return s
+
+
+class MemoryMixerBlock(nn.Module):
+ """Block 4a: Pack (cross-attn) → Mix (MLP-Mixer) → Local Update (MLP)."""
num_heads: int
num_kv_heads: int
d_model: int
d_ff: int
+ num_slots: int
num_layers: int
dtype: jnp.dtype = jnp.bfloat16
activation: str = "drelu"
@nn.compact
- def __call__(self, x, mask=None, rope=None):
- residual = x
- x = ZCRMSNorm(dtype=self.dtype)(x)
- x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype)(
- x, x, mask=mask, rope=rope
+ def __call__(self, x, s, mask=None):
+ s_norm = ZCRMSNorm(dtype=self.dtype, name="pack_s_norm")(s)
+ x_norm = ZCRMSNorm(dtype=self.dtype, name="pack_x_norm")(x)
+ s = s + MultiHeadAttention(
+ self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="pack_attn"
+ )(s_norm, x_norm, mask=mask)
+
+ s = s + MLPMixer(self.num_slots, self.d_model, self.d_ff, self.dtype, self.activation, name="mixer")(
+ ZCRMSNorm(dtype=self.dtype, name="mix_norm")(s)
)
- x = x + residual
residual = x
- x = ZCRMSNorm(dtype=self.dtype)(x)
- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x)
+ x = ZCRMSNorm(dtype=self.dtype, name="local_norm")(x)
+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="local_ffn")(x)
x = x + residual
- return x
+ return x, s
+
+
+class MemoryMixerEncoder(nn.Module):
+ """Encoder using MemoryMixer blocks. Output is the final memory slots S."""
+ config: TransformerConfig
+
+ @nn.compact
+ def __call__(self, x, mask=None):
+ cfg = self.config
+ dt = cfg.jax_dtype
+ x = x.astype(dt)
+
+ s = self.param("memory_slots", jinit.normal(stddev=0.02), (1, cfg.num_memory_slots, cfg.d_model))
+ s = jnp.broadcast_to(s.astype(dt), (x.shape[0], cfg.num_memory_slots, cfg.d_model))
+
+ for i in range(cfg.num_encoder_layers):
+ x, s = MemoryMixerBlock(
+ cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,
+ cfg.num_memory_slots, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"
+ )(x, s, mask=mask)
+
+ s = ZCRMSNorm(dtype=dt, name="final_norm")(s)
+ return s
+
class DecoderBlock(nn.Module):
@@ -175,7 +245,6 @@ class DecoderBlock(nn.Module):
@nn.compact
def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None):
- # Self-attention with RoPE
residual = x
x = ZCRMSNorm(dtype=self.dtype)(x)
x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(
@@ -183,7 +252,6 @@ class DecoderBlock(nn.Module):
)
x = x + residual
- # Cross-attention (no RoPE)
residual = x
x = ZCRMSNorm(dtype=self.dtype)(x)
x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="cross_attn")(
@@ -199,23 +267,6 @@ class DecoderBlock(nn.Module):
return x
-class Encoder(nn.Module):
- config: TransformerConfig
-
- @nn.compact
- def __call__(self, x, mask=None, rope=None):
- cfg = self.config
- dt = cfg.jax_dtype
- x = x.astype(dt)
-
- for i in range(cfg.num_encoder_layers):
- x = EncoderBlock(
- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"
- )(x, mask=mask, rope=rope)
-
- x = ZCRMSNorm(dtype=dt)(x)
- return x
-
class Decoder(nn.Module):
config: TransformerConfig
@@ -242,7 +293,7 @@ class EncoderDecoderTransformer(nn.Module):
def setup(self):
self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))
self.embed_scale = math.sqrt(self.config.d_model)
- self.encoder = Encoder(self.config)
+ self.encoder = MemoryMixerEncoder(self.config)
self.decoder = Decoder(self.config)
def _rope(self, seq_len):
@@ -251,14 +302,13 @@ class EncoderDecoderTransformer(nn.Module):
def encode(self, src, src_mask=None):
x = self.embedding(src) * self.embed_scale
- rope = self._rope(src.shape[1])
- return self.encoder(x, mask=src_mask, rope=rope)
+ return self.encoder(x, mask=src_mask)
def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None):
x = self.embedding(tgt) * self.embed_scale
rope = self._rope(tgt.shape[1])
x = self.decoder(
- x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope
+ x, encoder_out, self_mask=self_mask, cross_mask=None, rope=rope
)
logits = x.astype(jnp.float32) @ self.embedding.embedding.T
return logits
@@ -266,7 +316,7 @@ class EncoderDecoderTransformer(nn.Module):
def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):
encoder_out = self.encode(src, src_mask=src_mask)
logits = self.decode(
- tgt, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask
+ tgt, encoder_out, self_mask=tgt_mask
)
return logits
diff --git a/src/train.py b/src/train.py
index 3f2d7c5..76d7bba 100644
--- a/src/train.py
+++ b/src/train.py
@@ -388,6 +388,7 @@ def train(args):
max_seq_len=max(args.max_enc_len, args.max_dec_len),
dtype=args.dtype,
activation=getattr(args, "activation", "drelu"),
+ num_memory_slots=getattr(args, "num_memory_slots", 64),
)
# Set group size for QAT and sparsification (captured by _train_step closure)