Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Zero-centred RMS Norm
HenryNdubuaku812ae1fa229d
diff --git a/src/model.py b/src/model.pyindex 99624eb..1956e19 100644--- a/src/model.py+++ b/src/model.py@@ -17,6 +17,18 @@ def residual_init(num_layers):DTYPE_MAP = {"float32": jnp.float32, "bfloat16": jnp.bfloat16, "float16": jnp.float16}+class ZCRMSNorm(nn.Module):+ """Zero-centred RMSNorm: scale initialized to 0, applied as (1 + γ) * x / RMS(x)."""+ epsilon: float = 1e-6+ dtype: jnp.dtype = jnp.bfloat16++ @nn.compact+ def __call__(self, x):+ scale = self.param("scale", jinit.zeros, (x.shape[-1],))+ rms = jnp.sqrt(jnp.mean(x.astype(jnp.float32) ** 2, axis=-1, keepdims=True) + self.epsilon)+ return ((1 + scale) * x / rms).astype(self.dtype)++@dataclassclass TransformerConfig:vocab_size: int = 50257@@ -80,8 +92,8 @@ class MultiHeadAttention(nn.Module):k = k.reshape(B, -1, self.num_kv_heads, head_dim).transpose(0, 2, 1, 3)v = v.reshape(B, -1, self.num_kv_heads, head_dim).transpose(0, 2, 1, 3)- q = nn.RMSNorm(dtype=self.dtype, name="q_norm")(q)- k = nn.RMSNorm(dtype=self.dtype, name="k_norm")(k)+ q = ZCRMSNorm(dtype=self.dtype, name="q_norm")(q)+ k = ZCRMSNorm(dtype=self.dtype, name="k_norm")(k)# Repeat KV heads to match Q headsrepeats = self.num_heads // self.num_kv_heads@@ -137,7 +149,7 @@ class EncoderBlock(nn.Module):@nn.compactdef __call__(self, x, mask=None, rope=None, deterministic=True):residual = x- x = nn.RMSNorm(dtype=self.dtype)(x)+ x = ZCRMSNorm(dtype=self.dtype)(x)x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dropout_rate, self.dtype)(x, x, mask=mask, rope=rope, deterministic=deterministic)@@ -145,7 +157,7 @@ class EncoderBlock(nn.Module):x = x + residualresidual = x- x = nn.RMSNorm(dtype=self.dtype)(x)+ x = ZCRMSNorm(dtype=self.dtype)(x)x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dropout_rate, self.dtype)(x, deterministic=deterministic)x = x + residual@@ -165,7 +177,7 @@ class DecoderBlock(nn.Module):def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None, deterministic=True):# Self-attention with RoPEresidual = x- x = nn.RMSNorm(dtype=self.dtype)(x)+ x = ZCRMSNorm(dtype=self.dtype)(x)x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dropout_rate, self.dtype, name="self_attn")(x, x, mask=self_mask, rope=rope, deterministic=deterministic)@@ -174,7 +186,7 @@ class DecoderBlock(nn.Module):# Cross-attention (no RoPE)residual = x- x = nn.RMSNorm(dtype=self.dtype)(x)+ x = ZCRMSNorm(dtype=self.dtype)(x)x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dropout_rate, self.dtype, name="cross_attn")(x, encoder_out, mask=cross_mask, deterministic=deterministic)@@ -182,7 +194,7 @@ class DecoderBlock(nn.Module):x = x + residualresidual = x- x = nn.RMSNorm(dtype=self.dtype)(x)+ x = ZCRMSNorm(dtype=self.dtype)(x)x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dropout_rate, self.dtype)(x, deterministic=deterministic)x = x + residual@@ -204,7 +216,7 @@ class Encoder(nn.Module):cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, cfg.dropout_rate, dt, name=f"block_{i}")(x, mask=mask, rope=rope, deterministic=deterministic)- x = nn.RMSNorm(dtype=dt)(x)+ x = ZCRMSNorm(dtype=dt)(x)return x@@ -223,7 +235,7 @@ class Decoder(nn.Module):cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, cfg.dropout_rate, dt, name=f"block_{i}")(x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope, deterministic=deterministic)- x = nn.RMSNorm(dtype=dt)(x)+ x = ZCRMSNorm(dtype=dt)(x)return x