Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

strided convolution

HenryNdubuaku3c885b482ff0
diff --git a/src/model.py b/src/model.py
index a8140ba..6f8a5d2 100644
--- a/src/model.py
+++ b/src/model.py
@@ -1,6 +1,7 @@
import math
from dataclasses import dataclass
+import jax
import jax.numpy as jnp
import jax.nn.initializers as jinit
import flax.linen as nn
@@ -140,6 +141,66 @@ class FeedForward(nn.Module):
return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)
+class GatedDeltaNet(nn.Module):
+ """Gated Delta Network: linear-time recurrence replacing causal self-attention.
+
+ Maintains a (head_dim, head_dim) state matrix per head, updated via the delta rule:
+ S_t = S_{t-1} + β_t · k_t ⊗ (v_t − S_{t-1}^T k_t)
+ o_t = q_t · S_t
+ Inherently causal — no mask needed. O(T·d²) training, O(d²) per decode step.
+ """
+ num_heads: int
+ d_model: int
+ num_layers: int
+ dtype: jnp.dtype = jnp.bfloat16
+
+ @nn.compact
+ def __call__(self, x):
+ B, T, _ = x.shape
+ head_dim = self.d_model // self.num_heads
+
+ q = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,
+ kernel_init=default_init(), name="q_proj")(x)
+ k = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,
+ kernel_init=default_init(), name="k_proj")(x)
+ v = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,
+ kernel_init=default_init(), name="v_proj")(x)
+ beta = nn.Dense(self.num_heads, dtype=self.dtype, use_bias=True,
+ name="beta_proj")(x)
+ beta = nn.sigmoid(beta) # (B, T, H)
+
+ q = q.reshape(B, T, self.num_heads, head_dim).transpose(0, 2, 1, 3)
+ k = k.reshape(B, T, self.num_heads, head_dim).transpose(0, 2, 1, 3)
+ v = v.reshape(B, T, self.num_heads, head_dim).transpose(0, 2, 1, 3)
+ beta = beta.transpose(0, 2, 1) # (B, H, T)
+
+ # L2-normalise keys for a stable delta rule
+ k = k / (jnp.linalg.norm(k, axis=-1, keepdims=True) + 1e-6)
+
+ def delta_step(S, inputs):
+ q_t, k_t, v_t, b_t = inputs
+ v_old = k_t @ S # (head_dim,)
+ S = S + b_t * jnp.outer(k_t, v_t - v_old) # (hd, hd)
+ o_t = q_t @ S # (head_dim,)
+ return S, o_t
+
+ def per_head(q_h, k_h, v_h, b_h):
+ S0 = jnp.zeros((head_dim, head_dim), dtype=jnp.float32)
+ _, o = jax.lax.scan(delta_step, S0,
+ (q_h.astype(jnp.float32),
+ k_h.astype(jnp.float32),
+ v_h.astype(jnp.float32),
+ b_h.astype(jnp.float32)))
+ return o # (T, head_dim)
+
+ out = jax.vmap(jax.vmap(per_head))(q, k, v, beta) # (B, H, T, hd)
+ out = ZCRMSNorm(dtype=self.dtype, name="out_norm")(out)
+ out = out.transpose(0, 2, 1, 3).reshape(B, T, self.d_model)
+ return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,
+ kernel_init=residual_init(self.num_layers),
+ name="out_proj")(out)
+
+
class MLPMixer(nn.Module):
"""MLP-Mixer operating on fixed-size memory slots (B, M, d)."""
num_slots: int
@@ -228,6 +289,19 @@ class MemoryMixerEncoder(nn.Module):
s_bias = nn.Dense(cfg.d_model, dtype=dt, use_bias=False, kernel_init=jinit.zeros, name="slot_init")(x_pool)
s = jnp.broadcast_to(s_base.astype(dt), (x.shape[0], cfg.num_memory_slots, cfg.d_model)) + s_bias[:, None, :]
+ x = nn.Conv(features=cfg.d_model, kernel_size=(4,), strides=(2,),
+ padding='SAME', feature_group_count=cfg.d_model,
+ dtype=dt, use_bias=False, name="downsample_dw")(x)
+ x = nn.Dense(cfg.d_model, dtype=dt, use_bias=False,
+ kernel_init=default_init(), name="downsample_pw")(x)
+
+ if mask is not None:
+ T_new = x.shape[1]
+ if mask.shape[-1] % 2:
+ mask = jnp.pad(mask, ((0, 0), (0, 0), (0, 0), (0, 1)))
+ mask = mask.reshape(mask.shape[0], 1, 1, -1, 2).any(axis=-1)
+ mask = mask[..., :T_new]
+
for i in range(cfg.num_encoder_layers):
x, s = MemoryMixerBlock(
cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,
@@ -249,11 +323,10 @@ class DecoderBlock(nn.Module):
@nn.compact
def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None):
+ # Gated Delta Net (inherently causal — self_mask and rope unused)
residual = x
x = ZCRMSNorm(dtype=self.dtype)(x)
- x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(
- x, x, mask=self_mask, rope=rope
- )
+ x = GatedDeltaNet(self.num_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(x)
x = x + residual
residual = x
diff --git a/src/train.py b/src/train.py
index 668203e..ccf9a5a 100644
--- a/src/train.py
+++ b/src/train.py
@@ -53,7 +53,7 @@ def scale_by_muon(momentum=0.95, ns_steps=5):
del params
def ortho(g):
- if g.ndim >= 2:
+ if g.ndim == 2:
return _newton_schulz(g, steps=ns_steps)
return g
@@ -70,9 +70,9 @@ def scale_by_muon(momentum=0.95, ns_steps=5):
def _param_labels(params):
"""Label each param: 'muon' for Dense kernels, 'adam' for the rest."""
- def _label(path, _):
+ def _label(path, leaf):
name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])
- if name == "kernel":
+ if name == "kernel" and leaf.ndim == 2:
return "muon"
return "adam"
@@ -325,7 +325,7 @@ def _quantize_params(params, group_size=32):
"""Fake-quantize all Dense kernels in the param tree."""
def _maybe_quantize(path, leaf):
name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])
- if name == "kernel":
+ if name == "kernel" and leaf.ndim == 2:
return _fake_quantize_int4(leaf, group_size=group_size)
return leaf
return jax.tree_util.tree_map_with_path(_maybe_quantize, params)