Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
strided convolution
HenryNdubuaku3c885b482ff0
diff --git a/src/model.py b/src/model.pyindex a8140ba..6f8a5d2 100644--- a/src/model.py+++ b/src/model.py@@ -1,6 +1,7 @@import mathfrom dataclasses import dataclass+import jaximport jax.numpy as jnpimport jax.nn.initializers as jinitimport flax.linen as nn@@ -140,6 +141,66 @@ class FeedForward(nn.Module):return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)+class GatedDeltaNet(nn.Module):+ """Gated Delta Network: linear-time recurrence replacing causal self-attention.++ Maintains a (head_dim, head_dim) state matrix per head, updated via the delta rule:+ S_t = S_{t-1} + β_t · k_t ⊗ (v_t − S_{t-1}^T k_t)+ o_t = q_t · S_t+ Inherently causal — no mask needed. O(T·d²) training, O(d²) per decode step.+ """+ num_heads: int+ d_model: int+ num_layers: int+ dtype: jnp.dtype = jnp.bfloat16++ @nn.compact+ def __call__(self, x):+ B, T, _ = x.shape+ head_dim = self.d_model // self.num_heads++ q = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,+ kernel_init=default_init(), name="q_proj")(x)+ k = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,+ kernel_init=default_init(), name="k_proj")(x)+ v = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,+ kernel_init=default_init(), name="v_proj")(x)+ beta = nn.Dense(self.num_heads, dtype=self.dtype, use_bias=True,+ name="beta_proj")(x)+ beta = nn.sigmoid(beta) # (B, T, H)++ q = q.reshape(B, T, self.num_heads, head_dim).transpose(0, 2, 1, 3)+ k = k.reshape(B, T, self.num_heads, head_dim).transpose(0, 2, 1, 3)+ v = v.reshape(B, T, self.num_heads, head_dim).transpose(0, 2, 1, 3)+ beta = beta.transpose(0, 2, 1) # (B, H, T)++ # L2-normalise keys for a stable delta rule+ k = k / (jnp.linalg.norm(k, axis=-1, keepdims=True) + 1e-6)++ def delta_step(S, inputs):+ q_t, k_t, v_t, b_t = inputs+ v_old = k_t @ S # (head_dim,)+ S = S + b_t * jnp.outer(k_t, v_t - v_old) # (hd, hd)+ o_t = q_t @ S # (head_dim,)+ return S, o_t++ def per_head(q_h, k_h, v_h, b_h):+ S0 = jnp.zeros((head_dim, head_dim), dtype=jnp.float32)+ _, o = jax.lax.scan(delta_step, S0,+ (q_h.astype(jnp.float32),+ k_h.astype(jnp.float32),+ v_h.astype(jnp.float32),+ b_h.astype(jnp.float32)))+ return o # (T, head_dim)++ out = jax.vmap(jax.vmap(per_head))(q, k, v, beta) # (B, H, T, hd)+ out = ZCRMSNorm(dtype=self.dtype, name="out_norm")(out)+ out = out.transpose(0, 2, 1, 3).reshape(B, T, self.d_model)+ return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,+ kernel_init=residual_init(self.num_layers),+ name="out_proj")(out)++class MLPMixer(nn.Module):"""MLP-Mixer operating on fixed-size memory slots (B, M, d)."""num_slots: int@@ -228,6 +289,19 @@ class MemoryMixerEncoder(nn.Module):s_bias = nn.Dense(cfg.d_model, dtype=dt, use_bias=False, kernel_init=jinit.zeros, name="slot_init")(x_pool)s = jnp.broadcast_to(s_base.astype(dt), (x.shape[0], cfg.num_memory_slots, cfg.d_model)) + s_bias[:, None, :]+ x = nn.Conv(features=cfg.d_model, kernel_size=(4,), strides=(2,),+ padding='SAME', feature_group_count=cfg.d_model,+ dtype=dt, use_bias=False, name="downsample_dw")(x)+ x = nn.Dense(cfg.d_model, dtype=dt, use_bias=False,+ kernel_init=default_init(), name="downsample_pw")(x)++ if mask is not None:+ T_new = x.shape[1]+ if mask.shape[-1] % 2:+ mask = jnp.pad(mask, ((0, 0), (0, 0), (0, 0), (0, 1)))+ mask = mask.reshape(mask.shape[0], 1, 1, -1, 2).any(axis=-1)+ mask = mask[..., :T_new]+for i in range(cfg.num_encoder_layers):x, s = MemoryMixerBlock(cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,@@ -249,11 +323,10 @@ class DecoderBlock(nn.Module):@nn.compactdef __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None):+ # Gated Delta Net (inherently causal — self_mask and rope unused)residual = xx = ZCRMSNorm(dtype=self.dtype)(x)- x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(- x, x, mask=self_mask, rope=rope- )+ x = GatedDeltaNet(self.num_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(x)x = x + residualresidual = xdiff --git a/src/train.py b/src/train.pyindex 668203e..ccf9a5a 100644--- a/src/train.py+++ b/src/train.py@@ -53,7 +53,7 @@ def scale_by_muon(momentum=0.95, ns_steps=5):del paramsdef ortho(g):- if g.ndim >= 2:+ if g.ndim == 2:return _newton_schulz(g, steps=ns_steps)return g@@ -70,9 +70,9 @@ def scale_by_muon(momentum=0.95, ns_steps=5):def _param_labels(params):"""Label each param: 'muon' for Dense kernels, 'adam' for the rest."""- def _label(path, _):+ def _label(path, leaf):name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])- if name == "kernel":+ if name == "kernel" and leaf.ndim == 2:return "muon"return "adam"@@ -325,7 +325,7 @@ def _quantize_params(params, group_size=32):"""Fake-quantize all Dense kernels in the param tree."""def _maybe_quantize(path, leaf):name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])- if name == "kernel":+ if name == "kernel" and leaf.ndim == 2:return _fake_quantize_int4(leaf, group_size=group_size)return leafreturn jax.tree_util.tree_map_with_path(_maybe_quantize, params)