Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
revert gated delta net
HenryNdubuaku18ac111b92d0
diff --git a/src/cli.py b/src/cli.pyindex 160bbe8..b8b2255 100644--- a/src/cli.py+++ b/src/cli.py@@ -4,10 +4,10 @@ import sysHELP = """┌───────────────────────────────────────────────────────────────────┐│ │- │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │- │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │- │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │- │ ...the tiny model to rule them all... │+ │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │+ │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │+ │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │+ │ ...the tiny model to rule them all... ││ ││ train ││ --toy Use toy config for quick iteration │@@ -52,11 +52,6 @@ HELP = """│ --temperature FLOAT Sampling temperature (default: 0.8) ││ --throughput-runs INT Throughput runs (default: 10) ││ │- │ export │- │ --checkpoint PATH Path to model checkpoint (required) │- │ --dim INT Target d_model dimension (required) │- │ --output PATH Output path (default: auto-named) │- │ ││ evaluate ││ --checkpoint PATH Path to model checkpoint (required) ││ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │@@ -87,11 +82,9 @@ TOY_CONFIG = {}MAIN_CONFIG = {- "num_heads": 16,- "num_kv_heads": 8,- "num_layers": 28,- "num_dec_layers": 8,- "d_ff": 4096,+ "num_heads": 4,+ "num_layers": 24,+ "num_dec_layers": 4,"max_enc_len": 256,"max_dec_len": 256,"max_samples": None,@@ -124,10 +117,8 @@ def main():p.add_argument("--muon-lr", type=float, default=0.02)p.add_argument("--d-model", type=int, default=None)p.add_argument("--num-heads", type=int, default=None)- p.add_argument("--num-kv-heads", type=int, default=None)p.add_argument("--num-layers", type=int, default=None)p.add_argument("--num-dec-layers", type=int, default=None)- p.add_argument("--d-ff", type=int, default=None)p.add_argument("--max-enc-len", type=int, default=None)p.add_argument("--max-dec-len", type=int, default=None)p.add_argument("--max-samples", type=int, default=None)@@ -169,11 +160,6 @@ def main():p.add_argument("--temperature", type=float, default=0.8)p.add_argument("--throughput-runs", type=int, default=10)- p = sub.add_parser("export", add_help=False)- p.add_argument("--checkpoint", type=str, required=True)- p.add_argument("--dim", type=int, required=True)- p.add_argument("--output", type=str, default=None)-p = sub.add_parser("evaluate", add_help=False)p.add_argument("--checkpoint", type=str, required=True)p.add_argument("--benchmarks", type=str, nargs="*",@@ -227,9 +213,6 @@ def main():elif args.command == "test":from .test import main as test_maintest_main(args)- elif args.command == "export":- from .export import main as export_main- export_main(args)elif args.command == "evaluate":from .evaluate import main as eval_maineval_main(args)diff --git a/src/model.py b/src/model.pyindex 6f8a5d2..a60fa03 100644--- a/src/model.py+++ b/src/model.py@@ -1,7 +1,6 @@import mathfrom dataclasses import dataclass-import jaximport jax.numpy as jnpimport jax.nn.initializers as jinitimport flax.linen as nn@@ -141,66 +140,6 @@ class FeedForward(nn.Module):return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)-class GatedDeltaNet(nn.Module):- """Gated Delta Network: linear-time recurrence replacing causal self-attention.-- Maintains a (head_dim, head_dim) state matrix per head, updated via the delta rule:- S_t = S_{t-1} + β_t · k_t ⊗ (v_t − S_{t-1}^T k_t)- o_t = q_t · S_t- Inherently causal — no mask needed. O(T·d²) training, O(d²) per decode step.- """- num_heads: int- d_model: int- num_layers: int- dtype: jnp.dtype = jnp.bfloat16-- @nn.compact- def __call__(self, x):- B, T, _ = x.shape- head_dim = self.d_model // self.num_heads-- q = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,- kernel_init=default_init(), name="q_proj")(x)- k = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,- kernel_init=default_init(), name="k_proj")(x)- v = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,- kernel_init=default_init(), name="v_proj")(x)- beta = nn.Dense(self.num_heads, dtype=self.dtype, use_bias=True,- name="beta_proj")(x)- beta = nn.sigmoid(beta) # (B, T, H)-- q = q.reshape(B, T, self.num_heads, head_dim).transpose(0, 2, 1, 3)- k = k.reshape(B, T, self.num_heads, head_dim).transpose(0, 2, 1, 3)- v = v.reshape(B, T, self.num_heads, head_dim).transpose(0, 2, 1, 3)- beta = beta.transpose(0, 2, 1) # (B, H, T)-- # L2-normalise keys for a stable delta rule- k = k / (jnp.linalg.norm(k, axis=-1, keepdims=True) + 1e-6)-- def delta_step(S, inputs):- q_t, k_t, v_t, b_t = inputs- v_old = k_t @ S # (head_dim,)- S = S + b_t * jnp.outer(k_t, v_t - v_old) # (hd, hd)- o_t = q_t @ S # (head_dim,)- return S, o_t-- def per_head(q_h, k_h, v_h, b_h):- S0 = jnp.zeros((head_dim, head_dim), dtype=jnp.float32)- _, o = jax.lax.scan(delta_step, S0,- (q_h.astype(jnp.float32),- k_h.astype(jnp.float32),- v_h.astype(jnp.float32),- b_h.astype(jnp.float32)))- return o # (T, head_dim)-- out = jax.vmap(jax.vmap(per_head))(q, k, v, beta) # (B, H, T, hd)- out = ZCRMSNorm(dtype=self.dtype, name="out_norm")(out)- out = out.transpose(0, 2, 1, 3).reshape(B, T, self.d_model)- return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False,- kernel_init=residual_init(self.num_layers),- name="out_proj")(out)--class MLPMixer(nn.Module):"""MLP-Mixer operating on fixed-size memory slots (B, M, d)."""num_slots: int@@ -323,10 +262,11 @@ class DecoderBlock(nn.Module):@nn.compactdef __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None):- # Gated Delta Net (inherently causal — self_mask and rope unused)residual = xx = ZCRMSNorm(dtype=self.dtype)(x)- x = GatedDeltaNet(self.num_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(x)+ x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(+ x, x, mask=self_mask, rope=rope+ )x = x + residualresidual = x