Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
orthogonalisation
HenryNdubuaku779293f7175a
diff --git a/README.md b/README.mdindex 069fd1d..547d7bb 100644--- a/README.md+++ b/README.md@@ -1,46 +1,58 @@```- ┌────────────────────────────────────────────────────────┐- │ │- │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │- │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │- │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │- │ ...the tiny model to rule them all... │- └────────────────────────────────────────────────────────┘+ ┌────────────────────────────────────────────────────────┐+ │ │+ │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │+ │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │+ │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │+ │ ...the tiny model to rule them all... │+ └────────────────────────────────────────────────────────┘++ Architecture Training Pipeline+ ──────────── ─────────────────┌─────────────┐- │ Softmax │- └──────┬──────┘- ┌──────┴──────┐- │ Linear (T) │ ← tied weights- └──────┬──────┘- ┌──────┴──────┐- │ LayerNorm │- └──────┬──────┘- ┌─────────┴─────────┐- │ Decoder x 2 │- │ ┌───────────────┐ │- │ │ Masked Self │ │- ┌──────────────┐ │ │ Attn + RoPE │ │- │ │ │ ├───────────────┤ │- │ Encoder x 2 │ │ │ Cross │ │- │ ┌──────────┐ │────────▶ Attention │ │- │ │ Self │ │ │ ├───────────────┤ │- │ │Attn+RoPE │ │ │ │ Feed-Forward │ │- │ ├──────────┤ │ │ └───────────────┘ │- │ │ Feed- │ │ └─────────┬─────────┘- │ │ Forward │ │ ┌──────┴──────┐- │ └──────────┘ │ │ Embedding │ ← shared- └──────┬───────┘ └──────┬──────┘- ┌──────┴───────┐ │- │ Embedding │ ← shared │- └──────┬───────┘ │- │ │- ┌────┴────┐ ┌─────┴─────┐- │ Encoder │ │ Decoder │- │ Input │ │ Input │- └─────────┘ └───────────┘-- ~7.5M params · d=128 · 4 heads · 2+2 layers · RoPE+ │ Softmax │ ┌──────────────────────┐+ └──────┬──────┘ │ Forward pass with │+ ┌──────┴──────┐ │ INT4 fake-quantized │+ │ Linear (T) │ ← tied │ weights (g=32, STE) │+ └──────┬──────┘ └──────────┬───────────┘+ ┌──────┴──────┐ │+ │ ZCRMSNorm │ ┌──────────┴───────────┐+ └──────┬──────┘ │ Muon (2D kernels) │+ ┌─────────┴─────────┐ │ AdamW (everything │+ │ Decoder x N │ │ else) │+ │ ┌───────────────┐ │ │ WSD LR schedule │+ │ │ Masked Self │ │ └──────────┬───────────┘+ │ │ Attn + RoPE │ │ │+ │ ├───────────────┤ │ ┌──────────┴───────────┐+ ┌──────────────┐ S │ │ Cross │ │ │ EMA params (β=0.999) │+ │ MemoryMixer │─────────▶ Attention │ │ └──────────┬───────────┘+ │ Encoder x N │ │ ├───────────────┤ │ │+ │ │ │ │ Feed-Forward │ │ ┌──────────┴───────────┐+ │ ┌──────────┐ │ │ │ (dReLU) │ │ │ Block Prune │+ │ │Pack: │ │ │ └───────────────┘ │ │ after epoch 1 │+ │ │ S←X Attn │ │ └─────────┬─────────┘ │ group magnitude │+ │ ├──────────┤ │ ┌──────┴──────┐ │ lock sparsity mask │+ │ │Mix: │ │ │ Embedding │ ← shared └──────────┬───────────┘+ │ │ MLP-Mixer│ │ └──────┬──────┘ │+ │ │ on S │ │ │ ┌──────────┴───────────┐+ │ ├──────────┤ │ ┌─────┴─────┐ │ Layer Prune │+ │ │Local: │ │ │ Decoder │ │ after epoch 2 │+ │ │ FFN on X │ │ │ Input │ │ L1 block scoring │+ │ └──────────┘ │ └───────────┘ │ keep ≥1 enc & dec │+ │ S ∈ (M, d) │ └──────────┬───────────┘+ └──────┬───────┘ │+ ┌──────┴───────┐ ┌──────────┴───────────┐+ │ Embedding │ ← shared │ Checkpoint │+ └──────┬───────┘ │ sparse + INT4 │+ │ └──────────────────────┘+ ┌────┴────┐+ │ Encoder │+ │ Input │+ └─────────┘++ d=128 · 4 heads · 2 KV heads · 64 memory slots+ dReLU · ZCRMSNorm · RoPE · INT4 QAT · Muon```## Usage@@ -62,7 +74,6 @@ needle [command]│ --d-model INT Model dimension (default: 128) ││ --num-heads INT Attention heads (default: 4) ││ --num-layers INT Encoder/decoder layers (default: 2) │- │ --dropout FLOAT Dropout rate (default: 0.1) ││ --max-enc-len INT Max encoder seq length (default: 128) ││ --max-dec-len INT Max decoder seq length (default: 128) ││ --max-samples INT Training samples (default: 20000) │diff --git a/src/cli.py b/src/cli.pyindex 6478924..52a8359 100644--- a/src/cli.py+++ b/src/cli.py@@ -138,6 +138,7 @@ def main():p.add_argument("--group-size", type=int, default=32)p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])p.add_argument("--num-memory-slots", type=int, default=64)+ p.add_argument("--ortho-weight", type=float, default=1e-4)p = sub.add_parser("run", add_help=False)p.add_argument("--checkpoint", type=str, required=True)diff --git a/src/train.py b/src/train.pyindex 76d7bba..b58307a 100644--- a/src/train.py+++ b/src/train.py@@ -282,8 +282,29 @@ def _quantize_params(params, group_size=32):return jax.tree_util.tree_map_with_path(_maybe_quantize, params)-# Module-level group_size used by _train_step (set before pmap in train())+# Module-level config used by _train_step (set before pmap in train())_GROUP_SIZE = 32+_ORTHO_WEIGHT = 1e-4+++def _ortho_reg(params):+ """Orthogonal regularization: avg ||WᵀW - I||²_F over 2D kernels."""+ penalty = jnp.float32(0.0)+ count = 0+ for path, leaf in jax.tree_util.tree_leaves_with_path(params):+ name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])+ if name == "kernel" and leaf.ndim == 2:+ w = leaf.astype(jnp.float32)+ m, n = w.shape+ if m >= n:+ gram = w.T @ w+ eye = jnp.eye(n)+ else:+ gram = w @ w.T+ eye = jnp.eye(m)+ penalty = penalty + jnp.mean((gram - eye) ** 2)+ count += 1+ return penalty / max(count, 1)def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):@@ -309,7 +330,8 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):mask = (tgt_out != pad_id).astype(jnp.float32)ce_loss = jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)- return ce_loss + z_loss+ ortho_loss = _ORTHO_WEIGHT * _ortho_reg(params)+ return ce_loss + z_loss + ortho_lossloss, grads = jax.value_and_grad(loss_fn)(state.params)grads = jax.lax.pmean(grads, axis_name="batch")@@ -391,9 +413,10 @@ def train(args):num_memory_slots=getattr(args, "num_memory_slots", 64),)- # Set group size for QAT and sparsification (captured by _train_step closure)- global _GROUP_SIZE+ # Set module-level config (captured by _train_step closure)+ global _GROUP_SIZE, _ORTHO_WEIGHT_GROUP_SIZE = getattr(args, "group_size", 32)+ _ORTHO_WEIGHT = getattr(args, "ortho_weight", 1e-4)p_train_step = _make_p_train_step()np.random.seed(args.seed)