Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

orthogonalisation

HenryNdubuaku779293f7175a
diff --git a/README.md b/README.md
index 069fd1d..547d7bb 100644
--- a/README.md
+++ b/README.md
@@ -1,46 +1,58 @@
```
- ┌────────────────────────────────────────────────────────┐
- │ │
- │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │
- │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │
- │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │
- │ ...the tiny model to rule them all... │
- └────────────────────────────────────────────────────────┘
+ ┌────────────────────────────────────────────────────────┐
+ │ │
+ │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │
+ │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │
+ │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │
+ │ ...the tiny model to rule them all... │
+ └────────────────────────────────────────────────────────┘
+
+ Architecture Training Pipeline
+ ──────────── ─────────────────
┌─────────────┐
- │ Softmax │
- └──────┬──────┘
- ┌──────┴──────┐
- │ Linear (T) │ ← tied weights
- └──────┬──────┘
- ┌──────┴──────┐
- │ LayerNorm │
- └──────┬──────┘
- ┌─────────┴─────────┐
- │ Decoder x 2 │
- │ ┌───────────────┐ │
- │ │ Masked Self │ │
- ┌──────────────┐ │ │ Attn + RoPE │ │
- │ │ │ ├───────────────┤ │
- │ Encoder x 2 │ │ │ Cross │ │
- │ ┌──────────┐ │────────▶ Attention │ │
- │ │ Self │ │ │ ├───────────────┤ │
- │ │Attn+RoPE │ │ │ │ Feed-Forward │ │
- │ ├──────────┤ │ │ └───────────────┘ │
- │ │ Feed- │ │ └─────────┬─────────┘
- │ │ Forward │ │ ┌──────┴──────┐
- │ └──────────┘ │ │ Embedding │ ← shared
- └──────┬───────┘ └──────┬──────┘
- ┌──────┴───────┐ │
- │ Embedding │ ← shared │
- └──────┬───────┘ │
- │ │
- ┌────┴────┐ ┌─────┴─────┐
- │ Encoder │ │ Decoder │
- │ Input │ │ Input │
- └─────────┘ └───────────┘
-
- ~7.5M params · d=128 · 4 heads · 2+2 layers · RoPE
+ │ Softmax │ ┌──────────────────────┐
+ └──────┬──────┘ │ Forward pass with │
+ ┌──────┴──────┐ │ INT4 fake-quantized │
+ │ Linear (T) │ ← tied │ weights (g=32, STE) │
+ └──────┬──────┘ └──────────┬───────────┘
+ ┌──────┴──────┐ │
+ │ ZCRMSNorm │ ┌──────────┴───────────┐
+ └──────┬──────┘ │ Muon (2D kernels) │
+ ┌─────────┴─────────┐ │ AdamW (everything │
+ │ Decoder x N │ │ else) │
+ │ ┌───────────────┐ │ │ WSD LR schedule │
+ │ │ Masked Self │ │ └──────────┬───────────┘
+ │ │ Attn + RoPE │ │ │
+ │ ├───────────────┤ │ ┌──────────┴───────────┐
+ ┌──────────────┐ S │ │ Cross │ │ │ EMA params (β=0.999) │
+ │ MemoryMixer │─────────▶ Attention │ │ └──────────┬───────────┘
+ │ Encoder x N │ │ ├───────────────┤ │ │
+ │ │ │ │ Feed-Forward │ │ ┌──────────┴───────────┐
+ │ ┌──────────┐ │ │ │ (dReLU) │ │ │ Block Prune │
+ │ │Pack: │ │ │ └───────────────┘ │ │ after epoch 1 │
+ │ │ S←X Attn │ │ └─────────┬─────────┘ │ group magnitude │
+ │ ├──────────┤ │ ┌──────┴──────┐ │ lock sparsity mask │
+ │ │Mix: │ │ │ Embedding │ ← shared └──────────┬───────────┘
+ │ │ MLP-Mixer│ │ └──────┬──────┘ │
+ │ │ on S │ │ │ ┌──────────┴───────────┐
+ │ ├──────────┤ │ ┌─────┴─────┐ │ Layer Prune │
+ │ │Local: │ │ │ Decoder │ │ after epoch 2 │
+ │ │ FFN on X │ │ │ Input │ │ L1 block scoring │
+ │ └──────────┘ │ └───────────┘ │ keep ≥1 enc & dec │
+ │ S ∈ (M, d) │ └──────────┬───────────┘
+ └──────┬───────┘ │
+ ┌──────┴───────┐ ┌──────────┴───────────┐
+ │ Embedding │ ← shared │ Checkpoint │
+ └──────┬───────┘ │ sparse + INT4 │
+ │ └──────────────────────┘
+ ┌────┴────┐
+ │ Encoder │
+ │ Input │
+ └─────────┘
+
+ d=128 · 4 heads · 2 KV heads · 64 memory slots
+ dReLU · ZCRMSNorm · RoPE · INT4 QAT · Muon
```
## Usage
@@ -62,7 +74,6 @@ needle [command]
│ --d-model INT Model dimension (default: 128) │
│ --num-heads INT Attention heads (default: 4) │
│ --num-layers INT Encoder/decoder layers (default: 2) │
- │ --dropout FLOAT Dropout rate (default: 0.1) │
│ --max-enc-len INT Max encoder seq length (default: 128) │
│ --max-dec-len INT Max decoder seq length (default: 128) │
│ --max-samples INT Training samples (default: 20000) │
diff --git a/src/cli.py b/src/cli.py
index 6478924..52a8359 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -138,6 +138,7 @@ def main():
p.add_argument("--group-size", type=int, default=32)
p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])
p.add_argument("--num-memory-slots", type=int, default=64)
+ p.add_argument("--ortho-weight", type=float, default=1e-4)
p = sub.add_parser("run", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
diff --git a/src/train.py b/src/train.py
index 76d7bba..b58307a 100644
--- a/src/train.py
+++ b/src/train.py
@@ -282,8 +282,29 @@ def _quantize_params(params, group_size=32):
return jax.tree_util.tree_map_with_path(_maybe_quantize, params)
-# Module-level group_size used by _train_step (set before pmap in train())
+# Module-level config used by _train_step (set before pmap in train())
_GROUP_SIZE = 32
+_ORTHO_WEIGHT = 1e-4
+
+
+def _ortho_reg(params):
+ """Orthogonal regularization: avg ||WᵀW - I||²_F over 2D kernels."""
+ penalty = jnp.float32(0.0)
+ count = 0
+ for path, leaf in jax.tree_util.tree_leaves_with_path(params):
+ name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])
+ if name == "kernel" and leaf.ndim == 2:
+ w = leaf.astype(jnp.float32)
+ m, n = w.shape
+ if m >= n:
+ gram = w.T @ w
+ eye = jnp.eye(n)
+ else:
+ gram = w @ w.T
+ eye = jnp.eye(m)
+ penalty = penalty + jnp.mean((gram - eye) ** 2)
+ count += 1
+ return penalty / max(count, 1)
def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):
@@ -309,7 +330,8 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):
mask = (tgt_out != pad_id).astype(jnp.float32)
ce_loss = jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)
z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)
- return ce_loss + z_loss
+ ortho_loss = _ORTHO_WEIGHT * _ortho_reg(params)
+ return ce_loss + z_loss + ortho_loss
loss, grads = jax.value_and_grad(loss_fn)(state.params)
grads = jax.lax.pmean(grads, axis_name="batch")
@@ -391,9 +413,10 @@ def train(args):
num_memory_slots=getattr(args, "num_memory_slots", 64),
)
- # Set group size for QAT and sparsification (captured by _train_step closure)
- global _GROUP_SIZE
+ # Set module-level config (captured by _train_step closure)
+ global _GROUP_SIZE, _ORTHO_WEIGHT
_GROUP_SIZE = getattr(args, "group_size", 32)
+ _ORTHO_WEIGHT = getattr(args, "ortho_weight", 1e-4)
p_train_step = _make_p_train_step()
np.random.seed(args.seed)