Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
optimise
HenryNdubuaku73e4f4663ec0
diff --git a/README.md b/README.mdindex dfb37bc..692f384 100644--- a/README.md+++ b/README.md@@ -10,39 +10,43 @@Architecture Training Pipeline──────────── ─────────────────- ┌─────────────┐- │ Softmax │ ┌──────────────────────┐- └──────┬──────┘ │ MRL loss at dims │- ┌─────┬──────┴──────┬─────┐ │ {512,256,128,64} │- │ @E[:d'] for each d' │ │ + INT4 QAT (g=32) │- │ in mrl_dims │ │ + z-loss + slot div │- └─────┬──────┬──────┬─────┘ └──────────┬───────────┘- ┌──────┴──────┐ │- │ Linear (T) │ ← tied ┌──────────┴───────────┐- └──────┬──────┘ │ Grad clip (norm 1.0) │- ┌──────┴──────┐ │ Muon (2D kernels) │- │ ZCRMSNorm │ │ AdamW (all else) │- └──────┬──────┘ │ WSD LR schedule │- ┌─────────┴─────────┐ └──────────┬───────────┘- │ Decoder x N_dec │ │- │ ┌───────────────┐ │ ┌──────────┴───────────┐- │ │ Masked Self │ │ │ EMA params (β=0.999) │- │ │ Attn + RoPE │ │ └──────────┬───────────┘- │ ├───────────────┤ │ │- ┌──────────────┐ S │ │ Cross │ │ ┌──────────┴───────────┐- │ MemoryMixer │─────────▶ Attention │ │ │ Sparsification │- │ Encoder │ │ ├───────────────┤ │ │ cubic schedule │- │ x N_enc │ │ │ Gated FFN │ │ │ mask every N steps │- │ │ │ └───────────────┘ │ └──────────┬───────────┘- │ ┌──────────┐ │ └─────────┬─────────┘ │- │ │Pack: │ │ ┌──────┴──────┐ ┌──────────┴───────────┐- │ │ S←X Attn │ │ │ Embedding │ ← shared │ Checkpoint │- │ │ RoPE keys│ │ └──────┬──────┘ │ full params, can │- │ ├──────────┤ │ │ │ export MRL slices │- │ │Mix: │ │ ┌─────┴─────┐ └──────────────────────┘- │ │ MLP-Mixer│ │ │ Decoder │- │ │ on S │ │ │ Input │- │ ├──────────┤ │ └───────────┘+ ┌──────────────┐ ┌──────────────┐+ │ Transcription│ │ Tool JSON │ ┌──────────────────────┐+ └──────┬───────┘ └──────┬───────┘ │ MRL loss at dims │+ └───────┬─────────┘ │ + INT4 QAT (g=32) │+ ┌───┴───────┐ │ + z-loss + slot div │+ │ Softmax │ │ mixed text/speech │+ └─────┬─────┘ │ batches │+ ┌─────┬─────┴─────┬─────┐ └──────────┬───────────┘+ │ @E[:d'] for each d' │ │+ │ in mrl_dims │ ┌──────────┴───────────┐+ └─────┬─────┬─────┬─────┘ │ Grad clip (norm 1.0) │+ ┌─────┴─────┐ │ Muon (2D kernels) │+ │ Linear (T)│ ← tied │ AdamW (all else) │+ └─────┬─────┘ │ WSD LR schedule │+ ┌─────┴─────┐ └──────────┬───────────┘+ │ ZCRMSNorm │ │+ └─────┬─────┘ ┌──────────┴───────────┐+ ┌────────┴────────┐ │ EMA params (β=0.999) │+ │ Decoder x N_dec │ └──────────┬───────────┘+ │┌───────────────┐│ │+ ││ Masked Self ││ ┌──────────┴───────────┐+ ││ Attn + RoPE ││ │ Sparsification │+ │├───────────────┤│ │ cubic schedule │+ ┌──────────────┐ S ││ Cross ││ │ mask every N steps │+ │ MemoryMixer │────────▶ Attention ││ └──────────┬───────────┘+ │ Encoder │ │├───────────────┤│ │+ │ x N_enc │ ││ Gated FFN ││ ┌──────────┴───────────┐+ │ │ │└───────────────┘│ │ Checkpoint │+ │ ┌──────────┐ │ └────────┬────────┘ │ full params, can │+ │ │Pack: │ │ ┌─────┴─────┐ │ export MRL slices │+ │ │ S←X Attn │ │ │ Embedding │ ← shared └──────────────────────┘+ │ │ RoPE keys│ │ └─────┬─────┘+ │ ├──────────┤ │ ┌───────┴───────┐+ │ │Mix: │ │ │ <transcribe> │+ │ │ MLP-Mixer│ │ │ or <tools> │+ │ │ on S │ │ │ + target │+ │ ├──────────┤ │ └───────────────┘│ │Local: │ ││ │Gated FFN │ ││ │ on X │ │@@ -55,15 +59,15 @@┌──────┴───────┐│ Embedding │ ← shared└──────┬───────┘- │- ┌────┴────┐- │ Encoder │- │ Input │- └─────────┘+ ┌────┴─────────┐+ │ Text/Speech │+ │ Input │+ └──────────────┘- d=max(mrl_dims) · GQA (4 heads, 2 KV) · QK-norm · 64 slots+ d=512 (default) / 1536 (--full) · GQA · QK-normSentencePiece BPE (8192) · gated dReLU · ZCRMSNorm · RoPE- strided DW conv · Matryoshka dims · INT4 QAT · Muon + AdamW+ strided DW conv · MRL dims · INT4 QAT · Muon + AdamW+ text + speech encoder · <transcribe> / <tools> task routing```## Usage@@ -78,30 +82,32 @@ needle [command]┌───────────────────────────────────────────────────────────────────┐│ ││ train │- │ --epochs INT Training epochs (default: 1) │- │ --batch-size INT Batch size (default: 32) │- │ --lr FLOAT AdamW learning rate (default: 3e-4) │- │ --muon-lr FLOAT Muon learning rate (default: 0.02) │- │ --d-model INT Model dim (default: max of mrl-dims) │- │ --num-heads INT Attention heads (default: 4) │- │ --num-layers INT Encoder layers (default: 4) │- │ --num-dec-layers INT Decoder layers (default: 2) │- │ --max-enc-len INT Max encoder seq length (default: 256) │- │ --max-dec-len INT Max decoder seq length (default: 256) │- │ --max-samples INT Training samples (default: all) │- │ --mrl-dims INT [...] MRL dim targets (default: 1024 512..) │- │ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) │- │ --group-size INT Quant/prune group size (default: 32) │- │ --prune-interval INT Steps between mask updates (def: 100) │- │ --prune-start-frac FL Start pruning at this frac (def: 0.33) │- │ --prune-end-frac FL Lock mask at this frac (def: 0.67) │- │ --activation STR drelu|swiglu|geglu (default: drelu) │- │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │- │ --eval-every INT Val eval interval (default: 1000) │- │ --wandb Enable W&B logging │- │ --checkpoint PATH Resume from checkpoint │- │ --checkpoint-dir DIR Checkpoint directory │- │ --seed INT Random seed (default: 42) │+ │ --full Use full 1B config (~1.17B params) │+ │ --epochs INT Training epochs (default: 1) │+ │ --batch-size INT Batch size (default: 32) │+ │ --lr FLOAT AdamW learning rate (default: 3e-4) │+ │ --muon-lr FLOAT Muon learning rate (default: 0.02) │+ │ --d-model INT Model dim (default: 512) │+ │ --num-heads INT Attention heads (default: 8) │+ │ --num-kv-heads INT KV heads for GQA (default: num-heads) │+ │ --num-layers INT Encoder layers (default: 8) │+ │ --num-dec-layers INT Decoder layers (default: 4) │+ │ --max-enc-len INT Max encoder seq length (default: 256) │+ │ --max-dec-len INT Max decoder seq length (default: 256) │+ │ --max-samples INT Training samples (default: all) │+ │ --mrl-dims INT [...] MRL dim targets (default: 256 128 64) │+ │ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) │+ │ --group-size INT Quant/prune group size (default: 32) │+ │ --prune-interval INT Steps between mask updates (def: 100) │+ │ --prune-start-frac FL Start pruning at frac (def: 0.33) │+ │ --prune-end-frac FL Lock mask at this frac (def: 0.67) │+ │ --activation STR drelu|swiglu|geglu (default: drelu) │+ │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │+ │ --eval-every INT Val eval interval (default: 1000) │+ │ --wandb Enable W&B logging │+ │ --checkpoint PATH Resume from checkpoint │+ │ --checkpoint-dir DIR Checkpoint directory │+ │ --seed INT Random seed (default: 42) ││ ││ run ││ --checkpoint PATH Path to model checkpoint (required) │@@ -160,11 +166,29 @@ needle [command]~27 GB (from ~4.5 TB full dataset)──────────────────────────────────────────- 1B multimodal training estimates+ 100M quick-experiment estimates (--default)+ ──────────────────────────────────────────+ Total tokens 1B text + 1k hrs audio ≈ 1.2B+ Total FLOPs 6 × 102M × 1.2B ≈ 7.2e17+ MFU (100M model) ~15% → ~138 effective TFLOPS/chip++ ┌────────────────────┬──────────┬──────────┬──────────┐+ │ │ v6e-8 │ v6e-16 │ v6e-32 │+ ├────────────────────┼──────────┼──────────┼──────────┤+ │ Chips │ 8 │ 16 │ 32 │+ │ Total HBM │ 256 GB │ 512 GB │ 1024 GB │+ │ Scaling eff. │ 0.9× │ 0.8× │ 0.7× │+ │ Eff. TFLOPS │ 994 │ 1,766 │ 3,091 │+ │ Est. time │ ~12min │ ~7min │ ~4min │+ │ On-demand $/hr │ $21.60 │ $43.20 │ $86.40 │+ │ Est. total cost │ ~$4 │ ~$5 │ ~$6 │+ └────────────────────┴──────────┴──────────┴──────────┘++ 1B multimodal training estimates (--full)──────────────────────────────────────────- Total FLOPs 6 × 1B × 100B = 6e20+ Total tokens 100B text + 250k hrs audio ≈ 145B+ Total FLOPs 6 × 1.17B × 145B ≈ 1.0e21MFU (1B model) ~30% → ~275 effective TFLOPS/chip- Audio overhead <1% of total (negligible)┌────────────────────┬──────────┬──────────┬──────────┐│ │ v6e-8 │ v6e-16 │ v6e-32 │@@ -172,10 +196,10 @@ needle [command]│ Chips │ 8 │ 16 │ 32 ││ Total HBM │ 256 GB │ 512 GB │ 1024 GB ││ Scaling eff. │ 0.9× │ 0.8× │ 0.7× │- │ Eff. TFLOPS │ 1,983 │ 3,525 │ 6,160 │- │ Est. time │ ~84h │ ~47h │ ~27h │+ │ Eff. TFLOPS │ 1,980 │ 3,520 │ 6,160 │+ │ Est. time │ ~143h │ ~80h │ ~46h ││ On-demand $/hr │ $21.60 │ $43.20 │ $86.40 │- │ Est. total cost │ ~$1,810 │ ~$2,030 │ ~$2,330 │+ │ Est. total cost │ ~$3,090 │ ~$3,460 │ ~$3,970 │└────────────────────┴──────────┴──────────┴──────────┘```diff --git a/experiements/datasets.txt b/experiements/datasets.txtindex 555dc0c..e1ac4dc 100644--- a/experiements/datasets.txt+++ b/experiements/datasets.txt@@ -18,6 +18,7 @@ Note:1. LEAP makes sense as we wish the dataset to preprocess and pretokenize the inputs before injesting into the model.2. We are training a cutom sentencepiece tokenizer with a small vocab size, given how constrained the decode is.3. Requires solid cloud infra experience, latency in injestion will slow down training.+4. Can text to tools translate to voice to tools.4. Record all obersvations, they will go into the paper!5. Design the batching (mix of speech and text and tools) experimantally.6. We should have a <transcribe> and <tools> tag for the two tasks we train on.diff --git a/src/cli.py b/src/cli.pyindex 0349cec..037cbb6 100644--- a/src/cli.py+++ b/src/cli.py@@ -10,30 +10,32 @@ HELP = """│ ...the tiny model to rule them all... ││ ││ train │- │ --epochs INT Training epochs (default: 1) │- │ --batch-size INT Batch size (default: 32) │- │ --lr FLOAT AdamW learning rate (default: 3e-4) │- │ --muon-lr FLOAT Muon learning rate (default: 0.02) │- │ --d-model INT Model dim (default: max of mrl-dims) │- │ --num-heads INT Attention heads (default: 4) │- │ --num-layers INT Encoder layers (default: 4) │- │ --num-dec-layers INT Decoder layers (default: 2) │- │ --max-enc-len INT Max encoder seq length (default: 256) │- │ --max-dec-len INT Max decoder seq length (default: 256) │- │ --max-samples INT Training samples (default: all) │- │ --mrl-dims INT [...] MRL dim targets (default: 1024 512..) │- │ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) │- │ --group-size INT Quant/prune group size (default: 32) │- │ --prune-interval INT Steps between mask updates (def: 100) │- │ --prune-start-frac FL Start pruning at this frac (def: 0.33)│- │ --prune-end-frac FL Lock mask at this frac (def: 0.67) │- │ --activation STR drelu|swiglu|geglu (default: drelu) │- │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │- │ --eval-every INT Val eval interval (default: 1000) │- │ --wandb Enable W&B logging │- │ --checkpoint PATH Resume from checkpoint │- │ --checkpoint-dir DIR Checkpoint directory │- │ --seed INT Random seed (default: 42) │+ │ --full Use full 1B config (~1.17B params) │+ │ --epochs INT Training epochs (default: 1) │+ │ --batch-size INT Batch size (default: 32) │+ │ --lr FLOAT AdamW learning rate (default: 3e-4) │+ │ --muon-lr FLOAT Muon learning rate (default: 0.02) │+ │ --d-model INT Model dim (default: 512) │+ │ --num-heads INT Attention heads (default: 8) │+ │ --num-kv-heads INT KV heads for GQA (default: num-heads)│+ │ --num-layers INT Encoder layers (default: 8) │+ │ --num-dec-layers INT Decoder layers (default: 4) │+ │ --max-enc-len INT Max encoder seq length (default: 256)│+ │ --max-dec-len INT Max decoder seq length (default: 256)│+ │ --max-samples INT Training samples (default: all) │+ │ --mrl-dims INT [...] MRL dim targets (default: 256 128 64)│+ │ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) │+ │ --group-size INT Quant/prune group size (default: 32) │+ │ --prune-interval INT Steps between mask updates (def: 100)│+ │ --prune-start-frac FL Start pruning at frac (def: 0.33) │+ │ --prune-end-frac FL Lock mask at this frac (def: 0.67) │+ │ --activation STR drelu|swiglu|geglu (default: drelu) │+ │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │+ │ --eval-every INT Val eval interval (default: 1000) │+ │ --wandb Enable W&B logging │+ │ --checkpoint PATH Resume from checkpoint │+ │ --checkpoint-dir DIR Checkpoint directory │+ │ --seed INT Random seed (default: 42) ││ ││ run ││ --checkpoint PATH Path to model checkpoint (required) │@@ -79,15 +81,17 @@ def main():sub = parser.add_subparsers(dest="command")p = sub.add_parser("train", add_help=False)+ p.add_argument("--full", action="store_true")p.add_argument("--checkpoint", type=str, default=None)p.add_argument("--epochs", type=int, default=1)p.add_argument("--batch-size", type=int, default=32)p.add_argument("--lr", type=float, default=3e-4)p.add_argument("--muon-lr", type=float, default=0.02)- p.add_argument("--d-model", type=int, default=None)- p.add_argument("--num-heads", type=int, default=4)- p.add_argument("--num-layers", type=int, default=4)- p.add_argument("--num-dec-layers", type=int, default=2)+ p.add_argument("--d-model", type=int, default=512)+ p.add_argument("--num-heads", type=int, default=8)+ p.add_argument("--num-kv-heads", type=int, default=None)+ p.add_argument("--num-layers", type=int, default=8)+ p.add_argument("--num-dec-layers", type=int, default=4)p.add_argument("--max-enc-len", type=int, default=256)p.add_argument("--max-dec-len", type=int, default=256)p.add_argument("--max-samples", type=int, default=None)@@ -108,8 +112,8 @@ def main():help="Fraction of epoch at which pruning finishes and mask locks (default: 0.67)")p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])p.add_argument("--num-memory-slots", type=int, default=64)- p.add_argument("--mrl-dims", type=int, nargs="*", default=[1024, 512, 256, 128, 64],- help="MRL dimension pruning targets (default: 1024 512 256 128 64)")+ p.add_argument("--mrl-dims", type=int, nargs="*", default=[256, 128, 64],+ help="MRL dimension targets (default: 256 128 64)")p = sub.add_parser("run", add_help=False)p.add_argument("--checkpoint", type=str, required=True)@@ -172,8 +176,14 @@ def main():sys.exit(0)if args.command == "train":- if args.d_model is None:- args.d_model = max(args.mrl_dims)+ if getattr(args, "full", False):+ args.d_model = 1536+ args.num_heads = 24+ args.num_kv_heads = 8+ args.num_layers = 12+ args.num_dec_layers = 4+ args.num_memory_slots = 128+ args.mrl_dims = [1024, 768, 512, 256, 128]from .train import traintrain(args)elif args.command == "run":diff --git a/src/train.py b/src/train.pyindex 8dff7e1..3fafe3d 100644--- a/src/train.py+++ b/src/train.py@@ -218,7 +218,7 @@ def _cubic_sparsity_schedule(step, t_start, t_end, s_final):def _make_prune_mask(params, sparsity, group_size):"""Compute block-prune mask entirely on-device (no numpy round-trips).- Returns (pruned_params, binary_mask).+ Returns binary mask tree matching param shapes and dtypes."""# Pass 1: collect block L1 scores as JAX arraysall_scores = []@@ -233,12 +233,11 @@ def _make_prune_mask(params, sparsity, group_size):all_scores.append(scores)if not all_scores:- ones = jax.tree.map(jnp.ones_like, params)- return params, ones+ return jax.tree.map(jnp.ones_like, params)threshold = jnp.percentile(jnp.concatenate(all_scores), sparsity * 100)- # Pass 2: build per-leaf binary mask+ # Pass 2: build per-leaf binary mask (in param dtype to save memory)def _leaf_mask(path, leaf):if leaf.ndim != 2:return jnp.ones_like(leaf)@@ -248,11 +247,9 @@ def _make_prune_mask(params, sparsity, group_size):w = jnp.pad(leaf, ((0, pad), (0, 0))) if pad else leafw_grouped = w.reshape(-1, gs, out_feat)block_keep = (jnp.sum(jnp.abs(w_grouped), axis=1, keepdims=True) > threshold)- return jnp.broadcast_to(block_keep, w_grouped.shape).reshape(-1, out_feat)[:in_feat].astype(jnp.float32)+ return jnp.broadcast_to(block_keep, w_grouped.shape).reshape(-1, out_feat)[:in_feat].astype(leaf.dtype)- mask = jax.tree_util.tree_map_with_path(_leaf_mask, params)- pruned = jax.tree.map(lambda w, m: w * m, params, mask)- return pruned, mask+ return jax.tree_util.tree_map_with_path(_leaf_mask, params)def _quantize_params(params, group_size=32):@@ -559,8 +556,10 @@ def train(args):current_sparsity = _cubic_sparsity_schedule(epoch_step, t_start, t_end, sparsity_ratio)if epoch_step >= t_start and epoch_step % prune_interval == 0 and current_sparsity > 0:ema_unr = jax_utils.unreplicate(ema_params)- _, mask = _make_prune_mask(ema_unr, current_sparsity, _GROUP_SIZE)+ mask = _make_prune_mask(ema_unr, current_sparsity, _GROUP_SIZE)+ del ema_unrprune_mask = jax_utils.replicate(mask)+ del maskif prune_mask is not None:state = state.replace(params=jax.tree.map(lambda w, m: w * m, state.params, prune_mask))@@ -609,8 +608,10 @@ def train(args):if prune_mask is None:# Edge case: epoch had fewer steps than t_startema_unr = jax_utils.unreplicate(ema_params)- _, mask = _make_prune_mask(ema_unr, sparsity_ratio, _GROUP_SIZE)+ mask = _make_prune_mask(ema_unr, sparsity_ratio, _GROUP_SIZE)+ del ema_unrprune_mask = jax_utils.replicate(mask)+ del maskstate = state.replace(params=jax.tree.map(lambda w, m: w * m, state.params, prune_mask))ema_params = jax.tree.map(lambda w, m: w * m, ema_params, prune_mask)final_pruned = jax.tree.map(np.array, jax_utils.unreplicate(ema_params))