Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuaku794d7fa53c83
diff --git a/README.md b/README.mdindex 6a0a1be..1a124f6 100644--- a/README.md+++ b/README.md@@ -37,16 +37,16 @@│ ┌──────────┐ │ │ └───────────────┘ │ └──────────┬───────────┘│ │Pack: │ │ └─────────┬─────────┘ ││ │ S←X Attn │ │ ┌──────┴──────┐ ┌──────────┴───────────┐- │ │ RoPE keys│ │ │ Embedding │ ← shared │ Layer Prune │- │ ├──────────┤ │ └──────┬──────┘ │ after epoch 2 │- │ │Mix: │ │ │ │ L1 block scoring │- │ │ MLP-Mixer│ │ ┌─────┴─────┐ │ keep ≥1 enc & dec │- │ │ on S │ │ │ Decoder │ └──────────┬───────────┘- │ ├──────────┤ │ │ Input │ │- │ │Local: │ │ └───────────┘ ┌──────────┴───────────┐- │ │ FFN on X │ │ │ Checkpoint │- │ └──────────┘ │ │ MRL + sparse + INT4 │- │ S ∈ (M, d) │ └──────────────────────┘+ │ │ RoPE keys│ │ │ Embedding │ ← shared │ MRL Checkpoint │+ │ ├──────────┤ │ └──────┬──────┘ │ save per mrl_dim │+ │ │Mix: │ │ │ │ {512,256,128,64} │+ │ │ MLP-Mixer│ │ ┌─────┴─────┐ │ sparse + INT4 │+ │ │ on S │ │ │ Decoder │ └──────────────────────┘+ │ ├──────────┤ │ │ Input │+ │ │Local: │ │ └───────────┘+ │ │ FFN on X │ │+ │ └──────────┘ │+ │ S ∈ (M, d) │└──────┬───────┘┌──────┴───────┐│ Embedding │ ← shared@@ -75,7 +75,7 @@ needle [command]│ ││ train ││ --toy Use toy config for quick iteration │- │ --epochs INT Training epochs (default: 3) │+ │ --epochs INT Training epochs (default: 2) ││ --batch-size INT Batch size (default: 32) ││ --lr FLOAT AdamW learning rate (default: 3e-4) ││ --muon-lr FLOAT Muon learning rate (default: 0.02) │@@ -125,7 +125,7 @@ needle [command]│ ││ tpu ││ create NAME Create TPU (auto-finds zone) │- │ --type STR Accelerator (default: v6e-4) │+ │ --type STR Accelerator (default: v6e-8) ││ --version STR TPU OS (auto-detected from --type) ││ connect NAME SSH config + connect (auto-zone) ││ claude NAME Install Claude Code on instance │@@ -166,17 +166,17 @@ needle [command]MFU (300M model) ~15% → ~138 effective TFLOPS/chipAudio overhead <1% of total (negligible)- ┌────────────────────┬──────────┬──────────┬──────────┐- │ │ v6e-4 │ v6e-8 │ v6e-16 │- ├────────────────────┼──────────┼──────────┼──────────┤- │ Chips │ 4 │ 8 │ 16 │- │ Total HBM │ 128 GB │ 256 GB │ 512 GB │- │ Scaling eff. │ 1.0× │ 0.9× │ 0.8× │- │ Eff. TFLOPS │ 551 │ 992 │ 1,766 │- │ Est. time │ ~96h │ ~50h │ ~29h │- │ On-demand $/hr │ $10.80 │ $21.60 │ $43.20 │- │ Est. total cost │ ~$1,040 │ ~$1,080 │ ~$1,250 │- └────────────────────┴──────────┴──────────┴──────────┘+ ┌────────────────────┬──────────┬──────────┐+ │ │ v6e-8 │ v6e-16 │+ ├────────────────────┼──────────┼──────────┤+ │ Chips │ 8 │ 16 │+ │ Total HBM │ 256 GB │ 512 GB │+ │ Scaling eff. │ 0.9× │ 0.8× │+ │ Eff. TFLOPS │ 992 │ 1,766 │+ │ Est. time │ ~50h │ ~29h │+ │ On-demand $/hr │ $21.60 │ $43.20 │+ │ Est. total cost │ ~$1,080 │ ~$1,250 │+ └────────────────────┴──────────┴──────────┘```## Setup For TPU/GCP@@ -195,7 +195,7 @@ needle tpu [command]┌───────────────────────────────────────────────────────────────────┐│ ││ create NAME Create TPU (auto-finds zone) │- │ --type STR Accelerator type (default: v6e-4) │+ │ --type STR Accelerator type (default: v6e-8) ││ --version STR TPU OS (auto-detected from --type) ││ ││ connect NAME SSH config + first connect (auto-zone) │@@ -241,11 +241,17 @@ needle tpu [command]6. Use needle as you normally would locally, like trainingneedle train --wandb+7. Use tmux for long training runs (survives SSH disconnects)+ tmux new -s train # start a named session+ needle train --wandb # run training inside it+ Ctrl+B, then D # detach (keeps running)+ tmux attach -t train # reattach later+--- back on your Mac ----7. Stop when done (saves disk, stops billing)+8. Stop when done (saves disk, stops billing)needle tpu stop my-experiment-8. (Optional) Delete instance when no longer needed+9. (Optional) Delete instance when no longer neededneedle tpu delete my-experiment```diff --git a/src/cli.py b/src/cli.pyindex ebde0da..51803d6 100644--- a/src/cli.py+++ b/src/cli.py@@ -11,7 +11,7 @@ HELP = """│ ││ train ││ --toy Use toy config for quick iteration │- │ --epochs INT Training epochs (default: 3) │+ │ --epochs INT Training epochs (default: 2) ││ --batch-size INT Batch size (default: 32) ││ --lr FLOAT AdamW learning rate (default: 3e-4) ││ --muon-lr FLOAT Muon learning rate (default: 0.02) │@@ -61,7 +61,7 @@ HELP = """│ ││ tpu ││ create NAME Create TPU (auto-finds zone) │- │ --type STR Accelerator (default: v6e-4) │+ │ --type STR Accelerator (default: v6e-8) ││ --version STR TPU OS (auto-detected from --type) ││ connect NAME SSH config + connect (auto-zone) ││ claude NAME Install Claude Code on instance │@@ -113,7 +113,7 @@ def main():p = sub.add_parser("train", add_help=False)p.add_argument("--toy", action="store_true")p.add_argument("--checkpoint", type=str, default=None)- p.add_argument("--epochs", type=int, default=3)+ p.add_argument("--epochs", type=int, default=2)p.add_argument("--batch-size", type=int, default=32)p.add_argument("--lr", type=float, default=3e-4)p.add_argument("--muon-lr", type=float, default=0.02)@@ -186,7 +186,7 @@ def main():tp = tpu_sub.add_parser("create", add_help=False)tp.add_argument("name", type=str)- tp.add_argument("--type", dest="accel_type", type=str, default="v6e-4")+ tp.add_argument("--type", dest="accel_type", type=str, default="v6e-8")tp.add_argument("--version", type=str, default=None,help="Software version (auto-detected from --type if omitted)")diff --git a/src/train.py b/src/train.pyindex 9964749..d46b551 100644--- a/src/train.py+++ b/src/train.py@@ -775,7 +775,7 @@ def sweep(args):def parse_args():parser = argparse.ArgumentParser(description="Train encoder-decoder transformer on TinyStories")- parser.add_argument("--epochs", type=int, default=3)+ parser.add_argument("--epochs", type=int, default=2)parser.add_argument("--batch-size", type=int, default=32)parser.add_argument("--lr", type=float, default=3e-4)parser.add_argument("--muon-lr", type=float, default=0.02)