Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuaku794d7fa53c83
diff --git a/README.md b/README.md
index 6a0a1be..1a124f6 100644
--- a/README.md
+++ b/README.md
@@ -37,16 +37,16 @@
│ ┌──────────┐ │ │ └───────────────┘ │ └──────────┬───────────┘
│ │Pack: │ │ └─────────┬─────────┘ │
│ │ S←X Attn │ │ ┌──────┴──────┐ ┌──────────┴───────────┐
- │ │ RoPE keys│ │ │ Embedding │ ← shared │ Layer Prune │
- │ ├──────────┤ │ └──────┬──────┘ │ after epoch 2 │
- │ │Mix: │ │ │ │ L1 block scoring │
- │ │ MLP-Mixer│ │ ┌─────┴─────┐ │ keep ≥1 enc & dec │
- │ │ on S │ │ │ Decoder │ └──────────┬───────────┘
- │ ├──────────┤ │ │ Input │ │
- │ │Local: │ │ └───────────┘ ┌──────────┴───────────┐
- │ │ FFN on X │ │ │ Checkpoint │
- │ └──────────┘ │ │ MRL + sparse + INT4 │
- │ S ∈ (M, d) │ └──────────────────────┘
+ │ │ RoPE keys│ │ │ Embedding │ ← shared │ MRL Checkpoint │
+ │ ├──────────┤ │ └──────┬──────┘ │ save per mrl_dim │
+ │ │Mix: │ │ │ │ {512,256,128,64} │
+ │ │ MLP-Mixer│ │ ┌─────┴─────┐ │ sparse + INT4 │
+ │ │ on S │ │ │ Decoder │ └──────────────────────┘
+ │ ├──────────┤ │ │ Input │
+ │ │Local: │ │ └───────────┘
+ │ │ FFN on X │ │
+ │ └──────────┘ │
+ │ S ∈ (M, d) │
└──────┬───────┘
┌──────┴───────┐
│ Embedding │ ← shared
@@ -75,7 +75,7 @@ needle [command]
│ │
│ train │
│ --toy Use toy config for quick iteration │
- │ --epochs INT Training epochs (default: 3) │
+ │ --epochs INT Training epochs (default: 2) │
│ --batch-size INT Batch size (default: 32) │
│ --lr FLOAT AdamW learning rate (default: 3e-4) │
│ --muon-lr FLOAT Muon learning rate (default: 0.02) │
@@ -125,7 +125,7 @@ needle [command]
│ │
│ tpu │
│ create NAME Create TPU (auto-finds zone) │
- │ --type STR Accelerator (default: v6e-4) │
+ │ --type STR Accelerator (default: v6e-8) │
│ --version STR TPU OS (auto-detected from --type) │
│ connect NAME SSH config + connect (auto-zone) │
│ claude NAME Install Claude Code on instance │
@@ -166,17 +166,17 @@ needle [command]
MFU (300M model) ~15% → ~138 effective TFLOPS/chip
Audio overhead <1% of total (negligible)
- ┌────────────────────┬──────────┬──────────┬──────────┐
- │ │ v6e-4 │ v6e-8 │ v6e-16 │
- ├────────────────────┼──────────┼──────────┼──────────┤
- │ Chips │ 4 │ 8 │ 16 │
- │ Total HBM │ 128 GB │ 256 GB │ 512 GB │
- │ Scaling eff. │ 1.0× │ 0.9× │ 0.8× │
- │ Eff. TFLOPS │ 551 │ 992 │ 1,766 │
- │ Est. time │ ~96h │ ~50h │ ~29h │
- │ On-demand $/hr │ $10.80 │ $21.60 │ $43.20 │
- │ Est. total cost │ ~$1,040 │ ~$1,080 │ ~$1,250 │
- └────────────────────┴──────────┴──────────┴──────────┘
+ ┌────────────────────┬──────────┬──────────┐
+ │ │ v6e-8 │ v6e-16 │
+ ├────────────────────┼──────────┼──────────┤
+ │ Chips │ 8 │ 16 │
+ │ Total HBM │ 256 GB │ 512 GB │
+ │ Scaling eff. │ 0.9× │ 0.8× │
+ │ Eff. TFLOPS │ 992 │ 1,766 │
+ │ Est. time │ ~50h │ ~29h │
+ │ On-demand $/hr │ $21.60 │ $43.20 │
+ │ Est. total cost │ ~$1,080 │ ~$1,250 │
+ └────────────────────┴──────────┴──────────┘
```
## Setup For TPU/GCP
@@ -195,7 +195,7 @@ needle tpu [command]
┌───────────────────────────────────────────────────────────────────┐
│ │
│ create NAME Create TPU (auto-finds zone) │
- │ --type STR Accelerator type (default: v6e-4) │
+ │ --type STR Accelerator type (default: v6e-8) │
│ --version STR TPU OS (auto-detected from --type) │
│ │
│ connect NAME SSH config + first connect (auto-zone) │
@@ -241,11 +241,17 @@ needle tpu [command]
6. Use needle as you normally would locally, like training
needle train --wandb
+7. Use tmux for long training runs (survives SSH disconnects)
+ tmux new -s train # start a named session
+ needle train --wandb # run training inside it
+ Ctrl+B, then D # detach (keeps running)
+ tmux attach -t train # reattach later
+
--- back on your Mac ---
-7. Stop when done (saves disk, stops billing)
+8. Stop when done (saves disk, stops billing)
needle tpu stop my-experiment
-8. (Optional) Delete instance when no longer needed
+9. (Optional) Delete instance when no longer needed
needle tpu delete my-experiment
```
diff --git a/src/cli.py b/src/cli.py
index ebde0da..51803d6 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -11,7 +11,7 @@ HELP = """
│ │
│ train │
│ --toy Use toy config for quick iteration │
- │ --epochs INT Training epochs (default: 3) │
+ │ --epochs INT Training epochs (default: 2) │
│ --batch-size INT Batch size (default: 32) │
│ --lr FLOAT AdamW learning rate (default: 3e-4) │
│ --muon-lr FLOAT Muon learning rate (default: 0.02) │
@@ -61,7 +61,7 @@ HELP = """
│ │
│ tpu │
│ create NAME Create TPU (auto-finds zone) │
- │ --type STR Accelerator (default: v6e-4) │
+ │ --type STR Accelerator (default: v6e-8) │
│ --version STR TPU OS (auto-detected from --type) │
│ connect NAME SSH config + connect (auto-zone) │
│ claude NAME Install Claude Code on instance │
@@ -113,7 +113,7 @@ def main():
p = sub.add_parser("train", add_help=False)
p.add_argument("--toy", action="store_true")
p.add_argument("--checkpoint", type=str, default=None)
- p.add_argument("--epochs", type=int, default=3)
+ p.add_argument("--epochs", type=int, default=2)
p.add_argument("--batch-size", type=int, default=32)
p.add_argument("--lr", type=float, default=3e-4)
p.add_argument("--muon-lr", type=float, default=0.02)
@@ -186,7 +186,7 @@ def main():
tp = tpu_sub.add_parser("create", add_help=False)
tp.add_argument("name", type=str)
- tp.add_argument("--type", dest="accel_type", type=str, default="v6e-4")
+ tp.add_argument("--type", dest="accel_type", type=str, default="v6e-8")
tp.add_argument("--version", type=str, default=None,
help="Software version (auto-detected from --type if omitted)")
diff --git a/src/train.py b/src/train.py
index 9964749..d46b551 100644
--- a/src/train.py
+++ b/src/train.py
@@ -775,7 +775,7 @@ def sweep(args):
def parse_args():
parser = argparse.ArgumentParser(description="Train encoder-decoder transformer on TinyStories")
- parser.add_argument("--epochs", type=int, default=3)
+ parser.add_argument("--epochs", type=int, default=2)
parser.add_argument("--batch-size", type=int, default=32)
parser.add_argument("--lr", type=float, default=3e-4)
parser.add_argument("--muon-lr", type=float, default=0.02)