Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
update needle
HenryNdubuakue12277b679ae
diff --git a/README.md b/README.mdindex 78308cc..e4850f0 100644--- a/README.md+++ b/README.md@@ -46,7 +46,7 @@## Usage```-git clone https://github.com/cactus-compute/model.git+git clone https://github.com/cactus-compute/needle.gitsource ./setup@@ -99,15 +99,102 @@ needle [command]└───────────────────────────────────────────────────────────────────┘```-## Project Structure+## TPU Factsheet```- src/- ├── model.py ······ Transformer architecture- ├── data.py ······· TinyStories loading & preprocessing- ├── train.py ······ Training loop- ├── run.py ········ Story generation from prompts- ├── test.py ······· Throughput & quality benchmarks- ├── evaluate.py ··· NLP benchmark evaluation- └── cli.py ········ CLI entry point+┌────────────────────┬───────────────────┬──────────────────────┬────────────────────────────┐+│ │ v5e │ v5p │ v6e (Trillium) │+├────────────────────┼───────────────────┼──────────────────────┼────────────────────────────┤+│ Optimized for │ Train + inference │ Training (max perf) │ Train + inference │+│ HBM per chip │ 16 GB │ 95 GB │ 32 GB │+│ FLOPS (BF16) │ 197 TFLOPS │ 459 TFLOPS │ 918 TFLOPS │+│ HBM bandwidth │ 819 GB/s │ 2,765 GB/s │ 1,640 GB/s │+│ ICI bandwidth │ 1,600 Gbps │ 4,800 Gbps │ 3,584 Gbps │+│ On-demand/chip/hr │ $1.20 │ $4.20 │ $2.70 │+│ Spot/chip/hr │ $0.60 │ $2.10 │ $1.35 │+│ Perf per $ │ 1x (baseline) │ 0.5x │ 2x │+├────────────────────┼───────────────────┴──────────────────────┴────────────────────────────┤+│ │ │+│ DATASET │ │+│ Text │ 100B tokens │+│ Audio │ 200k × 20s = ~200M audio tokens + ~13M transcription tokens │+│ Effective total │ ~100.5B equivalent tokens (audio has ~2-3x encoder overhead) │+│ Storage │ ~5 GB audio (compressed) + ~400 GB text corpus │+│ │ │+├────────────────────┼───────────────────┬──────────────────────┬─────────────┬──────────────┤+│ 300M multimodal │ v5litepod-4 │ v6e-4 │ v6e-8 │ v6e-16 │+├────────────────────┼───────────────────┼──────────────────────┼─────────────┼──────────────┤+│ Chips │ 4 │ 4 │ 8 │ 16 │+│ Total HBM │ 64 GB │ 128 GB │ 256 GB │ 512 GB │+│ Est. time │ ~16-21 days │ ~4-5 days │ ~2-3 days │ ~1-1.5 days │+│ Spot $/hr │ $2.40 │ $5.40 │ $10.80 │ $21.60 │+│ Est. total cost │ ~$900-1,200 │ ~$550-700 │ ~$550-750 │ ~$550-750 │+└────────────────────┴───────────────────┴──────────────────────┴─────────────┴──────────────┘```++## Setup For TPU/GCP++- Setup gcloud 1: download the `macOS ARM` from [here](https://docs.cloud.google.com/sdk/docs/install-sdk) and uzip.+- Setup gcloud 2: open terminal, cd to ypur downloads and run `./google-cloud-sdk/install.sh`+- Setup gcloud 3: run `gloud init`, sign in with cactus email, should prompt for project+- Setup gcloud 4: else, set the project with `gcloud config set project needle-488623`+- setup gcloud 5: run `gcloud help` and read carefully++## TPU Guide++```+needle tpu [command]++ ┌───────────────────────────────────────────────────────────────────┐+ │ │+ │ create NAME Create TPU (auto-finds zone) │+ │ --type STR Accelerator type (default: v5litepod-4) │+ │ --version STR TPU OS (default: tpu-ubuntu2204-base) │+ │ │+ │ connect NAME SSH config + first connect (auto-zone) │+ │ claude NAME Install Claude Code on instance │+ │ │+ │ stop NAME Stop instance (keeps disk) │+ │ start NAME Restart a stopped instance │+ │ delete NAME Delete instance (prompts confirmation) │+ │ list List all TPU instances │+ │ │+ │ --zone ZONE Override auto-detected zone (optional) │+ │ │+ └───────────────────────────────────────────────────────────────────┘++ Quota increases:+ https://console.cloud.google.com/iam-admin/quotas?project=needle-488623+```++## Example Workflow++```+1. Create an instance (auto: finds zone → installs Claude Code → connects via SSH)+ needle tpu create my-experiment+ (exit with 'exit' or Ctrl+D)++2. Reconnect anytime (exit with 'exit' or Ctrl+D)+ ssh my-experiment+ or VS Code: click the '><' in the bottom left → select my-experiment++--- run from the instance ---++3. Clone the repo on your instance+ git clone https://github.com/cactus-compute/needle.git+ cd needle++4. Install needle+ source ./setup++5. Use needle as you normally would locally, like training+ needle train --wandb++--- back on your Mac ---++6. Stop when done (saves disk, stops billing)+ needle tpu stop my-experiment++7. (Optional) Delete instance when no longer needed+ needle tpu delete my-experiment+```\ No newline at end of filediff --git a/src/cli.py b/src/cli.pyindex 47e61ac..fbd0398 100644--- a/src/cli.py+++ b/src/cli.py@@ -52,6 +52,18 @@ HELP = """│ --benchmarks [...] wikitext2 lambada hellaswag arc_easy ││ --max-samples INT Samples per benchmark (default: 500) ││ │+ │ tpu │+ │ create NAME Create TPU (auto-finds zone) │+ │ --type STR Accelerator type (default:v5litepod-4)│+ │ --version STR TPU OS (default: tpu-ubuntu2204-base) │+ │ connect NAME SSH config + connect (auto-zone) │+ │ claude NAME Install Claude Code on instance │+ │ stop NAME Stop instance (auto-zone) │+ │ start NAME Start stopped instance (auto-zone) │+ │ delete NAME Delete instance (auto-zone) │+ │ list List all TPU instances │+ │ --zone ZONE Override auto-detected zone │+ │ │└───────────────────────────────────────────────────────────────────┘"""@@ -123,6 +135,36 @@ def main():choices=["wikitext2", "lambada", "hellaswag", "arc_easy"])p.add_argument("--max-samples", type=int, default=500)+ p = sub.add_parser("tpu", add_help=False)+ tpu_sub = p.add_subparsers(dest="tpu_action")++ tp = tpu_sub.add_parser("create", add_help=False)+ tp.add_argument("name", type=str)+ tp.add_argument("--type", dest="accel_type", type=str, default="v5litepod-4")+ tp.add_argument("--version", type=str, default="tpu-ubuntu2204-base")++ tp = tpu_sub.add_parser("connect", add_help=False)+ tp.add_argument("name", type=str)+ tp.add_argument("--zone", type=str, default=None)++ tp = tpu_sub.add_parser("claude", add_help=False)+ tp.add_argument("name", type=str)+ tp.add_argument("--zone", type=str, default=None)++ tp = tpu_sub.add_parser("stop", add_help=False)+ tp.add_argument("name", type=str)+ tp.add_argument("--zone", type=str, default=None)++ tp = tpu_sub.add_parser("start", add_help=False)+ tp.add_argument("name", type=str)+ tp.add_argument("--zone", type=str, default=None)++ tp = tpu_sub.add_parser("delete", add_help=False)+ tp.add_argument("name", type=str)+ tp.add_argument("--zone", type=str, default=None)++ tpu_sub.add_parser("list", add_help=False)+args = parser.parse_args()if not args.command:@@ -144,3 +186,6 @@ def main():elif args.command == "evaluate":from .evaluate import main as eval_maineval_main(args)+ elif args.command == "tpu":+ from .tpu import tpu_dispatch+ tpu_dispatch(args)diff --git a/src/tpu.py b/src/tpu.pynew file mode 100644index 0000000..e49042c--- /dev/null+++ b/src/tpu.py@@ -0,0 +1,228 @@+import getpass+import os+import re+import subprocess+import sys++PROJECT = "needle-488623"++ZONES = [+ "us-central1-a", "us-central1-b", "us-central1-f",+ "us-east1-b", "us-east1-c", "us-east1-d",+ "us-east5-a", "us-east5-b",+ "us-south1-a", "us-south1-b",+ "us-west1-a", "us-west1-b",+ "us-west4-a",+]++TPU_HELP = """+ tpu commands:+ needle tpu create NAME [--type TYPE] [--version VER]+ needle tpu connect NAME [--zone ZONE]+ needle tpu claude NAME [--zone ZONE]+ needle tpu stop NAME [--zone ZONE]+ needle tpu start NAME [--zone ZONE]+ needle tpu delete NAME [--zone ZONE]+ needle tpu list+"""+++def _run(cmd, check=True, capture=False, quiet=False):+ if not quiet:+ print(f"[tpu] $ {' '.join(cmd)}")+ try:+ result = subprocess.run(cmd, capture_output=capture, text=True)+ except FileNotFoundError:+ print(+ "[tpu] ERROR: 'gcloud' not found. "+ "Install: https://cloud.google.com/sdk/docs/install",+ file=sys.stderr,+ )+ sys.exit(1)+ if check and result.returncode != 0:+ if capture and result.stderr:+ print(result.stderr.strip(), file=sys.stderr)+ sys.exit(result.returncode)+ return result+++def _detect_zone(name):+ print(f"[tpu] Searching for '{name}'...")+ for zone in ZONES:+ result = _run(+ ["gcloud", "compute", "tpus", "tpu-vm", "describe", name,+ "--zone", zone, "--project", PROJECT,+ "--format", "value(name)"],+ check=False, capture=True, quiet=True,+ )+ if result.returncode == 0 and result.stdout.strip():+ print(f"[tpu] Found '{name}' in {zone}")+ return zone+ print(+ f"[tpu] ERROR: instance '{name}' not found. "+ "Run 'needle tpu list' to see instances.",+ file=sys.stderr,+ )+ sys.exit(1)+++def _update_ssh_config(path, host_name, new_block):+ os.makedirs(os.path.dirname(path), exist_ok=True)+ if os.path.exists(path):+ with open(path, "r") as f:+ content = f.read()+ pattern = rf"\n?Host {re.escape(host_name)}\n(?: .*\n)*"+ content = re.sub(pattern, "", content)+ else:+ content = ""+ with open(path, "w") as f:+ f.write(content.rstrip("\n") + "\n" if content.strip() else "")+ f.write(new_block)+++def tpu_create(args):+ for zone in ZONES:+ print(f"[tpu] Trying {zone}...")+ result = _run(+ ["gcloud", "compute", "tpus", "tpu-vm", "create", args.name,+ "--zone", zone,+ "--accelerator-type", args.accel_type,+ "--version", args.version,+ "--project", PROJECT],+ check=False, capture=True,+ )+ if result.returncode == 0:+ print(f"[tpu] SUCCESS: created '{args.name}' in {zone}")+ args.zone = zone+ tpu_claude(args)+ tpu_connect(args)+ return+ stderr = result.stderr.strip()+ last_line = stderr.splitlines()[-1] if stderr else "unknown error"+ print(f"[tpu] {zone}: {last_line}")+ print(+ f"[tpu] ERROR: could not create '{args.name}' in any zone.",+ file=sys.stderr,+ )+ print(+ f"[tpu] Check quota: "+ f"https://console.cloud.google.com/iam-admin/quotas?project={PROJECT}",+ file=sys.stderr,+ )+ sys.exit(1)+++def tpu_connect(args):+ zone = args.zone or _detect_zone(args.name)++ result = _run(+ ["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,+ "--zone", zone, "--project", PROJECT, "--dry-run"],+ capture=True,+ )++ ip_match = re.search(+ r"(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})",+ result.stdout + result.stderr,+ )+ if not ip_match:+ print("[tpu] ERROR: could not parse IP from dry-run output.", file=sys.stderr)+ print(f"[tpu] stdout: {result.stdout}", file=sys.stderr)+ sys.exit(1)+ ip = ip_match.group(1)+ print(f"[tpu] Detected IP: {ip}")++ ssh_config_path = os.path.expanduser("~/.ssh/config")+ user = getpass.getuser()+ block = (+ f"\nHost {args.name}\n"+ f" HostName {ip}\n"+ f" User {user}\n"+ f" IdentityFile ~/.ssh/google_compute_engine\n"+ f" CheckHostIP no\n"+ f" StrictHostKeyChecking no\n"+ )+ _update_ssh_config(ssh_config_path, args.name, block)+ print(f"[tpu] Updated {ssh_config_path} with host '{args.name}'")++ print(f"[tpu] Connecting to {args.name} (this propagates SSH keys)...")+ subprocess.run(+ ["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,+ "--zone", zone, "--project", PROJECT],+ )+++def tpu_stop(args):+ zone = args.zone or _detect_zone(args.name)+ _run(["gcloud", "compute", "tpus", "tpu-vm", "stop", args.name,+ "--zone", zone, "--project", PROJECT])+ print(f"[tpu] Stopped '{args.name}'")+++def tpu_start(args):+ zone = args.zone or _detect_zone(args.name)+ _run(["gcloud", "compute", "tpus", "tpu-vm", "start", args.name,+ "--zone", zone, "--project", PROJECT])+ print(f"[tpu] Started '{args.name}'")+++def tpu_delete(args):+ zone = args.zone or _detect_zone(args.name)+ answer = input(f"[tpu] Delete '{args.name}' in {zone}? This is permanent. [y/N] ")+ if answer.lower() != "y":+ print("[tpu] Aborted.")+ return+ _run(["gcloud", "compute", "tpus", "tpu-vm", "delete", args.name,+ "--zone", zone, "--project", PROJECT, "--quiet"])+ print(f"[tpu] Deleted '{args.name}'")+++def tpu_claude(args):+ zone = args.zone or _detect_zone(args.name)+ setup_script = (+ "curl -fsSL https://deb.nodesource.com/setup_22.x | sudo -E bash - && "+ "sudo apt-get install -y nodejs && "+ "sudo npm install -g @anthropic-ai/claude-code && "+ "echo '[tpu] Claude Code installed. Run: claude'"+ )+ print(f"[tpu] Installing Claude Code on '{args.name}'...")+ _run(+ ["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,+ "--zone", zone, "--project", PROJECT,+ "--command", setup_script],+ )+++def tpu_list(args):+ found = False+ for zone in ZONES:+ result = _run(+ ["gcloud", "compute", "tpus", "tpu-vm", "list",+ "--zone", zone, "--project", PROJECT],+ check=False, capture=True, quiet=True,+ )+ if result.returncode == 0 and result.stdout.strip():+ if not found:+ header = result.stdout.strip().splitlines()[0]+ print(f"{header} ZONE")+ found = True+ for line in result.stdout.strip().splitlines()[1:]:+ print(f"{line} {zone}")+ if not found:+ print("[tpu] No instances found.")+++def tpu_dispatch(args):+ actions = {+ "create": tpu_create,+ "connect": tpu_connect,+ "claude": tpu_claude,+ "stop": tpu_stop,+ "start": tpu_start,+ "delete": tpu_delete,+ "list": tpu_list,+ }+ if not args.tpu_action or args.tpu_action not in actions:+ print(TPU_HELP)+ sys.exit(0 if not args.tpu_action else 1)+ actions[args.tpu_action](args)