Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Updates
HenryNdubuakuc5f1fd9dfafe
diff --git a/launch_train.sh b/launch_train.shnew file mode 100644index 0000000..7f7d543--- /dev/null+++ b/launch_train.sh@@ -0,0 +1,33 @@+#!/bin/bash+# Launch multi-host training on all TPU workers.+# Run this from your LOCAL machine (not from the TPU VM).+#+# Usage: ./launch_train.sh [extra needle train args...]+# Example: ./launch_train.sh --epochs 3 --wandb --batch-size 64++set -euo pipefail++TPU_NAME="large"+ZONE="asia-northeast1-b"+PROJECT="needle-488623"++TRAIN_ARGS="${@}"++echo "[launch] Distributing SSH keys across workers..."+# Generate keypair on worker 0 if not present, then distribute to all workers+gcloud compute tpus tpu-vm ssh "$TPU_NAME" \+ --zone="$ZONE" --project="$PROJECT" --worker=0 \+ --command='[ -f ~/.ssh/id_rsa ] || ssh-keygen -t rsa -b 2048 -f ~/.ssh/id_rsa -N "" -q; cat ~/.ssh/id_rsa.pub' \+ 2>/dev/null | tail -1 > /tmp/_tpu_pubkey.txt++# Add worker 0's public key to all workers' authorized_keys+PUBKEY=$(cat /tmp/_tpu_pubkey.txt)+gcloud compute tpus tpu-vm ssh "$TPU_NAME" \+ --zone="$ZONE" --project="$PROJECT" --worker=all \+ --command="grep -qF '$(echo $PUBKEY | cut -d' ' -f2)' ~/.ssh/authorized_keys 2>/dev/null || echo '$PUBKEY' >> ~/.ssh/authorized_keys"++echo "[launch] Starting training on all workers..."+gcloud compute tpus tpu-vm ssh "$TPU_NAME" \+ --zone="$ZONE" --project="$PROJECT" --worker=all \+ --command="cd ~/needle && .venv/bin/needle train $TRAIN_ARGS" \+ 2>&1diff --git a/src/cli.py b/src/cli.pyindex dfc6398..29aedbe 100644--- a/src/cli.py+++ b/src/cli.py@@ -22,7 +22,7 @@ def main():p.add_argument("--d-model", type=int, default=512)p.add_argument("--num-heads", type=int, default=8)p.add_argument("--num-kv-heads", type=int, default=4)- p.add_argument("--num-layers", type=int, default=16)+ p.add_argument("--num-layers", type=int, default=12)p.add_argument("--num-dec-layers", type=int, default=8)p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)diff --git a/src/train.py b/src/train.pyindex d18c53c..3279d48 100644--- a/src/train.py+++ b/src/train.py@@ -498,16 +498,20 @@ def shard_batch(batch, num_devices):def train(args):+ # Multi-host TPU: initialize distributed JAX before anything else.+ # On single-host setups this is a no-op (coordinator_address is auto-detected).+ jax.distributed.initialize()+num_devices = jax.local_device_count()use_wandb = getattr(args, "wandb", False)- if use_wandb:+ if use_wandb and jax.process_index() == 0:import wandbif wandb.run is None:wandb.init(project="needle-v1", config=vars(args))print(f"\n[1/3] Detecting devices...")- print(f" {num_devices} device(s) for data-parallel training")+ print(f" {num_devices} local device(s), {jax.device_count()} total across {jax.process_count()} hosts")print(f"\n[2/3] Loading tokenizer...")tokenizer = get_tokenizer(max_samples=args.max_samples)@@ -541,7 +545,10 @@ def train(args):if has_contrastive:print(f" Contrastive: {len(cl_query_tokens):,} queries, {len(cl_tool_tokens):,} tools")- effective_batch_size = args.batch_size * num_devices+ num_hosts = jax.process_count()+ host_id = jax.process_index()+ total_devices = jax.device_count()+ effective_batch_size = args.batch_size * num_devices # per-host batch for pmapresume_checkpoint = getattr(args, "checkpoint", None)if resume_checkpoint:@@ -594,13 +601,15 @@ def train(args):rng, init_rng = jax.random.split(rng)unique_batch_size = (effective_batch_size // num_devices) * num_devices- text_batches_per_epoch = count_batches(len(enc_inputs), unique_batch_size)+ # Each host processes unique_batch_size samples; total across hosts = unique_batch_size * num_hosts+ global_batch_size = unique_batch_size * num_hosts+ text_batches_per_epoch = count_batches(len(enc_inputs), global_batch_size)num_batches = text_batches_per_epochtotal_steps = num_batches * args.epochswarmup_steps = max(1, int(total_steps * args.warmup_ratio))- scaled_lr = args.lr * num_devices- muon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(num_devices)+ scaled_lr = args.lr * total_devices+ muon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(total_devices)decay_ratio = getattr(args, "decay_ratio", 0.15)state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps, decay_ratio)val_loss_fn = _make_val_loss_fn(state.apply_fn)@@ -617,22 +626,25 @@ def train(args):best_call_f1 = 0.0best_ckpt_path = None- print(f"\n ─────────────────────────────────────")- print(f" Parameters {param_count:>12,}")- print(f" d_model {config.d_model:>12}")- print(f" Heads {config.num_heads:>7} ({config.num_kv_heads} KV)")- print(f" Layers {config.num_encoder_layers:>7} enc / {config.num_decoder_layers} dec")- print(f" Activation {config.activation:>12}")- print(f" Dtype {config.dtype:>12}")- print(f" ─────────────────────────────────────")- print(f" Devices {num_devices:>12}")- print(f" Batch {args.batch_size:>7} x {num_devices} = {effective_batch_size}")- print(f" Adam LR {args.lr:>7} x {num_devices} = {scaled_lr}")- print(f" Muon LR {args.muon_lr:>7.4f} -> {muon_lr:.4f}")- print(f" Schedule {warmup_steps}w / {stable_steps}s / {decay_steps}d (WSD)")- print(f" Total steps {total_steps:>12,}")- print(f" Epochs {args.epochs:>12}")- print(f" ─────────────────────────────────────\n")+ is_main = host_id == 0+ if is_main:+ print(f"\n ─────────────────────────────────────")+ print(f" Parameters {param_count:>12,}")+ print(f" d_model {config.d_model:>12}")+ print(f" Heads {config.num_heads:>7} ({config.num_kv_heads} KV)")+ print(f" Layers {config.num_encoder_layers:>7} enc / {config.num_decoder_layers} dec")+ print(f" Activation {config.activation:>12}")+ print(f" Dtype {config.dtype:>12}")+ print(f" ─────────────────────────────────────")+ print(f" Hosts {num_hosts:>12}")+ print(f" Devices {num_devices:>5}/host, {total_devices} total")+ print(f" Batch {args.batch_size:>7} x {total_devices} = {global_batch_size}")+ print(f" Adam LR {args.lr:>7} x {total_devices} = {scaled_lr}")+ print(f" Muon LR {args.muon_lr:>7.4f} -> {muon_lr:.4f}")+ print(f" Schedule {warmup_steps}w / {stable_steps}s / {decay_steps}d (WSD)")+ print(f" Total steps {total_steps:>12,}")+ print(f" Epochs {args.epochs:>12}")+ print(f" ─────────────────────────────────────\n")os.makedirs(args.checkpoint_dir, exist_ok=True)global_step = 0@@ -642,13 +654,13 @@ def train(args):text_ffn_mask[None, :, :],(num_devices, args.batch_size, config.d_ff),)- if n_widths > 1:+ if n_widths > 1 and is_main:print(f" Mat factors {n_widths} (full + {', '.join(str(f)+'x' for f in _MAT_FACTORS)})")print(f" Mat mode unique input ({args.batch_size}/dev, split by width)")adam_schedule = _wsd_schedule(scaled_lr, total_steps, warmup_steps)muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps)- tokens_per_batch = effective_batch_size * (args.max_enc_len + args.max_dec_len)+ tokens_per_batch = global_batch_size * (args.max_enc_len + args.max_dec_len)eval_model = EncoderDecoderTransformer(config)@@ -680,7 +692,7 @@ def train(args):text_losses = []text_batch_iter = PrefetchIterator(- lambda: get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size,+ lambda: get_batches(enc_inputs, dec_inputs, dec_targets, global_batch_size,loss_mask=train_loss_mask,enc_seg_ids=train_enc_seg,dec_seg_ids=train_dec_seg),@@ -693,7 +705,7 @@ def train(args):cl_batch_iter = PrefetchIterator(lambda: get_contrastive_batches(cl_query_tokens, cl_tool_tokens, cl_tool_ex_idx, cl_tool_is_pos,- unique_batch_size),+ global_batch_size),prefetch=4,)@@ -702,7 +714,11 @@ def train(args):for step_i in pbar:t0 = time.perf_counter()- src, tgt_in, tgt_out, lm, enc_seg, dec_seg = next(text_batch_iter)+ batch = next(text_batch_iter)++ # Slice this host's portion from the global batch+ host_slice = slice(host_id * unique_batch_size, (host_id + 1) * unique_batch_size)+ src, tgt_in, tgt_out, lm, enc_seg, dec_seg = [b[host_slice] for b in batch]# Upcast from int16/float16 storage to int32/float32 for JAXsrc = np.asarray(src, dtype=np.int32)@@ -729,6 +745,8 @@ def train(args):if do_contrastive:try:cl_q, cl_t = next(cl_batch_iter)+ cl_q = cl_q[host_slice]+ cl_t = cl_t[host_slice]cl_q_b = shard_batch(cl_q, num_devices)cl_t_b = shard_batch(cl_t, num_devices)rng, cl_rng = jax.random.split(rng)@@ -764,7 +782,7 @@ def train(args):dt = time.perf_counter() - t0eval_every = getattr(args, "eval_every", 100)- if global_step % eval_every == 0 or global_step == total_steps:+ if is_main and (global_step % eval_every == 0 or global_step == total_steps):_eval_params = jax_utils.unreplicate(state).paramstotal_loss, total_toks = 0.0, 0.0_val_n = count_batches(len(val_enc), args.batch_size)@@ -796,7 +814,7 @@ def train(args):postfix["sparsification"] = "done"pbar.set_postfix(**postfix)- if use_wandb:+ if use_wandb and is_main:log_dict = {"train/text_loss": text_loss_val,"train/grad_norm": step_grad_norm,@@ -836,6 +854,10 @@ def train(args):final_loss = text_losses[-1] if text_losses else float("nan")final_ppl = math.exp(min(final_loss, 20)) if not math.isnan(final_loss) else float("nan")+ if not is_main:+ # Non-main hosts wait here while host 0 runs eval/checkpoint+ continue+eval_params = jax_utils.unreplicate(state).paramsq_params = _quantize_params(eval_params, group_size=_GROUP_SIZE, precision=_PRECISION)@@ -1325,14 +1347,15 @@ def train(args):log_dict["epoch/retrieval_mrr"] = retrieval_metrics["mrr"]wandb.log(log_dict)- if use_wandb:+ if use_wandb and is_main:wandb.finish()- if best_ckpt_path:+ if is_main and best_ckpt_path:print(f"\nBest checkpoint (call_f1={best_call_f1:.1%}): {best_ckpt_path}")- print("\nTraining complete.")+ if is_main:+ print("\nTraining complete.")# Auto-calibrate confidence head on best checkpoint- if getattr(args, "calibrate", True) and best_ckpt_path:+ if is_main and getattr(args, "calibrate", True) and best_ckpt_path:print(f"\n{'='*50}")print(f"Running confidence head calibration on {best_ckpt_path}")print(f"{'='*50}")