Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuaku48ac5862343e
diff --git a/src/cli.py b/src/cli.pyindex 10cdfc8..854fee3 100644--- a/src/cli.py+++ b/src/cli.py@@ -77,6 +77,10 @@ def main():p = sub.add_parser("pretrain", add_help=False)p.add_argument("--name", type=str, default="pretrain",help="Experiment name for wandb (default: pretrain)")+ p.add_argument("--checkpoint", type=str, default=None,+ help="Resume from checkpoint (e.g. checkpoints/needle_base.pkl)")+ p.add_argument("--resume-step", type=int, default=None,+ help="Override resume step (skip this many batches)")p.add_argument("--epochs", type=int, default=1)p.add_argument("--batch-size", type=int, default=128)p.add_argument("--lr", type=float, default=3e-4)diff --git a/src/pretrain.py b/src/pretrain.pyindex cec5a93..7cee4be 100644--- a/src/pretrain.py+++ b/src/pretrain.py@@ -222,10 +222,30 @@ def pretrain(args):muon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(total_devices)decay_ratio = getattr(args, "decay_ratio", 0.05)+ resume_step = 0+ resume_checkpoint = getattr(args, "checkpoint", None)+print(f"\n[3/3] Initializing model...")rng = jax.random.PRNGKey(args.seed)rng, init_rng = jax.random.split(rng)state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps, decay_ratio)++ if resume_checkpoint:+ print(f" Resuming from {resume_checkpoint}")+ with open(resume_checkpoint, "rb") as f:+ ckpt_data = pickle.load(f)+ ckpt_params = jax.tree.map(jnp.array, ckpt_data["params"])+ state = state.replace(params=ckpt_params)+ del ckpt_params+ # Use --resume-step override, else read from checkpoint, else 0+ manual_step = getattr(args, "resume_step", None)+ if manual_step is not None:+ resume_step = manual_step+ else:+ resume_step = ckpt_data.get("pretrain_step", 0)+ if resume_step > 0:+ print(f" Resuming from step {resume_step}")+state = _replicate(state)param_count = sum(x.size for x in jax.tree.leaves(_unreplicate(state).params))@@ -259,7 +279,7 @@ def pretrain(args):print(f" Save every {save_every:>12,}")print(f" ─────────────────────────────────────\n")- global_step = 0+ global_step = resume_stepfor epoch in range(args.epochs):batch_stream = _PrefetchStream(@@ -270,7 +290,17 @@ def pretrain(args):)pbar = tqdm(desc=f"Pretrain epoch {epoch + 1}/{args.epochs}",- total=estimated_steps, disable=not is_main)+ total=estimated_steps, initial=resume_step,+ disable=not is_main)++ # Skip ahead if resuming+ if resume_step > 0:+ if is_main:+ print(f" Skipping {resume_step} batches to resume...")+ for i, _ in enumerate(batch_stream):+ if i + 1 >= resume_step:+ break+ resume_step = 0 # only skip oncefor batch in batch_stream:if max_steps and global_step >= max_steps:@@ -342,7 +372,8 @@ def _save_pretrain_checkpoint(state, config, checkpoint_dir, global_step):ckpt_path = os.path.join(checkpoint_dir, "needle_base.pkl")with open(ckpt_path, "wb") as f:- pickle.dump({"params": params_np, "config": config.__dict__}, f)+ pickle.dump({"params": params_np, "config": config.__dict__,+ "pretrain_step": global_step}, f)param_count = sum(x.size for x in jax.tree.leaves(params_np))size_mb = sum(x.nbytes for x in jax.tree.leaves(params_np)) / 1e6diff --git a/src/tpu.py b/src/tpu.pyindex 5864d57..e3e5d3a 100644--- a/src/tpu.py+++ b/src/tpu.py@@ -579,18 +579,25 @@ def _tpu_run_command(args, needle_cmd="train"):print(f"[tpu] Forwarding {env_count} env var(s) to workers")train_cmd = env_exports + " && " + train_cmd+ # Wrap with nohup so training survives SSH disconnects+ log_file = f"/tmp/needle_{needle_cmd}.log"+ train_cmd = (+ f"nohup bash -c '{train_cmd}' > {log_file} 2>&1 &"+ f" echo $!; sleep 1; head -5 {log_file}"+ )+if num_workers <= 1:- print(f"[tpu] Single-host TPU. Running directly...")- print(f"[tpu] $ gcloud ... --command 'needle {needle_cmd} ...'")+ print(f"[tpu] Single-host TPU. Running in background...")_run(["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,"--zone", zone, "--project", PROJECT,"--command", train_cmd],quiet=True,)+ print(f"[tpu] Running in background. Logs: ssh into worker and 'tail -f {log_file}'")return- print(f"[tpu] Launching 'needle {needle_cmd}' on {num_workers} workers...")+ print(f"[tpu] Launching 'needle {needle_cmd}' on {num_workers} workers (background)...")result = _run(["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,"--zone", zone, "--project", PROJECT,@@ -598,6 +605,8 @@ def _tpu_run_command(args, needle_cmd="train"):"--command", train_cmd],check=False, quiet=True,)+ print(f"[tpu] Processes launched. Monitor with:")+ print(f" gcloud compute tpus tpu-vm ssh large --zone=asia-northeast1-b --worker=0 --command='tail -f {log_file}'")if result.returncode != 0:print(f"[tpu] WARNING: 'needle {needle_cmd}' exited with errors on some workers.",file=sys.stderr)