Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuaku48ac5862343e
diff --git a/src/cli.py b/src/cli.py
index 10cdfc8..854fee3 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -77,6 +77,10 @@ def main():
p = sub.add_parser("pretrain", add_help=False)
p.add_argument("--name", type=str, default="pretrain",
help="Experiment name for wandb (default: pretrain)")
+ p.add_argument("--checkpoint", type=str, default=None,
+ help="Resume from checkpoint (e.g. checkpoints/needle_base.pkl)")
+ p.add_argument("--resume-step", type=int, default=None,
+ help="Override resume step (skip this many batches)")
p.add_argument("--epochs", type=int, default=1)
p.add_argument("--batch-size", type=int, default=128)
p.add_argument("--lr", type=float, default=3e-4)
diff --git a/src/pretrain.py b/src/pretrain.py
index cec5a93..7cee4be 100644
--- a/src/pretrain.py
+++ b/src/pretrain.py
@@ -222,10 +222,30 @@ def pretrain(args):
muon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(total_devices)
decay_ratio = getattr(args, "decay_ratio", 0.05)
+ resume_step = 0
+ resume_checkpoint = getattr(args, "checkpoint", None)
+
print(f"\n[3/3] Initializing model...")
rng = jax.random.PRNGKey(args.seed)
rng, init_rng = jax.random.split(rng)
state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps, decay_ratio)
+
+ if resume_checkpoint:
+ print(f" Resuming from {resume_checkpoint}")
+ with open(resume_checkpoint, "rb") as f:
+ ckpt_data = pickle.load(f)
+ ckpt_params = jax.tree.map(jnp.array, ckpt_data["params"])
+ state = state.replace(params=ckpt_params)
+ del ckpt_params
+ # Use --resume-step override, else read from checkpoint, else 0
+ manual_step = getattr(args, "resume_step", None)
+ if manual_step is not None:
+ resume_step = manual_step
+ else:
+ resume_step = ckpt_data.get("pretrain_step", 0)
+ if resume_step > 0:
+ print(f" Resuming from step {resume_step}")
+
state = _replicate(state)
param_count = sum(x.size for x in jax.tree.leaves(_unreplicate(state).params))
@@ -259,7 +279,7 @@ def pretrain(args):
print(f" Save every {save_every:>12,}")
print(f" ─────────────────────────────────────\n")
- global_step = 0
+ global_step = resume_step
for epoch in range(args.epochs):
batch_stream = _PrefetchStream(
@@ -270,7 +290,17 @@ def pretrain(args):
)
pbar = tqdm(desc=f"Pretrain epoch {epoch + 1}/{args.epochs}",
- total=estimated_steps, disable=not is_main)
+ total=estimated_steps, initial=resume_step,
+ disable=not is_main)
+
+ # Skip ahead if resuming
+ if resume_step > 0:
+ if is_main:
+ print(f" Skipping {resume_step} batches to resume...")
+ for i, _ in enumerate(batch_stream):
+ if i + 1 >= resume_step:
+ break
+ resume_step = 0 # only skip once
for batch in batch_stream:
if max_steps and global_step >= max_steps:
@@ -342,7 +372,8 @@ def _save_pretrain_checkpoint(state, config, checkpoint_dir, global_step):
ckpt_path = os.path.join(checkpoint_dir, "needle_base.pkl")
with open(ckpt_path, "wb") as f:
- pickle.dump({"params": params_np, "config": config.__dict__}, f)
+ pickle.dump({"params": params_np, "config": config.__dict__,
+ "pretrain_step": global_step}, f)
param_count = sum(x.size for x in jax.tree.leaves(params_np))
size_mb = sum(x.nbytes for x in jax.tree.leaves(params_np)) / 1e6
diff --git a/src/tpu.py b/src/tpu.py
index 5864d57..e3e5d3a 100644
--- a/src/tpu.py
+++ b/src/tpu.py
@@ -579,18 +579,25 @@ def _tpu_run_command(args, needle_cmd="train"):
print(f"[tpu] Forwarding {env_count} env var(s) to workers")
train_cmd = env_exports + " && " + train_cmd
+ # Wrap with nohup so training survives SSH disconnects
+ log_file = f"/tmp/needle_{needle_cmd}.log"
+ train_cmd = (
+ f"nohup bash -c '{train_cmd}' > {log_file} 2>&1 &"
+ f" echo $!; sleep 1; head -5 {log_file}"
+ )
+
if num_workers <= 1:
- print(f"[tpu] Single-host TPU. Running directly...")
- print(f"[tpu] $ gcloud ... --command 'needle {needle_cmd} ...'")
+ print(f"[tpu] Single-host TPU. Running in background...")
_run(
["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,
"--zone", zone, "--project", PROJECT,
"--command", train_cmd],
quiet=True,
)
+ print(f"[tpu] Running in background. Logs: ssh into worker and 'tail -f {log_file}'")
return
- print(f"[tpu] Launching 'needle {needle_cmd}' on {num_workers} workers...")
+ print(f"[tpu] Launching 'needle {needle_cmd}' on {num_workers} workers (background)...")
result = _run(
["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,
"--zone", zone, "--project", PROJECT,
@@ -598,6 +605,8 @@ def _tpu_run_command(args, needle_cmd="train"):
"--command", train_cmd],
check=False, quiet=True,
)
+ print(f"[tpu] Processes launched. Monitor with:")
+ print(f" gcloud compute tpus tpu-vm ssh large --zone=asia-northeast1-b --worker=0 --command='tail -f {log_file}'")
if result.returncode != 0:
print(f"[tpu] WARNING: 'needle {needle_cmd}' exited with errors on some workers.",
file=sys.stderr)