Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuaku458108d85172
diff --git a/src/tpu.py b/src/tpu.pyindex 89c1437..ec16ac5 100644--- a/src/tpu.py+++ b/src/tpu.py@@ -421,7 +421,10 @@ def tpu_list(args):def _sync_code_to_workers(name, zone, num_workers):- """Tarball source code and SCP to all workers. Returns True on success."""+ """Tarball source code and SCP to all workers. Returns True on success.++ Preserves .venv, .data_cache, and checkpoints on workers if they exist.+ """import tempfileproject_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))@@ -451,22 +454,32 @@ def _sync_code_to_workers(name, zone, num_workers):print("[tpu] ERROR: failed to copy tarball to workers.", file=sys.stderr)return False- # Extract on all workers (preserves .venv if it exists)+ # Extract on all workers, preserving .venv/.data_cache/checkpoints+ # 1. Move heavy dirs aside if they exist+ # 2. Remove old source code+ # 3. Extract fresh tarball+ # 4. Move heavy dirs backprint(f"[tpu] Extracting on all {num_workers} workers...")- result = _run_all_workers(name, zone,- "rm -rf /tmp/needle_new && tar -xzf /tmp/needle_sync.tar.gz -C /tmp "- "&& mv /tmp/needle_new 2>/dev/null; "- "rsync -a --exclude=.venv --exclude=.data_cache --exclude=checkpoints "- "/tmp/needle/ ~/needle/ "- "&& rm -rf /tmp/needle /tmp/needle_sync.tar.gz")+ extract_cmd = (+ "set -e; "+ "cd ~; "+ # Save heavy dirs if they exist+ "for d in .venv .data_cache checkpoints; do "+ " if [ -d ~/needle/$d ]; then mv ~/needle/$d /tmp/needle_save_$d; fi; "+ "done; "+ # Remove old code and extract new+ "rm -rf ~/needle; "+ "tar -xzf /tmp/needle_sync.tar.gz -C ~; "+ "rm -f /tmp/needle_sync.tar.gz; "+ # Restore heavy dirs+ "for d in .venv .data_cache checkpoints; do "+ " if [ -d /tmp/needle_save_$d ]; then mv /tmp/needle_save_$d ~/needle/$d; fi; "+ "done"+ )+ result = _run_all_workers(name, zone, extract_cmd)if result.returncode != 0:- # Fallback: simple overwrite (rsync may not be installed)- result = _run_all_workers(name, zone,- "rm -rf ~/needle && tar -xzf /tmp/needle_sync.tar.gz -C ~ "- "&& rm -f /tmp/needle_sync.tar.gz")- if result.returncode != 0:- print("[tpu] ERROR: failed to extract on workers.", file=sys.stderr)- return False+ print("[tpu] ERROR: failed to extract on workers.", file=sys.stderr)+ return Falsereturn True@@ -540,7 +553,9 @@ def _collect_env_exports():for var in ("HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "WANDB_API_KEY"):val = os.environ.get(var)if val:- exports.append(f"export {var}={val}")+ # Quote value to handle special characters in tokens+ safe_val = val.replace("'", "'\\''")+ exports.append(f"export {var}='{safe_val}'")return " && ".join(exports)@@ -558,19 +573,31 @@ def tpu_train(args):# Forward auth env vars to workersenv_exports = _collect_env_exports()if env_exports:+ env_count = len(env_exports.split(" && "))+ print(f"[tpu] Forwarding {env_count} env var(s) to workers")train_cmd = env_exports + " && " + train_cmdif num_workers <= 1:print(f"[tpu] Single-host TPU. Running training directly...")+ # Use quiet=True to avoid printing tokens, then log sanitized version+ print(f"[tpu] $ gcloud compute tpus tpu-vm ssh {args.name} --command 'needle train ...'")_run(["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,"--zone", zone, "--project", PROJECT,"--command", train_cmd],+ quiet=True,)returnprint(f"[tpu] Launching training on {num_workers} workers...")- result = _run_all_workers(args.name, zone, train_cmd)+ # Run with quiet=True to avoid printing tokens in command log+ result = _run(+ ["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,+ "--zone", zone, "--project", PROJECT,+ "--worker=all",+ "--command", train_cmd],+ check=False, quiet=True,+ )if result.returncode != 0:print("[tpu] WARNING: training exited with errors on some workers.",file=sys.stderr)diff --git a/src/train.py b/src/train.pyindex dac6283..0cffccf 100644--- a/src/train.py+++ b/src/train.py@@ -9,7 +9,6 @@ import jax.numpy as jnpimport numpy as npimport optaxfrom tqdm import tqdm-from flax import jax_utilsfrom flax.training import train_statefrom .data import (@@ -29,6 +28,26 @@ from .model import (_HF_CHECKPOINT_REPO = "Cactus-Compute/checkpoints"+def _replicate(tree):+ """Replicate a pytree across all local devices (multi-host safe)."""+ devices = jax.local_devices()+ return jax.tree.map(+ lambda x: jax.device_put_replicated(x, devices), tree)+++def _unreplicate(tree):+ """Get a single copy from a replicated pytree (multi-host safe).++ Uses addressable_shards instead of x[0] indexing, which fails+ in multi-host pmap because the array spans devices on other hosts.+ """+ return jax.tree.map(+ lambda x: jax.device_get(x.addressable_shards[0].data)+ if hasattr(x, 'addressable_shards') and x.addressable_shards+ else x,+ tree)++def _upload_checkpoint(ckpt_path):"""Upload a checkpoint file to HuggingFace Hub in a background thread."""import threading@@ -614,9 +633,9 @@ def train(args):state = state.replace(params=ckpt_params)print(f" Loaded checkpoint params into train state")- state = jax_utils.replicate(state)+ state = _replicate(state)- param_count = sum(x.size for x in jax.tree.leaves(jax_utils.unreplicate(state).params))+ param_count = sum(x.size for x in jax.tree.leaves(_unreplicate(state).params))decay_steps = max(1, int(total_steps * decay_ratio))stable_steps = total_steps - warmup_steps - decay_stepsbest_call_f1 = 0.0@@ -665,8 +684,8 @@ def train(args):prune_mask = Nonegradual_sparsify_done = False# Dummy all-ones prune mask (replicated) — used when pruning is inactive- _unrep_params = jax_utils.unreplicate(state).params- dummy_prune_mask = jax_utils.replicate(jax.tree.map(jnp.ones_like, _unrep_params))+ _unrep_params = _unreplicate(state).params+ dummy_prune_mask = _replicate(jax.tree.map(jnp.ones_like, _unrep_params))del _unrep_params# Dummy contrastive arrays — used when contrastive is inactivedummy_cl_tokens = jnp.zeros((num_devices, args.batch_size, 128), dtype=jnp.int32)@@ -761,25 +780,25 @@ def train(args):cl_q_b, cl_t_b, cl_rngs, do_qat, do_cl, do_prune,)- text_loss_val = float(loss[0])+ text_loss_val = float(loss.addressable_shards[0].data)text_losses.append(text_loss_val)- step_grad_norm = float(grad_norm[0])+ step_grad_norm = float(grad_norm.addressable_shards[0].data)global_step += 1if epoch == weight_prune_epoch and not gradual_sparsify_done:epoch_step += 1current_sparsity = _cubic_sparsity_schedule(epoch_step, t_start, t_end, sparsity_ratio)if epoch_step >= t_start and epoch_step % prune_interval == 0 and current_sparsity > 0:- ema_unr = jax_utils.unreplicate(state.params)+ ema_unr = _unreplicate(state.params)mask = _make_prune_mask(ema_unr, current_sparsity, _GROUP_SIZE)del ema_unr- prune_mask = jax_utils.replicate(mask)+ prune_mask = _replicate(mask)del maskdt = time.perf_counter() - t0eval_every = getattr(args, "eval_every", 100)if is_main and (global_step % eval_every == 0 or global_step == total_steps):- _eval_params = jax_utils.unreplicate(state).params+ _eval_params = _unreplicate(state).paramstotal_loss, total_toks = 0.0, 0.0_val_n = count_batches(len(val_enc), args.batch_size)for vb in tqdm(get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False,@@ -832,14 +851,14 @@ def train(args):if epoch == weight_prune_epoch and not gradual_sparsify_done:gradual_sparsify_done = Trueif prune_mask is None:- ema_unr = jax_utils.unreplicate(state.params)+ ema_unr = _unreplicate(state.params)mask = _make_prune_mask(ema_unr, sparsity_ratio, _GROUP_SIZE)del ema_unr- prune_mask = jax_utils.replicate(mask)+ prune_mask = _replicate(mask)del maskstate = state.replace(params=jax.tree.map(lambda w, m: w * m, state.params, prune_mask))- final_pruned = jax.tree.map(np.array, jax_utils.unreplicate(state).params)+ final_pruned = jax.tree.map(np.array, _unreplicate(state).params)total_p = sum(x.size for x in jax.tree.leaves(final_pruned))zero_p = sum(int(np.sum(np.abs(x) < 1e-6)) for x in jax.tree.leaves(final_pruned))print(f"\n Gradual sparsification complete — mask locked.")@@ -854,12 +873,12 @@ def train(args):# Non-main hosts wait here while host 0 runs eval/checkpointcontinue- eval_params = jax_utils.unreplicate(state).params+ eval_params = _unreplicate(state).paramsq_params = _quantize_params(eval_params, group_size=_GROUP_SIZE, precision=_PRECISION)mat_vl_fns = {}if _MAT_FACTORS:- _apply_fn = jax_utils.unreplicate(state).apply_fn+ _apply_fn = _unreplicate(state).apply_fnmat_vl_fns = {f: _make_mat_val_loss_fn(_apply_fn, fw)for f, fw in zip(_MAT_FACTORS, _MAT_FF_WIDTHS)}del _apply_fn