Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuaku458108d85172
diff --git a/src/tpu.py b/src/tpu.py
index 89c1437..ec16ac5 100644
--- a/src/tpu.py
+++ b/src/tpu.py
@@ -421,7 +421,10 @@ def tpu_list(args):
def _sync_code_to_workers(name, zone, num_workers):
- """Tarball source code and SCP to all workers. Returns True on success."""
+ """Tarball source code and SCP to all workers. Returns True on success.
+
+ Preserves .venv, .data_cache, and checkpoints on workers if they exist.
+ """
import tempfile
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -451,22 +454,32 @@ def _sync_code_to_workers(name, zone, num_workers):
print("[tpu] ERROR: failed to copy tarball to workers.", file=sys.stderr)
return False
- # Extract on all workers (preserves .venv if it exists)
+ # Extract on all workers, preserving .venv/.data_cache/checkpoints
+ # 1. Move heavy dirs aside if they exist
+ # 2. Remove old source code
+ # 3. Extract fresh tarball
+ # 4. Move heavy dirs back
print(f"[tpu] Extracting on all {num_workers} workers...")
- result = _run_all_workers(name, zone,
- "rm -rf /tmp/needle_new && tar -xzf /tmp/needle_sync.tar.gz -C /tmp "
- "&& mv /tmp/needle_new 2>/dev/null; "
- "rsync -a --exclude=.venv --exclude=.data_cache --exclude=checkpoints "
- "/tmp/needle/ ~/needle/ "
- "&& rm -rf /tmp/needle /tmp/needle_sync.tar.gz")
+ extract_cmd = (
+ "set -e; "
+ "cd ~; "
+ # Save heavy dirs if they exist
+ "for d in .venv .data_cache checkpoints; do "
+ " if [ -d ~/needle/$d ]; then mv ~/needle/$d /tmp/needle_save_$d; fi; "
+ "done; "
+ # Remove old code and extract new
+ "rm -rf ~/needle; "
+ "tar -xzf /tmp/needle_sync.tar.gz -C ~; "
+ "rm -f /tmp/needle_sync.tar.gz; "
+ # Restore heavy dirs
+ "for d in .venv .data_cache checkpoints; do "
+ " if [ -d /tmp/needle_save_$d ]; then mv /tmp/needle_save_$d ~/needle/$d; fi; "
+ "done"
+ )
+ result = _run_all_workers(name, zone, extract_cmd)
if result.returncode != 0:
- # Fallback: simple overwrite (rsync may not be installed)
- result = _run_all_workers(name, zone,
- "rm -rf ~/needle && tar -xzf /tmp/needle_sync.tar.gz -C ~ "
- "&& rm -f /tmp/needle_sync.tar.gz")
- if result.returncode != 0:
- print("[tpu] ERROR: failed to extract on workers.", file=sys.stderr)
- return False
+ print("[tpu] ERROR: failed to extract on workers.", file=sys.stderr)
+ return False
return True
@@ -540,7 +553,9 @@ def _collect_env_exports():
for var in ("HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "WANDB_API_KEY"):
val = os.environ.get(var)
if val:
- exports.append(f"export {var}={val}")
+ # Quote value to handle special characters in tokens
+ safe_val = val.replace("'", "'\\''")
+ exports.append(f"export {var}='{safe_val}'")
return " && ".join(exports)
@@ -558,19 +573,31 @@ def tpu_train(args):
# Forward auth env vars to workers
env_exports = _collect_env_exports()
if env_exports:
+ env_count = len(env_exports.split(" && "))
+ print(f"[tpu] Forwarding {env_count} env var(s) to workers")
train_cmd = env_exports + " && " + train_cmd
if num_workers <= 1:
print(f"[tpu] Single-host TPU. Running training directly...")
+ # Use quiet=True to avoid printing tokens, then log sanitized version
+ print(f"[tpu] $ gcloud compute tpus tpu-vm ssh {args.name} --command 'needle train ...'")
_run(
["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,
"--zone", zone, "--project", PROJECT,
"--command", train_cmd],
+ quiet=True,
)
return
print(f"[tpu] Launching training on {num_workers} workers...")
- result = _run_all_workers(args.name, zone, train_cmd)
+ # Run with quiet=True to avoid printing tokens in command log
+ result = _run(
+ ["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,
+ "--zone", zone, "--project", PROJECT,
+ "--worker=all",
+ "--command", train_cmd],
+ check=False, quiet=True,
+ )
if result.returncode != 0:
print("[tpu] WARNING: training exited with errors on some workers.",
file=sys.stderr)
diff --git a/src/train.py b/src/train.py
index dac6283..0cffccf 100644
--- a/src/train.py
+++ b/src/train.py
@@ -9,7 +9,6 @@ import jax.numpy as jnp
import numpy as np
import optax
from tqdm import tqdm
-from flax import jax_utils
from flax.training import train_state
from .data import (
@@ -29,6 +28,26 @@ from .model import (
_HF_CHECKPOINT_REPO = "Cactus-Compute/checkpoints"
+def _replicate(tree):
+ """Replicate a pytree across all local devices (multi-host safe)."""
+ devices = jax.local_devices()
+ return jax.tree.map(
+ lambda x: jax.device_put_replicated(x, devices), tree)
+
+
+def _unreplicate(tree):
+ """Get a single copy from a replicated pytree (multi-host safe).
+
+ Uses addressable_shards instead of x[0] indexing, which fails
+ in multi-host pmap because the array spans devices on other hosts.
+ """
+ return jax.tree.map(
+ lambda x: jax.device_get(x.addressable_shards[0].data)
+ if hasattr(x, 'addressable_shards') and x.addressable_shards
+ else x,
+ tree)
+
+
def _upload_checkpoint(ckpt_path):
"""Upload a checkpoint file to HuggingFace Hub in a background thread."""
import threading
@@ -614,9 +633,9 @@ def train(args):
state = state.replace(params=ckpt_params)
print(f" Loaded checkpoint params into train state")
- state = jax_utils.replicate(state)
+ state = _replicate(state)
- param_count = sum(x.size for x in jax.tree.leaves(jax_utils.unreplicate(state).params))
+ param_count = sum(x.size for x in jax.tree.leaves(_unreplicate(state).params))
decay_steps = max(1, int(total_steps * decay_ratio))
stable_steps = total_steps - warmup_steps - decay_steps
best_call_f1 = 0.0
@@ -665,8 +684,8 @@ def train(args):
prune_mask = None
gradual_sparsify_done = False
# Dummy all-ones prune mask (replicated) — used when pruning is inactive
- _unrep_params = jax_utils.unreplicate(state).params
- dummy_prune_mask = jax_utils.replicate(jax.tree.map(jnp.ones_like, _unrep_params))
+ _unrep_params = _unreplicate(state).params
+ dummy_prune_mask = _replicate(jax.tree.map(jnp.ones_like, _unrep_params))
del _unrep_params
# Dummy contrastive arrays — used when contrastive is inactive
dummy_cl_tokens = jnp.zeros((num_devices, args.batch_size, 128), dtype=jnp.int32)
@@ -761,25 +780,25 @@ def train(args):
cl_q_b, cl_t_b, cl_rngs, do_qat, do_cl, do_prune,
)
- text_loss_val = float(loss[0])
+ text_loss_val = float(loss.addressable_shards[0].data)
text_losses.append(text_loss_val)
- step_grad_norm = float(grad_norm[0])
+ step_grad_norm = float(grad_norm.addressable_shards[0].data)
global_step += 1
if epoch == weight_prune_epoch and not gradual_sparsify_done:
epoch_step += 1
current_sparsity = _cubic_sparsity_schedule(epoch_step, t_start, t_end, sparsity_ratio)
if epoch_step >= t_start and epoch_step % prune_interval == 0 and current_sparsity > 0:
- ema_unr = jax_utils.unreplicate(state.params)
+ ema_unr = _unreplicate(state.params)
mask = _make_prune_mask(ema_unr, current_sparsity, _GROUP_SIZE)
del ema_unr
- prune_mask = jax_utils.replicate(mask)
+ prune_mask = _replicate(mask)
del mask
dt = time.perf_counter() - t0
eval_every = getattr(args, "eval_every", 100)
if is_main and (global_step % eval_every == 0 or global_step == total_steps):
- _eval_params = jax_utils.unreplicate(state).params
+ _eval_params = _unreplicate(state).params
total_loss, total_toks = 0.0, 0.0
_val_n = count_batches(len(val_enc), args.batch_size)
for vb in tqdm(get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False,
@@ -832,14 +851,14 @@ def train(args):
if epoch == weight_prune_epoch and not gradual_sparsify_done:
gradual_sparsify_done = True
if prune_mask is None:
- ema_unr = jax_utils.unreplicate(state.params)
+ ema_unr = _unreplicate(state.params)
mask = _make_prune_mask(ema_unr, sparsity_ratio, _GROUP_SIZE)
del ema_unr
- prune_mask = jax_utils.replicate(mask)
+ prune_mask = _replicate(mask)
del mask
state = state.replace(
params=jax.tree.map(lambda w, m: w * m, state.params, prune_mask))
- final_pruned = jax.tree.map(np.array, jax_utils.unreplicate(state).params)
+ final_pruned = jax.tree.map(np.array, _unreplicate(state).params)
total_p = sum(x.size for x in jax.tree.leaves(final_pruned))
zero_p = sum(int(np.sum(np.abs(x) < 1e-6)) for x in jax.tree.leaves(final_pruned))
print(f"\n Gradual sparsification complete — mask locked.")
@@ -854,12 +873,12 @@ def train(args):
# Non-main hosts wait here while host 0 runs eval/checkpoint
continue
- eval_params = jax_utils.unreplicate(state).params
+ eval_params = _unreplicate(state).params
q_params = _quantize_params(eval_params, group_size=_GROUP_SIZE, precision=_PRECISION)
mat_vl_fns = {}
if _MAT_FACTORS:
- _apply_fn = jax_utils.unreplicate(state).apply_fn
+ _apply_fn = _unreplicate(state).apply_fn
mat_vl_fns = {f: _make_mat_val_loss_fn(_apply_fn, fw)
for f, fw in zip(_MAT_FACTORS, _MAT_FF_WIDTHS)}
del _apply_fn