Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Updates

HenryNdubuaku241841503242
diff --git a/pyproject.toml b/pyproject.toml
index 4a8b394..73a02c0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,12 +3,12 @@ name = "needle"
version = "0.1.0"
requires-python = ">=3.11"
dependencies = [
- "jax==0.9.0.1",
- "jaxlib==0.9.0.1",
- "flax==0.12.4",
- "optax==0.2.6",
- "datasets==4.5.0",
- "transformers==5.2.0",
+ "jax",
+ "jaxlib",
+ "flax",
+ "optax",
+ "datasets",
+ "transformers",
"wandb",
]
diff --git a/requirements.txt b/requirements.txt
index 16c0614..0b2e3f0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,8 +1,8 @@
-jax==0.9.0.1
-jaxlib==0.9.0.1
-flax==0.12.4
-optax==0.2.6
-datasets==4.5.0
-transformers==5.2.0
-wandb>=0.19.0
-pyyaml>=6.0
+jax
+jaxlib
+flax
+optax
+datasets
+transformers
+wandb
+pyyaml
diff --git a/setup b/setup
index 013fc20..527aaa7 100755
--- a/setup
+++ b/setup
@@ -53,6 +53,34 @@ echo "Installing dependencies..."
pip install --upgrade pip -q
pip install -e . -q
+# If running on a TPU VM, load drivers and install JAX with TPU support
+if lspci 2>/dev/null | grep -qi "google.*processing"; then
+ echo "TPU hardware detected..."
+
+ # Load kernel modules if not already loaded
+ if ! ls /dev/accel* &>/dev/null 2>&1; then
+ echo "Loading TPU kernel modules..."
+ for mod in gasket tpu_common tpu_v2 tpu_v3 tpu_v4 tpu_v4_lite; do
+ sudo modprobe "$mod" 2>/dev/null || true
+ done
+ fi
+
+ if ls /dev/accel* &>/dev/null 2>&1; then
+ echo "TPU devices: $(ls /dev/accel* 2>/dev/null | wc -w) chip(s)"
+ else
+ echo "WARNING: TPU PCI devices found but no /dev/accel* — kernel drivers may not match hardware."
+ echo " Recreate the VM with the correct --version for your accelerator type."
+ fi
+
+ echo "Installing jax[tpu]..."
+ pip install "jax[tpu]" -q -f https://storage.googleapis.com/jax-releases/libtpu_releases.html
+
+ if [ -f /sys/kernel/mm/transparent_hugepage/enabled ]; then
+ echo "Enabling transparent hugepages..."
+ sudo sh -c "echo always > /sys/kernel/mm/transparent_hugepage/enabled" 2>/dev/null || true
+ fi
+fi
+
if [ -z "$WANDB_API_KEY" ]; then
echo ""
echo " Enter your W&B API key (https://wandb.ai/authorize)"
diff --git a/src/cli.py b/src/cli.py
index fbd0398..49a5a12 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -54,8 +54,8 @@ HELP = """
│ │
│ tpu │
│ create NAME Create TPU (auto-finds zone) │
- │ --type STR Accelerator type (default:v5litepod-4)│
- │ --version STR TPU OS (default: tpu-ubuntu2204-base) │
+ │ --type STR Accelerator type (default: v6e-4) │
+ │ --version STR TPU OS (auto-detected from --type) │
│ connect NAME SSH config + connect (auto-zone) │
│ claude NAME Install Claude Code on instance │
│ stop NAME Stop instance (auto-zone) │
@@ -140,8 +140,9 @@ def main():
tp = tpu_sub.add_parser("create", add_help=False)
tp.add_argument("name", type=str)
- tp.add_argument("--type", dest="accel_type", type=str, default="v5litepod-4")
- tp.add_argument("--version", type=str, default="tpu-ubuntu2204-base")
+ tp.add_argument("--type", dest="accel_type", type=str, default="v6e-4")
+ tp.add_argument("--version", type=str, default=None,
+ help="Software version (auto-detected from --type if omitted)")
tp = tpu_sub.add_parser("connect", add_help=False)
tp.add_argument("name", type=str)
diff --git a/src/evaluate.py b/src/evaluate.py
index 816a25a..df544c0 100644
--- a/src/evaluate.py
+++ b/src/evaluate.py
@@ -6,6 +6,7 @@ import jax
import jax.numpy as jnp
import numpy as np
from datasets import load_dataset
+from flax import jax_utils
from .data import get_tokenizer
from .model import (
@@ -24,32 +25,88 @@ def load_checkpoint(path):
return params, config
-def score_sequence(model, params, enc_tokens, dec_tokens, pad_id):
+def _make_p_encode(model):
+ """Create a pmap'd encode function."""
+ def _encode(params, src, src_mask):
+ return model.apply(
+ {"params": params}, src, src_mask=src_mask, deterministic=True, method="encode",
+ )
+ return jax.pmap(_encode, axis_name="batch")
+
+
+def _make_p_decode(model):
+ """Create a pmap'd decode function."""
+ def _decode(params, dec_input, encoder_out, tgt_mask, cross_mask):
+ return model.apply(
+ {"params": params}, dec_input, encoder_out,
+ self_mask=tgt_mask, cross_mask=cross_mask, deterministic=True, method="decode",
+ )
+ return jax.pmap(_decode, axis_name="batch")
+
+
+def _make_p_forward(model):
+ """Create a pmap'd full forward function."""
+ def _forward(params, src, tgt_in, src_mask, tgt_mask, cross_mask):
+ return model.apply(
+ {"params": params}, src, tgt_in,
+ src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
+ deterministic=True,
+ )
+ return jax.pmap(_forward, axis_name="batch")
+
+
+def _shard_single(x, num_devices):
+ """Replicate a single-sample batch across all devices for pmap."""
+ return jnp.broadcast_to(x, (num_devices, *x.shape[1:]))
+
+
+def _shard_batch(x, num_devices):
+ """Reshape batch for pmap: (N, ...) -> (num_devices, per_device, ...)."""
+ return x.reshape(num_devices, -1, *x.shape[1:])
+
+
+def score_sequence(model, params, enc_tokens, dec_tokens, pad_id, p_encode=None, p_decode=None, num_devices=1):
"""Compute average negative log-likelihood of dec_tokens given enc_tokens."""
enc_input = jnp.array([enc_tokens])
src_mask = make_padding_mask(enc_input, pad_id)
- encoder_out = model.apply(
- {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",
- )
+ if p_encode is not None and num_devices > 1:
+ # Replicate single sample across devices, run pmap, take first result
+ enc_s = _shard_single(enc_input, num_devices)
+ src_mask_s = _shard_single(src_mask, num_devices)
+ encoder_out = p_encode(params, enc_s, src_mask_s)[0:1]
+ else:
+ p = params if num_devices <= 1 else jax_utils.unreplicate(params)
+ encoder_out = model.apply(
+ {"params": p}, enc_input, src_mask=src_mask, deterministic=True, method="encode",
+ )
dec_in = [pad_id] + list(dec_tokens[:-1])
dec_input = jnp.array([dec_in])
tgt_mask = make_causal_mask(len(dec_in))
cross_mask = src_mask
- logits = model.apply(
- {"params": params}, dec_input, encoder_out,
- self_mask=tgt_mask, cross_mask=cross_mask, deterministic=True, method="decode",
- )
+ if p_decode is not None and num_devices > 1:
+ dec_s = _shard_single(dec_input, num_devices)
+ tgt_mask_s = jnp.broadcast_to(tgt_mask, (num_devices, *tgt_mask.shape[1:]))
+ cross_mask_s = _shard_single(cross_mask, num_devices)
+ enc_out_s = _shard_single(encoder_out, num_devices)
+ logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, cross_mask_s)[0]
+ else:
+ p = params if num_devices <= 1 else jax_utils.unreplicate(params)
+ logits = model.apply(
+ {"params": p}, dec_input, encoder_out,
+ self_mask=tgt_mask, cross_mask=cross_mask, deterministic=True, method="decode",
+ )[0]
- log_probs = jax.nn.log_softmax(logits[0])
+ log_probs = jax.nn.log_softmax(logits if logits.ndim == 2 else logits[0])
target_ids = jnp.array(dec_tokens)
token_lls = log_probs[jnp.arange(len(dec_tokens)), target_ids]
return float(jnp.mean(token_lls))
-def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256):
+def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256,
+ num_devices=1, p_encode=None, p_decode=None):
"""Perplexity on WikiText-2 test split."""
ds = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="test")
@@ -58,6 +115,8 @@ def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256):
total_tokens = 0
evaluated = 0
+ single_params = jax_utils.unreplicate(params) if num_devices > 1 else params
+
for example in ds:
text = example["text"].strip()
if len(text) < 20:
@@ -74,20 +133,33 @@ def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256):
enc_input = jnp.array([enc_tokens])
src_mask = make_padding_mask(enc_input, pad_id)
- encoder_out = model.apply(
- {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",
- )
+
+ if p_encode is not None and num_devices > 1:
+ enc_s = _shard_single(enc_input, num_devices)
+ src_mask_s = _shard_single(src_mask, num_devices)
+ encoder_out = p_encode(params, enc_s, src_mask_s)[0:1]
+ else:
+ encoder_out = model.apply(
+ {"params": single_params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",
+ )
dec_in = [pad_id] + list(dec_tokens[:-1])
dec_input = jnp.array([dec_in])
tgt_mask = make_causal_mask(len(dec_in))
- logits = model.apply(
- {"params": params}, dec_input, encoder_out,
- self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",
- )
-
- log_probs = jax.nn.log_softmax(logits[0])
+ if p_decode is not None and num_devices > 1:
+ dec_s = _shard_single(dec_input, num_devices)
+ tgt_mask_s = jnp.broadcast_to(tgt_mask, (num_devices, *tgt_mask.shape[1:]))
+ cross_mask_s = _shard_single(src_mask, num_devices)
+ enc_out_s = _shard_single(encoder_out, num_devices)
+ logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, cross_mask_s)[0]
+ else:
+ logits = model.apply(
+ {"params": single_params}, dec_input, encoder_out,
+ self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",
+ )
+
+ log_probs = jax.nn.log_softmax(logits[0] if logits.ndim == 3 else logits[0])
target_ids = jnp.array(dec_tokens)
token_lls = log_probs[jnp.arange(len(dec_tokens)), target_ids]
@@ -102,7 +174,8 @@ def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256):
return {"perplexity": ppl, "samples": evaluated, "tokens": total_tokens}
-def eval_lambada(model, params, tokenizer, max_samples=500):
+def eval_lambada(model, params, tokenizer, max_samples=500,
+ num_devices=1, p_encode=None, p_decode=None):
"""Accuracy of predicting the final word on LAMBADA."""
ds = load_dataset("EleutherAI/lambada_openai", "default", split="test")
@@ -110,6 +183,8 @@ def eval_lambada(model, params, tokenizer, max_samples=500):
correct = 0
total = 0
+ single_params = jax_utils.unreplicate(params) if num_devices > 1 else params
+
for example in ds:
text = example["text"].strip()
words = text.rsplit(" ", 1)
@@ -125,19 +200,32 @@ def eval_lambada(model, params, tokenizer, max_samples=500):
enc_input = jnp.array([context_tokens])
src_mask = make_padding_mask(enc_input, pad_id)
- encoder_out = model.apply(
- {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",
- )
+
+ if p_encode is not None and num_devices > 1:
+ enc_s = _shard_single(enc_input, num_devices)
+ src_mask_s = _shard_single(src_mask, num_devices)
+ encoder_out = p_encode(params, enc_s, src_mask_s)[0:1]
+ else:
+ encoder_out = model.apply(
+ {"params": single_params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",
+ )
dec_in = jnp.array([[pad_id]])
tgt_mask = make_causal_mask(1)
- logits = model.apply(
- {"params": params}, dec_in, encoder_out,
- self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",
- )
-
- predicted = int(jnp.argmax(logits[0, 0]))
+ if p_decode is not None and num_devices > 1:
+ dec_s = _shard_single(dec_in, num_devices)
+ tgt_mask_s = jnp.broadcast_to(tgt_mask, (num_devices, *tgt_mask.shape[1:]))
+ cross_mask_s = _shard_single(src_mask, num_devices)
+ enc_out_s = _shard_single(encoder_out, num_devices)
+ logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, cross_mask_s)[0]
+ else:
+ logits = model.apply(
+ {"params": single_params}, dec_in, encoder_out,
+ self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",
+ )
+
+ predicted = int(jnp.argmax(logits[0, 0] if logits.ndim == 3 else logits[0, 0]))
if predicted == target_tokens[0]:
correct += 1
total += 1
@@ -149,7 +237,8 @@ def eval_lambada(model, params, tokenizer, max_samples=500):
return {"accuracy": acc, "correct": correct, "total": total}
-def eval_hellaswag(model, params, tokenizer, max_samples=500):
+def eval_hellaswag(model, params, tokenizer, max_samples=500,
+ num_devices=1, p_encode=None, p_decode=None):
"""Accuracy on HellaSwag by scoring each candidate ending."""
ds = load_dataset("Rowan/hellaswag", split="validation")
@@ -176,7 +265,8 @@ def eval_hellaswag(model, params, tokenizer, max_samples=500):
scores.append(float("-inf"))
continue
dec_tokens = dec_tokens[:64]
- score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id)
+ score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id,
+ p_encode=p_encode, p_decode=p_decode, num_devices=num_devices)
scores.append(score)
predicted = int(np.argmax(scores))
@@ -191,7 +281,8 @@ def eval_hellaswag(model, params, tokenizer, max_samples=500):
return {"accuracy": acc, "correct": correct, "total": total}
-def eval_arc_easy(model, params, tokenizer, max_samples=500):
+def eval_arc_easy(model, params, tokenizer, max_samples=500,
+ num_devices=1, p_encode=None, p_decode=None):
"""Accuracy on ARC-Easy by scoring each answer choice."""
ds = load_dataset("allenai/ai2_arc", "ARC-Easy", split="test")
@@ -218,7 +309,8 @@ def eval_arc_easy(model, params, tokenizer, max_samples=500):
scores.append(float("-inf"))
continue
dec_tokens = dec_tokens[:64]
- score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id)
+ score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id,
+ p_encode=p_encode, p_decode=p_decode, num_devices=num_devices)
scores.append(score)
predicted_idx = int(np.argmax(scores))
@@ -243,6 +335,9 @@ BENCHMARKS = {
def main(args):
+ num_devices = jax.local_device_count()
+ print(f"Detected {num_devices} device(s) for data-parallel evaluation")
+
print(f"Loading checkpoint: {args.checkpoint}")
params, config = load_checkpoint(args.checkpoint)
model = EncoderDecoderTransformer(config)
@@ -251,6 +346,16 @@ def main(args):
param_count = sum(x.size for x in jax.tree.leaves(params))
print(f"Model parameters: {param_count:,}")
+ # Replicate params across devices for pmap
+ if num_devices > 1:
+ params = jax_utils.replicate(params)
+ p_encode = _make_p_encode(model)
+ p_decode = _make_p_decode(model)
+ print(f"Params replicated across {num_devices} devices")
+ else:
+ p_encode = None
+ p_decode = None
+
benchmarks = args.benchmarks or list(BENCHMARKS.keys())
results = {}
@@ -263,7 +368,10 @@ def main(args):
print(f"Benchmark: {name}")
print("=" * 50)
- result = BENCHMARKS[name](model, params, tokenizer, max_samples=args.max_samples)
+ result = BENCHMARKS[name](
+ model, params, tokenizer, max_samples=args.max_samples,
+ num_devices=num_devices, p_encode=p_encode, p_decode=p_decode,
+ )
results[name] = result
for k, v in result.items():
diff --git a/src/tpu.py b/src/tpu.py
index e49042c..c4f9155 100644
--- a/src/tpu.py
+++ b/src/tpu.py
@@ -6,6 +6,37 @@ import sys
PROJECT = "needle-488623"
+VERSION_FOR_TYPE = {
+ "v2": "tpu-ubuntu2204-base",
+ "v3": "tpu-ubuntu2204-base",
+ "v4": "tpu-ubuntu2204-base",
+ "v5litepod": "v2-alpha-tpuv5-lite",
+ "v5e": "v2-alpha-tpuv5-lite",
+ "v5p": "v2-alpha-tpuv5",
+ "v6e": "v2-alpha-tpuv6e",
+}
+
+
+def _resolve_version(accel_type, explicit_version):
+ """Pick the right software version for the accelerator type.
+
+ If the user explicitly passed --version, use that.
+ Otherwise, match on the accelerator type prefix (e.g. 'v6e-4' → 'v6e').
+ """
+ if explicit_version is not None:
+ return explicit_version
+ for prefix, version in sorted(VERSION_FOR_TYPE.items(), key=lambda kv: -len(kv[0])):
+ if accel_type.startswith(prefix):
+ return version
+
+ print(
+ f"[tpu] WARNING: unknown accelerator type '{accel_type}', "
+ f"defaulting to tpu-ubuntu2204-base. Pass --version to override.",
+ file=sys.stderr,
+ )
+ return "tpu-ubuntu2204-base"
+
+
ZONES = [
"us-central1-a", "us-central1-b", "us-central1-f",
"us-east1-b", "us-east1-c", "us-east1-d",
@@ -80,20 +111,94 @@ def _update_ssh_config(path, host_name, new_block):
f.write(new_block)
+def _check_tpu_health(name, zone):
+ """SSH into the VM and verify /dev/accel* devices exist."""
+ print(f"[tpu] Verifying TPU devices on '{name}'...")
+ result = _run(
+ ["gcloud", "compute", "tpus", "tpu-vm", "ssh", name,
+ "--zone", zone, "--project", PROJECT,
+ "--command", "ls /dev/accel* 2>/dev/null && echo TPU_OK || echo TPU_MISSING"],
+ check=False, capture=True, quiet=True,
+ )
+ output = result.stdout.strip()
+ if "TPU_OK" in output:
+ devices = [l for l in output.splitlines() if l.startswith("/dev/accel")]
+ print(f"[tpu] TPU healthy: {len(devices)} device(s) found")
+ return True
+ else:
+ print(
+ f"[tpu] WARNING: no /dev/accel* devices found. "
+ f"The software version may not match the accelerator type.",
+ file=sys.stderr,
+ )
+ return False
+
+
+def _collect_git_config():
+ """Prompt for git user.name and user.email, with defaults from local git config."""
+ local_name = ""
+ local_email = ""
+ try:
+ local_name = subprocess.run(
+ ["git", "config", "user.name"], capture_output=True, text=True
+ ).stdout.strip()
+ local_email = subprocess.run(
+ ["git", "config", "user.email"], capture_output=True, text=True
+ ).stdout.strip()
+ except FileNotFoundError:
+ pass
+
+ prompt_name = f" git user.name [{local_name}]: " if local_name else " git user.name: "
+ prompt_email = f" git user.email [{local_email}]: " if local_email else " git user.email: "
+
+ print("[tpu] Configure git for the instance:")
+ name = input(prompt_name).strip() or local_name
+ email = input(prompt_email).strip() or local_email
+
+ if not name or not email:
+ print("[tpu] Skipping git config (name or email empty).")
+ return None, None
+ return name, email
+
+
+def _setup_git_on_instance(name, zone, git_name, git_email):
+ """SSH into the VM and configure git user.name/email."""
+ if not git_name or not git_email:
+ return
+ cmd = (
+ f'git config --global user.name "{git_name}" && '
+ f'git config --global user.email "{git_email}"'
+ )
+ print(f"[tpu] Configuring git as '{git_name} <{git_email}>'...")
+ _run(
+ ["gcloud", "compute", "tpus", "tpu-vm", "ssh", name,
+ "--zone", zone, "--project", PROJECT,
+ "--command", cmd],
+ check=False, capture=True, quiet=True,
+ )
+
+
def tpu_create(args):
+ version = _resolve_version(args.accel_type, args.version)
+ print(f"[tpu] Accelerator: {args.accel_type}, software version: {version}")
+
+ git_name, git_email = _collect_git_config()
+
for zone in ZONES:
print(f"[tpu] Trying {zone}...")
result = _run(
["gcloud", "compute", "tpus", "tpu-vm", "create", args.name,
"--zone", zone,
"--accelerator-type", args.accel_type,
- "--version", args.version,
+ "--version", version,
"--project", PROJECT],
check=False, capture=True,
)
if result.returncode == 0:
print(f"[tpu] SUCCESS: created '{args.name}' in {zone}")
args.zone = zone
+ _check_tpu_health(args.name, zone)
+ _setup_git_on_instance(args.name, zone, git_name, git_email)
tpu_claude(args)
tpu_connect(args)
return
diff --git a/src/train.py b/src/train.py
index 3e3e471..e479255 100644
--- a/src/train.py
+++ b/src/train.py
@@ -9,6 +9,7 @@ import jax.numpy as jnp
import numpy as np
import optax
from tqdm import tqdm
+from flax import jax_utils
from flax.training import train_state
from .data import get_batches, get_tokenizer, load_tinystories, prepare_encoder_decoder_pairs
@@ -127,8 +128,7 @@ def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_
)
-def train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):
+def _train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):
pad_id = 50256
def loss_fn(params):
@@ -152,12 +152,26 @@ def train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):
return jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)
loss, grads = jax.value_and_grad(loss_fn)(state.params)
+ # Average gradients and loss across devices
+ grads = jax.lax.pmean(grads, axis_name="batch")
+ loss = jax.lax.pmean(loss, axis_name="batch")
grad_norm = optax.global_norm(grads)
state = state.apply_gradients(grads=grads)
return state, loss, grad_norm
+p_train_step = jax.pmap(_train_step, axis_name="batch")
+
+
+def shard_batch(batch, num_devices):
+ """Reshape a batch array so leading dim is (num_devices, per_device_batch, ...)."""
+ return batch.reshape(num_devices, -1, *batch.shape[1:])
+
+
def train(args):
+ num_devices = jax.local_device_count()
+ print(f"Detected {num_devices} device(s) for data-parallel training")
+
use_wandb = getattr(args, "wandb", False)
if use_wandb:
import wandb
@@ -177,6 +191,9 @@ def train(args):
)
print(f"Prepared {len(enc_inputs)} training pairs")
+ # Effective batch size must be divisible by num_devices
+ effective_batch_size = args.batch_size * num_devices
+
config = TransformerConfig(
d_model=args.d_model,
num_heads=args.num_heads,
@@ -191,15 +208,19 @@ def train(args):
rng = jax.random.PRNGKey(args.seed)
rng, init_rng = jax.random.split(rng)
- num_batches = len(enc_inputs) // args.batch_size
+ num_batches = len(enc_inputs) // effective_batch_size
total_steps = num_batches * args.epochs
warmup_steps = max(1, int(total_steps * args.warmup_ratio))
muon_lr = getattr(args, "muon_lr", 0.02)
state = create_train_state(init_rng, config, args.lr, muon_lr, total_steps, warmup_steps)
- param_count = sum(x.size for x in jax.tree.leaves(state.params))
+ # Replicate state across all devices
+ state = jax_utils.replicate(state)
+
+ param_count = sum(x.size for x in jax.tree.leaves(jax_utils.unreplicate(state).params))
print(f"Model parameters: {param_count:,}")
+ print(f"Devices: {num_devices}, per-device batch: {args.batch_size}, effective batch: {effective_batch_size}")
print(f"LR schedule: warmup {warmup_steps} steps, cosine decay over {total_steps} steps")
os.makedirs(args.checkpoint_dir, exist_ok=True)
@@ -214,25 +235,30 @@ def train(args):
init_value=0.0, peak_value=muon_lr,
warmup_steps=warmup_steps, decay_steps=total_steps, end_value=muon_lr * 0.1,
)
- tokens_per_batch = args.batch_size * (args.max_enc_len + args.max_dec_len)
+ tokens_per_batch = effective_batch_size * (args.max_enc_len + args.max_dec_len)
for epoch in range(args.epochs):
losses = []
- batches = get_batches(enc_inputs, dec_inputs, dec_targets, args.batch_size)
+ batches = get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size)
pbar = tqdm(batches, total=num_batches, desc=f"Epoch {epoch + 1}/{args.epochs}")
for src, tgt_in, tgt_out in pbar:
rng, dropout_rng = jax.random.split(rng)
+ # Give each device a different dropout rng
+ dropout_rngs = jax.random.split(dropout_rng, num_devices)
+
t0 = time.perf_counter()
- state, loss, grad_norm = train_step(
+ # Shard batch across devices: (num_devices, per_device_batch, seq_len)
+ state, loss, grad_norm = p_train_step(
state,
- jnp.array(src),
- jnp.array(tgt_in),
- jnp.array(tgt_out),
+ shard_batch(jnp.array(src), num_devices),
+ shard_batch(jnp.array(tgt_in), num_devices),
+ shard_batch(jnp.array(tgt_out), num_devices),
causal_mask,
- dropout_rng,
+ dropout_rngs,
)
- loss_val = float(loss)
+ # loss/grad_norm are replicated across devices; take first
+ loss_val = float(loss[0])
losses.append(loss_val)
global_step += 1
dt = time.perf_counter() - t0
@@ -241,7 +267,7 @@ def train(args):
if use_wandb:
wandb.log({
"train/loss": loss_val,
- "train/grad_norm": float(grad_norm),
+ "train/grad_norm": float(grad_norm[0]),
"train/adam_lr": float(adam_schedule(global_step)),
"train/muon_lr": float(muon_schedule(global_step)),
"train/tokens_per_sec": tokens_per_batch / dt,
@@ -259,9 +285,10 @@ def train(args):
"epoch": epoch + 1,
})
+ # Unreplicate state for checkpointing (save single copy)
ckpt_name = f"needle_{args.num_layers}_{args.d_model}_{global_step}.pkl"
ckpt_path = os.path.join(args.checkpoint_dir, ckpt_name)
- params_np = jax.tree.map(np.array, state.params)
+ params_np = jax.tree.map(np.array, jax_utils.unreplicate(state).params)
with open(ckpt_path, "wb") as f:
pickle.dump({"params": params_np, "config": config.__dict__}, f)
print(f"Saved checkpoint: {ckpt_path}")
diff --git a/test.py b/test.py
new file mode 100644
index 0000000..c8640d9
--- /dev/null
+++ b/test.py
@@ -0,0 +1,11 @@
+import jax
+import jax.numpy as jnp
+
+print(f"JAX version: {jax.__version__}")
+print(f"Number of devices: {jax.device_count()}")
+print(f"Available devices: {jax.devices()}")
+
+# Run a simple calculation
+x = jnp.ones((2, 2))
+y = jnp.dot(x, x)
+print(f"Calculation result: {y}")