Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Updates
HenryNdubuaku241841503242
diff --git a/pyproject.toml b/pyproject.tomlindex 4a8b394..73a02c0 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -3,12 +3,12 @@ name = "needle"version = "0.1.0"requires-python = ">=3.11"dependencies = [- "jax==0.9.0.1",- "jaxlib==0.9.0.1",- "flax==0.12.4",- "optax==0.2.6",- "datasets==4.5.0",- "transformers==5.2.0",+ "jax",+ "jaxlib",+ "flax",+ "optax",+ "datasets",+ "transformers","wandb",]diff --git a/requirements.txt b/requirements.txtindex 16c0614..0b2e3f0 100644--- a/requirements.txt+++ b/requirements.txt@@ -1,8 +1,8 @@-jax==0.9.0.1-jaxlib==0.9.0.1-flax==0.12.4-optax==0.2.6-datasets==4.5.0-transformers==5.2.0-wandb>=0.19.0-pyyaml>=6.0+jax+jaxlib+flax+optax+datasets+transformers+wandb+pyyamldiff --git a/setup b/setupindex 013fc20..527aaa7 100755--- a/setup+++ b/setup@@ -53,6 +53,34 @@ echo "Installing dependencies..."pip install --upgrade pip -qpip install -e . -q+# If running on a TPU VM, load drivers and install JAX with TPU support+if lspci 2>/dev/null | grep -qi "google.*processing"; then+ echo "TPU hardware detected..."++ # Load kernel modules if not already loaded+ if ! ls /dev/accel* &>/dev/null 2>&1; then+ echo "Loading TPU kernel modules..."+ for mod in gasket tpu_common tpu_v2 tpu_v3 tpu_v4 tpu_v4_lite; do+ sudo modprobe "$mod" 2>/dev/null || true+ done+ fi++ if ls /dev/accel* &>/dev/null 2>&1; then+ echo "TPU devices: $(ls /dev/accel* 2>/dev/null | wc -w) chip(s)"+ else+ echo "WARNING: TPU PCI devices found but no /dev/accel* — kernel drivers may not match hardware."+ echo " Recreate the VM with the correct --version for your accelerator type."+ fi++ echo "Installing jax[tpu]..."+ pip install "jax[tpu]" -q -f https://storage.googleapis.com/jax-releases/libtpu_releases.html++ if [ -f /sys/kernel/mm/transparent_hugepage/enabled ]; then+ echo "Enabling transparent hugepages..."+ sudo sh -c "echo always > /sys/kernel/mm/transparent_hugepage/enabled" 2>/dev/null || true+ fi+fi+if [ -z "$WANDB_API_KEY" ]; thenecho ""echo " Enter your W&B API key (https://wandb.ai/authorize)"diff --git a/src/cli.py b/src/cli.pyindex fbd0398..49a5a12 100644--- a/src/cli.py+++ b/src/cli.py@@ -54,8 +54,8 @@ HELP = """│ ││ tpu ││ create NAME Create TPU (auto-finds zone) │- │ --type STR Accelerator type (default:v5litepod-4)│- │ --version STR TPU OS (default: tpu-ubuntu2204-base) │+ │ --type STR Accelerator type (default: v6e-4) │+ │ --version STR TPU OS (auto-detected from --type) ││ connect NAME SSH config + connect (auto-zone) ││ claude NAME Install Claude Code on instance ││ stop NAME Stop instance (auto-zone) │@@ -140,8 +140,9 @@ def main():tp = tpu_sub.add_parser("create", add_help=False)tp.add_argument("name", type=str)- tp.add_argument("--type", dest="accel_type", type=str, default="v5litepod-4")- tp.add_argument("--version", type=str, default="tpu-ubuntu2204-base")+ tp.add_argument("--type", dest="accel_type", type=str, default="v6e-4")+ tp.add_argument("--version", type=str, default=None,+ help="Software version (auto-detected from --type if omitted)")tp = tpu_sub.add_parser("connect", add_help=False)tp.add_argument("name", type=str)diff --git a/src/evaluate.py b/src/evaluate.pyindex 816a25a..df544c0 100644--- a/src/evaluate.py+++ b/src/evaluate.py@@ -6,6 +6,7 @@ import jaximport jax.numpy as jnpimport numpy as npfrom datasets import load_dataset+from flax import jax_utilsfrom .data import get_tokenizerfrom .model import (@@ -24,32 +25,88 @@ def load_checkpoint(path):return params, config-def score_sequence(model, params, enc_tokens, dec_tokens, pad_id):+def _make_p_encode(model):+ """Create a pmap'd encode function."""+ def _encode(params, src, src_mask):+ return model.apply(+ {"params": params}, src, src_mask=src_mask, deterministic=True, method="encode",+ )+ return jax.pmap(_encode, axis_name="batch")+++def _make_p_decode(model):+ """Create a pmap'd decode function."""+ def _decode(params, dec_input, encoder_out, tgt_mask, cross_mask):+ return model.apply(+ {"params": params}, dec_input, encoder_out,+ self_mask=tgt_mask, cross_mask=cross_mask, deterministic=True, method="decode",+ )+ return jax.pmap(_decode, axis_name="batch")+++def _make_p_forward(model):+ """Create a pmap'd full forward function."""+ def _forward(params, src, tgt_in, src_mask, tgt_mask, cross_mask):+ return model.apply(+ {"params": params}, src, tgt_in,+ src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,+ deterministic=True,+ )+ return jax.pmap(_forward, axis_name="batch")+++def _shard_single(x, num_devices):+ """Replicate a single-sample batch across all devices for pmap."""+ return jnp.broadcast_to(x, (num_devices, *x.shape[1:]))+++def _shard_batch(x, num_devices):+ """Reshape batch for pmap: (N, ...) -> (num_devices, per_device, ...)."""+ return x.reshape(num_devices, -1, *x.shape[1:])+++def score_sequence(model, params, enc_tokens, dec_tokens, pad_id, p_encode=None, p_decode=None, num_devices=1):"""Compute average negative log-likelihood of dec_tokens given enc_tokens."""enc_input = jnp.array([enc_tokens])src_mask = make_padding_mask(enc_input, pad_id)- encoder_out = model.apply(- {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",- )+ if p_encode is not None and num_devices > 1:+ # Replicate single sample across devices, run pmap, take first result+ enc_s = _shard_single(enc_input, num_devices)+ src_mask_s = _shard_single(src_mask, num_devices)+ encoder_out = p_encode(params, enc_s, src_mask_s)[0:1]+ else:+ p = params if num_devices <= 1 else jax_utils.unreplicate(params)+ encoder_out = model.apply(+ {"params": p}, enc_input, src_mask=src_mask, deterministic=True, method="encode",+ )dec_in = [pad_id] + list(dec_tokens[:-1])dec_input = jnp.array([dec_in])tgt_mask = make_causal_mask(len(dec_in))cross_mask = src_mask- logits = model.apply(- {"params": params}, dec_input, encoder_out,- self_mask=tgt_mask, cross_mask=cross_mask, deterministic=True, method="decode",- )+ if p_decode is not None and num_devices > 1:+ dec_s = _shard_single(dec_input, num_devices)+ tgt_mask_s = jnp.broadcast_to(tgt_mask, (num_devices, *tgt_mask.shape[1:]))+ cross_mask_s = _shard_single(cross_mask, num_devices)+ enc_out_s = _shard_single(encoder_out, num_devices)+ logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, cross_mask_s)[0]+ else:+ p = params if num_devices <= 1 else jax_utils.unreplicate(params)+ logits = model.apply(+ {"params": p}, dec_input, encoder_out,+ self_mask=tgt_mask, cross_mask=cross_mask, deterministic=True, method="decode",+ )[0]- log_probs = jax.nn.log_softmax(logits[0])+ log_probs = jax.nn.log_softmax(logits if logits.ndim == 2 else logits[0])target_ids = jnp.array(dec_tokens)token_lls = log_probs[jnp.arange(len(dec_tokens)), target_ids]return float(jnp.mean(token_lls))-def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256):+def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256,+ num_devices=1, p_encode=None, p_decode=None):"""Perplexity on WikiText-2 test split."""ds = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="test")@@ -58,6 +115,8 @@ def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256):total_tokens = 0evaluated = 0+ single_params = jax_utils.unreplicate(params) if num_devices > 1 else params+for example in ds:text = example["text"].strip()if len(text) < 20:@@ -74,20 +133,33 @@ def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256):enc_input = jnp.array([enc_tokens])src_mask = make_padding_mask(enc_input, pad_id)- encoder_out = model.apply(- {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",- )++ if p_encode is not None and num_devices > 1:+ enc_s = _shard_single(enc_input, num_devices)+ src_mask_s = _shard_single(src_mask, num_devices)+ encoder_out = p_encode(params, enc_s, src_mask_s)[0:1]+ else:+ encoder_out = model.apply(+ {"params": single_params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",+ )dec_in = [pad_id] + list(dec_tokens[:-1])dec_input = jnp.array([dec_in])tgt_mask = make_causal_mask(len(dec_in))- logits = model.apply(- {"params": params}, dec_input, encoder_out,- self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",- )-- log_probs = jax.nn.log_softmax(logits[0])+ if p_decode is not None and num_devices > 1:+ dec_s = _shard_single(dec_input, num_devices)+ tgt_mask_s = jnp.broadcast_to(tgt_mask, (num_devices, *tgt_mask.shape[1:]))+ cross_mask_s = _shard_single(src_mask, num_devices)+ enc_out_s = _shard_single(encoder_out, num_devices)+ logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, cross_mask_s)[0]+ else:+ logits = model.apply(+ {"params": single_params}, dec_input, encoder_out,+ self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",+ )++ log_probs = jax.nn.log_softmax(logits[0] if logits.ndim == 3 else logits[0])target_ids = jnp.array(dec_tokens)token_lls = log_probs[jnp.arange(len(dec_tokens)), target_ids]@@ -102,7 +174,8 @@ def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256):return {"perplexity": ppl, "samples": evaluated, "tokens": total_tokens}-def eval_lambada(model, params, tokenizer, max_samples=500):+def eval_lambada(model, params, tokenizer, max_samples=500,+ num_devices=1, p_encode=None, p_decode=None):"""Accuracy of predicting the final word on LAMBADA."""ds = load_dataset("EleutherAI/lambada_openai", "default", split="test")@@ -110,6 +183,8 @@ def eval_lambada(model, params, tokenizer, max_samples=500):correct = 0total = 0+ single_params = jax_utils.unreplicate(params) if num_devices > 1 else params+for example in ds:text = example["text"].strip()words = text.rsplit(" ", 1)@@ -125,19 +200,32 @@ def eval_lambada(model, params, tokenizer, max_samples=500):enc_input = jnp.array([context_tokens])src_mask = make_padding_mask(enc_input, pad_id)- encoder_out = model.apply(- {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",- )++ if p_encode is not None and num_devices > 1:+ enc_s = _shard_single(enc_input, num_devices)+ src_mask_s = _shard_single(src_mask, num_devices)+ encoder_out = p_encode(params, enc_s, src_mask_s)[0:1]+ else:+ encoder_out = model.apply(+ {"params": single_params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",+ )dec_in = jnp.array([[pad_id]])tgt_mask = make_causal_mask(1)- logits = model.apply(- {"params": params}, dec_in, encoder_out,- self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",- )-- predicted = int(jnp.argmax(logits[0, 0]))+ if p_decode is not None and num_devices > 1:+ dec_s = _shard_single(dec_in, num_devices)+ tgt_mask_s = jnp.broadcast_to(tgt_mask, (num_devices, *tgt_mask.shape[1:]))+ cross_mask_s = _shard_single(src_mask, num_devices)+ enc_out_s = _shard_single(encoder_out, num_devices)+ logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, cross_mask_s)[0]+ else:+ logits = model.apply(+ {"params": single_params}, dec_in, encoder_out,+ self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",+ )++ predicted = int(jnp.argmax(logits[0, 0] if logits.ndim == 3 else logits[0, 0]))if predicted == target_tokens[0]:correct += 1total += 1@@ -149,7 +237,8 @@ def eval_lambada(model, params, tokenizer, max_samples=500):return {"accuracy": acc, "correct": correct, "total": total}-def eval_hellaswag(model, params, tokenizer, max_samples=500):+def eval_hellaswag(model, params, tokenizer, max_samples=500,+ num_devices=1, p_encode=None, p_decode=None):"""Accuracy on HellaSwag by scoring each candidate ending."""ds = load_dataset("Rowan/hellaswag", split="validation")@@ -176,7 +265,8 @@ def eval_hellaswag(model, params, tokenizer, max_samples=500):scores.append(float("-inf"))continuedec_tokens = dec_tokens[:64]- score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id)+ score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id,+ p_encode=p_encode, p_decode=p_decode, num_devices=num_devices)scores.append(score)predicted = int(np.argmax(scores))@@ -191,7 +281,8 @@ def eval_hellaswag(model, params, tokenizer, max_samples=500):return {"accuracy": acc, "correct": correct, "total": total}-def eval_arc_easy(model, params, tokenizer, max_samples=500):+def eval_arc_easy(model, params, tokenizer, max_samples=500,+ num_devices=1, p_encode=None, p_decode=None):"""Accuracy on ARC-Easy by scoring each answer choice."""ds = load_dataset("allenai/ai2_arc", "ARC-Easy", split="test")@@ -218,7 +309,8 @@ def eval_arc_easy(model, params, tokenizer, max_samples=500):scores.append(float("-inf"))continuedec_tokens = dec_tokens[:64]- score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id)+ score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id,+ p_encode=p_encode, p_decode=p_decode, num_devices=num_devices)scores.append(score)predicted_idx = int(np.argmax(scores))@@ -243,6 +335,9 @@ BENCHMARKS = {def main(args):+ num_devices = jax.local_device_count()+ print(f"Detected {num_devices} device(s) for data-parallel evaluation")+print(f"Loading checkpoint: {args.checkpoint}")params, config = load_checkpoint(args.checkpoint)model = EncoderDecoderTransformer(config)@@ -251,6 +346,16 @@ def main(args):param_count = sum(x.size for x in jax.tree.leaves(params))print(f"Model parameters: {param_count:,}")+ # Replicate params across devices for pmap+ if num_devices > 1:+ params = jax_utils.replicate(params)+ p_encode = _make_p_encode(model)+ p_decode = _make_p_decode(model)+ print(f"Params replicated across {num_devices} devices")+ else:+ p_encode = None+ p_decode = None+benchmarks = args.benchmarks or list(BENCHMARKS.keys())results = {}@@ -263,7 +368,10 @@ def main(args):print(f"Benchmark: {name}")print("=" * 50)- result = BENCHMARKS[name](model, params, tokenizer, max_samples=args.max_samples)+ result = BENCHMARKS[name](+ model, params, tokenizer, max_samples=args.max_samples,+ num_devices=num_devices, p_encode=p_encode, p_decode=p_decode,+ )results[name] = resultfor k, v in result.items():diff --git a/src/tpu.py b/src/tpu.pyindex e49042c..c4f9155 100644--- a/src/tpu.py+++ b/src/tpu.py@@ -6,6 +6,37 @@ import sysPROJECT = "needle-488623"+VERSION_FOR_TYPE = {+ "v2": "tpu-ubuntu2204-base",+ "v3": "tpu-ubuntu2204-base",+ "v4": "tpu-ubuntu2204-base",+ "v5litepod": "v2-alpha-tpuv5-lite",+ "v5e": "v2-alpha-tpuv5-lite",+ "v5p": "v2-alpha-tpuv5",+ "v6e": "v2-alpha-tpuv6e",+}+++def _resolve_version(accel_type, explicit_version):+ """Pick the right software version for the accelerator type.++ If the user explicitly passed --version, use that.+ Otherwise, match on the accelerator type prefix (e.g. 'v6e-4' → 'v6e').+ """+ if explicit_version is not None:+ return explicit_version+ for prefix, version in sorted(VERSION_FOR_TYPE.items(), key=lambda kv: -len(kv[0])):+ if accel_type.startswith(prefix):+ return version++ print(+ f"[tpu] WARNING: unknown accelerator type '{accel_type}', "+ f"defaulting to tpu-ubuntu2204-base. Pass --version to override.",+ file=sys.stderr,+ )+ return "tpu-ubuntu2204-base"++ZONES = ["us-central1-a", "us-central1-b", "us-central1-f","us-east1-b", "us-east1-c", "us-east1-d",@@ -80,20 +111,94 @@ def _update_ssh_config(path, host_name, new_block):f.write(new_block)+def _check_tpu_health(name, zone):+ """SSH into the VM and verify /dev/accel* devices exist."""+ print(f"[tpu] Verifying TPU devices on '{name}'...")+ result = _run(+ ["gcloud", "compute", "tpus", "tpu-vm", "ssh", name,+ "--zone", zone, "--project", PROJECT,+ "--command", "ls /dev/accel* 2>/dev/null && echo TPU_OK || echo TPU_MISSING"],+ check=False, capture=True, quiet=True,+ )+ output = result.stdout.strip()+ if "TPU_OK" in output:+ devices = [l for l in output.splitlines() if l.startswith("/dev/accel")]+ print(f"[tpu] TPU healthy: {len(devices)} device(s) found")+ return True+ else:+ print(+ f"[tpu] WARNING: no /dev/accel* devices found. "+ f"The software version may not match the accelerator type.",+ file=sys.stderr,+ )+ return False+++def _collect_git_config():+ """Prompt for git user.name and user.email, with defaults from local git config."""+ local_name = ""+ local_email = ""+ try:+ local_name = subprocess.run(+ ["git", "config", "user.name"], capture_output=True, text=True+ ).stdout.strip()+ local_email = subprocess.run(+ ["git", "config", "user.email"], capture_output=True, text=True+ ).stdout.strip()+ except FileNotFoundError:+ pass++ prompt_name = f" git user.name [{local_name}]: " if local_name else " git user.name: "+ prompt_email = f" git user.email [{local_email}]: " if local_email else " git user.email: "++ print("[tpu] Configure git for the instance:")+ name = input(prompt_name).strip() or local_name+ email = input(prompt_email).strip() or local_email++ if not name or not email:+ print("[tpu] Skipping git config (name or email empty).")+ return None, None+ return name, email+++def _setup_git_on_instance(name, zone, git_name, git_email):+ """SSH into the VM and configure git user.name/email."""+ if not git_name or not git_email:+ return+ cmd = (+ f'git config --global user.name "{git_name}" && '+ f'git config --global user.email "{git_email}"'+ )+ print(f"[tpu] Configuring git as '{git_name} <{git_email}>'...")+ _run(+ ["gcloud", "compute", "tpus", "tpu-vm", "ssh", name,+ "--zone", zone, "--project", PROJECT,+ "--command", cmd],+ check=False, capture=True, quiet=True,+ )++def tpu_create(args):+ version = _resolve_version(args.accel_type, args.version)+ print(f"[tpu] Accelerator: {args.accel_type}, software version: {version}")++ git_name, git_email = _collect_git_config()+for zone in ZONES:print(f"[tpu] Trying {zone}...")result = _run(["gcloud", "compute", "tpus", "tpu-vm", "create", args.name,"--zone", zone,"--accelerator-type", args.accel_type,- "--version", args.version,+ "--version", version,"--project", PROJECT],check=False, capture=True,)if result.returncode == 0:print(f"[tpu] SUCCESS: created '{args.name}' in {zone}")args.zone = zone+ _check_tpu_health(args.name, zone)+ _setup_git_on_instance(args.name, zone, git_name, git_email)tpu_claude(args)tpu_connect(args)returndiff --git a/src/train.py b/src/train.pyindex 3e3e471..e479255 100644--- a/src/train.py+++ b/src/train.py@@ -9,6 +9,7 @@ import jax.numpy as jnpimport numpy as npimport optaxfrom tqdm import tqdm+from flax import jax_utilsfrom flax.training import train_statefrom .data import get_batches, get_tokenizer, load_tinystories, prepare_encoder_decoder_pairs@@ -127,8 +128,7 @@ def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_)-def train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):+def _train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):pad_id = 50256def loss_fn(params):@@ -152,12 +152,26 @@ def train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):return jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)loss, grads = jax.value_and_grad(loss_fn)(state.params)+ # Average gradients and loss across devices+ grads = jax.lax.pmean(grads, axis_name="batch")+ loss = jax.lax.pmean(loss, axis_name="batch")grad_norm = optax.global_norm(grads)state = state.apply_gradients(grads=grads)return state, loss, grad_norm+p_train_step = jax.pmap(_train_step, axis_name="batch")+++def shard_batch(batch, num_devices):+ """Reshape a batch array so leading dim is (num_devices, per_device_batch, ...)."""+ return batch.reshape(num_devices, -1, *batch.shape[1:])++def train(args):+ num_devices = jax.local_device_count()+ print(f"Detected {num_devices} device(s) for data-parallel training")+use_wandb = getattr(args, "wandb", False)if use_wandb:import wandb@@ -177,6 +191,9 @@ def train(args):)print(f"Prepared {len(enc_inputs)} training pairs")+ # Effective batch size must be divisible by num_devices+ effective_batch_size = args.batch_size * num_devices+config = TransformerConfig(d_model=args.d_model,num_heads=args.num_heads,@@ -191,15 +208,19 @@ def train(args):rng = jax.random.PRNGKey(args.seed)rng, init_rng = jax.random.split(rng)- num_batches = len(enc_inputs) // args.batch_size+ num_batches = len(enc_inputs) // effective_batch_sizetotal_steps = num_batches * args.epochswarmup_steps = max(1, int(total_steps * args.warmup_ratio))muon_lr = getattr(args, "muon_lr", 0.02)state = create_train_state(init_rng, config, args.lr, muon_lr, total_steps, warmup_steps)- param_count = sum(x.size for x in jax.tree.leaves(state.params))+ # Replicate state across all devices+ state = jax_utils.replicate(state)++ param_count = sum(x.size for x in jax.tree.leaves(jax_utils.unreplicate(state).params))print(f"Model parameters: {param_count:,}")+ print(f"Devices: {num_devices}, per-device batch: {args.batch_size}, effective batch: {effective_batch_size}")print(f"LR schedule: warmup {warmup_steps} steps, cosine decay over {total_steps} steps")os.makedirs(args.checkpoint_dir, exist_ok=True)@@ -214,25 +235,30 @@ def train(args):init_value=0.0, peak_value=muon_lr,warmup_steps=warmup_steps, decay_steps=total_steps, end_value=muon_lr * 0.1,)- tokens_per_batch = args.batch_size * (args.max_enc_len + args.max_dec_len)+ tokens_per_batch = effective_batch_size * (args.max_enc_len + args.max_dec_len)for epoch in range(args.epochs):losses = []- batches = get_batches(enc_inputs, dec_inputs, dec_targets, args.batch_size)+ batches = get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size)pbar = tqdm(batches, total=num_batches, desc=f"Epoch {epoch + 1}/{args.epochs}")for src, tgt_in, tgt_out in pbar:rng, dropout_rng = jax.random.split(rng)+ # Give each device a different dropout rng+ dropout_rngs = jax.random.split(dropout_rng, num_devices)+t0 = time.perf_counter()- state, loss, grad_norm = train_step(+ # Shard batch across devices: (num_devices, per_device_batch, seq_len)+ state, loss, grad_norm = p_train_step(state,- jnp.array(src),- jnp.array(tgt_in),- jnp.array(tgt_out),+ shard_batch(jnp.array(src), num_devices),+ shard_batch(jnp.array(tgt_in), num_devices),+ shard_batch(jnp.array(tgt_out), num_devices),causal_mask,- dropout_rng,+ dropout_rngs,)- loss_val = float(loss)+ # loss/grad_norm are replicated across devices; take first+ loss_val = float(loss[0])losses.append(loss_val)global_step += 1dt = time.perf_counter() - t0@@ -241,7 +267,7 @@ def train(args):if use_wandb:wandb.log({"train/loss": loss_val,- "train/grad_norm": float(grad_norm),+ "train/grad_norm": float(grad_norm[0]),"train/adam_lr": float(adam_schedule(global_step)),"train/muon_lr": float(muon_schedule(global_step)),"train/tokens_per_sec": tokens_per_batch / dt,@@ -259,9 +285,10 @@ def train(args):"epoch": epoch + 1,})+ # Unreplicate state for checkpointing (save single copy)ckpt_name = f"needle_{args.num_layers}_{args.d_model}_{global_step}.pkl"ckpt_path = os.path.join(args.checkpoint_dir, ckpt_name)- params_np = jax.tree.map(np.array, state.params)+ params_np = jax.tree.map(np.array, jax_utils.unreplicate(state).params)with open(ckpt_path, "wb") as f:pickle.dump({"params": params_np, "config": config.__dict__}, f)print(f"Saved checkpoint: {ckpt_path}")diff --git a/test.py b/test.pynew file mode 100644index 0000000..c8640d9--- /dev/null+++ b/test.py@@ -0,0 +1,11 @@+import jax+import jax.numpy as jnp++print(f"JAX version: {jax.__version__}")+print(f"Number of devices: {jax.device_count()}")+print(f"Available devices: {jax.devices()}")++# Run a simple calculation+x = jnp.ones((2, 2))+y = jnp.dot(x, x)+print(f"Calculation result: {y}")