Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
streamline codebase
HenryNdubuaku42ca7ececef1
diff --git a/README.md b/README.mdindex c222182..e638d91 100644--- a/README.md+++ b/README.md@@ -65,7 +65,7 @@┌─────────────────────────────────────────────────────────────┐│ ││ Contrastive Head CLIP-style tool retrieval │- │ encoder → mean_pool → Dense(d_model/4) → ReLU │+ │ encoder → mean_pool → Dense(d_model/4) → ReLU ││ → Dense(contrastive_dim) → L2-normalize ││ learnable temperature (log_temp) ││ │@@ -101,11 +101,6 @@│ ││ Gemini synthesis → Cactus-Compute/tool-calls (HuggingFace) ││ parallel workers · dedup · auto-merge · auto-upload │- │ │- │ Related: │- │ needle merge-xlam merge xlam-60k into dataset │- │ needle rebalance-tools trim over-represented bins │- │ needle split-dataset create train/val splits │└─────────────────────────────────────────────────────────────┘│▼@@ -152,17 +147,6 @@│ └──────────┬───────────┘ └──────────┬───────────┘ ││ ▼ ▼ ││ text + contrastive tool-call training (jax.pmap) │- └─────────────────────────────────────────────────────────────┘- │- ▼- ┌─────────────────────────────────────────────────────────────┐- │ needle calibrate │- │ │- │ Freeze encoder + decoder │- │ Compute per-example decoder perplexity │- │ Map PPL → [0,1] confidence via sigmoid calibration │- │ Train confidence head (2 Dense layers, MSE loss) │- │ Save calibrated checkpoint with mu, k params │└─────────────────────────────────────────────────────────────┘Hybrid Inference (confidence-based cloud routing)@@ -190,9 +174,9 @@│ ┌──┴──┐ │ ││ │ yes │──────────────┤ ││ └─────┘ ▼ │- │ ┌──────────┐ │+ │ ┌──────────┐ ││ │ Decoder │──▶ tool calls (local) │- │ └──────────┘ │+ │ └──────────┘ ││ ┌─────┐ ││ │ no │──▶ route to cloud (Gemini) ──▶ tool calls ││ └─────┘ (skip decoder entirely) │@@ -214,7 +198,7 @@ needle [command]│ train ││ --epochs INT Training epochs (default: 1) ││ --batch-size INT Batch size (default: 32) │- │ --lr FLOAT AdamW learning rate (default: 3e-4) │+ │ --lr FLOAT AdamW learning rate (default: 3e-5) ││ --muon-lr FLOAT Muon learning rate (default: 0.02) ││ --d-model INT Model dim (default: 512) ││ --num-heads INT Attention heads (default: 8) │@@ -224,7 +208,7 @@ needle [command]│ --max-enc-len INT Max encoder seq len (default: 1024) ││ --max-dec-len INT Max decoder seq len (default: 512) ││ --max-samples INT Training samples (default: all) │- │ --no-feedforward No FFN layers (default: on) │+ │ --no-feedforward No FFN layers (default: on) ││ --feedforward Enable FFN layers (off by default) ││ --activation STR swiglu|drelu|geglu (if FFN enabled) ││ --mat-factors INT [...] FFN shrink factors (if FFN enabled) │@@ -235,7 +219,7 @@ needle [command]│ --prune-end-frac FL Lock mask at this frac (def: 0.67) ││ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) ││ --decay-ratio FLOAT LR cosine decay ratio (default: 0.05)│- │ --dropout FLOAT Dropout rate (default: 0.0) │+ │ --dropout FLOAT Dropout rate (default: 0.0) ││ --eval-every INT Val eval interval (default: 1000) ││ --max-eval-samples INT Val samples limit (default: 5000) ││ --contrastive-weight FL CLIP loss weight (default: 0.1) │@@ -277,15 +261,6 @@ needle [command]│ --tool-call-samples INT Tool-call eval samples (default: 200)││ --no-constrained Disable constrained decoding ││ │- │ calibrate │- │ --checkpoint PATH Path to model checkpoint (required) │- │ --output PATH Output path (default: overwrite) │- │ --batch-size INT Batch size (default: 32) │- │ --num-samples INT Limit samples for PPL (default: all) │- │ --epochs INT Training epochs (default: 10) │- │ --lr FLOAT Learning rate (default: 1e-3) │- │ --k FLOAT Sigmoid steepness (default: 3.0) │- │ ││ generate-data ││ --num-samples INT Samples to generate (default: 5000) ││ --batch-size INT Examples per Gemini call (default:10)│@@ -295,15 +270,6 @@ needle [command]│ --output-jsonl PATH Also save raw generations to JSONL ││ --upload-every INT Merge+upload interval (def: 30000) ││ │- │ merge-xlam │- │ --dry-run Skip upload │- │ --max-samples INT Limit xlam samples │- │ │- │ rebalance-tools │- │ --dry-run Preview without modifying │- │ │- │ split-dataset Create train/val splits + upload │- │ ││ evaluate ││ --checkpoint PATH Path to model checkpoint (required) ││ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │@@ -390,21 +356,13 @@ needle [command]4. Clone the repo on your instance using your PATgit clone https://github.com/cactus-compute/needle.git- cd needle--5. Install needle (follow instruction to setup wandb)- source ./setup+ cd needle && source ./setup-6. Login to Hugging Face (required for private datasets)- huggingface-cli login+5. Login to Hugging Face (required for private datasets)+ hf auth login(paste your HF token — get one at https://huggingface.co/settings/tokens)-7. Full pipeline- needle tokenize # tokenize + pack + upload- needle train --wandb # train on TPU- needle eval --checkpoint <path> # evaluate--8. Use tmux for long training runs (survives SSH disconnects)+6. Use tmux for long training runs (survives SSH disconnects)tmux new -s train # start a named sessionneedle train --wandb # run training inside itCtrl+B, then D # detach (keeps running)@@ -448,12 +406,4 @@ all workers automatically.7. Stop/delete when doneneedle tpu stop my-experimentneedle tpu delete my-experiment-```--## Current TPU Instances--```- Name Zone Type Software Workers- ──────────────────────────────────────────────────────────────────- large asia-northeast1-b v6e-16 v2-alpha-tpuv6e 4-```+```\ No newline at end of filediff --git a/src/calibrate.py b/src/calibrate.pydeleted file mode 100644index f457969..0000000--- a/src/calibrate.py+++ /dev/null@@ -1,399 +0,0 @@-"""Post-training calibration of the confidence head.--Freezes the encoder and decoder, computes per-example decoder perplexity-on the training data, maps perplexity to [0, 1] confidence labels via a-sigmoid calibration curve, then trains only the confidence head (2 Dense layers)-to predict these labels from encoder output.--Two-pass design for memory efficiency:- Pass 1: Forward through encoder+decoder to compute per-example PPL (stores only scalars)- Pass 2: Re-encode through encoder only, train confidence head on the fly--All compute is distributed across all available devices via pmap.--Usage:- needle calibrate --checkpoint checkpoints/needle_12_512_50000.pkl- needle calibrate --checkpoint checkpoints/needle_12_512_50000.pkl --num-samples 10000-"""--import math-import os-import pickle--import jax-import jax.numpy as jnp-import numpy as np-import optax-from tqdm import tqdm--from .data import (- get_tokenizer, load_prepared_data, count_batches, get_batches,-)-from .model import (- EncoderDecoderTransformer, TransformerConfig,- make_packing_mask, make_causal_packing_mask, make_cross_packing_mask,-)---def _replicate(tree):- """Replicate params/state across all devices."""- return jax.tree.map(lambda x: jnp.broadcast_to(x, (jax.local_device_count(),) + x.shape), tree)---def _unreplicate(tree):- """Get single copy from replicated params/state."""- return jax.tree.map(lambda x: x[0], tree)---def compute_per_example_perplexity(model, params, train_data, batch_size, max_samples=None):- """Compute per-example perplexity using packed training data.-- Memory-efficient: only stores scalar PPL per example, not encoder outputs.- Distributed across all devices via pmap.-- Returns:- perplexities: numpy array of per-example perplexity values- """- enc = train_data["packed_enc"]- dec_in = train_data["packed_dec_in"]- dec_tgt = train_data["packed_dec_tgt"]- enc_seg = train_data["packed_enc_seg"]- dec_seg = train_data["packed_dec_seg"]-- n = len(enc)- if max_samples:- n = min(n, max_samples)-- num_devices = jax.local_device_count()- per_device_batch = batch_size // num_devices- assert batch_size % num_devices == 0, f"batch_size {batch_size} must be divisible by {num_devices} devices"-- all_ppl = []-- @jax.pmap- def forward_step(params, src, tgt_in, tgt_out, enc_seg_ids, dec_seg_ids):- src_mask = make_packing_mask(enc_seg_ids)- tgt_mask = make_causal_packing_mask(dec_seg_ids)- cross_mask = make_cross_packing_mask(enc_seg_ids, dec_seg_ids)-- logits = model.apply(- {"params": params}, src, tgt_in,- src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,- )-- ce = optax.softmax_cross_entropy_with_integer_labels(- logits.astype(jnp.float32), tgt_out- )- padding_mask = (dec_seg_ids > 0).astype(jnp.float32)- per_example_loss = jnp.sum(ce * padding_mask, axis=-1) / jnp.maximum(- jnp.sum(padding_mask, axis=-1), 1.0- )- per_example_ppl = jnp.exp(jnp.minimum(per_example_loss, 20.0))- return per_example_ppl-- r_params = _replicate(params)-- print(f"Computing per-example perplexity ({n} examples, batch_size={batch_size}, {num_devices} devices)...")- for i in tqdm(range(0, n, batch_size), desc="PPL forward"):- end = min(i + batch_size, n)- actual = end - i-- # Pad to full batch if needed- def _load(arr, i, end, actual):- batch = np.array(arr[i:end], dtype=np.int32)- if actual < batch_size:- pad = np.zeros((batch_size - actual, batch.shape[1]), dtype=np.int32)- batch = np.concatenate([batch, pad], axis=0)- return batch.reshape(num_devices, per_device_batch, -1)-- src = _load(enc, i, end, actual)- tgt_in = _load(dec_in, i, end, actual)- tgt_out = _load(dec_tgt, i, end, actual)- es = _load(enc_seg, i, end, actual)- ds = _load(dec_seg, i, end, actual)-- ppl = forward_step(r_params, src, tgt_in, tgt_out, es, ds)- # ppl shape: (num_devices, per_device_batch) → flatten and trim- ppl_flat = np.array(ppl).reshape(-1)[:actual]- all_ppl.append(ppl_flat)-- perplexities = np.concatenate(all_ppl)- print(f" Perplexity stats: min={perplexities.min():.2f}, median={np.median(perplexities):.2f}, "- f"mean={perplexities.mean():.2f}, max={perplexities.max():.2f}")-- return perplexities---def perplexity_to_confidence(perplexities, mu=None, k=5.0):- """Map perplexity values to [0, 1] confidence via sigmoid calibration.-- confidence = sigmoid(-k * (log(ppl) - log(mu)))- """- if mu is None:- mu = float(np.percentile(perplexities, 80))-- log_ppl = np.log(np.maximum(perplexities, 1.0))- log_mu = np.log(mu)-- confidence = 1.0 / (1.0 + np.exp(k * (log_ppl - log_mu)))-- return confidence.astype(np.float32), mu---def train_confidence_head(model, params, train_data, confidence_labels,- epochs=1, lr=1e-3, batch_size=128):- """Train only the confidence head parameters, re-encoding on the fly.-- Distributed across all devices via pmap. Memory-efficient: re-runs encoder- forward pass during training instead of caching all encoder outputs.- """- enc = train_data["packed_enc"]- enc_seg = train_data["packed_enc_seg"]- n = len(confidence_labels)-- num_devices = jax.local_device_count()- per_device_batch = batch_size // num_devices- assert batch_size % num_devices == 0, f"batch_size {batch_size} must be divisible by {num_devices} devices"-- conf_param_keys = {"confidence_hidden", "confidence_out"}-- def is_conf_param(path):- return any(k in str(path) for k in conf_param_keys)-- tx = optax.adam(lr)- opt_state = tx.init(params)-- train_step = jax.pmap(- lambda params, opt_state, src, enc_seg_ids, targets: _train_step_inner(- model, tx, is_conf_param, params, opt_state, src, enc_seg_ids, targets- ),- axis_name="devices",- )-- r_params = _replicate(params)- r_opt_state = _replicate(opt_state)-- print(f"\nTraining confidence head ({n} examples, {epochs} epoch(s), lr={lr}, "- f"batch_size={batch_size}, {num_devices} devices)...")-- for epoch in range(epochs):- perm = np.random.permutation(n)- epoch_losses = []-- for i in tqdm(range(0, n - batch_size + 1, batch_size),- desc=f"Calibrating epoch {epoch + 1}/{epochs}"):- idx = perm[i:i + batch_size]- src = np.array(enc[idx], dtype=np.int32).reshape(num_devices, per_device_batch, -1)- es = np.array(enc_seg[idx], dtype=np.int32).reshape(num_devices, per_device_batch, -1)- tgt = np.array(confidence_labels[idx]).reshape(num_devices, per_device_batch)-- r_params, r_opt_state, loss = train_step(r_params, r_opt_state, src, es, tgt)- epoch_losses.append(float(loss[0])) # loss is replicated, take first-- avg_loss = sum(epoch_losses) / len(epoch_losses) if epoch_losses else float("nan")- print(f" Epoch {epoch + 1}/{epochs}: MSE={avg_loss:.6f}")-- params = _unreplicate(r_params)- return params---def _train_step_inner(model, tx, is_conf_param, params, opt_state, src, enc_seg_ids, targets):- """Inner training step — separated for pmap with axis_name."""- def loss_fn(params):- src_mask = make_packing_mask(enc_seg_ids)- encoder_out, enc_mask = model.apply(- {"params": params}, src, src_mask=src_mask, method="encode_text",- )- pred = model.apply(- {"params": params}, encoder_out, enc_mask,- method="predict_confidence",- )- return jnp.mean((pred - targets) ** 2)-- loss, grads = jax.value_and_grad(loss_fn)(params)-- # Average gradients across devices- grads = jax.lax.pmean(grads, axis_name="devices")-- # Zero out gradients for non-confidence params- grads = jax.tree_util.tree_map_with_path(- lambda path, g: g if is_conf_param(path) else jnp.zeros_like(g),- grads,- )-- updates, opt_state_new = tx.update(grads, opt_state, params)- params_new = optax.apply_updates(params, updates)- return params_new, opt_state_new, loss---def evaluate_confidence(model, params, train_data, confidence_labels, perplexities, max_eval=10000):- """Evaluate calibrated confidence head and print diagnostic stats."""- enc = train_data["packed_enc"]- enc_seg = train_data["packed_enc_seg"]- n = min(len(confidence_labels), max_eval)-- num_devices = jax.local_device_count()- batch_size = 128- # Round down to multiple of num_devices- batch_size = (batch_size // num_devices) * num_devices- per_device_batch = batch_size // num_devices-- @jax.pmap- def predict_step(params, src, enc_seg_ids):- src_mask = make_packing_mask(enc_seg_ids)- encoder_out, enc_mask = model.apply(- {"params": params}, src, src_mask=src_mask, method="encode_text",- )- return model.apply(- {"params": params}, encoder_out, enc_mask,- method="predict_confidence",- )-- r_params = _replicate(params)-- predictions = []- for i in tqdm(range(0, n, batch_size), desc="Evaluating confidence"):- end = min(i + batch_size, n)- actual = end - i-- src = np.array(enc[i:end], dtype=np.int32)- es = np.array(enc_seg[i:end], dtype=np.int32)- if actual < batch_size:- pad_src = np.zeros((batch_size - actual, src.shape[1]), dtype=np.int32)- src = np.concatenate([src, pad_src], axis=0)- pad_es = np.zeros((batch_size - actual, es.shape[1]), dtype=np.int32)- es = np.concatenate([es, pad_es], axis=0)-- src = src.reshape(num_devices, per_device_batch, -1)- es = es.reshape(num_devices, per_device_batch, -1)-- pred = predict_step(r_params, src, es)- pred_flat = np.array(pred).reshape(-1)[:actual]- predictions.append(pred_flat)-- predictions = np.concatenate(predictions)-- labels = confidence_labels[:n]- ppl = perplexities[:n]-- # Stats- mse = np.mean((predictions - labels) ** 2)- mae = np.mean(np.abs(predictions - labels))- correlation = np.corrcoef(predictions, labels)[0, 1]-- print(f"\nConfidence head evaluation:")- print(f" MSE: {mse:.6f}")- print(f" MAE: {mae:.4f}")- print(f" Correlation: {correlation:.4f}")-- print(f"\n {'PPL Range':<15s} {'Count':>6s} {'Avg Conf':>10s} {'Avg Pred':>10s}")- print(f" {'─' * 45}")- buckets = [(0, 2), (2, 4), (4, 8), (8, 16), (16, 50), (50, 1000)]- for lo, hi in buckets:- mask = (ppl >= lo) & (ppl < hi)- if mask.sum() == 0:- continue- avg_conf = labels[mask].mean()- avg_pred = predictions[mask].mean()- print(f" {f'{lo}-{hi}':<15s} {mask.sum():>6d} {avg_conf:>10.3f} {avg_pred:>10.3f}")-- return predictions---def main(args):- num_devices = jax.local_device_count()- print(f"Devices: {num_devices}")-- # Ensure batch_size is divisible by num_devices- if args.batch_size % num_devices != 0:- args.batch_size = (args.batch_size // num_devices) * num_devices- print(f" Adjusted batch_size to {args.batch_size} (divisible by {num_devices})")-- # Load checkpoint- print(f"Loading checkpoint: {args.checkpoint}")- with open(args.checkpoint, "rb") as f:- data = pickle.load(f)- params = jax.tree.map(lambda x: jnp.array(x, dtype=jnp.bfloat16), data["params"])- config_dict = {k: v for k, v in data["config"].items()- if k in TransformerConfig.__dataclass_fields__}- config = TransformerConfig(**config_dict)- model = EncoderDecoderTransformer(config)-- param_count = sum(x.size for x in jax.tree.leaves(params))- print(f" Parameters: {param_count:,}")- print(f" Config: d={config.d_model}, heads={config.num_heads}, "- f"layers={config.num_encoder_layers}/{config.num_decoder_layers}")-- # Check if confidence head params need (re)initialization- rng = jax.random.PRNGKey(0)- dummy_src = jnp.ones((1, 128), dtype=jnp.int32)- dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)- init_vars = model.init({"params": rng}, dummy_src, dummy_tgt, method="init_all")- init_params = init_vars["params"]-- flat_keys = str(jax.tree.map(lambda x: x.shape, params))- needs_reinit = "confidence_hidden" not in flat_keys- if not needs_reinit:- # Check shape match- for key in ("confidence_hidden", "confidence_out"):- old_shapes = jax.tree.map(lambda x: x.shape, params[key])- new_shapes = jax.tree.map(lambda x: x.shape, init_params[key])- if str(old_shapes) != str(new_shapes):- needs_reinit = True- break-- if needs_reinit:- print(" Confidence head shape mismatch or missing — reinitializing...")- params = {**params, "confidence_hidden": init_params["confidence_hidden"],- "confidence_out": init_params["confidence_out"]}- print(" Confidence head initialized.")-- # Load training data- print(f"\nLoading prepared data...")- train_data = load_prepared_data("train", mmap=True)- print(f" {len(train_data['packed_enc']):,} packed bins")-- # Pass 1: Compute per-example perplexity (forward-only, no gradients → large batch)- ppl_batch_size = num_devices * 128 # 128 per device, forward-only no gradients- perplexities = compute_per_example_perplexity(- model, params, train_data, ppl_batch_size, max_samples=args.num_samples,- )-- # Map perplexity → confidence labels- confidence_labels, mu = perplexity_to_confidence(perplexities, k=args.k)- print(f"\nCalibration: mu (median PPL) = {mu:.2f}, k = {args.k}")- print(f" Confidence stats: min={confidence_labels.min():.3f}, "- f"median={np.median(confidence_labels):.3f}, "- f"mean={confidence_labels.mean():.3f}, max={confidence_labels.max():.3f}")-- # Pass 2: Train confidence head (has gradients → use args.batch_size)- params = train_confidence_head(- model, params, train_data, confidence_labels,- epochs=args.epochs, lr=args.lr, batch_size=args.batch_size,- )-- # Ensure params are on host (clear stale pmap sharding from training)- params = jax.tree.map(lambda x: jnp.array(np.array(x), dtype=x.dtype), params)-- # Evaluate- evaluate_confidence(model, params, train_data, confidence_labels, perplexities)-- # Save updated checkpoint- output_path = args.output or args.checkpoint- params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), params)-- save_config = dict(data["config"])- save_config["confidence_mu"] = float(mu)- save_config["confidence_k"] = float(args.k)-- with open(output_path, "wb") as f:- pickle.dump({"params": params_np, "config": save_config}, f)- print(f"\nSaved calibrated checkpoint: {output_path}")- print(f" Calibration params: mu={mu:.2f}, k={args.k}")- print(f"\n Usage at inference:")- print(f" confidence, encoder_out, enc_mask = model.apply(")- print(f' {{"params": params}}, src, src_mask=src_mask, method="forward_confidence")')- print(f" if confidence > 0.6: # user-configurable threshold")- print(f" # decode locally")- print(f" else:")- print(f" # route to cloud")diff --git a/src/categorize_failures.py b/src/categorize_failures.pydeleted file mode 100644index 8cf1594..0000000--- a/src/categorize_failures.py+++ /dev/null@@ -1,322 +0,0 @@-"""Categorize tool-call argument-value failures to quantify what a-draft-correction second pass could actually fix.--Runs unconstrained generation on a stratified single+multi-call val sample,-then sorts each argument-value mismatch into one of these buckets:-- correctable ref value appears as a substring of the query (a second- pass can re-extract it)- canonicalization ref and pred match after lowercase + whitespace collapse- + stripping a small set of prefixes ("at ", "today at ",- "on "). Data-inconsistency noise, not model error.- cross_lingual ref and pred use different scripts (Latin vs Cyrillic,- etc.) — translation mismatch, usually annotation noise.- unrecoverable neither ref nor pred appear in the query — a draft- corrector can't fix these without external info.- missing_key pred omitted a key the ref requires- extra_key pred emitted a key the ref doesn't require--Usage:- python -m src.categorize_failures --checkpoint checkpoints/needle.pkl- python -m src.categorize_failures --checkpoint needle.pkl --num-samples 500-"""--import argparse-import json-import re-import unicodedata-from collections import Counter--import jax-import numpy as np--from .data import get_tokenizer, load_tool_calls-from .model import EncoderDecoderTransformer-from .run import generate_batch, load_checkpoint-from .tokenizer import to_snake_case---_CANONICAL_PREFIXES = ("today at ", "tomorrow at ", "tonight at ",- "this evening at ", "at ", "on ", "the ")---def _strip_canonical(s):- """Lowercase, collapse whitespace, strip one canonical prefix."""- if not isinstance(s, str):- return s- s = s.strip().lower()- s = re.sub(r"\s+", " ", s)- for p in _CANONICAL_PREFIXES:- if s.startswith(p):- s = s[len(p):].strip()- break- return s---def _script_of(s):- """Return dominant script of alphabetic chars, or 'other' / 'empty'."""- if not isinstance(s, str) or not s:- return "empty"- scripts = set()- for ch in s:- if not ch.isalpha():- continue- try:- name = unicodedata.name(ch, "")- except ValueError:- continue- if "LATIN" in name:- scripts.add("latin")- elif "CYRILLIC" in name:- scripts.add("cyrillic")- elif "GREEK" in name:- scripts.add("greek")- elif "ARABIC" in name:- scripts.add("arabic")- elif "HEBREW" in name:- scripts.add("hebrew")- elif "HANGUL" in name:- scripts.add("hangul")- elif "DEVANAGARI" in name:- scripts.add("devanagari")- elif "CJK" in name or "HIRAGANA" in name or "KATAKANA" in name:- scripts.add("cjk")- else:- scripts.add("other")- if not scripts:- return "empty"- if len(scripts) == 1:- return next(iter(scripts))- return "mixed"---def _value_in_query(value, query):- """True if value appears in query (case-insensitive for strings)."""- if isinstance(value, bool):- return False- if isinstance(value, (int, float)):- return str(value) in query- if isinstance(value, str):- v = value.strip().lower()- if len(v) < 2:- return False- return v in query.lower()- return False---def categorize_value_mismatch(query, ref_value, pred_value):- if isinstance(ref_value, str) and isinstance(pred_value, str):- if _strip_canonical(ref_value) == _strip_canonical(pred_value):- return "canonicalization"- rs, ps = _script_of(ref_value), _script_of(pred_value)- if rs != ps and rs not in ("empty", "other", "mixed") and ps not in ("empty", "other", "mixed"):- return "cross_lingual"- if _value_in_query(ref_value, query):- return "correctable"- return "unrecoverable"---def categorize_pool(model, params, tokenizer, examples, max_gen_len, max_enc_len,- batch_size=32):- """Generate predictions for a pool and categorize each value mismatch.-- Returns (bucket_counts, bucket_examples).- """- queries = [ex["query"] for ex in examples]- tools_list = [ex["tools"] for ex in examples]-- all_preds = []- for i in range(0, len(examples), batch_size):- chunk_q = queries[i:i + batch_size]- chunk_t = tools_list[i:i + batch_size]- preds = generate_batch(- model, params, tokenizer, chunk_q, chunk_t,- max_gen_len=max_gen_len, max_enc_len=max_enc_len,- normalize=True, constrained=False,- )- all_preds.extend(preds)-- bucket_counts = Counter()- bucket_examples = {b: [] for b in- ("correctable", "canonicalization", "cross_lingual",- "unrecoverable", "missing_key", "extra_key")}-- for ex, pred_text in zip(examples, all_preds):- query = ex["query"]- try:- ref_calls = json.loads(ex["answers"])- except (ValueError, TypeError):- continue- try:- pred_calls = json.loads(pred_text)- if isinstance(pred_calls, dict):- pred_calls = [pred_calls]- elif not isinstance(pred_calls, list):- continue- except (ValueError, TypeError):- continue- if not isinstance(ref_calls, list):- continue-- for c in ref_calls:- if isinstance(c, dict) and "name" in c:- c["name"] = to_snake_case(c["name"])- for c in pred_calls:- if isinstance(c, dict) and "name" in c:- c["name"] = to_snake_case(c["name"])-- ref_by_name = {}- for c in ref_calls:- if isinstance(c, dict) and "name" in c:- ref_by_name.setdefault(c["name"], []).append(c.get("arguments", {}) or {})-- for c in pred_calls:- if not isinstance(c, dict) or "name" not in c:- continue- name = c["name"]- if name not in ref_by_name:- continue # tool-level mismatch — outside this script's scope- pred_args = c.get("arguments", {}) or {}- ref_args = ref_by_name[name][0]- if not isinstance(ref_args, dict):- continue-- all_keys = set(pred_args.keys()) | set(ref_args.keys())- for key in all_keys:- has_pred = key in pred_args- has_ref = key in ref_args-- if has_ref and not has_pred:- bucket_counts["missing_key"] += 1- if len(bucket_examples["missing_key"]) < 5:- bucket_examples["missing_key"].append(- {"query": query, "tool": name, "key": key,- "ref": ref_args[key], "pred": "<MISSING>"})- continue- if has_pred and not has_ref:- bucket_counts["extra_key"] += 1- if len(bucket_examples["extra_key"]) < 5:- bucket_examples["extra_key"].append(- {"query": query, "tool": name, "key": key,- "ref": "<MISSING>", "pred": pred_args[key]})- continue-- rv, pv = ref_args[key], pred_args[key]- if json.dumps(pv, sort_keys=True) == json.dumps(rv, sort_keys=True):- continue- cat = categorize_value_mismatch(query, rv, pv)- bucket_counts[cat] += 1- if len(bucket_examples[cat]) < 5:- bucket_examples[cat].append(- {"query": query, "tool": name, "key": key,- "ref": rv, "pred": pv})-- return bucket_counts, bucket_examples---def _print_block(label, counts, examples):- total = sum(counts.values())- print(f"\n=== {label} ===")- print(f" Total value-level mismatches: {total}")- if total == 0:- return- print()- order = ("correctable", "canonicalization", "cross_lingual",- "unrecoverable", "missing_key", "extra_key")- for b in order:- n = counts.get(b, 0)- pct = 100 * n / total- bar = "█" * int(pct / 2)- print(f" {b:<18} {n:>5} ({pct:>5.1f}%) {bar}")-- for b in order:- if not examples.get(b):- continue- print(f"\n -- {b} examples --")- for e in examples[b]:- q = e["query"][:110]- print(f" Q: {q}")- print(f" {e['tool']}.{e['key']}")- print(f" ref : {e['ref']!r}")- print(f" pred: {e['pred']!r}")---def main(args):- print(f"Loading checkpoint {args.checkpoint} ...")- params, config = load_checkpoint(args.checkpoint)- model = EncoderDecoderTransformer(config)- tokenizer = get_tokenizer()-- param_count = sum(x.size for x in jax.tree.leaves(params))- print(f" Parameters: {param_count:,}")-- val_ds = load_tool_calls("validation")- rng = np.random.RandomState(42)-- pools = {"single": [], "multi": []}- for k in range(len(val_ds)):- ex = val_ds[k]- try:- answers = json.loads(ex["answers"])- except (ValueError, TypeError):- continue- if not isinstance(answers, list) or not answers:- continue- pools["single" if len(answers) == 1 else "multi"].append(ex)-- for name in pools:- rng.shuffle(pools[name])- pools[name] = pools[name][:args.num_samples]- print(f" {name}-call sample: {len(pools[name])}")-- combined_counts = Counter()- for name in ("single", "multi"):- print(f"\n--- Running generation on {len(pools[name])} {name}-call examples ...")- counts, examples = categorize_pool(- model, params, tokenizer, pools[name],- max_gen_len=args.max_gen_len,- max_enc_len=args.max_enc_len,- batch_size=args.batch_size,- )- _print_block(f"{name.upper()}-CALL", counts, examples)- combined_counts += counts-- total = sum(combined_counts.values())- if total == 0:- return- print(f"\n=== COMBINED ({total} value mismatches) ===")- order = ("correctable", "canonicalization", "cross_lingual",- "unrecoverable", "missing_key", "extra_key")- for b in order:- n = combined_counts.get(b, 0)- pct = 100 * n / total- bar = "█" * int(pct / 2)- print(f" {b:<18} {n:>5} ({pct:>5.1f}%) {bar}")-- correctable = combined_counts.get("correctable", 0)- missing = combined_counts.get("missing_key", 0)- print(f"\n Upper bound for a perfect draft-corrector: "- f"{100 * (correctable + missing) / total:.1f}% of value errors "- f"({correctable + missing}/{total})")- noise = combined_counts.get("canonicalization", 0) + combined_counts.get("cross_lingual", 0)- print(f" Fixable by cleaning labels (no model change): "- f"{100 * noise / total:.1f}% ({noise}/{total})")- unrecov = combined_counts.get("unrecoverable", 0)- print(f" Floor a corrector can't beat: "- f"{100 * unrecov / total:.1f}% ({unrecov}/{total})")---def parse_args():- p = argparse.ArgumentParser(- description="Categorize tool-call value failures by correctability")- p.add_argument("--checkpoint", type=str, required=True)- p.add_argument("--num-samples", type=int, default=500,- help="Samples per pool (single-call + multi-call)")- p.add_argument("--batch-size", type=int, default=32)- p.add_argument("--max-enc-len", type=int, default=1024)- p.add_argument("--max-gen-len", type=int, default=512)- return p.parse_args()---if __name__ == "__main__":- main(parse_args())diff --git a/src/cli.py b/src/cli.pyindex 3be4001..f866549 100644--- a/src/cli.py+++ b/src/cli.py@@ -120,7 +120,7 @@ def main():"matching params, random-inits the rest.")p.add_argument("--epochs", type=int, default=1)p.add_argument("--batch-size", type=int, default=64)- p.add_argument("--lr", type=float, default=3e-4)+ p.add_argument("--lr", type=float, default=3e-5)p.add_argument("--muon-lr", type=float, default=0.02)p.add_argument("--d-model", type=int, default=512)p.add_argument("--num-heads", type=int, default=8)@@ -165,14 +165,6 @@ def main():help="Loss weight for argument key tokens (default: 1.5)")p.add_argument("--no-feedforward", action=argparse.BooleanOptionalAction, default=True,help="Remove feedforward layers entirely (default: True)")- p.add_argument("--calibrate", action=argparse.BooleanOptionalAction, default=False,- help="Run confidence head calibration after training (default: False)")- p.add_argument("--calibrate-epochs", type=int, default=1,- help="Epochs for confidence head calibration (default: 1)")- p.add_argument("--calibrate-lr", type=float, default=1e-3,- help="Learning rate for confidence head calibration (default: 1e-3)")- p.add_argument("--calibrate-k", type=float, default=5.0,- help="Sigmoid steepness for PPL→confidence mapping (default: 5.0)")p = sub.add_parser("pretrain", add_help=False)p.add_argument("--name", type=str, default="pretrain",@@ -205,31 +197,6 @@ def main():p.add_argument("--activation", type=str, default="swiglu", choices=["drelu", "swiglu", "geglu"])p.add_argument("--no-feedforward", action=argparse.BooleanOptionalAction, default=True)- p = sub.add_parser("grpo", add_help=False)- p.add_argument("--checkpoint", type=str, default="checkpoints/needle.pkl",- help="SFT checkpoint to init both policy and reference (default: checkpoints/needle.pkl)")- p.add_argument("--name", type=str, default="grpo")- p.add_argument("--batch-size", type=int, default=16,- help="Prompts per step (B)")- p.add_argument("--k", type=int, default=4,- help="Samples per prompt (K)")- p.add_argument("--lr", type=float, default=1e-6)- p.add_argument("--beta", type=float, default=0.04,- help="KL penalty weight against frozen reference")- p.add_argument("--max-steps", type=int, default=5000)- p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)- p.add_argument("--max-gen-len", type=int, default=DEFAULT_MAX_GEN_LEN)- p.add_argument("--eval-every", type=int, default=100,- help="Run quick val PPL every N steps (default: 100)")- p.add_argument("--eval-batch-size", type=int, default=32,- help="Batch size for val PPL eval")- p.add_argument("--tool-call-samples", type=int, default=100,- help="Samples for stratified tool-call benchmark at save time (split evenly single/multi)")- p.add_argument("--checkpoint-dir", type=str, default="checkpoints")- p.add_argument("--wandb", action="store_true")- p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])- p.add_argument("--seed", type=int, default=42)-p = sub.add_parser("tokenize", add_help=False)p.add_argument("--max-samples", type=int, default=None,help="Limit samples per split (for dev/test)")@@ -265,15 +232,6 @@ def main():p.add_argument("--no-constrained", action="store_true",help="Disable grammar-constrained decoding for tool names/arg keys")- p = sub.add_parser("calibrate", add_help=False)- p.add_argument("--checkpoint", type=str, required=True)- p.add_argument("--output", type=str, default=None, help="Output checkpoint path (default: overwrite input)")- p.add_argument("--batch-size", type=int, default=128)- p.add_argument("--num-samples", type=int, default=None, help="Limit training samples for PPL computation")- p.add_argument("--epochs", type=int, default=1)- p.add_argument("--lr", type=float, default=1e-3)- p.add_argument("--k", type=float, default=5.0, help="Sigmoid steepness for PPL→confidence mapping")-p = sub.add_parser("generate-data", add_help=False)p.add_argument("--num-samples", type=int, default=500, help="Number of samples to generate")p.add_argument("--batch-size", type=int, default=25, help="Examples per Gemini call")@@ -283,24 +241,6 @@ def main():p.add_argument("--output-jsonl", type=str, default=None, help="Also save raw generations to JSONL")p.add_argument("--upload-every", type=int, default=None, help="Merge+upload every N samples")- p = sub.add_parser("merge-xlam", add_help=False)- p.add_argument("--dry-run", action="store_true", help="Skip upload")- p.add_argument("--max-samples", type=int, default=None, help="Limit xlam samples")-- p = sub.add_parser("translate-xlam", add_help=False)- p.add_argument("--max-samples", type=int, default=None, help="Limit examples to translate")- p.add_argument("--workers", type=int, default=8, help="Parallel Gemini calls")- p.add_argument("--model", type=str, default=None, help="Gemini model for translation")- p.add_argument("--batch-size", type=int, default=10, help="Examples per Gemini call")- p.add_argument("--dry-run", action="store_true", help="Translate only, skip save/upload")-- p = sub.add_parser("rebalance-tools", add_help=False)- p.add_argument("--dry-run", action="store_true", help="Preview without modifying")-- p = sub.add_parser("split-dataset", add_help=False)- p.add_argument("--val-per-source", type=int, default=None,- help="Validation samples per source (default: 2500)")-p = sub.add_parser("evaluate", add_help=False)p.add_argument("--checkpoint", type=str, required=True)p.add_argument("--benchmarks", type=str, nargs="*",@@ -381,21 +321,12 @@ def main():jax.distributed.initialize()from .train import traintrain(args)- elif args.command == "grpo":- import jax- if os.path.exists("/dev/accel0"):- jax.distributed.initialize()- from .grpo import grpo- grpo(args)elif args.command == "run":from .run import main as run_mainrun_main(args)elif args.command == "eval":from .eval import main as eval_main_fneval_main_fn(args)- elif args.command == "calibrate":- from .calibrate import main as calibrate_main- calibrate_main(args)elif args.command == "generate-data":from .generate_data import main as gendata_main, MODEL as _MODEL, UPLOAD_EVERY as _UEif args.model is None:@@ -403,20 +334,6 @@ def main():if args.upload_every is None:args.upload_every = _UEgendata_main(args)- elif args.command == "merge-xlam":- from .merge_xlam import main as merge_main- merge_main(args)- elif args.command == "translate-xlam":- from .translate_xlam import main as translate_main, MODEL as _TMODEL- if args.model is None:- args.model = _TMODEL- translate_main(args)- elif args.command == "rebalance-tools":- from .rebalance_tools import main as rebalance_main- rebalance_main(args)- elif args.command == "split-dataset":- from .split_dataset import main as split_main- split_main(args)elif args.command == "evaluate":from .evaluate import main as eval_maineval_main(args)diff --git a/src/generate_data.py b/src/generate_data.pyindex 49d3c84..fa7cb83 100644--- a/src/generate_data.py+++ b/src/generate_data.py@@ -1826,7 +1826,6 @@ def _merge_and_upload(existing, new_examples):import shutil as _shutil_shutil.rmtree(parquet_dir)print(f"Upload complete: {HF_DATASET_REPO}")- print("NOTE: Run 'needle split-dataset' to create the validation split.")return mergeddiff --git a/src/grpo.py b/src/grpo.pydeleted file mode 100644index e2fd953..0000000--- a/src/grpo.py+++ /dev/null@@ -1,609 +0,0 @@-"""GRPO fine-tuning from an SFT checkpoint.--For each prompt we sample K completions from the current policy, score them-with a programmatic reward (exact call-set match against the reference answer),-normalize rewards within the group to get advantages, and take a policy-gradient step with a KL penalty against a frozen reference (the SFT ckpt).--No KV cache, no constrained decoding, no reward model.--Usage:- needle grpo --checkpoint checkpoints/needle.pkl --wandb-"""--import json-import math-import os-import pickle-import time--import jax-import jax.numpy as jnp-import numpy as np-import optax-from flax.training import train_state-from tqdm import tqdm--from .data import (- load_tool_calls, load_prepared_data, get_batches, to_snake_case,-)-from .distributed import (- _replicate, _unreplicate, _upload_checkpoint,-)--_HF_SFT_REPO = "Cactus-Compute/needle"---def _download_sft_checkpoint(path_or_name):- """Return a local path to the SFT checkpoint, downloading from HF if needed.-- The SFT checkpoint lives in Cactus-Compute/needle, not the generic- checkpoints repo that _download_checkpoint targets.- """- if os.path.exists(path_or_name):- return path_or_name- from huggingface_hub import hf_hub_download- local_dir = os.path.dirname(path_or_name) or "checkpoints"- os.makedirs(local_dir, exist_ok=True)- print(f" Downloading {os.path.basename(path_or_name)} "- f"from {_HF_SFT_REPO}...", flush=True)- local_path = hf_hub_download(- repo_id=_HF_SFT_REPO,- filename=os.path.basename(path_or_name),- repo_type="model",- local_dir=local_dir,- )- print(f" Downloaded to {local_path}", flush=True)- return local_path-from .model import (- EncoderDecoderTransformer, TransformerConfig,- make_causal_mask, make_padding_mask,- make_packing_mask, make_causal_packing_mask, make_cross_packing_mask,-)-from .optim import _wsd_schedule-from .run import _build_encoder_input-from .tokenizer import EOS_ID, PAD_ID, get_tokenizer---def _call_key_set(calls):- """Canonicalize a list of tool calls: snake_case names, sorted."""- keys = []- for c in calls:- if not isinstance(c, dict):- continue- name = to_snake_case(c.get("name", ""))- args = c.get("arguments", {}) or {}- keys.append(json.dumps({"name": name, "arguments": args}, sort_keys=True))- return sorted(keys)---def _score(pred_text, ref_answers):- """Binary reward: 1.0 if pred call-set == ref call-set (order-invariant)."""- if pred_text.startswith("<tool_call>"):- pred_text = pred_text[len("<tool_call>"):]- try:- pred_calls = json.loads(pred_text)- if isinstance(pred_calls, dict):- pred_calls = [pred_calls]- elif not isinstance(pred_calls, list):- return 0.0- except (json.JSONDecodeError, TypeError):- return 0.0- try:- ref_calls = json.loads(ref_answers)- if not isinstance(ref_calls, list):- ref_calls = []- except (json.JSONDecodeError, TypeError):- return 0.0- return 1.0 if _call_key_set(pred_calls) == _call_key_set(ref_calls) else 0.0---def _build_prompts(tokenizer, batch_examples, max_enc_len):- """Tokenize and pad encoder inputs for a batch of prompts."""- enc_lists = [- _build_encoder_input(tokenizer, ex["query"], ex["tools"], max_enc_len)- for ex in batch_examples- ]- B = len(enc_lists)- enc = np.full((B, max_enc_len), PAD_ID, dtype=np.int32)- for i, toks in enumerate(enc_lists):- enc[i, :len(toks)] = toks- return enc---def _make_fns(model, max_gen_len):- """Build pmapped encode / sample-step / log-prob functions."""-- @jax.pmap- def p_encode(params, enc_input):- src_mask = make_padding_mask(enc_input, PAD_ID)- return model.apply({"params": params}, enc_input,- src_mask=src_mask, method="encode")-- @jax.pmap- def p_sample_step(params, dec_buffer, encoder_out, cross_mask,- rng, step_idx, finished):- sm = make_causal_mask(max_gen_len)- logits = model.apply(- {"params": params}, dec_buffer, encoder_out,- self_mask=sm, cross_mask=cross_mask, method="decode",- )- next_logits = jax.lax.dynamic_index_in_dim(- logits, step_idx, axis=1, keepdims=False,- )- next_tokens = jax.random.categorical(rng, next_logits, axis=-1)- next_tokens = jnp.where(finished, PAD_ID, next_tokens)- new_finished = finished | (next_tokens == EOS_ID)- return next_tokens, new_finished-- @jax.pmap- def p_log_probs(params, enc_input, dec_in, dec_tgt):- src_mask = make_padding_mask(enc_input, PAD_ID)- tm = make_causal_mask(dec_in.shape[-1]) & make_padding_mask(dec_in, PAD_ID)- logits = model.apply({"params": params}, enc_input, dec_in,- src_mask=src_mask, tgt_mask=tm)- lp = jax.nn.log_softmax(logits.astype(jnp.float32), axis=-1)- return jnp.take_along_axis(lp, dec_tgt[..., None], axis=-1).squeeze(-1)-- return p_encode, p_sample_step, p_log_probs---def _make_grpo_step(model, beta):- """Build pmapped GRPO gradient step."""-- def loss_fn(params, enc_input, dec_in, dec_tgt, advantages,- response_mask, ref_logp):- src_mask = make_padding_mask(enc_input, PAD_ID)- tm = make_causal_mask(dec_in.shape[-1]) & make_padding_mask(dec_in, PAD_ID)- logits = model.apply({"params": params}, enc_input, dec_in,- src_mask=src_mask, tgt_mask=tm)- lp = jax.nn.log_softmax(logits.astype(jnp.float32), axis=-1)- pol_logp = jnp.take_along_axis(lp, dec_tgt[..., None], axis=-1).squeeze(-1)-- denom = jnp.maximum(response_mask.sum(), 1.0)- pg = -(advantages[:, None] * pol_logp * response_mask).sum() / denom-- # Schulman k3 KL estimator: unbiased, always non-negative.- log_ratio = ref_logp - pol_logp- kl_tok = (jnp.exp(log_ratio) - log_ratio - 1.0) * response_mask- kl = kl_tok.sum() / denom-- return pg + beta * kl, (pg, kl)-- def step(state, enc_input, dec_in, dec_tgt, advantages,- response_mask, ref_logp):- (loss, (pg, kl)), grads = jax.value_and_grad(loss_fn, has_aux=True)(- state.params, enc_input, dec_in, dec_tgt, advantages,- response_mask, ref_logp,- )- grads = jax.lax.pmean(grads, axis_name="batch")- loss = jax.lax.pmean(loss, axis_name="batch")- pg = jax.lax.pmean(pg, axis_name="batch")- kl = jax.lax.pmean(kl, axis_name="batch")- state = state.apply_gradients(grads=grads)- return state, loss, pg, kl-- return jax.pmap(step, axis_name="batch", donate_argnums=(0,))---def _sample_completions(p_encode, p_sample_step, r_params, enc_sharded,- num_devices, max_gen_len, rng):- """Sample one completion per input row, in parallel across devices.-- enc_sharded: (num_devices, per_device, max_enc_len) already on-device.- Returns numpy array (num_devices, per_device, max_gen_len): each row is- [EOS, sampled..., EOS, PAD...] or [EOS, sampled...] if EOS wasn't emitted.- """- encoder_out, cross_mask = p_encode(r_params, enc_sharded)- per_device = enc_sharded.shape[1]-- dec_buffer = np.full((num_devices, per_device, max_gen_len), PAD_ID, dtype=np.int32)- dec_buffer[:, :, 0] = EOS_ID- dec_buffer = jnp.asarray(dec_buffer)- finished = jnp.zeros((num_devices, per_device), dtype=jnp.bool_)-- for step in range(max_gen_len - 1):- rng, sub = jax.random.split(rng)- sub_per = jax.random.split(sub, num_devices)- step_idx = jnp.full((num_devices,), step, dtype=jnp.int32)- next_tokens, finished = p_sample_step(- r_params, dec_buffer, encoder_out, cross_mask,- sub_per, step_idx, finished,- )- dec_buffer = dec_buffer.at[:, :, step + 1].set(next_tokens)-- return np.asarray(dec_buffer), rng---def _make_val_loss_fn(model):- """Teacher-forced packed val loss (matches train.py)."""-- @jax.jit- def fn(params, src, dec_in, dec_tgt, enc_seg, dec_seg):- src_mask = make_packing_mask(enc_seg)- tgt_mask = make_causal_packing_mask(dec_seg)- cross_mask = make_cross_packing_mask(enc_seg, dec_seg)- logits = model.apply({"params": params}, src, dec_in,- src_mask=src_mask, tgt_mask=tgt_mask,- cross_mask=cross_mask)- loss = optax.softmax_cross_entropy_with_integer_labels(- logits.astype(jnp.float32), dec_tgt)- pmask = (dec_seg > 0).astype(jnp.float32)- return jnp.sum(loss * pmask), jnp.sum(pmask)-- return fn---def _compute_val_ppl(val_loss_fn, params, val_data, batch_size):- total_loss = 0.0- total_toks = 0.0- for vb in get_batches(- val_data["packed_enc"], val_data["packed_dec_in"], val_data["packed_dec_tgt"],- batch_size, shuffle=False,- loss_mask=val_data["packed_loss"],- enc_seg_ids=val_data["packed_enc_seg"],- dec_seg_ids=val_data["packed_dec_seg"],- ):- src, di, dt, _lm, es, ds = vb- vl, vt = val_loss_fn(- params,- jnp.asarray(src, dtype=jnp.int32),- jnp.asarray(di, dtype=jnp.int32),- jnp.asarray(dt, dtype=jnp.int32),- jnp.asarray(es, dtype=jnp.int32),- jnp.asarray(ds, dtype=jnp.int32),- )- total_loss += float(vl)- total_toks += float(vt)- return math.exp(min(total_loss / max(total_toks, 1), 20))---def _stratified_val_slice(val_ds, per_bucket):- """Pick up to per_bucket single-call + per_bucket multi-call examples."""- single, multi = [], []- for ex in val_ds:- try:- calls = json.loads(ex["answers"])- except (json.JSONDecodeError, TypeError):- continue- if not isinstance(calls, list) or not calls:- continue- (single if len(calls) == 1 else multi).append(ex)- if len(single) >= per_bucket and len(multi) >= per_bucket:- break- return single[:per_bucket], multi[:per_bucket]---def _run_tool_call_eval(model, params, tokenizer, val_ds, per_bucket,- max_enc_len, max_gen_len):- """Stratified single/multi-call benchmark with display samples.-- Mirrors train.py's end-of-epoch tool-call eval. Uses greedy decode WITHOUT- constrained decoding — this reveals whether the underlying model is- actually learning to produce correct tool names, rather than leaning on- the grammar. `constrained=True` is the right choice at deployment; for- training telemetry we want to see raw model behavior.- """- from .eval import benchmark_tool_calls-- single, multi = _stratified_val_slice(val_ds, per_bucket)- results = {}- for label, examples in (("single", single), ("multi", multi)):- if not examples:- continue- tc = benchmark_tool_calls(- model, params, tokenizer, num_samples=len(examples),- max_gen_len=max_gen_len, max_enc_len=max_enc_len,- constrained=False, ds=examples,- )- results[label] = {- "name_f1": tc["name_f1"],- "call_f1": tc["call_f1"],- "exact_match": tc["exact_match"],- "args_acc": tc["args_acc"],- "value_acc": tc["value_acc"],- "param_haluc": tc["param_haluc"],- "param_miss": tc["param_miss"],- "json_parse": tc["json_parse_rate"],- "samples": tc["samples"],- "n": len(examples),- }- return results---def _print_eval_block(step, val_ppl, tc_results, pbar):- """Mirror the end-of-epoch print layout from train.py."""- pbar.write(f"\n ─────── step {step} eval ───────")- if val_ppl is not None:- pbar.write(f" val_ppl {val_ppl:>10.3f}")- for label, m in tc_results.items():- pbar.write(f" ─── {label} ({m['n']} samples, unconstrained) ───")- pbar.write(f" JSON parse {m['json_parse']:>10.1%}")- pbar.write(f" Name F1 {m['name_f1']:>10.1%}")- pbar.write(f" Call F1 {m['call_f1']:>10.1%}")- pbar.write(f" Exact match {m['exact_match']:>10.1%}")- pbar.write(f" Args acc {m['args_acc']:>10.1%}")- pbar.write(f" Value acc {m['value_acc']:>10.1%}")- pbar.write(f" Param haluc {m['param_haluc']:>10.1%}")- pbar.write(f" Param miss {m['param_miss']:>10.1%}")- # Show up to 3 display samples from the "single" pool- single = tc_results.get("single") or tc_results.get("multi")- if single and single.get("samples"):- pbar.write(f" ─── samples ───")- for q, ref, pred in single["samples"][:3]:- pbar.write(f" Q: {q}")- pbar.write(f" R: {ref}")- pbar.write(f" P: {pred}")- pbar.write("")---def _decode_sample(tokenizer, token_row):- """Decode one sampled row to text, stopping at the first EOS after pos 0."""- sampled = token_row[1:]- eos_positions = np.where(sampled == EOS_ID)[0]- end = int(eos_positions[0]) if len(eos_positions) > 0 else len(sampled)- ids = [int(t) for t in sampled[:end] if t != PAD_ID]- return tokenizer.decode(ids)---def grpo(args):- num_devices = jax.local_device_count()- num_hosts = jax.process_count()- host_id = jax.process_index()- total_devices = jax.device_count()- is_main = host_id == 0-- use_wandb = getattr(args, "wandb", False) and is_main- if use_wandb:- import wandb- if wandb.run is None:- wandb.init(project="needle-v1", name=args.name, config=vars(args))-- print(f"\n[1/4] Devices: {num_devices} local, {total_devices} total "- f"across {num_hosts} host(s)", flush=True)-- print(f"\n[2/4] Loading checkpoint {args.checkpoint} ...", flush=True)- ckpt_path = _download_sft_checkpoint(args.checkpoint)- with open(ckpt_path, "rb") as f:- data = pickle.load(f)- config_dict = {k: v for k, v in data["config"].items()- if k in TransformerConfig.__dataclass_fields__}- config = TransformerConfig(**config_dict)- model = EncoderDecoderTransformer(config)-- dt = jnp.bfloat16 if args.dtype == "bfloat16" else jnp.float32- params = jax.tree.map(lambda x: jnp.array(x, dtype=dt), data["params"])- ref_params = jax.tree.map(lambda x: jnp.array(x, dtype=dt), data["params"])-- param_count = sum(x.size for x in jax.tree.leaves(params))- print(f" params: {param_count:,} "- f"(d={config.d_model}, heads={config.num_heads}, "- f"layers={config.num_encoder_layers}/{config.num_decoder_layers})")-- print(f"\n[3/4] Building optimizer + pmap ...", flush=True)- total_steps = args.max_steps- warmup = max(1, int(total_steps * 0.05))- scaled_lr = args.lr * total_devices- schedule = _wsd_schedule(scaled_lr, total_steps, warmup, decay_ratio=0.1)- tx = optax.chain(- optax.clip_by_global_norm(1.0),- optax.adamw(schedule, b2=0.95, weight_decay=0.0),- )- state = train_state.TrainState.create(apply_fn=model.apply, params=params, tx=tx)- del params-- r_state = _replicate(state)- r_ref_params = _replicate(ref_params)- del state, ref_params-- p_encode, p_sample_step, p_log_probs = _make_fns(model, args.max_gen_len)- p_grpo_step = _make_grpo_step(model, args.beta)-- print(f"\n[4/4] Loading tokenizer + data ...", flush=True)- tokenizer = get_tokenizer()- train_ds = load_tool_calls("train")- n_examples = len(train_ds)- print(f" train: {n_examples:,} examples")-- val_data = None- val_ds = None- val_loss_fn = None- if is_main:- try:- val_data = load_prepared_data("val", mmap=True)- print(f" val (packed): {len(val_data['packed_enc']):,} bins")- except FileNotFoundError:- print(" WARN: no packed val data — skipping val PPL "- "(run `needle tokenize` to generate)")- try:- val_ds = load_tool_calls("val")- print(f" val (tool-call): {len(val_ds):,} examples")- except Exception as e:- print(f" WARN: could not load tool-call val set: {e}")- val_loss_fn = _make_val_loss_fn(model)-- B = args.batch_size- K = args.k- BK = B * K- if BK % num_devices != 0:- raise ValueError(f"B*K ({BK}) must be divisible by num_devices ({num_devices})")- per_device = BK // num_devices-- if is_main:- print(f"\n ─────────────────────────────────────")- print(f" GRPO")- print(f" ─────────────────────────────────────")- print(f" B (prompts/step) {B}")- print(f" K (samples/prompt) {K}")- print(f" B*K {BK} ({per_device}/device)")- print(f" LR {args.lr} -> {scaled_lr}")- print(f" beta (KL) {args.beta}")- print(f" max_steps {args.max_steps}")- print(f" max_enc_len {args.max_enc_len}")- print(f" max_gen_len {args.max_gen_len}")- print(f" ─────────────────────────────────────\n")-- rng = jax.random.PRNGKey(args.seed)- perm = np.random.RandomState(args.seed).permutation(n_examples)- cursor = 0-- best_call_f1 = -1.0-- pbar = tqdm(total=args.max_steps, disable=not is_main, desc="GRPO")-- for step in range(args.max_steps):- if cursor + B > len(perm):- perm = np.random.RandomState(args.seed + step).permutation(n_examples)- cursor = 0- batch_idx = perm[cursor:cursor + B]- cursor += B- batch = [train_ds[int(i)] for i in batch_idx]-- # Build encoder inputs, replicate K times per prompt, shard across devices.- enc_np = _build_prompts(tokenizer, batch, args.max_enc_len)- enc_rep = np.repeat(enc_np, K, axis=0)- enc_sharded = jnp.asarray(- enc_rep.reshape(num_devices, per_device, args.max_enc_len)- )-- # Sample K completions per prompt under the current policy.- t0 = time.perf_counter()- dec_buffer_np, rng = _sample_completions(- p_encode, p_sample_step, r_state.params, enc_sharded,- num_devices, args.max_gen_len, rng,- )- sample_time = time.perf_counter() - t0- dec_buffer_flat = dec_buffer_np.reshape(BK, args.max_gen_len)-- # Score each completion.- rewards = np.zeros(BK, dtype=np.float32)- for i in range(BK):- prompt_idx = i // K- text = _decode_sample(tokenizer, dec_buffer_flat[i])- rewards[i] = _score(text, batch[prompt_idx]["answers"])-- # Group-relative advantages: normalize within each prompt's K samples.- rewards_grp = rewards.reshape(B, K)- mean = rewards_grp.mean(axis=1, keepdims=True)- std = rewards_grp.std(axis=1, keepdims=True)- advantages = ((rewards_grp - mean) / (std + 1e-4)).reshape(BK).astype(np.float32)-- # Teacher-forced shapes: dec_in = buffer[:-1], dec_tgt = buffer[1:].- dec_in_np = dec_buffer_flat[:, :-1].astype(np.int32)- dec_tgt_np = dec_buffer_flat[:, 1:].astype(np.int32)- response_mask_np = (dec_tgt_np != PAD_ID).astype(np.float32)- T_resp = dec_in_np.shape[1]-- dec_in_sh = jnp.asarray(dec_in_np.reshape(num_devices, per_device, T_resp))- dec_tgt_sh = jnp.asarray(dec_tgt_np.reshape(num_devices, per_device, T_resp))- resp_sh = jnp.asarray(response_mask_np.reshape(num_devices, per_device, T_resp))- adv_sh = jnp.asarray(advantages.reshape(num_devices, per_device))-- ref_logp_sh = p_log_probs(r_ref_params, enc_sharded, dec_in_sh, dec_tgt_sh)-- r_state, loss, pg, kl = p_grpo_step(- r_state, enc_sharded, dec_in_sh, dec_tgt_sh,- adv_sh, resp_sh, ref_logp_sh,- )-- loss_val = float(loss.addressable_shards[0].data[0])- pg_val = float(pg.addressable_shards[0].data[0])- kl_val = float(kl.addressable_shards[0].data[0])- mean_r = float(rewards.mean())- std_r = float(rewards.std())-- pbar.update(1)- pbar.set_postfix(- loss=f"{loss_val:+.4f}", pg=f"{pg_val:+.4f}",- kl=f"{kl_val:.4f}", R=f"{mean_r:.3f}", sT=f"{sample_time:.1f}s",- )-- if use_wandb:- import wandb- wandb.log({- "grpo/loss": loss_val,- "grpo/pg": pg_val,- "grpo/kl": kl_val,- "grpo/reward_mean": mean_r,- "grpo/reward_std": std_r,- "grpo/sample_time_s": sample_time,- "grpo/step": step,- })-- # Full eval — val PPL + stratified tool-call benchmark + samples.- # Mirrors the end-of-epoch eval in train.py, run every eval_every steps.- if is_main and (step + 1) % args.eval_every == 0:- eval_params = _unreplicate(r_state).params-- val_ppl = None- if val_data is not None:- val_ppl = _compute_val_ppl(- val_loss_fn, eval_params, val_data, args.eval_batch_size,- )-- tc_results = {}- if val_ds is not None:- tc_results = _run_tool_call_eval(- model, eval_params, tokenizer, val_ds,- per_bucket=args.tool_call_samples // 2,- max_enc_len=args.max_enc_len,- max_gen_len=args.max_gen_len,- )-- _print_eval_block(step + 1, val_ppl, tc_results, pbar)-- # Best-checkpoint tracking: match train.py, use single-call F1.- cur_f1 = tc_results.get("single", {}).get("call_f1", -1.0)- is_new_best = cur_f1 > best_call_f1- if is_new_best and cur_f1 >= 0:- best_call_f1 = cur_f1- _save_grpo_checkpoint(- r_state, config, args.checkpoint_dir, step + 1,- filename="needle_grpo_best.pkl",- )- pbar.write(f" ** New best single call_f1={best_call_f1:.1%}")-- if use_wandb:- import wandb- log = {"grpo/step": step + 1, "val/best_call_f1": best_call_f1}- if val_ppl is not None:- log["val/text_ppl"] = val_ppl- for label, metrics in tc_results.items():- for k, v in metrics.items():- if k in ("n", "samples"):- continue- log[f"val/{label}_{k}"] = v- wandb.log(log)-- del eval_params-- pbar.close()-- if is_main:- best_path = os.path.join(args.checkpoint_dir, "needle_grpo_best.pkl")- if best_call_f1 >= 0:- print(f"\nGRPO complete. Best single call_f1={best_call_f1:.1%} -> {best_path}")- else:- print(f"\nGRPO complete. No eval ran — no best checkpoint saved.")-- if use_wandb:- import wandb- wandb.finish()-- if num_hosts > 1:- jax.experimental.multihost_utils.sync_global_devices("grpo_done")---def _save_grpo_checkpoint(r_state, config, checkpoint_dir, step,- filename="needle_grpo.pkl"):- os.makedirs(checkpoint_dir, exist_ok=True)- params = _unreplicate(r_state).params- params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), params)- ckpt_path = os.path.join(checkpoint_dir, filename)- with open(ckpt_path, "wb") as f:- pickle.dump({"params": params_np, "config": config.__dict__,- "grpo_step": step}, f)- size_mb = sum(x.nbytes for x in jax.tree.leaves(params_np)) / 1e6- print(f"\n [step {step}] Saved {ckpt_path} ({size_mb:.1f} MB)", flush=True)- _upload_checkpoint(ckpt_path)- return ckpt_pathdiff --git a/src/merge_xlam.py b/src/merge_xlam.pydeleted file mode 100644index 08f6354..0000000--- a/src/merge_xlam.py+++ /dev/null@@ -1,113 +0,0 @@-#!/usr/bin/env python3-"""Merge Salesforce/xlam-function-calling-60k into the existing unified dataset-and re-upload to HuggingFace (Cactus-Compute/tool-calls).--Usage:- python scripts/merge_xlam.py # merge + upload- python scripts/merge_xlam.py --dry-run # merge only, no upload- python scripts/merge_xlam.py --max-samples 1000 # limit xlam samples (for testing)-"""--import argparse-import json-import os-import sys--from datasets import concatenate_datasets, load_dataset, load_from_disk--LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data", "tool_calls_unified")-HF_DATASET_REPO = "Cactus-Compute/tool-calls"---def load_existing():- local = os.path.abspath(LOCAL_UNIFIED_DIR)- if os.path.exists(local) and any(f.endswith(".arrow") for f in os.listdir(local)):- ds = load_from_disk(local)- print(f"Loaded existing dataset from disk: {len(ds)} rows")- else:- from .data import download_hf_split- print(f"Downloading existing dataset from {HF_DATASET_REPO}...")- ds = download_hf_split("train", HF_DATASET_REPO)- print(f"Downloaded: {len(ds)} rows")- return ds---def load_xlam(max_samples=None):- print("Loading Salesforce/xlam-function-calling-60k...")- ds = load_dataset("Salesforce/xlam-function-calling-60k", split="train")- print(f"Loaded xlam: {len(ds)} rows")-- if max_samples and max_samples < len(ds):- ds = ds.shuffle(seed=42).select(range(max_samples))- print(f"Sampled down to {len(ds)} rows")-- if "id" in ds.column_names:- ds = ds.remove_columns(["id"])-- ds = ds.add_column("source", ["xlam-function-calling-60k"] * len(ds))- ds = ds.add_column("model", ["human-curated"] * len(ds))-- return ds---def check_duplicates(existing, new):- """Quick check for query overlap."""- existing_queries = set(existing["query"][:10000])- new_queries = set(new["query"])- overlap = existing_queries & new_queries- if overlap:- print(f"Warning: {len(overlap)} overlapping queries found (sampling first 10k existing)")- else:- print("No duplicate queries detected (sampled check)")---def main(args):- existing = load_existing()- xlam = load_xlam(args.max_samples)-- xlam = xlam.select_columns(existing.column_names)-- check_duplicates(existing, xlam)-- merged = concatenate_datasets([existing, xlam])- print(f"\nMerged dataset: {len(merged)} rows ({len(existing)} existing + {len(xlam)} new)")-- from collections import Counter- counts = Counter(merged["source"])- print("\nBreakdown by source:")- for src, cnt in counts.most_common():- print(f" {src}: {cnt}")-- if args.dry_run:- print("\n--dry-run: skipping save and upload")- return-- import shutil- local = os.path.abspath(LOCAL_UNIFIED_DIR)- tmp_dir = local + "_tmp"- if os.path.exists(tmp_dir):- shutil.rmtree(tmp_dir)- print(f"\nSaving to {local}...")- merged.save_to_disk(tmp_dir)- if os.path.exists(local):- shutil.rmtree(local)- os.rename(tmp_dir, local)- print("Saved.")-- # Upload to HuggingFace (train split only)- from huggingface_hub import HfApi-- api = HfApi()- api.create_repo(HF_DATASET_REPO, repo_type="dataset", private=False, exist_ok=True)- print(f"\nUploading to {HF_DATASET_REPO} (train split)...")- merged.push_to_hub(HF_DATASET_REPO, split="train", token=True)- print(f"Upload complete: {HF_DATASET_REPO}")- print("NOTE: Run 'needle split-dataset' to create the validation split.")---if __name__ == "__main__":- import argparse as _ap- _p = _ap.ArgumentParser(description="Merge xlam into unified dataset")- _p.add_argument("--dry-run", action="store_true", help="Skip upload")- _p.add_argument("--max-samples", type=int, default=None, help="Limit xlam samples")- main(_p.parse_args())\ No newline at end of filediff --git a/src/mine_corrections.py b/src/mine_corrections.pydeleted file mode 100644index 26ceb14..0000000--- a/src/mine_corrections.py+++ /dev/null@@ -1,400 +0,0 @@-"""Mine self-correction training pairs from the SFT checkpoint.--For each training prompt, sample K completions at temperature T from the-frozen SFT model, classify each against the gold, and write-(query, tools, draft, gold, category) tuples to JSONL for downstream-tokenization.--Categories written: value_wrong, args_wrong, correct.-Categories dropped: tool_wrong, unparseable. They are counted in the-summary so you can see the distribution, but a corrector can't recover-tool-selection or parse errors from a draft, so we don't train on them.--Resumability: the script writes progress to <output>.cursor. On restart-it reads the cursor, reopens the output in append mode, and resumes from-the next unprocessed prompt. Crashing mid-chunk can cause up to one-batch of duplicates — acceptable for mining, downstream dedup is cheap.--Usage:- python -m src.mine_corrections \\- --checkpoint checkpoints/needle.pkl \\- --output data/corrections.jsonl \\- --max-samples 100000 \\- --k 1 --temperature 0.7-"""--import argparse-import json-import os-import time-from collections import Counter--import jax-import jax.numpy as jnp-import numpy as np-from tqdm import tqdm--from .data import get_tokenizer, load_tool_calls-from .distributed import _replicate-from .model import (- EncoderDecoderTransformer,- make_causal_mask,- make_padding_mask,-)-from .run import _build_encoder_input, load_checkpoint-from .tokenizer import EOS_ID, PAD_ID, to_snake_case---_HF_SFT_REPO = "Cactus-Compute/needle"---def _maybe_download_ckpt(path):- if os.path.exists(path):- return path- from huggingface_hub import hf_hub_download- local_dir = os.path.dirname(path) or "checkpoints"- os.makedirs(local_dir, exist_ok=True)- print(f" Downloading {os.path.basename(path)} from {_HF_SFT_REPO} ...")- return hf_hub_download(- repo_id=_HF_SFT_REPO,- filename=os.path.basename(path),- repo_type="model",- local_dir=local_dir,- )---def _call_key_set(calls):- keys = []- for c in calls:- if not isinstance(c, dict):- continue- name = to_snake_case(c.get("name", ""))- args = c.get("arguments", {}) or {}- keys.append(json.dumps({"name": name, "arguments": args}, sort_keys=True))- return sorted(keys)---def _classify(pred_text, ref_text):- """Return one of: correct, value_wrong, args_wrong, tool_wrong, unparseable."""- if pred_text.startswith("<tool_call>"):- pred_text = pred_text[len("<tool_call>"):]- try:- pred_calls = json.loads(pred_text)- if isinstance(pred_calls, dict):- pred_calls = [pred_calls]- elif not isinstance(pred_calls, list):- return "unparseable"- except (json.JSONDecodeError, TypeError):- return "unparseable"- try:- ref_calls = json.loads(ref_text)- if not isinstance(ref_calls, list):- return "unparseable"- except (json.JSONDecodeError, TypeError):- return "unparseable"-- for c in pred_calls:- if isinstance(c, dict) and isinstance(c.get("name"), str):- c["name"] = to_snake_case(c["name"])- else:- return "unparseable"- for c in ref_calls:- if isinstance(c, dict) and isinstance(c.get("name"), str):- c["name"] = to_snake_case(c["name"])-- if _call_key_set(pred_calls) == _call_key_set(ref_calls):- return "correct"-- ref_names = sorted(c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c)- pred_names = sorted(c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c)- if ref_names != pred_names:- return "tool_wrong"-- # Tool names match — check whether arg keys or just values differ- ref_by_name, pred_by_name = {}, {}- for c in ref_calls:- if isinstance(c, dict) and "name" in c:- ref_by_name.setdefault(c["name"], []).append(c.get("arguments", {}) or {})- for c in pred_calls:- if isinstance(c, dict) and "name" in c:- pred_by_name.setdefault(c["name"], []).append(c.get("arguments", {}) or {})-- for name, ref_list in ref_by_name.items():- pred_list = pred_by_name.get(name, [])- for ra, pa in zip(ref_list, pred_list):- if set((ra or {}).keys()) != set((pa or {}).keys()):- return "args_wrong"- return "value_wrong"---def _make_sampling_fns(model, max_gen_len, temperature):- """Build pmapped encode + sample-step functions with temperature baked in."""- inv_t = 1.0 / max(temperature, 1e-5)-- @jax.pmap- def p_encode(params, enc_input):- src_mask = make_padding_mask(enc_input, PAD_ID)- return model.apply(- {"params": params}, enc_input,- src_mask=src_mask, method="encode",- )-- @jax.pmap- def p_sample_step(params, dec_buffer, encoder_out, cross_mask,- rng, step_idx, finished):- sm = make_causal_mask(max_gen_len)- logits = model.apply(- {"params": params}, dec_buffer, encoder_out,- self_mask=sm, cross_mask=cross_mask, method="decode",- )- next_logits = jax.lax.dynamic_index_in_dim(- logits, step_idx, axis=1, keepdims=False,- ) * inv_t- next_tokens = jax.random.categorical(rng, next_logits, axis=-1)- next_tokens = jnp.where(finished, PAD_ID, next_tokens)- new_finished = finished | (next_tokens == EOS_ID)- return next_tokens, new_finished-- return p_encode, p_sample_step---def _sample_completions(p_encode, p_sample_step, r_params, enc_sharded,- num_devices, max_gen_len, rng):- encoder_out, cross_mask = p_encode(r_params, enc_sharded)- per_device = enc_sharded.shape[1]-- dec_buffer = np.full((num_devices, per_device, max_gen_len), PAD_ID, dtype=np.int32)- dec_buffer[:, :, 0] = EOS_ID- dec_buffer = jnp.asarray(dec_buffer)- finished = jnp.zeros((num_devices, per_device), dtype=jnp.bool_)-- for step in range(max_gen_len - 1):- rng, sub = jax.random.split(rng)- sub_per = jax.random.split(sub, num_devices)- step_idx = jnp.full((num_devices,), step, dtype=jnp.int32)- next_tokens, finished = p_sample_step(- r_params, dec_buffer, encoder_out, cross_mask,- sub_per, step_idx, finished,- )- dec_buffer = dec_buffer.at[:, :, step + 1].set(next_tokens)-- return np.asarray(dec_buffer), rng---def _decode_sample(tokenizer, token_row):- """Decode a sampled row and strip the leading <tool_call> task marker.-- Matches run.py's `generate_batch` post-processing: strip `<tool_call>`- if present so the stored draft is the raw JSON. Without this the draft- re-encodes to a double TOOL_CALL_ID at train time while inference (which- strips it) sees a single one — a train/inference format mismatch.- """- sampled = token_row[1:]- eos_positions = np.where(sampled == EOS_ID)[0]- end = int(eos_positions[0]) if len(eos_positions) > 0 else len(sampled)- ids = [int(t) for t in sampled[:end] if t != PAD_ID]- text = tokenizer.decode(ids)- if text.startswith("<tool_call>"):- text = text[len("<tool_call>"):]- return text---def _read_cursor(path):- if not os.path.exists(path):- return 0- with open(path) as f:- try:- return int(f.read().strip())- except ValueError:- return 0---def _write_cursor(path, value):- with open(path, "w") as f:- f.write(str(value))---def mine(args):- if os.path.exists("/dev/accel0"):- jax.distributed.initialize()-- num_devices = jax.local_device_count()- print(f"Devices: {num_devices} local, {jax.device_count()} total")-- ckpt_path = _maybe_download_ckpt(args.checkpoint)- print(f"Loading checkpoint {ckpt_path} ...")- params, config = load_checkpoint(ckpt_path)- model = EncoderDecoderTransformer(config)- tokenizer = get_tokenizer()-- param_count = sum(x.size for x in jax.tree.leaves(params))- print(f" Parameters: {param_count:,}")- print(f" Config: d={config.d_model}, heads={config.num_heads}, "- f"layers={config.num_encoder_layers}/{config.num_decoder_layers}")-- r_params = _replicate(params)- del params-- p_encode, p_sample_step = _make_sampling_fns(- model, args.max_gen_len, args.temperature)-- print("Loading train split ...")- train_ds = load_tool_calls("train")- n_total = len(train_ds)- print(f" {n_total:,} rows (empty-answer examples skipped inline)")-- os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)- cursor_path = args.output + ".cursor"- start_idx = _read_cursor(cursor_path)- if start_idx > 0:- print(f" Resuming from dataset index {start_idx}")-- B = args.batch_size- K = args.k- BK = B * K- if BK % num_devices != 0:- raise ValueError(- f"B*K ({BK}) must be divisible by num_devices ({num_devices}). "- f"Try --batch-size={num_devices * (BK // num_devices) // K}")- per_device = BK // num_devices-- print(f" B={B} prompts, K={K} samples/prompt, BK={BK} ({per_device}/device)")- print(f" temperature={args.temperature}, max_gen_len={args.max_gen_len}")-- rng = jax.random.PRNGKey(args.seed)- stats = Counter()-- out_f = open(args.output, "a", encoding="utf-8", buffering=1)- pbar_total = args.max_samples if args.max_samples else (n_total - start_idx)- pbar = tqdm(total=pbar_total, desc="Mining", unit="prompts", initial=0)- t0 = time.perf_counter()- processed = 0- last_ds_idx = start_idx - 1-- def _iter_prompts():- for i in range(start_idx, n_total):- ex = train_ds[int(i)]- if ex["answers"].strip() in ("", "[]"):- continue- yield i, ex-- from itertools import islice- prompt_iter = _iter_prompts()-- try:- while True:- pairs = list(islice(prompt_iter, B))- if not pairs:- break- batch_ds_idx = [p[0] for p in pairs]- batch = [p[1] for p in pairs]- actual_B = len(batch)- last_ds_idx = batch_ds_idx[-1]-- # Pad batch to full B by repeating last example (outputs ignored)- while len(batch) < B:- batch.append(batch[-1])-- enc_lists = [- _build_encoder_input(tokenizer, ex["query"], ex["tools"],- args.max_enc_len)- for ex in batch- ]- enc_np = np.full((B, args.max_enc_len), PAD_ID, dtype=np.int32)- for i, toks in enumerate(enc_lists):- enc_np[i, :len(toks)] = toks-- enc_rep = np.repeat(enc_np, K, axis=0)- enc_sharded = jnp.asarray(- enc_rep.reshape(num_devices, per_device, args.max_enc_len))-- dec_buffer_np, rng = _sample_completions(- p_encode, p_sample_step, r_params, enc_sharded,- num_devices, args.max_gen_len, rng,- )- dec_buffer_flat = dec_buffer_np.reshape(BK, args.max_gen_len)-- for prompt_idx in range(actual_B):- prompt = batch[prompt_idx]- kept_correct = False- for k in range(K):- i = prompt_idx * K + k- draft = _decode_sample(tokenizer, dec_buffer_flat[i])- category = _classify(draft, prompt["answers"])- stats[category] += 1-- if category in ("value_wrong", "args_wrong"):- out_f.write(json.dumps({- "query": prompt["query"],- "tools": prompt["tools"],- "draft": draft,- "answers": prompt["answers"],- "category": category,- "source_idx": int(batch_ds_idx[prompt_idx]),- }, ensure_ascii=False) + "\n")- elif category == "correct" and not kept_correct:- kept_correct = True- out_f.write(json.dumps({- "query": prompt["query"],- "tools": prompt["tools"],- "draft": draft,- "answers": prompt["answers"],- "category": "correct",- "source_idx": int(batch_ds_idx[prompt_idx]),- }, ensure_ascii=False) + "\n")-- _write_cursor(cursor_path, last_ds_idx + 1)- pbar.update(actual_B)- processed += actual_B-- if processed % (B * 10) == 0:- elapsed = time.perf_counter() - t0- rate = processed / max(elapsed, 1e-6)- total_gen = max(sum(stats.values()), 1)- pbar.set_postfix(- val=f"{stats['value_wrong']} ({100*stats['value_wrong']/total_gen:.0f}%)",- args=f"{stats['args_wrong']} ({100*stats['args_wrong']/total_gen:.0f}%)",- corr=f"{stats['correct']} ({100*stats['correct']/total_gen:.0f}%)",- tool=f"{stats['tool_wrong']} ({100*stats['tool_wrong']/total_gen:.0f}%)",- rate=f"{rate:.1f}/s",- )-- if args.max_samples and processed >= args.max_samples:- break- finally:- out_f.close()- pbar.close()- total = sum(stats.values())- written = stats["value_wrong"] + stats["args_wrong"] + stats["correct"]- print("\n=== Final stats ===")- for k in ("correct", "value_wrong", "args_wrong", "tool_wrong",- "unparseable"):- v = stats.get(k, 0)- pct = 100 * v / max(total, 1)- print(f" {k:<14} {v:>10} ({pct:>5.1f}%)")- print(f" total generations {total:>10}")- print(f" rows written {written:>10} -> {args.output}")---def parse_args():- p = argparse.ArgumentParser(- description="Mine self-correction pairs from an SFT checkpoint")- p.add_argument("--checkpoint", type=str, default="checkpoints/needle.pkl")- p.add_argument("--output", type=str, default="data/corrections.jsonl")- p.add_argument("--max-samples", type=int, default=None,- help="Limit number of training prompts (default: all)")- p.add_argument("--k", type=int, default=1,- help="Samples per prompt (default: 1)")- p.add_argument("--temperature", type=float, default=0.7)- p.add_argument("--batch-size", type=int, default=256,- help="Prompts per forward pass; B*K must divide num_devices")- p.add_argument("--max-enc-len", type=int, default=1024)- p.add_argument("--max-gen-len", type=int, default=256,- help="Generation cap — most tool calls fit in 256; "- "raise if you see truncations")- p.add_argument("--seed", type=int, default=42)- return p.parse_args()---if __name__ == "__main__":- mine(parse_args())diff --git a/src/rebalance_tools.py b/src/rebalance_tools.pydeleted file mode 100644index 58be96d..0000000--- a/src/rebalance_tools.py+++ /dev/null@@ -1,225 +0,0 @@-#!/usr/bin/env python3-"""Rebalance tool count distribution by trimming over-represented bins.--- 2/3 of 3-tool examples → trimmed to 1 or 2 tools-- 2/3 of 10-tool examples → trimmed to 5-9 tools--Keeps called tools + random subset of uncalled tools to reach target count.-Uploads the rebalanced dataset back to HuggingFace.--Usage:- python scripts/rebalance_tools.py- python scripts/rebalance_tools.py --dry-run-"""--import argparse-import json-import os-import random-from collections import Counter--from datasets import Dataset, concatenate_datasets-from tqdm import tqdm--HF_REPO = "Cactus-Compute/tool-calls"---def _get_called_names(answers_str):- """Extract set of called tool names from answers JSON."""- try:- calls = json.loads(answers_str)- except (ValueError, TypeError):- return set()- if not isinstance(calls, list):- return set()- return {c["name"] for c in calls if isinstance(c, dict) and "name" in c}---def _trim_tools(tools_str, called_names, target_n, rng):- """Trim tools list to target_n, keeping all called tools + random fillers."""- try:- tools = json.loads(tools_str)- except (ValueError, TypeError):- return tools_str- if not isinstance(tools, list):- return tools_str-- # Separate called vs uncalled- called = [t for t in tools if isinstance(t, dict) and t.get("name") in called_names]- uncalled = [t for t in tools if isinstance(t, dict) and t.get("name") not in called_names]-- # If we need fewer tools than called tools, can't trim — skip- if target_n < len(called):- return None-- # Pick random uncalled fillers to reach target- n_fillers = target_n - len(called)- if n_fillers > len(uncalled):- return None # not enough uncalled tools-- fillers = rng.sample(uncalled, n_fillers)- new_tools = called + fillers- rng.shuffle(new_tools)- return json.dumps(new_tools, separators=(",", ":"))---def rebalance(dry_run=False):- from .data import download_hf_split- print(f"Loading dataset from {HF_REPO}...")- ds = download_hf_split("train", HF_REPO)- print(f"Loaded {len(ds):,} examples")-- rng = random.Random(42)-- # Categorize by tool count- indices_by_n = {}- for i, ex in enumerate(tqdm(ds, desc="Categorizing")):- try:- tools = json.loads(ex["tools"])- n = len(tools) if isinstance(tools, list) else 0- except (ValueError, TypeError):- n = 0- indices_by_n.setdefault(n, []).append(i)-- print("\nBefore rebalancing:")- for n in sorted(indices_by_n.keys()):- if n <= 10:- print(f" {n:>2} tools: {len(indices_by_n[n]):>10,}")-- # Plan: which indices to modify- modify_3 = [] # (index, target_n)- modify_10 = [] # (index, target_n)-- # 2/3 of 3-tool → trim to 1 or 2- if 3 in indices_by_n:- pool = list(indices_by_n[3])- rng.shuffle(pool)- n_modify = len(pool) * 2 // 3- for idx in pool[:n_modify]:- target = rng.choice([1, 2])- modify_3.append((idx, target))-- # 2/3 of 10-tool → trim to 5-9- if 10 in indices_by_n:- pool = list(indices_by_n[10])- rng.shuffle(pool)- n_modify = len(pool) * 2 // 3- for idx in pool[:n_modify]:- target = rng.randint(5, 9)- modify_10.append((idx, target))-- all_modifications = modify_3 + modify_10- print(f"\nModifications planned:")- print(f" 3-tool → 1 or 2: {len(modify_3):,}")- print(f" 10-tool → 5-9: {len(modify_10):,}")- print(f" Total: {len(all_modifications):,}")-- if dry_run:- # Preview target distribution- target_counts = Counter()- for n in sorted(indices_by_n.keys()):- target_counts[n] = len(indices_by_n[n])- for idx, target in all_modifications:- ex = ds[idx]- try:- tools = json.loads(ex["tools"])- orig_n = len(tools)- except:- continue- target_counts[orig_n] -= 1- target_counts[target] += 1-- print("\nProjected distribution:")- for n in sorted(target_counts.keys()):- if n <= 10 and target_counts[n] > 0:- total = sum(target_counts.values())- print(f" {n:>2} tools: {target_counts[n]:>10,} ({target_counts[n]/total*100:5.1f}%)")- return-- # Apply modifications- mod_map = {idx: target for idx, target in all_modifications}- modified_indices = set(mod_map.keys())-- new_tools_col = []- skipped = 0- for i in tqdm(range(len(ds)), desc="Applying"):- ex = ds[i]- if i in modified_indices:- called = _get_called_names(ex["answers"])- new_tools = _trim_tools(ex["tools"], called, mod_map[i], rng)- if new_tools is None:- new_tools_col.append(ex["tools"]) # keep original- skipped += 1- else:- new_tools_col.append(new_tools)- else:- new_tools_col.append(ex["tools"])-- if skipped:- print(f" Skipped {skipped} examples (couldn't trim)")-- # Build new dataset- new_ds = ds.remove_columns(["tools"]).add_column("tools", new_tools_col)- # Reorder columns to match original- new_ds = new_ds.select_columns(ds.column_names)-- # Verify- print("\nAfter rebalancing:")- verify_counts = Counter()- for ex in tqdm(new_ds, desc="Verifying"):- try:- tools = json.loads(ex["tools"])- n = len(tools) if isinstance(tools, list) else 0- except:- n = 0- verify_counts[n] += 1- for n in sorted(verify_counts.keys()):- if n <= 10:- total = sum(verify_counts.values())- print(f" {n:>2} tools: {verify_counts[n]:>10,} ({verify_counts[n]/total*100:5.1f}%)")-- # Save locally then upload via HfApi to avoid push_to_hub dataset card bug- import tempfile- from huggingface_hub import HfApi, CommitOperationDelete-- local_dir = tempfile.mkdtemp(prefix="rebalanced_")- print(f"\nSaving to {local_dir}...")- new_ds.to_parquet(os.path.join(local_dir, "train.parquet"))-- api = HfApi()-- # Delete old train shards- files = api.list_repo_files(HF_REPO, repo_type="dataset", token=True)- old_train = [f for f in files if f.startswith("data/train-")]- if old_train:- print(f"Deleting {len(old_train)} old train shards...")- ops = [CommitOperationDelete(path_in_repo=f) for f in old_train]- api.create_commit(- repo_id=HF_REPO, repo_type="dataset", operations=ops,- commit_message="Remove old train shards", token=True,- )-- # Upload new train data- print(f"Uploading rebalanced train split...")- api.upload_file(- path_or_fileobj=os.path.join(local_dir, "train.parquet"),- path_in_repo="data/train-00000-of-00001.parquet",- repo_id=HF_REPO, repo_type="dataset", token=True,- commit_message="Upload rebalanced train data",- )-- # Clean up- import shutil- shutil.rmtree(local_dir)- print("Done.")---def main(args):- rebalance(dry_run=args.dry_run)---if __name__ == "__main__":- import argparse as _ap- _p = _ap.ArgumentParser()- _p.add_argument("--dry-run", action="store_true", help="Preview without modifying")- main(_p.parse_args())diff --git a/src/split_dataset.py b/src/split_dataset.pydeleted file mode 100644index 723bf1c..0000000--- a/src/split_dataset.py+++ /dev/null@@ -1,115 +0,0 @@-"""Split the existing Cactus-Compute/tool-calls dataset into train/validation splits.--Shuffles the full dataset with a fixed seed, then takes 10k rows as validation.-This ensures validation is representative of all data sources, not just the latest append.-Uploads via HfApi to avoid the push_to_hub dataset card bug.--Usage:- python scripts/split_dataset.py-"""--import os-import shutil-import tempfile--HF_DATASET_REPO = "Cactus-Compute/tool-calls"-VAL_PER_SOURCE = {- "gemini-3.1-flash-lite": 5000,- "xlam": 2500,- "xlam-translated": 2500,-}-SEED = 42---def main(args=None):- val_per_source = dict(VAL_PER_SOURCE)- if args is not None and getattr(args, "val_per_source", None) is not None:- # CLI override applies uniformly to all sources- val_per_source = {k: args.val_per_source for k in val_per_source}-- print(f"Downloading full dataset from {HF_DATASET_REPO} (train split only)...")- from .data import download_hf_split- ds = download_hf_split("train", HF_DATASET_REPO)- n = len(ds)- print(f"Total rows: {n:,}")-- # Stratified split: equal samples per source for balanced validation- from collections import Counter- source_counts = Counter(ds["source"])- print(f"\nSource distribution:")- for src, cnt in source_counts.most_common():- print(f" {src}: {cnt:,}")-- # Group indices by source- source_indices = {}- for i, src in enumerate(ds["source"]):- source_indices.setdefault(src, []).append(i)-- import random- rng = random.Random(SEED)-- val_indices = []- for src, indices in sorted(source_indices.items()):- rng.shuffle(indices)- take = min(val_per_source.get(src, 0), len(indices))- val_indices.extend(indices[:take])- print(f" {src}: taking {take} for validation")-- val_set = set(val_indices)- train_indices = [i for i in range(n) if i not in val_set]-- train_ds = ds.select(train_indices)- val_ds = ds.select(val_indices)-- print(f"\nTrain: {len(train_ds):,} rows")- print(f"Validation: {len(val_ds):,} rows")-- val_sources = Counter(val_ds["source"])- print(f"\nValidation source distribution:")- for src, cnt in val_sources.most_common():- print(f" {src}: {cnt} ({cnt/len(val_ds)*100:.1f}%)")-- # Upload via HfApi (avoids push_to_hub dataset card bug)- from huggingface_hub import HfApi, CommitOperationDelete-- api = HfApi()- api.create_repo(HF_DATASET_REPO, repo_type="dataset", private=False, exist_ok=True)-- parquet_dir = tempfile.mkdtemp(prefix="needle_split_")- print(f"\nExporting to parquet...")- train_ds.to_parquet(os.path.join(parquet_dir, "train.parquet"))- val_ds.to_parquet(os.path.join(parquet_dir, "validation.parquet"))-- # Delete old data shards- files = api.list_repo_files(HF_DATASET_REPO, repo_type="dataset", token=True)- old_shards = [f for f in files if f.startswith("data/")]- if old_shards:- print(f"Deleting {len(old_shards)} old shards...")- ops = [CommitOperationDelete(path_in_repo=f) for f in old_shards]- api.create_commit(- repo_id=HF_DATASET_REPO, repo_type="dataset", operations=ops,- commit_message="Remove old shards before split upload", token=True,- )-- print(f"Uploading train split ({len(train_ds):,} rows)...")- api.upload_file(- path_or_fileobj=os.path.join(parquet_dir, "train.parquet"),- path_in_repo="data/train-00000-of-00001.parquet",- repo_id=HF_DATASET_REPO, repo_type="dataset", token=True,- commit_message=f"Upload train split ({len(train_ds):,} rows)",- )-- print(f"Uploading validation split ({len(val_ds):,} rows)...")- api.upload_file(- path_or_fileobj=os.path.join(parquet_dir, "validation.parquet"),- path_in_repo="data/validation-00000-of-00001.parquet",- repo_id=HF_DATASET_REPO, repo_type="dataset", token=True,- commit_message=f"Upload validation split ({len(val_ds):,} rows)",- )-- shutil.rmtree(parquet_dir)- print("Done.")---if __name__ == "__main__":- main()diff --git a/src/train.py b/src/train.pyindex e104840..6f2deeb 100644--- a/src/train.py+++ b/src/train.py@@ -1163,23 +1163,4 @@ def train(args):if is_main:print("\nTraining complete.")- # Auto-calibrate confidence head on best checkpoint- if is_main and getattr(args, "calibrate", True) and best_ckpt_path:- print(f"\n{'='*50}")- print(f"Running confidence head calibration on {best_ckpt_path}")- print(f"{'='*50}")- from .calibrate import main as calibrate_main- from argparse import Namespace- cal_args = Namespace(- checkpoint=best_ckpt_path,- output=best_ckpt_path,- batch_size=args.batch_size * 4,- num_samples=None, # use full dataset- epochs=getattr(args, "calibrate_epochs", 1),- lr=getattr(args, "calibrate_lr", 1e-3),- k=getattr(args, "calibrate_k", 3.0),- )- calibrate_main(cal_args)- print(f"\nCalibration complete. Checkpoint updated: {best_ckpt_path}")-diff --git a/src/train_corrector.py b/src/train_corrector.pydeleted file mode 100644index d39bee2..0000000--- a/src/train_corrector.py+++ /dev/null@@ -1,854 +0,0 @@-"""Stage-2 correction fine-tuning.--Reads the JSONL produced by src.mine_corrections, tokenizes on-the-fly,-and fine-tunes a SEPARATE corrector model initialized from an SFT-checkpoint. The base SFT ckpt is not modified. Inference at deploy time-runs both models in sequence (base -> draft -> corrector).--Encoder input: [query, <tools>, tools_json, <tool_call>, draft_json]-Decoder input: [EOS, <tool_call>, gold_answer]-Decoder target: [<tool_call>, gold_answer, EOS]--The encoder uses <tool_call> (already in the tokenizer vocab) as the-tools->draft delimiter — no tokenizer changes required.--Eval mirrors train.py exactly: same stratified single/multi-call sampling-(50 per tool-count bucket, rng seed 42), same tool-call metrics, same-display samples. At init we run the frozen base SFT ckpt once over the-full val set and cache the drafts. Every subsequent eval step just-re-runs the CORRECTOR on those cached drafts — fast.--The eval report prints both `draft` (base only, baseline) and-`corrected` (base + corrector) side by side, so the improvement or-regression per bucket is a direct read.--Usage:- python -m src.train_corrector \\- --base-checkpoint checkpoints/needle.pkl \\- --corrector-init checkpoints/needle.pkl \\- --data data/corrections.jsonl \\- --epochs 1 --lr 1e-4 --batch-size 64 \\- --wandb-"""--import argparse-import json-import math-import os-import pickle-import queue-import threading-import time-from collections import Counter--import jax-import jax.numpy as jnp-import numpy as np-import optax-from tqdm import tqdm--from .data import (- _token_classes_for_answer, get_tokenizer, load_tool_calls,-)-from .distributed import (- _replicate, _unreplicate, shard_batch, _upload_checkpoint,-)-from .model import (- EncoderDecoderTransformer, TransformerConfig,- make_causal_mask, make_padding_mask,-)-from .optim import create_train_state, _wsd_schedule-from .run import generate_batch, load_checkpoint-from .tokenizer import EOS_ID, PAD_ID, TOOL_CALL_ID, TOOLS_ID---_LOSS_WEIGHT_MAP = jnp.array([1.0, 2.0, 4.0, 1.5], dtype=jnp.float32)---# ───────────────────────── filter: unrecoverable hallucinations ─────────────────────────--import re as _re--_ID_WITH_DIGIT = _re.compile(r"[a-zA-Z]_[a-zA-Z0-9]*\d")-_OPAQUE_CODE = _re.compile(r"^[A-Za-z0-9_-]+$")---def _looks_id_like(value):- """Detect ID-shaped values: emails, URLs, paths, coded names, compound IDs."""- if not isinstance(value, str) or len(value) < 4:- return False- if "@" in value:- left, _, right = value.rpartition("@")- if left and "." in right:- return True- if value.startswith(("http://", "https://", "www.")):- return True- if "/" in value or "\\" in value:- return True- if _ID_WITH_DIGIT.search(value):- return True- if len(value) > 15 and _OPAQUE_CODE.match(value) and any(c.isdigit() for c in value):- return True- if value.count("_") >= 2 and len(value) >= 10:- return True- return False---def _value_in_query(value, query):- if not isinstance(value, str):- return True- return value.lower() in query.lower()---def _gold_has_unrecoverable(query, gold_text):- """True if gold contains any ID-shaped arg value that isn't in the query."""- try:- gold_calls = json.loads(gold_text)- except (json.JSONDecodeError, TypeError):- return False- if not isinstance(gold_calls, list):- return False- for call in gold_calls:- if not isinstance(call, dict):- continue- args = call.get("arguments", {}) or {}- if not isinstance(args, dict):- continue- for _, value in args.items():- if _looks_id_like(value) and not _value_in_query(value, query):- return True- return False---# ───────────────────────── encoder format ─────────────────────────--def _build_correction_encoder_input(tokenizer, query, tools, draft, max_enc_len,- query_cap=256, draft_min=128):- """Build [query, <tools>, tools_json, <tool_call>, draft_json] up to max_enc_len.-- Truncation policy: query gets up to `query_cap` tokens, then tools and- draft share the remaining budget with tools getting priority but- always leaving at least `draft_min` tokens for the draft.- """- q_toks = tokenizer.encode(query)[:query_cap]- t_toks = tokenizer.encode(tools)- d_toks = tokenizer.encode(draft)-- budget = max_enc_len - len(q_toks) - 2 # two separator tokens- tools_cap = max(0, budget - draft_min)- t_toks = t_toks[:min(len(t_toks), tools_cap)]- budget -= len(t_toks)- d_toks = d_toks[:max(0, budget)]- return q_toks + [TOOLS_ID] + t_toks + [TOOL_CALL_ID] + d_toks---def _tokenize_example(tokenizer, sp_model, row, max_enc_len, max_dec_len):- """Build padded (enc, dec_in, dec_tgt, loss_mask) arrays for one row.-- Returns None if the row is filtered (unrecoverable hallucination) or the- decoder target doesn't fit in max_dec_len.- """- # Retroactive cleanup: drafts from pre-fix mining runs have a <tool_call>- # prefix that would re-encode to a duplicate TOOL_CALL_ID token. Strip it.- draft = row.get("draft", "")- if isinstance(draft, str) and draft.startswith("<tool_call>"):- draft = draft[len("<tool_call>"):]-- # Drop rows where the gold fabricates an ID-shaped value that isn't in the- # query. Pass-through correct rows are kept unconditionally — they teach- # "don't change a correct answer" even if the gold has an ID-looking value.- if row.get("category") != "correct":- if _gold_has_unrecoverable(row.get("query", ""), row.get("answers", "")):- return None-- enc_tokens = _build_correction_encoder_input(- tokenizer, row["query"], row["tools"], draft, max_enc_len)-- answer = row["answers"]- a_toks = tokenizer.encode(answer)- if 2 + len(a_toks) + 1 > max_dec_len:- return None-- dec_in_tokens = [EOS_ID, TOOL_CALL_ID] + list(a_toks)- dec_tgt_tokens = [TOOL_CALL_ID] + list(a_toks) + [EOS_ID]- loss_classes_full = np.zeros(len(dec_tgt_tokens), dtype=np.int8)- token_classes = _token_classes_for_answer(answer, a_toks, sp_model)- loss_classes_full[1:1 + len(token_classes)] = token_classes-- enc = np.full(max_enc_len, PAD_ID, dtype=np.int32)- enc[:len(enc_tokens)] = enc_tokens- dec_in = np.full(max_dec_len, PAD_ID, dtype=np.int32)- dec_in[:len(dec_in_tokens)] = dec_in_tokens- dec_tgt = np.full(max_dec_len, PAD_ID, dtype=np.int32)- dec_tgt[:len(dec_tgt_tokens)] = dec_tgt_tokens- loss = np.zeros(max_dec_len, dtype=np.int32)- loss[:len(loss_classes_full)] = loss_classes_full- return enc, dec_in, dec_tgt, loss---# ───────────────────────── streaming loader ─────────────────────────--def _read_jsonl_shuffled(path, seed):- """Generator over (row_dict) in shuffled order. Loads full index into memory."""- # Build line-offset index for random access- offsets = []- with open(path, "rb") as f:- pos = 0- for line in f:- offsets.append(pos)- pos += len(line)- rng = np.random.RandomState(seed)- order = rng.permutation(len(offsets))- with open(path, "rb") as f:- for idx in order:- f.seek(offsets[idx])- line = f.readline()- try:- yield json.loads(line)- except json.JSONDecodeError:- continue---def _stream_batches(tokenizer, sp_model, path, batch_size,- max_enc_len, max_dec_len, seed):- """Yield (enc, dec_in, dec_tgt, loss_mask) numpy batches from a JSONL file."""- enc_batch = np.full((batch_size, max_enc_len), PAD_ID, dtype=np.int32)- dec_in_batch = np.full((batch_size, max_dec_len), PAD_ID, dtype=np.int32)- dec_tgt_batch = np.full((batch_size, max_dec_len), PAD_ID, dtype=np.int32)- loss_batch = np.zeros((batch_size, max_dec_len), dtype=np.int32)- idx = 0-- for row in _read_jsonl_shuffled(path, seed):- result = _tokenize_example(tokenizer, sp_model, row,- max_enc_len, max_dec_len)- if result is None:- continue- enc, di, dt, lm = result- enc_batch[idx] = enc- dec_in_batch[idx] = di- dec_tgt_batch[idx] = dt- loss_batch[idx] = lm- idx += 1- if idx == batch_size:- yield (enc_batch.copy(), dec_in_batch.copy(),- dec_tgt_batch.copy(), loss_batch.copy())- enc_batch[:] = PAD_ID- dec_in_batch[:] = PAD_ID- dec_tgt_batch[:] = PAD_ID- loss_batch[:] = 0- idx = 0---class _PrefetchStream:- def __init__(self, gen_fn, prefetch=4):- self._queue = queue.Queue(maxsize=prefetch)- self._stop = threading.Event()- self._thread = threading.Thread(target=self._produce, args=(gen_fn,), daemon=True)- self._thread.start()-- def _produce(self, gen_fn):- try:- for batch in gen_fn():- if self._stop.is_set():- return- self._queue.put(batch)- self._queue.put(None)- except Exception as e:- self._queue.put(e)-- def __iter__(self): return self- def __next__(self):- item = self._queue.get()- if item is None:- raise StopIteration- if isinstance(item, Exception):- raise item- return item-- def close(self):- self._stop.set()---# ───────────────────────── train step ─────────────────────────--def _train_step(state, src, tgt_in, tgt_out, loss_mask, rng):- """CE + z_loss with token-class weighting. Mirrors train.py."""-- def loss_fn(params):- src_mask = make_padding_mask(src, PAD_ID)- tgt_mask = make_causal_mask(tgt_in.shape[1]) & make_padding_mask(tgt_in, PAD_ID)- logits = state.apply_fn(- {"params": params}, src, tgt_in,- src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=src_mask,- )- logits_f32 = logits.astype(jnp.float32)- token_weights = _LOSS_WEIGHT_MAP[loss_mask]- padding_mask = (tgt_out != PAD_ID).astype(jnp.float32)- mask = token_weights * padding_mask- num_tokens = jnp.maximum(jnp.sum(padding_mask), 1.0)- ce = jnp.sum(- optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask- ) / num_tokens- z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)- return ce + z_loss-- loss, grads = jax.value_and_grad(loss_fn)(state.params)- grads = jax.lax.pmean(grads, axis_name="batch")- loss = jax.lax.pmean(loss, axis_name="batch")- grad_norm = optax.global_norm(grads)- state = state.apply_gradients(grads=grads)- return state, loss, grad_norm---# ───────────────────────── eval: stratified sampling ─────────────────────────--def _stratified_val_pools(val_ds, per_bucket, seed):- """Return (single_examples, multi_examples) sampled identically to train.py.-- Same bucketing by min(len(tools), 10), same rng math.- """- pool_names = ["single", "multi"]- buckets = {n: {t: [] for t in range(11)} for n in pool_names}-- for k in range(len(val_ds)):- ex = val_ds[k]- try:- answers = json.loads(ex["answers"])- except (ValueError, TypeError):- continue- if not answers or not isinstance(answers, list):- continue- call_type = "single" if len(answers) == 1 else "multi"- try:- nc = min(len(json.loads(ex["tools"])), 10)- except (ValueError, TypeError):- nc = 0- buckets[call_type][nc].append(k)-- rng = np.random.RandomState(seed)- out = {}- for name in pool_names:- pool = []- for t in range(11):- b = np.array(buckets[name][t])- if len(b) > 0:- rng.shuffle(b)- pool.extend(b[:per_bucket].tolist())- rng.shuffle(pool)- out[name] = [val_ds[k] for k in pool]- return out["single"], out["multi"]---# ───────────────────────── eval: metrics ─────────────────────────--_PC_KEYS = ("n", "exact", "name_tp", "name_fp", "name_fn",- "call_tp", "call_fp", "call_fn", "parse_err")---def _call_key(c):- if not isinstance(c, dict):- return None- return json.dumps({"name": c.get("name"), "arguments": c.get("arguments")},- sort_keys=True)---def _score_pool(examples, preds):- """Compute tool-call metrics for one pool. Mirrors train.py._eval_pool."""- m_n = m_exact = m_parse_err = 0- m_name_tp = m_name_fp = m_name_fn = 0- m_call_tp = m_call_fp = m_call_fn = 0- m_args_correct = m_args_total = 0- m_halluc = m_total_pred_params = 0- m_missing = m_total_ref_params = 0- m_correct_values = m_matched_params = 0- m_per_count = {t: {k: 0 for k in _PC_KEYS} for t in range(11)}-- for ex, pred_text in zip(examples, preds):- try:- tool_defs = json.loads(ex["tools"])- num_tools = min(len(tool_defs), 10)- except (ValueError, TypeError):- tool_defs = []- num_tools = 0- pc = m_per_count[num_tools]-- ref_text = ex["answers"].strip()- pred_text = pred_text.strip()- ref_is_empty = ref_text in ("", "[]")- pred_is_empty = pred_text in ("", "[]")-- try:- ref_calls = json.loads(ref_text) if not ref_is_empty else []- except (ValueError, TypeError):- ref_calls = []- try:- pred_calls = json.loads(pred_text) if not pred_is_empty else []- if not isinstance(pred_calls, list):- pred_calls = [pred_calls] if isinstance(pred_calls, dict) else []- except (ValueError, TypeError):- m_parse_err += 1- pc["parse_err"] += 1- pred_calls = []-- m_n += 1- pc["n"] += 1- if ref_is_empty and pred_is_empty:- m_exact += 1- pc["exact"] += 1- elif not ref_is_empty and not pred_is_empty:- r = sorted([_call_key(c) for c in ref_calls if _call_key(c)])- p = sorted([_call_key(c) for c in pred_calls if _call_key(c)])- if r == p and len(r) == len(ref_calls) and len(p) == len(pred_calls):- m_exact += 1- pc["exact"] += 1-- ref_names = {c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c}- pred_names = {c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c}- m_name_tp += len(pred_names & ref_names); pc["name_tp"] += len(pred_names & ref_names)- m_name_fp += len(pred_names - ref_names); pc["name_fp"] += len(pred_names - ref_names)- m_name_fn += len(ref_names - pred_names); pc["name_fn"] += len(ref_names - pred_names)-- rk = {_call_key(c) for c in ref_calls} - {None}- pk = {_call_key(c) for c in pred_calls} - {None}- m_call_tp += len(pk & rk); pc["call_tp"] += len(pk & rk)- m_call_fp += len(pk - rk); pc["call_fp"] += len(pk - rk)- m_call_fn += len(rk - pk); pc["call_fn"] += len(rk - pk)-- ref_by_name = {}- for c in ref_calls:- if isinstance(c, dict) and "name" in c:- ref_by_name.setdefault(c["name"], []).append(c.get("arguments", {}))- for c in pred_calls:- if isinstance(c, dict) and "name" in c and c["name"] in ref_by_name:- m_args_total += 1- pa = json.dumps(c.get("arguments", {}), sort_keys=True)- if any(pa == json.dumps(ra, sort_keys=True) for ra in ref_by_name[c["name"]]):- m_args_correct += 1-- tool_param_map = {}- for t in tool_defs:- if isinstance(t, dict) and "name" in t:- tool_param_map[t["name"]] = set((t.get("parameters") or {}).keys())- for c in pred_calls:- if not isinstance(c, dict) or "name" not in c:- continue- cname = c["name"]- if cname not in tool_param_map:- continue- pred_keys = set((c.get("arguments") or {}).keys())- m_total_pred_params += len(pred_keys)- m_halluc += len(pred_keys - tool_param_map[cname])- if cname in ref_by_name:- ref_args = ref_by_name[cname][0]- ref_keys = set((ref_args if isinstance(ref_args, dict) else {}).keys())- m_total_ref_params += len(ref_keys)- m_missing += len(ref_keys - pred_keys)- matched = pred_keys & ref_keys- m_matched_params += len(matched)- for k in matched:- if json.dumps(c.get("arguments", {})[k], sort_keys=True) == \- json.dumps(ref_args[k], sort_keys=True):- m_correct_values += 1-- if m_n == 0:- return {}, m_per_count- np_ = m_name_tp + m_name_fp- nr_ = m_name_tp + m_name_fn- cp_ = m_call_tp + m_call_fp- cr_ = m_call_tp + m_call_fn- return {- "n": m_n,- "parse_rate": 1.0 - m_parse_err / m_n,- "exact_match": m_exact / m_n,- "name_f1": 2 * m_name_tp / max(np_ + nr_, 1),- "call_f1": 2 * m_call_tp / max(cp_ + cr_, 1),- "args_acc": m_args_correct / max(m_args_total, 1),- "param_haluc": m_halluc / max(m_total_pred_params, 1),- "param_miss": m_missing / max(m_total_ref_params, 1),- "value_acc": m_correct_values / max(m_matched_params, 1),- }, m_per_count---def _print_metrics(label, metrics, pc):- if not metrics:- return- print(f" ─── {label} ({metrics['n']} samples) ──")- for k, key in [("JSON parse", "parse_rate"), ("Name F1", "name_f1"),- ("Param haluc", "param_haluc"), ("Param miss", "param_miss"),- ("Value acc", "value_acc"), ("Args acc", "args_acc"),- ("Call F1", "call_f1"), ("Exact match", "exact_match")]:- print(f" {k:<14} {metrics[key]:>10.1%}")- has_any = any(pc[t]["n"] > 0 for t in range(11))- if has_any:- print(f" {'#tools':>6} {'n':>4} {'name_f1':>8} {'call_f1':>8} {'exact':>6}")- for t in range(11):- d = pc[t]- if d["n"] == 0:- continue- np_ = d["name_tp"] + d["name_fp"]; nr_ = d["name_tp"] + d["name_fn"]- cp_ = d["call_tp"] + d["call_fp"]; cr_ = d["call_tp"] + d["call_fn"]- nf1 = 2 * d["name_tp"] / max(np_ + nr_, 1)- cf1 = 2 * d["call_tp"] / max(cp_ + cr_, 1)- ex = d["exact"] / d["n"]- print(f" {t:>6} {d['n']:>4} {nf1:>7.1%} {cf1:>7.1%} {ex:>5.1%}")---def _print_comparison(label, draft_m, corr_m):- """Print side-by-side draft vs corrected metrics."""- if not draft_m or not corr_m:- return- print(f"\n ─── {label} — draft vs corrected ───")- print(f" {'metric':<14} {'draft':>8} {'corr':>8} {'Δ':>8}")- for k, key in [("JSON parse", "parse_rate"), ("Name F1", "name_f1"),- ("Param haluc", "param_haluc"), ("Param miss", "param_miss"),- ("Value acc", "value_acc"), ("Args acc", "args_acc"),- ("Call F1", "call_f1"), ("Exact match", "exact_match")]:- d, c = draft_m[key], corr_m[key]- sign = "+" if c >= d else ""- print(f" {k:<14} {d:>8.1%} {c:>8.1%} {sign}{c-d:>7.1%}")---# ───────────────────────── corrector generation ─────────────────────────--def _generate_corrector_batch(model, params, tokenizer, queries, tools_list,- drafts, max_gen_len, max_enc_len,- batch_size=32, desc="corrector-gen"):- """Greedy generate with encoder=[query, <tools>, tools, <tool_call>, draft]."""- from .run import _get_decode_fn- pad_id = PAD_ID- eos_id = EOS_ID- decode_fn = _get_decode_fn(model, max_gen_len)-- all_preds = []- n_total = len(queries)- pbar = tqdm(total=n_total, desc=desc, unit="ex", leave=False)- for start in range(0, n_total, batch_size):- chunk_q = queries[start:start + batch_size]- chunk_t = tools_list[start:start + batch_size]- chunk_d = drafts[start:start + batch_size]- B = len(chunk_q)-- enc_token_lists = [- _build_correction_encoder_input(tokenizer, q, t, d, max_enc_len)- for q, t, d in zip(chunk_q, chunk_t, chunk_d)- ]- max_enc = max(len(toks) for toks in enc_token_lists)- enc_input = np.full((B, max_enc), pad_id, dtype=np.int32)- for i, toks in enumerate(enc_token_lists):- enc_input[i, :len(toks)] = toks- enc_input = jnp.array(enc_input)- src_mask = make_padding_mask(enc_input, pad_id)-- encoder_out, enc_mask = model.apply(- {"params": params}, enc_input, src_mask=src_mask, method="encode")-- dec_buffer = np.full((B, max_gen_len), pad_id, dtype=np.int32)- dec_buffer[:, 0] = eos_id- dec_buffer = jnp.array(dec_buffer)-- finished = [False] * B- gen_tokens = [[] for _ in range(B)]- logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)-- for pos in range(max_gen_len - 1):- for i in range(B):- if finished[i]:- continue- next_token = int(jnp.argmax(logits[i, pos]))- if next_token == eos_id:- finished[i] = True- continue- gen_tokens[i].append(next_token)- dec_buffer = dec_buffer.at[i, pos + 1].set(next_token)- if all(finished):- break- logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)-- for i in range(B):- text = tokenizer.decode(gen_tokens[i])- if text.startswith("<tool_call>"):- text = text[len("<tool_call>"):]- all_preds.append(text)- pbar.update(B)- pbar.close()- return all_preds---# ───────────────────────── main ─────────────────────────--def train_corrector(args):- if os.path.exists("/dev/accel0"):- jax.distributed.initialize()-- num_devices = jax.local_device_count()- num_hosts = jax.process_count()- host_id = jax.process_index()- is_main = host_id == 0-- use_wandb = args.wandb and is_main- if use_wandb:- import wandb- if wandb.run is None:- wandb.init(project="needle-v1", name=args.name, config=vars(args))-- print(f"[1/5] Devices: {num_devices} local, {jax.device_count()} total")-- print(f"[2/5] Loading tokenizer and SP model ...")- tokenizer = get_tokenizer()- sp_model = tokenizer.sp # SentencePiece processor for class computation-- # ─── load base (frozen, for drafts) and corrector init ───- print(f"[3/5] Loading base SFT checkpoint: {args.base_checkpoint}")- base_params, base_config = load_checkpoint(args.base_checkpoint)- base_model = EncoderDecoderTransformer(base_config)-- print(f" Loading corrector init: {args.corrector_init}")- corr_params, corr_config = load_checkpoint(args.corrector_init)- corr_model = EncoderDecoderTransformer(corr_config)- param_count = sum(x.size for x in jax.tree.leaves(corr_params))- print(f" Corrector parameters: {param_count:,}")-- # ─── sample val pools (train.py-compatible) and cache base drafts ───- print(f"[4/5] Sampling stratified val pools and caching base drafts ...")- val_ds = load_tool_calls("val")- single_pool, multi_pool = _stratified_val_pools(- val_ds, per_bucket=args.tc_per_bucket, seed=args.eval_seed)- print(f" single-call: {len(single_pool)}, multi-call: {len(multi_pool)}")-- def _gen_drafts(pool, label, chunk=64):- queries = [ex["query"] for ex in pool]- tools_list = [ex["tools"] for ex in pool]- print(f" Generating {label} drafts from base model ...")- t0 = time.perf_counter()- params_j = jax.tree.map(jnp.array, base_params)- drafts = []- pbar = tqdm(total=len(queries), desc=f"base-draft-{label}",- unit="ex", leave=False)- for i in range(0, len(queries), chunk):- drafts.extend(generate_batch(- base_model, params_j, tokenizer,- queries[i:i + chunk], tools_list[i:i + chunk],- max_gen_len=args.max_gen_len, max_enc_len=args.max_enc_len,- constrained=False))- pbar.update(min(chunk, len(queries) - i))- pbar.close()- print(f" {label}: {len(drafts)} drafts in {time.perf_counter()-t0:.1f}s")- return drafts-- single_drafts = _gen_drafts(single_pool, "single") if is_main else []- multi_drafts = _gen_drafts(multi_pool, "multi") if is_main else []-- # Free base model weights — no longer needed- del base_params, base_model- import gc; gc.collect()-- # ─── estimate total steps for schedule ───- print(f"[5/5] Building optimizer + pmap ...")- with open(args.data) as f:- n_rows = sum(1 for _ in f)- effective_batch = args.batch_size * num_devices- global_batch = effective_batch * num_hosts- steps_per_epoch = n_rows // global_batch- total_steps = steps_per_epoch * args.epochs- warmup = max(1, int(total_steps * 0.05))- if args.max_steps:- total_steps = min(total_steps, args.max_steps)-- # ─── build state ───- scaled_lr = args.lr * jax.device_count()- muon_lr = 0.02 * math.sqrt(jax.device_count())- rng = jax.random.PRNGKey(args.seed)- rng, init_rng = jax.random.split(rng)- state = create_train_state(init_rng, corr_config, scaled_lr, muon_lr,- total_steps, warmup, decay_ratio=0.05)-- # Load corrector init weights into state- corr_params_cast = jax.tree.map(- lambda ref, loaded: jnp.asarray(loaded, dtype=ref.dtype),- state.params, jax.tree.map(jnp.array, corr_params))- state = state.replace(params=corr_params_cast)- del corr_params, corr_params_cast-- state = _replicate(state)- p_train_step = jax.pmap(_train_step, axis_name="batch", donate_argnums=(0,))-- adam_schedule = _wsd_schedule(scaled_lr, total_steps, warmup)-- if is_main:- print(f"\n ─────────────────────────────────────")- print(f" Corrector fine-tune")- print(f" ─────────────────────────────────────")- print(f" Train rows {n_rows:>12,}")- print(f" Steps/epoch {steps_per_epoch:>12,}")- print(f" Total steps {total_steps:>12,}")- print(f" Epochs {args.epochs:>12}")- print(f" Batch {args.batch_size:>4} x {jax.device_count()} = {global_batch}")- print(f" LR {args.lr:>7} x {jax.device_count()} = {scaled_lr}")- print(f" Encoder max {args.max_enc_len:>12}")- print(f" Decoder max {args.max_dec_len:>12}")- print(f" ─────────────────────────────────────\n")-- # ─── baseline eval (draft-only metrics) ───- baseline_single_m = baseline_multi_m = None- baseline_single_pc = baseline_multi_pc = None- if is_main:- print("Baseline eval (draft-only, no correction) ...")- baseline_single_m, baseline_single_pc = _score_pool(single_pool, single_drafts)- baseline_multi_m, baseline_multi_pc = _score_pool(multi_pool, multi_drafts)- _print_metrics("BASELINE Single-Call", baseline_single_m, baseline_single_pc)- _print_metrics("BASELINE Multi-Call", baseline_multi_m, baseline_multi_pc)-- os.makedirs(args.checkpoint_dir, exist_ok=True)- global_step = 0- best_call_f1 = -1.0- best_path = None-- def _run_eval():- nonlocal best_call_f1, best_path- if not is_main:- return- eval_params = _unreplicate(state).params- preds_single = _generate_corrector_batch(- corr_model, eval_params, tokenizer,- [ex["query"] for ex in single_pool],- [ex["tools"] for ex in single_pool],- single_drafts,- max_gen_len=args.max_gen_len, max_enc_len=args.max_enc_len,- desc=f"eval-step{global_step}-single")- preds_multi = _generate_corrector_batch(- corr_model, eval_params, tokenizer,- [ex["query"] for ex in multi_pool],- [ex["tools"] for ex in multi_pool],- multi_drafts,- max_gen_len=args.max_gen_len, max_enc_len=args.max_enc_len,- desc=f"eval-step{global_step}-multi")- single_m, single_pc = _score_pool(single_pool, preds_single)- multi_m, multi_pc = _score_pool(multi_pool, preds_multi)- _print_metrics("CORR Single-Call", single_m, single_pc)- _print_metrics("CORR Multi-Call", multi_m, multi_pc)- _print_comparison("Single-Call", baseline_single_m, single_m)- _print_comparison("Multi-Call", baseline_multi_m, multi_m)-- cur_f1 = single_m.get("call_f1", -1)- if cur_f1 > best_call_f1:- best_call_f1 = cur_f1- best_path = os.path.join(args.checkpoint_dir, "needle_corrector_best.pkl")- params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), eval_params)- with open(best_path, "wb") as f:- pickle.dump({"params": params_np, "config": corr_config.__dict__,- "step": global_step}, f)- print(f" ** New best single call_f1={best_call_f1:.1%} -> {best_path}")- _upload_checkpoint(best_path)-- if use_wandb:- import wandb- log = {"corrector/step": global_step,- "best/single_call_f1": best_call_f1}- for scope, m in (("single", single_m), ("multi", multi_m)):- for k in ("parse_rate", "exact_match", "name_f1", "call_f1",- "args_acc", "value_acc", "param_haluc", "param_miss"):- log[f"val/{scope}_{k}"] = m.get(k, 0)- for scope, m in (("single", baseline_single_m), ("multi", baseline_multi_m)):- for k in ("call_f1", "value_acc", "exact_match"):- log[f"baseline/{scope}_{k}"] = m.get(k, 0)- wandb.log(log)-- # ─── training loop ───- for epoch in range(args.epochs):- stream = _PrefetchStream(- lambda: _stream_batches(tokenizer, sp_model, args.data, global_batch,- args.max_enc_len, args.max_dec_len,- seed=args.seed + epoch),- prefetch=4)- pbar = tqdm(total=steps_per_epoch, desc=f"Corrector epoch {epoch+1}/{args.epochs}",- disable=not is_main)- losses = []- t0 = time.perf_counter()- try:- for batch in stream:- if args.max_steps and global_step >= args.max_steps:- break- src, tgt_in, tgt_out, lm = batch- host_slice = slice(host_id * effective_batch,- (host_id + 1) * effective_batch)- src = src[host_slice]; tgt_in = tgt_in[host_slice]- tgt_out = tgt_out[host_slice]; lm = lm[host_slice]-- src_b = shard_batch(src, num_devices)- tgt_in_b = shard_batch(tgt_in, num_devices)- tgt_out_b = shard_batch(tgt_out, num_devices)- lm_b = shard_batch(lm, num_devices)-- rng, step_rng = jax.random.split(rng)- rngs = jax.random.split(step_rng, num_devices)-- state, loss, gnorm = p_train_step(- state, src_b, tgt_in_b, tgt_out_b, lm_b, rngs)- loss_val = float(loss.addressable_shards[0].data[0])- gnorm_val = float(gnorm.addressable_shards[0].data[0])- losses.append(loss_val)- global_step += 1- pbar.update(1)- pbar.set_postfix(loss=f"{loss_val:.4f}",- gnorm=f"{gnorm_val:.2f}",- lr=f"{float(adam_schedule(global_step)):.2e}")- if use_wandb:- import wandb- wandb.log({"corrector/loss": loss_val,- "corrector/grad_norm": gnorm_val,- "corrector/lr": float(adam_schedule(global_step)),- "corrector/step": global_step})-- finally:- stream.close()- pbar.close()- _run_eval()-- # Final save (last eval already ran at end of final epoch)- if is_main:- final_path = os.path.join(args.checkpoint_dir, "needle_corrector.pkl")- final_params = _unreplicate(state).params- params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), final_params)- with open(final_path, "wb") as f:- pickle.dump({"params": params_np, "config": corr_config.__dict__,- "step": global_step}, f)- print(f"\nFinal corrector: {final_path}")- if best_path:- print(f"Best corrector: {best_path} (call_f1={best_call_f1:.1%})")- _upload_checkpoint(final_path)-- if use_wandb:- import wandb- wandb.finish()- if num_hosts > 1:- jax.experimental.multihost_utils.sync_global_devices("corrector_done")---def parse_args():- p = argparse.ArgumentParser(description="Stage-2 correction fine-tuning")- p.add_argument("--base-checkpoint", type=str, default="checkpoints/needle.pkl",- help="Frozen SFT checkpoint used to generate drafts for eval")- p.add_argument("--corrector-init", type=str, default="checkpoints/needle.pkl",- help="Checkpoint to initialize the corrector from (default: same as base)")- p.add_argument("--data", type=str, default="data/corrections.jsonl")- p.add_argument("--name", type=str, default="corrector")- p.add_argument("--epochs", type=int, default=1)- p.add_argument("--batch-size", type=int, default=32)- p.add_argument("--lr", type=float, default=1e-4)- p.add_argument("--max-steps", type=int, default=None)- p.add_argument("--max-enc-len", type=int, default=1024)- p.add_argument("--max-dec-len", type=int, default=512)- p.add_argument("--max-gen-len", type=int, default=256)- p.add_argument("--tc-per-bucket", type=int, default=50,- help="Eval samples per tool-count bucket (train.py default)")- p.add_argument("--eval-every", type=int, default=500,- help="Run full stratified eval every N steps")- p.add_argument("--eval-seed", type=int, default=42)- p.add_argument("--seed", type=int, default=42)- p.add_argument("--checkpoint-dir", type=str, default="checkpoints")- p.add_argument("--wandb", action="store_true")- return p.parse_args()---if __name__ == "__main__":- train_corrector(parse_args())diff --git a/src/translate_xlam.py b/src/translate_xlam.pydeleted file mode 100644index 71af90d..0000000--- a/src/translate_xlam.py+++ /dev/null@@ -1,414 +0,0 @@-#!/usr/bin/env python3-"""Translate xlam-sourced examples from the unified dataset into 24 non-English-languages using Gemini, then merge back into the dataset.--Translates both the query AND free-text argument values (message text, notes,-search queries, labels, destinations, etc.) so the data matches real-world usage-where a French speaker's message content is also in French. Structured values-(tool names, param keys, contact_ids, enum choices, booleans, numbers) stay-unchanged.--Examples are batched (default 10 per Gemini call) and grouped by language for-efficiency — ~60k xlam × 24 languages = ~1.4M rows in ~144k API calls instead-of 1.4M.--Usage:- python -m src.translate_xlam # translate + upload- python -m src.translate_xlam --dry-run # translate only- python -m src.translate_xlam --max-samples 100 # small test run- python -m src.translate_xlam --workers 16 # more parallelism-"""--import argparse-import concurrent.futures-import json-import os-import random-import sys-import threading--from tqdm import tqdm--try:- from google import genai-except ImportError:- print("Error: google-genai not installed. Run: pip install google-genai", file=sys.stderr)- sys.exit(1)---# Non-English target languages (English already covered by original xlam data)-LANGUAGES = [- "Bulgarian", "Croatian", "Czech", "Danish", "Dutch",- "Estonian", "Finnish", "French", "German", "Greek", "Hungarian",- "Italian", "Latvian", "Lithuanian", "Maltese", "Polish",- "Portuguese", "Romanian", "Slovak", "Slovenian", "Spanish",- "Swedish", "Russian", "Ukrainian",-]--MODEL = "gemini-3.1-flash-lite-preview"--# Param names/patterns where values should NOT be translated-_KEEP_ENGLISH = frozenset({- "contact_id", "password", "url", "source", "confirmation_code",- "order_id", "post_id", "file_name", "archive_name", "folder_name",- "destination_folder", "app_name", "device_name", "ssid",- "site", "username", "email", "to", "cc", "bcc",- "phone", "phone_number", "js",-})--# Enum-like params where the value is from a fixed set — keep as-is-_ENUM_HINTS = frozenset({- "action", "mode", "direction", "position", "camera", "cycle",- "ride_type", "service_type", "trigger", "data_type", "region",- "restriction_level", "level", "unit", "connection_type",- "setting", "goal_type", "input", "connector_type",-})---LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data", "tool_calls_unified")-HF_DATASET_REPO = "Cactus-Compute/tool-calls"---class ClientPool:- """Round-robin pool of Gemini clients."""-- def __init__(self, clients):- self._clients = clients- self._idx = 0- self._lock = threading.Lock()-- def get(self):- with self._lock:- client = self._clients[self._idx % len(self._clients)]- self._idx += 1- return client---def make_clients():- raw = os.environ.get("GEMINI_API_KEY", "")- keys = [k.strip() for k in raw.split(",") if k.strip()]- if not keys:- print("Error: GEMINI_API_KEY not set.", file=sys.stderr)- sys.exit(1)- clients = [genai.Client(api_key=k) for k in keys]- print(f"Using {len(clients)} API key(s)")- return clients---def _should_translate_param(pname, pval):- """Decide if a parameter value should be translated."""- if pname in _KEEP_ENGLISH:- return False- if pname in _ENUM_HINTS:- return False- if not isinstance(pval, str):- return False- # Short values that look like identifiers/codes — skip- if len(pval) < 3:- return False- # Values that look like file paths, URLs, or IDs- if "/" in pval or "@" in pval or pval.startswith(("http", "www")):- return False- # Pure numbers or booleans encoded as strings- try:- float(pval)- return False- except ValueError:- pass- if pval.lower() in ("true", "false", "null", "none"):- return False- return True---def _extract_translatable(example):- """Extract query + translatable arg values from an example.-- Returns (query, translatable_args) where translatable_args is a list of- (call_idx, param_name, param_value) tuples.- """- answers = json.loads(example["answers"])- translatable_args = []- for call_idx, call in enumerate(answers):- args = call.get("arguments", {})- for pname, pval in args.items():- if _should_translate_param(pname, pval):- translatable_args.append((call_idx, pname, pval))- return example["query"], translatable_args---def _apply_translation(example, translated_item, language):- """Apply a translated payload back onto an example."""- answers = json.loads(example["answers"])- query = example["query"]-- new_query = translated_item.get("query", query)-- # Reconstruct translatable_args to map arg_N keys back- _, translatable_args = _extract_translatable(example)- new_answers = json.loads(json.dumps(answers)) # deep copy- for i, (call_idx, pname, _) in enumerate(translatable_args):- key = f"arg_{i}"- if key in translated_item:- new_answers[call_idx]["arguments"][pname] = translated_item[key]-- return {- "query": new_query,- "tools": example["tools"],- "answers": json.dumps(new_answers, separators=(",", ":"), ensure_ascii=False),- "source": "xlam-translated",- "model": example.get("_model_tag", ""),- "language": language,- }---def translate_batch(client_pool, examples, language, model):- """Translate a batch of examples into a single language in one Gemini call.-- Packs all examples into one JSON array prompt, gets back a translated array.- Returns list of translated example dicts (may be shorter than input on errors).- """- # Build batch payload: list of {query, arg_0, arg_1, ...} per example- batch_payload = []- for ex in examples:- query, translatable_args = _extract_translatable(ex)- item = {"query": query}- for i, (_, _, pval) in enumerate(translatable_args):- item[f"arg_{i}"] = pval- batch_payload.append(item)-- prompt = f"""Translate each JSON object's values from English to {language}.--RULES:-- Translate ONLY the string values, not the keys-- Use natural, native-sounding {language} — not literal word-for-word translation-- Use culturally appropriate names, places, and expressions where relevant-- Keep proper nouns (brand names, app names) unchanged-- Keep numbers, dates, and times in their original format-- If a value is already a single common word that works in {language}, keep it-- Return a JSON array with the SAME number of objects in the SAME order-- Each object must have the SAME keys as the input — only values change--INPUT ({len(batch_payload)} items):-{json.dumps(batch_payload, ensure_ascii=False)}--OUTPUT (valid JSON array, no markdown, {len(batch_payload)} items):"""-- client = client_pool.get()- response = client.models.generate_content(- model=model, contents=prompt,- config={"temperature": 0.3, "max_output_tokens": 16384},- )-- text = response.text.strip()- if text.startswith("```"):- text = text.split("\n", 1)[1] if "\n" in text else text[3:]- if text.endswith("```"):- text = text[:-3]-- translated_batch = json.loads(text.strip())- if not isinstance(translated_batch, list):- return []-- # Apply translations back to examples- results = []- model_tag = f"{model}-translate"- for ex, translated_item in zip(examples, translated_batch):- if not isinstance(translated_item, dict):- continue- ex_with_tag = {**ex, "_model_tag": model_tag}- try:- result = _apply_translation(ex_with_tag, translated_item, language)- results.append(result)- except Exception:- continue-- return results---def main(args):- from datasets import Dataset, concatenate_datasets, load_dataset, load_from_disk-- client_pool = ClientPool(make_clients())-- # Load existing dataset- local = os.path.abspath(LOCAL_UNIFIED_DIR)- if os.path.exists(local) and any(f.endswith(".arrow") for f in os.listdir(local)):- ds = load_from_disk(local)- print(f"Loaded existing dataset: {len(ds)} rows")- else:- from .data import download_hf_split- print(f"Downloading from {HF_DATASET_REPO}...")- ds = download_hf_split("train", HF_DATASET_REPO)- os.makedirs(local, exist_ok=True)- ds.save_to_disk(local)- print(f"Downloaded: {len(ds)} rows")-- # Filter to xlam-sourced examples only- xlam_indices = [i for i, src in enumerate(ds["source"]) if "xlam" in src.lower()]- xlam_ds = ds.select(xlam_indices)- print(f"Found {len(xlam_ds)} xlam-sourced examples to translate")-- if args.max_samples and args.max_samples < len(xlam_ds):- xlam_ds = xlam_ds.shuffle(seed=42).select(range(args.max_samples))- print(f"Sampled down to {len(xlam_ds)} examples")-- # Prepare examples as dicts- examples = []- for i in range(len(xlam_ds)):- examples.append({- "query": xlam_ds[i]["query"],- "tools": xlam_ds[i]["tools"],- "answers": xlam_ds[i]["answers"],- })-- # Assign languages round-robin and shuffle to decorrelate from xlam order- rng = random.Random(42)- lang_assignments = [LANGUAGES[i % len(LANGUAGES)] for i in range(len(examples))]- combined = list(zip(examples, lang_assignments))- rng.shuffle(combined)- examples, lang_assignments = zip(*combined)- examples, lang_assignments = list(examples), list(lang_assignments)-- batches = []- lang_buckets = {}- for ex, lang in zip(examples, lang_assignments):- if lang not in lang_buckets:- lang_buckets[lang] = []- lang_buckets[lang].append(ex)- if len(lang_buckets[lang]) >= args.batch_size:- batches.append((lang, lang_buckets[lang]))- lang_buckets[lang] = []-- for lang, remaining in lang_buckets.items():- if remaining:- batches.append((lang, remaining))-- rng.shuffle(batches)- total_examples = sum(len(b) for _, b in batches)- print(f"Prepared {len(batches)} batches ({total_examples} examples, batch_size={args.batch_size})")-- # Translate in parallel — one future per batch- translated = []- failed_batches = 0- failed_examples = 0- pbar = tqdm(total=len(batches), desc="Translating", unit="batch")-- with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool:- futures = {}- for i, (lang, batch_examples) in enumerate(batches):- f = pool.submit(translate_batch, client_pool, batch_examples, lang, args.model)- futures[f] = (i, len(batch_examples))-- for f in concurrent.futures.as_completed(futures):- batch_idx, batch_len = futures[f]- try:- results = f.result()- translated.extend(results)- if len(results) < batch_len:- failed_examples += batch_len - len(results)- except Exception:- failed_batches += 1- failed_examples += batch_len- pbar.update(1)- pbar.set_postfix(- translated=len(translated),- failed_batches=failed_batches,- failed_ex=failed_examples,- )-- pbar.close()-- # Stats- from collections import Counter- lang_counts = Counter(ex["language"] for ex in translated)- print(f"\nTranslated {len(translated)} examples "- f"({failed_batches} failed batches, {failed_examples} failed examples)")- print(f"Language distribution: {dict(lang_counts.most_common())}")-- # Show samples- rng.shuffle(translated)- print(f"\nSample translations:")- for ex in translated[:20]:- print(f" [{ex['language']}] Q: {ex['query'][:120]}")- print(f" A: {ex['answers'][:150]}")- print()-- if args.dry_run:- print("--dry-run: skipping save and upload")- return-- # Merge into existing dataset- new_ds = Dataset.from_dict({- "query": [ex["query"] for ex in translated],- "tools": [ex["tools"] for ex in translated],- "answers": [ex["answers"] for ex in translated],- "source": [ex["source"] for ex in translated],- "model": [ex["model"] for ex in translated],- "language": [ex["language"] for ex in translated],- })-- # Handle schema compatibility- for col in new_ds.column_names:- if col not in ds.column_names:- ds = ds.add_column(col, ["English"] * len(ds) if col == "language" else [""] * len(ds))- new_ds = new_ds.select_columns(ds.column_names)-- merged = concatenate_datasets([ds, new_ds])- print(f"\nMerged: {len(merged)} rows (+{len(new_ds)} translated)")-- for src, cnt in Counter(merged["source"]).most_common():- print(f" {src}: {cnt}")-- # Save locally- import shutil- tmp_dir = local + "_tmp"- if os.path.exists(tmp_dir):- shutil.rmtree(tmp_dir)- merged.save_to_disk(tmp_dir)- if os.path.exists(local):- shutil.rmtree(local)- os.rename(tmp_dir, local)- print("Saved locally.")-- # Upload- import tempfile- from huggingface_hub import HfApi, CommitOperationDelete-- api = HfApi()- api.create_repo(HF_DATASET_REPO, repo_type="dataset", private=False, exist_ok=True)-- parquet_dir = tempfile.mkdtemp(prefix="needle_xlam_translate_")- print("Exporting to parquet...")- merged.to_parquet(os.path.join(parquet_dir, "train.parquet"))-- files = api.list_repo_files(HF_DATASET_REPO, repo_type="dataset", token=True)- old_train = [f for f in files if f.startswith("data/train-")]- if old_train:- print(f"Deleting {len(old_train)} old train shards...")- ops = [CommitOperationDelete(path_in_repo=f) for f in old_train]- api.create_commit(- repo_id=HF_DATASET_REPO, repo_type="dataset", operations=ops,- commit_message="Remove old train shards before re-upload", token=True,- )-- print(f"Uploading to {HF_DATASET_REPO}...")- api.upload_file(- path_or_fileobj=os.path.join(parquet_dir, "train.parquet"),- path_in_repo="data/train-00000-of-00001.parquet",- repo_id=HF_DATASET_REPO, repo_type="dataset", token=True,- commit_message=f"Upload train data with xlam translations ({len(merged)} rows)",- )-- shutil.rmtree(parquet_dir)- print(f"Upload complete: {HF_DATASET_REPO}")- print("NOTE: Run 'needle split-dataset' to create the validation split.")---if __name__ == "__main__":- p = argparse.ArgumentParser(description="Translate xlam examples to 24 languages")- p.add_argument("--max-samples", type=int, default=None, help="Limit examples to translate")- p.add_argument("--workers", type=int, default=8, help="Parallel Gemini calls")- p.add_argument("--model", type=str, default=MODEL, help="Gemini model for translation")- p.add_argument("--batch-size", type=int, default=10, help="Examples per Gemini call")- p.add_argument("--dry-run", action="store_true", help="Translate only, skip save/upload")- main(p.parse_args())