Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

streamline codebase

HenryNdubuaku42ca7ececef1
diff --git a/README.md b/README.md
index c222182..e638d91 100644
--- a/README.md
+++ b/README.md
@@ -65,7 +65,7 @@
┌─────────────────────────────────────────────────────────────┐
│ │
│ Contrastive Head CLIP-style tool retrieval │
- │ encoder → mean_pool → Dense(d_model/4) → ReLU │
+ │ encoder → mean_pool → Dense(d_model/4) → ReLU │
│ → Dense(contrastive_dim) → L2-normalize │
│ learnable temperature (log_temp) │
│ │
@@ -101,11 +101,6 @@
│ │
│ Gemini synthesis → Cactus-Compute/tool-calls (HuggingFace) │
│ parallel workers · dedup · auto-merge · auto-upload │
- │ │
- │ Related: │
- │ needle merge-xlam merge xlam-60k into dataset │
- │ needle rebalance-tools trim over-represented bins │
- │ needle split-dataset create train/val splits │
└─────────────────────────────────────────────────────────────┘
│
▼
@@ -152,17 +147,6 @@
│ └──────────┬───────────┘ └──────────┬───────────┘ │
│ ▼ ▼ │
│ text + contrastive tool-call training (jax.pmap) │
- └─────────────────────────────────────────────────────────────┘
- │
- ▼
- ┌─────────────────────────────────────────────────────────────┐
- │ needle calibrate │
- │ │
- │ Freeze encoder + decoder │
- │ Compute per-example decoder perplexity │
- │ Map PPL → [0,1] confidence via sigmoid calibration │
- │ Train confidence head (2 Dense layers, MSE loss) │
- │ Save calibrated checkpoint with mu, k params │
└─────────────────────────────────────────────────────────────┘
Hybrid Inference (confidence-based cloud routing)
@@ -190,9 +174,9 @@
│ ┌──┴──┐ │ │
│ │ yes │──────────────┤ │
│ └─────┘ ▼ │
- │ ┌──────────┐ │
+ │ ┌──────────┐ │
│ │ Decoder │──▶ tool calls (local) │
- │ └──────────┘ │
+ │ └──────────┘ │
│ ┌─────┐ │
│ │ no │──▶ route to cloud (Gemini) ──▶ tool calls │
│ └─────┘ (skip decoder entirely) │
@@ -214,7 +198,7 @@ needle [command]
│ train │
│ --epochs INT Training epochs (default: 1) │
│ --batch-size INT Batch size (default: 32) │
- │ --lr FLOAT AdamW learning rate (default: 3e-4) │
+ │ --lr FLOAT AdamW learning rate (default: 3e-5) │
│ --muon-lr FLOAT Muon learning rate (default: 0.02) │
│ --d-model INT Model dim (default: 512) │
│ --num-heads INT Attention heads (default: 8) │
@@ -224,7 +208,7 @@ needle [command]
│ --max-enc-len INT Max encoder seq len (default: 1024) │
│ --max-dec-len INT Max decoder seq len (default: 512) │
│ --max-samples INT Training samples (default: all) │
- │ --no-feedforward No FFN layers (default: on) │
+ │ --no-feedforward No FFN layers (default: on) │
│ --feedforward Enable FFN layers (off by default) │
│ --activation STR swiglu|drelu|geglu (if FFN enabled) │
│ --mat-factors INT [...] FFN shrink factors (if FFN enabled) │
@@ -235,7 +219,7 @@ needle [command]
│ --prune-end-frac FL Lock mask at this frac (def: 0.67) │
│ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │
│ --decay-ratio FLOAT LR cosine decay ratio (default: 0.05)│
- │ --dropout FLOAT Dropout rate (default: 0.0) │
+ │ --dropout FLOAT Dropout rate (default: 0.0) │
│ --eval-every INT Val eval interval (default: 1000) │
│ --max-eval-samples INT Val samples limit (default: 5000) │
│ --contrastive-weight FL CLIP loss weight (default: 0.1) │
@@ -277,15 +261,6 @@ needle [command]
│ --tool-call-samples INT Tool-call eval samples (default: 200)│
│ --no-constrained Disable constrained decoding │
│ │
- │ calibrate │
- │ --checkpoint PATH Path to model checkpoint (required) │
- │ --output PATH Output path (default: overwrite) │
- │ --batch-size INT Batch size (default: 32) │
- │ --num-samples INT Limit samples for PPL (default: all) │
- │ --epochs INT Training epochs (default: 10) │
- │ --lr FLOAT Learning rate (default: 1e-3) │
- │ --k FLOAT Sigmoid steepness (default: 3.0) │
- │ │
│ generate-data │
│ --num-samples INT Samples to generate (default: 5000) │
│ --batch-size INT Examples per Gemini call (default:10)│
@@ -295,15 +270,6 @@ needle [command]
│ --output-jsonl PATH Also save raw generations to JSONL │
│ --upload-every INT Merge+upload interval (def: 30000) │
│ │
- │ merge-xlam │
- │ --dry-run Skip upload │
- │ --max-samples INT Limit xlam samples │
- │ │
- │ rebalance-tools │
- │ --dry-run Preview without modifying │
- │ │
- │ split-dataset Create train/val splits + upload │
- │ │
│ evaluate │
│ --checkpoint PATH Path to model checkpoint (required) │
│ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │
@@ -390,21 +356,13 @@ needle [command]
4. Clone the repo on your instance using your PAT
git clone https://github.com/cactus-compute/needle.git
- cd needle
-
-5. Install needle (follow instruction to setup wandb)
- source ./setup
+ cd needle && source ./setup
-6. Login to Hugging Face (required for private datasets)
- huggingface-cli login
+5. Login to Hugging Face (required for private datasets)
+ hf auth login
(paste your HF token — get one at https://huggingface.co/settings/tokens)
-7. Full pipeline
- needle tokenize # tokenize + pack + upload
- needle train --wandb # train on TPU
- needle eval --checkpoint <path> # evaluate
-
-8. Use tmux for long training runs (survives SSH disconnects)
+6. Use tmux for long training runs (survives SSH disconnects)
tmux new -s train # start a named session
needle train --wandb # run training inside it
Ctrl+B, then D # detach (keeps running)
@@ -448,12 +406,4 @@ all workers automatically.
7. Stop/delete when done
needle tpu stop my-experiment
needle tpu delete my-experiment
-```
-
-## Current TPU Instances
-
-```
- Name Zone Type Software Workers
- ──────────────────────────────────────────────────────────────────
- large asia-northeast1-b v6e-16 v2-alpha-tpuv6e 4
-```
+```
\ No newline at end of file
diff --git a/src/calibrate.py b/src/calibrate.py
deleted file mode 100644
index f457969..0000000
--- a/src/calibrate.py
+++ /dev/null
@@ -1,399 +0,0 @@
-"""Post-training calibration of the confidence head.
-
-Freezes the encoder and decoder, computes per-example decoder perplexity
-on the training data, maps perplexity to [0, 1] confidence labels via a
-sigmoid calibration curve, then trains only the confidence head (2 Dense layers)
-to predict these labels from encoder output.
-
-Two-pass design for memory efficiency:
- Pass 1: Forward through encoder+decoder to compute per-example PPL (stores only scalars)
- Pass 2: Re-encode through encoder only, train confidence head on the fly
-
-All compute is distributed across all available devices via pmap.
-
-Usage:
- needle calibrate --checkpoint checkpoints/needle_12_512_50000.pkl
- needle calibrate --checkpoint checkpoints/needle_12_512_50000.pkl --num-samples 10000
-"""
-
-import math
-import os
-import pickle
-
-import jax
-import jax.numpy as jnp
-import numpy as np
-import optax
-from tqdm import tqdm
-
-from .data import (
- get_tokenizer, load_prepared_data, count_batches, get_batches,
-)
-from .model import (
- EncoderDecoderTransformer, TransformerConfig,
- make_packing_mask, make_causal_packing_mask, make_cross_packing_mask,
-)
-
-
-def _replicate(tree):
- """Replicate params/state across all devices."""
- return jax.tree.map(lambda x: jnp.broadcast_to(x, (jax.local_device_count(),) + x.shape), tree)
-
-
-def _unreplicate(tree):
- """Get single copy from replicated params/state."""
- return jax.tree.map(lambda x: x[0], tree)
-
-
-def compute_per_example_perplexity(model, params, train_data, batch_size, max_samples=None):
- """Compute per-example perplexity using packed training data.
-
- Memory-efficient: only stores scalar PPL per example, not encoder outputs.
- Distributed across all devices via pmap.
-
- Returns:
- perplexities: numpy array of per-example perplexity values
- """
- enc = train_data["packed_enc"]
- dec_in = train_data["packed_dec_in"]
- dec_tgt = train_data["packed_dec_tgt"]
- enc_seg = train_data["packed_enc_seg"]
- dec_seg = train_data["packed_dec_seg"]
-
- n = len(enc)
- if max_samples:
- n = min(n, max_samples)
-
- num_devices = jax.local_device_count()
- per_device_batch = batch_size // num_devices
- assert batch_size % num_devices == 0, f"batch_size {batch_size} must be divisible by {num_devices} devices"
-
- all_ppl = []
-
- @jax.pmap
- def forward_step(params, src, tgt_in, tgt_out, enc_seg_ids, dec_seg_ids):
- src_mask = make_packing_mask(enc_seg_ids)
- tgt_mask = make_causal_packing_mask(dec_seg_ids)
- cross_mask = make_cross_packing_mask(enc_seg_ids, dec_seg_ids)
-
- logits = model.apply(
- {"params": params}, src, tgt_in,
- src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
- )
-
- ce = optax.softmax_cross_entropy_with_integer_labels(
- logits.astype(jnp.float32), tgt_out
- )
- padding_mask = (dec_seg_ids > 0).astype(jnp.float32)
- per_example_loss = jnp.sum(ce * padding_mask, axis=-1) / jnp.maximum(
- jnp.sum(padding_mask, axis=-1), 1.0
- )
- per_example_ppl = jnp.exp(jnp.minimum(per_example_loss, 20.0))
- return per_example_ppl
-
- r_params = _replicate(params)
-
- print(f"Computing per-example perplexity ({n} examples, batch_size={batch_size}, {num_devices} devices)...")
- for i in tqdm(range(0, n, batch_size), desc="PPL forward"):
- end = min(i + batch_size, n)
- actual = end - i
-
- # Pad to full batch if needed
- def _load(arr, i, end, actual):
- batch = np.array(arr[i:end], dtype=np.int32)
- if actual < batch_size:
- pad = np.zeros((batch_size - actual, batch.shape[1]), dtype=np.int32)
- batch = np.concatenate([batch, pad], axis=0)
- return batch.reshape(num_devices, per_device_batch, -1)
-
- src = _load(enc, i, end, actual)
- tgt_in = _load(dec_in, i, end, actual)
- tgt_out = _load(dec_tgt, i, end, actual)
- es = _load(enc_seg, i, end, actual)
- ds = _load(dec_seg, i, end, actual)
-
- ppl = forward_step(r_params, src, tgt_in, tgt_out, es, ds)
- # ppl shape: (num_devices, per_device_batch) → flatten and trim
- ppl_flat = np.array(ppl).reshape(-1)[:actual]
- all_ppl.append(ppl_flat)
-
- perplexities = np.concatenate(all_ppl)
- print(f" Perplexity stats: min={perplexities.min():.2f}, median={np.median(perplexities):.2f}, "
- f"mean={perplexities.mean():.2f}, max={perplexities.max():.2f}")
-
- return perplexities
-
-
-def perplexity_to_confidence(perplexities, mu=None, k=5.0):
- """Map perplexity values to [0, 1] confidence via sigmoid calibration.
-
- confidence = sigmoid(-k * (log(ppl) - log(mu)))
- """
- if mu is None:
- mu = float(np.percentile(perplexities, 80))
-
- log_ppl = np.log(np.maximum(perplexities, 1.0))
- log_mu = np.log(mu)
-
- confidence = 1.0 / (1.0 + np.exp(k * (log_ppl - log_mu)))
-
- return confidence.astype(np.float32), mu
-
-
-def train_confidence_head(model, params, train_data, confidence_labels,
- epochs=1, lr=1e-3, batch_size=128):
- """Train only the confidence head parameters, re-encoding on the fly.
-
- Distributed across all devices via pmap. Memory-efficient: re-runs encoder
- forward pass during training instead of caching all encoder outputs.
- """
- enc = train_data["packed_enc"]
- enc_seg = train_data["packed_enc_seg"]
- n = len(confidence_labels)
-
- num_devices = jax.local_device_count()
- per_device_batch = batch_size // num_devices
- assert batch_size % num_devices == 0, f"batch_size {batch_size} must be divisible by {num_devices} devices"
-
- conf_param_keys = {"confidence_hidden", "confidence_out"}
-
- def is_conf_param(path):
- return any(k in str(path) for k in conf_param_keys)
-
- tx = optax.adam(lr)
- opt_state = tx.init(params)
-
- train_step = jax.pmap(
- lambda params, opt_state, src, enc_seg_ids, targets: _train_step_inner(
- model, tx, is_conf_param, params, opt_state, src, enc_seg_ids, targets
- ),
- axis_name="devices",
- )
-
- r_params = _replicate(params)
- r_opt_state = _replicate(opt_state)
-
- print(f"\nTraining confidence head ({n} examples, {epochs} epoch(s), lr={lr}, "
- f"batch_size={batch_size}, {num_devices} devices)...")
-
- for epoch in range(epochs):
- perm = np.random.permutation(n)
- epoch_losses = []
-
- for i in tqdm(range(0, n - batch_size + 1, batch_size),
- desc=f"Calibrating epoch {epoch + 1}/{epochs}"):
- idx = perm[i:i + batch_size]
- src = np.array(enc[idx], dtype=np.int32).reshape(num_devices, per_device_batch, -1)
- es = np.array(enc_seg[idx], dtype=np.int32).reshape(num_devices, per_device_batch, -1)
- tgt = np.array(confidence_labels[idx]).reshape(num_devices, per_device_batch)
-
- r_params, r_opt_state, loss = train_step(r_params, r_opt_state, src, es, tgt)
- epoch_losses.append(float(loss[0])) # loss is replicated, take first
-
- avg_loss = sum(epoch_losses) / len(epoch_losses) if epoch_losses else float("nan")
- print(f" Epoch {epoch + 1}/{epochs}: MSE={avg_loss:.6f}")
-
- params = _unreplicate(r_params)
- return params
-
-
-def _train_step_inner(model, tx, is_conf_param, params, opt_state, src, enc_seg_ids, targets):
- """Inner training step — separated for pmap with axis_name."""
- def loss_fn(params):
- src_mask = make_packing_mask(enc_seg_ids)
- encoder_out, enc_mask = model.apply(
- {"params": params}, src, src_mask=src_mask, method="encode_text",
- )
- pred = model.apply(
- {"params": params}, encoder_out, enc_mask,
- method="predict_confidence",
- )
- return jnp.mean((pred - targets) ** 2)
-
- loss, grads = jax.value_and_grad(loss_fn)(params)
-
- # Average gradients across devices
- grads = jax.lax.pmean(grads, axis_name="devices")
-
- # Zero out gradients for non-confidence params
- grads = jax.tree_util.tree_map_with_path(
- lambda path, g: g if is_conf_param(path) else jnp.zeros_like(g),
- grads,
- )
-
- updates, opt_state_new = tx.update(grads, opt_state, params)
- params_new = optax.apply_updates(params, updates)
- return params_new, opt_state_new, loss
-
-
-def evaluate_confidence(model, params, train_data, confidence_labels, perplexities, max_eval=10000):
- """Evaluate calibrated confidence head and print diagnostic stats."""
- enc = train_data["packed_enc"]
- enc_seg = train_data["packed_enc_seg"]
- n = min(len(confidence_labels), max_eval)
-
- num_devices = jax.local_device_count()
- batch_size = 128
- # Round down to multiple of num_devices
- batch_size = (batch_size // num_devices) * num_devices
- per_device_batch = batch_size // num_devices
-
- @jax.pmap
- def predict_step(params, src, enc_seg_ids):
- src_mask = make_packing_mask(enc_seg_ids)
- encoder_out, enc_mask = model.apply(
- {"params": params}, src, src_mask=src_mask, method="encode_text",
- )
- return model.apply(
- {"params": params}, encoder_out, enc_mask,
- method="predict_confidence",
- )
-
- r_params = _replicate(params)
-
- predictions = []
- for i in tqdm(range(0, n, batch_size), desc="Evaluating confidence"):
- end = min(i + batch_size, n)
- actual = end - i
-
- src = np.array(enc[i:end], dtype=np.int32)
- es = np.array(enc_seg[i:end], dtype=np.int32)
- if actual < batch_size:
- pad_src = np.zeros((batch_size - actual, src.shape[1]), dtype=np.int32)
- src = np.concatenate([src, pad_src], axis=0)
- pad_es = np.zeros((batch_size - actual, es.shape[1]), dtype=np.int32)
- es = np.concatenate([es, pad_es], axis=0)
-
- src = src.reshape(num_devices, per_device_batch, -1)
- es = es.reshape(num_devices, per_device_batch, -1)
-
- pred = predict_step(r_params, src, es)
- pred_flat = np.array(pred).reshape(-1)[:actual]
- predictions.append(pred_flat)
-
- predictions = np.concatenate(predictions)
-
- labels = confidence_labels[:n]
- ppl = perplexities[:n]
-
- # Stats
- mse = np.mean((predictions - labels) ** 2)
- mae = np.mean(np.abs(predictions - labels))
- correlation = np.corrcoef(predictions, labels)[0, 1]
-
- print(f"\nConfidence head evaluation:")
- print(f" MSE: {mse:.6f}")
- print(f" MAE: {mae:.4f}")
- print(f" Correlation: {correlation:.4f}")
-
- print(f"\n {'PPL Range':<15s} {'Count':>6s} {'Avg Conf':>10s} {'Avg Pred':>10s}")
- print(f" {'─' * 45}")
- buckets = [(0, 2), (2, 4), (4, 8), (8, 16), (16, 50), (50, 1000)]
- for lo, hi in buckets:
- mask = (ppl >= lo) & (ppl < hi)
- if mask.sum() == 0:
- continue
- avg_conf = labels[mask].mean()
- avg_pred = predictions[mask].mean()
- print(f" {f'{lo}-{hi}':<15s} {mask.sum():>6d} {avg_conf:>10.3f} {avg_pred:>10.3f}")
-
- return predictions
-
-
-def main(args):
- num_devices = jax.local_device_count()
- print(f"Devices: {num_devices}")
-
- # Ensure batch_size is divisible by num_devices
- if args.batch_size % num_devices != 0:
- args.batch_size = (args.batch_size // num_devices) * num_devices
- print(f" Adjusted batch_size to {args.batch_size} (divisible by {num_devices})")
-
- # Load checkpoint
- print(f"Loading checkpoint: {args.checkpoint}")
- with open(args.checkpoint, "rb") as f:
- data = pickle.load(f)
- params = jax.tree.map(lambda x: jnp.array(x, dtype=jnp.bfloat16), data["params"])
- config_dict = {k: v for k, v in data["config"].items()
- if k in TransformerConfig.__dataclass_fields__}
- config = TransformerConfig(**config_dict)
- model = EncoderDecoderTransformer(config)
-
- param_count = sum(x.size for x in jax.tree.leaves(params))
- print(f" Parameters: {param_count:,}")
- print(f" Config: d={config.d_model}, heads={config.num_heads}, "
- f"layers={config.num_encoder_layers}/{config.num_decoder_layers}")
-
- # Check if confidence head params need (re)initialization
- rng = jax.random.PRNGKey(0)
- dummy_src = jnp.ones((1, 128), dtype=jnp.int32)
- dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)
- init_vars = model.init({"params": rng}, dummy_src, dummy_tgt, method="init_all")
- init_params = init_vars["params"]
-
- flat_keys = str(jax.tree.map(lambda x: x.shape, params))
- needs_reinit = "confidence_hidden" not in flat_keys
- if not needs_reinit:
- # Check shape match
- for key in ("confidence_hidden", "confidence_out"):
- old_shapes = jax.tree.map(lambda x: x.shape, params[key])
- new_shapes = jax.tree.map(lambda x: x.shape, init_params[key])
- if str(old_shapes) != str(new_shapes):
- needs_reinit = True
- break
-
- if needs_reinit:
- print(" Confidence head shape mismatch or missing — reinitializing...")
- params = {**params, "confidence_hidden": init_params["confidence_hidden"],
- "confidence_out": init_params["confidence_out"]}
- print(" Confidence head initialized.")
-
- # Load training data
- print(f"\nLoading prepared data...")
- train_data = load_prepared_data("train", mmap=True)
- print(f" {len(train_data['packed_enc']):,} packed bins")
-
- # Pass 1: Compute per-example perplexity (forward-only, no gradients → large batch)
- ppl_batch_size = num_devices * 128 # 128 per device, forward-only no gradients
- perplexities = compute_per_example_perplexity(
- model, params, train_data, ppl_batch_size, max_samples=args.num_samples,
- )
-
- # Map perplexity → confidence labels
- confidence_labels, mu = perplexity_to_confidence(perplexities, k=args.k)
- print(f"\nCalibration: mu (median PPL) = {mu:.2f}, k = {args.k}")
- print(f" Confidence stats: min={confidence_labels.min():.3f}, "
- f"median={np.median(confidence_labels):.3f}, "
- f"mean={confidence_labels.mean():.3f}, max={confidence_labels.max():.3f}")
-
- # Pass 2: Train confidence head (has gradients → use args.batch_size)
- params = train_confidence_head(
- model, params, train_data, confidence_labels,
- epochs=args.epochs, lr=args.lr, batch_size=args.batch_size,
- )
-
- # Ensure params are on host (clear stale pmap sharding from training)
- params = jax.tree.map(lambda x: jnp.array(np.array(x), dtype=x.dtype), params)
-
- # Evaluate
- evaluate_confidence(model, params, train_data, confidence_labels, perplexities)
-
- # Save updated checkpoint
- output_path = args.output or args.checkpoint
- params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), params)
-
- save_config = dict(data["config"])
- save_config["confidence_mu"] = float(mu)
- save_config["confidence_k"] = float(args.k)
-
- with open(output_path, "wb") as f:
- pickle.dump({"params": params_np, "config": save_config}, f)
- print(f"\nSaved calibrated checkpoint: {output_path}")
- print(f" Calibration params: mu={mu:.2f}, k={args.k}")
- print(f"\n Usage at inference:")
- print(f" confidence, encoder_out, enc_mask = model.apply(")
- print(f' {{"params": params}}, src, src_mask=src_mask, method="forward_confidence")')
- print(f" if confidence > 0.6: # user-configurable threshold")
- print(f" # decode locally")
- print(f" else:")
- print(f" # route to cloud")
diff --git a/src/categorize_failures.py b/src/categorize_failures.py
deleted file mode 100644
index 8cf1594..0000000
--- a/src/categorize_failures.py
+++ /dev/null
@@ -1,322 +0,0 @@
-"""Categorize tool-call argument-value failures to quantify what a
-draft-correction second pass could actually fix.
-
-Runs unconstrained generation on a stratified single+multi-call val sample,
-then sorts each argument-value mismatch into one of these buckets:
-
- correctable ref value appears as a substring of the query (a second
- pass can re-extract it)
- canonicalization ref and pred match after lowercase + whitespace collapse
- + stripping a small set of prefixes ("at ", "today at ",
- "on "). Data-inconsistency noise, not model error.
- cross_lingual ref and pred use different scripts (Latin vs Cyrillic,
- etc.) — translation mismatch, usually annotation noise.
- unrecoverable neither ref nor pred appear in the query — a draft
- corrector can't fix these without external info.
- missing_key pred omitted a key the ref requires
- extra_key pred emitted a key the ref doesn't require
-
-Usage:
- python -m src.categorize_failures --checkpoint checkpoints/needle.pkl
- python -m src.categorize_failures --checkpoint needle.pkl --num-samples 500
-"""
-
-import argparse
-import json
-import re
-import unicodedata
-from collections import Counter
-
-import jax
-import numpy as np
-
-from .data import get_tokenizer, load_tool_calls
-from .model import EncoderDecoderTransformer
-from .run import generate_batch, load_checkpoint
-from .tokenizer import to_snake_case
-
-
-_CANONICAL_PREFIXES = ("today at ", "tomorrow at ", "tonight at ",
- "this evening at ", "at ", "on ", "the ")
-
-
-def _strip_canonical(s):
- """Lowercase, collapse whitespace, strip one canonical prefix."""
- if not isinstance(s, str):
- return s
- s = s.strip().lower()
- s = re.sub(r"\s+", " ", s)
- for p in _CANONICAL_PREFIXES:
- if s.startswith(p):
- s = s[len(p):].strip()
- break
- return s
-
-
-def _script_of(s):
- """Return dominant script of alphabetic chars, or 'other' / 'empty'."""
- if not isinstance(s, str) or not s:
- return "empty"
- scripts = set()
- for ch in s:
- if not ch.isalpha():
- continue
- try:
- name = unicodedata.name(ch, "")
- except ValueError:
- continue
- if "LATIN" in name:
- scripts.add("latin")
- elif "CYRILLIC" in name:
- scripts.add("cyrillic")
- elif "GREEK" in name:
- scripts.add("greek")
- elif "ARABIC" in name:
- scripts.add("arabic")
- elif "HEBREW" in name:
- scripts.add("hebrew")
- elif "HANGUL" in name:
- scripts.add("hangul")
- elif "DEVANAGARI" in name:
- scripts.add("devanagari")
- elif "CJK" in name or "HIRAGANA" in name or "KATAKANA" in name:
- scripts.add("cjk")
- else:
- scripts.add("other")
- if not scripts:
- return "empty"
- if len(scripts) == 1:
- return next(iter(scripts))
- return "mixed"
-
-
-def _value_in_query(value, query):
- """True if value appears in query (case-insensitive for strings)."""
- if isinstance(value, bool):
- return False
- if isinstance(value, (int, float)):
- return str(value) in query
- if isinstance(value, str):
- v = value.strip().lower()
- if len(v) < 2:
- return False
- return v in query.lower()
- return False
-
-
-def categorize_value_mismatch(query, ref_value, pred_value):
- if isinstance(ref_value, str) and isinstance(pred_value, str):
- if _strip_canonical(ref_value) == _strip_canonical(pred_value):
- return "canonicalization"
- rs, ps = _script_of(ref_value), _script_of(pred_value)
- if rs != ps and rs not in ("empty", "other", "mixed") and ps not in ("empty", "other", "mixed"):
- return "cross_lingual"
- if _value_in_query(ref_value, query):
- return "correctable"
- return "unrecoverable"
-
-
-def categorize_pool(model, params, tokenizer, examples, max_gen_len, max_enc_len,
- batch_size=32):
- """Generate predictions for a pool and categorize each value mismatch.
-
- Returns (bucket_counts, bucket_examples).
- """
- queries = [ex["query"] for ex in examples]
- tools_list = [ex["tools"] for ex in examples]
-
- all_preds = []
- for i in range(0, len(examples), batch_size):
- chunk_q = queries[i:i + batch_size]
- chunk_t = tools_list[i:i + batch_size]
- preds = generate_batch(
- model, params, tokenizer, chunk_q, chunk_t,
- max_gen_len=max_gen_len, max_enc_len=max_enc_len,
- normalize=True, constrained=False,
- )
- all_preds.extend(preds)
-
- bucket_counts = Counter()
- bucket_examples = {b: [] for b in
- ("correctable", "canonicalization", "cross_lingual",
- "unrecoverable", "missing_key", "extra_key")}
-
- for ex, pred_text in zip(examples, all_preds):
- query = ex["query"]
- try:
- ref_calls = json.loads(ex["answers"])
- except (ValueError, TypeError):
- continue
- try:
- pred_calls = json.loads(pred_text)
- if isinstance(pred_calls, dict):
- pred_calls = [pred_calls]
- elif not isinstance(pred_calls, list):
- continue
- except (ValueError, TypeError):
- continue
- if not isinstance(ref_calls, list):
- continue
-
- for c in ref_calls:
- if isinstance(c, dict) and "name" in c:
- c["name"] = to_snake_case(c["name"])
- for c in pred_calls:
- if isinstance(c, dict) and "name" in c:
- c["name"] = to_snake_case(c["name"])
-
- ref_by_name = {}
- for c in ref_calls:
- if isinstance(c, dict) and "name" in c:
- ref_by_name.setdefault(c["name"], []).append(c.get("arguments", {}) or {})
-
- for c in pred_calls:
- if not isinstance(c, dict) or "name" not in c:
- continue
- name = c["name"]
- if name not in ref_by_name:
- continue # tool-level mismatch — outside this script's scope
- pred_args = c.get("arguments", {}) or {}
- ref_args = ref_by_name[name][0]
- if not isinstance(ref_args, dict):
- continue
-
- all_keys = set(pred_args.keys()) | set(ref_args.keys())
- for key in all_keys:
- has_pred = key in pred_args
- has_ref = key in ref_args
-
- if has_ref and not has_pred:
- bucket_counts["missing_key"] += 1
- if len(bucket_examples["missing_key"]) < 5:
- bucket_examples["missing_key"].append(
- {"query": query, "tool": name, "key": key,
- "ref": ref_args[key], "pred": "<MISSING>"})
- continue
- if has_pred and not has_ref:
- bucket_counts["extra_key"] += 1
- if len(bucket_examples["extra_key"]) < 5:
- bucket_examples["extra_key"].append(
- {"query": query, "tool": name, "key": key,
- "ref": "<MISSING>", "pred": pred_args[key]})
- continue
-
- rv, pv = ref_args[key], pred_args[key]
- if json.dumps(pv, sort_keys=True) == json.dumps(rv, sort_keys=True):
- continue
- cat = categorize_value_mismatch(query, rv, pv)
- bucket_counts[cat] += 1
- if len(bucket_examples[cat]) < 5:
- bucket_examples[cat].append(
- {"query": query, "tool": name, "key": key,
- "ref": rv, "pred": pv})
-
- return bucket_counts, bucket_examples
-
-
-def _print_block(label, counts, examples):
- total = sum(counts.values())
- print(f"\n=== {label} ===")
- print(f" Total value-level mismatches: {total}")
- if total == 0:
- return
- print()
- order = ("correctable", "canonicalization", "cross_lingual",
- "unrecoverable", "missing_key", "extra_key")
- for b in order:
- n = counts.get(b, 0)
- pct = 100 * n / total
- bar = "█" * int(pct / 2)
- print(f" {b:<18} {n:>5} ({pct:>5.1f}%) {bar}")
-
- for b in order:
- if not examples.get(b):
- continue
- print(f"\n -- {b} examples --")
- for e in examples[b]:
- q = e["query"][:110]
- print(f" Q: {q}")
- print(f" {e['tool']}.{e['key']}")
- print(f" ref : {e['ref']!r}")
- print(f" pred: {e['pred']!r}")
-
-
-def main(args):
- print(f"Loading checkpoint {args.checkpoint} ...")
- params, config = load_checkpoint(args.checkpoint)
- model = EncoderDecoderTransformer(config)
- tokenizer = get_tokenizer()
-
- param_count = sum(x.size for x in jax.tree.leaves(params))
- print(f" Parameters: {param_count:,}")
-
- val_ds = load_tool_calls("validation")
- rng = np.random.RandomState(42)
-
- pools = {"single": [], "multi": []}
- for k in range(len(val_ds)):
- ex = val_ds[k]
- try:
- answers = json.loads(ex["answers"])
- except (ValueError, TypeError):
- continue
- if not isinstance(answers, list) or not answers:
- continue
- pools["single" if len(answers) == 1 else "multi"].append(ex)
-
- for name in pools:
- rng.shuffle(pools[name])
- pools[name] = pools[name][:args.num_samples]
- print(f" {name}-call sample: {len(pools[name])}")
-
- combined_counts = Counter()
- for name in ("single", "multi"):
- print(f"\n--- Running generation on {len(pools[name])} {name}-call examples ...")
- counts, examples = categorize_pool(
- model, params, tokenizer, pools[name],
- max_gen_len=args.max_gen_len,
- max_enc_len=args.max_enc_len,
- batch_size=args.batch_size,
- )
- _print_block(f"{name.upper()}-CALL", counts, examples)
- combined_counts += counts
-
- total = sum(combined_counts.values())
- if total == 0:
- return
- print(f"\n=== COMBINED ({total} value mismatches) ===")
- order = ("correctable", "canonicalization", "cross_lingual",
- "unrecoverable", "missing_key", "extra_key")
- for b in order:
- n = combined_counts.get(b, 0)
- pct = 100 * n / total
- bar = "█" * int(pct / 2)
- print(f" {b:<18} {n:>5} ({pct:>5.1f}%) {bar}")
-
- correctable = combined_counts.get("correctable", 0)
- missing = combined_counts.get("missing_key", 0)
- print(f"\n Upper bound for a perfect draft-corrector: "
- f"{100 * (correctable + missing) / total:.1f}% of value errors "
- f"({correctable + missing}/{total})")
- noise = combined_counts.get("canonicalization", 0) + combined_counts.get("cross_lingual", 0)
- print(f" Fixable by cleaning labels (no model change): "
- f"{100 * noise / total:.1f}% ({noise}/{total})")
- unrecov = combined_counts.get("unrecoverable", 0)
- print(f" Floor a corrector can't beat: "
- f"{100 * unrecov / total:.1f}% ({unrecov}/{total})")
-
-
-def parse_args():
- p = argparse.ArgumentParser(
- description="Categorize tool-call value failures by correctability")
- p.add_argument("--checkpoint", type=str, required=True)
- p.add_argument("--num-samples", type=int, default=500,
- help="Samples per pool (single-call + multi-call)")
- p.add_argument("--batch-size", type=int, default=32)
- p.add_argument("--max-enc-len", type=int, default=1024)
- p.add_argument("--max-gen-len", type=int, default=512)
- return p.parse_args()
-
-
-if __name__ == "__main__":
- main(parse_args())
diff --git a/src/cli.py b/src/cli.py
index 3be4001..f866549 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -120,7 +120,7 @@ def main():
"matching params, random-inits the rest.")
p.add_argument("--epochs", type=int, default=1)
p.add_argument("--batch-size", type=int, default=64)
- p.add_argument("--lr", type=float, default=3e-4)
+ p.add_argument("--lr", type=float, default=3e-5)
p.add_argument("--muon-lr", type=float, default=0.02)
p.add_argument("--d-model", type=int, default=512)
p.add_argument("--num-heads", type=int, default=8)
@@ -165,14 +165,6 @@ def main():
help="Loss weight for argument key tokens (default: 1.5)")
p.add_argument("--no-feedforward", action=argparse.BooleanOptionalAction, default=True,
help="Remove feedforward layers entirely (default: True)")
- p.add_argument("--calibrate", action=argparse.BooleanOptionalAction, default=False,
- help="Run confidence head calibration after training (default: False)")
- p.add_argument("--calibrate-epochs", type=int, default=1,
- help="Epochs for confidence head calibration (default: 1)")
- p.add_argument("--calibrate-lr", type=float, default=1e-3,
- help="Learning rate for confidence head calibration (default: 1e-3)")
- p.add_argument("--calibrate-k", type=float, default=5.0,
- help="Sigmoid steepness for PPL→confidence mapping (default: 5.0)")
p = sub.add_parser("pretrain", add_help=False)
p.add_argument("--name", type=str, default="pretrain",
@@ -205,31 +197,6 @@ def main():
p.add_argument("--activation", type=str, default="swiglu", choices=["drelu", "swiglu", "geglu"])
p.add_argument("--no-feedforward", action=argparse.BooleanOptionalAction, default=True)
- p = sub.add_parser("grpo", add_help=False)
- p.add_argument("--checkpoint", type=str, default="checkpoints/needle.pkl",
- help="SFT checkpoint to init both policy and reference (default: checkpoints/needle.pkl)")
- p.add_argument("--name", type=str, default="grpo")
- p.add_argument("--batch-size", type=int, default=16,
- help="Prompts per step (B)")
- p.add_argument("--k", type=int, default=4,
- help="Samples per prompt (K)")
- p.add_argument("--lr", type=float, default=1e-6)
- p.add_argument("--beta", type=float, default=0.04,
- help="KL penalty weight against frozen reference")
- p.add_argument("--max-steps", type=int, default=5000)
- p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)
- p.add_argument("--max-gen-len", type=int, default=DEFAULT_MAX_GEN_LEN)
- p.add_argument("--eval-every", type=int, default=100,
- help="Run quick val PPL every N steps (default: 100)")
- p.add_argument("--eval-batch-size", type=int, default=32,
- help="Batch size for val PPL eval")
- p.add_argument("--tool-call-samples", type=int, default=100,
- help="Samples for stratified tool-call benchmark at save time (split evenly single/multi)")
- p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
- p.add_argument("--wandb", action="store_true")
- p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])
- p.add_argument("--seed", type=int, default=42)
-
p = sub.add_parser("tokenize", add_help=False)
p.add_argument("--max-samples", type=int, default=None,
help="Limit samples per split (for dev/test)")
@@ -265,15 +232,6 @@ def main():
p.add_argument("--no-constrained", action="store_true",
help="Disable grammar-constrained decoding for tool names/arg keys")
- p = sub.add_parser("calibrate", add_help=False)
- p.add_argument("--checkpoint", type=str, required=True)
- p.add_argument("--output", type=str, default=None, help="Output checkpoint path (default: overwrite input)")
- p.add_argument("--batch-size", type=int, default=128)
- p.add_argument("--num-samples", type=int, default=None, help="Limit training samples for PPL computation")
- p.add_argument("--epochs", type=int, default=1)
- p.add_argument("--lr", type=float, default=1e-3)
- p.add_argument("--k", type=float, default=5.0, help="Sigmoid steepness for PPL→confidence mapping")
-
p = sub.add_parser("generate-data", add_help=False)
p.add_argument("--num-samples", type=int, default=500, help="Number of samples to generate")
p.add_argument("--batch-size", type=int, default=25, help="Examples per Gemini call")
@@ -283,24 +241,6 @@ def main():
p.add_argument("--output-jsonl", type=str, default=None, help="Also save raw generations to JSONL")
p.add_argument("--upload-every", type=int, default=None, help="Merge+upload every N samples")
- p = sub.add_parser("merge-xlam", add_help=False)
- p.add_argument("--dry-run", action="store_true", help="Skip upload")
- p.add_argument("--max-samples", type=int, default=None, help="Limit xlam samples")
-
- p = sub.add_parser("translate-xlam", add_help=False)
- p.add_argument("--max-samples", type=int, default=None, help="Limit examples to translate")
- p.add_argument("--workers", type=int, default=8, help="Parallel Gemini calls")
- p.add_argument("--model", type=str, default=None, help="Gemini model for translation")
- p.add_argument("--batch-size", type=int, default=10, help="Examples per Gemini call")
- p.add_argument("--dry-run", action="store_true", help="Translate only, skip save/upload")
-
- p = sub.add_parser("rebalance-tools", add_help=False)
- p.add_argument("--dry-run", action="store_true", help="Preview without modifying")
-
- p = sub.add_parser("split-dataset", add_help=False)
- p.add_argument("--val-per-source", type=int, default=None,
- help="Validation samples per source (default: 2500)")
-
p = sub.add_parser("evaluate", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
p.add_argument("--benchmarks", type=str, nargs="*",
@@ -381,21 +321,12 @@ def main():
jax.distributed.initialize()
from .train import train
train(args)
- elif args.command == "grpo":
- import jax
- if os.path.exists("/dev/accel0"):
- jax.distributed.initialize()
- from .grpo import grpo
- grpo(args)
elif args.command == "run":
from .run import main as run_main
run_main(args)
elif args.command == "eval":
from .eval import main as eval_main_fn
eval_main_fn(args)
- elif args.command == "calibrate":
- from .calibrate import main as calibrate_main
- calibrate_main(args)
elif args.command == "generate-data":
from .generate_data import main as gendata_main, MODEL as _MODEL, UPLOAD_EVERY as _UE
if args.model is None:
@@ -403,20 +334,6 @@ def main():
if args.upload_every is None:
args.upload_every = _UE
gendata_main(args)
- elif args.command == "merge-xlam":
- from .merge_xlam import main as merge_main
- merge_main(args)
- elif args.command == "translate-xlam":
- from .translate_xlam import main as translate_main, MODEL as _TMODEL
- if args.model is None:
- args.model = _TMODEL
- translate_main(args)
- elif args.command == "rebalance-tools":
- from .rebalance_tools import main as rebalance_main
- rebalance_main(args)
- elif args.command == "split-dataset":
- from .split_dataset import main as split_main
- split_main(args)
elif args.command == "evaluate":
from .evaluate import main as eval_main
eval_main(args)
diff --git a/src/generate_data.py b/src/generate_data.py
index 49d3c84..fa7cb83 100644
--- a/src/generate_data.py
+++ b/src/generate_data.py
@@ -1826,7 +1826,6 @@ def _merge_and_upload(existing, new_examples):
import shutil as _shutil
_shutil.rmtree(parquet_dir)
print(f"Upload complete: {HF_DATASET_REPO}")
- print("NOTE: Run 'needle split-dataset' to create the validation split.")
return merged
diff --git a/src/grpo.py b/src/grpo.py
deleted file mode 100644
index e2fd953..0000000
--- a/src/grpo.py
+++ /dev/null
@@ -1,609 +0,0 @@
-"""GRPO fine-tuning from an SFT checkpoint.
-
-For each prompt we sample K completions from the current policy, score them
-with a programmatic reward (exact call-set match against the reference answer),
-normalize rewards within the group to get advantages, and take a policy
-gradient step with a KL penalty against a frozen reference (the SFT ckpt).
-
-No KV cache, no constrained decoding, no reward model.
-
-Usage:
- needle grpo --checkpoint checkpoints/needle.pkl --wandb
-"""
-
-import json
-import math
-import os
-import pickle
-import time
-
-import jax
-import jax.numpy as jnp
-import numpy as np
-import optax
-from flax.training import train_state
-from tqdm import tqdm
-
-from .data import (
- load_tool_calls, load_prepared_data, get_batches, to_snake_case,
-)
-from .distributed import (
- _replicate, _unreplicate, _upload_checkpoint,
-)
-
-_HF_SFT_REPO = "Cactus-Compute/needle"
-
-
-def _download_sft_checkpoint(path_or_name):
- """Return a local path to the SFT checkpoint, downloading from HF if needed.
-
- The SFT checkpoint lives in Cactus-Compute/needle, not the generic
- checkpoints repo that _download_checkpoint targets.
- """
- if os.path.exists(path_or_name):
- return path_or_name
- from huggingface_hub import hf_hub_download
- local_dir = os.path.dirname(path_or_name) or "checkpoints"
- os.makedirs(local_dir, exist_ok=True)
- print(f" Downloading {os.path.basename(path_or_name)} "
- f"from {_HF_SFT_REPO}...", flush=True)
- local_path = hf_hub_download(
- repo_id=_HF_SFT_REPO,
- filename=os.path.basename(path_or_name),
- repo_type="model",
- local_dir=local_dir,
- )
- print(f" Downloaded to {local_path}", flush=True)
- return local_path
-from .model import (
- EncoderDecoderTransformer, TransformerConfig,
- make_causal_mask, make_padding_mask,
- make_packing_mask, make_causal_packing_mask, make_cross_packing_mask,
-)
-from .optim import _wsd_schedule
-from .run import _build_encoder_input
-from .tokenizer import EOS_ID, PAD_ID, get_tokenizer
-
-
-def _call_key_set(calls):
- """Canonicalize a list of tool calls: snake_case names, sorted."""
- keys = []
- for c in calls:
- if not isinstance(c, dict):
- continue
- name = to_snake_case(c.get("name", ""))
- args = c.get("arguments", {}) or {}
- keys.append(json.dumps({"name": name, "arguments": args}, sort_keys=True))
- return sorted(keys)
-
-
-def _score(pred_text, ref_answers):
- """Binary reward: 1.0 if pred call-set == ref call-set (order-invariant)."""
- if pred_text.startswith("<tool_call>"):
- pred_text = pred_text[len("<tool_call>"):]
- try:
- pred_calls = json.loads(pred_text)
- if isinstance(pred_calls, dict):
- pred_calls = [pred_calls]
- elif not isinstance(pred_calls, list):
- return 0.0
- except (json.JSONDecodeError, TypeError):
- return 0.0
- try:
- ref_calls = json.loads(ref_answers)
- if not isinstance(ref_calls, list):
- ref_calls = []
- except (json.JSONDecodeError, TypeError):
- return 0.0
- return 1.0 if _call_key_set(pred_calls) == _call_key_set(ref_calls) else 0.0
-
-
-def _build_prompts(tokenizer, batch_examples, max_enc_len):
- """Tokenize and pad encoder inputs for a batch of prompts."""
- enc_lists = [
- _build_encoder_input(tokenizer, ex["query"], ex["tools"], max_enc_len)
- for ex in batch_examples
- ]
- B = len(enc_lists)
- enc = np.full((B, max_enc_len), PAD_ID, dtype=np.int32)
- for i, toks in enumerate(enc_lists):
- enc[i, :len(toks)] = toks
- return enc
-
-
-def _make_fns(model, max_gen_len):
- """Build pmapped encode / sample-step / log-prob functions."""
-
- @jax.pmap
- def p_encode(params, enc_input):
- src_mask = make_padding_mask(enc_input, PAD_ID)
- return model.apply({"params": params}, enc_input,
- src_mask=src_mask, method="encode")
-
- @jax.pmap
- def p_sample_step(params, dec_buffer, encoder_out, cross_mask,
- rng, step_idx, finished):
- sm = make_causal_mask(max_gen_len)
- logits = model.apply(
- {"params": params}, dec_buffer, encoder_out,
- self_mask=sm, cross_mask=cross_mask, method="decode",
- )
- next_logits = jax.lax.dynamic_index_in_dim(
- logits, step_idx, axis=1, keepdims=False,
- )
- next_tokens = jax.random.categorical(rng, next_logits, axis=-1)
- next_tokens = jnp.where(finished, PAD_ID, next_tokens)
- new_finished = finished | (next_tokens == EOS_ID)
- return next_tokens, new_finished
-
- @jax.pmap
- def p_log_probs(params, enc_input, dec_in, dec_tgt):
- src_mask = make_padding_mask(enc_input, PAD_ID)
- tm = make_causal_mask(dec_in.shape[-1]) & make_padding_mask(dec_in, PAD_ID)
- logits = model.apply({"params": params}, enc_input, dec_in,
- src_mask=src_mask, tgt_mask=tm)
- lp = jax.nn.log_softmax(logits.astype(jnp.float32), axis=-1)
- return jnp.take_along_axis(lp, dec_tgt[..., None], axis=-1).squeeze(-1)
-
- return p_encode, p_sample_step, p_log_probs
-
-
-def _make_grpo_step(model, beta):
- """Build pmapped GRPO gradient step."""
-
- def loss_fn(params, enc_input, dec_in, dec_tgt, advantages,
- response_mask, ref_logp):
- src_mask = make_padding_mask(enc_input, PAD_ID)
- tm = make_causal_mask(dec_in.shape[-1]) & make_padding_mask(dec_in, PAD_ID)
- logits = model.apply({"params": params}, enc_input, dec_in,
- src_mask=src_mask, tgt_mask=tm)
- lp = jax.nn.log_softmax(logits.astype(jnp.float32), axis=-1)
- pol_logp = jnp.take_along_axis(lp, dec_tgt[..., None], axis=-1).squeeze(-1)
-
- denom = jnp.maximum(response_mask.sum(), 1.0)
- pg = -(advantages[:, None] * pol_logp * response_mask).sum() / denom
-
- # Schulman k3 KL estimator: unbiased, always non-negative.
- log_ratio = ref_logp - pol_logp
- kl_tok = (jnp.exp(log_ratio) - log_ratio - 1.0) * response_mask
- kl = kl_tok.sum() / denom
-
- return pg + beta * kl, (pg, kl)
-
- def step(state, enc_input, dec_in, dec_tgt, advantages,
- response_mask, ref_logp):
- (loss, (pg, kl)), grads = jax.value_and_grad(loss_fn, has_aux=True)(
- state.params, enc_input, dec_in, dec_tgt, advantages,
- response_mask, ref_logp,
- )
- grads = jax.lax.pmean(grads, axis_name="batch")
- loss = jax.lax.pmean(loss, axis_name="batch")
- pg = jax.lax.pmean(pg, axis_name="batch")
- kl = jax.lax.pmean(kl, axis_name="batch")
- state = state.apply_gradients(grads=grads)
- return state, loss, pg, kl
-
- return jax.pmap(step, axis_name="batch", donate_argnums=(0,))
-
-
-def _sample_completions(p_encode, p_sample_step, r_params, enc_sharded,
- num_devices, max_gen_len, rng):
- """Sample one completion per input row, in parallel across devices.
-
- enc_sharded: (num_devices, per_device, max_enc_len) already on-device.
- Returns numpy array (num_devices, per_device, max_gen_len): each row is
- [EOS, sampled..., EOS, PAD...] or [EOS, sampled...] if EOS wasn't emitted.
- """
- encoder_out, cross_mask = p_encode(r_params, enc_sharded)
- per_device = enc_sharded.shape[1]
-
- dec_buffer = np.full((num_devices, per_device, max_gen_len), PAD_ID, dtype=np.int32)
- dec_buffer[:, :, 0] = EOS_ID
- dec_buffer = jnp.asarray(dec_buffer)
- finished = jnp.zeros((num_devices, per_device), dtype=jnp.bool_)
-
- for step in range(max_gen_len - 1):
- rng, sub = jax.random.split(rng)
- sub_per = jax.random.split(sub, num_devices)
- step_idx = jnp.full((num_devices,), step, dtype=jnp.int32)
- next_tokens, finished = p_sample_step(
- r_params, dec_buffer, encoder_out, cross_mask,
- sub_per, step_idx, finished,
- )
- dec_buffer = dec_buffer.at[:, :, step + 1].set(next_tokens)
-
- return np.asarray(dec_buffer), rng
-
-
-def _make_val_loss_fn(model):
- """Teacher-forced packed val loss (matches train.py)."""
-
- @jax.jit
- def fn(params, src, dec_in, dec_tgt, enc_seg, dec_seg):
- src_mask = make_packing_mask(enc_seg)
- tgt_mask = make_causal_packing_mask(dec_seg)
- cross_mask = make_cross_packing_mask(enc_seg, dec_seg)
- logits = model.apply({"params": params}, src, dec_in,
- src_mask=src_mask, tgt_mask=tgt_mask,
- cross_mask=cross_mask)
- loss = optax.softmax_cross_entropy_with_integer_labels(
- logits.astype(jnp.float32), dec_tgt)
- pmask = (dec_seg > 0).astype(jnp.float32)
- return jnp.sum(loss * pmask), jnp.sum(pmask)
-
- return fn
-
-
-def _compute_val_ppl(val_loss_fn, params, val_data, batch_size):
- total_loss = 0.0
- total_toks = 0.0
- for vb in get_batches(
- val_data["packed_enc"], val_data["packed_dec_in"], val_data["packed_dec_tgt"],
- batch_size, shuffle=False,
- loss_mask=val_data["packed_loss"],
- enc_seg_ids=val_data["packed_enc_seg"],
- dec_seg_ids=val_data["packed_dec_seg"],
- ):
- src, di, dt, _lm, es, ds = vb
- vl, vt = val_loss_fn(
- params,
- jnp.asarray(src, dtype=jnp.int32),
- jnp.asarray(di, dtype=jnp.int32),
- jnp.asarray(dt, dtype=jnp.int32),
- jnp.asarray(es, dtype=jnp.int32),
- jnp.asarray(ds, dtype=jnp.int32),
- )
- total_loss += float(vl)
- total_toks += float(vt)
- return math.exp(min(total_loss / max(total_toks, 1), 20))
-
-
-def _stratified_val_slice(val_ds, per_bucket):
- """Pick up to per_bucket single-call + per_bucket multi-call examples."""
- single, multi = [], []
- for ex in val_ds:
- try:
- calls = json.loads(ex["answers"])
- except (json.JSONDecodeError, TypeError):
- continue
- if not isinstance(calls, list) or not calls:
- continue
- (single if len(calls) == 1 else multi).append(ex)
- if len(single) >= per_bucket and len(multi) >= per_bucket:
- break
- return single[:per_bucket], multi[:per_bucket]
-
-
-def _run_tool_call_eval(model, params, tokenizer, val_ds, per_bucket,
- max_enc_len, max_gen_len):
- """Stratified single/multi-call benchmark with display samples.
-
- Mirrors train.py's end-of-epoch tool-call eval. Uses greedy decode WITHOUT
- constrained decoding — this reveals whether the underlying model is
- actually learning to produce correct tool names, rather than leaning on
- the grammar. `constrained=True` is the right choice at deployment; for
- training telemetry we want to see raw model behavior.
- """
- from .eval import benchmark_tool_calls
-
- single, multi = _stratified_val_slice(val_ds, per_bucket)
- results = {}
- for label, examples in (("single", single), ("multi", multi)):
- if not examples:
- continue
- tc = benchmark_tool_calls(
- model, params, tokenizer, num_samples=len(examples),
- max_gen_len=max_gen_len, max_enc_len=max_enc_len,
- constrained=False, ds=examples,
- )
- results[label] = {
- "name_f1": tc["name_f1"],
- "call_f1": tc["call_f1"],
- "exact_match": tc["exact_match"],
- "args_acc": tc["args_acc"],
- "value_acc": tc["value_acc"],
- "param_haluc": tc["param_haluc"],
- "param_miss": tc["param_miss"],
- "json_parse": tc["json_parse_rate"],
- "samples": tc["samples"],
- "n": len(examples),
- }
- return results
-
-
-def _print_eval_block(step, val_ppl, tc_results, pbar):
- """Mirror the end-of-epoch print layout from train.py."""
- pbar.write(f"\n ─────── step {step} eval ───────")
- if val_ppl is not None:
- pbar.write(f" val_ppl {val_ppl:>10.3f}")
- for label, m in tc_results.items():
- pbar.write(f" ─── {label} ({m['n']} samples, unconstrained) ───")
- pbar.write(f" JSON parse {m['json_parse']:>10.1%}")
- pbar.write(f" Name F1 {m['name_f1']:>10.1%}")
- pbar.write(f" Call F1 {m['call_f1']:>10.1%}")
- pbar.write(f" Exact match {m['exact_match']:>10.1%}")
- pbar.write(f" Args acc {m['args_acc']:>10.1%}")
- pbar.write(f" Value acc {m['value_acc']:>10.1%}")
- pbar.write(f" Param haluc {m['param_haluc']:>10.1%}")
- pbar.write(f" Param miss {m['param_miss']:>10.1%}")
- # Show up to 3 display samples from the "single" pool
- single = tc_results.get("single") or tc_results.get("multi")
- if single and single.get("samples"):
- pbar.write(f" ─── samples ───")
- for q, ref, pred in single["samples"][:3]:
- pbar.write(f" Q: {q}")
- pbar.write(f" R: {ref}")
- pbar.write(f" P: {pred}")
- pbar.write("")
-
-
-def _decode_sample(tokenizer, token_row):
- """Decode one sampled row to text, stopping at the first EOS after pos 0."""
- sampled = token_row[1:]
- eos_positions = np.where(sampled == EOS_ID)[0]
- end = int(eos_positions[0]) if len(eos_positions) > 0 else len(sampled)
- ids = [int(t) for t in sampled[:end] if t != PAD_ID]
- return tokenizer.decode(ids)
-
-
-def grpo(args):
- num_devices = jax.local_device_count()
- num_hosts = jax.process_count()
- host_id = jax.process_index()
- total_devices = jax.device_count()
- is_main = host_id == 0
-
- use_wandb = getattr(args, "wandb", False) and is_main
- if use_wandb:
- import wandb
- if wandb.run is None:
- wandb.init(project="needle-v1", name=args.name, config=vars(args))
-
- print(f"\n[1/4] Devices: {num_devices} local, {total_devices} total "
- f"across {num_hosts} host(s)", flush=True)
-
- print(f"\n[2/4] Loading checkpoint {args.checkpoint} ...", flush=True)
- ckpt_path = _download_sft_checkpoint(args.checkpoint)
- with open(ckpt_path, "rb") as f:
- data = pickle.load(f)
- config_dict = {k: v for k, v in data["config"].items()
- if k in TransformerConfig.__dataclass_fields__}
- config = TransformerConfig(**config_dict)
- model = EncoderDecoderTransformer(config)
-
- dt = jnp.bfloat16 if args.dtype == "bfloat16" else jnp.float32
- params = jax.tree.map(lambda x: jnp.array(x, dtype=dt), data["params"])
- ref_params = jax.tree.map(lambda x: jnp.array(x, dtype=dt), data["params"])
-
- param_count = sum(x.size for x in jax.tree.leaves(params))
- print(f" params: {param_count:,} "
- f"(d={config.d_model}, heads={config.num_heads}, "
- f"layers={config.num_encoder_layers}/{config.num_decoder_layers})")
-
- print(f"\n[3/4] Building optimizer + pmap ...", flush=True)
- total_steps = args.max_steps
- warmup = max(1, int(total_steps * 0.05))
- scaled_lr = args.lr * total_devices
- schedule = _wsd_schedule(scaled_lr, total_steps, warmup, decay_ratio=0.1)
- tx = optax.chain(
- optax.clip_by_global_norm(1.0),
- optax.adamw(schedule, b2=0.95, weight_decay=0.0),
- )
- state = train_state.TrainState.create(apply_fn=model.apply, params=params, tx=tx)
- del params
-
- r_state = _replicate(state)
- r_ref_params = _replicate(ref_params)
- del state, ref_params
-
- p_encode, p_sample_step, p_log_probs = _make_fns(model, args.max_gen_len)
- p_grpo_step = _make_grpo_step(model, args.beta)
-
- print(f"\n[4/4] Loading tokenizer + data ...", flush=True)
- tokenizer = get_tokenizer()
- train_ds = load_tool_calls("train")
- n_examples = len(train_ds)
- print(f" train: {n_examples:,} examples")
-
- val_data = None
- val_ds = None
- val_loss_fn = None
- if is_main:
- try:
- val_data = load_prepared_data("val", mmap=True)
- print(f" val (packed): {len(val_data['packed_enc']):,} bins")
- except FileNotFoundError:
- print(" WARN: no packed val data — skipping val PPL "
- "(run `needle tokenize` to generate)")
- try:
- val_ds = load_tool_calls("val")
- print(f" val (tool-call): {len(val_ds):,} examples")
- except Exception as e:
- print(f" WARN: could not load tool-call val set: {e}")
- val_loss_fn = _make_val_loss_fn(model)
-
- B = args.batch_size
- K = args.k
- BK = B * K
- if BK % num_devices != 0:
- raise ValueError(f"B*K ({BK}) must be divisible by num_devices ({num_devices})")
- per_device = BK // num_devices
-
- if is_main:
- print(f"\n ─────────────────────────────────────")
- print(f" GRPO")
- print(f" ─────────────────────────────────────")
- print(f" B (prompts/step) {B}")
- print(f" K (samples/prompt) {K}")
- print(f" B*K {BK} ({per_device}/device)")
- print(f" LR {args.lr} -> {scaled_lr}")
- print(f" beta (KL) {args.beta}")
- print(f" max_steps {args.max_steps}")
- print(f" max_enc_len {args.max_enc_len}")
- print(f" max_gen_len {args.max_gen_len}")
- print(f" ─────────────────────────────────────\n")
-
- rng = jax.random.PRNGKey(args.seed)
- perm = np.random.RandomState(args.seed).permutation(n_examples)
- cursor = 0
-
- best_call_f1 = -1.0
-
- pbar = tqdm(total=args.max_steps, disable=not is_main, desc="GRPO")
-
- for step in range(args.max_steps):
- if cursor + B > len(perm):
- perm = np.random.RandomState(args.seed + step).permutation(n_examples)
- cursor = 0
- batch_idx = perm[cursor:cursor + B]
- cursor += B
- batch = [train_ds[int(i)] for i in batch_idx]
-
- # Build encoder inputs, replicate K times per prompt, shard across devices.
- enc_np = _build_prompts(tokenizer, batch, args.max_enc_len)
- enc_rep = np.repeat(enc_np, K, axis=0)
- enc_sharded = jnp.asarray(
- enc_rep.reshape(num_devices, per_device, args.max_enc_len)
- )
-
- # Sample K completions per prompt under the current policy.
- t0 = time.perf_counter()
- dec_buffer_np, rng = _sample_completions(
- p_encode, p_sample_step, r_state.params, enc_sharded,
- num_devices, args.max_gen_len, rng,
- )
- sample_time = time.perf_counter() - t0
- dec_buffer_flat = dec_buffer_np.reshape(BK, args.max_gen_len)
-
- # Score each completion.
- rewards = np.zeros(BK, dtype=np.float32)
- for i in range(BK):
- prompt_idx = i // K
- text = _decode_sample(tokenizer, dec_buffer_flat[i])
- rewards[i] = _score(text, batch[prompt_idx]["answers"])
-
- # Group-relative advantages: normalize within each prompt's K samples.
- rewards_grp = rewards.reshape(B, K)
- mean = rewards_grp.mean(axis=1, keepdims=True)
- std = rewards_grp.std(axis=1, keepdims=True)
- advantages = ((rewards_grp - mean) / (std + 1e-4)).reshape(BK).astype(np.float32)
-
- # Teacher-forced shapes: dec_in = buffer[:-1], dec_tgt = buffer[1:].
- dec_in_np = dec_buffer_flat[:, :-1].astype(np.int32)
- dec_tgt_np = dec_buffer_flat[:, 1:].astype(np.int32)
- response_mask_np = (dec_tgt_np != PAD_ID).astype(np.float32)
- T_resp = dec_in_np.shape[1]
-
- dec_in_sh = jnp.asarray(dec_in_np.reshape(num_devices, per_device, T_resp))
- dec_tgt_sh = jnp.asarray(dec_tgt_np.reshape(num_devices, per_device, T_resp))
- resp_sh = jnp.asarray(response_mask_np.reshape(num_devices, per_device, T_resp))
- adv_sh = jnp.asarray(advantages.reshape(num_devices, per_device))
-
- ref_logp_sh = p_log_probs(r_ref_params, enc_sharded, dec_in_sh, dec_tgt_sh)
-
- r_state, loss, pg, kl = p_grpo_step(
- r_state, enc_sharded, dec_in_sh, dec_tgt_sh,
- adv_sh, resp_sh, ref_logp_sh,
- )
-
- loss_val = float(loss.addressable_shards[0].data[0])
- pg_val = float(pg.addressable_shards[0].data[0])
- kl_val = float(kl.addressable_shards[0].data[0])
- mean_r = float(rewards.mean())
- std_r = float(rewards.std())
-
- pbar.update(1)
- pbar.set_postfix(
- loss=f"{loss_val:+.4f}", pg=f"{pg_val:+.4f}",
- kl=f"{kl_val:.4f}", R=f"{mean_r:.3f}", sT=f"{sample_time:.1f}s",
- )
-
- if use_wandb:
- import wandb
- wandb.log({
- "grpo/loss": loss_val,
- "grpo/pg": pg_val,
- "grpo/kl": kl_val,
- "grpo/reward_mean": mean_r,
- "grpo/reward_std": std_r,
- "grpo/sample_time_s": sample_time,
- "grpo/step": step,
- })
-
- # Full eval — val PPL + stratified tool-call benchmark + samples.
- # Mirrors the end-of-epoch eval in train.py, run every eval_every steps.
- if is_main and (step + 1) % args.eval_every == 0:
- eval_params = _unreplicate(r_state).params
-
- val_ppl = None
- if val_data is not None:
- val_ppl = _compute_val_ppl(
- val_loss_fn, eval_params, val_data, args.eval_batch_size,
- )
-
- tc_results = {}
- if val_ds is not None:
- tc_results = _run_tool_call_eval(
- model, eval_params, tokenizer, val_ds,
- per_bucket=args.tool_call_samples // 2,
- max_enc_len=args.max_enc_len,
- max_gen_len=args.max_gen_len,
- )
-
- _print_eval_block(step + 1, val_ppl, tc_results, pbar)
-
- # Best-checkpoint tracking: match train.py, use single-call F1.
- cur_f1 = tc_results.get("single", {}).get("call_f1", -1.0)
- is_new_best = cur_f1 > best_call_f1
- if is_new_best and cur_f1 >= 0:
- best_call_f1 = cur_f1
- _save_grpo_checkpoint(
- r_state, config, args.checkpoint_dir, step + 1,
- filename="needle_grpo_best.pkl",
- )
- pbar.write(f" ** New best single call_f1={best_call_f1:.1%}")
-
- if use_wandb:
- import wandb
- log = {"grpo/step": step + 1, "val/best_call_f1": best_call_f1}
- if val_ppl is not None:
- log["val/text_ppl"] = val_ppl
- for label, metrics in tc_results.items():
- for k, v in metrics.items():
- if k in ("n", "samples"):
- continue
- log[f"val/{label}_{k}"] = v
- wandb.log(log)
-
- del eval_params
-
- pbar.close()
-
- if is_main:
- best_path = os.path.join(args.checkpoint_dir, "needle_grpo_best.pkl")
- if best_call_f1 >= 0:
- print(f"\nGRPO complete. Best single call_f1={best_call_f1:.1%} -> {best_path}")
- else:
- print(f"\nGRPO complete. No eval ran — no best checkpoint saved.")
-
- if use_wandb:
- import wandb
- wandb.finish()
-
- if num_hosts > 1:
- jax.experimental.multihost_utils.sync_global_devices("grpo_done")
-
-
-def _save_grpo_checkpoint(r_state, config, checkpoint_dir, step,
- filename="needle_grpo.pkl"):
- os.makedirs(checkpoint_dir, exist_ok=True)
- params = _unreplicate(r_state).params
- params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), params)
- ckpt_path = os.path.join(checkpoint_dir, filename)
- with open(ckpt_path, "wb") as f:
- pickle.dump({"params": params_np, "config": config.__dict__,
- "grpo_step": step}, f)
- size_mb = sum(x.nbytes for x in jax.tree.leaves(params_np)) / 1e6
- print(f"\n [step {step}] Saved {ckpt_path} ({size_mb:.1f} MB)", flush=True)
- _upload_checkpoint(ckpt_path)
- return ckpt_path
diff --git a/src/merge_xlam.py b/src/merge_xlam.py
deleted file mode 100644
index 08f6354..0000000
--- a/src/merge_xlam.py
+++ /dev/null
@@ -1,113 +0,0 @@
-#!/usr/bin/env python3
-"""Merge Salesforce/xlam-function-calling-60k into the existing unified dataset
-and re-upload to HuggingFace (Cactus-Compute/tool-calls).
-
-Usage:
- python scripts/merge_xlam.py # merge + upload
- python scripts/merge_xlam.py --dry-run # merge only, no upload
- python scripts/merge_xlam.py --max-samples 1000 # limit xlam samples (for testing)
-"""
-
-import argparse
-import json
-import os
-import sys
-
-from datasets import concatenate_datasets, load_dataset, load_from_disk
-
-LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data", "tool_calls_unified")
-HF_DATASET_REPO = "Cactus-Compute/tool-calls"
-
-
-def load_existing():
- local = os.path.abspath(LOCAL_UNIFIED_DIR)
- if os.path.exists(local) and any(f.endswith(".arrow") for f in os.listdir(local)):
- ds = load_from_disk(local)
- print(f"Loaded existing dataset from disk: {len(ds)} rows")
- else:
- from .data import download_hf_split
- print(f"Downloading existing dataset from {HF_DATASET_REPO}...")
- ds = download_hf_split("train", HF_DATASET_REPO)
- print(f"Downloaded: {len(ds)} rows")
- return ds
-
-
-def load_xlam(max_samples=None):
- print("Loading Salesforce/xlam-function-calling-60k...")
- ds = load_dataset("Salesforce/xlam-function-calling-60k", split="train")
- print(f"Loaded xlam: {len(ds)} rows")
-
- if max_samples and max_samples < len(ds):
- ds = ds.shuffle(seed=42).select(range(max_samples))
- print(f"Sampled down to {len(ds)} rows")
-
- if "id" in ds.column_names:
- ds = ds.remove_columns(["id"])
-
- ds = ds.add_column("source", ["xlam-function-calling-60k"] * len(ds))
- ds = ds.add_column("model", ["human-curated"] * len(ds))
-
- return ds
-
-
-def check_duplicates(existing, new):
- """Quick check for query overlap."""
- existing_queries = set(existing["query"][:10000])
- new_queries = set(new["query"])
- overlap = existing_queries & new_queries
- if overlap:
- print(f"Warning: {len(overlap)} overlapping queries found (sampling first 10k existing)")
- else:
- print("No duplicate queries detected (sampled check)")
-
-
-def main(args):
- existing = load_existing()
- xlam = load_xlam(args.max_samples)
-
- xlam = xlam.select_columns(existing.column_names)
-
- check_duplicates(existing, xlam)
-
- merged = concatenate_datasets([existing, xlam])
- print(f"\nMerged dataset: {len(merged)} rows ({len(existing)} existing + {len(xlam)} new)")
-
- from collections import Counter
- counts = Counter(merged["source"])
- print("\nBreakdown by source:")
- for src, cnt in counts.most_common():
- print(f" {src}: {cnt}")
-
- if args.dry_run:
- print("\n--dry-run: skipping save and upload")
- return
-
- import shutil
- local = os.path.abspath(LOCAL_UNIFIED_DIR)
- tmp_dir = local + "_tmp"
- if os.path.exists(tmp_dir):
- shutil.rmtree(tmp_dir)
- print(f"\nSaving to {local}...")
- merged.save_to_disk(tmp_dir)
- if os.path.exists(local):
- shutil.rmtree(local)
- os.rename(tmp_dir, local)
- print("Saved.")
-
- # Upload to HuggingFace (train split only)
- from huggingface_hub import HfApi
-
- api = HfApi()
- api.create_repo(HF_DATASET_REPO, repo_type="dataset", private=False, exist_ok=True)
- print(f"\nUploading to {HF_DATASET_REPO} (train split)...")
- merged.push_to_hub(HF_DATASET_REPO, split="train", token=True)
- print(f"Upload complete: {HF_DATASET_REPO}")
- print("NOTE: Run 'needle split-dataset' to create the validation split.")
-
-
-if __name__ == "__main__":
- import argparse as _ap
- _p = _ap.ArgumentParser(description="Merge xlam into unified dataset")
- _p.add_argument("--dry-run", action="store_true", help="Skip upload")
- _p.add_argument("--max-samples", type=int, default=None, help="Limit xlam samples")
- main(_p.parse_args())
\ No newline at end of file
diff --git a/src/mine_corrections.py b/src/mine_corrections.py
deleted file mode 100644
index 26ceb14..0000000
--- a/src/mine_corrections.py
+++ /dev/null
@@ -1,400 +0,0 @@
-"""Mine self-correction training pairs from the SFT checkpoint.
-
-For each training prompt, sample K completions at temperature T from the
-frozen SFT model, classify each against the gold, and write
-(query, tools, draft, gold, category) tuples to JSONL for downstream
-tokenization.
-
-Categories written: value_wrong, args_wrong, correct.
-Categories dropped: tool_wrong, unparseable. They are counted in the
-summary so you can see the distribution, but a corrector can't recover
-tool-selection or parse errors from a draft, so we don't train on them.
-
-Resumability: the script writes progress to <output>.cursor. On restart
-it reads the cursor, reopens the output in append mode, and resumes from
-the next unprocessed prompt. Crashing mid-chunk can cause up to one
-batch of duplicates — acceptable for mining, downstream dedup is cheap.
-
-Usage:
- python -m src.mine_corrections \\
- --checkpoint checkpoints/needle.pkl \\
- --output data/corrections.jsonl \\
- --max-samples 100000 \\
- --k 1 --temperature 0.7
-"""
-
-import argparse
-import json
-import os
-import time
-from collections import Counter
-
-import jax
-import jax.numpy as jnp
-import numpy as np
-from tqdm import tqdm
-
-from .data import get_tokenizer, load_tool_calls
-from .distributed import _replicate
-from .model import (
- EncoderDecoderTransformer,
- make_causal_mask,
- make_padding_mask,
-)
-from .run import _build_encoder_input, load_checkpoint
-from .tokenizer import EOS_ID, PAD_ID, to_snake_case
-
-
-_HF_SFT_REPO = "Cactus-Compute/needle"
-
-
-def _maybe_download_ckpt(path):
- if os.path.exists(path):
- return path
- from huggingface_hub import hf_hub_download
- local_dir = os.path.dirname(path) or "checkpoints"
- os.makedirs(local_dir, exist_ok=True)
- print(f" Downloading {os.path.basename(path)} from {_HF_SFT_REPO} ...")
- return hf_hub_download(
- repo_id=_HF_SFT_REPO,
- filename=os.path.basename(path),
- repo_type="model",
- local_dir=local_dir,
- )
-
-
-def _call_key_set(calls):
- keys = []
- for c in calls:
- if not isinstance(c, dict):
- continue
- name = to_snake_case(c.get("name", ""))
- args = c.get("arguments", {}) or {}
- keys.append(json.dumps({"name": name, "arguments": args}, sort_keys=True))
- return sorted(keys)
-
-
-def _classify(pred_text, ref_text):
- """Return one of: correct, value_wrong, args_wrong, tool_wrong, unparseable."""
- if pred_text.startswith("<tool_call>"):
- pred_text = pred_text[len("<tool_call>"):]
- try:
- pred_calls = json.loads(pred_text)
- if isinstance(pred_calls, dict):
- pred_calls = [pred_calls]
- elif not isinstance(pred_calls, list):
- return "unparseable"
- except (json.JSONDecodeError, TypeError):
- return "unparseable"
- try:
- ref_calls = json.loads(ref_text)
- if not isinstance(ref_calls, list):
- return "unparseable"
- except (json.JSONDecodeError, TypeError):
- return "unparseable"
-
- for c in pred_calls:
- if isinstance(c, dict) and isinstance(c.get("name"), str):
- c["name"] = to_snake_case(c["name"])
- else:
- return "unparseable"
- for c in ref_calls:
- if isinstance(c, dict) and isinstance(c.get("name"), str):
- c["name"] = to_snake_case(c["name"])
-
- if _call_key_set(pred_calls) == _call_key_set(ref_calls):
- return "correct"
-
- ref_names = sorted(c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c)
- pred_names = sorted(c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c)
- if ref_names != pred_names:
- return "tool_wrong"
-
- # Tool names match — check whether arg keys or just values differ
- ref_by_name, pred_by_name = {}, {}
- for c in ref_calls:
- if isinstance(c, dict) and "name" in c:
- ref_by_name.setdefault(c["name"], []).append(c.get("arguments", {}) or {})
- for c in pred_calls:
- if isinstance(c, dict) and "name" in c:
- pred_by_name.setdefault(c["name"], []).append(c.get("arguments", {}) or {})
-
- for name, ref_list in ref_by_name.items():
- pred_list = pred_by_name.get(name, [])
- for ra, pa in zip(ref_list, pred_list):
- if set((ra or {}).keys()) != set((pa or {}).keys()):
- return "args_wrong"
- return "value_wrong"
-
-
-def _make_sampling_fns(model, max_gen_len, temperature):
- """Build pmapped encode + sample-step functions with temperature baked in."""
- inv_t = 1.0 / max(temperature, 1e-5)
-
- @jax.pmap
- def p_encode(params, enc_input):
- src_mask = make_padding_mask(enc_input, PAD_ID)
- return model.apply(
- {"params": params}, enc_input,
- src_mask=src_mask, method="encode",
- )
-
- @jax.pmap
- def p_sample_step(params, dec_buffer, encoder_out, cross_mask,
- rng, step_idx, finished):
- sm = make_causal_mask(max_gen_len)
- logits = model.apply(
- {"params": params}, dec_buffer, encoder_out,
- self_mask=sm, cross_mask=cross_mask, method="decode",
- )
- next_logits = jax.lax.dynamic_index_in_dim(
- logits, step_idx, axis=1, keepdims=False,
- ) * inv_t
- next_tokens = jax.random.categorical(rng, next_logits, axis=-1)
- next_tokens = jnp.where(finished, PAD_ID, next_tokens)
- new_finished = finished | (next_tokens == EOS_ID)
- return next_tokens, new_finished
-
- return p_encode, p_sample_step
-
-
-def _sample_completions(p_encode, p_sample_step, r_params, enc_sharded,
- num_devices, max_gen_len, rng):
- encoder_out, cross_mask = p_encode(r_params, enc_sharded)
- per_device = enc_sharded.shape[1]
-
- dec_buffer = np.full((num_devices, per_device, max_gen_len), PAD_ID, dtype=np.int32)
- dec_buffer[:, :, 0] = EOS_ID
- dec_buffer = jnp.asarray(dec_buffer)
- finished = jnp.zeros((num_devices, per_device), dtype=jnp.bool_)
-
- for step in range(max_gen_len - 1):
- rng, sub = jax.random.split(rng)
- sub_per = jax.random.split(sub, num_devices)
- step_idx = jnp.full((num_devices,), step, dtype=jnp.int32)
- next_tokens, finished = p_sample_step(
- r_params, dec_buffer, encoder_out, cross_mask,
- sub_per, step_idx, finished,
- )
- dec_buffer = dec_buffer.at[:, :, step + 1].set(next_tokens)
-
- return np.asarray(dec_buffer), rng
-
-
-def _decode_sample(tokenizer, token_row):
- """Decode a sampled row and strip the leading <tool_call> task marker.
-
- Matches run.py's `generate_batch` post-processing: strip `<tool_call>`
- if present so the stored draft is the raw JSON. Without this the draft
- re-encodes to a double TOOL_CALL_ID at train time while inference (which
- strips it) sees a single one — a train/inference format mismatch.
- """
- sampled = token_row[1:]
- eos_positions = np.where(sampled == EOS_ID)[0]
- end = int(eos_positions[0]) if len(eos_positions) > 0 else len(sampled)
- ids = [int(t) for t in sampled[:end] if t != PAD_ID]
- text = tokenizer.decode(ids)
- if text.startswith("<tool_call>"):
- text = text[len("<tool_call>"):]
- return text
-
-
-def _read_cursor(path):
- if not os.path.exists(path):
- return 0
- with open(path) as f:
- try:
- return int(f.read().strip())
- except ValueError:
- return 0
-
-
-def _write_cursor(path, value):
- with open(path, "w") as f:
- f.write(str(value))
-
-
-def mine(args):
- if os.path.exists("/dev/accel0"):
- jax.distributed.initialize()
-
- num_devices = jax.local_device_count()
- print(f"Devices: {num_devices} local, {jax.device_count()} total")
-
- ckpt_path = _maybe_download_ckpt(args.checkpoint)
- print(f"Loading checkpoint {ckpt_path} ...")
- params, config = load_checkpoint(ckpt_path)
- model = EncoderDecoderTransformer(config)
- tokenizer = get_tokenizer()
-
- param_count = sum(x.size for x in jax.tree.leaves(params))
- print(f" Parameters: {param_count:,}")
- print(f" Config: d={config.d_model}, heads={config.num_heads}, "
- f"layers={config.num_encoder_layers}/{config.num_decoder_layers}")
-
- r_params = _replicate(params)
- del params
-
- p_encode, p_sample_step = _make_sampling_fns(
- model, args.max_gen_len, args.temperature)
-
- print("Loading train split ...")
- train_ds = load_tool_calls("train")
- n_total = len(train_ds)
- print(f" {n_total:,} rows (empty-answer examples skipped inline)")
-
- os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
- cursor_path = args.output + ".cursor"
- start_idx = _read_cursor(cursor_path)
- if start_idx > 0:
- print(f" Resuming from dataset index {start_idx}")
-
- B = args.batch_size
- K = args.k
- BK = B * K
- if BK % num_devices != 0:
- raise ValueError(
- f"B*K ({BK}) must be divisible by num_devices ({num_devices}). "
- f"Try --batch-size={num_devices * (BK // num_devices) // K}")
- per_device = BK // num_devices
-
- print(f" B={B} prompts, K={K} samples/prompt, BK={BK} ({per_device}/device)")
- print(f" temperature={args.temperature}, max_gen_len={args.max_gen_len}")
-
- rng = jax.random.PRNGKey(args.seed)
- stats = Counter()
-
- out_f = open(args.output, "a", encoding="utf-8", buffering=1)
- pbar_total = args.max_samples if args.max_samples else (n_total - start_idx)
- pbar = tqdm(total=pbar_total, desc="Mining", unit="prompts", initial=0)
- t0 = time.perf_counter()
- processed = 0
- last_ds_idx = start_idx - 1
-
- def _iter_prompts():
- for i in range(start_idx, n_total):
- ex = train_ds[int(i)]
- if ex["answers"].strip() in ("", "[]"):
- continue
- yield i, ex
-
- from itertools import islice
- prompt_iter = _iter_prompts()
-
- try:
- while True:
- pairs = list(islice(prompt_iter, B))
- if not pairs:
- break
- batch_ds_idx = [p[0] for p in pairs]
- batch = [p[1] for p in pairs]
- actual_B = len(batch)
- last_ds_idx = batch_ds_idx[-1]
-
- # Pad batch to full B by repeating last example (outputs ignored)
- while len(batch) < B:
- batch.append(batch[-1])
-
- enc_lists = [
- _build_encoder_input(tokenizer, ex["query"], ex["tools"],
- args.max_enc_len)
- for ex in batch
- ]
- enc_np = np.full((B, args.max_enc_len), PAD_ID, dtype=np.int32)
- for i, toks in enumerate(enc_lists):
- enc_np[i, :len(toks)] = toks
-
- enc_rep = np.repeat(enc_np, K, axis=0)
- enc_sharded = jnp.asarray(
- enc_rep.reshape(num_devices, per_device, args.max_enc_len))
-
- dec_buffer_np, rng = _sample_completions(
- p_encode, p_sample_step, r_params, enc_sharded,
- num_devices, args.max_gen_len, rng,
- )
- dec_buffer_flat = dec_buffer_np.reshape(BK, args.max_gen_len)
-
- for prompt_idx in range(actual_B):
- prompt = batch[prompt_idx]
- kept_correct = False
- for k in range(K):
- i = prompt_idx * K + k
- draft = _decode_sample(tokenizer, dec_buffer_flat[i])
- category = _classify(draft, prompt["answers"])
- stats[category] += 1
-
- if category in ("value_wrong", "args_wrong"):
- out_f.write(json.dumps({
- "query": prompt["query"],
- "tools": prompt["tools"],
- "draft": draft,
- "answers": prompt["answers"],
- "category": category,
- "source_idx": int(batch_ds_idx[prompt_idx]),
- }, ensure_ascii=False) + "\n")
- elif category == "correct" and not kept_correct:
- kept_correct = True
- out_f.write(json.dumps({
- "query": prompt["query"],
- "tools": prompt["tools"],
- "draft": draft,
- "answers": prompt["answers"],
- "category": "correct",
- "source_idx": int(batch_ds_idx[prompt_idx]),
- }, ensure_ascii=False) + "\n")
-
- _write_cursor(cursor_path, last_ds_idx + 1)
- pbar.update(actual_B)
- processed += actual_B
-
- if processed % (B * 10) == 0:
- elapsed = time.perf_counter() - t0
- rate = processed / max(elapsed, 1e-6)
- total_gen = max(sum(stats.values()), 1)
- pbar.set_postfix(
- val=f"{stats['value_wrong']} ({100*stats['value_wrong']/total_gen:.0f}%)",
- args=f"{stats['args_wrong']} ({100*stats['args_wrong']/total_gen:.0f}%)",
- corr=f"{stats['correct']} ({100*stats['correct']/total_gen:.0f}%)",
- tool=f"{stats['tool_wrong']} ({100*stats['tool_wrong']/total_gen:.0f}%)",
- rate=f"{rate:.1f}/s",
- )
-
- if args.max_samples and processed >= args.max_samples:
- break
- finally:
- out_f.close()
- pbar.close()
- total = sum(stats.values())
- written = stats["value_wrong"] + stats["args_wrong"] + stats["correct"]
- print("\n=== Final stats ===")
- for k in ("correct", "value_wrong", "args_wrong", "tool_wrong",
- "unparseable"):
- v = stats.get(k, 0)
- pct = 100 * v / max(total, 1)
- print(f" {k:<14} {v:>10} ({pct:>5.1f}%)")
- print(f" total generations {total:>10}")
- print(f" rows written {written:>10} -> {args.output}")
-
-
-def parse_args():
- p = argparse.ArgumentParser(
- description="Mine self-correction pairs from an SFT checkpoint")
- p.add_argument("--checkpoint", type=str, default="checkpoints/needle.pkl")
- p.add_argument("--output", type=str, default="data/corrections.jsonl")
- p.add_argument("--max-samples", type=int, default=None,
- help="Limit number of training prompts (default: all)")
- p.add_argument("--k", type=int, default=1,
- help="Samples per prompt (default: 1)")
- p.add_argument("--temperature", type=float, default=0.7)
- p.add_argument("--batch-size", type=int, default=256,
- help="Prompts per forward pass; B*K must divide num_devices")
- p.add_argument("--max-enc-len", type=int, default=1024)
- p.add_argument("--max-gen-len", type=int, default=256,
- help="Generation cap — most tool calls fit in 256; "
- "raise if you see truncations")
- p.add_argument("--seed", type=int, default=42)
- return p.parse_args()
-
-
-if __name__ == "__main__":
- mine(parse_args())
diff --git a/src/rebalance_tools.py b/src/rebalance_tools.py
deleted file mode 100644
index 58be96d..0000000
--- a/src/rebalance_tools.py
+++ /dev/null
@@ -1,225 +0,0 @@
-#!/usr/bin/env python3
-"""Rebalance tool count distribution by trimming over-represented bins.
-
-- 2/3 of 3-tool examples → trimmed to 1 or 2 tools
-- 2/3 of 10-tool examples → trimmed to 5-9 tools
-
-Keeps called tools + random subset of uncalled tools to reach target count.
-Uploads the rebalanced dataset back to HuggingFace.
-
-Usage:
- python scripts/rebalance_tools.py
- python scripts/rebalance_tools.py --dry-run
-"""
-
-import argparse
-import json
-import os
-import random
-from collections import Counter
-
-from datasets import Dataset, concatenate_datasets
-from tqdm import tqdm
-
-HF_REPO = "Cactus-Compute/tool-calls"
-
-
-def _get_called_names(answers_str):
- """Extract set of called tool names from answers JSON."""
- try:
- calls = json.loads(answers_str)
- except (ValueError, TypeError):
- return set()
- if not isinstance(calls, list):
- return set()
- return {c["name"] for c in calls if isinstance(c, dict) and "name" in c}
-
-
-def _trim_tools(tools_str, called_names, target_n, rng):
- """Trim tools list to target_n, keeping all called tools + random fillers."""
- try:
- tools = json.loads(tools_str)
- except (ValueError, TypeError):
- return tools_str
- if not isinstance(tools, list):
- return tools_str
-
- # Separate called vs uncalled
- called = [t for t in tools if isinstance(t, dict) and t.get("name") in called_names]
- uncalled = [t for t in tools if isinstance(t, dict) and t.get("name") not in called_names]
-
- # If we need fewer tools than called tools, can't trim — skip
- if target_n < len(called):
- return None
-
- # Pick random uncalled fillers to reach target
- n_fillers = target_n - len(called)
- if n_fillers > len(uncalled):
- return None # not enough uncalled tools
-
- fillers = rng.sample(uncalled, n_fillers)
- new_tools = called + fillers
- rng.shuffle(new_tools)
- return json.dumps(new_tools, separators=(",", ":"))
-
-
-def rebalance(dry_run=False):
- from .data import download_hf_split
- print(f"Loading dataset from {HF_REPO}...")
- ds = download_hf_split("train", HF_REPO)
- print(f"Loaded {len(ds):,} examples")
-
- rng = random.Random(42)
-
- # Categorize by tool count
- indices_by_n = {}
- for i, ex in enumerate(tqdm(ds, desc="Categorizing")):
- try:
- tools = json.loads(ex["tools"])
- n = len(tools) if isinstance(tools, list) else 0
- except (ValueError, TypeError):
- n = 0
- indices_by_n.setdefault(n, []).append(i)
-
- print("\nBefore rebalancing:")
- for n in sorted(indices_by_n.keys()):
- if n <= 10:
- print(f" {n:>2} tools: {len(indices_by_n[n]):>10,}")
-
- # Plan: which indices to modify
- modify_3 = [] # (index, target_n)
- modify_10 = [] # (index, target_n)
-
- # 2/3 of 3-tool → trim to 1 or 2
- if 3 in indices_by_n:
- pool = list(indices_by_n[3])
- rng.shuffle(pool)
- n_modify = len(pool) * 2 // 3
- for idx in pool[:n_modify]:
- target = rng.choice([1, 2])
- modify_3.append((idx, target))
-
- # 2/3 of 10-tool → trim to 5-9
- if 10 in indices_by_n:
- pool = list(indices_by_n[10])
- rng.shuffle(pool)
- n_modify = len(pool) * 2 // 3
- for idx in pool[:n_modify]:
- target = rng.randint(5, 9)
- modify_10.append((idx, target))
-
- all_modifications = modify_3 + modify_10
- print(f"\nModifications planned:")
- print(f" 3-tool → 1 or 2: {len(modify_3):,}")
- print(f" 10-tool → 5-9: {len(modify_10):,}")
- print(f" Total: {len(all_modifications):,}")
-
- if dry_run:
- # Preview target distribution
- target_counts = Counter()
- for n in sorted(indices_by_n.keys()):
- target_counts[n] = len(indices_by_n[n])
- for idx, target in all_modifications:
- ex = ds[idx]
- try:
- tools = json.loads(ex["tools"])
- orig_n = len(tools)
- except:
- continue
- target_counts[orig_n] -= 1
- target_counts[target] += 1
-
- print("\nProjected distribution:")
- for n in sorted(target_counts.keys()):
- if n <= 10 and target_counts[n] > 0:
- total = sum(target_counts.values())
- print(f" {n:>2} tools: {target_counts[n]:>10,} ({target_counts[n]/total*100:5.1f}%)")
- return
-
- # Apply modifications
- mod_map = {idx: target for idx, target in all_modifications}
- modified_indices = set(mod_map.keys())
-
- new_tools_col = []
- skipped = 0
- for i in tqdm(range(len(ds)), desc="Applying"):
- ex = ds[i]
- if i in modified_indices:
- called = _get_called_names(ex["answers"])
- new_tools = _trim_tools(ex["tools"], called, mod_map[i], rng)
- if new_tools is None:
- new_tools_col.append(ex["tools"]) # keep original
- skipped += 1
- else:
- new_tools_col.append(new_tools)
- else:
- new_tools_col.append(ex["tools"])
-
- if skipped:
- print(f" Skipped {skipped} examples (couldn't trim)")
-
- # Build new dataset
- new_ds = ds.remove_columns(["tools"]).add_column("tools", new_tools_col)
- # Reorder columns to match original
- new_ds = new_ds.select_columns(ds.column_names)
-
- # Verify
- print("\nAfter rebalancing:")
- verify_counts = Counter()
- for ex in tqdm(new_ds, desc="Verifying"):
- try:
- tools = json.loads(ex["tools"])
- n = len(tools) if isinstance(tools, list) else 0
- except:
- n = 0
- verify_counts[n] += 1
- for n in sorted(verify_counts.keys()):
- if n <= 10:
- total = sum(verify_counts.values())
- print(f" {n:>2} tools: {verify_counts[n]:>10,} ({verify_counts[n]/total*100:5.1f}%)")
-
- # Save locally then upload via HfApi to avoid push_to_hub dataset card bug
- import tempfile
- from huggingface_hub import HfApi, CommitOperationDelete
-
- local_dir = tempfile.mkdtemp(prefix="rebalanced_")
- print(f"\nSaving to {local_dir}...")
- new_ds.to_parquet(os.path.join(local_dir, "train.parquet"))
-
- api = HfApi()
-
- # Delete old train shards
- files = api.list_repo_files(HF_REPO, repo_type="dataset", token=True)
- old_train = [f for f in files if f.startswith("data/train-")]
- if old_train:
- print(f"Deleting {len(old_train)} old train shards...")
- ops = [CommitOperationDelete(path_in_repo=f) for f in old_train]
- api.create_commit(
- repo_id=HF_REPO, repo_type="dataset", operations=ops,
- commit_message="Remove old train shards", token=True,
- )
-
- # Upload new train data
- print(f"Uploading rebalanced train split...")
- api.upload_file(
- path_or_fileobj=os.path.join(local_dir, "train.parquet"),
- path_in_repo="data/train-00000-of-00001.parquet",
- repo_id=HF_REPO, repo_type="dataset", token=True,
- commit_message="Upload rebalanced train data",
- )
-
- # Clean up
- import shutil
- shutil.rmtree(local_dir)
- print("Done.")
-
-
-def main(args):
- rebalance(dry_run=args.dry_run)
-
-
-if __name__ == "__main__":
- import argparse as _ap
- _p = _ap.ArgumentParser()
- _p.add_argument("--dry-run", action="store_true", help="Preview without modifying")
- main(_p.parse_args())
diff --git a/src/split_dataset.py b/src/split_dataset.py
deleted file mode 100644
index 723bf1c..0000000
--- a/src/split_dataset.py
+++ /dev/null
@@ -1,115 +0,0 @@
-"""Split the existing Cactus-Compute/tool-calls dataset into train/validation splits.
-
-Shuffles the full dataset with a fixed seed, then takes 10k rows as validation.
-This ensures validation is representative of all data sources, not just the latest append.
-Uploads via HfApi to avoid the push_to_hub dataset card bug.
-
-Usage:
- python scripts/split_dataset.py
-"""
-
-import os
-import shutil
-import tempfile
-
-HF_DATASET_REPO = "Cactus-Compute/tool-calls"
-VAL_PER_SOURCE = {
- "gemini-3.1-flash-lite": 5000,
- "xlam": 2500,
- "xlam-translated": 2500,
-}
-SEED = 42
-
-
-def main(args=None):
- val_per_source = dict(VAL_PER_SOURCE)
- if args is not None and getattr(args, "val_per_source", None) is not None:
- # CLI override applies uniformly to all sources
- val_per_source = {k: args.val_per_source for k in val_per_source}
-
- print(f"Downloading full dataset from {HF_DATASET_REPO} (train split only)...")
- from .data import download_hf_split
- ds = download_hf_split("train", HF_DATASET_REPO)
- n = len(ds)
- print(f"Total rows: {n:,}")
-
- # Stratified split: equal samples per source for balanced validation
- from collections import Counter
- source_counts = Counter(ds["source"])
- print(f"\nSource distribution:")
- for src, cnt in source_counts.most_common():
- print(f" {src}: {cnt:,}")
-
- # Group indices by source
- source_indices = {}
- for i, src in enumerate(ds["source"]):
- source_indices.setdefault(src, []).append(i)
-
- import random
- rng = random.Random(SEED)
-
- val_indices = []
- for src, indices in sorted(source_indices.items()):
- rng.shuffle(indices)
- take = min(val_per_source.get(src, 0), len(indices))
- val_indices.extend(indices[:take])
- print(f" {src}: taking {take} for validation")
-
- val_set = set(val_indices)
- train_indices = [i for i in range(n) if i not in val_set]
-
- train_ds = ds.select(train_indices)
- val_ds = ds.select(val_indices)
-
- print(f"\nTrain: {len(train_ds):,} rows")
- print(f"Validation: {len(val_ds):,} rows")
-
- val_sources = Counter(val_ds["source"])
- print(f"\nValidation source distribution:")
- for src, cnt in val_sources.most_common():
- print(f" {src}: {cnt} ({cnt/len(val_ds)*100:.1f}%)")
-
- # Upload via HfApi (avoids push_to_hub dataset card bug)
- from huggingface_hub import HfApi, CommitOperationDelete
-
- api = HfApi()
- api.create_repo(HF_DATASET_REPO, repo_type="dataset", private=False, exist_ok=True)
-
- parquet_dir = tempfile.mkdtemp(prefix="needle_split_")
- print(f"\nExporting to parquet...")
- train_ds.to_parquet(os.path.join(parquet_dir, "train.parquet"))
- val_ds.to_parquet(os.path.join(parquet_dir, "validation.parquet"))
-
- # Delete old data shards
- files = api.list_repo_files(HF_DATASET_REPO, repo_type="dataset", token=True)
- old_shards = [f for f in files if f.startswith("data/")]
- if old_shards:
- print(f"Deleting {len(old_shards)} old shards...")
- ops = [CommitOperationDelete(path_in_repo=f) for f in old_shards]
- api.create_commit(
- repo_id=HF_DATASET_REPO, repo_type="dataset", operations=ops,
- commit_message="Remove old shards before split upload", token=True,
- )
-
- print(f"Uploading train split ({len(train_ds):,} rows)...")
- api.upload_file(
- path_or_fileobj=os.path.join(parquet_dir, "train.parquet"),
- path_in_repo="data/train-00000-of-00001.parquet",
- repo_id=HF_DATASET_REPO, repo_type="dataset", token=True,
- commit_message=f"Upload train split ({len(train_ds):,} rows)",
- )
-
- print(f"Uploading validation split ({len(val_ds):,} rows)...")
- api.upload_file(
- path_or_fileobj=os.path.join(parquet_dir, "validation.parquet"),
- path_in_repo="data/validation-00000-of-00001.parquet",
- repo_id=HF_DATASET_REPO, repo_type="dataset", token=True,
- commit_message=f"Upload validation split ({len(val_ds):,} rows)",
- )
-
- shutil.rmtree(parquet_dir)
- print("Done.")
-
-
-if __name__ == "__main__":
- main()
diff --git a/src/train.py b/src/train.py
index e104840..6f2deeb 100644
--- a/src/train.py
+++ b/src/train.py
@@ -1163,23 +1163,4 @@ def train(args):
if is_main:
print("\nTraining complete.")
- # Auto-calibrate confidence head on best checkpoint
- if is_main and getattr(args, "calibrate", True) and best_ckpt_path:
- print(f"\n{'='*50}")
- print(f"Running confidence head calibration on {best_ckpt_path}")
- print(f"{'='*50}")
- from .calibrate import main as calibrate_main
- from argparse import Namespace
- cal_args = Namespace(
- checkpoint=best_ckpt_path,
- output=best_ckpt_path,
- batch_size=args.batch_size * 4,
- num_samples=None, # use full dataset
- epochs=getattr(args, "calibrate_epochs", 1),
- lr=getattr(args, "calibrate_lr", 1e-3),
- k=getattr(args, "calibrate_k", 3.0),
- )
- calibrate_main(cal_args)
- print(f"\nCalibration complete. Checkpoint updated: {best_ckpt_path}")
-
diff --git a/src/train_corrector.py b/src/train_corrector.py
deleted file mode 100644
index d39bee2..0000000
--- a/src/train_corrector.py
+++ /dev/null
@@ -1,854 +0,0 @@
-"""Stage-2 correction fine-tuning.
-
-Reads the JSONL produced by src.mine_corrections, tokenizes on-the-fly,
-and fine-tunes a SEPARATE corrector model initialized from an SFT
-checkpoint. The base SFT ckpt is not modified. Inference at deploy time
-runs both models in sequence (base -> draft -> corrector).
-
-Encoder input: [query, <tools>, tools_json, <tool_call>, draft_json]
-Decoder input: [EOS, <tool_call>, gold_answer]
-Decoder target: [<tool_call>, gold_answer, EOS]
-
-The encoder uses <tool_call> (already in the tokenizer vocab) as the
-tools->draft delimiter — no tokenizer changes required.
-
-Eval mirrors train.py exactly: same stratified single/multi-call sampling
-(50 per tool-count bucket, rng seed 42), same tool-call metrics, same
-display samples. At init we run the frozen base SFT ckpt once over the
-full val set and cache the drafts. Every subsequent eval step just
-re-runs the CORRECTOR on those cached drafts — fast.
-
-The eval report prints both `draft` (base only, baseline) and
-`corrected` (base + corrector) side by side, so the improvement or
-regression per bucket is a direct read.
-
-Usage:
- python -m src.train_corrector \\
- --base-checkpoint checkpoints/needle.pkl \\
- --corrector-init checkpoints/needle.pkl \\
- --data data/corrections.jsonl \\
- --epochs 1 --lr 1e-4 --batch-size 64 \\
- --wandb
-"""
-
-import argparse
-import json
-import math
-import os
-import pickle
-import queue
-import threading
-import time
-from collections import Counter
-
-import jax
-import jax.numpy as jnp
-import numpy as np
-import optax
-from tqdm import tqdm
-
-from .data import (
- _token_classes_for_answer, get_tokenizer, load_tool_calls,
-)
-from .distributed import (
- _replicate, _unreplicate, shard_batch, _upload_checkpoint,
-)
-from .model import (
- EncoderDecoderTransformer, TransformerConfig,
- make_causal_mask, make_padding_mask,
-)
-from .optim import create_train_state, _wsd_schedule
-from .run import generate_batch, load_checkpoint
-from .tokenizer import EOS_ID, PAD_ID, TOOL_CALL_ID, TOOLS_ID
-
-
-_LOSS_WEIGHT_MAP = jnp.array([1.0, 2.0, 4.0, 1.5], dtype=jnp.float32)
-
-
-# ───────────────────────── filter: unrecoverable hallucinations ─────────────────────────
-
-import re as _re
-
-_ID_WITH_DIGIT = _re.compile(r"[a-zA-Z]_[a-zA-Z0-9]*\d")
-_OPAQUE_CODE = _re.compile(r"^[A-Za-z0-9_-]+$")
-
-
-def _looks_id_like(value):
- """Detect ID-shaped values: emails, URLs, paths, coded names, compound IDs."""
- if not isinstance(value, str) or len(value) < 4:
- return False
- if "@" in value:
- left, _, right = value.rpartition("@")
- if left and "." in right:
- return True
- if value.startswith(("http://", "https://", "www.")):
- return True
- if "/" in value or "\\" in value:
- return True
- if _ID_WITH_DIGIT.search(value):
- return True
- if len(value) > 15 and _OPAQUE_CODE.match(value) and any(c.isdigit() for c in value):
- return True
- if value.count("_") >= 2 and len(value) >= 10:
- return True
- return False
-
-
-def _value_in_query(value, query):
- if not isinstance(value, str):
- return True
- return value.lower() in query.lower()
-
-
-def _gold_has_unrecoverable(query, gold_text):
- """True if gold contains any ID-shaped arg value that isn't in the query."""
- try:
- gold_calls = json.loads(gold_text)
- except (json.JSONDecodeError, TypeError):
- return False
- if not isinstance(gold_calls, list):
- return False
- for call in gold_calls:
- if not isinstance(call, dict):
- continue
- args = call.get("arguments", {}) or {}
- if not isinstance(args, dict):
- continue
- for _, value in args.items():
- if _looks_id_like(value) and not _value_in_query(value, query):
- return True
- return False
-
-
-# ───────────────────────── encoder format ─────────────────────────
-
-def _build_correction_encoder_input(tokenizer, query, tools, draft, max_enc_len,
- query_cap=256, draft_min=128):
- """Build [query, <tools>, tools_json, <tool_call>, draft_json] up to max_enc_len.
-
- Truncation policy: query gets up to `query_cap` tokens, then tools and
- draft share the remaining budget with tools getting priority but
- always leaving at least `draft_min` tokens for the draft.
- """
- q_toks = tokenizer.encode(query)[:query_cap]
- t_toks = tokenizer.encode(tools)
- d_toks = tokenizer.encode(draft)
-
- budget = max_enc_len - len(q_toks) - 2 # two separator tokens
- tools_cap = max(0, budget - draft_min)
- t_toks = t_toks[:min(len(t_toks), tools_cap)]
- budget -= len(t_toks)
- d_toks = d_toks[:max(0, budget)]
- return q_toks + [TOOLS_ID] + t_toks + [TOOL_CALL_ID] + d_toks
-
-
-def _tokenize_example(tokenizer, sp_model, row, max_enc_len, max_dec_len):
- """Build padded (enc, dec_in, dec_tgt, loss_mask) arrays for one row.
-
- Returns None if the row is filtered (unrecoverable hallucination) or the
- decoder target doesn't fit in max_dec_len.
- """
- # Retroactive cleanup: drafts from pre-fix mining runs have a <tool_call>
- # prefix that would re-encode to a duplicate TOOL_CALL_ID token. Strip it.
- draft = row.get("draft", "")
- if isinstance(draft, str) and draft.startswith("<tool_call>"):
- draft = draft[len("<tool_call>"):]
-
- # Drop rows where the gold fabricates an ID-shaped value that isn't in the
- # query. Pass-through correct rows are kept unconditionally — they teach
- # "don't change a correct answer" even if the gold has an ID-looking value.
- if row.get("category") != "correct":
- if _gold_has_unrecoverable(row.get("query", ""), row.get("answers", "")):
- return None
-
- enc_tokens = _build_correction_encoder_input(
- tokenizer, row["query"], row["tools"], draft, max_enc_len)
-
- answer = row["answers"]
- a_toks = tokenizer.encode(answer)
- if 2 + len(a_toks) + 1 > max_dec_len:
- return None
-
- dec_in_tokens = [EOS_ID, TOOL_CALL_ID] + list(a_toks)
- dec_tgt_tokens = [TOOL_CALL_ID] + list(a_toks) + [EOS_ID]
- loss_classes_full = np.zeros(len(dec_tgt_tokens), dtype=np.int8)
- token_classes = _token_classes_for_answer(answer, a_toks, sp_model)
- loss_classes_full[1:1 + len(token_classes)] = token_classes
-
- enc = np.full(max_enc_len, PAD_ID, dtype=np.int32)
- enc[:len(enc_tokens)] = enc_tokens
- dec_in = np.full(max_dec_len, PAD_ID, dtype=np.int32)
- dec_in[:len(dec_in_tokens)] = dec_in_tokens
- dec_tgt = np.full(max_dec_len, PAD_ID, dtype=np.int32)
- dec_tgt[:len(dec_tgt_tokens)] = dec_tgt_tokens
- loss = np.zeros(max_dec_len, dtype=np.int32)
- loss[:len(loss_classes_full)] = loss_classes_full
- return enc, dec_in, dec_tgt, loss
-
-
-# ───────────────────────── streaming loader ─────────────────────────
-
-def _read_jsonl_shuffled(path, seed):
- """Generator over (row_dict) in shuffled order. Loads full index into memory."""
- # Build line-offset index for random access
- offsets = []
- with open(path, "rb") as f:
- pos = 0
- for line in f:
- offsets.append(pos)
- pos += len(line)
- rng = np.random.RandomState(seed)
- order = rng.permutation(len(offsets))
- with open(path, "rb") as f:
- for idx in order:
- f.seek(offsets[idx])
- line = f.readline()
- try:
- yield json.loads(line)
- except json.JSONDecodeError:
- continue
-
-
-def _stream_batches(tokenizer, sp_model, path, batch_size,
- max_enc_len, max_dec_len, seed):
- """Yield (enc, dec_in, dec_tgt, loss_mask) numpy batches from a JSONL file."""
- enc_batch = np.full((batch_size, max_enc_len), PAD_ID, dtype=np.int32)
- dec_in_batch = np.full((batch_size, max_dec_len), PAD_ID, dtype=np.int32)
- dec_tgt_batch = np.full((batch_size, max_dec_len), PAD_ID, dtype=np.int32)
- loss_batch = np.zeros((batch_size, max_dec_len), dtype=np.int32)
- idx = 0
-
- for row in _read_jsonl_shuffled(path, seed):
- result = _tokenize_example(tokenizer, sp_model, row,
- max_enc_len, max_dec_len)
- if result is None:
- continue
- enc, di, dt, lm = result
- enc_batch[idx] = enc
- dec_in_batch[idx] = di
- dec_tgt_batch[idx] = dt
- loss_batch[idx] = lm
- idx += 1
- if idx == batch_size:
- yield (enc_batch.copy(), dec_in_batch.copy(),
- dec_tgt_batch.copy(), loss_batch.copy())
- enc_batch[:] = PAD_ID
- dec_in_batch[:] = PAD_ID
- dec_tgt_batch[:] = PAD_ID
- loss_batch[:] = 0
- idx = 0
-
-
-class _PrefetchStream:
- def __init__(self, gen_fn, prefetch=4):
- self._queue = queue.Queue(maxsize=prefetch)
- self._stop = threading.Event()
- self._thread = threading.Thread(target=self._produce, args=(gen_fn,), daemon=True)
- self._thread.start()
-
- def _produce(self, gen_fn):
- try:
- for batch in gen_fn():
- if self._stop.is_set():
- return
- self._queue.put(batch)
- self._queue.put(None)
- except Exception as e:
- self._queue.put(e)
-
- def __iter__(self): return self
- def __next__(self):
- item = self._queue.get()
- if item is None:
- raise StopIteration
- if isinstance(item, Exception):
- raise item
- return item
-
- def close(self):
- self._stop.set()
-
-
-# ───────────────────────── train step ─────────────────────────
-
-def _train_step(state, src, tgt_in, tgt_out, loss_mask, rng):
- """CE + z_loss with token-class weighting. Mirrors train.py."""
-
- def loss_fn(params):
- src_mask = make_padding_mask(src, PAD_ID)
- tgt_mask = make_causal_mask(tgt_in.shape[1]) & make_padding_mask(tgt_in, PAD_ID)
- logits = state.apply_fn(
- {"params": params}, src, tgt_in,
- src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=src_mask,
- )
- logits_f32 = logits.astype(jnp.float32)
- token_weights = _LOSS_WEIGHT_MAP[loss_mask]
- padding_mask = (tgt_out != PAD_ID).astype(jnp.float32)
- mask = token_weights * padding_mask
- num_tokens = jnp.maximum(jnp.sum(padding_mask), 1.0)
- ce = jnp.sum(
- optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask
- ) / num_tokens
- z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)
- return ce + z_loss
-
- loss, grads = jax.value_and_grad(loss_fn)(state.params)
- grads = jax.lax.pmean(grads, axis_name="batch")
- loss = jax.lax.pmean(loss, axis_name="batch")
- grad_norm = optax.global_norm(grads)
- state = state.apply_gradients(grads=grads)
- return state, loss, grad_norm
-
-
-# ───────────────────────── eval: stratified sampling ─────────────────────────
-
-def _stratified_val_pools(val_ds, per_bucket, seed):
- """Return (single_examples, multi_examples) sampled identically to train.py.
-
- Same bucketing by min(len(tools), 10), same rng math.
- """
- pool_names = ["single", "multi"]
- buckets = {n: {t: [] for t in range(11)} for n in pool_names}
-
- for k in range(len(val_ds)):
- ex = val_ds[k]
- try:
- answers = json.loads(ex["answers"])
- except (ValueError, TypeError):
- continue
- if not answers or not isinstance(answers, list):
- continue
- call_type = "single" if len(answers) == 1 else "multi"
- try:
- nc = min(len(json.loads(ex["tools"])), 10)
- except (ValueError, TypeError):
- nc = 0
- buckets[call_type][nc].append(k)
-
- rng = np.random.RandomState(seed)
- out = {}
- for name in pool_names:
- pool = []
- for t in range(11):
- b = np.array(buckets[name][t])
- if len(b) > 0:
- rng.shuffle(b)
- pool.extend(b[:per_bucket].tolist())
- rng.shuffle(pool)
- out[name] = [val_ds[k] for k in pool]
- return out["single"], out["multi"]
-
-
-# ───────────────────────── eval: metrics ─────────────────────────
-
-_PC_KEYS = ("n", "exact", "name_tp", "name_fp", "name_fn",
- "call_tp", "call_fp", "call_fn", "parse_err")
-
-
-def _call_key(c):
- if not isinstance(c, dict):
- return None
- return json.dumps({"name": c.get("name"), "arguments": c.get("arguments")},
- sort_keys=True)
-
-
-def _score_pool(examples, preds):
- """Compute tool-call metrics for one pool. Mirrors train.py._eval_pool."""
- m_n = m_exact = m_parse_err = 0
- m_name_tp = m_name_fp = m_name_fn = 0
- m_call_tp = m_call_fp = m_call_fn = 0
- m_args_correct = m_args_total = 0
- m_halluc = m_total_pred_params = 0
- m_missing = m_total_ref_params = 0
- m_correct_values = m_matched_params = 0
- m_per_count = {t: {k: 0 for k in _PC_KEYS} for t in range(11)}
-
- for ex, pred_text in zip(examples, preds):
- try:
- tool_defs = json.loads(ex["tools"])
- num_tools = min(len(tool_defs), 10)
- except (ValueError, TypeError):
- tool_defs = []
- num_tools = 0
- pc = m_per_count[num_tools]
-
- ref_text = ex["answers"].strip()
- pred_text = pred_text.strip()
- ref_is_empty = ref_text in ("", "[]")
- pred_is_empty = pred_text in ("", "[]")
-
- try:
- ref_calls = json.loads(ref_text) if not ref_is_empty else []
- except (ValueError, TypeError):
- ref_calls = []
- try:
- pred_calls = json.loads(pred_text) if not pred_is_empty else []
- if not isinstance(pred_calls, list):
- pred_calls = [pred_calls] if isinstance(pred_calls, dict) else []
- except (ValueError, TypeError):
- m_parse_err += 1
- pc["parse_err"] += 1
- pred_calls = []
-
- m_n += 1
- pc["n"] += 1
- if ref_is_empty and pred_is_empty:
- m_exact += 1
- pc["exact"] += 1
- elif not ref_is_empty and not pred_is_empty:
- r = sorted([_call_key(c) for c in ref_calls if _call_key(c)])
- p = sorted([_call_key(c) for c in pred_calls if _call_key(c)])
- if r == p and len(r) == len(ref_calls) and len(p) == len(pred_calls):
- m_exact += 1
- pc["exact"] += 1
-
- ref_names = {c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c}
- pred_names = {c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c}
- m_name_tp += len(pred_names & ref_names); pc["name_tp"] += len(pred_names & ref_names)
- m_name_fp += len(pred_names - ref_names); pc["name_fp"] += len(pred_names - ref_names)
- m_name_fn += len(ref_names - pred_names); pc["name_fn"] += len(ref_names - pred_names)
-
- rk = {_call_key(c) for c in ref_calls} - {None}
- pk = {_call_key(c) for c in pred_calls} - {None}
- m_call_tp += len(pk & rk); pc["call_tp"] += len(pk & rk)
- m_call_fp += len(pk - rk); pc["call_fp"] += len(pk - rk)
- m_call_fn += len(rk - pk); pc["call_fn"] += len(rk - pk)
-
- ref_by_name = {}
- for c in ref_calls:
- if isinstance(c, dict) and "name" in c:
- ref_by_name.setdefault(c["name"], []).append(c.get("arguments", {}))
- for c in pred_calls:
- if isinstance(c, dict) and "name" in c and c["name"] in ref_by_name:
- m_args_total += 1
- pa = json.dumps(c.get("arguments", {}), sort_keys=True)
- if any(pa == json.dumps(ra, sort_keys=True) for ra in ref_by_name[c["name"]]):
- m_args_correct += 1
-
- tool_param_map = {}
- for t in tool_defs:
- if isinstance(t, dict) and "name" in t:
- tool_param_map[t["name"]] = set((t.get("parameters") or {}).keys())
- for c in pred_calls:
- if not isinstance(c, dict) or "name" not in c:
- continue
- cname = c["name"]
- if cname not in tool_param_map:
- continue
- pred_keys = set((c.get("arguments") or {}).keys())
- m_total_pred_params += len(pred_keys)
- m_halluc += len(pred_keys - tool_param_map[cname])
- if cname in ref_by_name:
- ref_args = ref_by_name[cname][0]
- ref_keys = set((ref_args if isinstance(ref_args, dict) else {}).keys())
- m_total_ref_params += len(ref_keys)
- m_missing += len(ref_keys - pred_keys)
- matched = pred_keys & ref_keys
- m_matched_params += len(matched)
- for k in matched:
- if json.dumps(c.get("arguments", {})[k], sort_keys=True) == \
- json.dumps(ref_args[k], sort_keys=True):
- m_correct_values += 1
-
- if m_n == 0:
- return {}, m_per_count
- np_ = m_name_tp + m_name_fp
- nr_ = m_name_tp + m_name_fn
- cp_ = m_call_tp + m_call_fp
- cr_ = m_call_tp + m_call_fn
- return {
- "n": m_n,
- "parse_rate": 1.0 - m_parse_err / m_n,
- "exact_match": m_exact / m_n,
- "name_f1": 2 * m_name_tp / max(np_ + nr_, 1),
- "call_f1": 2 * m_call_tp / max(cp_ + cr_, 1),
- "args_acc": m_args_correct / max(m_args_total, 1),
- "param_haluc": m_halluc / max(m_total_pred_params, 1),
- "param_miss": m_missing / max(m_total_ref_params, 1),
- "value_acc": m_correct_values / max(m_matched_params, 1),
- }, m_per_count
-
-
-def _print_metrics(label, metrics, pc):
- if not metrics:
- return
- print(f" ─── {label} ({metrics['n']} samples) ──")
- for k, key in [("JSON parse", "parse_rate"), ("Name F1", "name_f1"),
- ("Param haluc", "param_haluc"), ("Param miss", "param_miss"),
- ("Value acc", "value_acc"), ("Args acc", "args_acc"),
- ("Call F1", "call_f1"), ("Exact match", "exact_match")]:
- print(f" {k:<14} {metrics[key]:>10.1%}")
- has_any = any(pc[t]["n"] > 0 for t in range(11))
- if has_any:
- print(f" {'#tools':>6} {'n':>4} {'name_f1':>8} {'call_f1':>8} {'exact':>6}")
- for t in range(11):
- d = pc[t]
- if d["n"] == 0:
- continue
- np_ = d["name_tp"] + d["name_fp"]; nr_ = d["name_tp"] + d["name_fn"]
- cp_ = d["call_tp"] + d["call_fp"]; cr_ = d["call_tp"] + d["call_fn"]
- nf1 = 2 * d["name_tp"] / max(np_ + nr_, 1)
- cf1 = 2 * d["call_tp"] / max(cp_ + cr_, 1)
- ex = d["exact"] / d["n"]
- print(f" {t:>6} {d['n']:>4} {nf1:>7.1%} {cf1:>7.1%} {ex:>5.1%}")
-
-
-def _print_comparison(label, draft_m, corr_m):
- """Print side-by-side draft vs corrected metrics."""
- if not draft_m or not corr_m:
- return
- print(f"\n ─── {label} — draft vs corrected ───")
- print(f" {'metric':<14} {'draft':>8} {'corr':>8} {'Δ':>8}")
- for k, key in [("JSON parse", "parse_rate"), ("Name F1", "name_f1"),
- ("Param haluc", "param_haluc"), ("Param miss", "param_miss"),
- ("Value acc", "value_acc"), ("Args acc", "args_acc"),
- ("Call F1", "call_f1"), ("Exact match", "exact_match")]:
- d, c = draft_m[key], corr_m[key]
- sign = "+" if c >= d else ""
- print(f" {k:<14} {d:>8.1%} {c:>8.1%} {sign}{c-d:>7.1%}")
-
-
-# ───────────────────────── corrector generation ─────────────────────────
-
-def _generate_corrector_batch(model, params, tokenizer, queries, tools_list,
- drafts, max_gen_len, max_enc_len,
- batch_size=32, desc="corrector-gen"):
- """Greedy generate with encoder=[query, <tools>, tools, <tool_call>, draft]."""
- from .run import _get_decode_fn
- pad_id = PAD_ID
- eos_id = EOS_ID
- decode_fn = _get_decode_fn(model, max_gen_len)
-
- all_preds = []
- n_total = len(queries)
- pbar = tqdm(total=n_total, desc=desc, unit="ex", leave=False)
- for start in range(0, n_total, batch_size):
- chunk_q = queries[start:start + batch_size]
- chunk_t = tools_list[start:start + batch_size]
- chunk_d = drafts[start:start + batch_size]
- B = len(chunk_q)
-
- enc_token_lists = [
- _build_correction_encoder_input(tokenizer, q, t, d, max_enc_len)
- for q, t, d in zip(chunk_q, chunk_t, chunk_d)
- ]
- max_enc = max(len(toks) for toks in enc_token_lists)
- enc_input = np.full((B, max_enc), pad_id, dtype=np.int32)
- for i, toks in enumerate(enc_token_lists):
- enc_input[i, :len(toks)] = toks
- enc_input = jnp.array(enc_input)
- src_mask = make_padding_mask(enc_input, pad_id)
-
- encoder_out, enc_mask = model.apply(
- {"params": params}, enc_input, src_mask=src_mask, method="encode")
-
- dec_buffer = np.full((B, max_gen_len), pad_id, dtype=np.int32)
- dec_buffer[:, 0] = eos_id
- dec_buffer = jnp.array(dec_buffer)
-
- finished = [False] * B
- gen_tokens = [[] for _ in range(B)]
- logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
-
- for pos in range(max_gen_len - 1):
- for i in range(B):
- if finished[i]:
- continue
- next_token = int(jnp.argmax(logits[i, pos]))
- if next_token == eos_id:
- finished[i] = True
- continue
- gen_tokens[i].append(next_token)
- dec_buffer = dec_buffer.at[i, pos + 1].set(next_token)
- if all(finished):
- break
- logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
-
- for i in range(B):
- text = tokenizer.decode(gen_tokens[i])
- if text.startswith("<tool_call>"):
- text = text[len("<tool_call>"):]
- all_preds.append(text)
- pbar.update(B)
- pbar.close()
- return all_preds
-
-
-# ───────────────────────── main ─────────────────────────
-
-def train_corrector(args):
- if os.path.exists("/dev/accel0"):
- jax.distributed.initialize()
-
- num_devices = jax.local_device_count()
- num_hosts = jax.process_count()
- host_id = jax.process_index()
- is_main = host_id == 0
-
- use_wandb = args.wandb and is_main
- if use_wandb:
- import wandb
- if wandb.run is None:
- wandb.init(project="needle-v1", name=args.name, config=vars(args))
-
- print(f"[1/5] Devices: {num_devices} local, {jax.device_count()} total")
-
- print(f"[2/5] Loading tokenizer and SP model ...")
- tokenizer = get_tokenizer()
- sp_model = tokenizer.sp # SentencePiece processor for class computation
-
- # ─── load base (frozen, for drafts) and corrector init ───
- print(f"[3/5] Loading base SFT checkpoint: {args.base_checkpoint}")
- base_params, base_config = load_checkpoint(args.base_checkpoint)
- base_model = EncoderDecoderTransformer(base_config)
-
- print(f" Loading corrector init: {args.corrector_init}")
- corr_params, corr_config = load_checkpoint(args.corrector_init)
- corr_model = EncoderDecoderTransformer(corr_config)
- param_count = sum(x.size for x in jax.tree.leaves(corr_params))
- print(f" Corrector parameters: {param_count:,}")
-
- # ─── sample val pools (train.py-compatible) and cache base drafts ───
- print(f"[4/5] Sampling stratified val pools and caching base drafts ...")
- val_ds = load_tool_calls("val")
- single_pool, multi_pool = _stratified_val_pools(
- val_ds, per_bucket=args.tc_per_bucket, seed=args.eval_seed)
- print(f" single-call: {len(single_pool)}, multi-call: {len(multi_pool)}")
-
- def _gen_drafts(pool, label, chunk=64):
- queries = [ex["query"] for ex in pool]
- tools_list = [ex["tools"] for ex in pool]
- print(f" Generating {label} drafts from base model ...")
- t0 = time.perf_counter()
- params_j = jax.tree.map(jnp.array, base_params)
- drafts = []
- pbar = tqdm(total=len(queries), desc=f"base-draft-{label}",
- unit="ex", leave=False)
- for i in range(0, len(queries), chunk):
- drafts.extend(generate_batch(
- base_model, params_j, tokenizer,
- queries[i:i + chunk], tools_list[i:i + chunk],
- max_gen_len=args.max_gen_len, max_enc_len=args.max_enc_len,
- constrained=False))
- pbar.update(min(chunk, len(queries) - i))
- pbar.close()
- print(f" {label}: {len(drafts)} drafts in {time.perf_counter()-t0:.1f}s")
- return drafts
-
- single_drafts = _gen_drafts(single_pool, "single") if is_main else []
- multi_drafts = _gen_drafts(multi_pool, "multi") if is_main else []
-
- # Free base model weights — no longer needed
- del base_params, base_model
- import gc; gc.collect()
-
- # ─── estimate total steps for schedule ───
- print(f"[5/5] Building optimizer + pmap ...")
- with open(args.data) as f:
- n_rows = sum(1 for _ in f)
- effective_batch = args.batch_size * num_devices
- global_batch = effective_batch * num_hosts
- steps_per_epoch = n_rows // global_batch
- total_steps = steps_per_epoch * args.epochs
- warmup = max(1, int(total_steps * 0.05))
- if args.max_steps:
- total_steps = min(total_steps, args.max_steps)
-
- # ─── build state ───
- scaled_lr = args.lr * jax.device_count()
- muon_lr = 0.02 * math.sqrt(jax.device_count())
- rng = jax.random.PRNGKey(args.seed)
- rng, init_rng = jax.random.split(rng)
- state = create_train_state(init_rng, corr_config, scaled_lr, muon_lr,
- total_steps, warmup, decay_ratio=0.05)
-
- # Load corrector init weights into state
- corr_params_cast = jax.tree.map(
- lambda ref, loaded: jnp.asarray(loaded, dtype=ref.dtype),
- state.params, jax.tree.map(jnp.array, corr_params))
- state = state.replace(params=corr_params_cast)
- del corr_params, corr_params_cast
-
- state = _replicate(state)
- p_train_step = jax.pmap(_train_step, axis_name="batch", donate_argnums=(0,))
-
- adam_schedule = _wsd_schedule(scaled_lr, total_steps, warmup)
-
- if is_main:
- print(f"\n ─────────────────────────────────────")
- print(f" Corrector fine-tune")
- print(f" ─────────────────────────────────────")
- print(f" Train rows {n_rows:>12,}")
- print(f" Steps/epoch {steps_per_epoch:>12,}")
- print(f" Total steps {total_steps:>12,}")
- print(f" Epochs {args.epochs:>12}")
- print(f" Batch {args.batch_size:>4} x {jax.device_count()} = {global_batch}")
- print(f" LR {args.lr:>7} x {jax.device_count()} = {scaled_lr}")
- print(f" Encoder max {args.max_enc_len:>12}")
- print(f" Decoder max {args.max_dec_len:>12}")
- print(f" ─────────────────────────────────────\n")
-
- # ─── baseline eval (draft-only metrics) ───
- baseline_single_m = baseline_multi_m = None
- baseline_single_pc = baseline_multi_pc = None
- if is_main:
- print("Baseline eval (draft-only, no correction) ...")
- baseline_single_m, baseline_single_pc = _score_pool(single_pool, single_drafts)
- baseline_multi_m, baseline_multi_pc = _score_pool(multi_pool, multi_drafts)
- _print_metrics("BASELINE Single-Call", baseline_single_m, baseline_single_pc)
- _print_metrics("BASELINE Multi-Call", baseline_multi_m, baseline_multi_pc)
-
- os.makedirs(args.checkpoint_dir, exist_ok=True)
- global_step = 0
- best_call_f1 = -1.0
- best_path = None
-
- def _run_eval():
- nonlocal best_call_f1, best_path
- if not is_main:
- return
- eval_params = _unreplicate(state).params
- preds_single = _generate_corrector_batch(
- corr_model, eval_params, tokenizer,
- [ex["query"] for ex in single_pool],
- [ex["tools"] for ex in single_pool],
- single_drafts,
- max_gen_len=args.max_gen_len, max_enc_len=args.max_enc_len,
- desc=f"eval-step{global_step}-single")
- preds_multi = _generate_corrector_batch(
- corr_model, eval_params, tokenizer,
- [ex["query"] for ex in multi_pool],
- [ex["tools"] for ex in multi_pool],
- multi_drafts,
- max_gen_len=args.max_gen_len, max_enc_len=args.max_enc_len,
- desc=f"eval-step{global_step}-multi")
- single_m, single_pc = _score_pool(single_pool, preds_single)
- multi_m, multi_pc = _score_pool(multi_pool, preds_multi)
- _print_metrics("CORR Single-Call", single_m, single_pc)
- _print_metrics("CORR Multi-Call", multi_m, multi_pc)
- _print_comparison("Single-Call", baseline_single_m, single_m)
- _print_comparison("Multi-Call", baseline_multi_m, multi_m)
-
- cur_f1 = single_m.get("call_f1", -1)
- if cur_f1 > best_call_f1:
- best_call_f1 = cur_f1
- best_path = os.path.join(args.checkpoint_dir, "needle_corrector_best.pkl")
- params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), eval_params)
- with open(best_path, "wb") as f:
- pickle.dump({"params": params_np, "config": corr_config.__dict__,
- "step": global_step}, f)
- print(f" ** New best single call_f1={best_call_f1:.1%} -> {best_path}")
- _upload_checkpoint(best_path)
-
- if use_wandb:
- import wandb
- log = {"corrector/step": global_step,
- "best/single_call_f1": best_call_f1}
- for scope, m in (("single", single_m), ("multi", multi_m)):
- for k in ("parse_rate", "exact_match", "name_f1", "call_f1",
- "args_acc", "value_acc", "param_haluc", "param_miss"):
- log[f"val/{scope}_{k}"] = m.get(k, 0)
- for scope, m in (("single", baseline_single_m), ("multi", baseline_multi_m)):
- for k in ("call_f1", "value_acc", "exact_match"):
- log[f"baseline/{scope}_{k}"] = m.get(k, 0)
- wandb.log(log)
-
- # ─── training loop ───
- for epoch in range(args.epochs):
- stream = _PrefetchStream(
- lambda: _stream_batches(tokenizer, sp_model, args.data, global_batch,
- args.max_enc_len, args.max_dec_len,
- seed=args.seed + epoch),
- prefetch=4)
- pbar = tqdm(total=steps_per_epoch, desc=f"Corrector epoch {epoch+1}/{args.epochs}",
- disable=not is_main)
- losses = []
- t0 = time.perf_counter()
- try:
- for batch in stream:
- if args.max_steps and global_step >= args.max_steps:
- break
- src, tgt_in, tgt_out, lm = batch
- host_slice = slice(host_id * effective_batch,
- (host_id + 1) * effective_batch)
- src = src[host_slice]; tgt_in = tgt_in[host_slice]
- tgt_out = tgt_out[host_slice]; lm = lm[host_slice]
-
- src_b = shard_batch(src, num_devices)
- tgt_in_b = shard_batch(tgt_in, num_devices)
- tgt_out_b = shard_batch(tgt_out, num_devices)
- lm_b = shard_batch(lm, num_devices)
-
- rng, step_rng = jax.random.split(rng)
- rngs = jax.random.split(step_rng, num_devices)
-
- state, loss, gnorm = p_train_step(
- state, src_b, tgt_in_b, tgt_out_b, lm_b, rngs)
- loss_val = float(loss.addressable_shards[0].data[0])
- gnorm_val = float(gnorm.addressable_shards[0].data[0])
- losses.append(loss_val)
- global_step += 1
- pbar.update(1)
- pbar.set_postfix(loss=f"{loss_val:.4f}",
- gnorm=f"{gnorm_val:.2f}",
- lr=f"{float(adam_schedule(global_step)):.2e}")
- if use_wandb:
- import wandb
- wandb.log({"corrector/loss": loss_val,
- "corrector/grad_norm": gnorm_val,
- "corrector/lr": float(adam_schedule(global_step)),
- "corrector/step": global_step})
-
- finally:
- stream.close()
- pbar.close()
- _run_eval()
-
- # Final save (last eval already ran at end of final epoch)
- if is_main:
- final_path = os.path.join(args.checkpoint_dir, "needle_corrector.pkl")
- final_params = _unreplicate(state).params
- params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), final_params)
- with open(final_path, "wb") as f:
- pickle.dump({"params": params_np, "config": corr_config.__dict__,
- "step": global_step}, f)
- print(f"\nFinal corrector: {final_path}")
- if best_path:
- print(f"Best corrector: {best_path} (call_f1={best_call_f1:.1%})")
- _upload_checkpoint(final_path)
-
- if use_wandb:
- import wandb
- wandb.finish()
- if num_hosts > 1:
- jax.experimental.multihost_utils.sync_global_devices("corrector_done")
-
-
-def parse_args():
- p = argparse.ArgumentParser(description="Stage-2 correction fine-tuning")
- p.add_argument("--base-checkpoint", type=str, default="checkpoints/needle.pkl",
- help="Frozen SFT checkpoint used to generate drafts for eval")
- p.add_argument("--corrector-init", type=str, default="checkpoints/needle.pkl",
- help="Checkpoint to initialize the corrector from (default: same as base)")
- p.add_argument("--data", type=str, default="data/corrections.jsonl")
- p.add_argument("--name", type=str, default="corrector")
- p.add_argument("--epochs", type=int, default=1)
- p.add_argument("--batch-size", type=int, default=32)
- p.add_argument("--lr", type=float, default=1e-4)
- p.add_argument("--max-steps", type=int, default=None)
- p.add_argument("--max-enc-len", type=int, default=1024)
- p.add_argument("--max-dec-len", type=int, default=512)
- p.add_argument("--max-gen-len", type=int, default=256)
- p.add_argument("--tc-per-bucket", type=int, default=50,
- help="Eval samples per tool-count bucket (train.py default)")
- p.add_argument("--eval-every", type=int, default=500,
- help="Run full stratified eval every N steps")
- p.add_argument("--eval-seed", type=int, default=42)
- p.add_argument("--seed", type=int, default=42)
- p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
- p.add_argument("--wandb", action="store_true")
- return p.parse_args()
-
-
-if __name__ == "__main__":
- train_corrector(parse_args())
diff --git a/src/translate_xlam.py b/src/translate_xlam.py
deleted file mode 100644
index 71af90d..0000000
--- a/src/translate_xlam.py
+++ /dev/null
@@ -1,414 +0,0 @@
-#!/usr/bin/env python3
-"""Translate xlam-sourced examples from the unified dataset into 24 non-English
-languages using Gemini, then merge back into the dataset.
-
-Translates both the query AND free-text argument values (message text, notes,
-search queries, labels, destinations, etc.) so the data matches real-world usage
-where a French speaker's message content is also in French. Structured values
-(tool names, param keys, contact_ids, enum choices, booleans, numbers) stay
-unchanged.
-
-Examples are batched (default 10 per Gemini call) and grouped by language for
-efficiency — ~60k xlam × 24 languages = ~1.4M rows in ~144k API calls instead
-of 1.4M.
-
-Usage:
- python -m src.translate_xlam # translate + upload
- python -m src.translate_xlam --dry-run # translate only
- python -m src.translate_xlam --max-samples 100 # small test run
- python -m src.translate_xlam --workers 16 # more parallelism
-"""
-
-import argparse
-import concurrent.futures
-import json
-import os
-import random
-import sys
-import threading
-
-from tqdm import tqdm
-
-try:
- from google import genai
-except ImportError:
- print("Error: google-genai not installed. Run: pip install google-genai", file=sys.stderr)
- sys.exit(1)
-
-
-# Non-English target languages (English already covered by original xlam data)
-LANGUAGES = [
- "Bulgarian", "Croatian", "Czech", "Danish", "Dutch",
- "Estonian", "Finnish", "French", "German", "Greek", "Hungarian",
- "Italian", "Latvian", "Lithuanian", "Maltese", "Polish",
- "Portuguese", "Romanian", "Slovak", "Slovenian", "Spanish",
- "Swedish", "Russian", "Ukrainian",
-]
-
-MODEL = "gemini-3.1-flash-lite-preview"
-
-# Param names/patterns where values should NOT be translated
-_KEEP_ENGLISH = frozenset({
- "contact_id", "password", "url", "source", "confirmation_code",
- "order_id", "post_id", "file_name", "archive_name", "folder_name",
- "destination_folder", "app_name", "device_name", "ssid",
- "site", "username", "email", "to", "cc", "bcc",
- "phone", "phone_number", "js",
-})
-
-# Enum-like params where the value is from a fixed set — keep as-is
-_ENUM_HINTS = frozenset({
- "action", "mode", "direction", "position", "camera", "cycle",
- "ride_type", "service_type", "trigger", "data_type", "region",
- "restriction_level", "level", "unit", "connection_type",
- "setting", "goal_type", "input", "connector_type",
-})
-
-
-LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data", "tool_calls_unified")
-HF_DATASET_REPO = "Cactus-Compute/tool-calls"
-
-
-class ClientPool:
- """Round-robin pool of Gemini clients."""
-
- def __init__(self, clients):
- self._clients = clients
- self._idx = 0
- self._lock = threading.Lock()
-
- def get(self):
- with self._lock:
- client = self._clients[self._idx % len(self._clients)]
- self._idx += 1
- return client
-
-
-def make_clients():
- raw = os.environ.get("GEMINI_API_KEY", "")
- keys = [k.strip() for k in raw.split(",") if k.strip()]
- if not keys:
- print("Error: GEMINI_API_KEY not set.", file=sys.stderr)
- sys.exit(1)
- clients = [genai.Client(api_key=k) for k in keys]
- print(f"Using {len(clients)} API key(s)")
- return clients
-
-
-def _should_translate_param(pname, pval):
- """Decide if a parameter value should be translated."""
- if pname in _KEEP_ENGLISH:
- return False
- if pname in _ENUM_HINTS:
- return False
- if not isinstance(pval, str):
- return False
- # Short values that look like identifiers/codes — skip
- if len(pval) < 3:
- return False
- # Values that look like file paths, URLs, or IDs
- if "/" in pval or "@" in pval or pval.startswith(("http", "www")):
- return False
- # Pure numbers or booleans encoded as strings
- try:
- float(pval)
- return False
- except ValueError:
- pass
- if pval.lower() in ("true", "false", "null", "none"):
- return False
- return True
-
-
-def _extract_translatable(example):
- """Extract query + translatable arg values from an example.
-
- Returns (query, translatable_args) where translatable_args is a list of
- (call_idx, param_name, param_value) tuples.
- """
- answers = json.loads(example["answers"])
- translatable_args = []
- for call_idx, call in enumerate(answers):
- args = call.get("arguments", {})
- for pname, pval in args.items():
- if _should_translate_param(pname, pval):
- translatable_args.append((call_idx, pname, pval))
- return example["query"], translatable_args
-
-
-def _apply_translation(example, translated_item, language):
- """Apply a translated payload back onto an example."""
- answers = json.loads(example["answers"])
- query = example["query"]
-
- new_query = translated_item.get("query", query)
-
- # Reconstruct translatable_args to map arg_N keys back
- _, translatable_args = _extract_translatable(example)
- new_answers = json.loads(json.dumps(answers)) # deep copy
- for i, (call_idx, pname, _) in enumerate(translatable_args):
- key = f"arg_{i}"
- if key in translated_item:
- new_answers[call_idx]["arguments"][pname] = translated_item[key]
-
- return {
- "query": new_query,
- "tools": example["tools"],
- "answers": json.dumps(new_answers, separators=(",", ":"), ensure_ascii=False),
- "source": "xlam-translated",
- "model": example.get("_model_tag", ""),
- "language": language,
- }
-
-
-def translate_batch(client_pool, examples, language, model):
- """Translate a batch of examples into a single language in one Gemini call.
-
- Packs all examples into one JSON array prompt, gets back a translated array.
- Returns list of translated example dicts (may be shorter than input on errors).
- """
- # Build batch payload: list of {query, arg_0, arg_1, ...} per example
- batch_payload = []
- for ex in examples:
- query, translatable_args = _extract_translatable(ex)
- item = {"query": query}
- for i, (_, _, pval) in enumerate(translatable_args):
- item[f"arg_{i}"] = pval
- batch_payload.append(item)
-
- prompt = f"""Translate each JSON object's values from English to {language}.
-
-RULES:
-- Translate ONLY the string values, not the keys
-- Use natural, native-sounding {language} — not literal word-for-word translation
-- Use culturally appropriate names, places, and expressions where relevant
-- Keep proper nouns (brand names, app names) unchanged
-- Keep numbers, dates, and times in their original format
-- If a value is already a single common word that works in {language}, keep it
-- Return a JSON array with the SAME number of objects in the SAME order
-- Each object must have the SAME keys as the input — only values change
-
-INPUT ({len(batch_payload)} items):
-{json.dumps(batch_payload, ensure_ascii=False)}
-
-OUTPUT (valid JSON array, no markdown, {len(batch_payload)} items):"""
-
- client = client_pool.get()
- response = client.models.generate_content(
- model=model, contents=prompt,
- config={"temperature": 0.3, "max_output_tokens": 16384},
- )
-
- text = response.text.strip()
- if text.startswith("```"):
- text = text.split("\n", 1)[1] if "\n" in text else text[3:]
- if text.endswith("```"):
- text = text[:-3]
-
- translated_batch = json.loads(text.strip())
- if not isinstance(translated_batch, list):
- return []
-
- # Apply translations back to examples
- results = []
- model_tag = f"{model}-translate"
- for ex, translated_item in zip(examples, translated_batch):
- if not isinstance(translated_item, dict):
- continue
- ex_with_tag = {**ex, "_model_tag": model_tag}
- try:
- result = _apply_translation(ex_with_tag, translated_item, language)
- results.append(result)
- except Exception:
- continue
-
- return results
-
-
-def main(args):
- from datasets import Dataset, concatenate_datasets, load_dataset, load_from_disk
-
- client_pool = ClientPool(make_clients())
-
- # Load existing dataset
- local = os.path.abspath(LOCAL_UNIFIED_DIR)
- if os.path.exists(local) and any(f.endswith(".arrow") for f in os.listdir(local)):
- ds = load_from_disk(local)
- print(f"Loaded existing dataset: {len(ds)} rows")
- else:
- from .data import download_hf_split
- print(f"Downloading from {HF_DATASET_REPO}...")
- ds = download_hf_split("train", HF_DATASET_REPO)
- os.makedirs(local, exist_ok=True)
- ds.save_to_disk(local)
- print(f"Downloaded: {len(ds)} rows")
-
- # Filter to xlam-sourced examples only
- xlam_indices = [i for i, src in enumerate(ds["source"]) if "xlam" in src.lower()]
- xlam_ds = ds.select(xlam_indices)
- print(f"Found {len(xlam_ds)} xlam-sourced examples to translate")
-
- if args.max_samples and args.max_samples < len(xlam_ds):
- xlam_ds = xlam_ds.shuffle(seed=42).select(range(args.max_samples))
- print(f"Sampled down to {len(xlam_ds)} examples")
-
- # Prepare examples as dicts
- examples = []
- for i in range(len(xlam_ds)):
- examples.append({
- "query": xlam_ds[i]["query"],
- "tools": xlam_ds[i]["tools"],
- "answers": xlam_ds[i]["answers"],
- })
-
- # Assign languages round-robin and shuffle to decorrelate from xlam order
- rng = random.Random(42)
- lang_assignments = [LANGUAGES[i % len(LANGUAGES)] for i in range(len(examples))]
- combined = list(zip(examples, lang_assignments))
- rng.shuffle(combined)
- examples, lang_assignments = zip(*combined)
- examples, lang_assignments = list(examples), list(lang_assignments)
-
- batches = []
- lang_buckets = {}
- for ex, lang in zip(examples, lang_assignments):
- if lang not in lang_buckets:
- lang_buckets[lang] = []
- lang_buckets[lang].append(ex)
- if len(lang_buckets[lang]) >= args.batch_size:
- batches.append((lang, lang_buckets[lang]))
- lang_buckets[lang] = []
-
- for lang, remaining in lang_buckets.items():
- if remaining:
- batches.append((lang, remaining))
-
- rng.shuffle(batches)
- total_examples = sum(len(b) for _, b in batches)
- print(f"Prepared {len(batches)} batches ({total_examples} examples, batch_size={args.batch_size})")
-
- # Translate in parallel — one future per batch
- translated = []
- failed_batches = 0
- failed_examples = 0
- pbar = tqdm(total=len(batches), desc="Translating", unit="batch")
-
- with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool:
- futures = {}
- for i, (lang, batch_examples) in enumerate(batches):
- f = pool.submit(translate_batch, client_pool, batch_examples, lang, args.model)
- futures[f] = (i, len(batch_examples))
-
- for f in concurrent.futures.as_completed(futures):
- batch_idx, batch_len = futures[f]
- try:
- results = f.result()
- translated.extend(results)
- if len(results) < batch_len:
- failed_examples += batch_len - len(results)
- except Exception:
- failed_batches += 1
- failed_examples += batch_len
- pbar.update(1)
- pbar.set_postfix(
- translated=len(translated),
- failed_batches=failed_batches,
- failed_ex=failed_examples,
- )
-
- pbar.close()
-
- # Stats
- from collections import Counter
- lang_counts = Counter(ex["language"] for ex in translated)
- print(f"\nTranslated {len(translated)} examples "
- f"({failed_batches} failed batches, {failed_examples} failed examples)")
- print(f"Language distribution: {dict(lang_counts.most_common())}")
-
- # Show samples
- rng.shuffle(translated)
- print(f"\nSample translations:")
- for ex in translated[:20]:
- print(f" [{ex['language']}] Q: {ex['query'][:120]}")
- print(f" A: {ex['answers'][:150]}")
- print()
-
- if args.dry_run:
- print("--dry-run: skipping save and upload")
- return
-
- # Merge into existing dataset
- new_ds = Dataset.from_dict({
- "query": [ex["query"] for ex in translated],
- "tools": [ex["tools"] for ex in translated],
- "answers": [ex["answers"] for ex in translated],
- "source": [ex["source"] for ex in translated],
- "model": [ex["model"] for ex in translated],
- "language": [ex["language"] for ex in translated],
- })
-
- # Handle schema compatibility
- for col in new_ds.column_names:
- if col not in ds.column_names:
- ds = ds.add_column(col, ["English"] * len(ds) if col == "language" else [""] * len(ds))
- new_ds = new_ds.select_columns(ds.column_names)
-
- merged = concatenate_datasets([ds, new_ds])
- print(f"\nMerged: {len(merged)} rows (+{len(new_ds)} translated)")
-
- for src, cnt in Counter(merged["source"]).most_common():
- print(f" {src}: {cnt}")
-
- # Save locally
- import shutil
- tmp_dir = local + "_tmp"
- if os.path.exists(tmp_dir):
- shutil.rmtree(tmp_dir)
- merged.save_to_disk(tmp_dir)
- if os.path.exists(local):
- shutil.rmtree(local)
- os.rename(tmp_dir, local)
- print("Saved locally.")
-
- # Upload
- import tempfile
- from huggingface_hub import HfApi, CommitOperationDelete
-
- api = HfApi()
- api.create_repo(HF_DATASET_REPO, repo_type="dataset", private=False, exist_ok=True)
-
- parquet_dir = tempfile.mkdtemp(prefix="needle_xlam_translate_")
- print("Exporting to parquet...")
- merged.to_parquet(os.path.join(parquet_dir, "train.parquet"))
-
- files = api.list_repo_files(HF_DATASET_REPO, repo_type="dataset", token=True)
- old_train = [f for f in files if f.startswith("data/train-")]
- if old_train:
- print(f"Deleting {len(old_train)} old train shards...")
- ops = [CommitOperationDelete(path_in_repo=f) for f in old_train]
- api.create_commit(
- repo_id=HF_DATASET_REPO, repo_type="dataset", operations=ops,
- commit_message="Remove old train shards before re-upload", token=True,
- )
-
- print(f"Uploading to {HF_DATASET_REPO}...")
- api.upload_file(
- path_or_fileobj=os.path.join(parquet_dir, "train.parquet"),
- path_in_repo="data/train-00000-of-00001.parquet",
- repo_id=HF_DATASET_REPO, repo_type="dataset", token=True,
- commit_message=f"Upload train data with xlam translations ({len(merged)} rows)",
- )
-
- shutil.rmtree(parquet_dir)
- print(f"Upload complete: {HF_DATASET_REPO}")
- print("NOTE: Run 'needle split-dataset' to create the validation split.")
-
-
-if __name__ == "__main__":
- p = argparse.ArgumentParser(description="Translate xlam examples to 24 languages")
- p.add_argument("--max-samples", type=int, default=None, help="Limit examples to translate")
- p.add_argument("--workers", type=int, default=8, help="Parallel Gemini calls")
- p.add_argument("--model", type=str, default=MODEL, help="Gemini model for translation")
- p.add_argument("--batch-size", type=int, default=10, help="Examples per Gemini call")
- p.add_argument("--dry-run", action="store_true", help="Translate only, skip save/upload")
- main(p.parse_args())