Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
cleanup
HenryNdubuaku21cfeffcfcad
diff --git a/src/calibrate.py b/src/calibrate.pyindex e61a956..1107b11 100644--- a/src/calibrate.py+++ b/src/calibrate.py@@ -5,6 +5,12 @@ on the training data, maps perplexity to [0, 1] confidence labels via asigmoid calibration curve, then trains only the confidence head (2 Dense layers)to predict these labels from encoder output.+Two-pass design for memory efficiency:+ Pass 1: Forward through encoder+decoder to compute per-example PPL (stores only scalars)+ Pass 2: Re-encode through encoder only, train confidence head on the fly++All compute is distributed across all available devices via pmap.+Usage:needle calibrate --checkpoint checkpoints/needle_12_512_50000.pklneedle calibrate --checkpoint checkpoints/needle_12_512_50000.pkl --num-samples 10000@@ -29,94 +35,99 @@ from .model import ()-def compute_per_example_perplexity(model, params, val_data, batch_size, max_samples=None):- """Compute per-example perplexity using packed validation data.+def _replicate(tree):+ """Replicate params/state across all devices."""+ return jax.tree.map(lambda x: jnp.broadcast_to(x, (jax.local_device_count(),) + x.shape), tree)+++def _unreplicate(tree):+ """Get single copy from replicated params/state."""+ return jax.tree.map(lambda x: x[0], tree)+++def compute_per_example_perplexity(model, params, train_data, batch_size, max_samples=None):+ """Compute per-example perplexity using packed training data.++ Memory-efficient: only stores scalar PPL per example, not encoder outputs.+ Distributed across all devices via pmap.Returns:- encoder_outputs: list of (encoder_out, enc_mask) tuplesperplexities: numpy array of per-example perplexity values"""- enc = val_data["packed_enc"]- dec_in = val_data["packed_dec_in"]- dec_tgt = val_data["packed_dec_tgt"]- enc_seg = val_data["packed_enc_seg"]- dec_seg = val_data["packed_dec_seg"]+ enc = train_data["packed_enc"]+ dec_in = train_data["packed_dec_in"]+ dec_tgt = train_data["packed_dec_tgt"]+ enc_seg = train_data["packed_enc_seg"]+ dec_seg = train_data["packed_dec_seg"]n = len(enc)if max_samples:n = min(n, max_samples)+ num_devices = jax.local_device_count()+ per_device_batch = batch_size // num_devices+ assert batch_size % num_devices == 0, f"batch_size {batch_size} must be divisible by {num_devices} devices"+all_ppl = []- all_enc_out = []- all_enc_mask = []- all_src = []- all_enc_seg_ids = []- @jax.jit+ @jax.pmapdef forward_step(params, src, tgt_in, tgt_out, enc_seg_ids, dec_seg_ids):src_mask = make_packing_mask(enc_seg_ids)tgt_mask = make_causal_packing_mask(dec_seg_ids)cross_mask = make_cross_packing_mask(enc_seg_ids, dec_seg_ids)- # Get encoder output- encoder_out, enc_mask = model.apply(- {"params": params}, src, src_mask=src_mask, method="encode_text",- )-- # Get decoder logitslogits = model.apply({"params": params}, src, tgt_in,src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,)- # Per-token cross entropyce = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)- # Per-example: average over non-padding tokenspadding_mask = (dec_seg_ids > 0).astype(jnp.float32)per_example_loss = jnp.sum(ce * padding_mask, axis=-1) / jnp.maximum(jnp.sum(padding_mask, axis=-1), 1.0)per_example_ppl = jnp.exp(jnp.minimum(per_example_loss, 20.0))+ return per_example_ppl- return per_example_ppl, encoder_out, enc_mask+ r_params = _replicate(params)- print(f"Computing per-example perplexity ({n} examples, batch_size={batch_size})...")+ print(f"Computing per-example perplexity ({n} examples, batch_size={batch_size}, {num_devices} devices)...")for i in tqdm(range(0, n, batch_size), desc="PPL forward"):end = min(i + batch_size, n)- src = jnp.asarray(enc[i:end], dtype=jnp.int32)- tgt_in = jnp.asarray(dec_in[i:end], dtype=jnp.int32)- tgt_out = jnp.asarray(dec_tgt[i:end], dtype=jnp.int32)- es = jnp.asarray(enc_seg[i:end], dtype=jnp.int32)- ds = jnp.asarray(dec_seg[i:end], dtype=jnp.int32)-- ppl, enc_out, enc_m = forward_step(params, src, tgt_in, tgt_out, es, ds)- all_ppl.append(np.array(ppl))- all_enc_out.append(np.array(enc_out))- all_enc_mask.append(np.array(enc_m))- all_src.append(np.array(src))- all_enc_seg_ids.append(np.array(es))+ actual = end - i++ # Pad to full batch if needed+ def _load(arr, i, end, actual):+ batch = np.array(arr[i:end], dtype=np.int32)+ if actual < batch_size:+ pad = np.zeros((batch_size - actual, batch.shape[1]), dtype=np.int32)+ batch = np.concatenate([batch, pad], axis=0)+ return batch.reshape(num_devices, per_device_batch, -1)++ src = _load(enc, i, end, actual)+ tgt_in = _load(dec_in, i, end, actual)+ tgt_out = _load(dec_tgt, i, end, actual)+ es = _load(enc_seg, i, end, actual)+ ds = _load(dec_seg, i, end, actual)++ ppl = forward_step(r_params, src, tgt_in, tgt_out, es, ds)+ # ppl shape: (num_devices, per_device_batch) → flatten and trim+ ppl_flat = np.array(ppl).reshape(-1)[:actual]+ all_ppl.append(ppl_flat)perplexities = np.concatenate(all_ppl)print(f" Perplexity stats: min={perplexities.min():.2f}, median={np.median(perplexities):.2f}, "f"mean={perplexities.mean():.2f}, max={perplexities.max():.2f}")- return all_enc_out, all_enc_mask, all_src, all_enc_seg_ids, perplexities+ return perplexitiesdef perplexity_to_confidence(perplexities, mu=None, k=3.0):"""Map perplexity values to [0, 1] confidence via sigmoid calibration.confidence = sigmoid(-k * (log(ppl) - log(mu)))-- Args:- perplexities: array of perplexity values- mu: center point (default: median perplexity)- k: steepness of the sigmoid curve-- Returns:- confidence labels in [0, 1]"""if mu is None:mu = float(np.median(perplexities))@@ -124,132 +135,165 @@ def perplexity_to_confidence(perplexities, mu=None, k=3.0):log_ppl = np.log(np.maximum(perplexities, 1.0))log_mu = np.log(mu)- # sigmoid(-k * (log_ppl - log_mu))- # PPL << mu → large positive → confidence ≈ 1- # PPL >> mu → large negative → confidence ≈ 0confidence = 1.0 / (1.0 + np.exp(k * (log_ppl - log_mu)))return confidence.astype(np.float32), mu-def train_confidence_head(model, params, enc_outputs, enc_masks, confidence_labels,- epochs=10, lr=1e-3, batch_size=64):- """Train only the confidence head parameters, freezing everything else.-- Args:- model: EncoderDecoderTransformer instance- params: full model params (only confidence_hidden and confidence_out will be updated)- enc_outputs: list of (B, T, d_model) encoder output arrays- enc_masks: list of (B, 1, 1, T) mask arrays- confidence_labels: (N,) float32 array of target confidence values+def train_confidence_head(model, params, train_data, confidence_labels,+ epochs=1, lr=1e-3, batch_size=128):+ """Train only the confidence head parameters, re-encoding on the fly.- Returns:- Updated params with trained confidence head.+ Distributed across all devices via pmap. Memory-efficient: re-runs encoder+ forward pass during training instead of caching all encoder outputs."""- # Flatten batched arrays- all_enc = np.concatenate(enc_outputs, axis=0)- all_mask = np.concatenate(enc_masks, axis=0)+ enc = train_data["packed_enc"]+ enc_seg = train_data["packed_enc_seg"]n = len(confidence_labels)- # Extract confidence head params+ num_devices = jax.local_device_count()+ per_device_batch = batch_size // num_devices+ assert batch_size % num_devices == 0, f"batch_size {batch_size} must be divisible by {num_devices} devices"+conf_param_keys = {"confidence_hidden", "confidence_out"}def is_conf_param(path):return any(k in str(path) for k in conf_param_keys)- # Create optimizer for confidence params onlytx = optax.adam(lr)-- # We need to separate confidence params from frozen params- conf_params = jax.tree_util.tree_map_with_path(- lambda path, x: x if is_conf_param(path) else jax.lax.stop_gradient(x),- params,- )opt_state = tx.init(params)- @jax.jit- def confidence_loss(params, enc_out, enc_mask, targets):- """MSE loss between predicted and target confidence."""- pred = model.apply(- {"params": params}, enc_out, enc_mask,- method="predict_confidence",- )- return jnp.mean((pred - targets) ** 2)-- @jax.jit- def train_step(params, opt_state, enc_out, enc_mask, targets):- loss, grads = jax.value_and_grad(confidence_loss)(params, enc_out, enc_mask, targets)-- # Zero out gradients for non-confidence params- grads = jax.tree_util.tree_map_with_path(- lambda path, g: g if is_conf_param(path) else jnp.zeros_like(g),- grads,- )+ train_step = jax.pmap(+ lambda params, opt_state, src, enc_seg_ids, targets: _train_step_inner(+ model, tx, is_conf_param, params, opt_state, src, enc_seg_ids, targets+ ),+ axis_name="devices",+ )- updates, opt_state_new = tx.update(grads, opt_state, params)- params_new = optax.apply_updates(params, updates)- return params_new, opt_state_new, loss+ r_params = _replicate(params)+ r_opt_state = _replicate(opt_state)- print(f"\nTraining confidence head ({n} examples, {epochs} epochs, lr={lr})...")+ print(f"\nTraining confidence head ({n} examples, {epochs} epoch(s), lr={lr}, "+ f"batch_size={batch_size}, {num_devices} devices)...")for epoch in range(epochs):perm = np.random.permutation(n)epoch_losses = []- for i in range(0, n - batch_size + 1, batch_size):+ for i in tqdm(range(0, n - batch_size + 1, batch_size),+ desc=f"Calibrating epoch {epoch + 1}/{epochs}"):idx = perm[i:i + batch_size]- enc_batch = jnp.asarray(all_enc[idx])- mask_batch = jnp.asarray(all_mask[idx])- target_batch = jnp.asarray(confidence_labels[idx])+ src = np.array(enc[idx], dtype=np.int32).reshape(num_devices, per_device_batch, -1)+ es = np.array(enc_seg[idx], dtype=np.int32).reshape(num_devices, per_device_batch, -1)+ tgt = np.array(confidence_labels[idx]).reshape(num_devices, per_device_batch)- params, opt_state, loss = train_step(params, opt_state, enc_batch, mask_batch, target_batch)- epoch_losses.append(float(loss))+ r_params, r_opt_state, loss = train_step(r_params, r_opt_state, src, es, tgt)+ epoch_losses.append(float(loss[0])) # loss is replicated, take firstavg_loss = sum(epoch_losses) / len(epoch_losses) if epoch_losses else float("nan")print(f" Epoch {epoch + 1}/{epochs}: MSE={avg_loss:.6f}")+ params = _unreplicate(r_params)return params-def evaluate_confidence(model, params, enc_outputs, enc_masks, confidence_labels, perplexities):+def _train_step_inner(model, tx, is_conf_param, params, opt_state, src, enc_seg_ids, targets):+ """Inner training step — separated for pmap with axis_name."""+ def loss_fn(params):+ src_mask = make_packing_mask(enc_seg_ids)+ encoder_out = model.apply(+ {"params": params}, src, src_mask=src_mask, method="encode_text",+ )+ pred = model.apply(+ {"params": params}, encoder_out, src_mask,+ method="predict_confidence",+ )+ return jnp.mean((pred - targets) ** 2)++ loss, grads = jax.value_and_grad(loss_fn)(params)++ # Average gradients across devices+ grads = jax.lax.pmean(grads, axis_name="devices")++ # Zero out gradients for non-confidence params+ grads = jax.tree_util.tree_map_with_path(+ lambda path, g: g if is_conf_param(path) else jnp.zeros_like(g),+ grads,+ )++ updates, opt_state_new = tx.update(grads, opt_state, params)+ params_new = optax.apply_updates(params, updates)+ return params_new, opt_state_new, loss+++def evaluate_confidence(model, params, train_data, confidence_labels, perplexities, max_eval=10000):"""Evaluate calibrated confidence head and print diagnostic stats."""- all_enc = np.concatenate(enc_outputs, axis=0)- all_mask = np.concatenate(enc_masks, axis=0)- n = len(confidence_labels)+ enc = train_data["packed_enc"]+ enc_seg = train_data["packed_enc_seg"]+ n = min(len(confidence_labels), max_eval)- # Predict in batches- predictions = []+ num_devices = jax.local_device_count()batch_size = 128- for i in range(0, n, batch_size):- end = min(i + batch_size, n)- pred = model.apply(- {"params": params},- jnp.asarray(all_enc[i:end]),- jnp.asarray(all_mask[i:end]),+ # Round down to multiple of num_devices+ batch_size = (batch_size // num_devices) * num_devices+ per_device_batch = batch_size // num_devices++ @jax.pmap+ def predict_step(params, src, enc_seg_ids):+ src_mask = make_packing_mask(enc_seg_ids)+ encoder_out = model.apply(+ {"params": params}, src, src_mask=src_mask, method="encode_text",+ )+ return model.apply(+ {"params": params}, encoder_out, src_mask,method="predict_confidence",)- predictions.append(np.array(pred))++ r_params = _replicate(params)++ predictions = []+ for i in tqdm(range(0, n, batch_size), desc="Evaluating confidence"):+ end = min(i + batch_size, n)+ actual = end - i++ src = np.array(enc[i:end], dtype=np.int32)+ es = np.array(enc_seg[i:end], dtype=np.int32)+ if actual < batch_size:+ pad_src = np.zeros((batch_size - actual, src.shape[1]), dtype=np.int32)+ src = np.concatenate([src, pad_src], axis=0)+ pad_es = np.zeros((batch_size - actual, es.shape[1]), dtype=np.int32)+ es = np.concatenate([es, pad_es], axis=0)++ src = src.reshape(num_devices, per_device_batch, -1)+ es = es.reshape(num_devices, per_device_batch, -1)++ pred = predict_step(r_params, src, es)+ pred_flat = np.array(pred).reshape(-1)[:actual]+ predictions.append(pred_flat)+predictions = np.concatenate(predictions)+ labels = confidence_labels[:n]+ ppl = perplexities[:n]+# Stats- mse = np.mean((predictions - confidence_labels) ** 2)- mae = np.mean(np.abs(predictions - confidence_labels))- correlation = np.corrcoef(predictions, confidence_labels)[0, 1]+ mse = np.mean((predictions - labels) ** 2)+ mae = np.mean(np.abs(predictions - labels))+ correlation = np.corrcoef(predictions, labels)[0, 1]print(f"\nConfidence head evaluation:")print(f" MSE: {mse:.6f}")print(f" MAE: {mae:.4f}")print(f" Correlation: {correlation:.4f}")- # Show confidence vs perplexity bucketsprint(f"\n {'PPL Range':<15s} {'Count':>6s} {'Avg Conf':>10s} {'Avg Pred':>10s}")print(f" {'─' * 45}")buckets = [(0, 2), (2, 4), (4, 8), (8, 16), (16, 50), (50, 1000)]for lo, hi in buckets:- mask = (perplexities >= lo) & (perplexities < hi)+ mask = (ppl >= lo) & (ppl < hi)if mask.sum() == 0:continue- avg_conf = confidence_labels[mask].mean()+ avg_conf = labels[mask].mean()avg_pred = predictions[mask].mean()print(f" {f'{lo}-{hi}':<15s} {mask.sum():>6d} {avg_conf:>10.3f} {avg_pred:>10.3f}")@@ -257,6 +301,14 @@ def evaluate_confidence(model, params, enc_outputs, enc_masks, confidence_labelsdef main(args):+ num_devices = jax.local_device_count()+ print(f"Devices: {num_devices}")++ # Ensure batch_size is divisible by num_devices+ if args.batch_size % num_devices != 0:+ args.batch_size = (args.batch_size // num_devices) * num_devices+ print(f" Adjusted batch_size to {args.batch_size} (divisible by {num_devices})")+# Load checkpointprint(f"Loading checkpoint: {args.checkpoint}")with open(args.checkpoint, "rb") as f:@@ -278,43 +330,42 @@ def main(args):dummy_src = jnp.ones((1, 128), dtype=jnp.int32)dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)init_vars = model.init({"params": rng}, dummy_src, dummy_tgt, method="init_all")- # Merge: keep existing params, add new confidence head paramsinit_params = init_vars["params"]params = {**params, "confidence_hidden": init_params["confidence_hidden"],"confidence_out": init_params["confidence_out"]}print(" Confidence head initialized.")- # Load training data for perplexity computation+ # Load training dataprint(f"\nLoading prepared data...")train_data = load_prepared_data("train", mmap=True)print(f" {len(train_data['packed_enc']):,} packed bins")- # Step 1: Compute per-example perplexity- enc_outputs, enc_masks, srcs, enc_seg_ids, perplexities = compute_per_example_perplexity(- model, params, train_data, args.batch_size, max_samples=args.num_samples,+ # Pass 1: Compute per-example perplexity (forward-only, no gradients → large batch)+ ppl_batch_size = num_devices * 128 # 128 per device, forward-only no gradients+ perplexities = compute_per_example_perplexity(+ model, params, train_data, ppl_batch_size, max_samples=args.num_samples,)- # Step 2: Map perplexity → confidence labels+ # Map perplexity → confidence labelsconfidence_labels, mu = perplexity_to_confidence(perplexities, k=args.k)print(f"\nCalibration: mu (median PPL) = {mu:.2f}, k = {args.k}")print(f" Confidence stats: min={confidence_labels.min():.3f}, "f"median={np.median(confidence_labels):.3f}, "f"mean={confidence_labels.mean():.3f}, max={confidence_labels.max():.3f}")- # Step 3: Train the confidence head+ # Pass 2: Train confidence head (has gradients → use args.batch_size)params = train_confidence_head(- model, params, enc_outputs, enc_masks, confidence_labels,+ model, params, train_data, confidence_labels,epochs=args.epochs, lr=args.lr, batch_size=args.batch_size,)- # Step 4: Evaluate- evaluate_confidence(model, params, enc_outputs, enc_masks, confidence_labels, perplexities)+ # Evaluate+ evaluate_confidence(model, params, train_data, confidence_labels, perplexities)- # Step 5: Save updated checkpoint+ # Save updated checkpointoutput_path = args.output or args.checkpointparams_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), params)- # Store calibration metadata alongside configsave_config = dict(data["config"])save_config["confidence_mu"] = float(mu)save_config["confidence_k"] = float(args.k)diff --git a/src/cli.py b/src/cli.pyindex d5c8902..b6c638c 100644--- a/src/cli.py+++ b/src/cli.py@@ -62,8 +62,8 @@ def main():help="Remove feedforward layers entirely (default: True)")p.add_argument("--calibrate", action=argparse.BooleanOptionalAction, default=True,help="Run confidence head calibration after training (default: True)")- p.add_argument("--calibrate-epochs", type=int, default=10,- help="Epochs for confidence head calibration (default: 10)")+ p.add_argument("--calibrate-epochs", type=int, default=1,+ help="Epochs for confidence head calibration (default: 1)")p.add_argument("--calibrate-lr", type=float, default=1e-3,help="Learning rate for confidence head calibration (default: 1e-3)")p.add_argument("--calibrate-k", type=float, default=3.0,@@ -107,9 +107,9 @@ def main():p = sub.add_parser("calibrate", add_help=False)p.add_argument("--checkpoint", type=str, required=True)p.add_argument("--output", type=str, default=None, help="Output checkpoint path (default: overwrite input)")- p.add_argument("--batch-size", type=int, default=32)+ p.add_argument("--batch-size", type=int, default=128)p.add_argument("--num-samples", type=int, default=None, help="Limit training samples for PPL computation")- p.add_argument("--epochs", type=int, default=10)+ p.add_argument("--epochs", type=int, default=1)p.add_argument("--lr", type=float, default=1e-3)p.add_argument("--k", type=float, default=3.0, help="Sigmoid steepness for PPL→confidence mapping")diff --git a/src/eval.py b/src/eval.pyindex 6231831..8ec3b69 100644--- a/src/eval.py+++ b/src/eval.py@@ -211,6 +211,7 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=correct_values = 0matched_params = 0samples = []+ failures = [] # (query, tools, ref, pred, reasons)def call_key(c):if not isinstance(c, dict):@@ -328,6 +329,58 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=if json.dumps(c.get("arguments", {})[k], sort_keys=True) == json.dumps(ref_args[k], sort_keys=True):correct_values += 1+ # Diagnose failures for this example+ is_exact = (ref_is_empty and pred_is_empty) or (+ not ref_is_empty and not pred_is_empty+ and json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True)+ )+ if not is_exact and len(failures) < 50:+ reasons = []+ if json_parse_errors > 0 and i == total - 1:+ # Check if this specific example had a parse error+ try:+ json.loads(pred_text)+ except (json.JSONDecodeError, TypeError):+ reasons.append("json_parse_error")++ ex_fp_names = pred_name_set - ref_name_set+ ex_fn_names = ref_name_set - pred_name_set+ if ex_fp_names:+ reasons.append(f"wrong_tools:{','.join(sorted(ex_fp_names))}")+ if ex_fn_names:+ reasons.append(f"missing_tools:{','.join(sorted(ex_fn_names))}")++ if not ex_fp_names and not ex_fn_names and pred_name_set:+ # Right tools but wrong args — find which values differ+ for c in pred_calls:+ if not isinstance(c, dict) or "name" not in c:+ continue+ cname = c["name"]+ if cname not in ref_by_name:+ continue+ pred_args = c.get("arguments", {})+ ref_args = ref_by_name[cname][0]+ for k in set(pred_args.keys()) | set(ref_args.keys()):+ pv = pred_args.get(k, "<MISSING>")+ rv = ref_args.get(k, "<MISSING>")+ if json.dumps(pv, sort_keys=True) != json.dumps(rv, sort_keys=True):+ reasons.append(f"value_mismatch:{cname}.{k}={json.dumps(pv)[:60]}!={json.dumps(rv)[:60]}")++ if ref_is_empty and not pred_is_empty:+ reasons.append("false_positive:should_be_empty")+ if not ref_is_empty and pred_is_empty:+ reasons.append("false_negative:predicted_empty")++ if not reasons:+ reasons.append("unknown")++ failures.append({+ "query": ex["query"][:200],+ "ref": ref_text[:300],+ "pred": pred_text[:300],+ "reasons": reasons,+ })+name_precision = tp_names / max(tp_names + fp_names, 1)name_recall = tp_names / max(tp_names + fn_names, 1)name_f1 = 2 * name_precision * name_recall / max(name_precision + name_recall, 1e-9)@@ -353,6 +406,7 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len="empty_ref_pct": empty_ref / max(total, 1),"empty_pred_pct": empty_pred / max(total, 1),"samples": samples,+ "failures": failures,}@@ -534,6 +588,14 @@ def main(args):print(f" R: {ref}")print(f" P: {pred}")print()+ if tc.get("failures"):+ print(f" ─── Failures ({len(tc['failures'])} captured) ───")+ for j, fail in enumerate(tc["failures"][:15]):+ print(f" [{j+1}] Q: {fail['query'][:120]}")+ print(f" Ref: {fail['ref'][:200]}")+ print(f" Pred: {fail['pred'][:200]}")+ print(f" Why: {', '.join(fail['reasons'])}")+ print()print(f"\nmeasuring throughput ({args.throughput_runs} runs)...")throughput = measure_throughput(model, params, tokenizer, num_runs=args.throughput_runs)diff --git a/src/train.py b/src/train.pyindex 345b695..681ef8b 100644--- a/src/train.py+++ b/src/train.py@@ -1005,6 +1005,7 @@ def train(args):m_missing, m_total_ref_params = 0, 0m_correct_values, m_matched_params = 0, 0m_per_count = {t: {k: 0 for k in _pc_keys} for t in range(11)}+ m_failures = []for ex, pred_text in zip(eval_pairs, preds):try:@@ -1091,6 +1092,42 @@ def train(args):if _json_mod.dumps(c.get("arguments", {})[k], sort_keys=True) == _json_mod.dumps(ref_args[k], sort_keys=True):m_correct_values += 1+ # Failure diagnosis+ is_exact = (ref_is_empty and pred_is_empty) or (+ not ref_is_empty and not pred_is_empty+ and _json_mod.dumps(pred_calls, sort_keys=True) == _json_mod.dumps(ref_calls, sort_keys=True)+ )+ if not is_exact and len(m_failures) < 30:+ reasons = []+ ex_fp = pred_names - ref_names+ ex_fn = ref_names - pred_names+ if ex_fp:+ reasons.append(f"wrong_tools:{','.join(sorted(ex_fp))}")+ if ex_fn:+ reasons.append(f"missing_tools:{','.join(sorted(ex_fn))}")+ if not ex_fp and not ex_fn and pred_names:+ for c in pred_calls:+ if not isinstance(c, dict) or "name" not in c or c["name"] not in ref_by_name:+ continue+ pa = c.get("arguments", {})+ ra = ref_by_name[c["name"]][0]+ for k in set(pa.keys()) | set(ra.keys()):+ pv, rv = pa.get(k, "<MISSING>"), ra.get(k, "<MISSING>")+ if _json_mod.dumps(pv, sort_keys=True) != _json_mod.dumps(rv, sort_keys=True):+ reasons.append(f"{c['name']}.{k}={_json_mod.dumps(pv)[:50]}!={_json_mod.dumps(rv)[:50]}")+ if ref_is_empty and not pred_is_empty:+ reasons.append("false_positive")+ if not ref_is_empty and pred_is_empty:+ reasons.append("false_negative")+ if not reasons:+ reasons.append("unknown")+ m_failures.append({+ "query": ex["query"][:150],+ "ref": ref_text[:200],+ "pred": pred_text[:200],+ "reasons": reasons,+ })+metrics = {}if m_n > 0:metrics["n"] = m_n@@ -1106,6 +1143,7 @@ def train(args):metrics["param_haluc"] = m_halluc / max(m_total_pred_params, 1)metrics["param_miss"] = m_missing / max(m_total_ref_params, 1)metrics["value_acc"] = m_correct_values / max(m_matched_params, 1)+ metrics["failures"] = m_failuresreturn metrics, m_per_countpool_metrics = {}@@ -1190,6 +1228,14 @@ def train(args):ex_ = d["exact"] / d["n"]pr_ = 1.0 - d["parse_err"] / d["n"]print(f" {t:>6} {d['n']:>4} {nf1:>7.1%} {d['name_tp']:>4} {d['name_fp']:>4} {d['name_fn']:>4} {cf1:>7.1%} {d['call_tp']:>4} {d['call_fp']:>4} {d['call_fn']:>4} {ex_:>5.1%} {pr_:>5.1%}")+ if metrics.get("failures"):+ print(f" ─── Failures ({len(metrics['failures'])} captured) ───")+ for j, fail in enumerate(metrics["failures"][:10]):+ print(f" [{j+1}] Q: {fail['query'][:120]}")+ print(f" Ref: {fail['ref'][:200]}")+ print(f" Pred: {fail['pred'][:200]}")+ print(f" Why: {', '.join(fail['reasons'])}")+ print()_label_map = {"single": "Single-Call",@@ -1261,9 +1307,9 @@ def train(args):cal_args = Namespace(checkpoint=best_ckpt_path,output=best_ckpt_path,- batch_size=args.batch_size,- num_samples=getattr(args, "max_eval_samples", 5000),- epochs=getattr(args, "calibrate_epochs", 10),+ batch_size=args.batch_size * 4,+ num_samples=None, # use full dataset+ epochs=getattr(args, "calibrate_epochs", 1),lr=getattr(args, "calibrate_lr", 1e-3),k=getattr(args, "calibrate_k", 3.0),)