Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

cleanup

HenryNdubuaku21cfeffcfcad
diff --git a/src/calibrate.py b/src/calibrate.py
index e61a956..1107b11 100644
--- a/src/calibrate.py
+++ b/src/calibrate.py
@@ -5,6 +5,12 @@ on the training data, maps perplexity to [0, 1] confidence labels via a
sigmoid calibration curve, then trains only the confidence head (2 Dense layers)
to predict these labels from encoder output.
+Two-pass design for memory efficiency:
+ Pass 1: Forward through encoder+decoder to compute per-example PPL (stores only scalars)
+ Pass 2: Re-encode through encoder only, train confidence head on the fly
+
+All compute is distributed across all available devices via pmap.
+
Usage:
needle calibrate --checkpoint checkpoints/needle_12_512_50000.pkl
needle calibrate --checkpoint checkpoints/needle_12_512_50000.pkl --num-samples 10000
@@ -29,94 +35,99 @@ from .model import (
)
-def compute_per_example_perplexity(model, params, val_data, batch_size, max_samples=None):
- """Compute per-example perplexity using packed validation data.
+def _replicate(tree):
+ """Replicate params/state across all devices."""
+ return jax.tree.map(lambda x: jnp.broadcast_to(x, (jax.local_device_count(),) + x.shape), tree)
+
+
+def _unreplicate(tree):
+ """Get single copy from replicated params/state."""
+ return jax.tree.map(lambda x: x[0], tree)
+
+
+def compute_per_example_perplexity(model, params, train_data, batch_size, max_samples=None):
+ """Compute per-example perplexity using packed training data.
+
+ Memory-efficient: only stores scalar PPL per example, not encoder outputs.
+ Distributed across all devices via pmap.
Returns:
- encoder_outputs: list of (encoder_out, enc_mask) tuples
perplexities: numpy array of per-example perplexity values
"""
- enc = val_data["packed_enc"]
- dec_in = val_data["packed_dec_in"]
- dec_tgt = val_data["packed_dec_tgt"]
- enc_seg = val_data["packed_enc_seg"]
- dec_seg = val_data["packed_dec_seg"]
+ enc = train_data["packed_enc"]
+ dec_in = train_data["packed_dec_in"]
+ dec_tgt = train_data["packed_dec_tgt"]
+ enc_seg = train_data["packed_enc_seg"]
+ dec_seg = train_data["packed_dec_seg"]
n = len(enc)
if max_samples:
n = min(n, max_samples)
+ num_devices = jax.local_device_count()
+ per_device_batch = batch_size // num_devices
+ assert batch_size % num_devices == 0, f"batch_size {batch_size} must be divisible by {num_devices} devices"
+
all_ppl = []
- all_enc_out = []
- all_enc_mask = []
- all_src = []
- all_enc_seg_ids = []
- @jax.jit
+ @jax.pmap
def forward_step(params, src, tgt_in, tgt_out, enc_seg_ids, dec_seg_ids):
src_mask = make_packing_mask(enc_seg_ids)
tgt_mask = make_causal_packing_mask(dec_seg_ids)
cross_mask = make_cross_packing_mask(enc_seg_ids, dec_seg_ids)
- # Get encoder output
- encoder_out, enc_mask = model.apply(
- {"params": params}, src, src_mask=src_mask, method="encode_text",
- )
-
- # Get decoder logits
logits = model.apply(
{"params": params}, src, tgt_in,
src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
)
- # Per-token cross entropy
ce = optax.softmax_cross_entropy_with_integer_labels(
logits.astype(jnp.float32), tgt_out
)
- # Per-example: average over non-padding tokens
padding_mask = (dec_seg_ids > 0).astype(jnp.float32)
per_example_loss = jnp.sum(ce * padding_mask, axis=-1) / jnp.maximum(
jnp.sum(padding_mask, axis=-1), 1.0
)
per_example_ppl = jnp.exp(jnp.minimum(per_example_loss, 20.0))
+ return per_example_ppl
- return per_example_ppl, encoder_out, enc_mask
+ r_params = _replicate(params)
- print(f"Computing per-example perplexity ({n} examples, batch_size={batch_size})...")
+ print(f"Computing per-example perplexity ({n} examples, batch_size={batch_size}, {num_devices} devices)...")
for i in tqdm(range(0, n, batch_size), desc="PPL forward"):
end = min(i + batch_size, n)
- src = jnp.asarray(enc[i:end], dtype=jnp.int32)
- tgt_in = jnp.asarray(dec_in[i:end], dtype=jnp.int32)
- tgt_out = jnp.asarray(dec_tgt[i:end], dtype=jnp.int32)
- es = jnp.asarray(enc_seg[i:end], dtype=jnp.int32)
- ds = jnp.asarray(dec_seg[i:end], dtype=jnp.int32)
-
- ppl, enc_out, enc_m = forward_step(params, src, tgt_in, tgt_out, es, ds)
- all_ppl.append(np.array(ppl))
- all_enc_out.append(np.array(enc_out))
- all_enc_mask.append(np.array(enc_m))
- all_src.append(np.array(src))
- all_enc_seg_ids.append(np.array(es))
+ actual = end - i
+
+ # Pad to full batch if needed
+ def _load(arr, i, end, actual):
+ batch = np.array(arr[i:end], dtype=np.int32)
+ if actual < batch_size:
+ pad = np.zeros((batch_size - actual, batch.shape[1]), dtype=np.int32)
+ batch = np.concatenate([batch, pad], axis=0)
+ return batch.reshape(num_devices, per_device_batch, -1)
+
+ src = _load(enc, i, end, actual)
+ tgt_in = _load(dec_in, i, end, actual)
+ tgt_out = _load(dec_tgt, i, end, actual)
+ es = _load(enc_seg, i, end, actual)
+ ds = _load(dec_seg, i, end, actual)
+
+ ppl = forward_step(r_params, src, tgt_in, tgt_out, es, ds)
+ # ppl shape: (num_devices, per_device_batch) → flatten and trim
+ ppl_flat = np.array(ppl).reshape(-1)[:actual]
+ all_ppl.append(ppl_flat)
perplexities = np.concatenate(all_ppl)
print(f" Perplexity stats: min={perplexities.min():.2f}, median={np.median(perplexities):.2f}, "
f"mean={perplexities.mean():.2f}, max={perplexities.max():.2f}")
- return all_enc_out, all_enc_mask, all_src, all_enc_seg_ids, perplexities
+ return perplexities
def perplexity_to_confidence(perplexities, mu=None, k=3.0):
"""Map perplexity values to [0, 1] confidence via sigmoid calibration.
confidence = sigmoid(-k * (log(ppl) - log(mu)))
-
- Args:
- perplexities: array of perplexity values
- mu: center point (default: median perplexity)
- k: steepness of the sigmoid curve
-
- Returns:
- confidence labels in [0, 1]
"""
if mu is None:
mu = float(np.median(perplexities))
@@ -124,132 +135,165 @@ def perplexity_to_confidence(perplexities, mu=None, k=3.0):
log_ppl = np.log(np.maximum(perplexities, 1.0))
log_mu = np.log(mu)
- # sigmoid(-k * (log_ppl - log_mu))
- # PPL << mu → large positive → confidence ≈ 1
- # PPL >> mu → large negative → confidence ≈ 0
confidence = 1.0 / (1.0 + np.exp(k * (log_ppl - log_mu)))
return confidence.astype(np.float32), mu
-def train_confidence_head(model, params, enc_outputs, enc_masks, confidence_labels,
- epochs=10, lr=1e-3, batch_size=64):
- """Train only the confidence head parameters, freezing everything else.
-
- Args:
- model: EncoderDecoderTransformer instance
- params: full model params (only confidence_hidden and confidence_out will be updated)
- enc_outputs: list of (B, T, d_model) encoder output arrays
- enc_masks: list of (B, 1, 1, T) mask arrays
- confidence_labels: (N,) float32 array of target confidence values
+def train_confidence_head(model, params, train_data, confidence_labels,
+ epochs=1, lr=1e-3, batch_size=128):
+ """Train only the confidence head parameters, re-encoding on the fly.
- Returns:
- Updated params with trained confidence head.
+ Distributed across all devices via pmap. Memory-efficient: re-runs encoder
+ forward pass during training instead of caching all encoder outputs.
"""
- # Flatten batched arrays
- all_enc = np.concatenate(enc_outputs, axis=0)
- all_mask = np.concatenate(enc_masks, axis=0)
+ enc = train_data["packed_enc"]
+ enc_seg = train_data["packed_enc_seg"]
n = len(confidence_labels)
- # Extract confidence head params
+ num_devices = jax.local_device_count()
+ per_device_batch = batch_size // num_devices
+ assert batch_size % num_devices == 0, f"batch_size {batch_size} must be divisible by {num_devices} devices"
+
conf_param_keys = {"confidence_hidden", "confidence_out"}
def is_conf_param(path):
return any(k in str(path) for k in conf_param_keys)
- # Create optimizer for confidence params only
tx = optax.adam(lr)
-
- # We need to separate confidence params from frozen params
- conf_params = jax.tree_util.tree_map_with_path(
- lambda path, x: x if is_conf_param(path) else jax.lax.stop_gradient(x),
- params,
- )
opt_state = tx.init(params)
- @jax.jit
- def confidence_loss(params, enc_out, enc_mask, targets):
- """MSE loss between predicted and target confidence."""
- pred = model.apply(
- {"params": params}, enc_out, enc_mask,
- method="predict_confidence",
- )
- return jnp.mean((pred - targets) ** 2)
-
- @jax.jit
- def train_step(params, opt_state, enc_out, enc_mask, targets):
- loss, grads = jax.value_and_grad(confidence_loss)(params, enc_out, enc_mask, targets)
-
- # Zero out gradients for non-confidence params
- grads = jax.tree_util.tree_map_with_path(
- lambda path, g: g if is_conf_param(path) else jnp.zeros_like(g),
- grads,
- )
+ train_step = jax.pmap(
+ lambda params, opt_state, src, enc_seg_ids, targets: _train_step_inner(
+ model, tx, is_conf_param, params, opt_state, src, enc_seg_ids, targets
+ ),
+ axis_name="devices",
+ )
- updates, opt_state_new = tx.update(grads, opt_state, params)
- params_new = optax.apply_updates(params, updates)
- return params_new, opt_state_new, loss
+ r_params = _replicate(params)
+ r_opt_state = _replicate(opt_state)
- print(f"\nTraining confidence head ({n} examples, {epochs} epochs, lr={lr})...")
+ print(f"\nTraining confidence head ({n} examples, {epochs} epoch(s), lr={lr}, "
+ f"batch_size={batch_size}, {num_devices} devices)...")
for epoch in range(epochs):
perm = np.random.permutation(n)
epoch_losses = []
- for i in range(0, n - batch_size + 1, batch_size):
+ for i in tqdm(range(0, n - batch_size + 1, batch_size),
+ desc=f"Calibrating epoch {epoch + 1}/{epochs}"):
idx = perm[i:i + batch_size]
- enc_batch = jnp.asarray(all_enc[idx])
- mask_batch = jnp.asarray(all_mask[idx])
- target_batch = jnp.asarray(confidence_labels[idx])
+ src = np.array(enc[idx], dtype=np.int32).reshape(num_devices, per_device_batch, -1)
+ es = np.array(enc_seg[idx], dtype=np.int32).reshape(num_devices, per_device_batch, -1)
+ tgt = np.array(confidence_labels[idx]).reshape(num_devices, per_device_batch)
- params, opt_state, loss = train_step(params, opt_state, enc_batch, mask_batch, target_batch)
- epoch_losses.append(float(loss))
+ r_params, r_opt_state, loss = train_step(r_params, r_opt_state, src, es, tgt)
+ epoch_losses.append(float(loss[0])) # loss is replicated, take first
avg_loss = sum(epoch_losses) / len(epoch_losses) if epoch_losses else float("nan")
print(f" Epoch {epoch + 1}/{epochs}: MSE={avg_loss:.6f}")
+ params = _unreplicate(r_params)
return params
-def evaluate_confidence(model, params, enc_outputs, enc_masks, confidence_labels, perplexities):
+def _train_step_inner(model, tx, is_conf_param, params, opt_state, src, enc_seg_ids, targets):
+ """Inner training step — separated for pmap with axis_name."""
+ def loss_fn(params):
+ src_mask = make_packing_mask(enc_seg_ids)
+ encoder_out = model.apply(
+ {"params": params}, src, src_mask=src_mask, method="encode_text",
+ )
+ pred = model.apply(
+ {"params": params}, encoder_out, src_mask,
+ method="predict_confidence",
+ )
+ return jnp.mean((pred - targets) ** 2)
+
+ loss, grads = jax.value_and_grad(loss_fn)(params)
+
+ # Average gradients across devices
+ grads = jax.lax.pmean(grads, axis_name="devices")
+
+ # Zero out gradients for non-confidence params
+ grads = jax.tree_util.tree_map_with_path(
+ lambda path, g: g if is_conf_param(path) else jnp.zeros_like(g),
+ grads,
+ )
+
+ updates, opt_state_new = tx.update(grads, opt_state, params)
+ params_new = optax.apply_updates(params, updates)
+ return params_new, opt_state_new, loss
+
+
+def evaluate_confidence(model, params, train_data, confidence_labels, perplexities, max_eval=10000):
"""Evaluate calibrated confidence head and print diagnostic stats."""
- all_enc = np.concatenate(enc_outputs, axis=0)
- all_mask = np.concatenate(enc_masks, axis=0)
- n = len(confidence_labels)
+ enc = train_data["packed_enc"]
+ enc_seg = train_data["packed_enc_seg"]
+ n = min(len(confidence_labels), max_eval)
- # Predict in batches
- predictions = []
+ num_devices = jax.local_device_count()
batch_size = 128
- for i in range(0, n, batch_size):
- end = min(i + batch_size, n)
- pred = model.apply(
- {"params": params},
- jnp.asarray(all_enc[i:end]),
- jnp.asarray(all_mask[i:end]),
+ # Round down to multiple of num_devices
+ batch_size = (batch_size // num_devices) * num_devices
+ per_device_batch = batch_size // num_devices
+
+ @jax.pmap
+ def predict_step(params, src, enc_seg_ids):
+ src_mask = make_packing_mask(enc_seg_ids)
+ encoder_out = model.apply(
+ {"params": params}, src, src_mask=src_mask, method="encode_text",
+ )
+ return model.apply(
+ {"params": params}, encoder_out, src_mask,
method="predict_confidence",
)
- predictions.append(np.array(pred))
+
+ r_params = _replicate(params)
+
+ predictions = []
+ for i in tqdm(range(0, n, batch_size), desc="Evaluating confidence"):
+ end = min(i + batch_size, n)
+ actual = end - i
+
+ src = np.array(enc[i:end], dtype=np.int32)
+ es = np.array(enc_seg[i:end], dtype=np.int32)
+ if actual < batch_size:
+ pad_src = np.zeros((batch_size - actual, src.shape[1]), dtype=np.int32)
+ src = np.concatenate([src, pad_src], axis=0)
+ pad_es = np.zeros((batch_size - actual, es.shape[1]), dtype=np.int32)
+ es = np.concatenate([es, pad_es], axis=0)
+
+ src = src.reshape(num_devices, per_device_batch, -1)
+ es = es.reshape(num_devices, per_device_batch, -1)
+
+ pred = predict_step(r_params, src, es)
+ pred_flat = np.array(pred).reshape(-1)[:actual]
+ predictions.append(pred_flat)
+
predictions = np.concatenate(predictions)
+ labels = confidence_labels[:n]
+ ppl = perplexities[:n]
+
# Stats
- mse = np.mean((predictions - confidence_labels) ** 2)
- mae = np.mean(np.abs(predictions - confidence_labels))
- correlation = np.corrcoef(predictions, confidence_labels)[0, 1]
+ mse = np.mean((predictions - labels) ** 2)
+ mae = np.mean(np.abs(predictions - labels))
+ correlation = np.corrcoef(predictions, labels)[0, 1]
print(f"\nConfidence head evaluation:")
print(f" MSE: {mse:.6f}")
print(f" MAE: {mae:.4f}")
print(f" Correlation: {correlation:.4f}")
- # Show confidence vs perplexity buckets
print(f"\n {'PPL Range':<15s} {'Count':>6s} {'Avg Conf':>10s} {'Avg Pred':>10s}")
print(f" {'─' * 45}")
buckets = [(0, 2), (2, 4), (4, 8), (8, 16), (16, 50), (50, 1000)]
for lo, hi in buckets:
- mask = (perplexities >= lo) & (perplexities < hi)
+ mask = (ppl >= lo) & (ppl < hi)
if mask.sum() == 0:
continue
- avg_conf = confidence_labels[mask].mean()
+ avg_conf = labels[mask].mean()
avg_pred = predictions[mask].mean()
print(f" {f'{lo}-{hi}':<15s} {mask.sum():>6d} {avg_conf:>10.3f} {avg_pred:>10.3f}")
@@ -257,6 +301,14 @@ def evaluate_confidence(model, params, enc_outputs, enc_masks, confidence_labels
def main(args):
+ num_devices = jax.local_device_count()
+ print(f"Devices: {num_devices}")
+
+ # Ensure batch_size is divisible by num_devices
+ if args.batch_size % num_devices != 0:
+ args.batch_size = (args.batch_size // num_devices) * num_devices
+ print(f" Adjusted batch_size to {args.batch_size} (divisible by {num_devices})")
+
# Load checkpoint
print(f"Loading checkpoint: {args.checkpoint}")
with open(args.checkpoint, "rb") as f:
@@ -278,43 +330,42 @@ def main(args):
dummy_src = jnp.ones((1, 128), dtype=jnp.int32)
dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)
init_vars = model.init({"params": rng}, dummy_src, dummy_tgt, method="init_all")
- # Merge: keep existing params, add new confidence head params
init_params = init_vars["params"]
params = {**params, "confidence_hidden": init_params["confidence_hidden"],
"confidence_out": init_params["confidence_out"]}
print(" Confidence head initialized.")
- # Load training data for perplexity computation
+ # Load training data
print(f"\nLoading prepared data...")
train_data = load_prepared_data("train", mmap=True)
print(f" {len(train_data['packed_enc']):,} packed bins")
- # Step 1: Compute per-example perplexity
- enc_outputs, enc_masks, srcs, enc_seg_ids, perplexities = compute_per_example_perplexity(
- model, params, train_data, args.batch_size, max_samples=args.num_samples,
+ # Pass 1: Compute per-example perplexity (forward-only, no gradients → large batch)
+ ppl_batch_size = num_devices * 128 # 128 per device, forward-only no gradients
+ perplexities = compute_per_example_perplexity(
+ model, params, train_data, ppl_batch_size, max_samples=args.num_samples,
)
- # Step 2: Map perplexity → confidence labels
+ # Map perplexity → confidence labels
confidence_labels, mu = perplexity_to_confidence(perplexities, k=args.k)
print(f"\nCalibration: mu (median PPL) = {mu:.2f}, k = {args.k}")
print(f" Confidence stats: min={confidence_labels.min():.3f}, "
f"median={np.median(confidence_labels):.3f}, "
f"mean={confidence_labels.mean():.3f}, max={confidence_labels.max():.3f}")
- # Step 3: Train the confidence head
+ # Pass 2: Train confidence head (has gradients → use args.batch_size)
params = train_confidence_head(
- model, params, enc_outputs, enc_masks, confidence_labels,
+ model, params, train_data, confidence_labels,
epochs=args.epochs, lr=args.lr, batch_size=args.batch_size,
)
- # Step 4: Evaluate
- evaluate_confidence(model, params, enc_outputs, enc_masks, confidence_labels, perplexities)
+ # Evaluate
+ evaluate_confidence(model, params, train_data, confidence_labels, perplexities)
- # Step 5: Save updated checkpoint
+ # Save updated checkpoint
output_path = args.output or args.checkpoint
params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), params)
- # Store calibration metadata alongside config
save_config = dict(data["config"])
save_config["confidence_mu"] = float(mu)
save_config["confidence_k"] = float(args.k)
diff --git a/src/cli.py b/src/cli.py
index d5c8902..b6c638c 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -62,8 +62,8 @@ def main():
help="Remove feedforward layers entirely (default: True)")
p.add_argument("--calibrate", action=argparse.BooleanOptionalAction, default=True,
help="Run confidence head calibration after training (default: True)")
- p.add_argument("--calibrate-epochs", type=int, default=10,
- help="Epochs for confidence head calibration (default: 10)")
+ p.add_argument("--calibrate-epochs", type=int, default=1,
+ help="Epochs for confidence head calibration (default: 1)")
p.add_argument("--calibrate-lr", type=float, default=1e-3,
help="Learning rate for confidence head calibration (default: 1e-3)")
p.add_argument("--calibrate-k", type=float, default=3.0,
@@ -107,9 +107,9 @@ def main():
p = sub.add_parser("calibrate", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
p.add_argument("--output", type=str, default=None, help="Output checkpoint path (default: overwrite input)")
- p.add_argument("--batch-size", type=int, default=32)
+ p.add_argument("--batch-size", type=int, default=128)
p.add_argument("--num-samples", type=int, default=None, help="Limit training samples for PPL computation")
- p.add_argument("--epochs", type=int, default=10)
+ p.add_argument("--epochs", type=int, default=1)
p.add_argument("--lr", type=float, default=1e-3)
p.add_argument("--k", type=float, default=3.0, help="Sigmoid steepness for PPL→confidence mapping")
diff --git a/src/eval.py b/src/eval.py
index 6231831..8ec3b69 100644
--- a/src/eval.py
+++ b/src/eval.py
@@ -211,6 +211,7 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
correct_values = 0
matched_params = 0
samples = []
+ failures = [] # (query, tools, ref, pred, reasons)
def call_key(c):
if not isinstance(c, dict):
@@ -328,6 +329,58 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
if json.dumps(c.get("arguments", {})[k], sort_keys=True) == json.dumps(ref_args[k], sort_keys=True):
correct_values += 1
+ # Diagnose failures for this example
+ is_exact = (ref_is_empty and pred_is_empty) or (
+ not ref_is_empty and not pred_is_empty
+ and json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True)
+ )
+ if not is_exact and len(failures) < 50:
+ reasons = []
+ if json_parse_errors > 0 and i == total - 1:
+ # Check if this specific example had a parse error
+ try:
+ json.loads(pred_text)
+ except (json.JSONDecodeError, TypeError):
+ reasons.append("json_parse_error")
+
+ ex_fp_names = pred_name_set - ref_name_set
+ ex_fn_names = ref_name_set - pred_name_set
+ if ex_fp_names:
+ reasons.append(f"wrong_tools:{','.join(sorted(ex_fp_names))}")
+ if ex_fn_names:
+ reasons.append(f"missing_tools:{','.join(sorted(ex_fn_names))}")
+
+ if not ex_fp_names and not ex_fn_names and pred_name_set:
+ # Right tools but wrong args — find which values differ
+ for c in pred_calls:
+ if not isinstance(c, dict) or "name" not in c:
+ continue
+ cname = c["name"]
+ if cname not in ref_by_name:
+ continue
+ pred_args = c.get("arguments", {})
+ ref_args = ref_by_name[cname][0]
+ for k in set(pred_args.keys()) | set(ref_args.keys()):
+ pv = pred_args.get(k, "<MISSING>")
+ rv = ref_args.get(k, "<MISSING>")
+ if json.dumps(pv, sort_keys=True) != json.dumps(rv, sort_keys=True):
+ reasons.append(f"value_mismatch:{cname}.{k}={json.dumps(pv)[:60]}!={json.dumps(rv)[:60]}")
+
+ if ref_is_empty and not pred_is_empty:
+ reasons.append("false_positive:should_be_empty")
+ if not ref_is_empty and pred_is_empty:
+ reasons.append("false_negative:predicted_empty")
+
+ if not reasons:
+ reasons.append("unknown")
+
+ failures.append({
+ "query": ex["query"][:200],
+ "ref": ref_text[:300],
+ "pred": pred_text[:300],
+ "reasons": reasons,
+ })
+
name_precision = tp_names / max(tp_names + fp_names, 1)
name_recall = tp_names / max(tp_names + fn_names, 1)
name_f1 = 2 * name_precision * name_recall / max(name_precision + name_recall, 1e-9)
@@ -353,6 +406,7 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
"empty_ref_pct": empty_ref / max(total, 1),
"empty_pred_pct": empty_pred / max(total, 1),
"samples": samples,
+ "failures": failures,
}
@@ -534,6 +588,14 @@ def main(args):
print(f" R: {ref}")
print(f" P: {pred}")
print()
+ if tc.get("failures"):
+ print(f" ─── Failures ({len(tc['failures'])} captured) ───")
+ for j, fail in enumerate(tc["failures"][:15]):
+ print(f" [{j+1}] Q: {fail['query'][:120]}")
+ print(f" Ref: {fail['ref'][:200]}")
+ print(f" Pred: {fail['pred'][:200]}")
+ print(f" Why: {', '.join(fail['reasons'])}")
+ print()
print(f"\nmeasuring throughput ({args.throughput_runs} runs)...")
throughput = measure_throughput(model, params, tokenizer, num_runs=args.throughput_runs)
diff --git a/src/train.py b/src/train.py
index 345b695..681ef8b 100644
--- a/src/train.py
+++ b/src/train.py
@@ -1005,6 +1005,7 @@ def train(args):
m_missing, m_total_ref_params = 0, 0
m_correct_values, m_matched_params = 0, 0
m_per_count = {t: {k: 0 for k in _pc_keys} for t in range(11)}
+ m_failures = []
for ex, pred_text in zip(eval_pairs, preds):
try:
@@ -1091,6 +1092,42 @@ def train(args):
if _json_mod.dumps(c.get("arguments", {})[k], sort_keys=True) == _json_mod.dumps(ref_args[k], sort_keys=True):
m_correct_values += 1
+ # Failure diagnosis
+ is_exact = (ref_is_empty and pred_is_empty) or (
+ not ref_is_empty and not pred_is_empty
+ and _json_mod.dumps(pred_calls, sort_keys=True) == _json_mod.dumps(ref_calls, sort_keys=True)
+ )
+ if not is_exact and len(m_failures) < 30:
+ reasons = []
+ ex_fp = pred_names - ref_names
+ ex_fn = ref_names - pred_names
+ if ex_fp:
+ reasons.append(f"wrong_tools:{','.join(sorted(ex_fp))}")
+ if ex_fn:
+ reasons.append(f"missing_tools:{','.join(sorted(ex_fn))}")
+ if not ex_fp and not ex_fn and pred_names:
+ for c in pred_calls:
+ if not isinstance(c, dict) or "name" not in c or c["name"] not in ref_by_name:
+ continue
+ pa = c.get("arguments", {})
+ ra = ref_by_name[c["name"]][0]
+ for k in set(pa.keys()) | set(ra.keys()):
+ pv, rv = pa.get(k, "<MISSING>"), ra.get(k, "<MISSING>")
+ if _json_mod.dumps(pv, sort_keys=True) != _json_mod.dumps(rv, sort_keys=True):
+ reasons.append(f"{c['name']}.{k}={_json_mod.dumps(pv)[:50]}!={_json_mod.dumps(rv)[:50]}")
+ if ref_is_empty and not pred_is_empty:
+ reasons.append("false_positive")
+ if not ref_is_empty and pred_is_empty:
+ reasons.append("false_negative")
+ if not reasons:
+ reasons.append("unknown")
+ m_failures.append({
+ "query": ex["query"][:150],
+ "ref": ref_text[:200],
+ "pred": pred_text[:200],
+ "reasons": reasons,
+ })
+
metrics = {}
if m_n > 0:
metrics["n"] = m_n
@@ -1106,6 +1143,7 @@ def train(args):
metrics["param_haluc"] = m_halluc / max(m_total_pred_params, 1)
metrics["param_miss"] = m_missing / max(m_total_ref_params, 1)
metrics["value_acc"] = m_correct_values / max(m_matched_params, 1)
+ metrics["failures"] = m_failures
return metrics, m_per_count
pool_metrics = {}
@@ -1190,6 +1228,14 @@ def train(args):
ex_ = d["exact"] / d["n"]
pr_ = 1.0 - d["parse_err"] / d["n"]
print(f" {t:>6} {d['n']:>4} {nf1:>7.1%} {d['name_tp']:>4} {d['name_fp']:>4} {d['name_fn']:>4} {cf1:>7.1%} {d['call_tp']:>4} {d['call_fp']:>4} {d['call_fn']:>4} {ex_:>5.1%} {pr_:>5.1%}")
+ if metrics.get("failures"):
+ print(f" ─── Failures ({len(metrics['failures'])} captured) ───")
+ for j, fail in enumerate(metrics["failures"][:10]):
+ print(f" [{j+1}] Q: {fail['query'][:120]}")
+ print(f" Ref: {fail['ref'][:200]}")
+ print(f" Pred: {fail['pred'][:200]}")
+ print(f" Why: {', '.join(fail['reasons'])}")
+ print()
_label_map = {
"single": "Single-Call",
@@ -1261,9 +1307,9 @@ def train(args):
cal_args = Namespace(
checkpoint=best_ckpt_path,
output=best_ckpt_path,
- batch_size=args.batch_size,
- num_samples=getattr(args, "max_eval_samples", 5000),
- epochs=getattr(args, "calibrate_epochs", 10),
+ batch_size=args.batch_size * 4,
+ num_samples=None, # use full dataset
+ epochs=getattr(args, "calibrate_epochs", 1),
lr=getattr(args, "calibrate_lr", 1e-3),
k=getattr(args, "calibrate_k", 3.0),
)