Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
innherent cloud fallback
HenryNdubuakuc16f4fa3a1f9
diff --git a/README.md b/README.mdindex 7f9694f..b85f66b 100644--- a/README.md+++ b/README.md@@ -7,50 +7,47 @@│ ...the tiny model to rule them all... │└────────────────────────────────────────────────────────┘- Architecture Training Pipeline- ──────────── ─────────────────-- ┌──────────────┐- │ Tool Call │ ┌──────────────────────┐- └──────┬───────┘ │ Mat loss at factors │- ┌┴──────────┐ │ + INT4 QAT (g=32) │- │ Softmax │ │ + z-loss │- └─────┬─────┘ │ + contrastive loss │- ┌─────┬─────┴─────┬─────┐ │ + token-level wt │- │ @E[:d_ff/f] for each │ └──────────┬──────────┘- │ f in mat_factors │ │- └─────┬─────┬─────┬─────┘ ┌──────────┴───────────┐- ┌─────┴─────┐ │ Grad clip (norm 1.0) │- │ Linear (T)│ ← tied │ Muon (2D kernels) │- └─────┬─────┘ │ AdamW (all else) │- ┌─────┴─────┐ │ WSD LR schedule │- │ ZCRMSNorm │ └──────────┬───────────┘- └─────┬─────┘ │- ┌────────┴────────┐ ┌──────────┴───────────┐- │ Decoder x N_dec │ │ Sparsification │- │┌───────────────┐│ │ cubic schedule │- ││ Masked Self ││ │ mask every N steps │- ││ Attn + RoPE ││ └──────────┬───────────┘- │├───────────────┤│ │- ┌──────────────┐ ││ Cross ││ ┌──────────┴───────────┐- │ Self-Attn │────────▶ Attention ││ │ EMA params (β=0.999) │- │ Encoder │ │├───────────────┤│ └──────────┬───────────┘- │ x N_enc │ ││ Gated FFN ││ │- │ │ │└───────────────┘│ ┌──────────┴───────────┐- │ ┌──────────┐ │ └────────┬────────┘ │ Checkpoint │- │ │Self Attn │ │ ┌─────┴─────┐ │ full params, can │- │ │ + RoPE │ │ │ Embedding │ ← shared │ export mat slices │- │ │ GQA │ │ └─────┬─────┘ └──────────────────────┘- │ ├──────────┤ │ ┌───────┴───────┐- │ │Gated FFN │ │ │[EOS]<tool_call>│- │ │ SwiGLU │ │ │ + answer │- │ ├──────────┤ │ └───────────────┘- │ │ZCRMSNorm │ │- │ │ Pre-norm │ │- │ └──────────┘ │- │ │+ Architecture+ ────────────++ Pure-attention encoder-decoder. No feedforward layers — each block is+ just gated self-attention (encoder) or gated self + cross attention+ (decoder). All mixing happens through attention.++ ┌──────────────┐+ │ Tool Call │+ └──────┬───────┘+ ┌┴──────────┐+ │ Softmax │+ └─────┬─────┘+ ┌─────┴─────┐+ │ Linear (T)│ ← tied+ └─────┬─────┘+ ┌─────┴─────┐+ │ ZCRMSNorm │+ └─────┬─────┘+ ┌────────┴────────┐+ │ Decoder x 12 │+ │┌───────────────┐│+ ││ ZCRMSNorm ││+ ││ Masked Self ││+ ││ Attn + RoPE ││+ ││ Gated Residual││+ │├───────────────┤│+ ┌──────────────┐ ││ ZCRMSNorm ││+ │ Encoder x 12 │──────────────────────▶Cross Attn ││+ │ │ ││ Gated Residual││+ │ ┌──────────┐ │ │└───────────────┘│+ │ │ZCRMSNorm │ │ └────────┬────────┘+ │ │Self Attn │ │ ┌─────┴─────┐+ │ │ GQA+RoPE │ │ │ Embedding │ ← shared+ │ │Gated Res │ │ └─────┬─────┘+ │ │ │ │ ┌───────┴───────-┐+ │ │ (no FFN) │ │ │[EOS]<tool_call>│+ │ └──────────┘ │ │ + answer │+ │ │ └───────────────-┘│ DW Conv ↓2 │ stride-2 depthwise-separable- │ + Pointwise │ downsamples seq before encoder+ │ + Pointwise │ (speech pathway only)└──────┬───────┘│┌────┴──────┐ ┌──────────────┐@@ -63,15 +60,55 @@│ query │ │ waveform │└───────────┘ └──────────────┘- d=512 · 8 enc / 8 dec layers · GQA (8H / 4KV) · QK-norm- SentencePiece BPE (8192) · SwiGLU · ZCRMSNorm · RoPE- strided DW conv · mat factors · INT4 QAT · Muon + AdamW+ Heads+ ─────+ ┌─────────────────────────────────────────────────────────────┐+ │ │+ │ Contrastive Head CLIP-style tool retrieval │+ │ encoder → mean_pool → Dense(d_model/4) → ReLU │+ │ → Dense(contrastive_dim) → L2-normalize │+ │ learnable temperature (log_temp) │+ │ │+ │ Confidence Head cloud routing / hybrid inference │+ │ encoder → mean_pool → Dense(d_model/4) │+ │ → ReLU → Dense(1) → sigmoid → [0, 1] │+ │ trained post-hoc on perplexity-derived labels │+ │ high = handle locally, low = route to cloud │+ │ │+ └─────────────────────────────────────────────────────────────┘++ Training Pipeline+ ─────────────────+ ┌─────────────────────────────────────────────────────────────┐+ │ Contrastive loss + token-level weighting + z-loss │+ │ Grad clip (norm 1.0) · Muon (2D kernels) · AdamW (rest) │+ │ WSD LR schedule (warmup → hold → cosine decay) │+ │ Grammar-constrained decoding (trie-based token masking) │+ └─────────────────────────────────────────────────────────────┘++ d=512 · 12 enc / 12 dec layers · GQA (8H / 4KV) · QK-norm+ no FFN (pure attention) · gated residuals (init=0.5)+ SentencePiece BPE (8192) · ZCRMSNorm · RoPEtext + speech encoder · <tool_call> task routing- CLIP contrastive retrieval · token-level loss weighting+ CLIP contrastive retrieval · confidence-based cloud routing+ token-level loss weighting · grammar-constrained decoding- Data Pipeline (needle tokenize → needle train)- ───────────────────────────────────────────────+ Data Pipeline (needle generate-data → needle tokenize → needle train)+ ─────────────────────────────────────────────────────────────────────+ ┌─────────────────────────────────────────────────────────────┐+ │ needle generate-data │+ │ │+ │ Gemini synthesis → Cactus-Compute/tool-calls (HuggingFace) │+ │ parallel workers · dedup · auto-merge · auto-upload │+ │ │+ │ Related: │+ │ needle merge-xlam merge xlam-60k into dataset │+ │ needle rebalance-tools trim over-represented bins │+ │ needle split-dataset create train/val splits │+ └─────────────────────────────────────────────────────────────┘+ │+ ▼┌─────────────────────────────────────────────────────────────┐│ needle tokenize ││ │@@ -115,6 +152,51 @@│ └──────────┬───────────┘ └──────────┬───────────┘ ││ ▼ ▼ ││ text + contrastive tool-call training (jax.pmap) │+ └─────────────────────────────────────────────────────────────┘+ │+ ▼+ ┌─────────────────────────────────────────────────────────────┐+ │ needle calibrate │+ │ │+ │ Freeze encoder + decoder │+ │ Compute per-example decoder perplexity │+ │ Map PPL → [0,1] confidence via sigmoid calibration │+ │ Train confidence head (2 Dense layers, MSE loss) │+ │ Save calibrated checkpoint with mu, k params │+ └─────────────────────────────────────────────────────────────┘++ Hybrid Inference (confidence-based cloud routing)+ ─────────────────────────────────────────────────++ ┌─────────────────────────────────────────────────────────────┐+ │ │+ │ query + tools │+ │ │ │+ │ ▼ │+ │ ┌──────────┐ │+ │ │ Encoder │ │+ │ └────┬─────┘ │+ │ │ │+ │ ├──────────────────┐ │+ │ ▼ ▼ │+ │ ┌──────────┐ ┌──────────────┐ │+ │ │Confidence│ │ (encoder │ │+ │ │ Head │ │ output) │ │+ │ └────┬─────┘ └──────┬───────┘ │+ │ │ │ │+ │ ▼ │ │+ │ confidence > threshold?│ │+ │ │ │ │+ │ ┌──┴──┐ │ │+ │ │ yes │──────────────┤ │+ │ └─────┘ ▼ │+ │ ┌──────────┐ │+ │ │ Decoder │──▶ tool calls (local) │+ │ └──────────┘ │+ │ ┌─────┐ │+ │ │ no │──▶ route to cloud (Gemini) ──▶ tool calls │+ │ └─────┘ (skip decoder entirely) │+ │ │└─────────────────────────────────────────────────────────────┘```@@ -130,33 +212,37 @@ needle [command]┌───────────────────────────────────────────────────────────────────┐│ ││ train │- │ --epochs INT Training epochs (default: 10) │+ │ --epochs INT Training epochs (default: 1) ││ --batch-size INT Batch size (default: 32) ││ --lr FLOAT AdamW learning rate (default: 3e-4) ││ --muon-lr FLOAT Muon learning rate (default: 0.02) ││ --d-model INT Model dim (default: 512) ││ --num-heads INT Attention heads (default: 8) ││ --num-kv-heads INT KV heads for GQA (default: 4) │- │ --num-layers INT Encoder layers (default: 8) │- │ --num-dec-layers INT Decoder layers (default: 8) │+ │ --num-layers INT Encoder layers (default: 12) │+ │ --num-dec-layers INT Decoder layers (default: 12) ││ --max-enc-len INT Max encoder seq len (default: 1024) ││ --max-dec-len INT Max decoder seq len (default: 512) ││ --max-samples INT Training samples (default: all) │- │ --mat-factors INT [...] FFN shrink factors (default: 2 4) │+ │ --no-feedforward No FFN layers (default: on) │+ │ --feedforward Enable FFN layers (off by default) │+ │ --activation STR swiglu|drelu|geglu (if FFN enabled) │+ │ --mat-factors INT [...] FFN shrink factors (if FFN enabled) ││ --sparsity-ratio FLOAT Block prune ratio (default: 0.0) ││ --group-size INT Quant/prune group size (default: 32) ││ --prune-interval INT Steps between mask updates (def: 100)││ --prune-start-frac FL Start pruning at frac (def: 0.33) ││ --prune-end-frac FL Lock mask at this frac (def: 0.67) │- │ --activation STR swiglu|drelu|geglu (default: swiglu) ││ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) ││ --decay-ratio FLOAT LR cosine decay ratio (default: 0.05)│- │ --dropout FLOAT Dropout rate (default: 0.1) │+ │ --dropout FLOAT Dropout rate (default: 0.0) ││ --eval-every INT Val eval interval (default: 1000) ││ --max-eval-samples INT Val samples limit (default: 5000) ││ --contrastive-weight FL CLIP loss weight (default: 0.1) ││ --contrastive-dim INT Projection dim (default: 128) │- │ --curriculum Sort batches easy→hard by tool count │+ │ --w-name FLOAT Loss weight: tool names (def: 2.0) │+ │ --w-value FLOAT Loss weight: arg values (def: 4.0) │+ │ --w-key FLOAT Loss weight: arg keys (def: 1.5) ││ --wandb Enable W&B logging ││ --checkpoint PATH Resume from checkpoint ││ --checkpoint-dir DIR Checkpoint directory │@@ -167,9 +253,6 @@ needle [command]│ --max-samples INT Limit samples per split (dev/test) ││ --max-enc-len INT Max encoder seq len (default: 1024) ││ --max-dec-len INT Max decoder seq len (default: 512) │- │ --w-name FLOAT Loss weight: tool names (def: 3.0) │- │ --w-value FLOAT Loss weight: arg values (def: 2.0) │- │ --w-key FLOAT Loss weight: arg keys (def: 1.5) ││ --shuffle-tools Shuffle tool order (default: on) ││ --no-shuffle-tools Disable tool shuffling ││ --max-tool-len INT Max tool desc tokens (default: 256) │@@ -181,6 +264,7 @@ needle [command]│ --audio PATH [...] Audio files for voice-to-tool-call ││ --max-len INT Max tokens to generate (default: 512)││ --seed INT Random seed (default: 0) │+ │ --no-constrained Disable constrained decoding ││ ││ eval ││ --checkpoint PATH Path to model checkpoint (required) │@@ -191,6 +275,34 @@ needle [command]│ --max-gen-len INT Max generation length (default: 512) ││ --throughput-runs INT Throughput runs (default: 10) ││ --tool-call-samples INT Tool-call eval samples (default: 200)│+ │ --no-constrained Disable constrained decoding │+ │ │+ │ calibrate │+ │ --checkpoint PATH Path to model checkpoint (required) │+ │ --output PATH Output path (default: overwrite) │+ │ --batch-size INT Batch size (default: 32) │+ │ --num-samples INT Limit samples for PPL (default: all) │+ │ --epochs INT Training epochs (default: 10) │+ │ --lr FLOAT Learning rate (default: 1e-3) │+ │ --k FLOAT Sigmoid steepness (default: 3.0) │+ │ │+ │ generate-data │+ │ --num-samples INT Samples to generate (default: 5000) │+ │ --batch-size INT Examples per Gemini call (default:10)│+ │ --workers INT Parallel Gemini calls (default: 8) │+ │ --model STR Gemini model override │+ │ --dry-run Generate only, skip upload │+ │ --output-jsonl PATH Also save raw generations to JSONL │+ │ --upload-every INT Merge+upload interval (def: 30000) │+ │ │+ │ merge-xlam │+ │ --dry-run Skip upload │+ │ --max-samples INT Limit xlam samples │+ │ │+ │ rebalance-tools │+ │ --dry-run Preview without modifying │+ │ │+ │ split-dataset Create train/val splits + upload ││ ││ evaluate ││ --checkpoint PATH Path to model checkpoint (required) │@@ -247,11 +359,11 @@ needle [command]## Setup For TPU/GCP-- Setup gcloud 1: download the `macOS ARM` from [here](https://docs.cloud.google.com/sdk/docs/install-sdk) and uzip.-- Setup gcloud 2: open terminal, cd to ypur downloads and run `./google-cloud-sdk/install.sh`-- Setup gcloud 3: restart terminal and run `gloud init`, sign in with cactus email, should prompt for project+- Setup gcloud 1: download the `macOS ARM` from [here](https://docs.cloud.google.com/sdk/docs/install-sdk) and unzip.+- Setup gcloud 2: open terminal, cd to your downloads and run `./google-cloud-sdk/install.sh`+- Setup gcloud 3: restart terminal and run `gcloud init`, sign in with cactus email, should prompt for project- Setup gcloud 4: else, set the project with `gcloud config set project needle-488623`-- setup gcloud 5: run `gcloud help` and read carefully+- Setup gcloud 5: run `gcloud help` and read carefully## Example Workflow@@ -281,8 +393,12 @@ needle [command]huggingface-cli login(paste your HF token — get one at https://huggingface.co/settings/tokens)-7. Use needle as you normally would locally, like training- needle train --wandb+7. Full pipeline+ needle generate-data --num-samples 5000 # synthesize training data+ needle tokenize # tokenize + pack + upload+ needle train --wandb # train on TPU+ needle calibrate --checkpoint <path> # calibrate confidence head+ needle eval --checkpoint <path> # evaluate8. Use tmux for long training runs (survives SSH disconnects)tmux new -s train # start a named sessiondiff --git a/notes/voice-to-action.txt b/notes/dataset.txtsimilarity index 100%rename from notes/voice-to-action.txtrename to notes/dataset.txtdiff --git a/src/calibrate.py b/src/calibrate.pynew file mode 100644index 0000000..e61a956--- /dev/null+++ b/src/calibrate.py@@ -0,0 +1,332 @@+"""Post-training calibration of the confidence head.++Freezes the encoder and decoder, computes per-example decoder perplexity+on the training data, maps perplexity to [0, 1] confidence labels via a+sigmoid calibration curve, then trains only the confidence head (2 Dense layers)+to predict these labels from encoder output.++Usage:+ needle calibrate --checkpoint checkpoints/needle_12_512_50000.pkl+ needle calibrate --checkpoint checkpoints/needle_12_512_50000.pkl --num-samples 10000+"""++import math+import os+import pickle++import jax+import jax.numpy as jnp+import numpy as np+import optax+from tqdm import tqdm++from .data import (+ get_tokenizer, load_prepared_data, count_batches, get_batches,+)+from .model import (+ EncoderDecoderTransformer, TransformerConfig,+ make_packing_mask, make_causal_packing_mask, make_cross_packing_mask,+)+++def compute_per_example_perplexity(model, params, val_data, batch_size, max_samples=None):+ """Compute per-example perplexity using packed validation data.++ Returns:+ encoder_outputs: list of (encoder_out, enc_mask) tuples+ perplexities: numpy array of per-example perplexity values+ """+ enc = val_data["packed_enc"]+ dec_in = val_data["packed_dec_in"]+ dec_tgt = val_data["packed_dec_tgt"]+ enc_seg = val_data["packed_enc_seg"]+ dec_seg = val_data["packed_dec_seg"]++ n = len(enc)+ if max_samples:+ n = min(n, max_samples)++ all_ppl = []+ all_enc_out = []+ all_enc_mask = []+ all_src = []+ all_enc_seg_ids = []++ @jax.jit+ def forward_step(params, src, tgt_in, tgt_out, enc_seg_ids, dec_seg_ids):+ src_mask = make_packing_mask(enc_seg_ids)+ tgt_mask = make_causal_packing_mask(dec_seg_ids)+ cross_mask = make_cross_packing_mask(enc_seg_ids, dec_seg_ids)++ # Get encoder output+ encoder_out, enc_mask = model.apply(+ {"params": params}, src, src_mask=src_mask, method="encode_text",+ )++ # Get decoder logits+ logits = model.apply(+ {"params": params}, src, tgt_in,+ src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,+ )++ # Per-token cross entropy+ ce = optax.softmax_cross_entropy_with_integer_labels(+ logits.astype(jnp.float32), tgt_out+ )+ # Per-example: average over non-padding tokens+ padding_mask = (dec_seg_ids > 0).astype(jnp.float32)+ per_example_loss = jnp.sum(ce * padding_mask, axis=-1) / jnp.maximum(+ jnp.sum(padding_mask, axis=-1), 1.0+ )+ per_example_ppl = jnp.exp(jnp.minimum(per_example_loss, 20.0))++ return per_example_ppl, encoder_out, enc_mask++ print(f"Computing per-example perplexity ({n} examples, batch_size={batch_size})...")+ for i in tqdm(range(0, n, batch_size), desc="PPL forward"):+ end = min(i + batch_size, n)+ src = jnp.asarray(enc[i:end], dtype=jnp.int32)+ tgt_in = jnp.asarray(dec_in[i:end], dtype=jnp.int32)+ tgt_out = jnp.asarray(dec_tgt[i:end], dtype=jnp.int32)+ es = jnp.asarray(enc_seg[i:end], dtype=jnp.int32)+ ds = jnp.asarray(dec_seg[i:end], dtype=jnp.int32)++ ppl, enc_out, enc_m = forward_step(params, src, tgt_in, tgt_out, es, ds)+ all_ppl.append(np.array(ppl))+ all_enc_out.append(np.array(enc_out))+ all_enc_mask.append(np.array(enc_m))+ all_src.append(np.array(src))+ all_enc_seg_ids.append(np.array(es))++ perplexities = np.concatenate(all_ppl)+ print(f" Perplexity stats: min={perplexities.min():.2f}, median={np.median(perplexities):.2f}, "+ f"mean={perplexities.mean():.2f}, max={perplexities.max():.2f}")++ return all_enc_out, all_enc_mask, all_src, all_enc_seg_ids, perplexities+++def perplexity_to_confidence(perplexities, mu=None, k=3.0):+ """Map perplexity values to [0, 1] confidence via sigmoid calibration.++ confidence = sigmoid(-k * (log(ppl) - log(mu)))++ Args:+ perplexities: array of perplexity values+ mu: center point (default: median perplexity)+ k: steepness of the sigmoid curve++ Returns:+ confidence labels in [0, 1]+ """+ if mu is None:+ mu = float(np.median(perplexities))++ log_ppl = np.log(np.maximum(perplexities, 1.0))+ log_mu = np.log(mu)++ # sigmoid(-k * (log_ppl - log_mu))+ # PPL << mu → large positive → confidence ≈ 1+ # PPL >> mu → large negative → confidence ≈ 0+ confidence = 1.0 / (1.0 + np.exp(k * (log_ppl - log_mu)))++ return confidence.astype(np.float32), mu+++def train_confidence_head(model, params, enc_outputs, enc_masks, confidence_labels,+ epochs=10, lr=1e-3, batch_size=64):+ """Train only the confidence head parameters, freezing everything else.++ Args:+ model: EncoderDecoderTransformer instance+ params: full model params (only confidence_hidden and confidence_out will be updated)+ enc_outputs: list of (B, T, d_model) encoder output arrays+ enc_masks: list of (B, 1, 1, T) mask arrays+ confidence_labels: (N,) float32 array of target confidence values++ Returns:+ Updated params with trained confidence head.+ """+ # Flatten batched arrays+ all_enc = np.concatenate(enc_outputs, axis=0)+ all_mask = np.concatenate(enc_masks, axis=0)+ n = len(confidence_labels)++ # Extract confidence head params+ conf_param_keys = {"confidence_hidden", "confidence_out"}++ def is_conf_param(path):+ return any(k in str(path) for k in conf_param_keys)++ # Create optimizer for confidence params only+ tx = optax.adam(lr)++ # We need to separate confidence params from frozen params+ conf_params = jax.tree_util.tree_map_with_path(+ lambda path, x: x if is_conf_param(path) else jax.lax.stop_gradient(x),+ params,+ )+ opt_state = tx.init(params)++ @jax.jit+ def confidence_loss(params, enc_out, enc_mask, targets):+ """MSE loss between predicted and target confidence."""+ pred = model.apply(+ {"params": params}, enc_out, enc_mask,+ method="predict_confidence",+ )+ return jnp.mean((pred - targets) ** 2)++ @jax.jit+ def train_step(params, opt_state, enc_out, enc_mask, targets):+ loss, grads = jax.value_and_grad(confidence_loss)(params, enc_out, enc_mask, targets)++ # Zero out gradients for non-confidence params+ grads = jax.tree_util.tree_map_with_path(+ lambda path, g: g if is_conf_param(path) else jnp.zeros_like(g),+ grads,+ )++ updates, opt_state_new = tx.update(grads, opt_state, params)+ params_new = optax.apply_updates(params, updates)+ return params_new, opt_state_new, loss++ print(f"\nTraining confidence head ({n} examples, {epochs} epochs, lr={lr})...")++ for epoch in range(epochs):+ perm = np.random.permutation(n)+ epoch_losses = []++ for i in range(0, n - batch_size + 1, batch_size):+ idx = perm[i:i + batch_size]+ enc_batch = jnp.asarray(all_enc[idx])+ mask_batch = jnp.asarray(all_mask[idx])+ target_batch = jnp.asarray(confidence_labels[idx])++ params, opt_state, loss = train_step(params, opt_state, enc_batch, mask_batch, target_batch)+ epoch_losses.append(float(loss))++ avg_loss = sum(epoch_losses) / len(epoch_losses) if epoch_losses else float("nan")+ print(f" Epoch {epoch + 1}/{epochs}: MSE={avg_loss:.6f}")++ return params+++def evaluate_confidence(model, params, enc_outputs, enc_masks, confidence_labels, perplexities):+ """Evaluate calibrated confidence head and print diagnostic stats."""+ all_enc = np.concatenate(enc_outputs, axis=0)+ all_mask = np.concatenate(enc_masks, axis=0)+ n = len(confidence_labels)++ # Predict in batches+ predictions = []+ batch_size = 128+ for i in range(0, n, batch_size):+ end = min(i + batch_size, n)+ pred = model.apply(+ {"params": params},+ jnp.asarray(all_enc[i:end]),+ jnp.asarray(all_mask[i:end]),+ method="predict_confidence",+ )+ predictions.append(np.array(pred))+ predictions = np.concatenate(predictions)++ # Stats+ mse = np.mean((predictions - confidence_labels) ** 2)+ mae = np.mean(np.abs(predictions - confidence_labels))+ correlation = np.corrcoef(predictions, confidence_labels)[0, 1]++ print(f"\nConfidence head evaluation:")+ print(f" MSE: {mse:.6f}")+ print(f" MAE: {mae:.4f}")+ print(f" Correlation: {correlation:.4f}")++ # Show confidence vs perplexity buckets+ print(f"\n {'PPL Range':<15s} {'Count':>6s} {'Avg Conf':>10s} {'Avg Pred':>10s}")+ print(f" {'─' * 45}")+ buckets = [(0, 2), (2, 4), (4, 8), (8, 16), (16, 50), (50, 1000)]+ for lo, hi in buckets:+ mask = (perplexities >= lo) & (perplexities < hi)+ if mask.sum() == 0:+ continue+ avg_conf = confidence_labels[mask].mean()+ avg_pred = predictions[mask].mean()+ print(f" {f'{lo}-{hi}':<15s} {mask.sum():>6d} {avg_conf:>10.3f} {avg_pred:>10.3f}")++ return predictions+++def main(args):+ # Load checkpoint+ print(f"Loading checkpoint: {args.checkpoint}")+ with open(args.checkpoint, "rb") as f:+ data = pickle.load(f)+ params = jax.tree.map(lambda x: jnp.array(x, dtype=jnp.bfloat16), data["params"])+ config = TransformerConfig(**data["config"])+ model = EncoderDecoderTransformer(config)++ param_count = sum(x.size for x in jax.tree.leaves(params))+ print(f" Parameters: {param_count:,}")+ print(f" Config: d={config.d_model}, heads={config.num_heads}, "+ f"layers={config.num_encoder_layers}/{config.num_decoder_layers}")++ # Check if confidence head params exist; if not, initialize them+ flat_keys = str(jax.tree.map(lambda x: x.shape, params))+ if "confidence_hidden" not in flat_keys:+ print(" Confidence head not found in checkpoint — initializing...")+ rng = jax.random.PRNGKey(0)+ dummy_src = jnp.ones((1, 128), dtype=jnp.int32)+ dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)+ init_vars = model.init({"params": rng}, dummy_src, dummy_tgt, method="init_all")+ # Merge: keep existing params, add new confidence head params+ init_params = init_vars["params"]+ params = {**params, "confidence_hidden": init_params["confidence_hidden"],+ "confidence_out": init_params["confidence_out"]}+ print(" Confidence head initialized.")++ # Load training data for perplexity computation+ print(f"\nLoading prepared data...")+ train_data = load_prepared_data("train", mmap=True)+ print(f" {len(train_data['packed_enc']):,} packed bins")++ # Step 1: Compute per-example perplexity+ enc_outputs, enc_masks, srcs, enc_seg_ids, perplexities = compute_per_example_perplexity(+ model, params, train_data, args.batch_size, max_samples=args.num_samples,+ )++ # Step 2: Map perplexity → confidence labels+ confidence_labels, mu = perplexity_to_confidence(perplexities, k=args.k)+ print(f"\nCalibration: mu (median PPL) = {mu:.2f}, k = {args.k}")+ print(f" Confidence stats: min={confidence_labels.min():.3f}, "+ f"median={np.median(confidence_labels):.3f}, "+ f"mean={confidence_labels.mean():.3f}, max={confidence_labels.max():.3f}")++ # Step 3: Train the confidence head+ params = train_confidence_head(+ model, params, enc_outputs, enc_masks, confidence_labels,+ epochs=args.epochs, lr=args.lr, batch_size=args.batch_size,+ )++ # Step 4: Evaluate+ evaluate_confidence(model, params, enc_outputs, enc_masks, confidence_labels, perplexities)++ # Step 5: Save updated checkpoint+ output_path = args.output or args.checkpoint+ params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), params)++ # Store calibration metadata alongside config+ save_config = dict(data["config"])+ save_config["confidence_mu"] = float(mu)+ save_config["confidence_k"] = float(args.k)++ with open(output_path, "wb") as f:+ pickle.dump({"params": params_np, "config": save_config}, f)+ print(f"\nSaved calibrated checkpoint: {output_path}")+ print(f" Calibration params: mu={mu:.2f}, k={args.k}")+ print(f"\n Usage at inference:")+ print(f" confidence, encoder_out, enc_mask = model.apply(")+ print(f' {{"params": params}}, src, src_mask=src_mask, method="forward_confidence")')+ print(f" if confidence > 0.6: # user-configurable threshold")+ print(f" # decode locally")+ print(f" else:")+ print(f" # route to cloud")diff --git a/src/cli.py b/src/cli.pyindex dedb175..9f60261 100644--- a/src/cli.py+++ b/src/cli.py@@ -96,6 +96,33 @@ def main():p.add_argument("--no-constrained", action="store_true",help="Disable grammar-constrained decoding for tool names/arg keys")+ p = sub.add_parser("calibrate", add_help=False)+ p.add_argument("--checkpoint", type=str, required=True)+ p.add_argument("--output", type=str, default=None, help="Output checkpoint path (default: overwrite input)")+ p.add_argument("--batch-size", type=int, default=32)+ p.add_argument("--num-samples", type=int, default=None, help="Limit training samples for PPL computation")+ p.add_argument("--epochs", type=int, default=10)+ p.add_argument("--lr", type=float, default=1e-3)+ p.add_argument("--k", type=float, default=3.0, help="Sigmoid steepness for PPL→confidence mapping")++ p = sub.add_parser("generate-data", add_help=False)+ p.add_argument("--num-samples", type=int, default=5000, help="Number of samples to generate")+ p.add_argument("--batch-size", type=int, default=10, help="Examples per Gemini call")+ p.add_argument("--workers", type=int, default=8, help="Parallel Gemini calls")+ p.add_argument("--model", type=str, default=None, help="Gemini model override")+ p.add_argument("--dry-run", action="store_true", help="Generate only, skip save and upload")+ p.add_argument("--output-jsonl", type=str, default=None, help="Also save raw generations to JSONL")+ p.add_argument("--upload-every", type=int, default=None, help="Merge+upload every N samples")++ p = sub.add_parser("merge-xlam", add_help=False)+ p.add_argument("--dry-run", action="store_true", help="Skip upload")+ p.add_argument("--max-samples", type=int, default=None, help="Limit xlam samples")++ p = sub.add_parser("rebalance-tools", add_help=False)+ p.add_argument("--dry-run", action="store_true", help="Preview without modifying")++ p = sub.add_parser("split-dataset", add_help=False)+p = sub.add_parser("evaluate", add_help=False)p.add_argument("--checkpoint", type=str, required=True)p.add_argument("--benchmarks", type=str, nargs="*",@@ -151,6 +178,25 @@ def main():elif args.command == "eval":from .eval import main as eval_main_fneval_main_fn(args)+ elif args.command == "calibrate":+ from .calibrate import main as calibrate_main+ calibrate_main(args)+ elif args.command == "generate-data":+ from .generate_data import main as gendata_main, MODEL as _MODEL, UPLOAD_EVERY as _UE+ if args.model is None:+ args.model = _MODEL+ if args.upload_every is None:+ args.upload_every = _UE+ gendata_main(args)+ elif args.command == "merge-xlam":+ from .merge_xlam import main as merge_main+ merge_main(args)+ elif args.command == "rebalance-tools":+ from .rebalance_tools import main as rebalance_main+ rebalance_main(args)+ elif args.command == "split-dataset":+ from .split_dataset import main as split_main+ split_main(args)elif args.command == "evaluate":from .evaluate import main as eval_maineval_main(args)diff --git a/scripts/generate_data.py b/src/generate_data.pysimilarity index 99%rename from scripts/generate_data.pyrename to src/generate_data.pyindex dde1423..f646a73 100644--- a/scripts/generate_data.py+++ b/src/generate_data.py@@ -1698,7 +1698,7 @@ def generate_all(num_samples, workers=8, batch_size=25, model=MODEL, client_pool-LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(__file__), "..", "data", "tool_calls_unified")+LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data", "tool_calls_unified")HF_DATASET_REPO = "Cactus-Compute/tool-calls"UPLOAD_EVERY = 30000@@ -1791,21 +1791,12 @@ def _merge_and_upload(existing, new_examples):import shutil as _shutil_shutil.rmtree(parquet_dir)print(f"Upload complete: {HF_DATASET_REPO}")- print("NOTE: Run 'python scripts/split_dataset.py' to create the validation split.")+ print("NOTE: Run 'needle split-dataset' to create the validation split.")return merged-def main():- parser = argparse.ArgumentParser(description="Generate on-device assistant tool-calling data with Gemini")- parser.add_argument("--num-samples", type=int, default=5000, help="Number of samples to generate")- parser.add_argument("--batch-size", type=int, default=10, help="Examples per Gemini call (smaller = more tool diversity, more API calls)")- parser.add_argument("--workers", type=int, default=8, help="Parallel Gemini calls")- parser.add_argument("--model", type=str, default=MODEL, help="Gemini model")- parser.add_argument("--dry-run", action="store_true", help="Generate only, skip save and upload")- parser.add_argument("--output-jsonl", type=str, default=None, help="Also save raw generations to JSONL")- parser.add_argument("--upload-every", type=int, default=UPLOAD_EVERY, help="Merge+upload every N samples (default 1000)")- args = parser.parse_args()+def main(args):client_pool = ClientPool(make_clients())@@ -1897,4 +1888,13 @@ def main():if __name__ == "__main__":- main()+ import argparse as _ap+ _p = _ap.ArgumentParser(description="Generate on-device assistant tool-calling data with Gemini")+ _p.add_argument("--num-samples", type=int, default=500)+ _p.add_argument("--batch-size", type=int, default=10)+ _p.add_argument("--workers", type=int, default=8)+ _p.add_argument("--model", type=str, default=MODEL)+ _p.add_argument("--dry-run", action="store_true")+ _p.add_argument("--output-jsonl", type=str, default=None)+ _p.add_argument("--upload-every", type=int, default=UPLOAD_EVERY)+ main(_p.parse_args())diff --git a/scripts/merge_xlam.py b/src/merge_xlam.pysimilarity index 87%rename from scripts/merge_xlam.pyrename to src/merge_xlam.pyindex d439e54..b085daa 100644--- a/scripts/merge_xlam.py+++ b/src/merge_xlam.py@@ -15,7 +15,7 @@ import sysfrom datasets import concatenate_datasets, load_dataset, load_from_disk-LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(__file__), "..", "data", "tool_calls_unified")+LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data", "tool_calls_unified")HF_DATASET_REPO = "Cactus-Compute/tool-calls"@@ -60,12 +60,7 @@ def check_duplicates(existing, new):print("No duplicate queries detected (sampled check)")-def main():- parser = argparse.ArgumentParser(description="Merge xlam into unified dataset")- parser.add_argument("--dry-run", action="store_true", help="Skip upload")- parser.add_argument("--max-samples", type=int, default=None, help="Limit xlam samples")- args = parser.parse_args()-+def main(args):existing = load_existing()xlam = load_xlam(args.max_samples)@@ -106,8 +101,12 @@ def main():print(f"\nUploading to {HF_DATASET_REPO} (train split)...")merged.push_to_hub(HF_DATASET_REPO, split="train", token=True)print(f"Upload complete: {HF_DATASET_REPO}")- print("NOTE: Run 'python scripts/split_dataset.py' to create the validation split.")+ print("NOTE: Run 'needle split-dataset' to create the validation split.")if __name__ == "__main__":- main()\ No newline at end of file+ import argparse as _ap+ _p = _ap.ArgumentParser(description="Merge xlam into unified dataset")+ _p.add_argument("--dry-run", action="store_true", help="Skip upload")+ _p.add_argument("--max-samples", type=int, default=None, help="Limit xlam samples")+ main(_p.parse_args())\ No newline at end of filediff --git a/src/model.py b/src/model.pyindex 731b8c0..b555c91 100644--- a/src/model.py+++ b/src/model.py@@ -412,11 +412,24 @@ class EncoderDecoderTransformer(nn.Module):self.config.d_model, dtype=self.config.jax_dtype, name="speech_subsample")self.encoder = Encoder(self.config)self.decoder = Decoder(self.config)+ self.contrastive_hidden = nn.Dense(+ self.config.d_model // 4, dtype=self.config.jax_dtype,+ kernel_init=default_init(), name="contrastive_hidden",+ )self.contrastive_proj = nn.Dense(self.config.contrastive_dim, dtype=self.config.jax_dtype,use_bias=False, kernel_init=default_init(), name="contrastive_proj",)self.log_temp = self.param("log_temp", jinit.zeros, ())++ self.confidence_hidden = nn.Dense(+ self.config.d_model // 4, dtype=self.config.jax_dtype,+ kernel_init=default_init(), name="confidence_hidden",+ )+ self.confidence_out = nn.Dense(+ 1, dtype=jnp.float32,+ kernel_init=jinit.zeros, name="confidence_out",+ )def _rope(self, seq_len):head_dim = self.config.d_model // self.config.num_heads@@ -467,12 +480,40 @@ class EncoderDecoderTransformer(nn.Module):counts = jnp.maximum(jnp.sum(mask_2d, axis=1, keepdims=True), 1.0)return summed / counts+ def predict_confidence(self, encoder_out, enc_mask):+ """Predict confidence score [0, 1] from encoder output.++ High confidence (→1.0) means the model can handle this query locally.+ Low confidence (→0.0) means the query should be routed to cloud.++ Args:+ encoder_out: (B, T, d_model) encoder hidden states+ enc_mask: (B, 1, 1, T) padding mask++ Returns:+ (B,) confidence scores in [0, 1]+ """+ pooled = self._mean_pool(encoder_out, enc_mask) # (B, d_model)+ h = nn.relu(self.confidence_hidden(pooled)) # (B, d_model // 4)+ return nn.sigmoid(self.confidence_out(h)).squeeze(-1) # (B,)++ def forward_confidence(self, src, src_mask=None):+ """End-to-end confidence prediction from token input.++ Returns (confidence, encoder_out, enc_mask) so the caller can+ reuse encoder output if confidence is high enough to decode.+ """+ encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)+ confidence = self.predict_confidence(encoder_out, enc_mask)+ return confidence, encoder_out, enc_mask+def encode_contrastive(self, tokens, deterministic=True):"""Encode tokens and project to L2-normalized contrastive space. Returns (B, contrastive_dim)."""src_mask = make_padding_mask(tokens, self.config.pad_token_id)encoder_out, enc_mask = self.encode_text(tokens, src_mask=src_mask, deterministic=deterministic)pooled = self._mean_pool(encoder_out, enc_mask)- projected = self.contrastive_proj(pooled)+ h = nn.relu(self.contrastive_hidden(pooled))+ projected = self.contrastive_proj(h)return projected / jnp.maximum(jnp.linalg.norm(projected, axis=-1, keepdims=True), 1e-8)def forward_contrastive(self, query_tokens, tool_tokens, deterministic=True):@@ -567,6 +608,7 @@ class EncoderDecoderTransformer(nn.Module):speech_out, speech_enc_mask = self.encode_speech(mel, src_mask=mel_mask, deterministic=True)_ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask, cross_mask=speech_enc_mask)_ = self.encode_contrastive(src)+ _ = self.predict_confidence(text_out, text_enc_mask)return jnp.zeros(())diff --git a/scripts/rebalance_tools.py b/src/rebalance_tools.pysimilarity index 97%rename from scripts/rebalance_tools.pyrename to src/rebalance_tools.pyindex 9e834cb..fc51a8d 100644--- a/scripts/rebalance_tools.py+++ b/src/rebalance_tools.py@@ -213,8 +213,12 @@ def rebalance(dry_run=False):print("Done.")-if __name__ == "__main__":- parser = argparse.ArgumentParser()- parser.add_argument("--dry-run", action="store_true", help="Preview without modifying")- args = parser.parse_args()+def main(args):rebalance(dry_run=args.dry_run)+++if __name__ == "__main__":+ import argparse as _ap+ _p = _ap.ArgumentParser()+ _p.add_argument("--dry-run", action="store_true", help="Preview without modifying")+ main(_p.parse_args())diff --git a/src/run.py b/src/run.pyindex 3d6d01b..537d434 100644--- a/src/run.py+++ b/src/run.py@@ -180,6 +180,86 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MAreturn result+def generate_with_confidence(model, params, tokenizer, query, tools="[]",+ max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN,+ seed=0, stream=True, normalize=True, constrained=True,+ confidence_threshold=None):+ """Generate tool calls with confidence score.++ Returns (result, confidence) where confidence is a float in [0, 1].+ If confidence_threshold is set and confidence is below it, returns+ (None, confidence) without running the decoder — saving compute for+ queries that should be routed to cloud.+ """+ name_map = {}+ if normalize:+ tools, name_map = normalize_tools(tools)++ enc_tokens = _build_encoder_input(tokenizer, query, tools, max_enc_len)+ enc_input = jnp.array([enc_tokens])+ pad_id = tokenizer.pad_token_id+ src_mask = make_padding_mask(enc_input, pad_id)++ # Get confidence + encoder output in one pass+ confidence, encoder_out, enc_mask = model.apply(+ {"params": params}, enc_input, src_mask=src_mask,+ method="forward_confidence",+ )+ confidence_val = float(confidence[0])++ # Early exit if below threshold — skip decoder entirely+ if confidence_threshold is not None and confidence_val < confidence_threshold:+ return None, confidence_val++ # Proceed with normal decoding using already-computed encoder output+ eos_id = tokenizer.eos_token_id+ dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)+ dec_buffer = dec_buffer.at[0, 0].set(eos_id)++ decode_fn = _get_decode_fn(model, max_gen_len)++ constrained_decoder = None+ if constrained:+ from .constrained import build_constrained_decoder+ constrained_decoder = build_constrained_decoder([tools], tokenizer)++ if stream:+ sys.stdout.write(f"\n")+ sys.stdout.flush()++ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)+ generated_tokens = []++ for i in range(0, max_gen_len - 1):+ next_logits = logits[0, i]+ if constrained_decoder and constrained_decoder.is_active(0):+ logits_np = np.array(next_logits)+ logits_np = constrained_decoder.constrain_logits(logits_np, 0)+ next_token = int(np.argmax(logits_np))+ else:+ next_token = int(jnp.argmax(next_logits))+ if constrained_decoder:+ constrained_decoder.update(0, next_token)+ if next_token == eos_id:+ break+ generated_tokens.append(next_token)+ dec_buffer = dec_buffer.at[0, i + 1].set(next_token)+ if stream:+ sys.stdout.write(tokenizer.decode([next_token]))+ sys.stdout.flush()+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)++ if stream:+ sys.stdout.write("\n")++ result = tokenizer.decode(generated_tokens)+ if result.startswith("<tool_call>"):+ result = result[len("<tool_call>"):]+ if normalize and name_map:+ result = restore_tool_names(result, name_map)+ return result, confidence_val++def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN, normalize=True, constrained=True):"""Batch-generate tool-call outputs for multiple examples at once.diff --git a/scripts/split_dataset.py b/src/split_dataset.pysimilarity index 99%rename from scripts/split_dataset.pyrename to src/split_dataset.pyindex bceef65..dc0d184 100644--- a/scripts/split_dataset.py+++ b/src/split_dataset.py@@ -20,7 +20,7 @@ VAL_SIZE = 10000SEED = 42-def main():+def main(args=None):print(f"Downloading full dataset from {HF_DATASET_REPO}...")ds = load_dataset(HF_DATASET_REPO, split="train", token=True)n = len(ds)