Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

innherent cloud fallback

HenryNdubuakuc16f4fa3a1f9
diff --git a/README.md b/README.md
index 7f9694f..b85f66b 100644
--- a/README.md
+++ b/README.md
@@ -7,50 +7,47 @@
│ ...the tiny model to rule them all... │
└────────────────────────────────────────────────────────┘
- Architecture Training Pipeline
- ──────────── ─────────────────
-
- ┌──────────────┐
- │ Tool Call │ ┌──────────────────────┐
- └──────┬───────┘ │ Mat loss at factors │
- ┌┴──────────┐ │ + INT4 QAT (g=32) │
- │ Softmax │ │ + z-loss │
- └─────┬─────┘ │ + contrastive loss │
- ┌─────┬─────┴─────┬─────┐ │ + token-level wt │
- │ @E[:d_ff/f] for each │ └──────────┬──────────┘
- │ f in mat_factors │ │
- └─────┬─────┬─────┬─────┘ ┌──────────┴───────────┐
- ┌─────┴─────┐ │ Grad clip (norm 1.0) │
- │ Linear (T)│ ← tied │ Muon (2D kernels) │
- └─────┬─────┘ │ AdamW (all else) │
- ┌─────┴─────┐ │ WSD LR schedule │
- │ ZCRMSNorm │ └──────────┬───────────┘
- └─────┬─────┘ │
- ┌────────┴────────┐ ┌──────────┴───────────┐
- │ Decoder x N_dec │ │ Sparsification │
- │┌───────────────┐│ │ cubic schedule │
- ││ Masked Self ││ │ mask every N steps │
- ││ Attn + RoPE ││ └──────────┬───────────┘
- │├───────────────┤│ │
- ┌──────────────┐ ││ Cross ││ ┌──────────┴───────────┐
- │ Self-Attn │────────▶ Attention ││ │ EMA params (β=0.999) │
- │ Encoder │ │├───────────────┤│ └──────────┬───────────┘
- │ x N_enc │ ││ Gated FFN ││ │
- │ │ │└───────────────┘│ ┌──────────┴───────────┐
- │ ┌──────────┐ │ └────────┬────────┘ │ Checkpoint │
- │ │Self Attn │ │ ┌─────┴─────┐ │ full params, can │
- │ │ + RoPE │ │ │ Embedding │ ← shared │ export mat slices │
- │ │ GQA │ │ └─────┬─────┘ └──────────────────────┘
- │ ├──────────┤ │ ┌───────┴───────┐
- │ │Gated FFN │ │ │[EOS]<tool_call>│
- │ │ SwiGLU │ │ │ + answer │
- │ ├──────────┤ │ └───────────────┘
- │ │ZCRMSNorm │ │
- │ │ Pre-norm │ │
- │ └──────────┘ │
- │ │
+ Architecture
+ ────────────
+
+ Pure-attention encoder-decoder. No feedforward layers — each block is
+ just gated self-attention (encoder) or gated self + cross attention
+ (decoder). All mixing happens through attention.
+
+ ┌──────────────┐
+ │ Tool Call │
+ └──────┬───────┘
+ ┌┴──────────┐
+ │ Softmax │
+ └─────┬─────┘
+ ┌─────┴─────┐
+ │ Linear (T)│ ← tied
+ └─────┬─────┘
+ ┌─────┴─────┐
+ │ ZCRMSNorm │
+ └─────┬─────┘
+ ┌────────┴────────┐
+ │ Decoder x 12 │
+ │┌───────────────┐│
+ ││ ZCRMSNorm ││
+ ││ Masked Self ││
+ ││ Attn + RoPE ││
+ ││ Gated Residual││
+ │├───────────────┤│
+ ┌──────────────┐ ││ ZCRMSNorm ││
+ │ Encoder x 12 │──────────────────────▶Cross Attn ││
+ │ │ ││ Gated Residual││
+ │ ┌──────────┐ │ │└───────────────┘│
+ │ │ZCRMSNorm │ │ └────────┬────────┘
+ │ │Self Attn │ │ ┌─────┴─────┐
+ │ │ GQA+RoPE │ │ │ Embedding │ ← shared
+ │ │Gated Res │ │ └─────┬─────┘
+ │ │ │ │ ┌───────┴───────-┐
+ │ │ (no FFN) │ │ │[EOS]<tool_call>│
+ │ └──────────┘ │ │ + answer │
+ │ │ └───────────────-┘
│ DW Conv ↓2 │ stride-2 depthwise-separable
- │ + Pointwise │ downsamples seq before encoder
+ │ + Pointwise │ (speech pathway only)
└──────┬───────┘
│
┌────┴──────┐ ┌──────────────┐
@@ -63,15 +60,55 @@
│ query │ │ waveform │
└───────────┘ └──────────────┘
- d=512 · 8 enc / 8 dec layers · GQA (8H / 4KV) · QK-norm
- SentencePiece BPE (8192) · SwiGLU · ZCRMSNorm · RoPE
- strided DW conv · mat factors · INT4 QAT · Muon + AdamW
+ Heads
+ ─────
+ ┌─────────────────────────────────────────────────────────────┐
+ │ │
+ │ Contrastive Head CLIP-style tool retrieval │
+ │ encoder → mean_pool → Dense(d_model/4) → ReLU │
+ │ → Dense(contrastive_dim) → L2-normalize │
+ │ learnable temperature (log_temp) │
+ │ │
+ │ Confidence Head cloud routing / hybrid inference │
+ │ encoder → mean_pool → Dense(d_model/4) │
+ │ → ReLU → Dense(1) → sigmoid → [0, 1] │
+ │ trained post-hoc on perplexity-derived labels │
+ │ high = handle locally, low = route to cloud │
+ │ │
+ └─────────────────────────────────────────────────────────────┘
+
+ Training Pipeline
+ ─────────────────
+ ┌─────────────────────────────────────────────────────────────┐
+ │ Contrastive loss + token-level weighting + z-loss │
+ │ Grad clip (norm 1.0) · Muon (2D kernels) · AdamW (rest) │
+ │ WSD LR schedule (warmup → hold → cosine decay) │
+ │ Grammar-constrained decoding (trie-based token masking) │
+ └─────────────────────────────────────────────────────────────┘
+
+ d=512 · 12 enc / 12 dec layers · GQA (8H / 4KV) · QK-norm
+ no FFN (pure attention) · gated residuals (init=0.5)
+ SentencePiece BPE (8192) · ZCRMSNorm · RoPE
text + speech encoder · <tool_call> task routing
- CLIP contrastive retrieval · token-level loss weighting
+ CLIP contrastive retrieval · confidence-based cloud routing
+ token-level loss weighting · grammar-constrained decoding
- Data Pipeline (needle tokenize → needle train)
- ───────────────────────────────────────────────
+ Data Pipeline (needle generate-data → needle tokenize → needle train)
+ ─────────────────────────────────────────────────────────────────────
+ ┌─────────────────────────────────────────────────────────────┐
+ │ needle generate-data │
+ │ │
+ │ Gemini synthesis → Cactus-Compute/tool-calls (HuggingFace) │
+ │ parallel workers · dedup · auto-merge · auto-upload │
+ │ │
+ │ Related: │
+ │ needle merge-xlam merge xlam-60k into dataset │
+ │ needle rebalance-tools trim over-represented bins │
+ │ needle split-dataset create train/val splits │
+ └─────────────────────────────────────────────────────────────┘
+ │
+ ▼
┌─────────────────────────────────────────────────────────────┐
│ needle tokenize │
│ │
@@ -115,6 +152,51 @@
│ └──────────┬───────────┘ └──────────┬───────────┘ │
│ ▼ ▼ │
│ text + contrastive tool-call training (jax.pmap) │
+ └─────────────────────────────────────────────────────────────┘
+ │
+ ▼
+ ┌─────────────────────────────────────────────────────────────┐
+ │ needle calibrate │
+ │ │
+ │ Freeze encoder + decoder │
+ │ Compute per-example decoder perplexity │
+ │ Map PPL → [0,1] confidence via sigmoid calibration │
+ │ Train confidence head (2 Dense layers, MSE loss) │
+ │ Save calibrated checkpoint with mu, k params │
+ └─────────────────────────────────────────────────────────────┘
+
+ Hybrid Inference (confidence-based cloud routing)
+ ─────────────────────────────────────────────────
+
+ ┌─────────────────────────────────────────────────────────────┐
+ │ │
+ │ query + tools │
+ │ │ │
+ │ ▼ │
+ │ ┌──────────┐ │
+ │ │ Encoder │ │
+ │ └────┬─────┘ │
+ │ │ │
+ │ ├──────────────────┐ │
+ │ ▼ ▼ │
+ │ ┌──────────┐ ┌──────────────┐ │
+ │ │Confidence│ │ (encoder │ │
+ │ │ Head │ │ output) │ │
+ │ └────┬─────┘ └──────┬───────┘ │
+ │ │ │ │
+ │ ▼ │ │
+ │ confidence > threshold?│ │
+ │ │ │ │
+ │ ┌──┴──┐ │ │
+ │ │ yes │──────────────┤ │
+ │ └─────┘ ▼ │
+ │ ┌──────────┐ │
+ │ │ Decoder │──▶ tool calls (local) │
+ │ └──────────┘ │
+ │ ┌─────┐ │
+ │ │ no │──▶ route to cloud (Gemini) ──▶ tool calls │
+ │ └─────┘ (skip decoder entirely) │
+ │ │
└─────────────────────────────────────────────────────────────┘
```
@@ -130,33 +212,37 @@ needle [command]
┌───────────────────────────────────────────────────────────────────┐
│ │
│ train │
- │ --epochs INT Training epochs (default: 10) │
+ │ --epochs INT Training epochs (default: 1) │
│ --batch-size INT Batch size (default: 32) │
│ --lr FLOAT AdamW learning rate (default: 3e-4) │
│ --muon-lr FLOAT Muon learning rate (default: 0.02) │
│ --d-model INT Model dim (default: 512) │
│ --num-heads INT Attention heads (default: 8) │
│ --num-kv-heads INT KV heads for GQA (default: 4) │
- │ --num-layers INT Encoder layers (default: 8) │
- │ --num-dec-layers INT Decoder layers (default: 8) │
+ │ --num-layers INT Encoder layers (default: 12) │
+ │ --num-dec-layers INT Decoder layers (default: 12) │
│ --max-enc-len INT Max encoder seq len (default: 1024) │
│ --max-dec-len INT Max decoder seq len (default: 512) │
│ --max-samples INT Training samples (default: all) │
- │ --mat-factors INT [...] FFN shrink factors (default: 2 4) │
+ │ --no-feedforward No FFN layers (default: on) │
+ │ --feedforward Enable FFN layers (off by default) │
+ │ --activation STR swiglu|drelu|geglu (if FFN enabled) │
+ │ --mat-factors INT [...] FFN shrink factors (if FFN enabled) │
│ --sparsity-ratio FLOAT Block prune ratio (default: 0.0) │
│ --group-size INT Quant/prune group size (default: 32) │
│ --prune-interval INT Steps between mask updates (def: 100)│
│ --prune-start-frac FL Start pruning at frac (def: 0.33) │
│ --prune-end-frac FL Lock mask at this frac (def: 0.67) │
- │ --activation STR swiglu|drelu|geglu (default: swiglu) │
│ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │
│ --decay-ratio FLOAT LR cosine decay ratio (default: 0.05)│
- │ --dropout FLOAT Dropout rate (default: 0.1) │
+ │ --dropout FLOAT Dropout rate (default: 0.0) │
│ --eval-every INT Val eval interval (default: 1000) │
│ --max-eval-samples INT Val samples limit (default: 5000) │
│ --contrastive-weight FL CLIP loss weight (default: 0.1) │
│ --contrastive-dim INT Projection dim (default: 128) │
- │ --curriculum Sort batches easy→hard by tool count │
+ │ --w-name FLOAT Loss weight: tool names (def: 2.0) │
+ │ --w-value FLOAT Loss weight: arg values (def: 4.0) │
+ │ --w-key FLOAT Loss weight: arg keys (def: 1.5) │
│ --wandb Enable W&B logging │
│ --checkpoint PATH Resume from checkpoint │
│ --checkpoint-dir DIR Checkpoint directory │
@@ -167,9 +253,6 @@ needle [command]
│ --max-samples INT Limit samples per split (dev/test) │
│ --max-enc-len INT Max encoder seq len (default: 1024) │
│ --max-dec-len INT Max decoder seq len (default: 512) │
- │ --w-name FLOAT Loss weight: tool names (def: 3.0) │
- │ --w-value FLOAT Loss weight: arg values (def: 2.0) │
- │ --w-key FLOAT Loss weight: arg keys (def: 1.5) │
│ --shuffle-tools Shuffle tool order (default: on) │
│ --no-shuffle-tools Disable tool shuffling │
│ --max-tool-len INT Max tool desc tokens (default: 256) │
@@ -181,6 +264,7 @@ needle [command]
│ --audio PATH [...] Audio files for voice-to-tool-call │
│ --max-len INT Max tokens to generate (default: 512)│
│ --seed INT Random seed (default: 0) │
+ │ --no-constrained Disable constrained decoding │
│ │
│ eval │
│ --checkpoint PATH Path to model checkpoint (required) │
@@ -191,6 +275,34 @@ needle [command]
│ --max-gen-len INT Max generation length (default: 512) │
│ --throughput-runs INT Throughput runs (default: 10) │
│ --tool-call-samples INT Tool-call eval samples (default: 200)│
+ │ --no-constrained Disable constrained decoding │
+ │ │
+ │ calibrate │
+ │ --checkpoint PATH Path to model checkpoint (required) │
+ │ --output PATH Output path (default: overwrite) │
+ │ --batch-size INT Batch size (default: 32) │
+ │ --num-samples INT Limit samples for PPL (default: all) │
+ │ --epochs INT Training epochs (default: 10) │
+ │ --lr FLOAT Learning rate (default: 1e-3) │
+ │ --k FLOAT Sigmoid steepness (default: 3.0) │
+ │ │
+ │ generate-data │
+ │ --num-samples INT Samples to generate (default: 5000) │
+ │ --batch-size INT Examples per Gemini call (default:10)│
+ │ --workers INT Parallel Gemini calls (default: 8) │
+ │ --model STR Gemini model override │
+ │ --dry-run Generate only, skip upload │
+ │ --output-jsonl PATH Also save raw generations to JSONL │
+ │ --upload-every INT Merge+upload interval (def: 30000) │
+ │ │
+ │ merge-xlam │
+ │ --dry-run Skip upload │
+ │ --max-samples INT Limit xlam samples │
+ │ │
+ │ rebalance-tools │
+ │ --dry-run Preview without modifying │
+ │ │
+ │ split-dataset Create train/val splits + upload │
│ │
│ evaluate │
│ --checkpoint PATH Path to model checkpoint (required) │
@@ -247,11 +359,11 @@ needle [command]
## Setup For TPU/GCP
-- Setup gcloud 1: download the `macOS ARM` from [here](https://docs.cloud.google.com/sdk/docs/install-sdk) and uzip.
-- Setup gcloud 2: open terminal, cd to ypur downloads and run `./google-cloud-sdk/install.sh`
-- Setup gcloud 3: restart terminal and run `gloud init`, sign in with cactus email, should prompt for project
+- Setup gcloud 1: download the `macOS ARM` from [here](https://docs.cloud.google.com/sdk/docs/install-sdk) and unzip.
+- Setup gcloud 2: open terminal, cd to your downloads and run `./google-cloud-sdk/install.sh`
+- Setup gcloud 3: restart terminal and run `gcloud init`, sign in with cactus email, should prompt for project
- Setup gcloud 4: else, set the project with `gcloud config set project needle-488623`
-- setup gcloud 5: run `gcloud help` and read carefully
+- Setup gcloud 5: run `gcloud help` and read carefully
## Example Workflow
@@ -281,8 +393,12 @@ needle [command]
huggingface-cli login
(paste your HF token — get one at https://huggingface.co/settings/tokens)
-7. Use needle as you normally would locally, like training
- needle train --wandb
+7. Full pipeline
+ needle generate-data --num-samples 5000 # synthesize training data
+ needle tokenize # tokenize + pack + upload
+ needle train --wandb # train on TPU
+ needle calibrate --checkpoint <path> # calibrate confidence head
+ needle eval --checkpoint <path> # evaluate
8. Use tmux for long training runs (survives SSH disconnects)
tmux new -s train # start a named session
diff --git a/notes/voice-to-action.txt b/notes/dataset.txt
similarity index 100%
rename from notes/voice-to-action.txt
rename to notes/dataset.txt
diff --git a/src/calibrate.py b/src/calibrate.py
new file mode 100644
index 0000000..e61a956
--- /dev/null
+++ b/src/calibrate.py
@@ -0,0 +1,332 @@
+"""Post-training calibration of the confidence head.
+
+Freezes the encoder and decoder, computes per-example decoder perplexity
+on the training data, maps perplexity to [0, 1] confidence labels via a
+sigmoid calibration curve, then trains only the confidence head (2 Dense layers)
+to predict these labels from encoder output.
+
+Usage:
+ needle calibrate --checkpoint checkpoints/needle_12_512_50000.pkl
+ needle calibrate --checkpoint checkpoints/needle_12_512_50000.pkl --num-samples 10000
+"""
+
+import math
+import os
+import pickle
+
+import jax
+import jax.numpy as jnp
+import numpy as np
+import optax
+from tqdm import tqdm
+
+from .data import (
+ get_tokenizer, load_prepared_data, count_batches, get_batches,
+)
+from .model import (
+ EncoderDecoderTransformer, TransformerConfig,
+ make_packing_mask, make_causal_packing_mask, make_cross_packing_mask,
+)
+
+
+def compute_per_example_perplexity(model, params, val_data, batch_size, max_samples=None):
+ """Compute per-example perplexity using packed validation data.
+
+ Returns:
+ encoder_outputs: list of (encoder_out, enc_mask) tuples
+ perplexities: numpy array of per-example perplexity values
+ """
+ enc = val_data["packed_enc"]
+ dec_in = val_data["packed_dec_in"]
+ dec_tgt = val_data["packed_dec_tgt"]
+ enc_seg = val_data["packed_enc_seg"]
+ dec_seg = val_data["packed_dec_seg"]
+
+ n = len(enc)
+ if max_samples:
+ n = min(n, max_samples)
+
+ all_ppl = []
+ all_enc_out = []
+ all_enc_mask = []
+ all_src = []
+ all_enc_seg_ids = []
+
+ @jax.jit
+ def forward_step(params, src, tgt_in, tgt_out, enc_seg_ids, dec_seg_ids):
+ src_mask = make_packing_mask(enc_seg_ids)
+ tgt_mask = make_causal_packing_mask(dec_seg_ids)
+ cross_mask = make_cross_packing_mask(enc_seg_ids, dec_seg_ids)
+
+ # Get encoder output
+ encoder_out, enc_mask = model.apply(
+ {"params": params}, src, src_mask=src_mask, method="encode_text",
+ )
+
+ # Get decoder logits
+ logits = model.apply(
+ {"params": params}, src, tgt_in,
+ src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
+ )
+
+ # Per-token cross entropy
+ ce = optax.softmax_cross_entropy_with_integer_labels(
+ logits.astype(jnp.float32), tgt_out
+ )
+ # Per-example: average over non-padding tokens
+ padding_mask = (dec_seg_ids > 0).astype(jnp.float32)
+ per_example_loss = jnp.sum(ce * padding_mask, axis=-1) / jnp.maximum(
+ jnp.sum(padding_mask, axis=-1), 1.0
+ )
+ per_example_ppl = jnp.exp(jnp.minimum(per_example_loss, 20.0))
+
+ return per_example_ppl, encoder_out, enc_mask
+
+ print(f"Computing per-example perplexity ({n} examples, batch_size={batch_size})...")
+ for i in tqdm(range(0, n, batch_size), desc="PPL forward"):
+ end = min(i + batch_size, n)
+ src = jnp.asarray(enc[i:end], dtype=jnp.int32)
+ tgt_in = jnp.asarray(dec_in[i:end], dtype=jnp.int32)
+ tgt_out = jnp.asarray(dec_tgt[i:end], dtype=jnp.int32)
+ es = jnp.asarray(enc_seg[i:end], dtype=jnp.int32)
+ ds = jnp.asarray(dec_seg[i:end], dtype=jnp.int32)
+
+ ppl, enc_out, enc_m = forward_step(params, src, tgt_in, tgt_out, es, ds)
+ all_ppl.append(np.array(ppl))
+ all_enc_out.append(np.array(enc_out))
+ all_enc_mask.append(np.array(enc_m))
+ all_src.append(np.array(src))
+ all_enc_seg_ids.append(np.array(es))
+
+ perplexities = np.concatenate(all_ppl)
+ print(f" Perplexity stats: min={perplexities.min():.2f}, median={np.median(perplexities):.2f}, "
+ f"mean={perplexities.mean():.2f}, max={perplexities.max():.2f}")
+
+ return all_enc_out, all_enc_mask, all_src, all_enc_seg_ids, perplexities
+
+
+def perplexity_to_confidence(perplexities, mu=None, k=3.0):
+ """Map perplexity values to [0, 1] confidence via sigmoid calibration.
+
+ confidence = sigmoid(-k * (log(ppl) - log(mu)))
+
+ Args:
+ perplexities: array of perplexity values
+ mu: center point (default: median perplexity)
+ k: steepness of the sigmoid curve
+
+ Returns:
+ confidence labels in [0, 1]
+ """
+ if mu is None:
+ mu = float(np.median(perplexities))
+
+ log_ppl = np.log(np.maximum(perplexities, 1.0))
+ log_mu = np.log(mu)
+
+ # sigmoid(-k * (log_ppl - log_mu))
+ # PPL << mu → large positive → confidence ≈ 1
+ # PPL >> mu → large negative → confidence ≈ 0
+ confidence = 1.0 / (1.0 + np.exp(k * (log_ppl - log_mu)))
+
+ return confidence.astype(np.float32), mu
+
+
+def train_confidence_head(model, params, enc_outputs, enc_masks, confidence_labels,
+ epochs=10, lr=1e-3, batch_size=64):
+ """Train only the confidence head parameters, freezing everything else.
+
+ Args:
+ model: EncoderDecoderTransformer instance
+ params: full model params (only confidence_hidden and confidence_out will be updated)
+ enc_outputs: list of (B, T, d_model) encoder output arrays
+ enc_masks: list of (B, 1, 1, T) mask arrays
+ confidence_labels: (N,) float32 array of target confidence values
+
+ Returns:
+ Updated params with trained confidence head.
+ """
+ # Flatten batched arrays
+ all_enc = np.concatenate(enc_outputs, axis=0)
+ all_mask = np.concatenate(enc_masks, axis=0)
+ n = len(confidence_labels)
+
+ # Extract confidence head params
+ conf_param_keys = {"confidence_hidden", "confidence_out"}
+
+ def is_conf_param(path):
+ return any(k in str(path) for k in conf_param_keys)
+
+ # Create optimizer for confidence params only
+ tx = optax.adam(lr)
+
+ # We need to separate confidence params from frozen params
+ conf_params = jax.tree_util.tree_map_with_path(
+ lambda path, x: x if is_conf_param(path) else jax.lax.stop_gradient(x),
+ params,
+ )
+ opt_state = tx.init(params)
+
+ @jax.jit
+ def confidence_loss(params, enc_out, enc_mask, targets):
+ """MSE loss between predicted and target confidence."""
+ pred = model.apply(
+ {"params": params}, enc_out, enc_mask,
+ method="predict_confidence",
+ )
+ return jnp.mean((pred - targets) ** 2)
+
+ @jax.jit
+ def train_step(params, opt_state, enc_out, enc_mask, targets):
+ loss, grads = jax.value_and_grad(confidence_loss)(params, enc_out, enc_mask, targets)
+
+ # Zero out gradients for non-confidence params
+ grads = jax.tree_util.tree_map_with_path(
+ lambda path, g: g if is_conf_param(path) else jnp.zeros_like(g),
+ grads,
+ )
+
+ updates, opt_state_new = tx.update(grads, opt_state, params)
+ params_new = optax.apply_updates(params, updates)
+ return params_new, opt_state_new, loss
+
+ print(f"\nTraining confidence head ({n} examples, {epochs} epochs, lr={lr})...")
+
+ for epoch in range(epochs):
+ perm = np.random.permutation(n)
+ epoch_losses = []
+
+ for i in range(0, n - batch_size + 1, batch_size):
+ idx = perm[i:i + batch_size]
+ enc_batch = jnp.asarray(all_enc[idx])
+ mask_batch = jnp.asarray(all_mask[idx])
+ target_batch = jnp.asarray(confidence_labels[idx])
+
+ params, opt_state, loss = train_step(params, opt_state, enc_batch, mask_batch, target_batch)
+ epoch_losses.append(float(loss))
+
+ avg_loss = sum(epoch_losses) / len(epoch_losses) if epoch_losses else float("nan")
+ print(f" Epoch {epoch + 1}/{epochs}: MSE={avg_loss:.6f}")
+
+ return params
+
+
+def evaluate_confidence(model, params, enc_outputs, enc_masks, confidence_labels, perplexities):
+ """Evaluate calibrated confidence head and print diagnostic stats."""
+ all_enc = np.concatenate(enc_outputs, axis=0)
+ all_mask = np.concatenate(enc_masks, axis=0)
+ n = len(confidence_labels)
+
+ # Predict in batches
+ predictions = []
+ batch_size = 128
+ for i in range(0, n, batch_size):
+ end = min(i + batch_size, n)
+ pred = model.apply(
+ {"params": params},
+ jnp.asarray(all_enc[i:end]),
+ jnp.asarray(all_mask[i:end]),
+ method="predict_confidence",
+ )
+ predictions.append(np.array(pred))
+ predictions = np.concatenate(predictions)
+
+ # Stats
+ mse = np.mean((predictions - confidence_labels) ** 2)
+ mae = np.mean(np.abs(predictions - confidence_labels))
+ correlation = np.corrcoef(predictions, confidence_labels)[0, 1]
+
+ print(f"\nConfidence head evaluation:")
+ print(f" MSE: {mse:.6f}")
+ print(f" MAE: {mae:.4f}")
+ print(f" Correlation: {correlation:.4f}")
+
+ # Show confidence vs perplexity buckets
+ print(f"\n {'PPL Range':<15s} {'Count':>6s} {'Avg Conf':>10s} {'Avg Pred':>10s}")
+ print(f" {'─' * 45}")
+ buckets = [(0, 2), (2, 4), (4, 8), (8, 16), (16, 50), (50, 1000)]
+ for lo, hi in buckets:
+ mask = (perplexities >= lo) & (perplexities < hi)
+ if mask.sum() == 0:
+ continue
+ avg_conf = confidence_labels[mask].mean()
+ avg_pred = predictions[mask].mean()
+ print(f" {f'{lo}-{hi}':<15s} {mask.sum():>6d} {avg_conf:>10.3f} {avg_pred:>10.3f}")
+
+ return predictions
+
+
+def main(args):
+ # Load checkpoint
+ print(f"Loading checkpoint: {args.checkpoint}")
+ with open(args.checkpoint, "rb") as f:
+ data = pickle.load(f)
+ params = jax.tree.map(lambda x: jnp.array(x, dtype=jnp.bfloat16), data["params"])
+ config = TransformerConfig(**data["config"])
+ model = EncoderDecoderTransformer(config)
+
+ param_count = sum(x.size for x in jax.tree.leaves(params))
+ print(f" Parameters: {param_count:,}")
+ print(f" Config: d={config.d_model}, heads={config.num_heads}, "
+ f"layers={config.num_encoder_layers}/{config.num_decoder_layers}")
+
+ # Check if confidence head params exist; if not, initialize them
+ flat_keys = str(jax.tree.map(lambda x: x.shape, params))
+ if "confidence_hidden" not in flat_keys:
+ print(" Confidence head not found in checkpoint — initializing...")
+ rng = jax.random.PRNGKey(0)
+ dummy_src = jnp.ones((1, 128), dtype=jnp.int32)
+ dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)
+ init_vars = model.init({"params": rng}, dummy_src, dummy_tgt, method="init_all")
+ # Merge: keep existing params, add new confidence head params
+ init_params = init_vars["params"]
+ params = {**params, "confidence_hidden": init_params["confidence_hidden"],
+ "confidence_out": init_params["confidence_out"]}
+ print(" Confidence head initialized.")
+
+ # Load training data for perplexity computation
+ print(f"\nLoading prepared data...")
+ train_data = load_prepared_data("train", mmap=True)
+ print(f" {len(train_data['packed_enc']):,} packed bins")
+
+ # Step 1: Compute per-example perplexity
+ enc_outputs, enc_masks, srcs, enc_seg_ids, perplexities = compute_per_example_perplexity(
+ model, params, train_data, args.batch_size, max_samples=args.num_samples,
+ )
+
+ # Step 2: Map perplexity → confidence labels
+ confidence_labels, mu = perplexity_to_confidence(perplexities, k=args.k)
+ print(f"\nCalibration: mu (median PPL) = {mu:.2f}, k = {args.k}")
+ print(f" Confidence stats: min={confidence_labels.min():.3f}, "
+ f"median={np.median(confidence_labels):.3f}, "
+ f"mean={confidence_labels.mean():.3f}, max={confidence_labels.max():.3f}")
+
+ # Step 3: Train the confidence head
+ params = train_confidence_head(
+ model, params, enc_outputs, enc_masks, confidence_labels,
+ epochs=args.epochs, lr=args.lr, batch_size=args.batch_size,
+ )
+
+ # Step 4: Evaluate
+ evaluate_confidence(model, params, enc_outputs, enc_masks, confidence_labels, perplexities)
+
+ # Step 5: Save updated checkpoint
+ output_path = args.output or args.checkpoint
+ params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), params)
+
+ # Store calibration metadata alongside config
+ save_config = dict(data["config"])
+ save_config["confidence_mu"] = float(mu)
+ save_config["confidence_k"] = float(args.k)
+
+ with open(output_path, "wb") as f:
+ pickle.dump({"params": params_np, "config": save_config}, f)
+ print(f"\nSaved calibrated checkpoint: {output_path}")
+ print(f" Calibration params: mu={mu:.2f}, k={args.k}")
+ print(f"\n Usage at inference:")
+ print(f" confidence, encoder_out, enc_mask = model.apply(")
+ print(f' {{"params": params}}, src, src_mask=src_mask, method="forward_confidence")')
+ print(f" if confidence > 0.6: # user-configurable threshold")
+ print(f" # decode locally")
+ print(f" else:")
+ print(f" # route to cloud")
diff --git a/src/cli.py b/src/cli.py
index dedb175..9f60261 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -96,6 +96,33 @@ def main():
p.add_argument("--no-constrained", action="store_true",
help="Disable grammar-constrained decoding for tool names/arg keys")
+ p = sub.add_parser("calibrate", add_help=False)
+ p.add_argument("--checkpoint", type=str, required=True)
+ p.add_argument("--output", type=str, default=None, help="Output checkpoint path (default: overwrite input)")
+ p.add_argument("--batch-size", type=int, default=32)
+ p.add_argument("--num-samples", type=int, default=None, help="Limit training samples for PPL computation")
+ p.add_argument("--epochs", type=int, default=10)
+ p.add_argument("--lr", type=float, default=1e-3)
+ p.add_argument("--k", type=float, default=3.0, help="Sigmoid steepness for PPL→confidence mapping")
+
+ p = sub.add_parser("generate-data", add_help=False)
+ p.add_argument("--num-samples", type=int, default=5000, help="Number of samples to generate")
+ p.add_argument("--batch-size", type=int, default=10, help="Examples per Gemini call")
+ p.add_argument("--workers", type=int, default=8, help="Parallel Gemini calls")
+ p.add_argument("--model", type=str, default=None, help="Gemini model override")
+ p.add_argument("--dry-run", action="store_true", help="Generate only, skip save and upload")
+ p.add_argument("--output-jsonl", type=str, default=None, help="Also save raw generations to JSONL")
+ p.add_argument("--upload-every", type=int, default=None, help="Merge+upload every N samples")
+
+ p = sub.add_parser("merge-xlam", add_help=False)
+ p.add_argument("--dry-run", action="store_true", help="Skip upload")
+ p.add_argument("--max-samples", type=int, default=None, help="Limit xlam samples")
+
+ p = sub.add_parser("rebalance-tools", add_help=False)
+ p.add_argument("--dry-run", action="store_true", help="Preview without modifying")
+
+ p = sub.add_parser("split-dataset", add_help=False)
+
p = sub.add_parser("evaluate", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
p.add_argument("--benchmarks", type=str, nargs="*",
@@ -151,6 +178,25 @@ def main():
elif args.command == "eval":
from .eval import main as eval_main_fn
eval_main_fn(args)
+ elif args.command == "calibrate":
+ from .calibrate import main as calibrate_main
+ calibrate_main(args)
+ elif args.command == "generate-data":
+ from .generate_data import main as gendata_main, MODEL as _MODEL, UPLOAD_EVERY as _UE
+ if args.model is None:
+ args.model = _MODEL
+ if args.upload_every is None:
+ args.upload_every = _UE
+ gendata_main(args)
+ elif args.command == "merge-xlam":
+ from .merge_xlam import main as merge_main
+ merge_main(args)
+ elif args.command == "rebalance-tools":
+ from .rebalance_tools import main as rebalance_main
+ rebalance_main(args)
+ elif args.command == "split-dataset":
+ from .split_dataset import main as split_main
+ split_main(args)
elif args.command == "evaluate":
from .evaluate import main as eval_main
eval_main(args)
diff --git a/scripts/generate_data.py b/src/generate_data.py
similarity index 99%
rename from scripts/generate_data.py
rename to src/generate_data.py
index dde1423..f646a73 100644
--- a/scripts/generate_data.py
+++ b/src/generate_data.py
@@ -1698,7 +1698,7 @@ def generate_all(num_samples, workers=8, batch_size=25, model=MODEL, client_pool
-LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(__file__), "..", "data", "tool_calls_unified")
+LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data", "tool_calls_unified")
HF_DATASET_REPO = "Cactus-Compute/tool-calls"
UPLOAD_EVERY = 30000
@@ -1791,21 +1791,12 @@ def _merge_and_upload(existing, new_examples):
import shutil as _shutil
_shutil.rmtree(parquet_dir)
print(f"Upload complete: {HF_DATASET_REPO}")
- print("NOTE: Run 'python scripts/split_dataset.py' to create the validation split.")
+ print("NOTE: Run 'needle split-dataset' to create the validation split.")
return merged
-def main():
- parser = argparse.ArgumentParser(description="Generate on-device assistant tool-calling data with Gemini")
- parser.add_argument("--num-samples", type=int, default=5000, help="Number of samples to generate")
- parser.add_argument("--batch-size", type=int, default=10, help="Examples per Gemini call (smaller = more tool diversity, more API calls)")
- parser.add_argument("--workers", type=int, default=8, help="Parallel Gemini calls")
- parser.add_argument("--model", type=str, default=MODEL, help="Gemini model")
- parser.add_argument("--dry-run", action="store_true", help="Generate only, skip save and upload")
- parser.add_argument("--output-jsonl", type=str, default=None, help="Also save raw generations to JSONL")
- parser.add_argument("--upload-every", type=int, default=UPLOAD_EVERY, help="Merge+upload every N samples (default 1000)")
- args = parser.parse_args()
+def main(args):
client_pool = ClientPool(make_clients())
@@ -1897,4 +1888,13 @@ def main():
if __name__ == "__main__":
- main()
+ import argparse as _ap
+ _p = _ap.ArgumentParser(description="Generate on-device assistant tool-calling data with Gemini")
+ _p.add_argument("--num-samples", type=int, default=500)
+ _p.add_argument("--batch-size", type=int, default=10)
+ _p.add_argument("--workers", type=int, default=8)
+ _p.add_argument("--model", type=str, default=MODEL)
+ _p.add_argument("--dry-run", action="store_true")
+ _p.add_argument("--output-jsonl", type=str, default=None)
+ _p.add_argument("--upload-every", type=int, default=UPLOAD_EVERY)
+ main(_p.parse_args())
diff --git a/scripts/merge_xlam.py b/src/merge_xlam.py
similarity index 87%
rename from scripts/merge_xlam.py
rename to src/merge_xlam.py
index d439e54..b085daa 100644
--- a/scripts/merge_xlam.py
+++ b/src/merge_xlam.py
@@ -15,7 +15,7 @@ import sys
from datasets import concatenate_datasets, load_dataset, load_from_disk
-LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(__file__), "..", "data", "tool_calls_unified")
+LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data", "tool_calls_unified")
HF_DATASET_REPO = "Cactus-Compute/tool-calls"
@@ -60,12 +60,7 @@ def check_duplicates(existing, new):
print("No duplicate queries detected (sampled check)")
-def main():
- parser = argparse.ArgumentParser(description="Merge xlam into unified dataset")
- parser.add_argument("--dry-run", action="store_true", help="Skip upload")
- parser.add_argument("--max-samples", type=int, default=None, help="Limit xlam samples")
- args = parser.parse_args()
-
+def main(args):
existing = load_existing()
xlam = load_xlam(args.max_samples)
@@ -106,8 +101,12 @@ def main():
print(f"\nUploading to {HF_DATASET_REPO} (train split)...")
merged.push_to_hub(HF_DATASET_REPO, split="train", token=True)
print(f"Upload complete: {HF_DATASET_REPO}")
- print("NOTE: Run 'python scripts/split_dataset.py' to create the validation split.")
+ print("NOTE: Run 'needle split-dataset' to create the validation split.")
if __name__ == "__main__":
- main()
\ No newline at end of file
+ import argparse as _ap
+ _p = _ap.ArgumentParser(description="Merge xlam into unified dataset")
+ _p.add_argument("--dry-run", action="store_true", help="Skip upload")
+ _p.add_argument("--max-samples", type=int, default=None, help="Limit xlam samples")
+ main(_p.parse_args())
\ No newline at end of file
diff --git a/src/model.py b/src/model.py
index 731b8c0..b555c91 100644
--- a/src/model.py
+++ b/src/model.py
@@ -412,11 +412,24 @@ class EncoderDecoderTransformer(nn.Module):
self.config.d_model, dtype=self.config.jax_dtype, name="speech_subsample")
self.encoder = Encoder(self.config)
self.decoder = Decoder(self.config)
+ self.contrastive_hidden = nn.Dense(
+ self.config.d_model // 4, dtype=self.config.jax_dtype,
+ kernel_init=default_init(), name="contrastive_hidden",
+ )
self.contrastive_proj = nn.Dense(
self.config.contrastive_dim, dtype=self.config.jax_dtype,
use_bias=False, kernel_init=default_init(), name="contrastive_proj",
)
self.log_temp = self.param("log_temp", jinit.zeros, ())
+
+ self.confidence_hidden = nn.Dense(
+ self.config.d_model // 4, dtype=self.config.jax_dtype,
+ kernel_init=default_init(), name="confidence_hidden",
+ )
+ self.confidence_out = nn.Dense(
+ 1, dtype=jnp.float32,
+ kernel_init=jinit.zeros, name="confidence_out",
+ )
def _rope(self, seq_len):
head_dim = self.config.d_model // self.config.num_heads
@@ -467,12 +480,40 @@ class EncoderDecoderTransformer(nn.Module):
counts = jnp.maximum(jnp.sum(mask_2d, axis=1, keepdims=True), 1.0)
return summed / counts
+ def predict_confidence(self, encoder_out, enc_mask):
+ """Predict confidence score [0, 1] from encoder output.
+
+ High confidence (→1.0) means the model can handle this query locally.
+ Low confidence (→0.0) means the query should be routed to cloud.
+
+ Args:
+ encoder_out: (B, T, d_model) encoder hidden states
+ enc_mask: (B, 1, 1, T) padding mask
+
+ Returns:
+ (B,) confidence scores in [0, 1]
+ """
+ pooled = self._mean_pool(encoder_out, enc_mask) # (B, d_model)
+ h = nn.relu(self.confidence_hidden(pooled)) # (B, d_model // 4)
+ return nn.sigmoid(self.confidence_out(h)).squeeze(-1) # (B,)
+
+ def forward_confidence(self, src, src_mask=None):
+ """End-to-end confidence prediction from token input.
+
+ Returns (confidence, encoder_out, enc_mask) so the caller can
+ reuse encoder output if confidence is high enough to decode.
+ """
+ encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)
+ confidence = self.predict_confidence(encoder_out, enc_mask)
+ return confidence, encoder_out, enc_mask
+
def encode_contrastive(self, tokens, deterministic=True):
"""Encode tokens and project to L2-normalized contrastive space. Returns (B, contrastive_dim)."""
src_mask = make_padding_mask(tokens, self.config.pad_token_id)
encoder_out, enc_mask = self.encode_text(tokens, src_mask=src_mask, deterministic=deterministic)
pooled = self._mean_pool(encoder_out, enc_mask)
- projected = self.contrastive_proj(pooled)
+ h = nn.relu(self.contrastive_hidden(pooled))
+ projected = self.contrastive_proj(h)
return projected / jnp.maximum(jnp.linalg.norm(projected, axis=-1, keepdims=True), 1e-8)
def forward_contrastive(self, query_tokens, tool_tokens, deterministic=True):
@@ -567,6 +608,7 @@ class EncoderDecoderTransformer(nn.Module):
speech_out, speech_enc_mask = self.encode_speech(mel, src_mask=mel_mask, deterministic=True)
_ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask, cross_mask=speech_enc_mask)
_ = self.encode_contrastive(src)
+ _ = self.predict_confidence(text_out, text_enc_mask)
return jnp.zeros(())
diff --git a/scripts/rebalance_tools.py b/src/rebalance_tools.py
similarity index 97%
rename from scripts/rebalance_tools.py
rename to src/rebalance_tools.py
index 9e834cb..fc51a8d 100644
--- a/scripts/rebalance_tools.py
+++ b/src/rebalance_tools.py
@@ -213,8 +213,12 @@ def rebalance(dry_run=False):
print("Done.")
-if __name__ == "__main__":
- parser = argparse.ArgumentParser()
- parser.add_argument("--dry-run", action="store_true", help="Preview without modifying")
- args = parser.parse_args()
+def main(args):
rebalance(dry_run=args.dry_run)
+
+
+if __name__ == "__main__":
+ import argparse as _ap
+ _p = _ap.ArgumentParser()
+ _p.add_argument("--dry-run", action="store_true", help="Preview without modifying")
+ main(_p.parse_args())
diff --git a/src/run.py b/src/run.py
index 3d6d01b..537d434 100644
--- a/src/run.py
+++ b/src/run.py
@@ -180,6 +180,86 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=DEFAULT_MA
return result
+def generate_with_confidence(model, params, tokenizer, query, tools="[]",
+ max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN,
+ seed=0, stream=True, normalize=True, constrained=True,
+ confidence_threshold=None):
+ """Generate tool calls with confidence score.
+
+ Returns (result, confidence) where confidence is a float in [0, 1].
+ If confidence_threshold is set and confidence is below it, returns
+ (None, confidence) without running the decoder — saving compute for
+ queries that should be routed to cloud.
+ """
+ name_map = {}
+ if normalize:
+ tools, name_map = normalize_tools(tools)
+
+ enc_tokens = _build_encoder_input(tokenizer, query, tools, max_enc_len)
+ enc_input = jnp.array([enc_tokens])
+ pad_id = tokenizer.pad_token_id
+ src_mask = make_padding_mask(enc_input, pad_id)
+
+ # Get confidence + encoder output in one pass
+ confidence, encoder_out, enc_mask = model.apply(
+ {"params": params}, enc_input, src_mask=src_mask,
+ method="forward_confidence",
+ )
+ confidence_val = float(confidence[0])
+
+ # Early exit if below threshold — skip decoder entirely
+ if confidence_threshold is not None and confidence_val < confidence_threshold:
+ return None, confidence_val
+
+ # Proceed with normal decoding using already-computed encoder output
+ eos_id = tokenizer.eos_token_id
+ dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
+ dec_buffer = dec_buffer.at[0, 0].set(eos_id)
+
+ decode_fn = _get_decode_fn(model, max_gen_len)
+
+ constrained_decoder = None
+ if constrained:
+ from .constrained import build_constrained_decoder
+ constrained_decoder = build_constrained_decoder([tools], tokenizer)
+
+ if stream:
+ sys.stdout.write(f"\n")
+ sys.stdout.flush()
+
+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
+ generated_tokens = []
+
+ for i in range(0, max_gen_len - 1):
+ next_logits = logits[0, i]
+ if constrained_decoder and constrained_decoder.is_active(0):
+ logits_np = np.array(next_logits)
+ logits_np = constrained_decoder.constrain_logits(logits_np, 0)
+ next_token = int(np.argmax(logits_np))
+ else:
+ next_token = int(jnp.argmax(next_logits))
+ if constrained_decoder:
+ constrained_decoder.update(0, next_token)
+ if next_token == eos_id:
+ break
+ generated_tokens.append(next_token)
+ dec_buffer = dec_buffer.at[0, i + 1].set(next_token)
+ if stream:
+ sys.stdout.write(tokenizer.decode([next_token]))
+ sys.stdout.flush()
+ logits = decode_fn(params, dec_buffer, encoder_out, enc_mask)
+
+ if stream:
+ sys.stdout.write("\n")
+
+ result = tokenizer.decode(generated_tokens)
+ if result.startswith("<tool_call>"):
+ result = result[len("<tool_call>"):]
+ if normalize and name_map:
+ result = restore_tool_names(result, name_map)
+ return result, confidence_val
+
+
def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=DEFAULT_MAX_GEN_LEN, max_enc_len=DEFAULT_MAX_ENC_LEN, normalize=True, constrained=True):
"""Batch-generate tool-call outputs for multiple examples at once.
diff --git a/scripts/split_dataset.py b/src/split_dataset.py
similarity index 99%
rename from scripts/split_dataset.py
rename to src/split_dataset.py
index bceef65..dc0d184 100644
--- a/scripts/split_dataset.py
+++ b/src/split_dataset.py
@@ -20,7 +20,7 @@ VAL_SIZE = 10000
SEED = 42
-def main():
+def main(args=None):
print(f"Downloading full dataset from {HF_DATASET_REPO}...")
ds = load_dataset(HF_DATASET_REPO, split="train", token=True)
n = len(ds)