Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
first push
HenryNdubuaku8132d7682b23
diff --git a/src/cli.py b/src/cli.pyindex 7419065..dd4aa88 100644--- a/src/cli.py+++ b/src/cli.py@@ -4,10 +4,10 @@ import sysHELP = """┌───────────────────────────────────────────────────────────────────┐│ │- │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │- │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │- │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │- │ ...the tiny model to rule them all... │+ │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │+ │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │+ │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │+ │ ...the tiny model to rule them all... ││ ││ train ││ --full Use full 1B config (~1.17B params) │@@ -20,17 +20,17 @@ HELP = """│ --num-kv-heads INT KV heads for GQA (default: num-heads)││ --num-layers INT Encoder layers (default: 8) ││ --num-dec-layers INT Decoder layers (default: 4) │- │ --max-enc-len INT Max encoder seq length (default: 256)│- │ --max-dec-len INT Max decoder seq length (default: 256)│+ │ --max-enc-len INT Max encoder seq len (default: 1024) │+ │ --max-dec-len INT Max decoder seq length (default: 512)││ --max-samples INT Training samples (default: all) │- │ --mat-factors INT [...] FFN shrink factors (default: 2 4 8) │- │ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) │+ │ --mat-factors INT [...] FFN shrink factors (default: 2 4 8) │+ │ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) ││ --group-size INT Quant/prune group size (default: 32) ││ --prune-interval INT Steps between mask updates (def: 100)│- │ --prune-start-frac FL Start pruning at frac (def: 0.33) │- │ --prune-end-frac FL Lock mask at this frac (def: 0.67) │+ │ --prune-start-frac FL Start pruning at frac (def: 0.33) │+ │ --prune-end-frac FL Lock mask at this frac (def: 0.67) ││ --activation STR drelu|swiglu|geglu (default: drelu) │- │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │+ │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) ││ --eval-every INT Val eval interval (default: 1000) ││ --wandb Enable W&B logging ││ --checkpoint PATH Resume from checkpoint │@@ -44,9 +44,11 @@ HELP = """│ ││ run ││ --checkpoint PATH Path to model checkpoint (required) │- │ --prompts STR [...] One or more prompts to continue │+ │ --query STR Query text for tool-call generation │+ │ --tools STR Tools JSON for tool-call generation │+ │ --prompts STR [...] Raw prompts (query+tools combined) ││ --audio PATH [...] Audio files to transcribe │- │ --max-len INT Max tokens to generate (default: 128) │+ │ --max-len INT Max tokens to generate (default: 512) ││ --temperature FLOAT Sampling temperature (default: 0.8) ││ --seed INT Random seed (default: 0) ││ │@@ -54,9 +56,12 @@ HELP = """│ --checkpoint PATH Path to model checkpoint (required) ││ --batch-size INT Batch size (default: 32) ││ --max-eval-samples INT Evaluation samples (default: 1000) │- │ --max-gen-len INT Max generation length (default: 128) │+ │ --max-enc-len INT Max encoder length (default: 1024) │+ │ --max-dec-len INT Max decoder length (default: 512) │+ │ --max-gen-len INT Max generation length (default: 512) ││ --temperature FLOAT Sampling temperature (default: 0.8) ││ --throughput-runs INT Throughput runs (default: 10) │+ │ --tool-call-samples INT Tool-call eval samples (default: 200) ││ ││ evaluate ││ --checkpoint PATH Path to model checkpoint (required) │@@ -98,8 +103,8 @@ def main():p.add_argument("--num-kv-heads", type=int, default=None)p.add_argument("--num-layers", type=int, default=8)p.add_argument("--num-dec-layers", type=int, default=4)- p.add_argument("--max-enc-len", type=int, default=256)- p.add_argument("--max-dec-len", type=int, default=256)+ p.add_argument("--max-enc-len", type=int, default=1024)+ p.add_argument("--max-dec-len", type=int, default=512)p.add_argument("--max-samples", type=int, default=None)p.add_argument("--warmup-ratio", type=float, default=0.05)p.add_argument("--wandb", action="store_true")@@ -134,9 +139,11 @@ def main():p = sub.add_parser("run", add_help=False)p.add_argument("--checkpoint", type=str, required=True)+ p.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")+ p.add_argument("--tools", type=str, default=None, help="Tools JSON for tool-call generation")p.add_argument("--prompts", type=str, nargs="*")p.add_argument("--audio", type=str, nargs="*", help="Audio file paths to transcribe")- p.add_argument("--max-len", type=int, default=128)+ p.add_argument("--max-len", type=int, default=512)p.add_argument("--temperature", type=float, default=0.8)p.add_argument("--seed", type=int, default=0)@@ -144,9 +151,11 @@ def main():p.add_argument("--checkpoint", type=str, required=True)p.add_argument("--batch-size", type=int, default=32)p.add_argument("--max-eval-samples", type=int, default=1000)- p.add_argument("--max-enc-len", type=int, default=128)- p.add_argument("--max-dec-len", type=int, default=128)- p.add_argument("--max-gen-len", type=int, default=128)+ p.add_argument("--max-enc-len", type=int, default=1024)+ p.add_argument("--max-dec-len", type=int, default=512)+ p.add_argument("--max-gen-len", type=int, default=512)+ p.add_argument("--tool-call-samples", type=int, default=200,+ help="Samples for tool-call accuracy eval (default: 200)")p.add_argument("--temperature", type=float, default=0.8)p.add_argument("--throughput-runs", type=int, default=10)diff --git a/src/data.py b/src/data.pyindex b89a420..fc9cf5a 100644--- a/src/data.py+++ b/src/data.py@@ -10,19 +10,20 @@ logging.getLogger("transformers").setLevel(logging.ERROR)logging.getLogger("huggingface_hub").setLevel(logging.ERROR)import numpy as np-from datasets import Audio, load_dataset+from datasets import Audio, load_dataset, load_from_diskfrom tqdm import tqdmimport sentencepiece as spmCACHE_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), ".data_cache")TOKENIZER_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tokenizer")TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")+TOOL_CALLS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "tool_calls")PAD_ID = 0EOS_ID = 1BOS_ID = 2UNK_ID = 3-STORY_ID = 4+TOOL_CALL_ID = 4TRANSCRIBE_ID = 5@@ -46,8 +47,8 @@ class NeedleTokenizer:return BOS_ID@property- def story_token_id(self):- return STORY_ID+ def tool_call_token_id(self):+ return TOOL_CALL_ID@propertydef transcribe_token_id(self):@@ -93,7 +94,7 @@ def _tokenize_chunk(texts):def train_tokenizer(vocab_size=8192, max_samples=None):- """Train a SentencePiece BPE tokenizer on TinyStories."""+ """Train a SentencePiece BPE tokenizer on tool-calling corpus."""model_path = TOKENIZER_PREFIX + ".model"if os.path.exists(model_path):print(f"Tokenizer already exists at {model_path}")@@ -101,7 +102,7 @@ def train_tokenizer(vocab_size=8192, max_samples=None):os.makedirs(TOKENIZER_DIR, exist_ok=True)- ds = load_dataset("roneneldan/TinyStories", split="train")+ ds = load_from_disk(TOOL_CALLS_DIR)if max_samples:ds = ds.select(range(min(max_samples, len(ds))))@@ -110,9 +111,10 @@ def train_tokenizer(vocab_size=8192, max_samples=None):corpus_path = os.path.join(TOKENIZER_DIR, "corpus.txt")with open(corpus_path, "w") as f:for example in tqdm(ds, desc="Writing corpus"):- text = example["text"].strip()- if text:- f.write(text + "\n")+ for field in ("query", "tools", "answers"):+ text = example[field].strip()+ if text:+ f.write(text + "\n")spm.SentencePieceTrainer.Train(input=corpus_path,@@ -123,7 +125,7 @@ def train_tokenizer(vocab_size=8192, max_samples=None):eos_id=EOS_ID,bos_id=BOS_ID,unk_id=UNK_ID,- user_defined_symbols=["<story>", "<transcribe>"],+ user_defined_symbols=["<tool_call>", "<transcribe>"],byte_fallback=True,normalization_rule_name="identity",num_threads=os.cpu_count(),@@ -143,8 +145,14 @@ def get_tokenizer(max_samples=None):return NeedleTokenizer(model_path)-def load_tinystories(split="train", max_samples=None):- ds = load_dataset("roneneldan/TinyStories", split=split)+def load_tool_calls(split="train", max_samples=None):+ """Load tool-calling dataset, splitting 90/10 for train/val."""+ ds = load_from_disk(TOOL_CALLS_DIR)+ n = len(ds)+ if split in ("validation", "val", "test"):+ ds = ds.select(range(int(n * 0.9), n))+ elif split == "train":+ ds = ds.select(range(int(n * 0.9)))if max_samples:ds = ds.select(range(min(max_samples, len(ds))))return ds@@ -165,13 +173,18 @@ def _cache_key(prefix, n_samples, max_enc_len, max_dec_len):return hashlib.md5(key.encode()).hexdigest()[:12]-def prepare_text_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):- """Prepare text encoder-decoder pairs with <story> task token."""+def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=1024, max_dec_len=512):+ """Prepare tool-call encoder-decoder pairs with <tool_call> task token.++ Encoder input: tokenize(query + " " + tools), truncated to max_enc_len.+ Decoder input: [BOS, <tool_call>, answer_tokens...]+ Decoder target: [<tool_call>, answer_tokens..., EOS]+ """- cache_id = _cache_key("text", len(ds), max_enc_len, max_dec_len)+ cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len)cache_path = os.path.join(CACHE_DIR, cache_id)if os.path.exists(cache_path + "_enc.npy"):- print(f"Loading cached text data ({cache_id})...")+ print(f"Loading cached tool-call data ({cache_id})...")return (np.load(cache_path + "_enc.npy"),np.load(cache_path + "_dec_in.npy"),@@ -180,71 +193,72 @@ def prepare_text_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_rapad_id = tokenizer.pad_token_idbos_id = tokenizer.eos_token_id- story_id = tokenizer.story_token_id- max_total = max_enc_len + max_dec_len+ eos_id = tokenizer.eos_token_id+ tool_call_id = tokenizer.tool_call_token_id++ # Build encoder texts (query + tools) and decoder texts (answers)+ enc_texts = [ex["query"] + " " + ex["tools"] for ex in ds]+ dec_texts = [ex["answers"] for ex in ds]- # --- Parallel tokenization ---- texts = list(ds["text"])num_workers = min(os.cpu_count() or 1, 8)model_path = TOKENIZER_PREFIX + ".model"- chunk_size = max(1, len(texts) // (num_workers * 4))- chunks = [texts[i:i + chunk_size] for i in range(0, len(texts), chunk_size)]- print(f"Tokenizing ({num_workers} workers)...")+ # Parallel tokenization for encoder inputs+ chunk_size = max(1, len(enc_texts) // (num_workers * 4))+ enc_chunks = [enc_texts[i:i + chunk_size] for i in range(0, len(enc_texts), chunk_size)]++ print(f"Tokenizing encoder inputs ({num_workers} workers)...")with mp.Pool(num_workers, initializer=_init_worker,- initargs=(model_path, max_total)) as pool:- results = pool.map(_tokenize_chunk, chunks)- all_tokens = [tok for chunk in results for tok in chunk]-- # --- Pre-filter by length to reduce loop iterations ---- lengths = np.array([len(t) for t in all_tokens])- valid_mask = lengths >= 6- valid_indices = np.where(valid_mask)[0]-- split_points = np.maximum(2, (lengths[valid_indices] * split_ratio).astype(np.int32))- enc_lens = np.minimum(split_points, max_enc_len)- # Account for <story> token taking one slot in decoder- dec_lens = np.minimum(lengths[valid_indices] - split_points, max_dec_len - 1)- keep = dec_lens >= 2-- valid_indices = valid_indices[keep]- split_points = split_points[keep]- enc_lens = enc_lens[keep]- dec_lens = dec_lens[keep]-- n_keep = len(valid_indices)- enc_inputs = np.full((n_keep, max_enc_len), pad_id, dtype=np.int32)- dec_inputs = np.full((n_keep, max_dec_len), pad_id, dtype=np.int32)- dec_targets = np.full((n_keep, max_dec_len), pad_id, dtype=np.int32)-- for i, (idx, sp, el, dl) in enumerate(- zip(valid_indices, split_points, enc_lens, dec_lens)- ):- tokens = all_tokens[idx]- enc_inputs[i, :el] = tokens[:sp][:max_enc_len]- # Decoder input: [BOS, <story>, tok1, tok2, ...]+ initargs=(model_path, max_enc_len)) as pool:+ enc_results = pool.map(_tokenize_chunk, enc_chunks)+ all_enc_tokens = [tok for chunk in enc_results for tok in chunk]++ # Parallel tokenization for decoder inputs (max_dec_len - 2 for BOS + task token)+ dec_chunks = [dec_texts[i:i + chunk_size] for i in range(0, len(dec_texts), chunk_size)]++ print(f"Tokenizing decoder inputs ({num_workers} workers)...")+ with mp.Pool(num_workers, initializer=_init_worker,+ initargs=(model_path, max_dec_len - 2)) as pool:+ dec_results = pool.map(_tokenize_chunk, dec_chunks)+ all_dec_tokens = [tok for chunk in dec_results for tok in chunk]++ n = len(ds)+ enc_inputs = np.full((n, max_enc_len), pad_id, dtype=np.int32)+ dec_inputs = np.full((n, max_dec_len), pad_id, dtype=np.int32)+ dec_targets = np.full((n, max_dec_len), pad_id, dtype=np.int32)++ for i in range(n):+ # Encoder: truncated tokens+ e_tok = all_enc_tokens[i]+ el = len(e_tok)+ enc_inputs[i, :el] = e_tok++ # Decoder answer tokens+ d_tok = all_dec_tokens[i]+ dl = len(d_tok)++ # Decoder input: [BOS, <tool_call>, answer_tokens...]dec_inputs[i, 0] = bos_id- dec_inputs[i, 1] = story_id- dec_inputs[i, 2:dl + 1] = tokens[sp:][:dl - 1]- # Decoder target: [<story>, tok1, tok2, ..., EOS]- dec_targets[i, 0] = story_id- dec_targets[i, 1:dl + 1] = tokens[sp:][:dl]+ dec_inputs[i, 1] = tool_call_id+ if dl > 0:+ dec_inputs[i, 2:2 + dl] = d_tok++ # Decoder target: [<tool_call>, answer_tokens..., EOS]+ dec_targets[i, 0] = tool_call_id+ if dl > 0:+ dec_targets[i, 1:1 + dl] = d_tok+ dec_targets[i, 1 + dl] = eos_id# Save cacheos.makedirs(CACHE_DIR, exist_ok=True)np.save(cache_path + "_enc.npy", enc_inputs)np.save(cache_path + "_dec_in.npy", dec_inputs)np.save(cache_path + "_dec_tgt.npy", dec_targets)- print(f"Cached {n_keep:,} text pairs to {CACHE_DIR}/{cache_id}")+ print(f"Cached {n:,} tool-call pairs to {CACHE_DIR}/{cache_id}")return enc_inputs, dec_inputs, dec_targets-# Backward compatibility alias-def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):- return prepare_text_pairs(ds, tokenizer, max_enc_len, max_dec_len, split_ratio)--def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True):n = len(enc_inputs)if shuffle:diff --git a/src/run.py b/src/run.pyindex 3d84a91..45842ed 100644--- a/src/run.py+++ b/src/run.py@@ -24,7 +24,7 @@ def load_checkpoint(path):return params, config-def generate(model, params, tokenizer, prompt, max_gen_len=128, temperature=0.8, seed=0, stream=True):+def generate(model, params, tokenizer, prompt, max_gen_len=512, temperature=0.8, seed=0, stream=True, task_token_id=None):enc_tokens = tokenizer.encode(prompt)enc_input = jnp.array([enc_tokens])@@ -41,6 +41,12 @@ def generate(model, params, tokenizer, prompt, max_gen_len=128, temperature=0.8,dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)dec_buffer = dec_buffer.at[0, 0].set(eos_id)+ # If task_token_id provided, set it at position 1 (matching transcribe() pattern)+ start_pos = 0+ if task_token_id is not None:+ dec_buffer = dec_buffer.at[0, 1].set(task_token_id)+ start_pos = 1+@jax.jitdef decode_step(dec_buffer, encoder_out, src_mask):logits = model.apply(@@ -57,13 +63,13 @@ def generate(model, params, tokenizer, prompt, max_gen_len=128, temperature=0.8,generated_tokens = []if stream:- sys.stdout.write(f"\n{prompt}")+ sys.stdout.write(f"\n")sys.stdout.flush()# Compile once on first calllogits = decode_step(dec_buffer, encoder_out, src_mask)- for i in range(max_gen_len - 1):+ for i in range(start_pos, max_gen_len - 1):next_logits = logits[0, i] / temperaturerng, sample_rng = jax.random.split(rng)next_token = jax.random.categorical(sample_rng, next_logits).item()@@ -196,8 +202,20 @@ def main(args):)return- # --- Text generation mode ---- prompts = args.prompts or ["Once upon a time", "The little dog", "She was very happy because"]+ # --- Tool-call generation mode ---+ query = getattr(args, "query", None)+ tools = getattr(args, "tools", None)+ prompts = args.prompts++ if query:+ prompt = query + " " + (tools or "[]")+ prompts = [prompt]+ elif not prompts:+ prompts = [+ 'What is the weather in San Francisco? [{"name": "get_weather", "parameters": {"location": "string"}}]',+ 'Send an email to [email protected] saying hello [{"name": "send_email", "parameters": {"to": "string", "body": "string"}}]',+ 'Get the current stock price of AAPL [{"name": "get_stock_price", "parameters": {"symbol": "string"}}]',+ ]for i, prompt in enumerate(prompts):generate(@@ -209,14 +227,18 @@ def main(args):temperature=args.temperature,seed=args.seed + i,stream=True,+ task_token_id=tokenizer.tool_call_token_id,)def parse_args():- parser = argparse.ArgumentParser(description="Generate stories with trained transformer")+ parser = argparse.ArgumentParser(description="Generate tool calls with trained transformer")parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")- parser.add_argument("--prompts", type=str, nargs="*", help="Prompts to continue")- parser.add_argument("--max-len", type=int, default=128)+ parser.add_argument("--prompts", type=str, nargs="*", help="Raw prompts (query + tools combined)")+ parser.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")+ parser.add_argument("--tools", type=str, default=None, help="Tools JSON for tool-call generation")+ parser.add_argument("--audio", type=str, nargs="*", help="Audio file paths to transcribe")+ parser.add_argument("--max-len", type=int, default=512)parser.add_argument("--temperature", type=float, default=0.8)parser.add_argument("--seed", type=int, default=0)return parser.parse_args()diff --git a/src/test.py b/src/test.pyindex 93e45e5..01884ef 100644--- a/src/test.py+++ b/src/test.py@@ -8,7 +8,7 @@ import jax.numpy as jnpimport numpy as npimport optax-from .data import get_batches, get_tokenizer, load_tinystories, prepare_text_pairs, load_librispeech+from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs, load_librispeechfrom .model import (EncoderDecoderTransformer,TransformerConfig,@@ -50,7 +50,7 @@ def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batchreturn math.exp(avg_nll)-def measure_throughput(model, params, tokenizer, num_runs=10, prompt="Once upon a time", max_gen_len=64):+def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is the weather? [{"name": "get_weather"}]', max_gen_len=64):enc_tokens = tokenizer.encode(prompt)enc_input = jnp.array([enc_tokens])pad_id = tokenizer.pad_token_id@@ -180,6 +180,117 @@ def compute_wer(hypotheses, references):return total_edits / max(total_ref_words, 1)+def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=512):+ """Generate tool-call predictions and compute structured metrics."""+ import json+ from .run import generate+ from .data import load_tool_calls++ ds = load_tool_calls("validation", max_samples=num_samples)++ total = 0+ exact_match = 0+ # For function-name-level P/R/F1+ tp_names = 0 # predicted name in reference+ fp_names = 0 # predicted name not in reference+ fn_names = 0 # reference name not predicted+ # For full call-level (name + args) P/R/F1+ tp_calls = 0+ fp_calls = 0+ fn_calls = 0+ json_parse_errors = 0+ empty_ref = 0+ empty_pred = 0++ samples = []++ for i, ex in enumerate(ds):+ prompt = ex["query"] + " " + ex["tools"]+ ref_text = ex["answers"]++ pred_text = generate(+ model, params, tokenizer, prompt,+ max_gen_len=max_gen_len, temperature=0.0, seed=i, stream=False,+ task_token_id=tokenizer.tool_call_token_id,+ ).strip()++ # Parse reference+ try:+ ref_calls = json.loads(ref_text)+ except (json.JSONDecodeError, TypeError):+ ref_calls = []++ # Parse prediction+ try:+ pred_calls = json.loads(pred_text)+ if not isinstance(pred_calls, list):+ pred_calls = [pred_calls] if isinstance(pred_calls, dict) else []+ except (json.JSONDecodeError, TypeError):+ json_parse_errors += 1+ pred_calls = []++ total += 1+ if i < 10:+ samples.append((ex["query"][:80], ref_text[:120], pred_text[:120]))++ # Track empty+ if len(ref_calls) == 0:+ empty_ref += 1+ if len(pred_calls) == 0:+ empty_pred += 1++ # Exact match (normalized)+ if json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):+ exact_match += 1++ # Function name P/R/F1+ ref_names = [c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c]+ pred_names = [c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c]+ ref_name_set = set(ref_names)+ pred_name_set = set(pred_names)+ tp_names += len(pred_name_set & ref_name_set)+ fp_names += len(pred_name_set - ref_name_set)+ fn_names += len(ref_name_set - pred_name_set)++ # Full call P/R/F1 (name + args must match)+ def call_key(c):+ if not isinstance(c, dict):+ return None+ return json.dumps({"name": c.get("name"), "arguments": c.get("arguments")}, sort_keys=True)++ ref_keys = set(call_key(c) for c in ref_calls)+ pred_keys = set(call_key(c) for c in pred_calls)+ ref_keys.discard(None)+ pred_keys.discard(None)+ tp_calls += len(pred_keys & ref_keys)+ fp_calls += len(pred_keys - ref_keys)+ fn_calls += len(ref_keys - pred_keys)++ # Compute metrics+ name_precision = tp_names / max(tp_names + fp_names, 1)+ name_recall = tp_names / max(tp_names + fn_names, 1)+ name_f1 = 2 * name_precision * name_recall / max(name_precision + name_recall, 1e-9)++ call_precision = tp_calls / max(tp_calls + fp_calls, 1)+ call_recall = tp_calls / max(tp_calls + fn_calls, 1)+ call_f1 = 2 * call_precision * call_recall / max(call_precision + call_recall, 1e-9)++ return {+ "num_samples": total,+ "exact_match": exact_match / max(total, 1),+ "json_parse_rate": 1.0 - json_parse_errors / max(total, 1),+ "name_precision": name_precision,+ "name_recall": name_recall,+ "name_f1": name_f1,+ "call_precision": call_precision,+ "call_recall": call_recall,+ "call_f1": call_f1,+ "empty_ref_pct": empty_ref / max(total, 1),+ "empty_pred_pct": empty_pred / max(total, 1),+ "samples": samples,+ }++def benchmark_asr(model, params, tokenizer, num_samples=100):"""Transcribe LibriSpeech test-clean samples and report WER."""from .run import transcribe@@ -218,13 +329,43 @@ def main(args):print(f"parameters: {param_count:,}")print(f"config: d={config.d_model}, heads={config.num_heads}, layers={config.num_encoder_layers}/{config.num_decoder_layers}")- print(f"\nevaluating text perplexity ({args.max_eval_samples} samples)...")- ds = load_tinystories("validation", max_samples=args.max_eval_samples)- enc_inputs, dec_inputs, dec_targets = prepare_text_pairs(+ print(f"\nevaluating tool-call perplexity ({args.max_eval_samples} samples)...")+ ds = load_tool_calls("validation", max_samples=args.max_eval_samples)+ enc_inputs, dec_inputs, dec_targets = prepare_tool_call_pairs(ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len)ppl = compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, args.batch_size, config.pad_token_id)- print(f"text ppl: {ppl:.2f}")++ tc_samples = getattr(args, "tool_call_samples", 200)+ tc = None+ if tc_samples > 0:+ print(f"\nevaluating tool-call accuracy ({tc_samples} samples)...")+ tc = benchmark_tool_calls(model, params, tokenizer, num_samples=tc_samples, max_gen_len=args.max_gen_len)++ print(f"\n ─────────────────────────────────────")+ print(f" Tool-Call Metrics")+ print(f" ─────────────────────────────────────")+ print(f" Perplexity {ppl:>10.2f} ({args.max_eval_samples} samples)")+ if tc:+ print(f" JSON parse rate {tc['json_parse_rate']:>10.1%}")+ print(f" Exact match {tc['exact_match']:>10.1%}")+ print(f" ─── Function Name ───────────────────")+ print(f" Precision {tc['name_precision']:>10.3f}")+ print(f" Recall {tc['name_recall']:>10.3f}")+ print(f" F1 {tc['name_f1']:>10.3f}")+ print(f" ─── Full Call (name+args) ───────────")+ print(f" Precision {tc['call_precision']:>10.3f}")+ print(f" Recall {tc['call_recall']:>10.3f}")+ print(f" F1 {tc['call_f1']:>10.3f}")+ print(f" ─────────────────────────────────────")+ print(f" Empty ref/pred {tc['empty_ref_pct']:.1%} / {tc['empty_pred_pct']:.1%}")+ if tc["samples"]:+ print(f"\n samples:")+ for query, ref, pred in tc["samples"][:5]:+ print(f" Q: {query}")+ print(f" R: {ref}")+ print(f" P: {pred}")+ print()asr_samples = getattr(args, "asr_samples", 50)if asr_samples > 0:@@ -238,37 +379,19 @@ def main(args):print(f"avg latency: {throughput['avg_latency_s']:.3f}s")print(f"tokens/sec: {throughput['tokens_per_second']:.1f}")- prompts = [- "Once upon a time",- "The little dog",- "She was very happy because",- "One day, a boy named",- "The cat and the mouse",- ]- print(f"\ngenerating samples (t={args.temperature})...")- quality = benchmark_generation_quality(- model, params, tokenizer, prompts, max_gen_len=args.max_gen_len, temperature=args.temperature- )- print(f"avg length: {quality['avg_generation_length']:.1f} tokens")- print(f"repetition: {quality['bigram_repetition_rate']:.3f}")-- print("\nsamples:")- for prompt, gen in quality["generations"]:- print(f" [{prompt}]")- print(f" {gen[:120]}")- print()-def parse_args():parser = argparse.ArgumentParser(description="Benchmark transformer on generation tasks")parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")parser.add_argument("--batch-size", type=int, default=32)parser.add_argument("--max-eval-samples", type=int, default=1000)- parser.add_argument("--max-enc-len", type=int, default=128)- parser.add_argument("--max-dec-len", type=int, default=128)- parser.add_argument("--max-gen-len", type=int, default=128)+ parser.add_argument("--max-enc-len", type=int, default=1024)+ parser.add_argument("--max-dec-len", type=int, default=512)+ parser.add_argument("--max-gen-len", type=int, default=512)parser.add_argument("--temperature", type=float, default=0.8)parser.add_argument("--throughput-runs", type=int, default=10)+ parser.add_argument("--tool-call-samples", type=int, default=200,+ help="Samples for tool-call accuracy eval (default: 200)")return parser.parse_args()diff --git a/src/train.py b/src/train.pyindex f4143eb..91ce210 100644--- a/src/train.py+++ b/src/train.py@@ -14,7 +14,7 @@ from flax import jax_utilsfrom flax.training import train_statefrom .data import (- get_batches, get_tokenizer, load_tinystories, prepare_text_pairs,+ get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs,load_librispeech, prepare_speech_pairs, get_speech_batches,)from .model import (@@ -474,22 +474,22 @@ def train(args):tokenizer = get_tokenizer(max_samples=args.max_samples)step_idx += 1- print(f"\n[{step_idx}/{total_data_steps}] Loading TinyStories dataset...")- ds = load_tinystories("train", max_samples=args.max_samples)- val_ds = load_tinystories("validation", max_samples=getattr(args, "max_eval_samples", None))+ print(f"\n[{step_idx}/{total_data_steps}] Loading tool-calling dataset...")+ ds = load_tool_calls("train", max_samples=args.max_samples)+ val_ds = load_tool_calls("validation", max_samples=getattr(args, "max_eval_samples", None))step_idx += 1- print(f"\n[{step_idx}/{total_data_steps}] Preparing text pairs...")- enc_inputs, dec_inputs, dec_targets = prepare_text_pairs(+ print(f"\n[{step_idx}/{total_data_steps}] Preparing tool-call pairs...")+ enc_inputs, dec_inputs, dec_targets = prepare_tool_call_pairs(ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len)- val_enc, val_dec_in, val_dec_tgt = prepare_text_pairs(+ val_enc, val_dec_in, val_dec_tgt = prepare_tool_call_pairs(val_ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len)val_enc = np.array(val_enc)val_dec_in = np.array(val_dec_in)val_dec_tgt = np.array(val_dec_tgt)- print(f" {len(enc_inputs):,} train / {len(val_enc):,} val text pairs")+ print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs")# --- Speech data ---speech_mel = speech_dec_in = speech_dec_tgt = None@@ -891,15 +891,14 @@ def train(args):with open(ckpt_path, "wb") as f:pickle.dump({"params": params_np, "config": config.__dict__}, f)- from .test import measure_throughput, benchmark_generation_quality, compute_wer- from .run import generate, transcribe+ from .test import measure_throughput, benchmark_tool_calls, compute_wer+ from .run import transcribefrom .data import _decode_audioeval_params_jnp = jax.tree.map(jnp.array, params_np)del params_np # free CPU memorymodel = EncoderDecoderTransformer(config)tp = measure_throughput(model, eval_params_jnp, tokenizer, num_runs=5)- prompts = ["Once upon a time", "The little dog", "She was very happy because"]- quality = benchmark_generation_quality(model, eval_params_jnp, tokenizer, prompts, max_gen_len=64, temperature=0.8)+ tc_metrics = benchmark_tool_calls(model, eval_params_jnp, tokenizer, num_samples=20, max_gen_len=128)speech_wer = Nonetranscribe_samples = []@@ -926,7 +925,7 @@ def train(args):print(f" Avg loss {epoch_avg_loss:>12.4f}")print(f" Final loss {final_loss:>12.4f}")print(f" Train ppl {final_ppl:>12.2f}")- print(f" Text val ppl {last_val_ppl:>12.2f}")+ print(f" TC val ppl {last_val_ppl:>12.2f}")if speech_wer is not None:print(f" Speech WER {speech_wer:>12.4f}")epoch_avg_speech = sum(speech_losses) / len(speech_losses) if speech_losses else float("nan")@@ -946,12 +945,18 @@ def train(args):print(f" ─────────────────────────────────────")print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")print(f" Latency {tp['avg_latency_s']:>11.3f}s")- print(f" Gen length {quality['avg_generation_length']:>10.1f} tok")- print(f" Repetition {quality['bigram_repetition_rate']:>12.3f}")- print(f" ─────────────────────────────────────")- for prompt, gen in quality["generations"]:- gen_clean = gen.replace("<story>", "").replace("<transcribe>", "").strip()- print(f" <story>[{prompt}] {gen_clean[:80]}")+ print(f" ─── Tool-Call Metrics ───────────────")+ print(f" JSON parse {tc_metrics['json_parse_rate']:>10.1%}")+ print(f" Exact match {tc_metrics['exact_match']:>10.1%}")+ print(f" Name F1 {tc_metrics['name_f1']:>10.3f} (P={tc_metrics['name_precision']:.3f} R={tc_metrics['name_recall']:.3f})")+ print(f" Call F1 {tc_metrics['call_f1']:>10.3f} (P={tc_metrics['call_precision']:.3f} R={tc_metrics['call_recall']:.3f})")+ if tc_metrics["samples"]:+ print(f" ─────────────────────────────────────")+ for query, ref, pred in tc_metrics["samples"][:3]:+ print(f" Q: {query}")+ print(f" R: {ref}")+ print(f" P: {pred}")+ print()if transcribe_samples:print(f" ─────────────────────────────────────")print(f" Transcribe (WER={speech_wer:.4f}):")@@ -971,6 +976,10 @@ def train(args):"epoch/weight_sparsity": sparsity,"epoch": epoch + 1,}+ log_dict["epoch/tc_exact_match"] = tc_metrics["exact_match"]+ log_dict["epoch/tc_name_f1"] = tc_metrics["name_f1"]+ log_dict["epoch/tc_call_f1"] = tc_metrics["call_f1"]+ log_dict["epoch/tc_json_parse_rate"] = tc_metrics["json_parse_rate"]if speech_wer is not None:log_dict["epoch/speech_wer"] = speech_werif speech_losses: