Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

first push

HenryNdubuaku8132d7682b23
diff --git a/src/cli.py b/src/cli.py
index 7419065..dd4aa88 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -4,10 +4,10 @@ import sys
HELP = """
┌───────────────────────────────────────────────────────────────────┐
│ │
- │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │
- │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │
- │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │
- │ ...the tiny model to rule them all... │
+ │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │
+ │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │
+ │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │
+ │ ...the tiny model to rule them all... │
│ │
│ train │
│ --full Use full 1B config (~1.17B params) │
@@ -20,17 +20,17 @@ HELP = """
│ --num-kv-heads INT KV heads for GQA (default: num-heads)│
│ --num-layers INT Encoder layers (default: 8) │
│ --num-dec-layers INT Decoder layers (default: 4) │
- │ --max-enc-len INT Max encoder seq length (default: 256)│
- │ --max-dec-len INT Max decoder seq length (default: 256)│
+ │ --max-enc-len INT Max encoder seq len (default: 1024) │
+ │ --max-dec-len INT Max decoder seq length (default: 512)│
│ --max-samples INT Training samples (default: all) │
- │ --mat-factors INT [...] FFN shrink factors (default: 2 4 8) │
- │ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) │
+ │ --mat-factors INT [...] FFN shrink factors (default: 2 4 8) │
+ │ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) │
│ --group-size INT Quant/prune group size (default: 32) │
│ --prune-interval INT Steps between mask updates (def: 100)│
- │ --prune-start-frac FL Start pruning at frac (def: 0.33) │
- │ --prune-end-frac FL Lock mask at this frac (def: 0.67) │
+ │ --prune-start-frac FL Start pruning at frac (def: 0.33) │
+ │ --prune-end-frac FL Lock mask at this frac (def: 0.67) │
│ --activation STR drelu|swiglu|geglu (default: drelu) │
- │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │
+ │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │
│ --eval-every INT Val eval interval (default: 1000) │
│ --wandb Enable W&B logging │
│ --checkpoint PATH Resume from checkpoint │
@@ -44,9 +44,11 @@ HELP = """
│ │
│ run │
│ --checkpoint PATH Path to model checkpoint (required) │
- │ --prompts STR [...] One or more prompts to continue │
+ │ --query STR Query text for tool-call generation │
+ │ --tools STR Tools JSON for tool-call generation │
+ │ --prompts STR [...] Raw prompts (query+tools combined) │
│ --audio PATH [...] Audio files to transcribe │
- │ --max-len INT Max tokens to generate (default: 128) │
+ │ --max-len INT Max tokens to generate (default: 512) │
│ --temperature FLOAT Sampling temperature (default: 0.8) │
│ --seed INT Random seed (default: 0) │
│ │
@@ -54,9 +56,12 @@ HELP = """
│ --checkpoint PATH Path to model checkpoint (required) │
│ --batch-size INT Batch size (default: 32) │
│ --max-eval-samples INT Evaluation samples (default: 1000) │
- │ --max-gen-len INT Max generation length (default: 128) │
+ │ --max-enc-len INT Max encoder length (default: 1024) │
+ │ --max-dec-len INT Max decoder length (default: 512) │
+ │ --max-gen-len INT Max generation length (default: 512) │
│ --temperature FLOAT Sampling temperature (default: 0.8) │
│ --throughput-runs INT Throughput runs (default: 10) │
+ │ --tool-call-samples INT Tool-call eval samples (default: 200) │
│ │
│ evaluate │
│ --checkpoint PATH Path to model checkpoint (required) │
@@ -98,8 +103,8 @@ def main():
p.add_argument("--num-kv-heads", type=int, default=None)
p.add_argument("--num-layers", type=int, default=8)
p.add_argument("--num-dec-layers", type=int, default=4)
- p.add_argument("--max-enc-len", type=int, default=256)
- p.add_argument("--max-dec-len", type=int, default=256)
+ p.add_argument("--max-enc-len", type=int, default=1024)
+ p.add_argument("--max-dec-len", type=int, default=512)
p.add_argument("--max-samples", type=int, default=None)
p.add_argument("--warmup-ratio", type=float, default=0.05)
p.add_argument("--wandb", action="store_true")
@@ -134,9 +139,11 @@ def main():
p = sub.add_parser("run", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
+ p.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")
+ p.add_argument("--tools", type=str, default=None, help="Tools JSON for tool-call generation")
p.add_argument("--prompts", type=str, nargs="*")
p.add_argument("--audio", type=str, nargs="*", help="Audio file paths to transcribe")
- p.add_argument("--max-len", type=int, default=128)
+ p.add_argument("--max-len", type=int, default=512)
p.add_argument("--temperature", type=float, default=0.8)
p.add_argument("--seed", type=int, default=0)
@@ -144,9 +151,11 @@ def main():
p.add_argument("--checkpoint", type=str, required=True)
p.add_argument("--batch-size", type=int, default=32)
p.add_argument("--max-eval-samples", type=int, default=1000)
- p.add_argument("--max-enc-len", type=int, default=128)
- p.add_argument("--max-dec-len", type=int, default=128)
- p.add_argument("--max-gen-len", type=int, default=128)
+ p.add_argument("--max-enc-len", type=int, default=1024)
+ p.add_argument("--max-dec-len", type=int, default=512)
+ p.add_argument("--max-gen-len", type=int, default=512)
+ p.add_argument("--tool-call-samples", type=int, default=200,
+ help="Samples for tool-call accuracy eval (default: 200)")
p.add_argument("--temperature", type=float, default=0.8)
p.add_argument("--throughput-runs", type=int, default=10)
diff --git a/src/data.py b/src/data.py
index b89a420..fc9cf5a 100644
--- a/src/data.py
+++ b/src/data.py
@@ -10,19 +10,20 @@ logging.getLogger("transformers").setLevel(logging.ERROR)
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
import numpy as np
-from datasets import Audio, load_dataset
+from datasets import Audio, load_dataset, load_from_disk
from tqdm import tqdm
import sentencepiece as spm
CACHE_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), ".data_cache")
TOKENIZER_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tokenizer")
TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")
+TOOL_CALLS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "tool_calls")
PAD_ID = 0
EOS_ID = 1
BOS_ID = 2
UNK_ID = 3
-STORY_ID = 4
+TOOL_CALL_ID = 4
TRANSCRIBE_ID = 5
@@ -46,8 +47,8 @@ class NeedleTokenizer:
return BOS_ID
@property
- def story_token_id(self):
- return STORY_ID
+ def tool_call_token_id(self):
+ return TOOL_CALL_ID
@property
def transcribe_token_id(self):
@@ -93,7 +94,7 @@ def _tokenize_chunk(texts):
def train_tokenizer(vocab_size=8192, max_samples=None):
- """Train a SentencePiece BPE tokenizer on TinyStories."""
+ """Train a SentencePiece BPE tokenizer on tool-calling corpus."""
model_path = TOKENIZER_PREFIX + ".model"
if os.path.exists(model_path):
print(f"Tokenizer already exists at {model_path}")
@@ -101,7 +102,7 @@ def train_tokenizer(vocab_size=8192, max_samples=None):
os.makedirs(TOKENIZER_DIR, exist_ok=True)
- ds = load_dataset("roneneldan/TinyStories", split="train")
+ ds = load_from_disk(TOOL_CALLS_DIR)
if max_samples:
ds = ds.select(range(min(max_samples, len(ds))))
@@ -110,9 +111,10 @@ def train_tokenizer(vocab_size=8192, max_samples=None):
corpus_path = os.path.join(TOKENIZER_DIR, "corpus.txt")
with open(corpus_path, "w") as f:
for example in tqdm(ds, desc="Writing corpus"):
- text = example["text"].strip()
- if text:
- f.write(text + "\n")
+ for field in ("query", "tools", "answers"):
+ text = example[field].strip()
+ if text:
+ f.write(text + "\n")
spm.SentencePieceTrainer.Train(
input=corpus_path,
@@ -123,7 +125,7 @@ def train_tokenizer(vocab_size=8192, max_samples=None):
eos_id=EOS_ID,
bos_id=BOS_ID,
unk_id=UNK_ID,
- user_defined_symbols=["<story>", "<transcribe>"],
+ user_defined_symbols=["<tool_call>", "<transcribe>"],
byte_fallback=True,
normalization_rule_name="identity",
num_threads=os.cpu_count(),
@@ -143,8 +145,14 @@ def get_tokenizer(max_samples=None):
return NeedleTokenizer(model_path)
-def load_tinystories(split="train", max_samples=None):
- ds = load_dataset("roneneldan/TinyStories", split=split)
+def load_tool_calls(split="train", max_samples=None):
+ """Load tool-calling dataset, splitting 90/10 for train/val."""
+ ds = load_from_disk(TOOL_CALLS_DIR)
+ n = len(ds)
+ if split in ("validation", "val", "test"):
+ ds = ds.select(range(int(n * 0.9), n))
+ elif split == "train":
+ ds = ds.select(range(int(n * 0.9)))
if max_samples:
ds = ds.select(range(min(max_samples, len(ds))))
return ds
@@ -165,13 +173,18 @@ def _cache_key(prefix, n_samples, max_enc_len, max_dec_len):
return hashlib.md5(key.encode()).hexdigest()[:12]
-def prepare_text_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):
- """Prepare text encoder-decoder pairs with <story> task token."""
+def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=1024, max_dec_len=512):
+ """Prepare tool-call encoder-decoder pairs with <tool_call> task token.
+
+ Encoder input: tokenize(query + " " + tools), truncated to max_enc_len.
+ Decoder input: [BOS, <tool_call>, answer_tokens...]
+ Decoder target: [<tool_call>, answer_tokens..., EOS]
+ """
- cache_id = _cache_key("text", len(ds), max_enc_len, max_dec_len)
+ cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len)
cache_path = os.path.join(CACHE_DIR, cache_id)
if os.path.exists(cache_path + "_enc.npy"):
- print(f"Loading cached text data ({cache_id})...")
+ print(f"Loading cached tool-call data ({cache_id})...")
return (
np.load(cache_path + "_enc.npy"),
np.load(cache_path + "_dec_in.npy"),
@@ -180,71 +193,72 @@ def prepare_text_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_ra
pad_id = tokenizer.pad_token_id
bos_id = tokenizer.eos_token_id
- story_id = tokenizer.story_token_id
- max_total = max_enc_len + max_dec_len
+ eos_id = tokenizer.eos_token_id
+ tool_call_id = tokenizer.tool_call_token_id
+
+ # Build encoder texts (query + tools) and decoder texts (answers)
+ enc_texts = [ex["query"] + " " + ex["tools"] for ex in ds]
+ dec_texts = [ex["answers"] for ex in ds]
- # --- Parallel tokenization ---
- texts = list(ds["text"])
num_workers = min(os.cpu_count() or 1, 8)
model_path = TOKENIZER_PREFIX + ".model"
- chunk_size = max(1, len(texts) // (num_workers * 4))
- chunks = [texts[i:i + chunk_size] for i in range(0, len(texts), chunk_size)]
- print(f"Tokenizing ({num_workers} workers)...")
+ # Parallel tokenization for encoder inputs
+ chunk_size = max(1, len(enc_texts) // (num_workers * 4))
+ enc_chunks = [enc_texts[i:i + chunk_size] for i in range(0, len(enc_texts), chunk_size)]
+
+ print(f"Tokenizing encoder inputs ({num_workers} workers)...")
with mp.Pool(num_workers, initializer=_init_worker,
- initargs=(model_path, max_total)) as pool:
- results = pool.map(_tokenize_chunk, chunks)
- all_tokens = [tok for chunk in results for tok in chunk]
-
- # --- Pre-filter by length to reduce loop iterations ---
- lengths = np.array([len(t) for t in all_tokens])
- valid_mask = lengths >= 6
- valid_indices = np.where(valid_mask)[0]
-
- split_points = np.maximum(2, (lengths[valid_indices] * split_ratio).astype(np.int32))
- enc_lens = np.minimum(split_points, max_enc_len)
- # Account for <story> token taking one slot in decoder
- dec_lens = np.minimum(lengths[valid_indices] - split_points, max_dec_len - 1)
- keep = dec_lens >= 2
-
- valid_indices = valid_indices[keep]
- split_points = split_points[keep]
- enc_lens = enc_lens[keep]
- dec_lens = dec_lens[keep]
-
- n_keep = len(valid_indices)
- enc_inputs = np.full((n_keep, max_enc_len), pad_id, dtype=np.int32)
- dec_inputs = np.full((n_keep, max_dec_len), pad_id, dtype=np.int32)
- dec_targets = np.full((n_keep, max_dec_len), pad_id, dtype=np.int32)
-
- for i, (idx, sp, el, dl) in enumerate(
- zip(valid_indices, split_points, enc_lens, dec_lens)
- ):
- tokens = all_tokens[idx]
- enc_inputs[i, :el] = tokens[:sp][:max_enc_len]
- # Decoder input: [BOS, <story>, tok1, tok2, ...]
+ initargs=(model_path, max_enc_len)) as pool:
+ enc_results = pool.map(_tokenize_chunk, enc_chunks)
+ all_enc_tokens = [tok for chunk in enc_results for tok in chunk]
+
+ # Parallel tokenization for decoder inputs (max_dec_len - 2 for BOS + task token)
+ dec_chunks = [dec_texts[i:i + chunk_size] for i in range(0, len(dec_texts), chunk_size)]
+
+ print(f"Tokenizing decoder inputs ({num_workers} workers)...")
+ with mp.Pool(num_workers, initializer=_init_worker,
+ initargs=(model_path, max_dec_len - 2)) as pool:
+ dec_results = pool.map(_tokenize_chunk, dec_chunks)
+ all_dec_tokens = [tok for chunk in dec_results for tok in chunk]
+
+ n = len(ds)
+ enc_inputs = np.full((n, max_enc_len), pad_id, dtype=np.int32)
+ dec_inputs = np.full((n, max_dec_len), pad_id, dtype=np.int32)
+ dec_targets = np.full((n, max_dec_len), pad_id, dtype=np.int32)
+
+ for i in range(n):
+ # Encoder: truncated tokens
+ e_tok = all_enc_tokens[i]
+ el = len(e_tok)
+ enc_inputs[i, :el] = e_tok
+
+ # Decoder answer tokens
+ d_tok = all_dec_tokens[i]
+ dl = len(d_tok)
+
+ # Decoder input: [BOS, <tool_call>, answer_tokens...]
dec_inputs[i, 0] = bos_id
- dec_inputs[i, 1] = story_id
- dec_inputs[i, 2:dl + 1] = tokens[sp:][:dl - 1]
- # Decoder target: [<story>, tok1, tok2, ..., EOS]
- dec_targets[i, 0] = story_id
- dec_targets[i, 1:dl + 1] = tokens[sp:][:dl]
+ dec_inputs[i, 1] = tool_call_id
+ if dl > 0:
+ dec_inputs[i, 2:2 + dl] = d_tok
+
+ # Decoder target: [<tool_call>, answer_tokens..., EOS]
+ dec_targets[i, 0] = tool_call_id
+ if dl > 0:
+ dec_targets[i, 1:1 + dl] = d_tok
+ dec_targets[i, 1 + dl] = eos_id
# Save cache
os.makedirs(CACHE_DIR, exist_ok=True)
np.save(cache_path + "_enc.npy", enc_inputs)
np.save(cache_path + "_dec_in.npy", dec_inputs)
np.save(cache_path + "_dec_tgt.npy", dec_targets)
- print(f"Cached {n_keep:,} text pairs to {CACHE_DIR}/{cache_id}")
+ print(f"Cached {n:,} tool-call pairs to {CACHE_DIR}/{cache_id}")
return enc_inputs, dec_inputs, dec_targets
-# Backward compatibility alias
-def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):
- return prepare_text_pairs(ds, tokenizer, max_enc_len, max_dec_len, split_ratio)
-
-
def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True):
n = len(enc_inputs)
if shuffle:
diff --git a/src/run.py b/src/run.py
index 3d84a91..45842ed 100644
--- a/src/run.py
+++ b/src/run.py
@@ -24,7 +24,7 @@ def load_checkpoint(path):
return params, config
-def generate(model, params, tokenizer, prompt, max_gen_len=128, temperature=0.8, seed=0, stream=True):
+def generate(model, params, tokenizer, prompt, max_gen_len=512, temperature=0.8, seed=0, stream=True, task_token_id=None):
enc_tokens = tokenizer.encode(prompt)
enc_input = jnp.array([enc_tokens])
@@ -41,6 +41,12 @@ def generate(model, params, tokenizer, prompt, max_gen_len=128, temperature=0.8,
dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
dec_buffer = dec_buffer.at[0, 0].set(eos_id)
+ # If task_token_id provided, set it at position 1 (matching transcribe() pattern)
+ start_pos = 0
+ if task_token_id is not None:
+ dec_buffer = dec_buffer.at[0, 1].set(task_token_id)
+ start_pos = 1
+
@jax.jit
def decode_step(dec_buffer, encoder_out, src_mask):
logits = model.apply(
@@ -57,13 +63,13 @@ def generate(model, params, tokenizer, prompt, max_gen_len=128, temperature=0.8,
generated_tokens = []
if stream:
- sys.stdout.write(f"\n{prompt}")
+ sys.stdout.write(f"\n")
sys.stdout.flush()
# Compile once on first call
logits = decode_step(dec_buffer, encoder_out, src_mask)
- for i in range(max_gen_len - 1):
+ for i in range(start_pos, max_gen_len - 1):
next_logits = logits[0, i] / temperature
rng, sample_rng = jax.random.split(rng)
next_token = jax.random.categorical(sample_rng, next_logits).item()
@@ -196,8 +202,20 @@ def main(args):
)
return
- # --- Text generation mode ---
- prompts = args.prompts or ["Once upon a time", "The little dog", "She was very happy because"]
+ # --- Tool-call generation mode ---
+ query = getattr(args, "query", None)
+ tools = getattr(args, "tools", None)
+ prompts = args.prompts
+
+ if query:
+ prompt = query + " " + (tools or "[]")
+ prompts = [prompt]
+ elif not prompts:
+ prompts = [
+ 'What is the weather in San Francisco? [{"name": "get_weather", "parameters": {"location": "string"}}]',
+ 'Send an email to [email protected] saying hello [{"name": "send_email", "parameters": {"to": "string", "body": "string"}}]',
+ 'Get the current stock price of AAPL [{"name": "get_stock_price", "parameters": {"symbol": "string"}}]',
+ ]
for i, prompt in enumerate(prompts):
generate(
@@ -209,14 +227,18 @@ def main(args):
temperature=args.temperature,
seed=args.seed + i,
stream=True,
+ task_token_id=tokenizer.tool_call_token_id,
)
def parse_args():
- parser = argparse.ArgumentParser(description="Generate stories with trained transformer")
+ parser = argparse.ArgumentParser(description="Generate tool calls with trained transformer")
parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")
- parser.add_argument("--prompts", type=str, nargs="*", help="Prompts to continue")
- parser.add_argument("--max-len", type=int, default=128)
+ parser.add_argument("--prompts", type=str, nargs="*", help="Raw prompts (query + tools combined)")
+ parser.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")
+ parser.add_argument("--tools", type=str, default=None, help="Tools JSON for tool-call generation")
+ parser.add_argument("--audio", type=str, nargs="*", help="Audio file paths to transcribe")
+ parser.add_argument("--max-len", type=int, default=512)
parser.add_argument("--temperature", type=float, default=0.8)
parser.add_argument("--seed", type=int, default=0)
return parser.parse_args()
diff --git a/src/test.py b/src/test.py
index 93e45e5..01884ef 100644
--- a/src/test.py
+++ b/src/test.py
@@ -8,7 +8,7 @@ import jax.numpy as jnp
import numpy as np
import optax
-from .data import get_batches, get_tokenizer, load_tinystories, prepare_text_pairs, load_librispeech
+from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs, load_librispeech
from .model import (
EncoderDecoderTransformer,
TransformerConfig,
@@ -50,7 +50,7 @@ def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batch
return math.exp(avg_nll)
-def measure_throughput(model, params, tokenizer, num_runs=10, prompt="Once upon a time", max_gen_len=64):
+def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is the weather? [{"name": "get_weather"}]', max_gen_len=64):
enc_tokens = tokenizer.encode(prompt)
enc_input = jnp.array([enc_tokens])
pad_id = tokenizer.pad_token_id
@@ -180,6 +180,117 @@ def compute_wer(hypotheses, references):
return total_edits / max(total_ref_words, 1)
+def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=512):
+ """Generate tool-call predictions and compute structured metrics."""
+ import json
+ from .run import generate
+ from .data import load_tool_calls
+
+ ds = load_tool_calls("validation", max_samples=num_samples)
+
+ total = 0
+ exact_match = 0
+ # For function-name-level P/R/F1
+ tp_names = 0 # predicted name in reference
+ fp_names = 0 # predicted name not in reference
+ fn_names = 0 # reference name not predicted
+ # For full call-level (name + args) P/R/F1
+ tp_calls = 0
+ fp_calls = 0
+ fn_calls = 0
+ json_parse_errors = 0
+ empty_ref = 0
+ empty_pred = 0
+
+ samples = []
+
+ for i, ex in enumerate(ds):
+ prompt = ex["query"] + " " + ex["tools"]
+ ref_text = ex["answers"]
+
+ pred_text = generate(
+ model, params, tokenizer, prompt,
+ max_gen_len=max_gen_len, temperature=0.0, seed=i, stream=False,
+ task_token_id=tokenizer.tool_call_token_id,
+ ).strip()
+
+ # Parse reference
+ try:
+ ref_calls = json.loads(ref_text)
+ except (json.JSONDecodeError, TypeError):
+ ref_calls = []
+
+ # Parse prediction
+ try:
+ pred_calls = json.loads(pred_text)
+ if not isinstance(pred_calls, list):
+ pred_calls = [pred_calls] if isinstance(pred_calls, dict) else []
+ except (json.JSONDecodeError, TypeError):
+ json_parse_errors += 1
+ pred_calls = []
+
+ total += 1
+ if i < 10:
+ samples.append((ex["query"][:80], ref_text[:120], pred_text[:120]))
+
+ # Track empty
+ if len(ref_calls) == 0:
+ empty_ref += 1
+ if len(pred_calls) == 0:
+ empty_pred += 1
+
+ # Exact match (normalized)
+ if json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):
+ exact_match += 1
+
+ # Function name P/R/F1
+ ref_names = [c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c]
+ pred_names = [c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c]
+ ref_name_set = set(ref_names)
+ pred_name_set = set(pred_names)
+ tp_names += len(pred_name_set & ref_name_set)
+ fp_names += len(pred_name_set - ref_name_set)
+ fn_names += len(ref_name_set - pred_name_set)
+
+ # Full call P/R/F1 (name + args must match)
+ def call_key(c):
+ if not isinstance(c, dict):
+ return None
+ return json.dumps({"name": c.get("name"), "arguments": c.get("arguments")}, sort_keys=True)
+
+ ref_keys = set(call_key(c) for c in ref_calls)
+ pred_keys = set(call_key(c) for c in pred_calls)
+ ref_keys.discard(None)
+ pred_keys.discard(None)
+ tp_calls += len(pred_keys & ref_keys)
+ fp_calls += len(pred_keys - ref_keys)
+ fn_calls += len(ref_keys - pred_keys)
+
+ # Compute metrics
+ name_precision = tp_names / max(tp_names + fp_names, 1)
+ name_recall = tp_names / max(tp_names + fn_names, 1)
+ name_f1 = 2 * name_precision * name_recall / max(name_precision + name_recall, 1e-9)
+
+ call_precision = tp_calls / max(tp_calls + fp_calls, 1)
+ call_recall = tp_calls / max(tp_calls + fn_calls, 1)
+ call_f1 = 2 * call_precision * call_recall / max(call_precision + call_recall, 1e-9)
+
+ return {
+ "num_samples": total,
+ "exact_match": exact_match / max(total, 1),
+ "json_parse_rate": 1.0 - json_parse_errors / max(total, 1),
+ "name_precision": name_precision,
+ "name_recall": name_recall,
+ "name_f1": name_f1,
+ "call_precision": call_precision,
+ "call_recall": call_recall,
+ "call_f1": call_f1,
+ "empty_ref_pct": empty_ref / max(total, 1),
+ "empty_pred_pct": empty_pred / max(total, 1),
+ "samples": samples,
+ }
+
+
def benchmark_asr(model, params, tokenizer, num_samples=100):
"""Transcribe LibriSpeech test-clean samples and report WER."""
from .run import transcribe
@@ -218,13 +329,43 @@ def main(args):
print(f"parameters: {param_count:,}")
print(f"config: d={config.d_model}, heads={config.num_heads}, layers={config.num_encoder_layers}/{config.num_decoder_layers}")
- print(f"\nevaluating text perplexity ({args.max_eval_samples} samples)...")
- ds = load_tinystories("validation", max_samples=args.max_eval_samples)
- enc_inputs, dec_inputs, dec_targets = prepare_text_pairs(
+ print(f"\nevaluating tool-call perplexity ({args.max_eval_samples} samples)...")
+ ds = load_tool_calls("validation", max_samples=args.max_eval_samples)
+ enc_inputs, dec_inputs, dec_targets = prepare_tool_call_pairs(
ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
)
ppl = compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, args.batch_size, config.pad_token_id)
- print(f"text ppl: {ppl:.2f}")
+
+ tc_samples = getattr(args, "tool_call_samples", 200)
+ tc = None
+ if tc_samples > 0:
+ print(f"\nevaluating tool-call accuracy ({tc_samples} samples)...")
+ tc = benchmark_tool_calls(model, params, tokenizer, num_samples=tc_samples, max_gen_len=args.max_gen_len)
+
+ print(f"\n ─────────────────────────────────────")
+ print(f" Tool-Call Metrics")
+ print(f" ─────────────────────────────────────")
+ print(f" Perplexity {ppl:>10.2f} ({args.max_eval_samples} samples)")
+ if tc:
+ print(f" JSON parse rate {tc['json_parse_rate']:>10.1%}")
+ print(f" Exact match {tc['exact_match']:>10.1%}")
+ print(f" ─── Function Name ───────────────────")
+ print(f" Precision {tc['name_precision']:>10.3f}")
+ print(f" Recall {tc['name_recall']:>10.3f}")
+ print(f" F1 {tc['name_f1']:>10.3f}")
+ print(f" ─── Full Call (name+args) ───────────")
+ print(f" Precision {tc['call_precision']:>10.3f}")
+ print(f" Recall {tc['call_recall']:>10.3f}")
+ print(f" F1 {tc['call_f1']:>10.3f}")
+ print(f" ─────────────────────────────────────")
+ print(f" Empty ref/pred {tc['empty_ref_pct']:.1%} / {tc['empty_pred_pct']:.1%}")
+ if tc["samples"]:
+ print(f"\n samples:")
+ for query, ref, pred in tc["samples"][:5]:
+ print(f" Q: {query}")
+ print(f" R: {ref}")
+ print(f" P: {pred}")
+ print()
asr_samples = getattr(args, "asr_samples", 50)
if asr_samples > 0:
@@ -238,37 +379,19 @@ def main(args):
print(f"avg latency: {throughput['avg_latency_s']:.3f}s")
print(f"tokens/sec: {throughput['tokens_per_second']:.1f}")
- prompts = [
- "Once upon a time",
- "The little dog",
- "She was very happy because",
- "One day, a boy named",
- "The cat and the mouse",
- ]
- print(f"\ngenerating samples (t={args.temperature})...")
- quality = benchmark_generation_quality(
- model, params, tokenizer, prompts, max_gen_len=args.max_gen_len, temperature=args.temperature
- )
- print(f"avg length: {quality['avg_generation_length']:.1f} tokens")
- print(f"repetition: {quality['bigram_repetition_rate']:.3f}")
-
- print("\nsamples:")
- for prompt, gen in quality["generations"]:
- print(f" [{prompt}]")
- print(f" {gen[:120]}")
- print()
-
def parse_args():
parser = argparse.ArgumentParser(description="Benchmark transformer on generation tasks")
parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")
parser.add_argument("--batch-size", type=int, default=32)
parser.add_argument("--max-eval-samples", type=int, default=1000)
- parser.add_argument("--max-enc-len", type=int, default=128)
- parser.add_argument("--max-dec-len", type=int, default=128)
- parser.add_argument("--max-gen-len", type=int, default=128)
+ parser.add_argument("--max-enc-len", type=int, default=1024)
+ parser.add_argument("--max-dec-len", type=int, default=512)
+ parser.add_argument("--max-gen-len", type=int, default=512)
parser.add_argument("--temperature", type=float, default=0.8)
parser.add_argument("--throughput-runs", type=int, default=10)
+ parser.add_argument("--tool-call-samples", type=int, default=200,
+ help="Samples for tool-call accuracy eval (default: 200)")
return parser.parse_args()
diff --git a/src/train.py b/src/train.py
index f4143eb..91ce210 100644
--- a/src/train.py
+++ b/src/train.py
@@ -14,7 +14,7 @@ from flax import jax_utils
from flax.training import train_state
from .data import (
- get_batches, get_tokenizer, load_tinystories, prepare_text_pairs,
+ get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs,
load_librispeech, prepare_speech_pairs, get_speech_batches,
)
from .model import (
@@ -474,22 +474,22 @@ def train(args):
tokenizer = get_tokenizer(max_samples=args.max_samples)
step_idx += 1
- print(f"\n[{step_idx}/{total_data_steps}] Loading TinyStories dataset...")
- ds = load_tinystories("train", max_samples=args.max_samples)
- val_ds = load_tinystories("validation", max_samples=getattr(args, "max_eval_samples", None))
+ print(f"\n[{step_idx}/{total_data_steps}] Loading tool-calling dataset...")
+ ds = load_tool_calls("train", max_samples=args.max_samples)
+ val_ds = load_tool_calls("validation", max_samples=getattr(args, "max_eval_samples", None))
step_idx += 1
- print(f"\n[{step_idx}/{total_data_steps}] Preparing text pairs...")
- enc_inputs, dec_inputs, dec_targets = prepare_text_pairs(
+ print(f"\n[{step_idx}/{total_data_steps}] Preparing tool-call pairs...")
+ enc_inputs, dec_inputs, dec_targets = prepare_tool_call_pairs(
ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
)
- val_enc, val_dec_in, val_dec_tgt = prepare_text_pairs(
+ val_enc, val_dec_in, val_dec_tgt = prepare_tool_call_pairs(
val_ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
)
val_enc = np.array(val_enc)
val_dec_in = np.array(val_dec_in)
val_dec_tgt = np.array(val_dec_tgt)
- print(f" {len(enc_inputs):,} train / {len(val_enc):,} val text pairs")
+ print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs")
# --- Speech data ---
speech_mel = speech_dec_in = speech_dec_tgt = None
@@ -891,15 +891,14 @@ def train(args):
with open(ckpt_path, "wb") as f:
pickle.dump({"params": params_np, "config": config.__dict__}, f)
- from .test import measure_throughput, benchmark_generation_quality, compute_wer
- from .run import generate, transcribe
+ from .test import measure_throughput, benchmark_tool_calls, compute_wer
+ from .run import transcribe
from .data import _decode_audio
eval_params_jnp = jax.tree.map(jnp.array, params_np)
del params_np # free CPU memory
model = EncoderDecoderTransformer(config)
tp = measure_throughput(model, eval_params_jnp, tokenizer, num_runs=5)
- prompts = ["Once upon a time", "The little dog", "She was very happy because"]
- quality = benchmark_generation_quality(model, eval_params_jnp, tokenizer, prompts, max_gen_len=64, temperature=0.8)
+ tc_metrics = benchmark_tool_calls(model, eval_params_jnp, tokenizer, num_samples=20, max_gen_len=128)
speech_wer = None
transcribe_samples = []
@@ -926,7 +925,7 @@ def train(args):
print(f" Avg loss {epoch_avg_loss:>12.4f}")
print(f" Final loss {final_loss:>12.4f}")
print(f" Train ppl {final_ppl:>12.2f}")
- print(f" Text val ppl {last_val_ppl:>12.2f}")
+ print(f" TC val ppl {last_val_ppl:>12.2f}")
if speech_wer is not None:
print(f" Speech WER {speech_wer:>12.4f}")
epoch_avg_speech = sum(speech_losses) / len(speech_losses) if speech_losses else float("nan")
@@ -946,12 +945,18 @@ def train(args):
print(f" ─────────────────────────────────────")
print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")
print(f" Latency {tp['avg_latency_s']:>11.3f}s")
- print(f" Gen length {quality['avg_generation_length']:>10.1f} tok")
- print(f" Repetition {quality['bigram_repetition_rate']:>12.3f}")
- print(f" ─────────────────────────────────────")
- for prompt, gen in quality["generations"]:
- gen_clean = gen.replace("<story>", "").replace("<transcribe>", "").strip()
- print(f" <story>[{prompt}] {gen_clean[:80]}")
+ print(f" ─── Tool-Call Metrics ───────────────")
+ print(f" JSON parse {tc_metrics['json_parse_rate']:>10.1%}")
+ print(f" Exact match {tc_metrics['exact_match']:>10.1%}")
+ print(f" Name F1 {tc_metrics['name_f1']:>10.3f} (P={tc_metrics['name_precision']:.3f} R={tc_metrics['name_recall']:.3f})")
+ print(f" Call F1 {tc_metrics['call_f1']:>10.3f} (P={tc_metrics['call_precision']:.3f} R={tc_metrics['call_recall']:.3f})")
+ if tc_metrics["samples"]:
+ print(f" ─────────────────────────────────────")
+ for query, ref, pred in tc_metrics["samples"][:3]:
+ print(f" Q: {query}")
+ print(f" R: {ref}")
+ print(f" P: {pred}")
+ print()
if transcribe_samples:
print(f" ─────────────────────────────────────")
print(f" Transcribe (WER={speech_wer:.4f}):")
@@ -971,6 +976,10 @@ def train(args):
"epoch/weight_sparsity": sparsity,
"epoch": epoch + 1,
}
+ log_dict["epoch/tc_exact_match"] = tc_metrics["exact_match"]
+ log_dict["epoch/tc_name_f1"] = tc_metrics["name_f1"]
+ log_dict["epoch/tc_call_f1"] = tc_metrics["call_f1"]
+ log_dict["epoch/tc_json_parse_rate"] = tc_metrics["json_parse_rate"]
if speech_wer is not None:
log_dict["epoch/speech_wer"] = speech_wer
if speech_losses: