Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Tool call refactor
HenryNdubuakue1c146854483
diff --git a/README.md b/README.mdindex ba4c9a7..38b549b 100644--- a/README.md+++ b/README.md@@ -10,42 +10,41 @@Architecture Training Pipeline──────────── ─────────────────- ┌──────────────┐ ┌──────────────┐- │ Transcription│ │ Story │ ┌──────────────────────┐- └──────┬───────┘ └──────┬───────┘ │ Mat loss at factors │- └───────┬─────────┘ │ + INT4 QAT (g=32) │- ┌───┴───────┐ │ + z-loss + slot div │- │ Softmax │ │ mixed text/speech │- └─────┬─────┘ │ batches │- ┌─────┬─────┴─────┬─────┐ └──────────┬───────────┘- │ @E[:d_ff/f] for each │ │- │ f in mat_factors │ ┌──────────┴───────────┐- └─────┬─────┬─────┬─────┘ │ Grad clip (norm 1.0) │- ┌─────┴─────┐ │ Muon (2D kernels) │- │ Linear (T)│ ← tied │ AdamW (all else) │- └─────┬─────┘ │ WSD LR schedule │- ┌─────┴─────┐ └──────────┬───────────┘- │ ZCRMSNorm │ │- └─────┬─────┘ ┌──────────┴───────────┐- ┌────────┴────────┐ │ Sparsification │- │ Decoder x N_dec │ │ cubic schedule │- │┌───────────────┐│ │ mask every N steps │- ││ Masked Self ││ └──────────┬───────────┘- ││ Attn + RoPE ││ │- │├───────────────┤│ ┌──────────┴───────────┐- ┌──────────────┐ S ││ Cross ││ │ EMA params (β=0.999) │- │ MemoryMixer │────────▶ Attention ││ └──────────┬───────────┘- │ Encoder │ │├───────────────┤│ │- │ x N_enc │ ││ Gated FFN ││ ┌──────────┴───────────┐- │ │ │└───────────────┘│ │ Checkpoint │- │ ┌──────────┐ │ └────────┬────────┘ │ full params, can │- │ │Pack: │ │ ┌─────┴─────┐ │ export mat slices │- │ │ S←X Attn │ │ │ Embedding │ ← shared └──────────────────────┘- │ │ RoPE keys│ │ └─────┬─────┘+ ┌──────────────┐+ │ Tool Call │ ┌──────────────────────┐+ └──────┬───────┘ │ Mat loss at factors │+ ┌┴──────────┐ │ + INT4 QAT (g=32) │+ │ Softmax │ │ + z-loss + slot div │+ └─────┬─────┘ │ 1:1 text/speech │+ ┌─────┬─────┴─────┬─────┐ │ alternation │+ │ @E[:d_ff/f] for each │ └──────────┬───────────┘+ │ f in mat_factors │ │+ └─────┬─────┬─────┬─────┘ ┌──────────┴───────────┐+ ┌─────┴─────┐ │ Grad clip (norm 1.0) │+ │ Linear (T)│ ← tied │ Muon (2D kernels) │+ └─────┬─────┘ │ AdamW (all else) │+ ┌─────┴─────┐ │ WSD LR schedule │+ │ ZCRMSNorm │ └──────────┬───────────┘+ └─────┬─────┘ │+ ┌────────┴────────┐ ┌──────────┴───────────┐+ │ Decoder x N_dec │ │ Sparsification │+ │┌───────────────┐│ │ cubic schedule │+ ││ Masked Self ││ │ mask every N steps │+ ││ Attn + RoPE ││ └──────────┬───────────┘+ │├───────────────┤│ │+ ┌──────────────┐ S ││ Cross ││ ┌──────────┴───────────┐+ │ MemoryMixer │────────▶ Attention ││ │ EMA params (β=0.999) │+ │ Encoder │ │├───────────────┤│ └──────────┬───────────┘+ │ x N_enc │ ││ Gated FFN ││ │+ │ │ │└───────────────┘│ ┌──────────┴───────────┐+ │ ┌──────────┐ │ └────────┬────────┘ │ Checkpoint │+ │ │Pack: │ │ ┌─────┴─────┐ │ full params, can │+ │ │ S←X Attn │ │ │ Embedding │ ← shared │ export mat slices │+ │ │ RoPE keys│ │ └─────┬─────┘ └──────────────────────┘│ ├──────────┤ │ ┌───────┴───────┐- │ │Mix: │ │ │ <transcribe> │- │ │ MLP-Mixer│ │ │ or <story> │- │ │ on S │ │ │ + target │+ │ │Mix: │ │ │[EOS]<tool_call>│+ │ │ MLP-Mixer│ │ │ + tools │+ │ │ on S │ │ │ + answer ││ ├──────────┤ │ └───────────────┘│ │Local: │ ││ │Gated FFN │ │@@ -64,56 +63,63 @@│ │┌────┴──────┐ ┌──────┴───────┐│ Text │ │ Audio │- │ tokens │ │ mel spec │+ │ query │ │ waveform │└───────────┘ └──────────────┘d=512 (default) / 1536 (--full) · GQA · QK-normSentencePiece BPE (8192) · gated dReLU · ZCRMSNorm · RoPEstrided DW conv · mat factors · INT4 QAT · Muon + AdamW- text + speech encoder · <transcribe> / <story> task routing+ text + speech encoder · <tool_call> task routingData Preprocessing──────────────────- Text (TinyStories) Speech (LibriSpeech-100)- ────────────────── ────────────────────────+ Text (Tool-Call Dataset) Speech (Voice-Tool-Call)+ ──────────────────────── ────────────────────────- TinyStories LibriSpeech-100+ Unified dataset Unified dataset+ (GCS / local) (GCS / local)│ │▼ ▼┌──────────────┐ ┌────────────────┐- │ SentencePiece│ │ soundfile │- │ BPE tokenize │ │ decode (16kHz) │+ │ SentencePiece│ │ HF Audio col │+ │ BPE tokenize │ │ decode (16kHz) ││ (8192 vocab) │ └───────┬────────┘└──────┬───────┘ ││ ▼▼ ┌────────────────┐- ┌──────────────┐ │ Log-mel spec │- │ Split at 30% │ │ 80 bins, 25ms │- │ enc / dec │ │ window, 10ms │- └──┬───────┬───┘ │ hop (~100 f/s) │- │ │ └───────┬────────┘- ▼ ▼ │- ┌─────┐ ┌──────────────┐ ▼- │ enc │ │ [BOS]<story> │ ┌────────────────┐- │ ids │ │ + tokens │ │ Pad/trunc to │- └─────┘ │ → dec_in │ │ max_mel_len │- ├──────────────┤ └───────┬────────┘- │ <story> │ │- │ + tokens │ ▼- │ + [EOS] │ ┌────────────────┐- │ → dec_tgt │ │ [BOS]<transcr> │- └──────────────┘ │ + tokens │- │ → dec_in │- Both cached as ├────────────────┤- .npy in .data_cache/ │ <transcr> │- │ + tokens │- ┌──────────────────────┐ │ + [EOS] │- │ Interleaved batches: │ │ → dec_tgt │- │ 1 speech every N │ └────────────────┘- │ text steps │- │ (--speech-every N) │ Eval: WER on 3 val samples- └──────────────────────┘ at end of each epoch+ ┌──────────────┐ │ Augmentation │+ │ Encoder: │ │ (noise, pitch, │+ │ query tokens │ │ stretch, gain)│+ └──────┬───────┘ └───────┬────────┘+ │ │+ ▼ ▼+ ┌──────────────────┐ ┌────────────────┐+ │ Decoder: │ │ Log-mel spec │+ │ [EOS]<tool_call> │ │ 80 bins, 25ms │+ │ + tools tokens │ │ window, 10ms │+ │ + answer tokens │ │ hop (~100 f/s) │+ │ → dec_in │ └───────┬────────┘+ ├──────────────────┤ │+ │ <tool_call> │ ▼+ │ + tools tokens │ ┌────────────────┐+ │ + answer tokens │ │ Pad/trunc to │+ │ + [EOS] │ │ max_mel_len │+ │ → dec_tgt │ └───────┬────────┘+ ├──────────────────┤ │+ │ Loss mask: 1.0 │ ▼+ │ on answer + EOS │ ┌────────────────┐+ │ only (not tools) │ │ Same decoder │+ └──────────────────┘ │ format as text │+ └────────────────┘+ Both cached as+ .npy in .data_cache/ Mel computed on-the-fly+ (augment before mel)+ ┌──────────────────────┐+ │ Interleaved batches: │+ │ 1:1 text / speech │ Eval: voice-to-tool-call on+ │ alternation │ 3 val samples per epoch+ └──────────────────────┘```## Usage@@ -128,70 +134,72 @@ needle [command]┌───────────────────────────────────────────────────────────────────┐│ ││ train │- │ --full Use full 1B config (~1.17B params) │- │ --epochs INT Training epochs (default: 1) │- │ --batch-size INT Batch size (default: 32) │- │ --lr FLOAT AdamW learning rate (default: 3e-4) │- │ --muon-lr FLOAT Muon learning rate (default: 0.02) │- │ --d-model INT Model dim (default: 512) │- │ --num-heads INT Attention heads (default: 8) │- │ --num-kv-heads INT KV heads for GQA (default: num-heads) │- │ --num-layers INT Encoder layers (default: 8) │- │ --num-dec-layers INT Decoder layers (default: 4) │- │ --max-enc-len INT Max encoder seq length (default: 256) │- │ --max-dec-len INT Max decoder seq length (default: 256) │- │ --max-samples INT Training samples (default: all) │- │ --mat-factors INT [...] FFN shrink factors (default: 2 4 8) │- │ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) │- │ --group-size INT Quant/prune group size (default: 32) │- │ --prune-interval INT Steps between mask updates (def: 100) │- │ --prune-start-frac FL Start pruning at frac (def: 0.33) │- │ --prune-end-frac FL Lock mask at this frac (def: 0.67) │- │ --activation STR drelu|swiglu|geglu (default: drelu) │- │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │- │ --eval-every INT Val eval interval (default: 1000) │- │ --wandb Enable W&B logging │- │ --checkpoint PATH Resume from checkpoint │- │ --checkpoint-dir DIR Checkpoint directory │- │ --seed INT Random seed (default: 42) │+ │ --full Use full 1B config (~1.17B params) │+ │ --epochs INT Training epochs (default: 1) │+ │ --batch-size INT Batch size (default: 32) │+ │ --lr FLOAT AdamW learning rate (default: 3e-4) │+ │ --muon-lr FLOAT Muon learning rate (default: 0.02) │+ │ --d-model INT Model dim (default: 512) │+ │ --num-heads INT Attention heads (default: 8) │+ │ --num-kv-heads INT KV heads for GQA (default: num-heads)│+ │ --num-layers INT Encoder layers (default: 4) │+ │ --num-dec-layers INT Decoder layers (default: 4) │+ │ --max-enc-len INT Max encoder seq len (default: 256) │+ │ --max-dec-len INT Max decoder seq len (default: 1024) │+ │ --max-samples INT Training samples (default: all) │+ │ --mat-factors INT [...] FFN shrink factors (default: 2 4 8) │+ │ --sparsity-ratio FLOAT Block prune ratio (default: 0.0) │+ │ --group-size INT Quant/prune group size (default: 32) │+ │ --prune-interval INT Steps between mask updates (def: 100)│+ │ --prune-start-frac FL Start pruning at frac (def: 0.33) │+ │ --prune-end-frac FL Lock mask at this frac (def: 0.67) │+ │ --activation STR drelu|swiglu|geglu (default: drelu) │+ │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │+ │ --eval-every INT Val eval interval (default: 1000) │+ │ --wandb Enable W&B logging │+ │ --checkpoint PATH Resume from checkpoint │+ │ --checkpoint-dir DIR Checkpoint directory │+ │ --seed INT Random seed (default: 42) ││ --no-speech Disable speech (text-only training) │- │ --speech-every INT Speech step every N text (default: 3) ││ --max-mel-len INT Max mel frames (default: 1024) ││ --n-mels INT Mel frequency bins (default: 80) │- │ --max-speech-samples INT Max LibriSpeech samples │+ │ --max-speech-samples INT Max voice-tool-call samples ││ ││ run │- │ --checkpoint PATH Path to model checkpoint (required) │- │ --prompts STR [...] One or more prompts to continue │- │ --audio PATH [...] Audio files to transcribe │- │ --max-len INT Max tokens to generate (default: 128) │- │ --temperature FLOAT Sampling temperature (default: 0.8) │- │ --seed INT Random seed (default: 0) │+ │ --checkpoint PATH Path to model checkpoint (required) │+ │ --query STR Query text for tool-call generation │+ │ --tools STR Tools JSON for tool-call generation │+ │ --audio PATH [...] Audio files for voice-to-tool-call │+ │ --max-len INT Max tokens to generate (default: 512) │+ │ --seed INT Random seed (default: 0) ││ ││ test │- │ --checkpoint PATH Path to model checkpoint (required) │- │ --batch-size INT Batch size (default: 32) │- │ --max-eval-samples INT Evaluation samples (default: 1000) │- │ --max-gen-len INT Max generation length (default: 128) │- │ --temperature FLOAT Sampling temperature (default: 0.8) │- │ --throughput-runs INT Throughput runs (default: 10) │+ │ --checkpoint PATH Path to model checkpoint (required) │+ │ --batch-size INT Batch size (default: 32) │+ │ --max-eval-samples INT Evaluation samples (default: 1000) │+ │ --max-enc-len INT Max encoder length (default: 256) │+ │ --max-dec-len INT Max decoder length (default: 1024) │+ │ --max-gen-len INT Max generation length (default: 512) │+ │ --throughput-runs INT Throughput runs (default: 10) │+ │ --tool-call-samples INT Tool-call eval samples (default: 200) │+ │ --voice-tc-samples INT Voice-TC eval samples (default: 50) ││ ││ evaluate │- │ --checkpoint PATH Path to model checkpoint (required) │- │ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │- │ --max-samples INT Samples per benchmark (default: 500) │+ │ --checkpoint PATH Path to model checkpoint (required) │+ │ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │+ │ --max-samples INT Samples per benchmark (default: 500) ││ ││ tpu │- │ create NAME Create TPU (auto-finds zone) │- │ --type STR Accelerator (default: v6e-8) │- │ --version STR TPU OS (auto-detected from --type) │- │ connect NAME SSH config + connect (auto-zone) │- │ claude NAME Install Claude Code on instance │- │ stop NAME Stop instance (auto-zone) │- │ start NAME Start stopped instance (auto-zone) │- │ delete NAME Delete instance (auto-zone) │- │ list List all TPU instances │- │ --zone ZONE Override auto-detected zone │+ │ create NAME Create TPU (auto-finds zone) │+ │ --type STR Accelerator (default: v6e-8) │+ │ --version STR TPU OS (auto-detected from --type) │+ │ connect NAME SSH config + connect (auto-zone) │+ │ claude NAME Install Claude Code on instance │+ │ stop NAME Stop instance (auto-zone) │+ │ start NAME Start stopped instance (auto-zone) │+ │ delete NAME Delete instance (auto-zone) │+ │ list List all TPU instances │+ │ --zone ZONE Override auto-detected zone ││ │└───────────────────────────────────────────────────────────────────┘```@@ -210,19 +218,21 @@ needle [command]Dataset──────────────────────────────────────────- Text 100B tokens (~400 GB)- Audio Emilia subset- 200k × 20s clips (~1.1k hrs)- ~200M audio tokens- ~13M transcription tokens- ~27 GB (from ~4.5 TB full dataset)+ Text Tool-call pairs+ (query, tools, answers)+ from GCS unified dataset+ Audio TTS-generated speech+ (GCP Cloud TTS voices)+ 16kHz WAV, HF Audio col──────────────────────────────────────────- 100M quick-experiment estimates (--default)+ 100M params — 10M multimodal triples (5 epochs)──────────────────────────────────────────- Total tokens 1B text + 1k hrs audio ≈ 1.2B- Total FLOPs 6 × 102M × 1.2B ≈ 7.2e17- MFU (100M model) ~15% → ~138 effective TFLOPS/chip+ Triples 10M (text / voice / image → JSON tool calls)+ Tokens/triple ~300 avg (text ~180, voice ~390, image ~336)+ Total tokens 10M × 300 × 5 epochs ≈ 15B+ Total FLOPs 6 × 100M × 15B ≈ 9e18+ MFU (100M) ~15% → ~138 effective TFLOPS/chip┌────────────────────┬──────────┬──────────┬──────────┐│ │ v6e-8 │ v6e-16 │ v6e-32 │@@ -231,27 +241,9 @@ needle [command]│ Total HBM │ 256 GB │ 512 GB │ 1024 GB ││ Scaling eff. │ 0.9× │ 0.8× │ 0.7× ││ Eff. TFLOPS │ 994 │ 1,766 │ 3,091 │- │ Est. time │ ~12min │ ~7min │ ~4min │- │ On-demand $/hr │ $21.60 │ $43.20 │ $86.40 │- │ Est. total cost │ ~$4 │ ~$5 │ ~$6 │- └────────────────────┴──────────┴──────────┴──────────┘-- 1B multimodal training estimates (--full)- ──────────────────────────────────────────- Total tokens 100B text + 250k hrs audio ≈ 145B- Total FLOPs 6 × 1.17B × 145B ≈ 1.0e21- MFU (1B model) ~30% → ~275 effective TFLOPS/chip-- ┌────────────────────┬──────────┬──────────┬──────────┐- │ │ v6e-8 │ v6e-16 │ v6e-32 │- ├────────────────────┼──────────┼──────────┼──────────┤- │ Chips │ 8 │ 16 │ 32 │- │ Total HBM │ 256 GB │ 512 GB │ 1024 GB │- │ Scaling eff. │ 0.9× │ 0.8× │ 0.7× │- │ Eff. TFLOPS │ 1,980 │ 3,520 │ 6,160 │- │ Est. time │ ~143h │ ~80h │ ~46h │+ │ Est. time │ ~2.5h │ ~1.4h │ ~49min ││ On-demand $/hr │ $21.60 │ $43.20 │ $86.40 │- │ Est. total cost │ ~$3,090 │ ~$3,460 │ ~$3,970 │+ │ Est. total cost │ ~$54 │ ~$61 │ ~$71 │└────────────────────┴──────────┴──────────┴──────────┘```diff --git a/requirements.txt b/requirements.txtindex 4e3232f..ec5f4fb 100644--- a/requirements.txt+++ b/requirements.txt@@ -3,7 +3,11 @@ jaxlibflaxoptaxdatasets+gcsfstransformerswandbpyyamlsentencepiece+google-cloud-texttospeech+audiomentations+gcsfs\ No newline at end of filediff --git a/scripts/build_dataset.py b/scripts/build_dataset.pynew file mode 100644index 0000000..72f9385--- /dev/null+++ b/scripts/build_dataset.py@@ -0,0 +1,335 @@+#!/usr/bin/env python3+"""Unified dataset builder: text + TTS audio -> HuggingFace Dataset on GCS.++Combines the text pipeline (tools_data.py) with TTS audio generation into+one script that produces a single HF Dataset with an Audio column.++Usage:+ python scripts/build_dataset.py --max-samples 1000 --workers 16+ python scripts/build_dataset.py --max-samples 1000 --gcs-upload gs://cactus-dataset/tool_calls+"""++import argparse+import io+import logging+import os+import random+import subprocess+import sys+import time+import wave+from concurrent.futures import ThreadPoolExecutor, as_completed++from datasets import Audio, Dataset, load_from_disk+from tqdm import tqdm++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from tools_data import load_and_combine++logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")+logger = logging.getLogger(__name__)++VOICES = [+ ("en-US", "en-US-Journey-D", "MALE", False, False),+ ("en-US", "en-US-Journey-F", "FEMALE", False, False),+ ("en-US", "en-US-Journey-O", "FEMALE", False, False),+ ("en-GB", "en-GB-Wavenet-B", "MALE", True, False),+ ("en-GB", "en-GB-Wavenet-A", "FEMALE", True, False),+ ("en-AU", "en-AU-Wavenet-B", "MALE", True, False),+ ("en-AU", "en-AU-Wavenet-C", "FEMALE", True, False),+ ("en-US", "en-US-Wavenet-D", "MALE", True, False),+ ("en-IN", "en-IN-Wavenet-A", "FEMALE", True, False),+ ("en-IN", "en-IN-Wavenet-B", "MALE", True, False),+ ("en-IN", "en-IN-Wavenet-C", "MALE", True, False),+ ("en-GB", "en-GB-Wavenet-D", "MALE", True, False),+ ("en-AU", "en-AU-Wavenet-D", "MALE", True, False),+ ("en-US", "en-US-Wavenet-E", "FEMALE", True, True),+ ("en-GB", "en-GB-Wavenet-A", "FEMALE", True, True),+ ("en-IN", "en-IN-Wavenet-A", "FEMALE", True, True),+]+++def _pcm_to_wav_bytes(pcm_bytes: bytes, sample_rate: int) -> bytes:+ """Convert raw LINEAR16 PCM bytes to a WAV byte buffer."""+ buf = io.BytesIO()+ with wave.open(buf, "wb") as wf:+ wf.setnchannels(1)+ wf.setsampwidth(2)+ wf.setframerate(sample_rate)+ wf.writeframes(pcm_bytes)+ return buf.getvalue()+++def _synthesize_one(client, idx, query, max_retries=5):+ """Synthesize TTS audio for a single query via GCP Cloud TTS.++ Selects a voice from VOICES based on idx, applies randomized prosody+ for configurable voices, and retries on transient API errors.++ Returns:+ Tuple of (idx, wav_bytes, voice_name, error_msg).+ wav_bytes is None on failure; error_msg is None on success.+ """+ from google.cloud import texttospeech+ from google.api_core.exceptions import ResourceExhausted, ServiceUnavailable++ lang, voice_name, ssml_gender, configurable, child = VOICES[idx % len(VOICES)]+ gender = getattr(texttospeech.SsmlVoiceGender, ssml_gender)++ synthesis_input = texttospeech.SynthesisInput(text=query)+ voice_params = texttospeech.VoiceSelectionParams(+ language_code=lang,+ name=voice_name,+ ssml_gender=gender,+ )++ if configurable:+ rng = random.Random(idx)+ if child:+ pitch = rng.uniform(4.0, 8.0)+ rate = rng.uniform(1.0, 1.15)+ else:+ pitch = rng.uniform(-3.0, 3.0)+ rate = rng.uniform(0.9, 1.1)+ audio_config = texttospeech.AudioConfig(+ audio_encoding=texttospeech.AudioEncoding.LINEAR16,+ sample_rate_hertz=16000,+ pitch=pitch,+ speaking_rate=rate,+ volume_gain_db=rng.uniform(-1.5, 1.5),+ )+ else:+ audio_config = texttospeech.AudioConfig(+ audio_encoding=texttospeech.AudioEncoding.LINEAR16,+ sample_rate_hertz=16000,+ )++ for attempt in range(max_retries):+ try:+ response = client.synthesize_speech(+ input=synthesis_input,+ voice=voice_params,+ audio_config=audio_config,+ )+ wav_bytes = _pcm_to_wav_bytes(response.audio_content, 16000)+ return idx, wav_bytes, voice_name, None+ except (ResourceExhausted, ServiceUnavailable):+ wait = min(2 ** attempt, 30)+ time.sleep(wait)+ except Exception as e:+ return idx, None, voice_name, str(e)++ return idx, None, voice_name, "max retries exceeded"+++def _load_text_dataset(output_path, max_samples):+ """Load the text dataset and optionally resume from an existing build.++ Returns:+ Tuple of (combined_dataset, existing_dataset_or_None).+ """+ existing_ds = None+ if os.path.exists(output_path):+ logger.info(f"Found existing dataset at {output_path}, loading for resume...")+ existing_ds = load_from_disk(output_path)+ logger.info(f" Existing dataset: {len(existing_ds)} rows")++ logger.info("Loading and combining text datasets...")+ combined = load_and_combine()++ if max_samples:+ combined = combined.select(range(min(max_samples, len(combined))))+ logger.info(f"Text dataset: {len(combined)} rows")++ return combined, existing_ds+++def _find_rows_needing_tts(combined, existing_ds):+ """Identify which rows lack audio by inspecting the raw Arrow column.++ Accesses the underlying Arrow table directly to avoid triggering+ HF audio decoding (which requires torchcodec).++ Returns:+ Set of row indices that need TTS synthesis.+ """+ n = len(combined)+ needs_tts = set(range(n))++ if existing_ds is not None and len(existing_ds) == n and "audio" in existing_ds.column_names:+ audio_column = existing_ds.data.column("audio")+ for i in range(n):+ if audio_column[i].as_py() is not None:+ needs_tts.discard(i)+ logger.info(f" {n - len(needs_tts)} rows already have audio, {len(needs_tts)} need TTS")++ return needs_tts+++def _recover_existing_audio(existing_ds, n, needs_tts):+ """Extract audio bytes and voice names from a previously saved dataset.++ Reads raw Arrow data to avoid triggering audio decoding.++ Returns:+ Tuple of (audio_data list, voice_names list).+ """+ audio_data = [None] * n+ voice_names = [""] * n++ if existing_ds is not None and len(existing_ds) == n and "audio" in existing_ds.column_names:+ audio_column = existing_ds.data.column("audio")+ voice_col = existing_ds["voice"] if "voice" in existing_ds.column_names else None+ for i in range(n):+ if i not in needs_tts:+ raw = audio_column[i].as_py()+ if isinstance(raw, dict) and raw.get("bytes"):+ audio_data[i] = raw["bytes"]+ elif isinstance(raw, dict) and raw.get("path"):+ with open(raw["path"], "rb") as f:+ audio_data[i] = f.read()+ if voice_col is not None:+ voice_names[i] = voice_col[i] or ""++ return audio_data, voice_names+++def _synthesize_missing(combined, needs_tts, audio_data, voice_names, workers, rate_limit):+ """Run TTS synthesis for all rows that lack audio.++ Uses a thread pool with rate-limited submission to stay within+ GCP Cloud TTS API quotas.+ """+ if not needs_tts:+ return++ from google.cloud import texttospeech++ queries_to_synth = [(i, combined[i]["query"]) for i in sorted(needs_tts)]+ logger.info(f"Synthesizing TTS for {len(queries_to_synth)} rows with {workers} workers...")++ def _worker(item):+ idx, query = item+ client = texttospeech.TextToSpeechClient()+ return _synthesize_one(client, idx, query)++ success, failed = 0, 0+ delay = 1.0 / rate_limit if rate_limit > 0 else 0+ with ThreadPoolExecutor(max_workers=workers) as pool:+ futures = {}+ for q in queries_to_synth:+ futures[pool.submit(_worker, q)] = q+ time.sleep(delay)+ with tqdm(total=len(futures), desc="TTS") as pbar:+ for future in as_completed(futures):+ idx, wav_bytes, voice_name, err = future.result()+ if wav_bytes is not None:+ audio_data[idx] = wav_bytes+ voice_names[idx] = voice_name+ success += 1+ else:+ failed += 1+ logger.warning(f"Failed idx={idx}: {err}")+ pbar.update(1)++ logger.info(f"TTS done: {success} generated, {failed} failed")+++def _backfill_voice_names(voice_names):+ """Assign default voice names to any rows that don't have one."""+ for i in range(len(voice_names)):+ if not voice_names[i]:+ _, voice_name, _, _, _ = VOICES[i % len(VOICES)]+ voice_names[i] = voice_name+++def _build_and_save(combined, audio_data, voice_names, output_path):+ """Assemble the unified HF Dataset and save to disk.++ Returns:+ The saved Dataset.+ """+ n = len(combined)+ logger.info("Building unified HF Dataset...")+ ds_dict = {+ "query": [combined[i]["query"] for i in range(n)],+ "audio": audio_data,+ "tools": [combined[i]["tools"] for i in range(n)],+ "answers": [combined[i]["answers"] for i in range(n)],+ "source": [combined[i]["source"] for i in range(n)],+ "voice": voice_names,+ }++ unified = Dataset.from_dict(ds_dict)+ unified = unified.cast_column("audio", Audio(sampling_rate=16000))++ os.makedirs(os.path.dirname(output_path) if os.path.dirname(output_path) else ".", exist_ok=True)+ unified.save_to_disk(output_path)+ logger.info(f"Saved unified dataset to {output_path}/ ({len(unified)} rows)")+ return unified+++def _train_and_save_tokenizer(max_samples):+ """Train a SentencePiece tokenizer on the unified dataset."""+ from src.data import train_tokenizer, TOKENIZER_PREFIX+ logger.info("Training tokenizer on unified dataset...")+ model_path = train_tokenizer(max_samples=max_samples)+ logger.info(f"Tokenizer ready at {model_path}")+++def _upload_to_gcs(output_path, gcs_upload):+ """Upload the dataset and tokenizer to GCS via gcloud CLI."""+ from src.data import TOKENIZER_PREFIX++ logger.info(f"Uploading dataset to {gcs_upload}...")+ result = subprocess.run(+ ["gcloud", "storage", "cp", "-r", output_path, gcs_upload],+ capture_output=True, text=True,+ )+ if result.returncode == 0:+ logger.info(f"Dataset upload complete: {gcs_upload}")+ else:+ logger.error(f"Dataset upload failed: {result.stderr}")+ sys.exit(1)++ gcs_bucket = gcs_upload.rsplit("/", 1)[0]+ gcs_tokenizer_path = f"{gcs_bucket}/tokenizer/"+ logger.info(f"Uploading tokenizer to {gcs_tokenizer_path}...")+ tokenizer_dir = os.path.dirname(TOKENIZER_PREFIX)+ result = subprocess.run(+ ["gcloud", "storage", "cp", "-r", tokenizer_dir, gcs_tokenizer_path],+ capture_output=True, text=True,+ )+ if result.returncode == 0:+ logger.info(f"Tokenizer upload complete: {gcs_tokenizer_path}")+ else:+ logger.error(f"Tokenizer upload failed: {result.stderr}")+ sys.exit(1)+++def main():+ """Entry point for the unified dataset build pipeline."""+ parser = argparse.ArgumentParser(description="Build unified tool-call dataset with TTS audio")+ parser.add_argument("--max-samples", type=int, default=None, help="Max samples to process")+ parser.add_argument("--workers", type=int, default=16, help="TTS concurrent workers (default: 16)")+ parser.add_argument("--output", type=str, default="data/tool_calls_unified", help="Output directory")+ parser.add_argument("--rate-limit", type=float, default=100.0,+ help="Max TTS requests per second (default: 100)")+ parser.add_argument("--gcs-upload", type=str, default=None,+ help="GCS path to upload dataset (e.g. gs://cactus-dataset/tool_calls)")+ args = parser.parse_args()++ combined, existing_ds = _load_text_dataset(args.output, args.max_samples)+ needs_tts = _find_rows_needing_tts(combined, existing_ds)+ audio_data, voice_names = _recover_existing_audio(existing_ds, len(combined), needs_tts)+ _synthesize_missing(combined, needs_tts, audio_data, voice_names, args.workers, args.rate_limit)+ _backfill_voice_names(voice_names)+ _build_and_save(combined, audio_data, voice_names, args.output)+ _train_and_save_tokenizer(args.max_samples)++ if args.gcs_upload:+ _upload_to_gcs(args.output, args.gcs_upload)+++if __name__ == "__main__":+ main()diff --git a/src/cli.py b/src/cli.pyindex 9fd59e9..928ae95 100644--- a/src/cli.py+++ b/src/cli.py@@ -1,87 +1,7 @@import argparseimport sys-HELP = """- ┌───────────────────────────────────────────────────────────────────┐- │ │- │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │- │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │- │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │- │ ...the tiny model to rule them all... │- │ │- │ train │- │ --full Use full 1B config (~1.17B params) │- │ --epochs INT Training epochs (default: 1) │- │ --batch-size INT Batch size (default: 32) │- │ --lr FLOAT AdamW learning rate (default: 3e-4) │- │ --muon-lr FLOAT Muon learning rate (default: 0.02) │- │ --d-model INT Model dim (default: 512) │- │ --num-heads INT Attention heads (default: 8) │- │ --num-kv-heads INT KV heads for GQA (default: num-heads)│- │ --num-layers INT Encoder layers (default: 4) │- │ --num-dec-layers INT Decoder layers (default: 4) │- │ --max-enc-len INT Max encoder seq len (default: 256) │- │ --max-dec-len INT Max decoder seq len (default: 1024) │- │ --max-samples INT Training samples (default: all) │- │ --mat-factors INT [...] FFN shrink factors (default: 2 4 8) │- │ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) │- │ --group-size INT Quant/prune group size (default: 32) │- │ --prune-interval INT Steps between mask updates (def: 100)│- │ --prune-start-frac FL Start pruning at frac (def: 0.33) │- │ --prune-end-frac FL Lock mask at this frac (def: 0.67) │- │ --activation STR drelu|swiglu|geglu (default: drelu) │- │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │- │ --eval-every INT Val eval interval (default: 1000) │- │ --wandb Enable W&B logging │- │ --checkpoint PATH Resume from checkpoint │- │ --checkpoint-dir DIR Checkpoint directory │- │ --seed INT Random seed (default: 42) │- │ --no-speech Disable speech (text-only training) │- │ --speech-every INT Speech step every N text (default: 3) │- │ --max-mel-len INT Max mel frames (default: 1024) │- │ --n-mels INT Mel frequency bins (default: 80) │- │ --max-speech-samples INT Max LibriSpeech samples │- │ │- │ run │- │ --checkpoint PATH Path to model checkpoint (required) │- │ --query STR Query text for tool-call generation │- │ --tools STR Tools JSON for tool-call generation │- │ --prompts STR [...] Raw prompts (query+tools combined) │- │ --audio PATH [...] Audio files to transcribe │- │ --max-len INT Max tokens to generate (default: 512) │- │ --temperature FLOAT Sampling temperature (default: 0.8) │- │ --seed INT Random seed (default: 0) │- │ │- │ test │- │ --checkpoint PATH Path to model checkpoint (required) │- │ --batch-size INT Batch size (default: 32) │- │ --max-eval-samples INT Evaluation samples (default: 1000) │- │ --max-enc-len INT Max encoder length (default: 256) │- │ --max-dec-len INT Max decoder length (default: 1024) │- │ --max-gen-len INT Max generation length (default: 512) │- │ --temperature FLOAT Sampling temperature (default: 0.8) │- │ --throughput-runs INT Throughput runs (default: 10) │- │ --tool-call-samples INT Tool-call eval samples (default: 200) │- │ │- │ evaluate │- │ --checkpoint PATH Path to model checkpoint (required) │- │ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │- │ --max-samples INT Samples per benchmark (default: 500) │- │ │- │ tpu │- │ create NAME Create TPU (auto-finds zone) │- │ --type STR Accelerator (default: v6e-8) │- │ --version STR TPU OS (auto-detected from --type) │- │ connect NAME SSH config + connect (auto-zone) │- │ claude NAME Install Claude Code on instance │- │ stop NAME Stop instance (auto-zone) │- │ start NAME Start stopped instance (auto-zone) │- │ delete NAME Delete instance (auto-zone) │- │ list List all TPU instances │- │ --zone ZONE Override auto-detected zone │- │ │- └───────────────────────────────────────────────────────────────────┘-"""+HELP = """Check the readme"""def main():if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help", "help"):@@ -113,7 +33,7 @@ def main():p.add_argument("--seed", type=int, default=42)p.add_argument("--eval-every", type=int, default=1000)p.add_argument("--max-eval-samples", type=int, default=None)- p.add_argument("--sparsity-ratio", type=float, default=0.5)+ p.add_argument("--sparsity-ratio", type=float, default=0.0)p.add_argument("--group-size", type=int, default=32)p.add_argument("--prune-interval", type=int, default=100,help="Steps between mask updates during gradual pruning (default: 100)")@@ -128,20 +48,18 @@ def main():p.add_argument("--mat-shared-input", action="store_true",help="Each unique input is repeated across all mat widths (default: unique input per width)")p.add_argument("--no-speech", action="store_true", help="Disable speech training (text-only)")- p.add_argument("--speech-every", type=int, default=3,- help="Do one speech step every N text steps (default: 3)")p.add_argument("--max-mel-len", type=int, default=1024,help="Max mel spectrogram frames (default: 1024)")p.add_argument("--n-mels", type=int, default=80,help="Number of mel frequency bins (default: 80)")p.add_argument("--max-speech-samples", type=int, default=None,- help="Max LibriSpeech training samples (default: all)")+ help="Max voice-tool-call training samples (default: all)")p = sub.add_parser("run", add_help=False)p.add_argument("--checkpoint", type=str, required=True)p.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")p.add_argument("--tools", type=str, default=None, help="Tools JSON for tool-call generation")- p.add_argument("--audio", type=str, nargs="*", help="Audio file paths to transcribe")+ p.add_argument("--audio", type=str, nargs="*", help="Audio file paths for voice-to-tool-call")p.add_argument("--max-len", type=int, default=512)p.add_argument("--seed", type=int, default=0)@@ -154,6 +72,8 @@ def main():p.add_argument("--max-gen-len", type=int, default=512)p.add_argument("--tool-call-samples", type=int, default=200,help="Samples for tool-call accuracy eval (default: 200)")+ p.add_argument("--voice-tc-samples", type=int, default=50,+ help="Samples for voice-to-tool-call eval (default: 50)")p.add_argument("--throughput-runs", type=int, default=10)p = sub.add_parser("evaluate", add_help=False)diff --git a/src/data.py b/src/data.pyindex 4ebc294..a09d62f 100644--- a/src/data.py+++ b/src/data.py@@ -10,22 +10,71 @@ logging.getLogger("transformers").setLevel(logging.ERROR)logging.getLogger("huggingface_hub").setLevel(logging.ERROR)import numpy as np-from datasets import Audio, load_dataset, load_from_disk+from datasets import Audio, load_from_diskfrom tqdm import tqdmimport sentencepiece as spmCACHE_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), ".data_cache")TOKENIZER_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tokenizer")TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")-TOOL_CALLS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "tool_calls")+LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "tool_calls_unified")+GCS_DATASET_PATH = "gs://cactus-dataset/tool_calls"PAD_ID = 0EOS_ID = 1-BOS_ID = 2+BOS_ID = 2 # reserved for SentencePiece, unused at runtime (EOS_ID serves as SOS)UNK_ID = 3TOOL_CALL_ID = 4TRANSCRIBE_ID = 5+_unified_dataset_cache = None+++def _load_unified_dataset():+ """Load the unified dataset from GCS (via gcsfs) or local fallback.++ Caches the result in memory after first load.+ Uses soundfile as the audio decoding backend.+ """+ global _unified_dataset_cache+ if _unified_dataset_cache is not None:+ return _unified_dataset_cache++ # Try GCS first+ try:+ ds = load_from_disk(GCS_DATASET_PATH)+ print(f"Loaded unified dataset from GCS ({len(ds)} rows)")+ # Cache locally for faster subsequent loads+ if not os.path.exists(LOCAL_UNIFIED_DIR):+ ds.save_to_disk(LOCAL_UNIFIED_DIR)+ print(f" Cached locally to {LOCAL_UNIFIED_DIR}")+ _unified_dataset_cache = _set_audio_backend(ds)+ return _unified_dataset_cache+ except Exception:+ pass++ # Fallback to local+ if os.path.exists(LOCAL_UNIFIED_DIR):+ ds = load_from_disk(LOCAL_UNIFIED_DIR)+ print(f"Loaded unified dataset from {LOCAL_UNIFIED_DIR} ({len(ds)} rows)")+ _unified_dataset_cache = _set_audio_backend(ds)+ return _unified_dataset_cache++ raise FileNotFoundError(+ f"Unified dataset not found. Run scripts/build_dataset.py first, "+ f"or ensure GCS path {GCS_DATASET_PATH} is accessible (pip install gcsfs)."+ )+++def _set_audio_backend(ds):+ """Disable automatic audio decoding to avoid torchcodec dependency.++ Audio is decoded manually via soundfile in load_tool_call_audio().+ """+ if "audio" in ds.column_names:+ ds = ds.cast_column("audio", Audio(sampling_rate=16000, decode=False))+ return ds+class NeedleTokenizer:"""Wrapper around SentencePiece providing the interface the codebase expects."""@@ -102,7 +151,7 @@ def train_tokenizer(vocab_size=8192, max_samples=None):os.makedirs(TOKENIZER_DIR, exist_ok=True)- ds = load_from_disk(TOOL_CALLS_DIR)+ ds = _load_unified_dataset()if max_samples:ds = ds.select(range(min(max_samples, len(ds))))@@ -138,16 +187,37 @@ def train_tokenizer(vocab_size=8192, max_samples=None):return model_path+GCS_TOKENIZER_PATH = "gs://cactus-dataset/tokenizer/"++def get_tokenizer(max_samples=None):model_path = TOKENIZER_PREFIX + ".model"if not os.path.exists(model_path):+ # Try downloading from GCS first+ if _download_tokenizer_from_gcs():+ return NeedleTokenizer(model_path)train_tokenizer(max_samples=max_samples)return NeedleTokenizer(model_path)+def _download_tokenizer_from_gcs():+ """Download tokenizer files from GCS. Returns True on success."""+ import subprocess+ os.makedirs(TOKENIZER_DIR, exist_ok=True)+ result = subprocess.run(+ ["gcloud", "storage", "cp", "-r", GCS_TOKENIZER_PATH + "*", TOKENIZER_DIR + "/"],+ capture_output=True, text=True,+ )+ model_path = TOKENIZER_PREFIX + ".model"+ if result.returncode == 0 and os.path.exists(model_path):+ print(f"Downloaded tokenizer from {GCS_TOKENIZER_PATH}")+ return True+ return False++def load_tool_calls(split="train", max_samples=None):"""Load tool-calling dataset, splitting 90/10 for train/val."""- ds = load_from_disk(TOOL_CALLS_DIR)+ ds = _load_unified_dataset()n = len(ds)if split in ("validation", "val", "test"):ds = ds.select(range(int(n * 0.9), n))@@ -196,11 +266,9 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):)pad_id = tokenizer.pad_token_id- bos_id = tokenizer.eos_token_ideos_id = tokenizer.eos_token_idtool_call_id = tokenizer.tool_call_token_id- # Encoder: query only.enc_texts = [ex["query"] for ex in ds]tools_texts = [ex["tools"] for ex in ds]ans_texts = [ex["answers"] for ex in ds]@@ -260,12 +328,10 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):prefix_len = 2 + len(t_tok)total_dec = prefix_len + len(a_tok) + 1- # Encoderel = len(e_tok)enc_inputs[i, :el] = e_tok- # Decoder input: [BOS, <tool_call>, tools..., answer...]- dec_inputs[i, 0] = bos_id+ dec_inputs[i, 0] = eos_iddec_inputs[i, 1] = tool_call_idtl = len(t_tok)if tl > 0:@@ -322,57 +388,56 @@ def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True, lyield batch-get_text_batches = get_batches+def load_tool_call_audio(split="train", max_samples=None):+ """Load tool-call dataset entries that have TTS audio.-# ─── Speech pipeline ───────────────────────────────────────────────────────────+ Applies the same 90/10 split as load_tool_calls, then filters to entries+ where the audio column is not None.-def load_librispeech(split="train", max_samples=None):- """Load LibriSpeech-100 from HuggingFace (SPRINGLab/LibriSpeech-100).-- The dataset has a single 'train' split, so we partition it:- - train / train.clean.100: first 90%- - validation.clean / test.clean / validation / test: last 10%-- Audio decoding is disabled (to avoid torchcodec/torch dependency);- prepare_speech_pairs decodes audio bytes via soundfile instead.+ Returns list of dicts: {query, answers, tools, audio_bytes, sampling_rate}."""- ds = load_dataset("SPRINGLab/LibriSpeech-100", split="train")- ds = ds.cast_column("audio", Audio(decode=False))+ ds = _load_unified_dataset()n = len(ds)- if split in ("validation.clean", "validation", "test.clean", "test"):- ds = ds.select(range(int(n * 0.9), n))- elif split.startswith("train"):- ds = ds.select(range(int(n * 0.9)))- if max_samples:- ds = ds.select(range(min(max_samples, len(ds))))- return ds+ if split in ("validation", "val", "test"):+ indices = list(range(int(n * 0.9), n))+ elif split == "train":+ indices = list(range(int(n * 0.9)))+ else:+ indices = list(range(n))-def _decode_audio(audio_data, target_sr=16000):- """Decode raw audio bytes from a datasets Audio(decode=False) field using soundfile."""import ioimport soundfile as sf- audio_bytes = audio_data["bytes"]- if audio_bytes is None:- path = audio_data["path"]- audio, sr = sf.read(path, dtype="float32")- else:- audio, sr = sf.read(io.BytesIO(audio_bytes), dtype="float32")-- # Convert stereo to mono- if audio.ndim > 1:- audio = audio.mean(axis=1)-- # Resample if needed- if sr != target_sr:- from scipy.signal import resample- num_samples = int(len(audio) * target_sr / sr)- audio = resample(audio, num_samples).astype(np.float32)- sr = target_sr-- return audio.astype(np.float32), sr+ pairs = []+ for idx in indices:+ ex = ds[idx]+ audio_val = ex.get("audio")+ if audio_val is None:+ continue+ # With decode=False, audio_val is {"bytes": b"...", "path": ...}+ raw_bytes = None+ if isinstance(audio_val, dict):+ raw_bytes = audio_val.get("bytes")+ elif isinstance(audio_val, bytes):+ raw_bytes = audio_val+ if raw_bytes is None:+ continue+ audio_array, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")+ if audio_array.ndim > 1:+ audio_array = audio_array.mean(axis=1)+ pairs.append({+ "query": ex["query"],+ "answers": ex["answers"],+ "tools": ex["tools"],+ "audio_array": audio_array.astype(np.float32),+ "sampling_rate": sr,+ })+ if max_samples and len(pairs) >= max_samples:+ break++ return pairsdef compute_mel_spectrogram(audio, sr=16000, n_mels=80, n_fft=400, hop_length=160):@@ -429,85 +494,168 @@ def _speech_cache_key(n_samples, n_mels, max_mel_len, max_dec_len):return hashlib.md5(key.encode()).hexdigest()[:12]-def prepare_speech_pairs(ds, tokenizer, n_mels=80, max_mel_len=1024, max_dec_len=128):- """Prepare speech encoder-decoder pairs with <transcribe> task token."""+def build_audio_augmenter(sr=16000):+ """Build an audiomentations augmentation pipeline for training.++ Returns an augmenter callable or None if audiomentations is unavailable.+ """+ try:+ import audiomentations as A+ except ImportError:+ print(" WARNING: audiomentations not installed — no waveform augmentation")+ return None++ return A.Compose([+ A.AddGaussianSNR(min_snr_db=10.0, max_snr_db=35.0, p=0.5),+ A.AddGaussianNoise(min_amplitude=0.001, max_amplitude=0.015, p=0.3),+ A.TimeStretch(min_rate=0.9, max_rate=1.1, p=0.3),+ A.PitchShift(min_semitones=-2, max_semitones=2, p=0.3),+ A.Gain(min_gain_db=-6, max_gain_db=6, p=0.4),+ A.LowPassFilter(min_cutoff_freq=3000, max_cutoff_freq=7500, p=0.2),+ A.HighPassFilter(min_cutoff_freq=50, max_cutoff_freq=400, p=0.2),+ A.ClippingDistortion(min_percentile_threshold=0, max_percentile_threshold=5, p=0.1),+ ])+++def prepare_voice_tool_call_pairs(pairs, tokenizer, n_mels=80, max_mel_len=1024, max_dec_len=1024):+ """Prepare voice-tool-call decoder arrays with <tool_call> task token.++ Caches only decoder tokens + loss mask + audio arrays (NOT mels).+ Mel spectrograms are computed on-the-fly by get_speech_batches to allow+ online waveform augmentation during training.++ Returns (audio_arrays, dec_inputs, dec_targets, loss_mask).+ """- cache_id = _speech_cache_key(len(ds), n_mels, max_mel_len, max_dec_len)+ cache_id = _speech_cache_key(len(pairs), n_mels, max_mel_len, max_dec_len)cache_path = os.path.join(CACHE_DIR, cache_id)- if os.path.exists(cache_path + "_mel.npy"):- print(f"Loading cached speech data ({cache_id})...")+ arrays_file = cache_path + "_arrays.npy"+ if os.path.exists(arrays_file):+ print(f"Loading cached voice-tool-call data ({cache_id})...")return (- np.load(cache_path + "_mel.npy"),+ np.load(arrays_file, allow_pickle=True),np.load(cache_path + "_dec_in.npy"),np.load(cache_path + "_dec_tgt.npy"),+ np.load(cache_path + "_loss_mask.npy"),)pad_id = tokenizer.pad_token_id- bos_id = tokenizer.eos_token_ideos_id = tokenizer.eos_token_id- transcribe_id = tokenizer.transcribe_token_id+ tool_call_id = tokenizer.tool_call_token_id- mel_features = []+ audio_arrays = []dec_inputs_list = []dec_targets_list = []+ loss_mask_list = []- for example in tqdm(ds, desc="Processing speech"):- audio, sr = _decode_audio(example["audio"])-- mel = compute_mel_spectrogram(audio, sr=sr, n_mels=n_mels)+ skipped = 0+ for pair in tqdm(pairs, desc="Processing voice-tool-call"):+ t_tok = tokenizer.encode(pair["tools"])[:max_dec_len - 2]+ a_tok = tokenizer.encode(pair["answers"])- # Pad or truncate mel to max_mel_len- if mel.shape[0] > max_mel_len:- mel = mel[:max_mel_len]- elif mel.shape[0] < max_mel_len:- pad_len = max_mel_len - mel.shape[0]- mel = np.pad(mel, ((0, pad_len), (0, 0)))+ prefix_len = 2 + len(t_tok)+ total_dec = prefix_len + len(a_tok) + 1- # Tokenize transcript- text = example["text"].strip().lower()- tokens = tokenizer.encode(text)- # Leave room for <transcribe> prefix and EOS- tokens = tokens[:max_dec_len - 2]+ if total_dec > max_dec_len:+ available_for_tools = max_dec_len - 2 - len(a_tok) - 1+ if available_for_tools < 1:+ skipped += 1+ continue+ t_tok = t_tok[:available_for_tools]+ prefix_len = 2 + len(t_tok)- # Decoder input: [BOS, <transcribe>, tok1, tok2, ...]dec_in = np.full(max_dec_len, pad_id, dtype=np.int32)- dec_in[0] = bos_id- dec_in[1] = transcribe_id- n_tok = len(tokens)- if n_tok > 0:- dec_in[2:2 + n_tok] = tokens+ dec_in[0] = eos_id+ dec_in[1] = tool_call_id+ tl = len(t_tok)+ if tl > 0:+ dec_in[2:2 + tl] = t_tok+ al = len(a_tok)+ if al > 0:+ dec_in[prefix_len:prefix_len + al] = a_tok- # Decoder target: [<transcribe>, tok1, tok2, ..., EOS]+ # Decoder target: [<tool_call>, tools..., answer..., EOS]dec_tgt = np.full(max_dec_len, pad_id, dtype=np.int32)- dec_tgt[0] = transcribe_id- if n_tok > 0:- dec_tgt[1:1 + n_tok] = tokens- dec_tgt[1 + n_tok] = eos_id+ dec_tgt[0] = tool_call_id+ if tl > 0:+ dec_tgt[1:1 + tl] = t_tok+ if al > 0:+ dec_tgt[prefix_len - 1:prefix_len - 1 + al] = a_tok+ dec_tgt[prefix_len - 1 + al] = eos_id- mel_features.append(mel)+ # Loss mask: 1 on answer positions + EOS only+ lm = np.zeros(max_dec_len, dtype=np.float32)+ lm[prefix_len - 1:prefix_len - 1 + al + 1] = 1.0++ audio_arrays.append(pair["audio_array"])dec_inputs_list.append(dec_in)dec_targets_list.append(dec_tgt)+ loss_mask_list.append(lm)- mel_features = np.stack(mel_features).astype(np.float32)+ if skipped > 0:+ print(f" Skipped {skipped} examples (too long for max_dec_len={max_dec_len})")++ audio_arrays = np.array(audio_arrays, dtype=object)dec_inputs = np.stack(dec_inputs_list)dec_targets = np.stack(dec_targets_list)+ loss_mask = np.stack(loss_mask_list)os.makedirs(CACHE_DIR, exist_ok=True)- np.save(cache_path + "_mel.npy", mel_features)+ np.save(arrays_file, audio_arrays)np.save(cache_path + "_dec_in.npy", dec_inputs)np.save(cache_path + "_dec_tgt.npy", dec_targets)- print(f"Cached {len(mel_features):,} speech pairs to {CACHE_DIR}/{cache_id}")+ np.save(cache_path + "_loss_mask.npy", loss_mask)+ print(f"Cached {len(audio_arrays):,} voice-tool-call pairs to {CACHE_DIR}/{cache_id}")- return mel_features, dec_inputs, dec_targets+ return audio_arrays, dec_inputs, dec_targets, loss_mask-def get_speech_batches(mel_features, dec_inputs, dec_targets, batch_size, shuffle=True):- n = len(mel_features)- if shuffle:- perm = np.random.permutation(n)- mel_features = mel_features[perm]- dec_inputs = dec_inputs[perm]- dec_targets = dec_targets[perm]+def _load_mel_batch(audio_arrays, n_mels, max_mel_len, augmenter=None, sr=16000):+ """Compute mel spectrograms for a batch of audio arrays.++ If augmenter is provided, applies waveform augmentation before mel computation.+ """+ mels = []+ for audio in audio_arrays:+ audio = np.array(audio, dtype=np.float32)+ if audio.ndim > 1:+ audio = audio.mean(axis=1)++ if augmenter is not None:+ audio = augmenter(samples=audio, sample_rate=sr)++ mel = compute_mel_spectrogram(audio, sr=sr, n_mels=n_mels)++ if mel.shape[0] > max_mel_len:+ mel = mel[:max_mel_len]+ elif mel.shape[0] < max_mel_len:+ pad_len = max_mel_len - mel.shape[0]+ mel = np.pad(mel, ((0, pad_len), (0, 0)))++ mels.append(mel)++ return np.stack(mels).astype(np.float32)+++def get_speech_batches(audio_arrays, dec_inputs, dec_targets, batch_size,+ shuffle=True, loss_mask=None, n_mels=80, max_mel_len=1024,+ augmenter=None):+ """Yield speech batches with on-the-fly mel computation and optional augmentation.++ Args:+ audio_arrays: array of audio waveform arrays (numpy float32)+ dec_inputs, dec_targets, loss_mask: pre-tokenized decoder arrays+ augmenter: audiomentations Compose pipeline (None = no augmentation)+ """+ n = len(audio_arrays)+ indices = np.random.permutation(n) if shuffle else np.arange(n)for i in range(0, n - batch_size + 1, batch_size):- yield mel_features[i : i + batch_size], dec_inputs[i : i + batch_size], dec_targets[i : i + batch_size]+ batch_idx = indices[i : i + batch_size]+ batch_audio = audio_arrays[batch_idx]+ mel_batch = _load_mel_batch(batch_audio, n_mels, max_mel_len, augmenter=augmenter)++ batch = (mel_batch, dec_inputs[batch_idx], dec_targets[batch_idx])+ if loss_mask is not None:+ batch = batch + (loss_mask[batch_idx],)+ yield batchdiff --git a/src/evaluate.py b/src/evaluate.pyindex 39a8a7c..911b853 100644--- a/src/evaluate.py+++ b/src/evaluate.py@@ -1,6 +1,5 @@import argparseimport math-import pickleimport jaximport jax.numpy as jnp@@ -15,14 +14,7 @@ from .model import (make_causal_mask,make_padding_mask,)---def load_checkpoint(path):- with open(path, "rb") as f:- data = pickle.load(f)- params = jax.tree.map(jnp.array, data["params"])- config = TransformerConfig(**data["config"])- return params, config+from .run import load_checkpointdef _make_p_encode(model):@@ -36,41 +28,27 @@ def _make_p_encode(model):def _make_p_decode(model):"""Create a pmap'd decode function."""- def _decode(params, dec_input, encoder_out, tgt_mask, cross_mask):+ def _decode(params, dec_input, encoder_out, tgt_mask, _unused_cross_mask):return model.apply({"params": params}, dec_input, encoder_out,- self_mask=tgt_mask, cross_mask=cross_mask, method="decode",+ self_mask=tgt_mask, method="decode",)return jax.pmap(_decode, axis_name="batch")-def _make_p_forward(model):- """Create a pmap'd full forward function."""- def _forward(params, src, tgt_in, src_mask, tgt_mask, cross_mask):- return model.apply(- {"params": params}, src, tgt_in,- src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,- )- return jax.pmap(_forward, axis_name="batch")--def _shard_single(x, num_devices):"""Replicate a single-sample batch across all devices for pmap."""return jnp.broadcast_to(x, (num_devices, *x.shape[1:]))-def _shard_batch(x, num_devices):- """Reshape batch for pmap: (N, ...) -> (num_devices, per_device, ...)."""- return x.reshape(num_devices, -1, *x.shape[1:])---def score_sequence(model, params, enc_tokens, dec_tokens, pad_id, p_encode=None, p_decode=None, num_devices=1):+def score_sequence(model, params, enc_tokens, dec_tokens, pad_id, sos_id=None,+ p_encode=None, p_decode=None, num_devices=1):"""Compute average negative log-likelihood of dec_tokens given enc_tokens."""+ sos = sos_id if sos_id is not None else pad_idenc_input = jnp.array([enc_tokens])src_mask = make_padding_mask(enc_input, pad_id)if p_encode is not None and num_devices > 1:- # Replicate single sample across devices, run pmap, take first resultenc_s = _shard_single(enc_input, num_devices)src_mask_s = _shard_single(src_mask, num_devices)encoder_out = p_encode(params, enc_s, src_mask_s)[0:1]@@ -80,22 +58,20 @@ def score_sequence(model, params, enc_tokens, dec_tokens, pad_id, p_encode=None,{"params": p}, enc_input, src_mask=src_mask, method="encode",)- dec_in = [pad_id] + list(dec_tokens[:-1])+ dec_in = [sos] + list(dec_tokens[:-1])dec_input = jnp.array([dec_in])tgt_mask = make_causal_mask(len(dec_in))- cross_mask = src_maskif p_decode is not None and num_devices > 1:dec_s = _shard_single(dec_input, num_devices)tgt_mask_s = jnp.broadcast_to(tgt_mask, (num_devices, *tgt_mask.shape[1:]))- cross_mask_s = _shard_single(cross_mask, num_devices)enc_out_s = _shard_single(encoder_out, num_devices)- logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, cross_mask_s)[0]+ logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, None)[0]else:p = params if num_devices <= 1 else jax_utils.unreplicate(params)logits = model.apply({"params": p}, dec_input, encoder_out,- self_mask=tgt_mask, cross_mask=cross_mask, method="decode",+ self_mask=tgt_mask, method="decode",)[0]log_probs = jax.nn.log_softmax(logits if logits.ndim == 2 else logits[0])@@ -110,6 +86,7 @@ def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256,ds = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="test")pad_id = tokenizer.pad_token_id+ sos_id = tokenizer.eos_token_idtotal_nll = 0.0total_tokens = 0evaluated = 0@@ -142,20 +119,19 @@ def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256,{"params": single_params}, enc_input, src_mask=src_mask, method="encode",)- dec_in = [pad_id] + list(dec_tokens[:-1])+ dec_in = [sos_id] + list(dec_tokens[:-1])dec_input = jnp.array([dec_in])tgt_mask = make_causal_mask(len(dec_in))if p_decode is not None and num_devices > 1:dec_s = _shard_single(dec_input, num_devices)tgt_mask_s = jnp.broadcast_to(tgt_mask, (num_devices, *tgt_mask.shape[1:]))- cross_mask_s = _shard_single(src_mask, num_devices)enc_out_s = _shard_single(encoder_out, num_devices)- logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, cross_mask_s)[0]+ logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, None)[0]else:logits = model.apply({"params": single_params}, dec_input, encoder_out,- self_mask=tgt_mask, cross_mask=src_mask, method="decode",+ self_mask=tgt_mask, method="decode",)log_probs = jax.nn.log_softmax(logits[0] if logits.ndim == 3 else logits[0])@@ -179,6 +155,7 @@ def eval_lambada(model, params, tokenizer, max_samples=500,ds = load_dataset("EleutherAI/lambada_openai", "default", split="test")pad_id = tokenizer.pad_token_id+ sos_id = tokenizer.eos_token_idcorrect = 0total = 0@@ -209,19 +186,18 @@ def eval_lambada(model, params, tokenizer, max_samples=500,{"params": single_params}, enc_input, src_mask=src_mask, method="encode",)- dec_in = jnp.array([[pad_id]])+ dec_in = jnp.array([[sos_id]])tgt_mask = make_causal_mask(1)if p_decode is not None and num_devices > 1:dec_s = _shard_single(dec_in, num_devices)tgt_mask_s = jnp.broadcast_to(tgt_mask, (num_devices, *tgt_mask.shape[1:]))- cross_mask_s = _shard_single(src_mask, num_devices)enc_out_s = _shard_single(encoder_out, num_devices)- logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, cross_mask_s)[0]+ logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, None)[0]else:logits = model.apply({"params": single_params}, dec_in, encoder_out,- self_mask=tgt_mask, cross_mask=src_mask, method="decode",+ self_mask=tgt_mask, method="decode",)predicted = int(jnp.argmax(logits[0, 0] if logits.ndim == 3 else logits[0, 0]))@@ -242,6 +218,7 @@ def eval_hellaswag(model, params, tokenizer, max_samples=500,ds = load_dataset("Rowan/hellaswag", split="validation")pad_id = tokenizer.pad_token_id+ sos_id = tokenizer.eos_token_idcorrect = 0total = 0@@ -254,7 +231,6 @@ def eval_hellaswag(model, params, tokenizer, max_samples=500,if not enc_tokens:continue- # Truncate context to keep it manageableenc_tokens = enc_tokens[:192]scores = []@@ -265,7 +241,8 @@ def eval_hellaswag(model, params, tokenizer, max_samples=500,continuedec_tokens = dec_tokens[:64]score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id,- p_encode=p_encode, p_decode=p_decode, num_devices=num_devices)+ sos_id=sos_id, p_encode=p_encode, p_decode=p_decode,+ num_devices=num_devices)scores.append(score)predicted = int(np.argmax(scores))@@ -286,6 +263,7 @@ def eval_arc_easy(model, params, tokenizer, max_samples=500,ds = load_dataset("allenai/ai2_arc", "ARC-Easy", split="test")pad_id = tokenizer.pad_token_id+ sos_id = tokenizer.eos_token_idcorrect = 0total = 0@@ -309,7 +287,8 @@ def eval_arc_easy(model, params, tokenizer, max_samples=500,continuedec_tokens = dec_tokens[:64]score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id,- p_encode=p_encode, p_decode=p_decode, num_devices=num_devices)+ sos_id=sos_id, p_encode=p_encode, p_decode=p_decode,+ num_devices=num_devices)scores.append(score)predicted_idx = int(np.argmax(scores))diff --git a/src/export.py b/src/export.pyindex add15d7..8b5d00e 100644--- a/src/export.py+++ b/src/export.py@@ -15,7 +15,6 @@ import numpy as npfrom .model import TransformerConfig-# FFN kernel names that should be sliced in the d_ff dimension_FFN_KERNEL_NAMES = {"gate_proj", "up_proj", "down_proj"}@@ -42,7 +41,6 @@ def export_submodel(checkpoint_path, factor, output_path):if arr.ndim != 2:return arr- # Check if this is an FFN kernel by looking at parent module nameparent_name = Nonefor part in key_path:name = part.key if hasattr(part, "key") else str(part)@@ -53,15 +51,11 @@ def export_submodel(checkpoint_path, factor, output_path):if parent_name is None:return arr- # Both token_mix and channel_mix are sliced via ffn_mask-rows, cols = arr.shapeif parent_name in ("gate_proj", "up_proj"):- # (d_in, d_ff) → (d_in, d_ff_new)if cols == d_ff:return arr[:, :d_ff_new]elif parent_name == "down_proj":- # (d_ff, d_out) → (d_ff_new, d_out)if rows == d_ff:return arr[:d_ff_new, :]diff --git a/src/model.py b/src/model.pyindex 7abbd21..9a16661 100644--- a/src/model.py+++ b/src/model.py@@ -292,26 +292,6 @@ class Decoder(nn.Module):return x-class ConvSubsampling(nn.Module):- """Whisper-style convolutional frontend: two Conv1d layers with GELU, 4x temporal downsampling."""- d_model: int- n_mels: int- dtype: jnp.dtype = jnp.bfloat16-- @nn.compact- def __call__(self, mel):- # mel: (B, T, n_mels)- x = nn.Conv(features=self.d_model, kernel_size=(3,), strides=(2,),- padding='SAME', use_bias=True, kernel_init=default_init(),- dtype=self.dtype, name="conv1")(mel)- x = nn.gelu(x)- x = nn.Conv(features=self.d_model, kernel_size=(3,), strides=(2,),- padding='SAME', use_bias=True, kernel_init=default_init(),- dtype=self.dtype, name="conv2")(x)- x = nn.gelu(x)- return x--class SpecAugment(nn.Module):"""SpecAugment: time and frequency masking for speech regularization."""num_time_masks: int = 2@@ -354,7 +334,8 @@ class EncoderDecoderTransformer(nn.Module):def setup(self):self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))self.embed_scale = math.sqrt(self.config.d_model)- self.conv_subsample = ConvSubsampling(self.config.d_model, self.config.n_mels, self.config.jax_dtype)+ self.mel_proj = nn.Dense(self.config.d_model, dtype=self.config.jax_dtype,+ use_bias=True, kernel_init=default_init(), name="mel_proj")self.spec_augment = SpecAugment()self.encoder = MemoryMixerEncoder(self.config)self.decoder = Decoder(self.config)@@ -372,40 +353,23 @@ class EncoderDecoderTransformer(nn.Module):"""Backward-compatible alias for encode_text."""return self.encode_text(src, src_mask=src_mask)- @staticmethod- def _downsample_mask(mask, factor):- """Downsample a padding mask by OR-pooling groups of `factor` frames."""- if mask is None:- return None- # mask: (B, 1, 1, T)- T = mask.shape[-1]- pad = (factor - T % factor) % factor- if pad > 0:- mask = jnp.pad(mask, ((0, 0), (0, 0), (0, 0), (0, pad)))- mask = mask.reshape(mask.shape[0], 1, 1, -1, factor).any(axis=-1)- return mask-def encode_speech(self, mel, src_mask=None, ffn_mask=None, deterministic=True):mel = self.spec_augment(mel, deterministic=deterministic)- x = self.conv_subsample(mel) * self.embed_scale- src_mask = self._downsample_mask(src_mask, 4)+ x = self.mel_proj(mel) * self.embed_scalerope = self._rope(x.shape[1])return self.encoder(x, mask=src_mask, rope=rope, ffn_mask=ffn_mask)- def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None):+ def decode(self, tgt, encoder_out, self_mask=None):+ """Decode from encoder memory slots. No cross_mask needed (fixed-size slots)."""x = self.embedding(tgt) * self.embed_scalerope = self._rope(tgt.shape[1])- x = self.decoder(- x, encoder_out, self_mask=self_mask, cross_mask=None, rope=rope- )+ x = self.decoder(x, encoder_out, self_mask=self_mask, cross_mask=None, rope=rope)logits = x.astype(jnp.float32) @ self.embedding.embedding.Treturn logits- def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):+ def __call__(self, src, tgt, src_mask=None, tgt_mask=None):encoder_out = self.encode_text(src, src_mask=src_mask)- logits = self.decode(- tgt, encoder_out, self_mask=tgt_mask- )+ logits = self.decode(tgt, encoder_out, self_mask=tgt_mask)return logitsdef _run_decoder(self, encoder_out, tgt, tgt_mask=None, ffn_mask=None):@@ -442,7 +406,7 @@ class EncoderDecoderTransformer(nn.Module):mask = (jnp.arange(self.config.d_ff) < ff_width).astype(dtype)return jnp.broadcast_to(mask[None, :], (B, self.config.d_ff))- def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mat_ff_widths=None):+ def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, mat_ff_widths=None):"""Eval-only: separate per-width forwards for reporting per-width PPL.mat_ff_widths: list of FFN widths to evaluate (e.g. [1024, 512, 256]).diff --git a/src/run.py b/src/run.pyindex 0dba654..a9a00ba 100644--- a/src/run.py+++ b/src/run.py@@ -55,13 +55,12 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=dec_buffer = dec_buffer.at[0, j].set(tok)@jax.jit- def decode_step(dec_buffer, encoder_out, src_mask):+ def decode_step(dec_buffer, encoder_out):logits = model.apply({"params": params},dec_buffer,encoder_out,self_mask=tgt_mask,- cross_mask=src_mask,method="decode",)return logits@@ -72,10 +71,8 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=sys.stdout.write(f"\n")sys.stdout.flush()- # Compile once on first call- logits = decode_step(dec_buffer, encoder_out, src_mask)+ logits = decode_step(dec_buffer, encoder_out)- # Greedy decoding from end of prefixfor i in range(prefix_len - 1, max_gen_len - 1):next_logits = logits[0, i]next_token = int(jnp.argmax(next_logits))@@ -90,7 +87,7 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=sys.stdout.write(tokenizer.decode([next_token]))sys.stdout.flush()- logits = decode_step(dec_buffer, encoder_out, src_mask)+ logits = decode_step(dec_buffer, encoder_out)if stream:sys.stdout.write("\n")@@ -114,52 +111,57 @@ def load_audio(path, target_sr=16000):return audio, sr-def transcribe(model, params, tokenizer, audio_array, sr=16000, max_gen_len=128, temperature=0.8, seed=0, stream=True):- """Transcribe audio using the speech encoder pathway."""+def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="[]", max_gen_len=512, seed=0, stream=True):+ """Generate tool-call output from audio using the speech encoder pathway.++ mel -> encode_speech -> decoder [BOS, <tool_call>, tools_tokens...] -> greedy decode.+ Same structure as generate() but uses speech encoder + mel padding mask.+ """n_mels = model.config.n_melsmel = compute_mel_spectrogram(audio_array, sr=sr, n_mels=n_mels)mel_input = jnp.array(mel)[None, :, :] # (1, T_mel, n_mels)pad_id = tokenizer.pad_token_ideos_id = tokenizer.eos_token_id- transcribe_id = tokenizer.transcribe_token_id+ tool_call_id = tokenizer.tool_call_token_idsrc_mask = make_mel_padding_mask(mel_input)encoder_out = model.apply({"params": params}, mel_input, src_mask=src_mask, deterministic=True, method="encode_speech")+ # Build decoder prefix: [BOS, <tool_call>, tools_tokens...]+ tools_tokens = tokenizer.encode(tools)+ prefix = [eos_id, tool_call_id] + tools_tokens+ prefix_len = len(prefix)+tgt_mask = make_causal_mask(max_gen_len)dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)- dec_buffer = dec_buffer.at[0, 0].set(eos_id) # BOS = EOS in this tokenizer- dec_buffer = dec_buffer.at[0, 1].set(transcribe_id)+ for j, tok in enumerate(prefix[:max_gen_len]):+ dec_buffer = dec_buffer.at[0, j].set(tok)@jax.jit- def decode_step(dec_buffer, encoder_out, src_mask):+ def decode_step(dec_buffer, encoder_out):logits = model.apply({"params": params},dec_buffer,encoder_out,self_mask=tgt_mask,- cross_mask=src_mask,method="decode",)return logits- rng = jax.random.PRNGKey(seed)generated_tokens = []if stream:sys.stdout.write("\n")sys.stdout.flush()- logits = decode_step(dec_buffer, encoder_out, src_mask)+ logits = decode_step(dec_buffer, encoder_out)- # Start generating from position 2 (after BOS + <transcribe>)- for i in range(1, max_gen_len - 1):- next_logits = logits[0, i] / temperature- rng, sample_rng = jax.random.split(rng)- next_token = jax.random.categorical(sample_rng, next_logits).item()+ for i in range(prefix_len - 1, max_gen_len - 1):+ next_logits = logits[0, i]+ next_token = int(jnp.argmax(next_logits))if next_token == eos_id:break@@ -171,7 +173,7 @@ def transcribe(model, params, tokenizer, audio_array, sr=16000, max_gen_len=128,sys.stdout.write(tokenizer.decode([next_token]))sys.stdout.flush()- logits = decode_step(dec_buffer, encoder_out, src_mask)+ logits = decode_step(dec_buffer, encoder_out)if stream:sys.stdout.write("\n")@@ -189,20 +191,22 @@ def main(args):param_count = sum(x.size for x in jax.tree.leaves(params))print(f"Model parameters: {param_count:,}")- # --- Audio transcription mode ---+ # --- Voice-to-tool-call mode ---audio_files = getattr(args, "audio", None)if audio_files:+ tools = getattr(args, "tools", None) or "[]"for i, audio_path in enumerate(audio_files):- print(f"\nTranscribing: {audio_path}")+ print(f"\nVoice-to-tool-call: {audio_path}")+ print(f"Tools: {tools[:80]}{'...' if len(tools) > 80 else ''}")audio, sr = load_audio(audio_path)- transcribe(+ generate_from_audio(model,params,tokenizer,audio,sr=sr,+ tools=tools,max_gen_len=args.max_len,- temperature=args.temperature,seed=args.seed + i,stream=True,)@@ -241,7 +245,7 @@ def parse_args():parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")parser.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")parser.add_argument("--tools", type=str, default=None, help="Tools JSON for tool-call generation")- parser.add_argument("--audio", type=str, nargs="*", help="Audio file paths to transcribe")+ parser.add_argument("--audio", type=str, nargs="*", help="Audio file paths for voice-to-tool-call")parser.add_argument("--max-len", type=int, default=512)parser.add_argument("--seed", type=int, default=0)return parser.parse_args()diff --git a/src/test.py b/src/test.pyindex 515b516..b454a97 100644--- a/src/test.py+++ b/src/test.py@@ -1,6 +1,5 @@import argparseimport math-import pickleimport timeimport jax@@ -8,21 +7,14 @@ import jax.numpy as jnpimport numpy as npimport optax-from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs, load_librispeech+from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs, load_tool_call_audiofrom .model import (EncoderDecoderTransformer,TransformerConfig,make_causal_mask,make_padding_mask,)---def load_checkpoint(path):- with open(path, "rb") as f:- data = pickle.load(f)- params = jax.tree.map(jnp.array, data["params"])- config = TransformerConfig(**data["config"])- return params, config+from .run import load_checkpointdef compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batch_size, pad_id, loss_mask=None):@@ -41,11 +33,10 @@ def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batchsrc_mask = make_padding_mask(src, pad_id)tgt_mask = make_causal_mask(tgt_in.shape[1]) & make_padding_mask(tgt_in, pad_id)- cross_mask = make_padding_mask(src, pad_id)logits = model.apply({"params": params}, src, tgt_in,- src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,+ src_mask=src_mask, tgt_mask=tgt_mask,)loss = optax.softmax_cross_entropy_with_integer_labels(logits, tgt_out)@@ -66,22 +57,20 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is thtgt_mask = make_causal_mask(max_gen_len)@jax.jit- def decode_step(dec_buffer, encoder_out, src_mask):+ def decode_step(dec_buffer, encoder_out):logits = model.apply({"params": params}, dec_buffer, encoder_out,- self_mask=tgt_mask, cross_mask=src_mask, method="decode",+ self_mask=tgt_mask, method="decode",)return logits- # Warmup — compile onceencoder_out = model.apply({"params": params}, enc_input, src_mask=src_mask, method="encode")dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)dec_buffer = dec_buffer.at[0, 0].set(eos_id)- decode_step(dec_buffer, encoder_out, src_mask)+ decode_step(dec_buffer, encoder_out)- # Benchmark generationtokens_generated = []latencies = []@@ -94,7 +83,7 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is thencoder_out = model.apply({"params": params}, enc_input, src_mask=src_mask, method="encode")- logits = decode_step(dec_buffer, encoder_out, src_mask)+ logits = decode_step(dec_buffer, encoder_out)num_tokens = 0for i in range(max_gen_len - 1):@@ -106,7 +95,7 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is thnum_tokens += 1dec_buffer = dec_buffer.at[0, i + 1].set(next_token)- logits = decode_step(dec_buffer, encoder_out, src_mask)+ logits = decode_step(dec_buffer, encoder_out)elapsed = time.perf_counter() - starttokens_generated.append(num_tokens)@@ -196,11 +185,9 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=total = 0exact_match = 0- # For function-name-level P/R/F1tp_names = 0 # predicted name in referencefp_names = 0 # predicted name not in referencefn_names = 0 # reference name not predicted- # For full call-level (name + args) P/R/F1tp_calls = 0fp_calls = 0fn_calls = 0@@ -218,13 +205,11 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=tools=ex["tools"], max_gen_len=max_gen_len, seed=i, stream=False,).strip()- # Parse referencetry:ref_calls = json.loads(ref_text)except (json.JSONDecodeError, TypeError):ref_calls = []- # Parse predictiontry:pred_calls = json.loads(pred_text)if not isinstance(pred_calls, list):@@ -237,17 +222,14 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=if i < 10:samples.append((ex["query"][:80], ref_text[:120], pred_text[:120]))- # Track emptyif len(ref_calls) == 0:empty_ref += 1if len(pred_calls) == 0:empty_pred += 1- # Exact match (normalized)if json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):exact_match += 1- # Function name P/R/F1ref_names = [c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c]pred_names = [c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c]ref_name_set = set(ref_names)@@ -256,7 +238,6 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=fp_names += len(pred_name_set - ref_name_set)fn_names += len(ref_name_set - pred_name_set)- # Full call P/R/F1 (name + args must match)def call_key(c):if not isinstance(c, dict):return None@@ -270,7 +251,6 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=fp_calls += len(pred_keys - ref_keys)fn_calls += len(ref_keys - pred_keys)- # Compute metricsname_precision = tp_names / max(tp_names + fp_names, 1)name_recall = tp_names / max(tp_names + fn_names, 1)name_f1 = 2 * name_precision * name_recall / max(name_precision + name_recall, 1e-9)@@ -295,32 +275,104 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=}-def benchmark_asr(model, params, tokenizer, num_samples=100):- """Transcribe LibriSpeech test-clean samples and report WER."""- from .run import transcribe- from .data import load_librispeech+def benchmark_voice_tool_calls(model, params, tokenizer, num_samples=100, max_gen_len=512):+ """Generate tool-call predictions from audio and compute structured metrics."""+ import json+ from .run import generate_from_audio- ds = load_librispeech("test.clean", max_samples=num_samples)+ pairs = load_tool_call_audio("validation", max_samples=num_samples)- hypotheses = []- references = []+ total = 0+ exact_match = 0+ tp_names = 0+ fp_names = 0+ fn_names = 0+ tp_calls = 0+ fp_calls = 0+ fn_calls = 0+ json_parse_errors = 0+ empty_ref = 0+ empty_pred = 0+ samples = []- for example in ds:- audio_data = example["audio"]- audio = np.array(audio_data["array"], dtype=np.float32)- sr = audio_data["sampling_rate"]- ref_text = example["text"].strip().lower()+ for i, pair in enumerate(pairs):+ ref_text = pair["answers"]+ audio = pair["audio_array"]+ sr = pair["sampling_rate"]- hyp_text = transcribe(+ pred_text = generate_from_audio(model, params, tokenizer, audio, sr=sr,- max_gen_len=128, temperature=0.0, seed=0, stream=False,- ).strip().lower()+ tools=pair["tools"], max_gen_len=max_gen_len, seed=i, stream=False,+ ).strip()++ try:+ ref_calls = json.loads(ref_text)+ except (json.JSONDecodeError, TypeError):+ ref_calls = []++ try:+ pred_calls = json.loads(pred_text)+ if not isinstance(pred_calls, list):+ pred_calls = [pred_calls] if isinstance(pred_calls, dict) else []+ except (json.JSONDecodeError, TypeError):+ json_parse_errors += 1+ pred_calls = []- hypotheses.append(hyp_text)- references.append(ref_text)+ total += 1+ if i < 10:+ samples.append((pair["query"][:80], ref_text[:120], pred_text[:120]))++ if len(ref_calls) == 0:+ empty_ref += 1+ if len(pred_calls) == 0:+ empty_pred += 1- wer = compute_wer(hypotheses, references)- return {"wer": wer, "num_samples": len(hypotheses)}+ if json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):+ exact_match += 1++ ref_names = [c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c]+ pred_names = [c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c]+ ref_name_set = set(ref_names)+ pred_name_set = set(pred_names)+ tp_names += len(pred_name_set & ref_name_set)+ fp_names += len(pred_name_set - ref_name_set)+ fn_names += len(ref_name_set - pred_name_set)++ def call_key(c):+ if not isinstance(c, dict):+ return None+ return json.dumps({"name": c.get("name"), "arguments": c.get("arguments")}, sort_keys=True)++ ref_keys = set(call_key(c) for c in ref_calls)+ pred_keys = set(call_key(c) for c in pred_calls)+ ref_keys.discard(None)+ pred_keys.discard(None)+ tp_calls += len(pred_keys & ref_keys)+ fp_calls += len(pred_keys - ref_keys)+ fn_calls += len(ref_keys - pred_keys)++ name_precision = tp_names / max(tp_names + fp_names, 1)+ name_recall = tp_names / max(tp_names + fn_names, 1)+ name_f1 = 2 * name_precision * name_recall / max(name_precision + name_recall, 1e-9)++ call_precision = tp_calls / max(tp_calls + fp_calls, 1)+ call_recall = tp_calls / max(tp_calls + fn_calls, 1)+ call_f1 = 2 * call_precision * call_recall / max(call_precision + call_recall, 1e-9)++ return {+ "num_samples": total,+ "exact_match": exact_match / max(total, 1),+ "json_parse_rate": 1.0 - json_parse_errors / max(total, 1),+ "name_precision": name_precision,+ "name_recall": name_recall,+ "name_f1": name_f1,+ "call_precision": call_precision,+ "call_recall": call_recall,+ "call_f1": call_f1,+ "empty_ref_pct": empty_ref / max(total, 1),+ "empty_pred_pct": empty_pred / max(total, 1),+ "samples": samples,+ }def main(args):@@ -371,11 +423,29 @@ def main(args):print(f" P: {pred}")print()- asr_samples = getattr(args, "asr_samples", 50)- if asr_samples > 0:- print(f"\nevaluating ASR WER ({asr_samples} samples)...")- asr_results = benchmark_asr(model, params, tokenizer, num_samples=asr_samples)- print(f"ASR WER: {asr_results['wer']:.4f} ({asr_results['num_samples']} samples)")+ voice_tc_samples = getattr(args, "voice_tc_samples", 50)+ if voice_tc_samples > 0:+ print(f"\nevaluating voice-to-tool-call ({voice_tc_samples} samples)...")+ vtc = benchmark_voice_tool_calls(model, params, tokenizer, num_samples=voice_tc_samples, max_gen_len=args.max_gen_len)+ print(f"\n ─── Voice-Tool-Call Metrics ─────────")+ print(f" JSON parse rate {vtc['json_parse_rate']:>10.1%}")+ print(f" Exact match {vtc['exact_match']:>10.1%}")+ print(f" ─── Function Name ───────────────────")+ print(f" Precision {vtc['name_precision']:>10.3f}")+ print(f" Recall {vtc['name_recall']:>10.3f}")+ print(f" F1 {vtc['name_f1']:>10.3f}")+ print(f" ─── Full Call (name+args) ───────────")+ print(f" Precision {vtc['call_precision']:>10.3f}")+ print(f" Recall {vtc['call_recall']:>10.3f}")+ print(f" F1 {vtc['call_f1']:>10.3f}")+ print(f" ─────────────────────────────────────")+ if vtc["samples"]:+ print(f"\n samples:")+ for query, ref, pred in vtc["samples"][:5]:+ print(f" Q: {query}")+ print(f" R: {ref}")+ print(f" P: {pred}")+ print()print(f"\nmeasuring throughput ({args.throughput_runs} runs)...")throughput = measure_throughput(model, params, tokenizer, num_runs=args.throughput_runs)@@ -396,6 +466,8 @@ def parse_args():parser.add_argument("--throughput-runs", type=int, default=10)parser.add_argument("--tool-call-samples", type=int, default=200,help="Samples for tool-call accuracy eval (default: 200)")+ parser.add_argument("--voice-tc-samples", type=int, default=50,+ help="Samples for voice-to-tool-call eval (default: 50)")return parser.parse_args()diff --git a/src/train.py b/src/train.pyindex 0484b4f..9e14141 100644--- a/src/train.py+++ b/src/train.py@@ -15,7 +15,8 @@ from flax.training import train_statefrom .data import (get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs,- load_librispeech, prepare_speech_pairs, get_speech_batches,+ load_tool_call_audio, prepare_voice_tool_call_pairs, get_speech_batches,+ build_audio_augmenter,)from .model import (EncoderDecoderTransformer,@@ -181,7 +182,6 @@ def _make_prune_mask(params, sparsity, group_size):Returns binary mask tree matching param shapes and dtypes."""- # Pass 1: collect block L1 scores as JAX arraysall_scores = []for _, leaf in jax.tree_util.tree_leaves_with_path(params):if leaf.ndim != 2:@@ -198,7 +198,6 @@ def _make_prune_mask(params, sparsity, group_size):threshold = jnp.percentile(jnp.concatenate(all_scores), sparsity * 100)- # Pass 2: build per-leaf binary mask (in param dtype to save memory)def _leaf_mask(path, leaf):if leaf.ndim != 2:return jnp.ones_like(leaf)@@ -225,8 +224,8 @@ def _quantize_params(params, group_size=32):_GROUP_SIZE = 32_MAT_FACTORS = ()-_MAT_FF_WIDTHS = () # precomputed d_ff widths per factor-_D_FF = 2048 # set in train()+_MAT_FF_WIDTHS = ()+_D_FF = 2048def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, loss_mask):@@ -249,7 +248,7 @@ def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, loreturn ce_loss + z_loss + div_loss-def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng):+def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask):pad_id = 0src_mask = make_mel_padding_mask(mel)tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)@@ -262,7 +261,7 @@ def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask, ffn_mask,rngs={"specaugment": rng},)logits_f32 = logits.astype(jnp.float32)- mask = (tgt_out != pad_id).astype(jnp.float32)+ mask = loss_maskce_loss = jnp.sum(optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask) / jnp.maximum(jnp.sum(mask), 1.0)@@ -318,10 +317,10 @@ def _train_step_text_masked(state, ema_params, src, tgt_in, tgt_out, causal_maskreturn state, ema_params, loss, grad_norm-def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng):+def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask):ema_decay = 0.999loss, grads = jax.value_and_grad(- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng)+ lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask))(state.params)grads = jax.lax.pmean(grads, axis_name="batch")loss = jax.lax.pmean(loss, axis_name="batch")@@ -331,11 +330,11 @@ def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask, ffnreturn state, ema_params, loss, grad_norm-def _train_step_speech_masked(state, ema_params, mel, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng):+def _train_step_speech_masked(state, ema_params, mel, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng, loss_mask):"""Speech training step with fused prune mask application."""ema_decay = 0.999loss, grads = jax.value_and_grad(- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng)+ lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask))(state.params)grads = jax.lax.pmean(grads, axis_name="batch")loss = jax.lax.pmean(loss, axis_name="batch")@@ -371,7 +370,7 @@ def _make_val_loss_fn(apply_fn):tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)logits = apply_fn({"params": params}, src, tgt_in,- src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=src_mask,+ src_mask=src_mask, tgt_mask=tgt_mask,)loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)return jnp.sum(loss * loss_mask), jnp.sum(loss_mask)@@ -387,7 +386,7 @@ def _make_mat_val_loss_fn(apply_fn, ff_width):tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)logits, _, mat_logits = apply_fn({"params": params}, src, tgt_in,- src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=src_mask,+ src_mask=src_mask, tgt_mask=tgt_mask,mat_ff_widths=(ff_width,),method="forward_with_aux",)@@ -399,7 +398,7 @@ def _make_mat_val_loss_fn(apply_fn, ff_width):def _make_speech_val_loss_fn(apply_fn):@jax.jit- def val_loss_batch(params, mel, tgt_in, tgt_out, causal_mask):+ def val_loss_batch(params, mel, tgt_in, tgt_out, causal_mask, loss_mask):pad_id = 0src_mask = make_mel_padding_mask(mel)tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)@@ -410,7 +409,7 @@ def _make_speech_val_loss_fn(apply_fn):method="forward_speech_with_aux",)loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)- mask = (tgt_out != pad_id).astype(jnp.float32)+ mask = loss_maskreturn jnp.sum(loss * mask), jnp.sum(mask)return val_loss_batch@@ -446,11 +445,8 @@ def shard_batch(batch, num_devices):def train(args):- import itertools-num_devices = jax.local_device_count()no_speech = getattr(args, "no_speech", False)- speech_every_override = getattr(args, "speech_every", None)n_mels = getattr(args, "n_mels", 80)max_mel_len = getattr(args, "max_mel_len", 1024)@@ -490,31 +486,35 @@ def train(args):val_loss_mask = np.array(val_loss_mask)print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs")- # --- Speech data ---- speech_mel = speech_dec_in = speech_dec_tgt = None- val_speech_mel = val_speech_dec_in = val_speech_dec_tgt = None+ speech_audio_arrays = speech_dec_in = speech_dec_tgt = speech_loss_mask_arr = None+ val_speech_audio_arrays = val_speech_dec_in = val_speech_dec_tgt = val_speech_loss_mask = None+ audio_augmenter = Noneif not no_speech:step_idx += 1max_speech_samples = getattr(args, "max_speech_samples", None)- print(f"\n[{step_idx}/{total_data_steps}] Loading LibriSpeech...")- speech_ds = load_librispeech("train.clean.100", max_samples=max_speech_samples)- speech_val_ds = load_librispeech("validation.clean", max_samples=getattr(args, "max_eval_samples", None))+ print(f"\n[{step_idx}/{total_data_steps}] Loading voice-tool-call audio...")+ speech_pairs = load_tool_call_audio("train", max_samples=max_speech_samples)+ speech_val_pairs = load_tool_call_audio("validation", max_samples=getattr(args, "max_eval_samples", None))step_idx += 1- print(f"\n[{step_idx}/{total_data_steps}] Preparing speech pairs...")- speech_mel, speech_dec_in, speech_dec_tgt = prepare_speech_pairs(- speech_ds, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len+ print(f"\n[{step_idx}/{total_data_steps}] Preparing voice-tool-call pairs...")+ speech_audio_arrays, speech_dec_in, speech_dec_tgt, speech_loss_mask_arr = prepare_voice_tool_call_pairs(+ speech_pairs, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len)- val_speech_mel, val_speech_dec_in, val_speech_dec_tgt = prepare_speech_pairs(- speech_val_ds, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len+ val_speech_audio_arrays, val_speech_dec_in, val_speech_dec_tgt, val_speech_loss_mask = prepare_voice_tool_call_pairs(+ speech_val_pairs, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len)- speech_mel = np.array(speech_mel)speech_dec_in = np.array(speech_dec_in)speech_dec_tgt = np.array(speech_dec_tgt)- val_speech_mel = np.array(val_speech_mel)+ speech_loss_mask_arr = np.array(speech_loss_mask_arr)val_speech_dec_in = np.array(val_speech_dec_in)val_speech_dec_tgt = np.array(val_speech_dec_tgt)- print(f" {len(speech_mel):,} train / {len(val_speech_mel):,} val speech pairs")+ val_speech_loss_mask = np.array(val_speech_loss_mask)+ print(f" {len(speech_audio_arrays):,} train / {len(val_speech_audio_arrays):,} val voice-tool-call pairs")++ audio_augmenter = build_audio_augmenter(sr=16000)+ if audio_augmenter is not None:+ print(f" Audio augmentation: enabled ({len(audio_augmenter.transforms)} transforms)")effective_batch_size = args.batch_size * num_devices@@ -562,18 +562,12 @@ def train(args):rng, init_rng = jax.random.split(rng)mat_shared_input = getattr(args, "mat_shared_input", False)- # With shared input, each unique sample is repeated n_widths times,- # so we fetch smaller batches but take more steps per epoch.unique_batch_size = effective_batch_size // n_widths if (mat_shared_input and n_widths > 1) else effective_batch_sizetext_batches_per_epoch = len(enc_inputs) // unique_batch_size- if speech_every_override is not None:- speech_every = speech_every_override- elif not no_speech and speech_mel is not None:- speech_batches_available = len(speech_mel) // effective_batch_size- speech_every = max(1, text_batches_per_epoch // max(1, speech_batches_available))+ if not no_speech and speech_audio_arrays is not None:+ speech_batches_per_epoch = len(speech_audio_arrays) // unique_batch_sizeelse:- speech_every = 1- speech_batches_per_epoch = text_batches_per_epoch // speech_every if (not no_speech and speech_mel is not None) else 0+ speech_batches_per_epoch = 0num_batches = text_batches_per_epoch + speech_batches_per_epochtotal_steps = num_batches * args.epochswarmup_steps = max(1, int(total_steps * args.warmup_ratio))@@ -605,7 +599,7 @@ def train(args):print(f" Activation {config.activation:>12}")print(f" Dtype {config.dtype:>12}")if not no_speech:- print(f" Speech every {speech_every} text steps")+ print(f" Speech {speech_batches_per_epoch} batches/epoch")print(f" n_mels {n_mels:>12}")print(f" max_mel_len {max_mel_len:>12}")else:@@ -627,8 +621,6 @@ def train(args):(num_devices, 1, args.max_dec_len, args.max_dec_len),)- # Precompute FFN mask for heterogeneous matryoshka training.- # Batch is split: 1/N full-width + 1/N per mat factor.text_ffn_mask = _make_ffn_mask(args.batch_size, config.d_ff, _MAT_FF_WIDTHS)text_ffn_mask = jnp.broadcast_to(text_ffn_mask[None, :, :],@@ -670,61 +662,73 @@ def train(args):text_losses = []speech_losses = []- text_batches = get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size, loss_mask=train_loss_mask)+ text_batches = list(get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size, loss_mask=train_loss_mask))++ speech_batch_list = []+ if not no_speech and speech_audio_arrays is not None:+ speech_batch_list = list(+ get_speech_batches(speech_audio_arrays, speech_dec_in, speech_dec_tgt, unique_batch_size,+ loss_mask=speech_loss_mask_arr, n_mels=n_mels, max_mel_len=max_mel_len,+ augmenter=audio_augmenter)+ )++ steps_this_epoch = len(text_batches) + len(speech_batch_list)+ text_idx = 0+ speech_idx = 0+ pbar = tqdm(range(steps_this_epoch), desc=f"Epoch {epoch + 1}/{args.epochs}")++ for step_i in pbar:+ t0 = time.perf_counter()- # Speech batches cycle via itertools.cycle- speech_iter = None- if not no_speech and speech_mel is not None:- speech_iter = iter(itertools.cycle(- get_speech_batches(speech_mel, speech_dec_in, speech_dec_tgt, unique_batch_size)- ))+ do_speech = (step_i % 2 == 1) and speech_idx < len(speech_batch_list)+ do_text = not do_speech and text_idx < len(text_batches)+ if not do_speech and not do_text:+ if text_idx < len(text_batches):+ do_text = True+ elif speech_idx < len(speech_batch_list):+ do_speech = True+ else:+ break- text_step_in_epoch = 0- pbar = tqdm(text_batches, total=text_batches_per_epoch, desc=f"Epoch {epoch + 1}/{args.epochs}")+ speech_loss_val = None+ step_grad_norm = None- for batch in pbar:- src, tgt_in, tgt_out, lm = batch- # --- Text step ---- t0 = time.perf_counter()+ if do_text:+ src, tgt_in, tgt_out, lm = text_batches[text_idx]+ text_idx += 1- if n_widths > 1 and mat_shared_input:- # Shared: each unique input sees all widths. Shard unique samples- # across devices, then tile within each device's portion so the- # ffn_mask width sections each get the same unique inputs.- per_width = args.batch_size // n_widths- def _tile_for_mat(arr):- # (unique_batch, ...) -> (devices, per_width, ...) -> tile -> (devices, batch, ...)- s = arr.reshape(num_devices, per_width, *arr.shape[1:])- s = np.tile(s, (1, n_widths) + (1,) * (arr.ndim - 1))- return s- src_b = _tile_for_mat(src)- tgt_in_b = _tile_for_mat(tgt_in)- tgt_out_b = _tile_for_mat(tgt_out)- lm_b = _tile_for_mat(lm)- else:- src_b = shard_batch(src, num_devices)- tgt_in_b = shard_batch(tgt_in, num_devices)- tgt_out_b = shard_batch(tgt_out, num_devices)- lm_b = shard_batch(lm, num_devices)-- if prune_mask is not None:- state, ema_params, loss, grad_norm = p_train_step_masked(- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, lm_b,- )- else:- state, ema_params, loss, grad_norm = p_train_step(- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, lm_b,- )+ if n_widths > 1 and mat_shared_input:+ per_width = args.batch_size // n_widths+ def _tile_for_mat(arr):+ s = arr.reshape(num_devices, per_width, *arr.shape[1:])+ return np.tile(s, (1, n_widths) + (1,) * (arr.ndim - 1))+ src_b = _tile_for_mat(src)+ tgt_in_b = _tile_for_mat(tgt_in)+ tgt_out_b = _tile_for_mat(tgt_out)+ lm_b = _tile_for_mat(lm)+ else:+ src_b = shard_batch(src, num_devices)+ tgt_in_b = shard_batch(tgt_in, num_devices)+ tgt_out_b = shard_batch(tgt_out, num_devices)+ lm_b = shard_batch(lm, num_devices)- text_loss_val = float(loss[0])- text_losses.append(text_loss_val)- global_step += 1- text_step_in_epoch += 1+ if prune_mask is not None:+ state, ema_params, loss, grad_norm = p_train_step_masked(+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, lm_b,+ )+ else:+ state, ema_params, loss, grad_norm = p_train_step(+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, lm_b,+ )- # --- Speech step (every speech_every text steps) ---- speech_loss_val = None- if speech_iter is not None and text_step_in_epoch % speech_every == 0:- mel_batch, sp_tgt_in, sp_tgt_out = next(speech_iter)+ text_loss_val = float(loss[0])+ text_losses.append(text_loss_val)+ step_grad_norm = float(grad_norm[0])+ global_step += 1++ else:+ mel_batch, sp_tgt_in, sp_tgt_out, sp_lm = speech_batch_list[speech_idx]+ speech_idx += 1if n_widths > 1 and mat_shared_input:per_width = args.batch_size // n_widths@@ -734,28 +738,30 @@ def train(args):mel_b = _tile_sp(mel_batch)sp_tgt_in_b = _tile_sp(sp_tgt_in)sp_tgt_out_b = _tile_sp(sp_tgt_out)+ sp_lm_b = _tile_sp(sp_lm)else:mel_b = shard_batch(mel_batch, num_devices)sp_tgt_in_b = shard_batch(sp_tgt_in, num_devices)sp_tgt_out_b = shard_batch(sp_tgt_out, num_devices)+ sp_lm_b = shard_batch(sp_lm, num_devices)- speech_ffn_mask = text_ffn_maskrng, spec_rng = jax.random.split(rng)spec_rngs = jax.random.split(spec_rng, num_devices)if prune_mask is not None:state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech_masked(- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, prune_mask, speech_ffn_mask, spec_rngs,+ state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, prune_mask, text_ffn_mask, spec_rngs, sp_lm_b,)else:state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech(- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, speech_ffn_mask, spec_rngs,+ state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, text_ffn_mask, spec_rngs, sp_lm_b,)speech_loss_val = float(sp_loss[0])speech_losses.append(speech_loss_val)+ step_grad_norm = float(sp_grad_norm[0])+ text_loss_val = text_losses[-1] if text_losses else float("nan")global_step += 1- # --- Gradual magnitude sparsification (epoch 1 only) ---if epoch == weight_prune_epoch and not gradual_sparsify_done:epoch_step += 1current_sparsity = _cubic_sparsity_schedule(epoch_step, t_start, t_end, sparsity_ratio)@@ -776,11 +782,10 @@ def train(args):vl, vt = val_loss_fn(_eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])total_loss += float(vl)total_toks += float(vt)- last_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))+ last_val_ppl = float(math.exp(min(total_loss / max(total_toks, 1), 20)))del _eval_params- # --- tqdm postfix (fixed keys to prevent bar flickering) ---postfix = {"speech_loss": f"{speech_loss_val:.4f}" if speech_loss_val is not None else "-","text_loss": f"{text_loss_val:.4f}",@@ -796,7 +801,7 @@ def train(args):if use_wandb:log_dict = {"train/text_loss": text_loss_val,- "train/grad_norm": float(grad_norm[0]),+ "train/grad_norm": step_grad_norm,"train/adam_lr": float(adam_schedule(global_step)),"train/muon_lr": float(muon_schedule(global_step)),"train/tokens_per_sec": tokens_per_batch / dt,@@ -810,17 +815,14 @@ def train(args):log_dict["val/text_ppl"] = last_val_pplwandb.log(log_dict)- # --- Lock mask after epoch 1 gradual sparsification ---if epoch == weight_prune_epoch and not gradual_sparsify_done:gradual_sparsify_done = Trueif prune_mask is None:- # Edge case: epoch had fewer steps than t_startema_unr = jax_utils.unreplicate(ema_params)mask = _make_prune_mask(ema_unr, sparsity_ratio, _GROUP_SIZE)del ema_unrprune_mask = jax_utils.replicate(mask)del mask- # Apply final mask to both params and EMA (EMA lags due to slow decay)state = state.replace(params=jax.tree.map(lambda w, m: w * m, state.params, prune_mask))ema_params = jax.tree.map(lambda w, m: w * m, ema_params, prune_mask)@@ -833,7 +835,7 @@ def train(args):epoch_avg_loss = sum(text_losses) / len(text_losses) if text_losses else float("nan")final_loss = text_losses[-1] if text_losses else float("nan")- final_ppl = math.exp(final_loss) if not math.isnan(final_loss) else float("nan")+ final_ppl = math.exp(min(final_loss, 20)) if not math.isnan(final_loss) else float("nan")eval_params = jax_utils.unreplicate(ema_params)val_causal = make_causal_mask(args.max_dec_len)@@ -842,13 +844,15 @@ def train(args):vl, vt = val_loss_fn(eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])total_loss += float(vl)total_toks += float(vt)- last_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))+ last_val_ppl = float(math.exp(min(total_loss / max(total_toks, 1), 20)))speech_val_ppl = None- if speech_vl_fn is not None and val_speech_mel is not None:+ if speech_vl_fn is not None and val_speech_audio_arrays is not None:sp_total_loss, sp_total_toks = 0.0, 0.0- for sp_batch in get_speech_batches(val_speech_mel, val_speech_dec_in, val_speech_dec_tgt, args.batch_size, shuffle=False):- vl, vt = speech_vl_fn(eval_params, sp_batch[0], sp_batch[1], sp_batch[2], val_causal)+ for sp_batch in get_speech_batches(val_speech_audio_arrays, val_speech_dec_in, val_speech_dec_tgt, args.batch_size,+ shuffle=False, loss_mask=val_speech_loss_mask,+ n_mels=n_mels, max_mel_len=max_mel_len, augmenter=None):+ vl, vt = speech_vl_fn(eval_params, sp_batch[0], sp_batch[1], sp_batch[2], val_causal, sp_batch[3])sp_total_loss += float(vl)sp_total_toks += float(vt)speech_val_loss = sp_total_loss / max(sp_total_toks, 1)@@ -860,10 +864,9 @@ def train(args):vl, vt = val_loss_fn(q_params, vb[0], vb[1], vb[2], val_causal, vb[3])q_total_loss += float(vl)q_total_toks += float(vt)- quant_val_ppl = float(math.exp(q_total_loss / max(q_total_toks, 1)))- del q_params # free device memory+ quant_val_ppl = float(math.exp(min(q_total_loss / max(q_total_toks, 1), 20)))+ del q_params- # Matryoshka per-factor evaluation (reuse val_loss_fn's apply_fn)mat_results = {}if _MAT_FACTORS:apply_fn = jax_utils.unreplicate(state).apply_fn@@ -880,9 +883,8 @@ def train(args):mat_results[factor] = (mat_ppl, mat_params, ff_w)del apply_fn- # Sparsity + checkpoint — move to CPU, reuse single copyparams_np = jax.tree.map(np.array, eval_params)- del eval_params # free device memory+ del eval_paramstotal_params = sum(x.size for x in jax.tree.leaves(params_np))near_zero = sum(int(np.sum(np.abs(x) < 1e-6)) for x in jax.tree.leaves(params_np))sparsity = near_zero / total_params * 100@@ -893,33 +895,27 @@ def train(args):with open(ckpt_path, "wb") as f:pickle.dump({"params": params_np, "config": config.__dict__}, f)- from .test import measure_throughput, benchmark_tool_calls, compute_wer- from .run import transcribe- from .data import _decode_audio+ from .test import measure_throughput, benchmark_tool_calls+ from .run import generate_from_audioeval_params_jnp = jax.tree.map(jnp.array, params_np)- del params_np # free CPU memory+ del params_npmodel = EncoderDecoderTransformer(config)tp = measure_throughput(model, eval_params_jnp, tokenizer, num_runs=5)tc_metrics = benchmark_tool_calls(model, eval_params_jnp, tokenizer, num_samples=20, max_gen_len=128)- speech_wer = None- transcribe_samples = []- if not no_speech and val_speech_mel is not None:- speech_val_ds = load_librispeech("validation.clean", max_samples=3)- hyps, refs = [], []- for i, example in enumerate(speech_val_ds):- audio, sr = _decode_audio(example["audio"])- ref_text = example["text"].strip().lower()- hyp_text = transcribe(+ voice_tc_samples = []+ if not no_speech and val_speech_audio_arrays is not None:+ val_pairs = load_tool_call_audio("validation", max_samples=3)+ for i, pair in enumerate(val_pairs):+ audio = pair["audio_array"]+ sr = pair["sampling_rate"]+ pred_text = generate_from_audio(model, eval_params_jnp, tokenizer, audio, sr=sr,- max_gen_len=64, temperature=0.0, seed=i, stream=False,- ).strip().lower()- hyps.append(hyp_text)- refs.append(ref_text)- transcribe_samples.append((ref_text, hyp_text))- speech_wer = compute_wer(hyps, refs)+ tools=pair["tools"], max_gen_len=128, seed=i, stream=False,+ ).strip()+ voice_tc_samples.append((pair["query"][:80], pair["answers"][:120], pred_text[:120]))- del eval_params_jnp # free device memory+ del eval_params_jnpprint(f"\n ─────────────────────────────────────")print(f" Epoch {epoch + 1}/{args.epochs}")@@ -928,9 +924,8 @@ def train(args):print(f" Final loss {final_loss:>12.4f}")print(f" Train ppl {final_ppl:>12.2f}")print(f" TC val ppl {last_val_ppl:>12.2f}")- if speech_wer is not None:- print(f" Speech WER {speech_wer:>12.4f}")- epoch_avg_speech = sum(speech_losses) / len(speech_losses) if speech_losses else float("nan")+ if speech_losses:+ epoch_avg_speech = sum(speech_losses) / len(speech_losses)print(f" Avg speech loss{epoch_avg_speech:>12.4f}")if speech_val_ppl is not None:print(f" Speech val ppl {speech_val_ppl:>12.2f}")@@ -959,12 +954,14 @@ def train(args):print(f" R: {ref}")print(f" P: {pred}")print()- if transcribe_samples:+ if voice_tc_samples:print(f" ─────────────────────────────────────")- print(f" Transcribe (WER={speech_wer:.4f}):")- for ref, hyp in transcribe_samples:- print(f" <transcribe> ref: {ref[:80]}")- print(f" hyp: {hyp[:80]}")+ print(f" Voice-to-Tool-Call samples:")+ for query, ref, pred in voice_tc_samples:+ print(f" Q: {query}")+ print(f" R: {ref}")+ print(f" P: {pred}")+ print()print(f" ─────────────────────────────────────")print(f" Checkpoint: {ckpt_path}")print(f" ─────────────────────────────────────\n")@@ -982,8 +979,6 @@ def train(args):log_dict["epoch/tc_name_f1"] = tc_metrics["name_f1"]log_dict["epoch/tc_call_f1"] = tc_metrics["call_f1"]log_dict["epoch/tc_json_parse_rate"] = tc_metrics["json_parse_rate"]- if speech_wer is not None:- log_dict["epoch/speech_wer"] = speech_werif speech_losses:log_dict["epoch/avg_speech_loss"] = sum(speech_losses) / len(speech_losses)if speech_val_ppl is not None: