Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Tool call refactor

HenryNdubuakue1c146854483
diff --git a/README.md b/README.md
index ba4c9a7..38b549b 100644
--- a/README.md
+++ b/README.md
@@ -10,42 +10,41 @@
Architecture Training Pipeline
──────────── ─────────────────
- ┌──────────────┐ ┌──────────────┐
- │ Transcription│ │ Story │ ┌──────────────────────┐
- └──────┬───────┘ └──────┬───────┘ │ Mat loss at factors │
- └───────┬─────────┘ │ + INT4 QAT (g=32) │
- ┌───┴───────┐ │ + z-loss + slot div │
- │ Softmax │ │ mixed text/speech │
- └─────┬─────┘ │ batches │
- ┌─────┬─────┴─────┬─────┐ └──────────┬───────────┘
- │ @E[:d_ff/f] for each │ │
- │ f in mat_factors │ ┌──────────┴───────────┐
- └─────┬─────┬─────┬─────┘ │ Grad clip (norm 1.0) │
- ┌─────┴─────┐ │ Muon (2D kernels) │
- │ Linear (T)│ ← tied │ AdamW (all else) │
- └─────┬─────┘ │ WSD LR schedule │
- ┌─────┴─────┐ └──────────┬───────────┘
- │ ZCRMSNorm │ │
- └─────┬─────┘ ┌──────────┴───────────┐
- ┌────────┴────────┐ │ Sparsification │
- │ Decoder x N_dec │ │ cubic schedule │
- │┌───────────────┐│ │ mask every N steps │
- ││ Masked Self ││ └──────────┬───────────┘
- ││ Attn + RoPE ││ │
- │├───────────────┤│ ┌──────────┴───────────┐
- ┌──────────────┐ S ││ Cross ││ │ EMA params (β=0.999) │
- │ MemoryMixer │────────▶ Attention ││ └──────────┬───────────┘
- │ Encoder │ │├───────────────┤│ │
- │ x N_enc │ ││ Gated FFN ││ ┌──────────┴───────────┐
- │ │ │└───────────────┘│ │ Checkpoint │
- │ ┌──────────┐ │ └────────┬────────┘ │ full params, can │
- │ │Pack: │ │ ┌─────┴─────┐ │ export mat slices │
- │ │ S←X Attn │ │ │ Embedding │ ← shared └──────────────────────┘
- │ │ RoPE keys│ │ └─────┬─────┘
+ ┌──────────────┐
+ │ Tool Call │ ┌──────────────────────┐
+ └──────┬───────┘ │ Mat loss at factors │
+ ┌┴──────────┐ │ + INT4 QAT (g=32) │
+ │ Softmax │ │ + z-loss + slot div │
+ └─────┬─────┘ │ 1:1 text/speech │
+ ┌─────┬─────┴─────┬─────┐ │ alternation │
+ │ @E[:d_ff/f] for each │ └──────────┬───────────┘
+ │ f in mat_factors │ │
+ └─────┬─────┬─────┬─────┘ ┌──────────┴───────────┐
+ ┌─────┴─────┐ │ Grad clip (norm 1.0) │
+ │ Linear (T)│ ← tied │ Muon (2D kernels) │
+ └─────┬─────┘ │ AdamW (all else) │
+ ┌─────┴─────┐ │ WSD LR schedule │
+ │ ZCRMSNorm │ └──────────┬───────────┘
+ └─────┬─────┘ │
+ ┌────────┴────────┐ ┌──────────┴───────────┐
+ │ Decoder x N_dec │ │ Sparsification │
+ │┌───────────────┐│ │ cubic schedule │
+ ││ Masked Self ││ │ mask every N steps │
+ ││ Attn + RoPE ││ └──────────┬───────────┘
+ │├───────────────┤│ │
+ ┌──────────────┐ S ││ Cross ││ ┌──────────┴───────────┐
+ │ MemoryMixer │────────▶ Attention ││ │ EMA params (β=0.999) │
+ │ Encoder │ │├───────────────┤│ └──────────┬───────────┘
+ │ x N_enc │ ││ Gated FFN ││ │
+ │ │ │└───────────────┘│ ┌──────────┴───────────┐
+ │ ┌──────────┐ │ └────────┬────────┘ │ Checkpoint │
+ │ │Pack: │ │ ┌─────┴─────┐ │ full params, can │
+ │ │ S←X Attn │ │ │ Embedding │ ← shared │ export mat slices │
+ │ │ RoPE keys│ │ └─────┬─────┘ └──────────────────────┘
│ ├──────────┤ │ ┌───────┴───────┐
- │ │Mix: │ │ │ <transcribe> │
- │ │ MLP-Mixer│ │ │ or <story> │
- │ │ on S │ │ │ + target │
+ │ │Mix: │ │ │[EOS]<tool_call>│
+ │ │ MLP-Mixer│ │ │ + tools │
+ │ │ on S │ │ │ + answer │
│ ├──────────┤ │ └───────────────┘
│ │Local: │ │
│ │Gated FFN │ │
@@ -64,56 +63,63 @@
│ │
┌────┴──────┐ ┌──────┴───────┐
│ Text │ │ Audio │
- │ tokens │ │ mel spec │
+ │ query │ │ waveform │
└───────────┘ └──────────────┘
d=512 (default) / 1536 (--full) · GQA · QK-norm
SentencePiece BPE (8192) · gated dReLU · ZCRMSNorm · RoPE
strided DW conv · mat factors · INT4 QAT · Muon + AdamW
- text + speech encoder · <transcribe> / <story> task routing
+ text + speech encoder · <tool_call> task routing
Data Preprocessing
──────────────────
- Text (TinyStories) Speech (LibriSpeech-100)
- ────────────────── ────────────────────────
+ Text (Tool-Call Dataset) Speech (Voice-Tool-Call)
+ ──────────────────────── ────────────────────────
- TinyStories LibriSpeech-100
+ Unified dataset Unified dataset
+ (GCS / local) (GCS / local)
│ │
▼ ▼
┌──────────────┐ ┌────────────────┐
- │ SentencePiece│ │ soundfile │
- │ BPE tokenize │ │ decode (16kHz) │
+ │ SentencePiece│ │ HF Audio col │
+ │ BPE tokenize │ │ decode (16kHz) │
│ (8192 vocab) │ └───────┬────────┘
└──────┬───────┘ │
│ ▼
▼ ┌────────────────┐
- ┌──────────────┐ │ Log-mel spec │
- │ Split at 30% │ │ 80 bins, 25ms │
- │ enc / dec │ │ window, 10ms │
- └──┬───────┬───┘ │ hop (~100 f/s) │
- │ │ └───────┬────────┘
- ▼ ▼ │
- ┌─────┐ ┌──────────────┐ ▼
- │ enc │ │ [BOS]<story> │ ┌────────────────┐
- │ ids │ │ + tokens │ │ Pad/trunc to │
- └─────┘ │ → dec_in │ │ max_mel_len │
- ├──────────────┤ └───────┬────────┘
- │ <story> │ │
- │ + tokens │ ▼
- │ + [EOS] │ ┌────────────────┐
- │ → dec_tgt │ │ [BOS]<transcr> │
- └──────────────┘ │ + tokens │
- │ → dec_in │
- Both cached as ├────────────────┤
- .npy in .data_cache/ │ <transcr> │
- │ + tokens │
- ┌──────────────────────┐ │ + [EOS] │
- │ Interleaved batches: │ │ → dec_tgt │
- │ 1 speech every N │ └────────────────┘
- │ text steps │
- │ (--speech-every N) │ Eval: WER on 3 val samples
- └──────────────────────┘ at end of each epoch
+ ┌──────────────┐ │ Augmentation │
+ │ Encoder: │ │ (noise, pitch, │
+ │ query tokens │ │ stretch, gain)│
+ └──────┬───────┘ └───────┬────────┘
+ │ │
+ ▼ ▼
+ ┌──────────────────┐ ┌────────────────┐
+ │ Decoder: │ │ Log-mel spec │
+ │ [EOS]<tool_call> │ │ 80 bins, 25ms │
+ │ + tools tokens │ │ window, 10ms │
+ │ + answer tokens │ │ hop (~100 f/s) │
+ │ → dec_in │ └───────┬────────┘
+ ├──────────────────┤ │
+ │ <tool_call> │ ▼
+ │ + tools tokens │ ┌────────────────┐
+ │ + answer tokens │ │ Pad/trunc to │
+ │ + [EOS] │ │ max_mel_len │
+ │ → dec_tgt │ └───────┬────────┘
+ ├──────────────────┤ │
+ │ Loss mask: 1.0 │ ▼
+ │ on answer + EOS │ ┌────────────────┐
+ │ only (not tools) │ │ Same decoder │
+ └──────────────────┘ │ format as text │
+ └────────────────┘
+ Both cached as
+ .npy in .data_cache/ Mel computed on-the-fly
+ (augment before mel)
+ ┌──────────────────────┐
+ │ Interleaved batches: │
+ │ 1:1 text / speech │ Eval: voice-to-tool-call on
+ │ alternation │ 3 val samples per epoch
+ └──────────────────────┘
```
## Usage
@@ -128,70 +134,72 @@ needle [command]
┌───────────────────────────────────────────────────────────────────┐
│ │
│ train │
- │ --full Use full 1B config (~1.17B params) │
- │ --epochs INT Training epochs (default: 1) │
- │ --batch-size INT Batch size (default: 32) │
- │ --lr FLOAT AdamW learning rate (default: 3e-4) │
- │ --muon-lr FLOAT Muon learning rate (default: 0.02) │
- │ --d-model INT Model dim (default: 512) │
- │ --num-heads INT Attention heads (default: 8) │
- │ --num-kv-heads INT KV heads for GQA (default: num-heads) │
- │ --num-layers INT Encoder layers (default: 8) │
- │ --num-dec-layers INT Decoder layers (default: 4) │
- │ --max-enc-len INT Max encoder seq length (default: 256) │
- │ --max-dec-len INT Max decoder seq length (default: 256) │
- │ --max-samples INT Training samples (default: all) │
- │ --mat-factors INT [...] FFN shrink factors (default: 2 4 8) │
- │ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) │
- │ --group-size INT Quant/prune group size (default: 32) │
- │ --prune-interval INT Steps between mask updates (def: 100) │
- │ --prune-start-frac FL Start pruning at frac (def: 0.33) │
- │ --prune-end-frac FL Lock mask at this frac (def: 0.67) │
- │ --activation STR drelu|swiglu|geglu (default: drelu) │
- │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │
- │ --eval-every INT Val eval interval (default: 1000) │
- │ --wandb Enable W&B logging │
- │ --checkpoint PATH Resume from checkpoint │
- │ --checkpoint-dir DIR Checkpoint directory │
- │ --seed INT Random seed (default: 42) │
+ │ --full Use full 1B config (~1.17B params) │
+ │ --epochs INT Training epochs (default: 1) │
+ │ --batch-size INT Batch size (default: 32) │
+ │ --lr FLOAT AdamW learning rate (default: 3e-4) │
+ │ --muon-lr FLOAT Muon learning rate (default: 0.02) │
+ │ --d-model INT Model dim (default: 512) │
+ │ --num-heads INT Attention heads (default: 8) │
+ │ --num-kv-heads INT KV heads for GQA (default: num-heads)│
+ │ --num-layers INT Encoder layers (default: 4) │
+ │ --num-dec-layers INT Decoder layers (default: 4) │
+ │ --max-enc-len INT Max encoder seq len (default: 256) │
+ │ --max-dec-len INT Max decoder seq len (default: 1024) │
+ │ --max-samples INT Training samples (default: all) │
+ │ --mat-factors INT [...] FFN shrink factors (default: 2 4 8) │
+ │ --sparsity-ratio FLOAT Block prune ratio (default: 0.0) │
+ │ --group-size INT Quant/prune group size (default: 32) │
+ │ --prune-interval INT Steps between mask updates (def: 100)│
+ │ --prune-start-frac FL Start pruning at frac (def: 0.33) │
+ │ --prune-end-frac FL Lock mask at this frac (def: 0.67) │
+ │ --activation STR drelu|swiglu|geglu (default: drelu) │
+ │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │
+ │ --eval-every INT Val eval interval (default: 1000) │
+ │ --wandb Enable W&B logging │
+ │ --checkpoint PATH Resume from checkpoint │
+ │ --checkpoint-dir DIR Checkpoint directory │
+ │ --seed INT Random seed (default: 42) │
│ --no-speech Disable speech (text-only training) │
- │ --speech-every INT Speech step every N text (default: 3) │
│ --max-mel-len INT Max mel frames (default: 1024) │
│ --n-mels INT Mel frequency bins (default: 80) │
- │ --max-speech-samples INT Max LibriSpeech samples │
+ │ --max-speech-samples INT Max voice-tool-call samples │
│ │
│ run │
- │ --checkpoint PATH Path to model checkpoint (required) │
- │ --prompts STR [...] One or more prompts to continue │
- │ --audio PATH [...] Audio files to transcribe │
- │ --max-len INT Max tokens to generate (default: 128) │
- │ --temperature FLOAT Sampling temperature (default: 0.8) │
- │ --seed INT Random seed (default: 0) │
+ │ --checkpoint PATH Path to model checkpoint (required) │
+ │ --query STR Query text for tool-call generation │
+ │ --tools STR Tools JSON for tool-call generation │
+ │ --audio PATH [...] Audio files for voice-to-tool-call │
+ │ --max-len INT Max tokens to generate (default: 512) │
+ │ --seed INT Random seed (default: 0) │
│ │
│ test │
- │ --checkpoint PATH Path to model checkpoint (required) │
- │ --batch-size INT Batch size (default: 32) │
- │ --max-eval-samples INT Evaluation samples (default: 1000) │
- │ --max-gen-len INT Max generation length (default: 128) │
- │ --temperature FLOAT Sampling temperature (default: 0.8) │
- │ --throughput-runs INT Throughput runs (default: 10) │
+ │ --checkpoint PATH Path to model checkpoint (required) │
+ │ --batch-size INT Batch size (default: 32) │
+ │ --max-eval-samples INT Evaluation samples (default: 1000) │
+ │ --max-enc-len INT Max encoder length (default: 256) │
+ │ --max-dec-len INT Max decoder length (default: 1024) │
+ │ --max-gen-len INT Max generation length (default: 512) │
+ │ --throughput-runs INT Throughput runs (default: 10) │
+ │ --tool-call-samples INT Tool-call eval samples (default: 200) │
+ │ --voice-tc-samples INT Voice-TC eval samples (default: 50) │
│ │
│ evaluate │
- │ --checkpoint PATH Path to model checkpoint (required) │
- │ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │
- │ --max-samples INT Samples per benchmark (default: 500) │
+ │ --checkpoint PATH Path to model checkpoint (required) │
+ │ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │
+ │ --max-samples INT Samples per benchmark (default: 500) │
│ │
│ tpu │
- │ create NAME Create TPU (auto-finds zone) │
- │ --type STR Accelerator (default: v6e-8) │
- │ --version STR TPU OS (auto-detected from --type) │
- │ connect NAME SSH config + connect (auto-zone) │
- │ claude NAME Install Claude Code on instance │
- │ stop NAME Stop instance (auto-zone) │
- │ start NAME Start stopped instance (auto-zone) │
- │ delete NAME Delete instance (auto-zone) │
- │ list List all TPU instances │
- │ --zone ZONE Override auto-detected zone │
+ │ create NAME Create TPU (auto-finds zone) │
+ │ --type STR Accelerator (default: v6e-8) │
+ │ --version STR TPU OS (auto-detected from --type) │
+ │ connect NAME SSH config + connect (auto-zone) │
+ │ claude NAME Install Claude Code on instance │
+ │ stop NAME Stop instance (auto-zone) │
+ │ start NAME Start stopped instance (auto-zone) │
+ │ delete NAME Delete instance (auto-zone) │
+ │ list List all TPU instances │
+ │ --zone ZONE Override auto-detected zone │
│ │
└───────────────────────────────────────────────────────────────────┘
```
@@ -210,19 +218,21 @@ needle [command]
Dataset
──────────────────────────────────────────
- Text 100B tokens (~400 GB)
- Audio Emilia subset
- 200k × 20s clips (~1.1k hrs)
- ~200M audio tokens
- ~13M transcription tokens
- ~27 GB (from ~4.5 TB full dataset)
+ Text Tool-call pairs
+ (query, tools, answers)
+ from GCS unified dataset
+ Audio TTS-generated speech
+ (GCP Cloud TTS voices)
+ 16kHz WAV, HF Audio col
──────────────────────────────────────────
- 100M quick-experiment estimates (--default)
+ 100M params — 10M multimodal triples (5 epochs)
──────────────────────────────────────────
- Total tokens 1B text + 1k hrs audio ≈ 1.2B
- Total FLOPs 6 × 102M × 1.2B ≈ 7.2e17
- MFU (100M model) ~15% → ~138 effective TFLOPS/chip
+ Triples 10M (text / voice / image → JSON tool calls)
+ Tokens/triple ~300 avg (text ~180, voice ~390, image ~336)
+ Total tokens 10M × 300 × 5 epochs ≈ 15B
+ Total FLOPs 6 × 100M × 15B ≈ 9e18
+ MFU (100M) ~15% → ~138 effective TFLOPS/chip
┌────────────────────┬──────────┬──────────┬──────────┐
│ │ v6e-8 │ v6e-16 │ v6e-32 │
@@ -231,27 +241,9 @@ needle [command]
│ Total HBM │ 256 GB │ 512 GB │ 1024 GB │
│ Scaling eff. │ 0.9× │ 0.8× │ 0.7× │
│ Eff. TFLOPS │ 994 │ 1,766 │ 3,091 │
- │ Est. time │ ~12min │ ~7min │ ~4min │
- │ On-demand $/hr │ $21.60 │ $43.20 │ $86.40 │
- │ Est. total cost │ ~$4 │ ~$5 │ ~$6 │
- └────────────────────┴──────────┴──────────┴──────────┘
-
- 1B multimodal training estimates (--full)
- ──────────────────────────────────────────
- Total tokens 100B text + 250k hrs audio ≈ 145B
- Total FLOPs 6 × 1.17B × 145B ≈ 1.0e21
- MFU (1B model) ~30% → ~275 effective TFLOPS/chip
-
- ┌────────────────────┬──────────┬──────────┬──────────┐
- │ │ v6e-8 │ v6e-16 │ v6e-32 │
- ├────────────────────┼──────────┼──────────┼──────────┤
- │ Chips │ 8 │ 16 │ 32 │
- │ Total HBM │ 256 GB │ 512 GB │ 1024 GB │
- │ Scaling eff. │ 0.9× │ 0.8× │ 0.7× │
- │ Eff. TFLOPS │ 1,980 │ 3,520 │ 6,160 │
- │ Est. time │ ~143h │ ~80h │ ~46h │
+ │ Est. time │ ~2.5h │ ~1.4h │ ~49min │
│ On-demand $/hr │ $21.60 │ $43.20 │ $86.40 │
- │ Est. total cost │ ~$3,090 │ ~$3,460 │ ~$3,970 │
+ │ Est. total cost │ ~$54 │ ~$61 │ ~$71 │
└────────────────────┴──────────┴──────────┴──────────┘
```
diff --git a/requirements.txt b/requirements.txt
index 4e3232f..ec5f4fb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,11 @@ jaxlib
flax
optax
datasets
+gcsfs
transformers
wandb
pyyaml
sentencepiece
+google-cloud-texttospeech
+audiomentations
+gcsfs
\ No newline at end of file
diff --git a/scripts/build_dataset.py b/scripts/build_dataset.py
new file mode 100644
index 0000000..72f9385
--- /dev/null
+++ b/scripts/build_dataset.py
@@ -0,0 +1,335 @@
+#!/usr/bin/env python3
+"""Unified dataset builder: text + TTS audio -> HuggingFace Dataset on GCS.
+
+Combines the text pipeline (tools_data.py) with TTS audio generation into
+one script that produces a single HF Dataset with an Audio column.
+
+Usage:
+ python scripts/build_dataset.py --max-samples 1000 --workers 16
+ python scripts/build_dataset.py --max-samples 1000 --gcs-upload gs://cactus-dataset/tool_calls
+"""
+
+import argparse
+import io
+import logging
+import os
+import random
+import subprocess
+import sys
+import time
+import wave
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from datasets import Audio, Dataset, load_from_disk
+from tqdm import tqdm
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from tools_data import load_and_combine
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+logger = logging.getLogger(__name__)
+
+VOICES = [
+ ("en-US", "en-US-Journey-D", "MALE", False, False),
+ ("en-US", "en-US-Journey-F", "FEMALE", False, False),
+ ("en-US", "en-US-Journey-O", "FEMALE", False, False),
+ ("en-GB", "en-GB-Wavenet-B", "MALE", True, False),
+ ("en-GB", "en-GB-Wavenet-A", "FEMALE", True, False),
+ ("en-AU", "en-AU-Wavenet-B", "MALE", True, False),
+ ("en-AU", "en-AU-Wavenet-C", "FEMALE", True, False),
+ ("en-US", "en-US-Wavenet-D", "MALE", True, False),
+ ("en-IN", "en-IN-Wavenet-A", "FEMALE", True, False),
+ ("en-IN", "en-IN-Wavenet-B", "MALE", True, False),
+ ("en-IN", "en-IN-Wavenet-C", "MALE", True, False),
+ ("en-GB", "en-GB-Wavenet-D", "MALE", True, False),
+ ("en-AU", "en-AU-Wavenet-D", "MALE", True, False),
+ ("en-US", "en-US-Wavenet-E", "FEMALE", True, True),
+ ("en-GB", "en-GB-Wavenet-A", "FEMALE", True, True),
+ ("en-IN", "en-IN-Wavenet-A", "FEMALE", True, True),
+]
+
+
+def _pcm_to_wav_bytes(pcm_bytes: bytes, sample_rate: int) -> bytes:
+ """Convert raw LINEAR16 PCM bytes to a WAV byte buffer."""
+ buf = io.BytesIO()
+ with wave.open(buf, "wb") as wf:
+ wf.setnchannels(1)
+ wf.setsampwidth(2)
+ wf.setframerate(sample_rate)
+ wf.writeframes(pcm_bytes)
+ return buf.getvalue()
+
+
+def _synthesize_one(client, idx, query, max_retries=5):
+ """Synthesize TTS audio for a single query via GCP Cloud TTS.
+
+ Selects a voice from VOICES based on idx, applies randomized prosody
+ for configurable voices, and retries on transient API errors.
+
+ Returns:
+ Tuple of (idx, wav_bytes, voice_name, error_msg).
+ wav_bytes is None on failure; error_msg is None on success.
+ """
+ from google.cloud import texttospeech
+ from google.api_core.exceptions import ResourceExhausted, ServiceUnavailable
+
+ lang, voice_name, ssml_gender, configurable, child = VOICES[idx % len(VOICES)]
+ gender = getattr(texttospeech.SsmlVoiceGender, ssml_gender)
+
+ synthesis_input = texttospeech.SynthesisInput(text=query)
+ voice_params = texttospeech.VoiceSelectionParams(
+ language_code=lang,
+ name=voice_name,
+ ssml_gender=gender,
+ )
+
+ if configurable:
+ rng = random.Random(idx)
+ if child:
+ pitch = rng.uniform(4.0, 8.0)
+ rate = rng.uniform(1.0, 1.15)
+ else:
+ pitch = rng.uniform(-3.0, 3.0)
+ rate = rng.uniform(0.9, 1.1)
+ audio_config = texttospeech.AudioConfig(
+ audio_encoding=texttospeech.AudioEncoding.LINEAR16,
+ sample_rate_hertz=16000,
+ pitch=pitch,
+ speaking_rate=rate,
+ volume_gain_db=rng.uniform(-1.5, 1.5),
+ )
+ else:
+ audio_config = texttospeech.AudioConfig(
+ audio_encoding=texttospeech.AudioEncoding.LINEAR16,
+ sample_rate_hertz=16000,
+ )
+
+ for attempt in range(max_retries):
+ try:
+ response = client.synthesize_speech(
+ input=synthesis_input,
+ voice=voice_params,
+ audio_config=audio_config,
+ )
+ wav_bytes = _pcm_to_wav_bytes(response.audio_content, 16000)
+ return idx, wav_bytes, voice_name, None
+ except (ResourceExhausted, ServiceUnavailable):
+ wait = min(2 ** attempt, 30)
+ time.sleep(wait)
+ except Exception as e:
+ return idx, None, voice_name, str(e)
+
+ return idx, None, voice_name, "max retries exceeded"
+
+
+def _load_text_dataset(output_path, max_samples):
+ """Load the text dataset and optionally resume from an existing build.
+
+ Returns:
+ Tuple of (combined_dataset, existing_dataset_or_None).
+ """
+ existing_ds = None
+ if os.path.exists(output_path):
+ logger.info(f"Found existing dataset at {output_path}, loading for resume...")
+ existing_ds = load_from_disk(output_path)
+ logger.info(f" Existing dataset: {len(existing_ds)} rows")
+
+ logger.info("Loading and combining text datasets...")
+ combined = load_and_combine()
+
+ if max_samples:
+ combined = combined.select(range(min(max_samples, len(combined))))
+ logger.info(f"Text dataset: {len(combined)} rows")
+
+ return combined, existing_ds
+
+
+def _find_rows_needing_tts(combined, existing_ds):
+ """Identify which rows lack audio by inspecting the raw Arrow column.
+
+ Accesses the underlying Arrow table directly to avoid triggering
+ HF audio decoding (which requires torchcodec).
+
+ Returns:
+ Set of row indices that need TTS synthesis.
+ """
+ n = len(combined)
+ needs_tts = set(range(n))
+
+ if existing_ds is not None and len(existing_ds) == n and "audio" in existing_ds.column_names:
+ audio_column = existing_ds.data.column("audio")
+ for i in range(n):
+ if audio_column[i].as_py() is not None:
+ needs_tts.discard(i)
+ logger.info(f" {n - len(needs_tts)} rows already have audio, {len(needs_tts)} need TTS")
+
+ return needs_tts
+
+
+def _recover_existing_audio(existing_ds, n, needs_tts):
+ """Extract audio bytes and voice names from a previously saved dataset.
+
+ Reads raw Arrow data to avoid triggering audio decoding.
+
+ Returns:
+ Tuple of (audio_data list, voice_names list).
+ """
+ audio_data = [None] * n
+ voice_names = [""] * n
+
+ if existing_ds is not None and len(existing_ds) == n and "audio" in existing_ds.column_names:
+ audio_column = existing_ds.data.column("audio")
+ voice_col = existing_ds["voice"] if "voice" in existing_ds.column_names else None
+ for i in range(n):
+ if i not in needs_tts:
+ raw = audio_column[i].as_py()
+ if isinstance(raw, dict) and raw.get("bytes"):
+ audio_data[i] = raw["bytes"]
+ elif isinstance(raw, dict) and raw.get("path"):
+ with open(raw["path"], "rb") as f:
+ audio_data[i] = f.read()
+ if voice_col is not None:
+ voice_names[i] = voice_col[i] or ""
+
+ return audio_data, voice_names
+
+
+def _synthesize_missing(combined, needs_tts, audio_data, voice_names, workers, rate_limit):
+ """Run TTS synthesis for all rows that lack audio.
+
+ Uses a thread pool with rate-limited submission to stay within
+ GCP Cloud TTS API quotas.
+ """
+ if not needs_tts:
+ return
+
+ from google.cloud import texttospeech
+
+ queries_to_synth = [(i, combined[i]["query"]) for i in sorted(needs_tts)]
+ logger.info(f"Synthesizing TTS for {len(queries_to_synth)} rows with {workers} workers...")
+
+ def _worker(item):
+ idx, query = item
+ client = texttospeech.TextToSpeechClient()
+ return _synthesize_one(client, idx, query)
+
+ success, failed = 0, 0
+ delay = 1.0 / rate_limit if rate_limit > 0 else 0
+ with ThreadPoolExecutor(max_workers=workers) as pool:
+ futures = {}
+ for q in queries_to_synth:
+ futures[pool.submit(_worker, q)] = q
+ time.sleep(delay)
+ with tqdm(total=len(futures), desc="TTS") as pbar:
+ for future in as_completed(futures):
+ idx, wav_bytes, voice_name, err = future.result()
+ if wav_bytes is not None:
+ audio_data[idx] = wav_bytes
+ voice_names[idx] = voice_name
+ success += 1
+ else:
+ failed += 1
+ logger.warning(f"Failed idx={idx}: {err}")
+ pbar.update(1)
+
+ logger.info(f"TTS done: {success} generated, {failed} failed")
+
+
+def _backfill_voice_names(voice_names):
+ """Assign default voice names to any rows that don't have one."""
+ for i in range(len(voice_names)):
+ if not voice_names[i]:
+ _, voice_name, _, _, _ = VOICES[i % len(VOICES)]
+ voice_names[i] = voice_name
+
+
+def _build_and_save(combined, audio_data, voice_names, output_path):
+ """Assemble the unified HF Dataset and save to disk.
+
+ Returns:
+ The saved Dataset.
+ """
+ n = len(combined)
+ logger.info("Building unified HF Dataset...")
+ ds_dict = {
+ "query": [combined[i]["query"] for i in range(n)],
+ "audio": audio_data,
+ "tools": [combined[i]["tools"] for i in range(n)],
+ "answers": [combined[i]["answers"] for i in range(n)],
+ "source": [combined[i]["source"] for i in range(n)],
+ "voice": voice_names,
+ }
+
+ unified = Dataset.from_dict(ds_dict)
+ unified = unified.cast_column("audio", Audio(sampling_rate=16000))
+
+ os.makedirs(os.path.dirname(output_path) if os.path.dirname(output_path) else ".", exist_ok=True)
+ unified.save_to_disk(output_path)
+ logger.info(f"Saved unified dataset to {output_path}/ ({len(unified)} rows)")
+ return unified
+
+
+def _train_and_save_tokenizer(max_samples):
+ """Train a SentencePiece tokenizer on the unified dataset."""
+ from src.data import train_tokenizer, TOKENIZER_PREFIX
+ logger.info("Training tokenizer on unified dataset...")
+ model_path = train_tokenizer(max_samples=max_samples)
+ logger.info(f"Tokenizer ready at {model_path}")
+
+
+def _upload_to_gcs(output_path, gcs_upload):
+ """Upload the dataset and tokenizer to GCS via gcloud CLI."""
+ from src.data import TOKENIZER_PREFIX
+
+ logger.info(f"Uploading dataset to {gcs_upload}...")
+ result = subprocess.run(
+ ["gcloud", "storage", "cp", "-r", output_path, gcs_upload],
+ capture_output=True, text=True,
+ )
+ if result.returncode == 0:
+ logger.info(f"Dataset upload complete: {gcs_upload}")
+ else:
+ logger.error(f"Dataset upload failed: {result.stderr}")
+ sys.exit(1)
+
+ gcs_bucket = gcs_upload.rsplit("/", 1)[0]
+ gcs_tokenizer_path = f"{gcs_bucket}/tokenizer/"
+ logger.info(f"Uploading tokenizer to {gcs_tokenizer_path}...")
+ tokenizer_dir = os.path.dirname(TOKENIZER_PREFIX)
+ result = subprocess.run(
+ ["gcloud", "storage", "cp", "-r", tokenizer_dir, gcs_tokenizer_path],
+ capture_output=True, text=True,
+ )
+ if result.returncode == 0:
+ logger.info(f"Tokenizer upload complete: {gcs_tokenizer_path}")
+ else:
+ logger.error(f"Tokenizer upload failed: {result.stderr}")
+ sys.exit(1)
+
+
+def main():
+ """Entry point for the unified dataset build pipeline."""
+ parser = argparse.ArgumentParser(description="Build unified tool-call dataset with TTS audio")
+ parser.add_argument("--max-samples", type=int, default=None, help="Max samples to process")
+ parser.add_argument("--workers", type=int, default=16, help="TTS concurrent workers (default: 16)")
+ parser.add_argument("--output", type=str, default="data/tool_calls_unified", help="Output directory")
+ parser.add_argument("--rate-limit", type=float, default=100.0,
+ help="Max TTS requests per second (default: 100)")
+ parser.add_argument("--gcs-upload", type=str, default=None,
+ help="GCS path to upload dataset (e.g. gs://cactus-dataset/tool_calls)")
+ args = parser.parse_args()
+
+ combined, existing_ds = _load_text_dataset(args.output, args.max_samples)
+ needs_tts = _find_rows_needing_tts(combined, existing_ds)
+ audio_data, voice_names = _recover_existing_audio(existing_ds, len(combined), needs_tts)
+ _synthesize_missing(combined, needs_tts, audio_data, voice_names, args.workers, args.rate_limit)
+ _backfill_voice_names(voice_names)
+ _build_and_save(combined, audio_data, voice_names, args.output)
+ _train_and_save_tokenizer(args.max_samples)
+
+ if args.gcs_upload:
+ _upload_to_gcs(args.output, args.gcs_upload)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/cli.py b/src/cli.py
index 9fd59e9..928ae95 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -1,87 +1,7 @@
import argparse
import sys
-HELP = """
- ┌───────────────────────────────────────────────────────────────────┐
- │ │
- │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │
- │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │
- │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │
- │ ...the tiny model to rule them all... │
- │ │
- │ train │
- │ --full Use full 1B config (~1.17B params) │
- │ --epochs INT Training epochs (default: 1) │
- │ --batch-size INT Batch size (default: 32) │
- │ --lr FLOAT AdamW learning rate (default: 3e-4) │
- │ --muon-lr FLOAT Muon learning rate (default: 0.02) │
- │ --d-model INT Model dim (default: 512) │
- │ --num-heads INT Attention heads (default: 8) │
- │ --num-kv-heads INT KV heads for GQA (default: num-heads)│
- │ --num-layers INT Encoder layers (default: 4) │
- │ --num-dec-layers INT Decoder layers (default: 4) │
- │ --max-enc-len INT Max encoder seq len (default: 256) │
- │ --max-dec-len INT Max decoder seq len (default: 1024) │
- │ --max-samples INT Training samples (default: all) │
- │ --mat-factors INT [...] FFN shrink factors (default: 2 4 8) │
- │ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) │
- │ --group-size INT Quant/prune group size (default: 32) │
- │ --prune-interval INT Steps between mask updates (def: 100)│
- │ --prune-start-frac FL Start pruning at frac (def: 0.33) │
- │ --prune-end-frac FL Lock mask at this frac (def: 0.67) │
- │ --activation STR drelu|swiglu|geglu (default: drelu) │
- │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │
- │ --eval-every INT Val eval interval (default: 1000) │
- │ --wandb Enable W&B logging │
- │ --checkpoint PATH Resume from checkpoint │
- │ --checkpoint-dir DIR Checkpoint directory │
- │ --seed INT Random seed (default: 42) │
- │ --no-speech Disable speech (text-only training) │
- │ --speech-every INT Speech step every N text (default: 3) │
- │ --max-mel-len INT Max mel frames (default: 1024) │
- │ --n-mels INT Mel frequency bins (default: 80) │
- │ --max-speech-samples INT Max LibriSpeech samples │
- │ │
- │ run │
- │ --checkpoint PATH Path to model checkpoint (required) │
- │ --query STR Query text for tool-call generation │
- │ --tools STR Tools JSON for tool-call generation │
- │ --prompts STR [...] Raw prompts (query+tools combined) │
- │ --audio PATH [...] Audio files to transcribe │
- │ --max-len INT Max tokens to generate (default: 512) │
- │ --temperature FLOAT Sampling temperature (default: 0.8) │
- │ --seed INT Random seed (default: 0) │
- │ │
- │ test │
- │ --checkpoint PATH Path to model checkpoint (required) │
- │ --batch-size INT Batch size (default: 32) │
- │ --max-eval-samples INT Evaluation samples (default: 1000) │
- │ --max-enc-len INT Max encoder length (default: 256) │
- │ --max-dec-len INT Max decoder length (default: 1024) │
- │ --max-gen-len INT Max generation length (default: 512) │
- │ --temperature FLOAT Sampling temperature (default: 0.8) │
- │ --throughput-runs INT Throughput runs (default: 10) │
- │ --tool-call-samples INT Tool-call eval samples (default: 200) │
- │ │
- │ evaluate │
- │ --checkpoint PATH Path to model checkpoint (required) │
- │ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │
- │ --max-samples INT Samples per benchmark (default: 500) │
- │ │
- │ tpu │
- │ create NAME Create TPU (auto-finds zone) │
- │ --type STR Accelerator (default: v6e-8) │
- │ --version STR TPU OS (auto-detected from --type) │
- │ connect NAME SSH config + connect (auto-zone) │
- │ claude NAME Install Claude Code on instance │
- │ stop NAME Stop instance (auto-zone) │
- │ start NAME Start stopped instance (auto-zone) │
- │ delete NAME Delete instance (auto-zone) │
- │ list List all TPU instances │
- │ --zone ZONE Override auto-detected zone │
- │ │
- └───────────────────────────────────────────────────────────────────┘
-"""
+HELP = """Check the readme"""
def main():
if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help", "help"):
@@ -113,7 +33,7 @@ def main():
p.add_argument("--seed", type=int, default=42)
p.add_argument("--eval-every", type=int, default=1000)
p.add_argument("--max-eval-samples", type=int, default=None)
- p.add_argument("--sparsity-ratio", type=float, default=0.5)
+ p.add_argument("--sparsity-ratio", type=float, default=0.0)
p.add_argument("--group-size", type=int, default=32)
p.add_argument("--prune-interval", type=int, default=100,
help="Steps between mask updates during gradual pruning (default: 100)")
@@ -128,20 +48,18 @@ def main():
p.add_argument("--mat-shared-input", action="store_true",
help="Each unique input is repeated across all mat widths (default: unique input per width)")
p.add_argument("--no-speech", action="store_true", help="Disable speech training (text-only)")
- p.add_argument("--speech-every", type=int, default=3,
- help="Do one speech step every N text steps (default: 3)")
p.add_argument("--max-mel-len", type=int, default=1024,
help="Max mel spectrogram frames (default: 1024)")
p.add_argument("--n-mels", type=int, default=80,
help="Number of mel frequency bins (default: 80)")
p.add_argument("--max-speech-samples", type=int, default=None,
- help="Max LibriSpeech training samples (default: all)")
+ help="Max voice-tool-call training samples (default: all)")
p = sub.add_parser("run", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
p.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")
p.add_argument("--tools", type=str, default=None, help="Tools JSON for tool-call generation")
- p.add_argument("--audio", type=str, nargs="*", help="Audio file paths to transcribe")
+ p.add_argument("--audio", type=str, nargs="*", help="Audio file paths for voice-to-tool-call")
p.add_argument("--max-len", type=int, default=512)
p.add_argument("--seed", type=int, default=0)
@@ -154,6 +72,8 @@ def main():
p.add_argument("--max-gen-len", type=int, default=512)
p.add_argument("--tool-call-samples", type=int, default=200,
help="Samples for tool-call accuracy eval (default: 200)")
+ p.add_argument("--voice-tc-samples", type=int, default=50,
+ help="Samples for voice-to-tool-call eval (default: 50)")
p.add_argument("--throughput-runs", type=int, default=10)
p = sub.add_parser("evaluate", add_help=False)
diff --git a/src/data.py b/src/data.py
index 4ebc294..a09d62f 100644
--- a/src/data.py
+++ b/src/data.py
@@ -10,22 +10,71 @@ logging.getLogger("transformers").setLevel(logging.ERROR)
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
import numpy as np
-from datasets import Audio, load_dataset, load_from_disk
+from datasets import Audio, load_from_disk
from tqdm import tqdm
import sentencepiece as spm
CACHE_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), ".data_cache")
TOKENIZER_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tokenizer")
TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")
-TOOL_CALLS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "tool_calls")
+LOCAL_UNIFIED_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "tool_calls_unified")
+GCS_DATASET_PATH = "gs://cactus-dataset/tool_calls"
PAD_ID = 0
EOS_ID = 1
-BOS_ID = 2
+BOS_ID = 2 # reserved for SentencePiece, unused at runtime (EOS_ID serves as SOS)
UNK_ID = 3
TOOL_CALL_ID = 4
TRANSCRIBE_ID = 5
+_unified_dataset_cache = None
+
+
+def _load_unified_dataset():
+ """Load the unified dataset from GCS (via gcsfs) or local fallback.
+
+ Caches the result in memory after first load.
+ Uses soundfile as the audio decoding backend.
+ """
+ global _unified_dataset_cache
+ if _unified_dataset_cache is not None:
+ return _unified_dataset_cache
+
+ # Try GCS first
+ try:
+ ds = load_from_disk(GCS_DATASET_PATH)
+ print(f"Loaded unified dataset from GCS ({len(ds)} rows)")
+ # Cache locally for faster subsequent loads
+ if not os.path.exists(LOCAL_UNIFIED_DIR):
+ ds.save_to_disk(LOCAL_UNIFIED_DIR)
+ print(f" Cached locally to {LOCAL_UNIFIED_DIR}")
+ _unified_dataset_cache = _set_audio_backend(ds)
+ return _unified_dataset_cache
+ except Exception:
+ pass
+
+ # Fallback to local
+ if os.path.exists(LOCAL_UNIFIED_DIR):
+ ds = load_from_disk(LOCAL_UNIFIED_DIR)
+ print(f"Loaded unified dataset from {LOCAL_UNIFIED_DIR} ({len(ds)} rows)")
+ _unified_dataset_cache = _set_audio_backend(ds)
+ return _unified_dataset_cache
+
+ raise FileNotFoundError(
+ f"Unified dataset not found. Run scripts/build_dataset.py first, "
+ f"or ensure GCS path {GCS_DATASET_PATH} is accessible (pip install gcsfs)."
+ )
+
+
+def _set_audio_backend(ds):
+ """Disable automatic audio decoding to avoid torchcodec dependency.
+
+ Audio is decoded manually via soundfile in load_tool_call_audio().
+ """
+ if "audio" in ds.column_names:
+ ds = ds.cast_column("audio", Audio(sampling_rate=16000, decode=False))
+ return ds
+
class NeedleTokenizer:
"""Wrapper around SentencePiece providing the interface the codebase expects."""
@@ -102,7 +151,7 @@ def train_tokenizer(vocab_size=8192, max_samples=None):
os.makedirs(TOKENIZER_DIR, exist_ok=True)
- ds = load_from_disk(TOOL_CALLS_DIR)
+ ds = _load_unified_dataset()
if max_samples:
ds = ds.select(range(min(max_samples, len(ds))))
@@ -138,16 +187,37 @@ def train_tokenizer(vocab_size=8192, max_samples=None):
return model_path
+GCS_TOKENIZER_PATH = "gs://cactus-dataset/tokenizer/"
+
+
def get_tokenizer(max_samples=None):
model_path = TOKENIZER_PREFIX + ".model"
if not os.path.exists(model_path):
+ # Try downloading from GCS first
+ if _download_tokenizer_from_gcs():
+ return NeedleTokenizer(model_path)
train_tokenizer(max_samples=max_samples)
return NeedleTokenizer(model_path)
+def _download_tokenizer_from_gcs():
+ """Download tokenizer files from GCS. Returns True on success."""
+ import subprocess
+ os.makedirs(TOKENIZER_DIR, exist_ok=True)
+ result = subprocess.run(
+ ["gcloud", "storage", "cp", "-r", GCS_TOKENIZER_PATH + "*", TOKENIZER_DIR + "/"],
+ capture_output=True, text=True,
+ )
+ model_path = TOKENIZER_PREFIX + ".model"
+ if result.returncode == 0 and os.path.exists(model_path):
+ print(f"Downloaded tokenizer from {GCS_TOKENIZER_PATH}")
+ return True
+ return False
+
+
def load_tool_calls(split="train", max_samples=None):
"""Load tool-calling dataset, splitting 90/10 for train/val."""
- ds = load_from_disk(TOOL_CALLS_DIR)
+ ds = _load_unified_dataset()
n = len(ds)
if split in ("validation", "val", "test"):
ds = ds.select(range(int(n * 0.9), n))
@@ -196,11 +266,9 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):
)
pad_id = tokenizer.pad_token_id
- bos_id = tokenizer.eos_token_id
eos_id = tokenizer.eos_token_id
tool_call_id = tokenizer.tool_call_token_id
- # Encoder: query only.
enc_texts = [ex["query"] for ex in ds]
tools_texts = [ex["tools"] for ex in ds]
ans_texts = [ex["answers"] for ex in ds]
@@ -260,12 +328,10 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):
prefix_len = 2 + len(t_tok)
total_dec = prefix_len + len(a_tok) + 1
- # Encoder
el = len(e_tok)
enc_inputs[i, :el] = e_tok
- # Decoder input: [BOS, <tool_call>, tools..., answer...]
- dec_inputs[i, 0] = bos_id
+ dec_inputs[i, 0] = eos_id
dec_inputs[i, 1] = tool_call_id
tl = len(t_tok)
if tl > 0:
@@ -322,57 +388,56 @@ def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True, l
yield batch
-get_text_batches = get_batches
+def load_tool_call_audio(split="train", max_samples=None):
+ """Load tool-call dataset entries that have TTS audio.
-# ─── Speech pipeline ───────────────────────────────────────────────────────────
+ Applies the same 90/10 split as load_tool_calls, then filters to entries
+ where the audio column is not None.
-def load_librispeech(split="train", max_samples=None):
- """Load LibriSpeech-100 from HuggingFace (SPRINGLab/LibriSpeech-100).
-
- The dataset has a single 'train' split, so we partition it:
- - train / train.clean.100: first 90%
- - validation.clean / test.clean / validation / test: last 10%
-
- Audio decoding is disabled (to avoid torchcodec/torch dependency);
- prepare_speech_pairs decodes audio bytes via soundfile instead.
+ Returns list of dicts: {query, answers, tools, audio_bytes, sampling_rate}.
"""
- ds = load_dataset("SPRINGLab/LibriSpeech-100", split="train")
- ds = ds.cast_column("audio", Audio(decode=False))
+ ds = _load_unified_dataset()
n = len(ds)
- if split in ("validation.clean", "validation", "test.clean", "test"):
- ds = ds.select(range(int(n * 0.9), n))
- elif split.startswith("train"):
- ds = ds.select(range(int(n * 0.9)))
- if max_samples:
- ds = ds.select(range(min(max_samples, len(ds))))
- return ds
+ if split in ("validation", "val", "test"):
+ indices = list(range(int(n * 0.9), n))
+ elif split == "train":
+ indices = list(range(int(n * 0.9)))
+ else:
+ indices = list(range(n))
-def _decode_audio(audio_data, target_sr=16000):
- """Decode raw audio bytes from a datasets Audio(decode=False) field using soundfile."""
import io
import soundfile as sf
- audio_bytes = audio_data["bytes"]
- if audio_bytes is None:
- path = audio_data["path"]
- audio, sr = sf.read(path, dtype="float32")
- else:
- audio, sr = sf.read(io.BytesIO(audio_bytes), dtype="float32")
-
- # Convert stereo to mono
- if audio.ndim > 1:
- audio = audio.mean(axis=1)
-
- # Resample if needed
- if sr != target_sr:
- from scipy.signal import resample
- num_samples = int(len(audio) * target_sr / sr)
- audio = resample(audio, num_samples).astype(np.float32)
- sr = target_sr
-
- return audio.astype(np.float32), sr
+ pairs = []
+ for idx in indices:
+ ex = ds[idx]
+ audio_val = ex.get("audio")
+ if audio_val is None:
+ continue
+ # With decode=False, audio_val is {"bytes": b"...", "path": ...}
+ raw_bytes = None
+ if isinstance(audio_val, dict):
+ raw_bytes = audio_val.get("bytes")
+ elif isinstance(audio_val, bytes):
+ raw_bytes = audio_val
+ if raw_bytes is None:
+ continue
+ audio_array, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")
+ if audio_array.ndim > 1:
+ audio_array = audio_array.mean(axis=1)
+ pairs.append({
+ "query": ex["query"],
+ "answers": ex["answers"],
+ "tools": ex["tools"],
+ "audio_array": audio_array.astype(np.float32),
+ "sampling_rate": sr,
+ })
+ if max_samples and len(pairs) >= max_samples:
+ break
+
+ return pairs
def compute_mel_spectrogram(audio, sr=16000, n_mels=80, n_fft=400, hop_length=160):
@@ -429,85 +494,168 @@ def _speech_cache_key(n_samples, n_mels, max_mel_len, max_dec_len):
return hashlib.md5(key.encode()).hexdigest()[:12]
-def prepare_speech_pairs(ds, tokenizer, n_mels=80, max_mel_len=1024, max_dec_len=128):
- """Prepare speech encoder-decoder pairs with <transcribe> task token."""
+def build_audio_augmenter(sr=16000):
+ """Build an audiomentations augmentation pipeline for training.
+
+ Returns an augmenter callable or None if audiomentations is unavailable.
+ """
+ try:
+ import audiomentations as A
+ except ImportError:
+ print(" WARNING: audiomentations not installed — no waveform augmentation")
+ return None
+
+ return A.Compose([
+ A.AddGaussianSNR(min_snr_db=10.0, max_snr_db=35.0, p=0.5),
+ A.AddGaussianNoise(min_amplitude=0.001, max_amplitude=0.015, p=0.3),
+ A.TimeStretch(min_rate=0.9, max_rate=1.1, p=0.3),
+ A.PitchShift(min_semitones=-2, max_semitones=2, p=0.3),
+ A.Gain(min_gain_db=-6, max_gain_db=6, p=0.4),
+ A.LowPassFilter(min_cutoff_freq=3000, max_cutoff_freq=7500, p=0.2),
+ A.HighPassFilter(min_cutoff_freq=50, max_cutoff_freq=400, p=0.2),
+ A.ClippingDistortion(min_percentile_threshold=0, max_percentile_threshold=5, p=0.1),
+ ])
+
+
+def prepare_voice_tool_call_pairs(pairs, tokenizer, n_mels=80, max_mel_len=1024, max_dec_len=1024):
+ """Prepare voice-tool-call decoder arrays with <tool_call> task token.
+
+ Caches only decoder tokens + loss mask + audio arrays (NOT mels).
+ Mel spectrograms are computed on-the-fly by get_speech_batches to allow
+ online waveform augmentation during training.
+
+ Returns (audio_arrays, dec_inputs, dec_targets, loss_mask).
+ """
- cache_id = _speech_cache_key(len(ds), n_mels, max_mel_len, max_dec_len)
+ cache_id = _speech_cache_key(len(pairs), n_mels, max_mel_len, max_dec_len)
cache_path = os.path.join(CACHE_DIR, cache_id)
- if os.path.exists(cache_path + "_mel.npy"):
- print(f"Loading cached speech data ({cache_id})...")
+ arrays_file = cache_path + "_arrays.npy"
+ if os.path.exists(arrays_file):
+ print(f"Loading cached voice-tool-call data ({cache_id})...")
return (
- np.load(cache_path + "_mel.npy"),
+ np.load(arrays_file, allow_pickle=True),
np.load(cache_path + "_dec_in.npy"),
np.load(cache_path + "_dec_tgt.npy"),
+ np.load(cache_path + "_loss_mask.npy"),
)
pad_id = tokenizer.pad_token_id
- bos_id = tokenizer.eos_token_id
eos_id = tokenizer.eos_token_id
- transcribe_id = tokenizer.transcribe_token_id
+ tool_call_id = tokenizer.tool_call_token_id
- mel_features = []
+ audio_arrays = []
dec_inputs_list = []
dec_targets_list = []
+ loss_mask_list = []
- for example in tqdm(ds, desc="Processing speech"):
- audio, sr = _decode_audio(example["audio"])
-
- mel = compute_mel_spectrogram(audio, sr=sr, n_mels=n_mels)
+ skipped = 0
+ for pair in tqdm(pairs, desc="Processing voice-tool-call"):
+ t_tok = tokenizer.encode(pair["tools"])[:max_dec_len - 2]
+ a_tok = tokenizer.encode(pair["answers"])
- # Pad or truncate mel to max_mel_len
- if mel.shape[0] > max_mel_len:
- mel = mel[:max_mel_len]
- elif mel.shape[0] < max_mel_len:
- pad_len = max_mel_len - mel.shape[0]
- mel = np.pad(mel, ((0, pad_len), (0, 0)))
+ prefix_len = 2 + len(t_tok)
+ total_dec = prefix_len + len(a_tok) + 1
- # Tokenize transcript
- text = example["text"].strip().lower()
- tokens = tokenizer.encode(text)
- # Leave room for <transcribe> prefix and EOS
- tokens = tokens[:max_dec_len - 2]
+ if total_dec > max_dec_len:
+ available_for_tools = max_dec_len - 2 - len(a_tok) - 1
+ if available_for_tools < 1:
+ skipped += 1
+ continue
+ t_tok = t_tok[:available_for_tools]
+ prefix_len = 2 + len(t_tok)
- # Decoder input: [BOS, <transcribe>, tok1, tok2, ...]
dec_in = np.full(max_dec_len, pad_id, dtype=np.int32)
- dec_in[0] = bos_id
- dec_in[1] = transcribe_id
- n_tok = len(tokens)
- if n_tok > 0:
- dec_in[2:2 + n_tok] = tokens
+ dec_in[0] = eos_id
+ dec_in[1] = tool_call_id
+ tl = len(t_tok)
+ if tl > 0:
+ dec_in[2:2 + tl] = t_tok
+ al = len(a_tok)
+ if al > 0:
+ dec_in[prefix_len:prefix_len + al] = a_tok
- # Decoder target: [<transcribe>, tok1, tok2, ..., EOS]
+ # Decoder target: [<tool_call>, tools..., answer..., EOS]
dec_tgt = np.full(max_dec_len, pad_id, dtype=np.int32)
- dec_tgt[0] = transcribe_id
- if n_tok > 0:
- dec_tgt[1:1 + n_tok] = tokens
- dec_tgt[1 + n_tok] = eos_id
+ dec_tgt[0] = tool_call_id
+ if tl > 0:
+ dec_tgt[1:1 + tl] = t_tok
+ if al > 0:
+ dec_tgt[prefix_len - 1:prefix_len - 1 + al] = a_tok
+ dec_tgt[prefix_len - 1 + al] = eos_id
- mel_features.append(mel)
+ # Loss mask: 1 on answer positions + EOS only
+ lm = np.zeros(max_dec_len, dtype=np.float32)
+ lm[prefix_len - 1:prefix_len - 1 + al + 1] = 1.0
+
+ audio_arrays.append(pair["audio_array"])
dec_inputs_list.append(dec_in)
dec_targets_list.append(dec_tgt)
+ loss_mask_list.append(lm)
- mel_features = np.stack(mel_features).astype(np.float32)
+ if skipped > 0:
+ print(f" Skipped {skipped} examples (too long for max_dec_len={max_dec_len})")
+
+ audio_arrays = np.array(audio_arrays, dtype=object)
dec_inputs = np.stack(dec_inputs_list)
dec_targets = np.stack(dec_targets_list)
+ loss_mask = np.stack(loss_mask_list)
os.makedirs(CACHE_DIR, exist_ok=True)
- np.save(cache_path + "_mel.npy", mel_features)
+ np.save(arrays_file, audio_arrays)
np.save(cache_path + "_dec_in.npy", dec_inputs)
np.save(cache_path + "_dec_tgt.npy", dec_targets)
- print(f"Cached {len(mel_features):,} speech pairs to {CACHE_DIR}/{cache_id}")
+ np.save(cache_path + "_loss_mask.npy", loss_mask)
+ print(f"Cached {len(audio_arrays):,} voice-tool-call pairs to {CACHE_DIR}/{cache_id}")
- return mel_features, dec_inputs, dec_targets
+ return audio_arrays, dec_inputs, dec_targets, loss_mask
-def get_speech_batches(mel_features, dec_inputs, dec_targets, batch_size, shuffle=True):
- n = len(mel_features)
- if shuffle:
- perm = np.random.permutation(n)
- mel_features = mel_features[perm]
- dec_inputs = dec_inputs[perm]
- dec_targets = dec_targets[perm]
+def _load_mel_batch(audio_arrays, n_mels, max_mel_len, augmenter=None, sr=16000):
+ """Compute mel spectrograms for a batch of audio arrays.
+
+ If augmenter is provided, applies waveform augmentation before mel computation.
+ """
+ mels = []
+ for audio in audio_arrays:
+ audio = np.array(audio, dtype=np.float32)
+ if audio.ndim > 1:
+ audio = audio.mean(axis=1)
+
+ if augmenter is not None:
+ audio = augmenter(samples=audio, sample_rate=sr)
+
+ mel = compute_mel_spectrogram(audio, sr=sr, n_mels=n_mels)
+
+ if mel.shape[0] > max_mel_len:
+ mel = mel[:max_mel_len]
+ elif mel.shape[0] < max_mel_len:
+ pad_len = max_mel_len - mel.shape[0]
+ mel = np.pad(mel, ((0, pad_len), (0, 0)))
+
+ mels.append(mel)
+
+ return np.stack(mels).astype(np.float32)
+
+
+def get_speech_batches(audio_arrays, dec_inputs, dec_targets, batch_size,
+ shuffle=True, loss_mask=None, n_mels=80, max_mel_len=1024,
+ augmenter=None):
+ """Yield speech batches with on-the-fly mel computation and optional augmentation.
+
+ Args:
+ audio_arrays: array of audio waveform arrays (numpy float32)
+ dec_inputs, dec_targets, loss_mask: pre-tokenized decoder arrays
+ augmenter: audiomentations Compose pipeline (None = no augmentation)
+ """
+ n = len(audio_arrays)
+ indices = np.random.permutation(n) if shuffle else np.arange(n)
for i in range(0, n - batch_size + 1, batch_size):
- yield mel_features[i : i + batch_size], dec_inputs[i : i + batch_size], dec_targets[i : i + batch_size]
+ batch_idx = indices[i : i + batch_size]
+ batch_audio = audio_arrays[batch_idx]
+ mel_batch = _load_mel_batch(batch_audio, n_mels, max_mel_len, augmenter=augmenter)
+
+ batch = (mel_batch, dec_inputs[batch_idx], dec_targets[batch_idx])
+ if loss_mask is not None:
+ batch = batch + (loss_mask[batch_idx],)
+ yield batch
diff --git a/src/evaluate.py b/src/evaluate.py
index 39a8a7c..911b853 100644
--- a/src/evaluate.py
+++ b/src/evaluate.py
@@ -1,6 +1,5 @@
import argparse
import math
-import pickle
import jax
import jax.numpy as jnp
@@ -15,14 +14,7 @@ from .model import (
make_causal_mask,
make_padding_mask,
)
-
-
-def load_checkpoint(path):
- with open(path, "rb") as f:
- data = pickle.load(f)
- params = jax.tree.map(jnp.array, data["params"])
- config = TransformerConfig(**data["config"])
- return params, config
+from .run import load_checkpoint
def _make_p_encode(model):
@@ -36,41 +28,27 @@ def _make_p_encode(model):
def _make_p_decode(model):
"""Create a pmap'd decode function."""
- def _decode(params, dec_input, encoder_out, tgt_mask, cross_mask):
+ def _decode(params, dec_input, encoder_out, tgt_mask, _unused_cross_mask):
return model.apply(
{"params": params}, dec_input, encoder_out,
- self_mask=tgt_mask, cross_mask=cross_mask, method="decode",
+ self_mask=tgt_mask, method="decode",
)
return jax.pmap(_decode, axis_name="batch")
-def _make_p_forward(model):
- """Create a pmap'd full forward function."""
- def _forward(params, src, tgt_in, src_mask, tgt_mask, cross_mask):
- return model.apply(
- {"params": params}, src, tgt_in,
- src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
- )
- return jax.pmap(_forward, axis_name="batch")
-
-
def _shard_single(x, num_devices):
"""Replicate a single-sample batch across all devices for pmap."""
return jnp.broadcast_to(x, (num_devices, *x.shape[1:]))
-def _shard_batch(x, num_devices):
- """Reshape batch for pmap: (N, ...) -> (num_devices, per_device, ...)."""
- return x.reshape(num_devices, -1, *x.shape[1:])
-
-
-def score_sequence(model, params, enc_tokens, dec_tokens, pad_id, p_encode=None, p_decode=None, num_devices=1):
+def score_sequence(model, params, enc_tokens, dec_tokens, pad_id, sos_id=None,
+ p_encode=None, p_decode=None, num_devices=1):
"""Compute average negative log-likelihood of dec_tokens given enc_tokens."""
+ sos = sos_id if sos_id is not None else pad_id
enc_input = jnp.array([enc_tokens])
src_mask = make_padding_mask(enc_input, pad_id)
if p_encode is not None and num_devices > 1:
- # Replicate single sample across devices, run pmap, take first result
enc_s = _shard_single(enc_input, num_devices)
src_mask_s = _shard_single(src_mask, num_devices)
encoder_out = p_encode(params, enc_s, src_mask_s)[0:1]
@@ -80,22 +58,20 @@ def score_sequence(model, params, enc_tokens, dec_tokens, pad_id, p_encode=None,
{"params": p}, enc_input, src_mask=src_mask, method="encode",
)
- dec_in = [pad_id] + list(dec_tokens[:-1])
+ dec_in = [sos] + list(dec_tokens[:-1])
dec_input = jnp.array([dec_in])
tgt_mask = make_causal_mask(len(dec_in))
- cross_mask = src_mask
if p_decode is not None and num_devices > 1:
dec_s = _shard_single(dec_input, num_devices)
tgt_mask_s = jnp.broadcast_to(tgt_mask, (num_devices, *tgt_mask.shape[1:]))
- cross_mask_s = _shard_single(cross_mask, num_devices)
enc_out_s = _shard_single(encoder_out, num_devices)
- logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, cross_mask_s)[0]
+ logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, None)[0]
else:
p = params if num_devices <= 1 else jax_utils.unreplicate(params)
logits = model.apply(
{"params": p}, dec_input, encoder_out,
- self_mask=tgt_mask, cross_mask=cross_mask, method="decode",
+ self_mask=tgt_mask, method="decode",
)[0]
log_probs = jax.nn.log_softmax(logits if logits.ndim == 2 else logits[0])
@@ -110,6 +86,7 @@ def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256,
ds = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="test")
pad_id = tokenizer.pad_token_id
+ sos_id = tokenizer.eos_token_id
total_nll = 0.0
total_tokens = 0
evaluated = 0
@@ -142,20 +119,19 @@ def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256,
{"params": single_params}, enc_input, src_mask=src_mask, method="encode",
)
- dec_in = [pad_id] + list(dec_tokens[:-1])
+ dec_in = [sos_id] + list(dec_tokens[:-1])
dec_input = jnp.array([dec_in])
tgt_mask = make_causal_mask(len(dec_in))
if p_decode is not None and num_devices > 1:
dec_s = _shard_single(dec_input, num_devices)
tgt_mask_s = jnp.broadcast_to(tgt_mask, (num_devices, *tgt_mask.shape[1:]))
- cross_mask_s = _shard_single(src_mask, num_devices)
enc_out_s = _shard_single(encoder_out, num_devices)
- logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, cross_mask_s)[0]
+ logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, None)[0]
else:
logits = model.apply(
{"params": single_params}, dec_input, encoder_out,
- self_mask=tgt_mask, cross_mask=src_mask, method="decode",
+ self_mask=tgt_mask, method="decode",
)
log_probs = jax.nn.log_softmax(logits[0] if logits.ndim == 3 else logits[0])
@@ -179,6 +155,7 @@ def eval_lambada(model, params, tokenizer, max_samples=500,
ds = load_dataset("EleutherAI/lambada_openai", "default", split="test")
pad_id = tokenizer.pad_token_id
+ sos_id = tokenizer.eos_token_id
correct = 0
total = 0
@@ -209,19 +186,18 @@ def eval_lambada(model, params, tokenizer, max_samples=500,
{"params": single_params}, enc_input, src_mask=src_mask, method="encode",
)
- dec_in = jnp.array([[pad_id]])
+ dec_in = jnp.array([[sos_id]])
tgt_mask = make_causal_mask(1)
if p_decode is not None and num_devices > 1:
dec_s = _shard_single(dec_in, num_devices)
tgt_mask_s = jnp.broadcast_to(tgt_mask, (num_devices, *tgt_mask.shape[1:]))
- cross_mask_s = _shard_single(src_mask, num_devices)
enc_out_s = _shard_single(encoder_out, num_devices)
- logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, cross_mask_s)[0]
+ logits = p_decode(params, dec_s, enc_out_s, tgt_mask_s, None)[0]
else:
logits = model.apply(
{"params": single_params}, dec_in, encoder_out,
- self_mask=tgt_mask, cross_mask=src_mask, method="decode",
+ self_mask=tgt_mask, method="decode",
)
predicted = int(jnp.argmax(logits[0, 0] if logits.ndim == 3 else logits[0, 0]))
@@ -242,6 +218,7 @@ def eval_hellaswag(model, params, tokenizer, max_samples=500,
ds = load_dataset("Rowan/hellaswag", split="validation")
pad_id = tokenizer.pad_token_id
+ sos_id = tokenizer.eos_token_id
correct = 0
total = 0
@@ -254,7 +231,6 @@ def eval_hellaswag(model, params, tokenizer, max_samples=500,
if not enc_tokens:
continue
- # Truncate context to keep it manageable
enc_tokens = enc_tokens[:192]
scores = []
@@ -265,7 +241,8 @@ def eval_hellaswag(model, params, tokenizer, max_samples=500,
continue
dec_tokens = dec_tokens[:64]
score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id,
- p_encode=p_encode, p_decode=p_decode, num_devices=num_devices)
+ sos_id=sos_id, p_encode=p_encode, p_decode=p_decode,
+ num_devices=num_devices)
scores.append(score)
predicted = int(np.argmax(scores))
@@ -286,6 +263,7 @@ def eval_arc_easy(model, params, tokenizer, max_samples=500,
ds = load_dataset("allenai/ai2_arc", "ARC-Easy", split="test")
pad_id = tokenizer.pad_token_id
+ sos_id = tokenizer.eos_token_id
correct = 0
total = 0
@@ -309,7 +287,8 @@ def eval_arc_easy(model, params, tokenizer, max_samples=500,
continue
dec_tokens = dec_tokens[:64]
score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id,
- p_encode=p_encode, p_decode=p_decode, num_devices=num_devices)
+ sos_id=sos_id, p_encode=p_encode, p_decode=p_decode,
+ num_devices=num_devices)
scores.append(score)
predicted_idx = int(np.argmax(scores))
diff --git a/src/export.py b/src/export.py
index add15d7..8b5d00e 100644
--- a/src/export.py
+++ b/src/export.py
@@ -15,7 +15,6 @@ import numpy as np
from .model import TransformerConfig
-# FFN kernel names that should be sliced in the d_ff dimension
_FFN_KERNEL_NAMES = {"gate_proj", "up_proj", "down_proj"}
@@ -42,7 +41,6 @@ def export_submodel(checkpoint_path, factor, output_path):
if arr.ndim != 2:
return arr
- # Check if this is an FFN kernel by looking at parent module name
parent_name = None
for part in key_path:
name = part.key if hasattr(part, "key") else str(part)
@@ -53,15 +51,11 @@ def export_submodel(checkpoint_path, factor, output_path):
if parent_name is None:
return arr
- # Both token_mix and channel_mix are sliced via ffn_mask
-
rows, cols = arr.shape
if parent_name in ("gate_proj", "up_proj"):
- # (d_in, d_ff) → (d_in, d_ff_new)
if cols == d_ff:
return arr[:, :d_ff_new]
elif parent_name == "down_proj":
- # (d_ff, d_out) → (d_ff_new, d_out)
if rows == d_ff:
return arr[:d_ff_new, :]
diff --git a/src/model.py b/src/model.py
index 7abbd21..9a16661 100644
--- a/src/model.py
+++ b/src/model.py
@@ -292,26 +292,6 @@ class Decoder(nn.Module):
return x
-class ConvSubsampling(nn.Module):
- """Whisper-style convolutional frontend: two Conv1d layers with GELU, 4x temporal downsampling."""
- d_model: int
- n_mels: int
- dtype: jnp.dtype = jnp.bfloat16
-
- @nn.compact
- def __call__(self, mel):
- # mel: (B, T, n_mels)
- x = nn.Conv(features=self.d_model, kernel_size=(3,), strides=(2,),
- padding='SAME', use_bias=True, kernel_init=default_init(),
- dtype=self.dtype, name="conv1")(mel)
- x = nn.gelu(x)
- x = nn.Conv(features=self.d_model, kernel_size=(3,), strides=(2,),
- padding='SAME', use_bias=True, kernel_init=default_init(),
- dtype=self.dtype, name="conv2")(x)
- x = nn.gelu(x)
- return x
-
-
class SpecAugment(nn.Module):
"""SpecAugment: time and frequency masking for speech regularization."""
num_time_masks: int = 2
@@ -354,7 +334,8 @@ class EncoderDecoderTransformer(nn.Module):
def setup(self):
self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))
self.embed_scale = math.sqrt(self.config.d_model)
- self.conv_subsample = ConvSubsampling(self.config.d_model, self.config.n_mels, self.config.jax_dtype)
+ self.mel_proj = nn.Dense(self.config.d_model, dtype=self.config.jax_dtype,
+ use_bias=True, kernel_init=default_init(), name="mel_proj")
self.spec_augment = SpecAugment()
self.encoder = MemoryMixerEncoder(self.config)
self.decoder = Decoder(self.config)
@@ -372,40 +353,23 @@ class EncoderDecoderTransformer(nn.Module):
"""Backward-compatible alias for encode_text."""
return self.encode_text(src, src_mask=src_mask)
- @staticmethod
- def _downsample_mask(mask, factor):
- """Downsample a padding mask by OR-pooling groups of `factor` frames."""
- if mask is None:
- return None
- # mask: (B, 1, 1, T)
- T = mask.shape[-1]
- pad = (factor - T % factor) % factor
- if pad > 0:
- mask = jnp.pad(mask, ((0, 0), (0, 0), (0, 0), (0, pad)))
- mask = mask.reshape(mask.shape[0], 1, 1, -1, factor).any(axis=-1)
- return mask
-
def encode_speech(self, mel, src_mask=None, ffn_mask=None, deterministic=True):
mel = self.spec_augment(mel, deterministic=deterministic)
- x = self.conv_subsample(mel) * self.embed_scale
- src_mask = self._downsample_mask(src_mask, 4)
+ x = self.mel_proj(mel) * self.embed_scale
rope = self._rope(x.shape[1])
return self.encoder(x, mask=src_mask, rope=rope, ffn_mask=ffn_mask)
- def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None):
+ def decode(self, tgt, encoder_out, self_mask=None):
+ """Decode from encoder memory slots. No cross_mask needed (fixed-size slots)."""
x = self.embedding(tgt) * self.embed_scale
rope = self._rope(tgt.shape[1])
- x = self.decoder(
- x, encoder_out, self_mask=self_mask, cross_mask=None, rope=rope
- )
+ x = self.decoder(x, encoder_out, self_mask=self_mask, cross_mask=None, rope=rope)
logits = x.astype(jnp.float32) @ self.embedding.embedding.T
return logits
- def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):
+ def __call__(self, src, tgt, src_mask=None, tgt_mask=None):
encoder_out = self.encode_text(src, src_mask=src_mask)
- logits = self.decode(
- tgt, encoder_out, self_mask=tgt_mask
- )
+ logits = self.decode(tgt, encoder_out, self_mask=tgt_mask)
return logits
def _run_decoder(self, encoder_out, tgt, tgt_mask=None, ffn_mask=None):
@@ -442,7 +406,7 @@ class EncoderDecoderTransformer(nn.Module):
mask = (jnp.arange(self.config.d_ff) < ff_width).astype(dtype)
return jnp.broadcast_to(mask[None, :], (B, self.config.d_ff))
- def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mat_ff_widths=None):
+ def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, mat_ff_widths=None):
"""Eval-only: separate per-width forwards for reporting per-width PPL.
mat_ff_widths: list of FFN widths to evaluate (e.g. [1024, 512, 256]).
diff --git a/src/run.py b/src/run.py
index 0dba654..a9a00ba 100644
--- a/src/run.py
+++ b/src/run.py
@@ -55,13 +55,12 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=
dec_buffer = dec_buffer.at[0, j].set(tok)
@jax.jit
- def decode_step(dec_buffer, encoder_out, src_mask):
+ def decode_step(dec_buffer, encoder_out):
logits = model.apply(
{"params": params},
dec_buffer,
encoder_out,
self_mask=tgt_mask,
- cross_mask=src_mask,
method="decode",
)
return logits
@@ -72,10 +71,8 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=
sys.stdout.write(f"\n")
sys.stdout.flush()
- # Compile once on first call
- logits = decode_step(dec_buffer, encoder_out, src_mask)
+ logits = decode_step(dec_buffer, encoder_out)
- # Greedy decoding from end of prefix
for i in range(prefix_len - 1, max_gen_len - 1):
next_logits = logits[0, i]
next_token = int(jnp.argmax(next_logits))
@@ -90,7 +87,7 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=
sys.stdout.write(tokenizer.decode([next_token]))
sys.stdout.flush()
- logits = decode_step(dec_buffer, encoder_out, src_mask)
+ logits = decode_step(dec_buffer, encoder_out)
if stream:
sys.stdout.write("\n")
@@ -114,52 +111,57 @@ def load_audio(path, target_sr=16000):
return audio, sr
-def transcribe(model, params, tokenizer, audio_array, sr=16000, max_gen_len=128, temperature=0.8, seed=0, stream=True):
- """Transcribe audio using the speech encoder pathway."""
+def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="[]", max_gen_len=512, seed=0, stream=True):
+ """Generate tool-call output from audio using the speech encoder pathway.
+
+ mel -> encode_speech -> decoder [BOS, <tool_call>, tools_tokens...] -> greedy decode.
+ Same structure as generate() but uses speech encoder + mel padding mask.
+ """
n_mels = model.config.n_mels
mel = compute_mel_spectrogram(audio_array, sr=sr, n_mels=n_mels)
mel_input = jnp.array(mel)[None, :, :] # (1, T_mel, n_mels)
pad_id = tokenizer.pad_token_id
eos_id = tokenizer.eos_token_id
- transcribe_id = tokenizer.transcribe_token_id
+ tool_call_id = tokenizer.tool_call_token_id
src_mask = make_mel_padding_mask(mel_input)
encoder_out = model.apply(
{"params": params}, mel_input, src_mask=src_mask, deterministic=True, method="encode_speech"
)
+ # Build decoder prefix: [BOS, <tool_call>, tools_tokens...]
+ tools_tokens = tokenizer.encode(tools)
+ prefix = [eos_id, tool_call_id] + tools_tokens
+ prefix_len = len(prefix)
+
tgt_mask = make_causal_mask(max_gen_len)
dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
- dec_buffer = dec_buffer.at[0, 0].set(eos_id) # BOS = EOS in this tokenizer
- dec_buffer = dec_buffer.at[0, 1].set(transcribe_id)
+ for j, tok in enumerate(prefix[:max_gen_len]):
+ dec_buffer = dec_buffer.at[0, j].set(tok)
@jax.jit
- def decode_step(dec_buffer, encoder_out, src_mask):
+ def decode_step(dec_buffer, encoder_out):
logits = model.apply(
{"params": params},
dec_buffer,
encoder_out,
self_mask=tgt_mask,
- cross_mask=src_mask,
method="decode",
)
return logits
- rng = jax.random.PRNGKey(seed)
generated_tokens = []
if stream:
sys.stdout.write("\n")
sys.stdout.flush()
- logits = decode_step(dec_buffer, encoder_out, src_mask)
+ logits = decode_step(dec_buffer, encoder_out)
- # Start generating from position 2 (after BOS + <transcribe>)
- for i in range(1, max_gen_len - 1):
- next_logits = logits[0, i] / temperature
- rng, sample_rng = jax.random.split(rng)
- next_token = jax.random.categorical(sample_rng, next_logits).item()
+ for i in range(prefix_len - 1, max_gen_len - 1):
+ next_logits = logits[0, i]
+ next_token = int(jnp.argmax(next_logits))
if next_token == eos_id:
break
@@ -171,7 +173,7 @@ def transcribe(model, params, tokenizer, audio_array, sr=16000, max_gen_len=128,
sys.stdout.write(tokenizer.decode([next_token]))
sys.stdout.flush()
- logits = decode_step(dec_buffer, encoder_out, src_mask)
+ logits = decode_step(dec_buffer, encoder_out)
if stream:
sys.stdout.write("\n")
@@ -189,20 +191,22 @@ def main(args):
param_count = sum(x.size for x in jax.tree.leaves(params))
print(f"Model parameters: {param_count:,}")
- # --- Audio transcription mode ---
+ # --- Voice-to-tool-call mode ---
audio_files = getattr(args, "audio", None)
if audio_files:
+ tools = getattr(args, "tools", None) or "[]"
for i, audio_path in enumerate(audio_files):
- print(f"\nTranscribing: {audio_path}")
+ print(f"\nVoice-to-tool-call: {audio_path}")
+ print(f"Tools: {tools[:80]}{'...' if len(tools) > 80 else ''}")
audio, sr = load_audio(audio_path)
- transcribe(
+ generate_from_audio(
model,
params,
tokenizer,
audio,
sr=sr,
+ tools=tools,
max_gen_len=args.max_len,
- temperature=args.temperature,
seed=args.seed + i,
stream=True,
)
@@ -241,7 +245,7 @@ def parse_args():
parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")
parser.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")
parser.add_argument("--tools", type=str, default=None, help="Tools JSON for tool-call generation")
- parser.add_argument("--audio", type=str, nargs="*", help="Audio file paths to transcribe")
+ parser.add_argument("--audio", type=str, nargs="*", help="Audio file paths for voice-to-tool-call")
parser.add_argument("--max-len", type=int, default=512)
parser.add_argument("--seed", type=int, default=0)
return parser.parse_args()
diff --git a/src/test.py b/src/test.py
index 515b516..b454a97 100644
--- a/src/test.py
+++ b/src/test.py
@@ -1,6 +1,5 @@
import argparse
import math
-import pickle
import time
import jax
@@ -8,21 +7,14 @@ import jax.numpy as jnp
import numpy as np
import optax
-from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs, load_librispeech
+from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs, load_tool_call_audio
from .model import (
EncoderDecoderTransformer,
TransformerConfig,
make_causal_mask,
make_padding_mask,
)
-
-
-def load_checkpoint(path):
- with open(path, "rb") as f:
- data = pickle.load(f)
- params = jax.tree.map(jnp.array, data["params"])
- config = TransformerConfig(**data["config"])
- return params, config
+from .run import load_checkpoint
def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batch_size, pad_id, loss_mask=None):
@@ -41,11 +33,10 @@ def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batch
src_mask = make_padding_mask(src, pad_id)
tgt_mask = make_causal_mask(tgt_in.shape[1]) & make_padding_mask(tgt_in, pad_id)
- cross_mask = make_padding_mask(src, pad_id)
logits = model.apply(
{"params": params}, src, tgt_in,
- src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
+ src_mask=src_mask, tgt_mask=tgt_mask,
)
loss = optax.softmax_cross_entropy_with_integer_labels(logits, tgt_out)
@@ -66,22 +57,20 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is th
tgt_mask = make_causal_mask(max_gen_len)
@jax.jit
- def decode_step(dec_buffer, encoder_out, src_mask):
+ def decode_step(dec_buffer, encoder_out):
logits = model.apply(
{"params": params}, dec_buffer, encoder_out,
- self_mask=tgt_mask, cross_mask=src_mask, method="decode",
+ self_mask=tgt_mask, method="decode",
)
return logits
- # Warmup — compile once
encoder_out = model.apply(
{"params": params}, enc_input, src_mask=src_mask, method="encode"
)
dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
dec_buffer = dec_buffer.at[0, 0].set(eos_id)
- decode_step(dec_buffer, encoder_out, src_mask)
+ decode_step(dec_buffer, encoder_out)
- # Benchmark generation
tokens_generated = []
latencies = []
@@ -94,7 +83,7 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is th
encoder_out = model.apply(
{"params": params}, enc_input, src_mask=src_mask, method="encode"
)
- logits = decode_step(dec_buffer, encoder_out, src_mask)
+ logits = decode_step(dec_buffer, encoder_out)
num_tokens = 0
for i in range(max_gen_len - 1):
@@ -106,7 +95,7 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is th
num_tokens += 1
dec_buffer = dec_buffer.at[0, i + 1].set(next_token)
- logits = decode_step(dec_buffer, encoder_out, src_mask)
+ logits = decode_step(dec_buffer, encoder_out)
elapsed = time.perf_counter() - start
tokens_generated.append(num_tokens)
@@ -196,11 +185,9 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
total = 0
exact_match = 0
- # For function-name-level P/R/F1
tp_names = 0 # predicted name in reference
fp_names = 0 # predicted name not in reference
fn_names = 0 # reference name not predicted
- # For full call-level (name + args) P/R/F1
tp_calls = 0
fp_calls = 0
fn_calls = 0
@@ -218,13 +205,11 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
tools=ex["tools"], max_gen_len=max_gen_len, seed=i, stream=False,
).strip()
- # Parse reference
try:
ref_calls = json.loads(ref_text)
except (json.JSONDecodeError, TypeError):
ref_calls = []
- # Parse prediction
try:
pred_calls = json.loads(pred_text)
if not isinstance(pred_calls, list):
@@ -237,17 +222,14 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
if i < 10:
samples.append((ex["query"][:80], ref_text[:120], pred_text[:120]))
- # Track empty
if len(ref_calls) == 0:
empty_ref += 1
if len(pred_calls) == 0:
empty_pred += 1
- # Exact match (normalized)
if json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):
exact_match += 1
- # Function name P/R/F1
ref_names = [c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c]
pred_names = [c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c]
ref_name_set = set(ref_names)
@@ -256,7 +238,6 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
fp_names += len(pred_name_set - ref_name_set)
fn_names += len(ref_name_set - pred_name_set)
- # Full call P/R/F1 (name + args must match)
def call_key(c):
if not isinstance(c, dict):
return None
@@ -270,7 +251,6 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
fp_calls += len(pred_keys - ref_keys)
fn_calls += len(ref_keys - pred_keys)
- # Compute metrics
name_precision = tp_names / max(tp_names + fp_names, 1)
name_recall = tp_names / max(tp_names + fn_names, 1)
name_f1 = 2 * name_precision * name_recall / max(name_precision + name_recall, 1e-9)
@@ -295,32 +275,104 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
}
-def benchmark_asr(model, params, tokenizer, num_samples=100):
- """Transcribe LibriSpeech test-clean samples and report WER."""
- from .run import transcribe
- from .data import load_librispeech
+def benchmark_voice_tool_calls(model, params, tokenizer, num_samples=100, max_gen_len=512):
+ """Generate tool-call predictions from audio and compute structured metrics."""
+ import json
+ from .run import generate_from_audio
- ds = load_librispeech("test.clean", max_samples=num_samples)
+ pairs = load_tool_call_audio("validation", max_samples=num_samples)
- hypotheses = []
- references = []
+ total = 0
+ exact_match = 0
+ tp_names = 0
+ fp_names = 0
+ fn_names = 0
+ tp_calls = 0
+ fp_calls = 0
+ fn_calls = 0
+ json_parse_errors = 0
+ empty_ref = 0
+ empty_pred = 0
+ samples = []
- for example in ds:
- audio_data = example["audio"]
- audio = np.array(audio_data["array"], dtype=np.float32)
- sr = audio_data["sampling_rate"]
- ref_text = example["text"].strip().lower()
+ for i, pair in enumerate(pairs):
+ ref_text = pair["answers"]
+ audio = pair["audio_array"]
+ sr = pair["sampling_rate"]
- hyp_text = transcribe(
+ pred_text = generate_from_audio(
model, params, tokenizer, audio, sr=sr,
- max_gen_len=128, temperature=0.0, seed=0, stream=False,
- ).strip().lower()
+ tools=pair["tools"], max_gen_len=max_gen_len, seed=i, stream=False,
+ ).strip()
+
+ try:
+ ref_calls = json.loads(ref_text)
+ except (json.JSONDecodeError, TypeError):
+ ref_calls = []
+
+ try:
+ pred_calls = json.loads(pred_text)
+ if not isinstance(pred_calls, list):
+ pred_calls = [pred_calls] if isinstance(pred_calls, dict) else []
+ except (json.JSONDecodeError, TypeError):
+ json_parse_errors += 1
+ pred_calls = []
- hypotheses.append(hyp_text)
- references.append(ref_text)
+ total += 1
+ if i < 10:
+ samples.append((pair["query"][:80], ref_text[:120], pred_text[:120]))
+
+ if len(ref_calls) == 0:
+ empty_ref += 1
+ if len(pred_calls) == 0:
+ empty_pred += 1
- wer = compute_wer(hypotheses, references)
- return {"wer": wer, "num_samples": len(hypotheses)}
+ if json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):
+ exact_match += 1
+
+ ref_names = [c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c]
+ pred_names = [c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c]
+ ref_name_set = set(ref_names)
+ pred_name_set = set(pred_names)
+ tp_names += len(pred_name_set & ref_name_set)
+ fp_names += len(pred_name_set - ref_name_set)
+ fn_names += len(ref_name_set - pred_name_set)
+
+ def call_key(c):
+ if not isinstance(c, dict):
+ return None
+ return json.dumps({"name": c.get("name"), "arguments": c.get("arguments")}, sort_keys=True)
+
+ ref_keys = set(call_key(c) for c in ref_calls)
+ pred_keys = set(call_key(c) for c in pred_calls)
+ ref_keys.discard(None)
+ pred_keys.discard(None)
+ tp_calls += len(pred_keys & ref_keys)
+ fp_calls += len(pred_keys - ref_keys)
+ fn_calls += len(ref_keys - pred_keys)
+
+ name_precision = tp_names / max(tp_names + fp_names, 1)
+ name_recall = tp_names / max(tp_names + fn_names, 1)
+ name_f1 = 2 * name_precision * name_recall / max(name_precision + name_recall, 1e-9)
+
+ call_precision = tp_calls / max(tp_calls + fp_calls, 1)
+ call_recall = tp_calls / max(tp_calls + fn_calls, 1)
+ call_f1 = 2 * call_precision * call_recall / max(call_precision + call_recall, 1e-9)
+
+ return {
+ "num_samples": total,
+ "exact_match": exact_match / max(total, 1),
+ "json_parse_rate": 1.0 - json_parse_errors / max(total, 1),
+ "name_precision": name_precision,
+ "name_recall": name_recall,
+ "name_f1": name_f1,
+ "call_precision": call_precision,
+ "call_recall": call_recall,
+ "call_f1": call_f1,
+ "empty_ref_pct": empty_ref / max(total, 1),
+ "empty_pred_pct": empty_pred / max(total, 1),
+ "samples": samples,
+ }
def main(args):
@@ -371,11 +423,29 @@ def main(args):
print(f" P: {pred}")
print()
- asr_samples = getattr(args, "asr_samples", 50)
- if asr_samples > 0:
- print(f"\nevaluating ASR WER ({asr_samples} samples)...")
- asr_results = benchmark_asr(model, params, tokenizer, num_samples=asr_samples)
- print(f"ASR WER: {asr_results['wer']:.4f} ({asr_results['num_samples']} samples)")
+ voice_tc_samples = getattr(args, "voice_tc_samples", 50)
+ if voice_tc_samples > 0:
+ print(f"\nevaluating voice-to-tool-call ({voice_tc_samples} samples)...")
+ vtc = benchmark_voice_tool_calls(model, params, tokenizer, num_samples=voice_tc_samples, max_gen_len=args.max_gen_len)
+ print(f"\n ─── Voice-Tool-Call Metrics ─────────")
+ print(f" JSON parse rate {vtc['json_parse_rate']:>10.1%}")
+ print(f" Exact match {vtc['exact_match']:>10.1%}")
+ print(f" ─── Function Name ───────────────────")
+ print(f" Precision {vtc['name_precision']:>10.3f}")
+ print(f" Recall {vtc['name_recall']:>10.3f}")
+ print(f" F1 {vtc['name_f1']:>10.3f}")
+ print(f" ─── Full Call (name+args) ───────────")
+ print(f" Precision {vtc['call_precision']:>10.3f}")
+ print(f" Recall {vtc['call_recall']:>10.3f}")
+ print(f" F1 {vtc['call_f1']:>10.3f}")
+ print(f" ─────────────────────────────────────")
+ if vtc["samples"]:
+ print(f"\n samples:")
+ for query, ref, pred in vtc["samples"][:5]:
+ print(f" Q: {query}")
+ print(f" R: {ref}")
+ print(f" P: {pred}")
+ print()
print(f"\nmeasuring throughput ({args.throughput_runs} runs)...")
throughput = measure_throughput(model, params, tokenizer, num_runs=args.throughput_runs)
@@ -396,6 +466,8 @@ def parse_args():
parser.add_argument("--throughput-runs", type=int, default=10)
parser.add_argument("--tool-call-samples", type=int, default=200,
help="Samples for tool-call accuracy eval (default: 200)")
+ parser.add_argument("--voice-tc-samples", type=int, default=50,
+ help="Samples for voice-to-tool-call eval (default: 50)")
return parser.parse_args()
diff --git a/src/train.py b/src/train.py
index 0484b4f..9e14141 100644
--- a/src/train.py
+++ b/src/train.py
@@ -15,7 +15,8 @@ from flax.training import train_state
from .data import (
get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs,
- load_librispeech, prepare_speech_pairs, get_speech_batches,
+ load_tool_call_audio, prepare_voice_tool_call_pairs, get_speech_batches,
+ build_audio_augmenter,
)
from .model import (
EncoderDecoderTransformer,
@@ -181,7 +182,6 @@ def _make_prune_mask(params, sparsity, group_size):
Returns binary mask tree matching param shapes and dtypes.
"""
- # Pass 1: collect block L1 scores as JAX arrays
all_scores = []
for _, leaf in jax.tree_util.tree_leaves_with_path(params):
if leaf.ndim != 2:
@@ -198,7 +198,6 @@ def _make_prune_mask(params, sparsity, group_size):
threshold = jnp.percentile(jnp.concatenate(all_scores), sparsity * 100)
- # Pass 2: build per-leaf binary mask (in param dtype to save memory)
def _leaf_mask(path, leaf):
if leaf.ndim != 2:
return jnp.ones_like(leaf)
@@ -225,8 +224,8 @@ def _quantize_params(params, group_size=32):
_GROUP_SIZE = 32
_MAT_FACTORS = ()
-_MAT_FF_WIDTHS = () # precomputed d_ff widths per factor
-_D_FF = 2048 # set in train()
+_MAT_FF_WIDTHS = ()
+_D_FF = 2048
def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, loss_mask):
@@ -249,7 +248,7 @@ def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, lo
return ce_loss + z_loss + div_loss
-def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng):
+def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask):
pad_id = 0
src_mask = make_mel_padding_mask(mel)
tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
@@ -262,7 +261,7 @@ def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask, ffn_mask,
rngs={"specaugment": rng},
)
logits_f32 = logits.astype(jnp.float32)
- mask = (tgt_out != pad_id).astype(jnp.float32)
+ mask = loss_mask
ce_loss = jnp.sum(
optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask
) / jnp.maximum(jnp.sum(mask), 1.0)
@@ -318,10 +317,10 @@ def _train_step_text_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask
return state, ema_params, loss, grad_norm
-def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng):
+def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask):
ema_decay = 0.999
loss, grads = jax.value_and_grad(
- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng)
+ lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)
)(state.params)
grads = jax.lax.pmean(grads, axis_name="batch")
loss = jax.lax.pmean(loss, axis_name="batch")
@@ -331,11 +330,11 @@ def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask, ffn
return state, ema_params, loss, grad_norm
-def _train_step_speech_masked(state, ema_params, mel, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng):
+def _train_step_speech_masked(state, ema_params, mel, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng, loss_mask):
"""Speech training step with fused prune mask application."""
ema_decay = 0.999
loss, grads = jax.value_and_grad(
- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng)
+ lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)
)(state.params)
grads = jax.lax.pmean(grads, axis_name="batch")
loss = jax.lax.pmean(loss, axis_name="batch")
@@ -371,7 +370,7 @@ def _make_val_loss_fn(apply_fn):
tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
logits = apply_fn(
{"params": params}, src, tgt_in,
- src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=src_mask,
+ src_mask=src_mask, tgt_mask=tgt_mask,
)
loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)
return jnp.sum(loss * loss_mask), jnp.sum(loss_mask)
@@ -387,7 +386,7 @@ def _make_mat_val_loss_fn(apply_fn, ff_width):
tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
logits, _, mat_logits = apply_fn(
{"params": params}, src, tgt_in,
- src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=src_mask,
+ src_mask=src_mask, tgt_mask=tgt_mask,
mat_ff_widths=(ff_width,),
method="forward_with_aux",
)
@@ -399,7 +398,7 @@ def _make_mat_val_loss_fn(apply_fn, ff_width):
def _make_speech_val_loss_fn(apply_fn):
@jax.jit
- def val_loss_batch(params, mel, tgt_in, tgt_out, causal_mask):
+ def val_loss_batch(params, mel, tgt_in, tgt_out, causal_mask, loss_mask):
pad_id = 0
src_mask = make_mel_padding_mask(mel)
tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
@@ -410,7 +409,7 @@ def _make_speech_val_loss_fn(apply_fn):
method="forward_speech_with_aux",
)
loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)
- mask = (tgt_out != pad_id).astype(jnp.float32)
+ mask = loss_mask
return jnp.sum(loss * mask), jnp.sum(mask)
return val_loss_batch
@@ -446,11 +445,8 @@ def shard_batch(batch, num_devices):
def train(args):
- import itertools
-
num_devices = jax.local_device_count()
no_speech = getattr(args, "no_speech", False)
- speech_every_override = getattr(args, "speech_every", None)
n_mels = getattr(args, "n_mels", 80)
max_mel_len = getattr(args, "max_mel_len", 1024)
@@ -490,31 +486,35 @@ def train(args):
val_loss_mask = np.array(val_loss_mask)
print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs")
- # --- Speech data ---
- speech_mel = speech_dec_in = speech_dec_tgt = None
- val_speech_mel = val_speech_dec_in = val_speech_dec_tgt = None
+ speech_audio_arrays = speech_dec_in = speech_dec_tgt = speech_loss_mask_arr = None
+ val_speech_audio_arrays = val_speech_dec_in = val_speech_dec_tgt = val_speech_loss_mask = None
+ audio_augmenter = None
if not no_speech:
step_idx += 1
max_speech_samples = getattr(args, "max_speech_samples", None)
- print(f"\n[{step_idx}/{total_data_steps}] Loading LibriSpeech...")
- speech_ds = load_librispeech("train.clean.100", max_samples=max_speech_samples)
- speech_val_ds = load_librispeech("validation.clean", max_samples=getattr(args, "max_eval_samples", None))
+ print(f"\n[{step_idx}/{total_data_steps}] Loading voice-tool-call audio...")
+ speech_pairs = load_tool_call_audio("train", max_samples=max_speech_samples)
+ speech_val_pairs = load_tool_call_audio("validation", max_samples=getattr(args, "max_eval_samples", None))
step_idx += 1
- print(f"\n[{step_idx}/{total_data_steps}] Preparing speech pairs...")
- speech_mel, speech_dec_in, speech_dec_tgt = prepare_speech_pairs(
- speech_ds, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len
+ print(f"\n[{step_idx}/{total_data_steps}] Preparing voice-tool-call pairs...")
+ speech_audio_arrays, speech_dec_in, speech_dec_tgt, speech_loss_mask_arr = prepare_voice_tool_call_pairs(
+ speech_pairs, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len
)
- val_speech_mel, val_speech_dec_in, val_speech_dec_tgt = prepare_speech_pairs(
- speech_val_ds, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len
+ val_speech_audio_arrays, val_speech_dec_in, val_speech_dec_tgt, val_speech_loss_mask = prepare_voice_tool_call_pairs(
+ speech_val_pairs, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len
)
- speech_mel = np.array(speech_mel)
speech_dec_in = np.array(speech_dec_in)
speech_dec_tgt = np.array(speech_dec_tgt)
- val_speech_mel = np.array(val_speech_mel)
+ speech_loss_mask_arr = np.array(speech_loss_mask_arr)
val_speech_dec_in = np.array(val_speech_dec_in)
val_speech_dec_tgt = np.array(val_speech_dec_tgt)
- print(f" {len(speech_mel):,} train / {len(val_speech_mel):,} val speech pairs")
+ val_speech_loss_mask = np.array(val_speech_loss_mask)
+ print(f" {len(speech_audio_arrays):,} train / {len(val_speech_audio_arrays):,} val voice-tool-call pairs")
+
+ audio_augmenter = build_audio_augmenter(sr=16000)
+ if audio_augmenter is not None:
+ print(f" Audio augmentation: enabled ({len(audio_augmenter.transforms)} transforms)")
effective_batch_size = args.batch_size * num_devices
@@ -562,18 +562,12 @@ def train(args):
rng, init_rng = jax.random.split(rng)
mat_shared_input = getattr(args, "mat_shared_input", False)
- # With shared input, each unique sample is repeated n_widths times,
- # so we fetch smaller batches but take more steps per epoch.
unique_batch_size = effective_batch_size // n_widths if (mat_shared_input and n_widths > 1) else effective_batch_size
text_batches_per_epoch = len(enc_inputs) // unique_batch_size
- if speech_every_override is not None:
- speech_every = speech_every_override
- elif not no_speech and speech_mel is not None:
- speech_batches_available = len(speech_mel) // effective_batch_size
- speech_every = max(1, text_batches_per_epoch // max(1, speech_batches_available))
+ if not no_speech and speech_audio_arrays is not None:
+ speech_batches_per_epoch = len(speech_audio_arrays) // unique_batch_size
else:
- speech_every = 1
- speech_batches_per_epoch = text_batches_per_epoch // speech_every if (not no_speech and speech_mel is not None) else 0
+ speech_batches_per_epoch = 0
num_batches = text_batches_per_epoch + speech_batches_per_epoch
total_steps = num_batches * args.epochs
warmup_steps = max(1, int(total_steps * args.warmup_ratio))
@@ -605,7 +599,7 @@ def train(args):
print(f" Activation {config.activation:>12}")
print(f" Dtype {config.dtype:>12}")
if not no_speech:
- print(f" Speech every {speech_every} text steps")
+ print(f" Speech {speech_batches_per_epoch} batches/epoch")
print(f" n_mels {n_mels:>12}")
print(f" max_mel_len {max_mel_len:>12}")
else:
@@ -627,8 +621,6 @@ def train(args):
(num_devices, 1, args.max_dec_len, args.max_dec_len),
)
- # Precompute FFN mask for heterogeneous matryoshka training.
- # Batch is split: 1/N full-width + 1/N per mat factor.
text_ffn_mask = _make_ffn_mask(args.batch_size, config.d_ff, _MAT_FF_WIDTHS)
text_ffn_mask = jnp.broadcast_to(
text_ffn_mask[None, :, :],
@@ -670,61 +662,73 @@ def train(args):
text_losses = []
speech_losses = []
- text_batches = get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size, loss_mask=train_loss_mask)
+ text_batches = list(get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size, loss_mask=train_loss_mask))
+
+ speech_batch_list = []
+ if not no_speech and speech_audio_arrays is not None:
+ speech_batch_list = list(
+ get_speech_batches(speech_audio_arrays, speech_dec_in, speech_dec_tgt, unique_batch_size,
+ loss_mask=speech_loss_mask_arr, n_mels=n_mels, max_mel_len=max_mel_len,
+ augmenter=audio_augmenter)
+ )
+
+ steps_this_epoch = len(text_batches) + len(speech_batch_list)
+ text_idx = 0
+ speech_idx = 0
+ pbar = tqdm(range(steps_this_epoch), desc=f"Epoch {epoch + 1}/{args.epochs}")
+
+ for step_i in pbar:
+ t0 = time.perf_counter()
- # Speech batches cycle via itertools.cycle
- speech_iter = None
- if not no_speech and speech_mel is not None:
- speech_iter = iter(itertools.cycle(
- get_speech_batches(speech_mel, speech_dec_in, speech_dec_tgt, unique_batch_size)
- ))
+ do_speech = (step_i % 2 == 1) and speech_idx < len(speech_batch_list)
+ do_text = not do_speech and text_idx < len(text_batches)
+ if not do_speech and not do_text:
+ if text_idx < len(text_batches):
+ do_text = True
+ elif speech_idx < len(speech_batch_list):
+ do_speech = True
+ else:
+ break
- text_step_in_epoch = 0
- pbar = tqdm(text_batches, total=text_batches_per_epoch, desc=f"Epoch {epoch + 1}/{args.epochs}")
+ speech_loss_val = None
+ step_grad_norm = None
- for batch in pbar:
- src, tgt_in, tgt_out, lm = batch
- # --- Text step ---
- t0 = time.perf_counter()
+ if do_text:
+ src, tgt_in, tgt_out, lm = text_batches[text_idx]
+ text_idx += 1
- if n_widths > 1 and mat_shared_input:
- # Shared: each unique input sees all widths. Shard unique samples
- # across devices, then tile within each device's portion so the
- # ffn_mask width sections each get the same unique inputs.
- per_width = args.batch_size // n_widths
- def _tile_for_mat(arr):
- # (unique_batch, ...) -> (devices, per_width, ...) -> tile -> (devices, batch, ...)
- s = arr.reshape(num_devices, per_width, *arr.shape[1:])
- s = np.tile(s, (1, n_widths) + (1,) * (arr.ndim - 1))
- return s
- src_b = _tile_for_mat(src)
- tgt_in_b = _tile_for_mat(tgt_in)
- tgt_out_b = _tile_for_mat(tgt_out)
- lm_b = _tile_for_mat(lm)
- else:
- src_b = shard_batch(src, num_devices)
- tgt_in_b = shard_batch(tgt_in, num_devices)
- tgt_out_b = shard_batch(tgt_out, num_devices)
- lm_b = shard_batch(lm, num_devices)
-
- if prune_mask is not None:
- state, ema_params, loss, grad_norm = p_train_step_masked(
- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, lm_b,
- )
- else:
- state, ema_params, loss, grad_norm = p_train_step(
- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, lm_b,
- )
+ if n_widths > 1 and mat_shared_input:
+ per_width = args.batch_size // n_widths
+ def _tile_for_mat(arr):
+ s = arr.reshape(num_devices, per_width, *arr.shape[1:])
+ return np.tile(s, (1, n_widths) + (1,) * (arr.ndim - 1))
+ src_b = _tile_for_mat(src)
+ tgt_in_b = _tile_for_mat(tgt_in)
+ tgt_out_b = _tile_for_mat(tgt_out)
+ lm_b = _tile_for_mat(lm)
+ else:
+ src_b = shard_batch(src, num_devices)
+ tgt_in_b = shard_batch(tgt_in, num_devices)
+ tgt_out_b = shard_batch(tgt_out, num_devices)
+ lm_b = shard_batch(lm, num_devices)
- text_loss_val = float(loss[0])
- text_losses.append(text_loss_val)
- global_step += 1
- text_step_in_epoch += 1
+ if prune_mask is not None:
+ state, ema_params, loss, grad_norm = p_train_step_masked(
+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, lm_b,
+ )
+ else:
+ state, ema_params, loss, grad_norm = p_train_step(
+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, lm_b,
+ )
- # --- Speech step (every speech_every text steps) ---
- speech_loss_val = None
- if speech_iter is not None and text_step_in_epoch % speech_every == 0:
- mel_batch, sp_tgt_in, sp_tgt_out = next(speech_iter)
+ text_loss_val = float(loss[0])
+ text_losses.append(text_loss_val)
+ step_grad_norm = float(grad_norm[0])
+ global_step += 1
+
+ else:
+ mel_batch, sp_tgt_in, sp_tgt_out, sp_lm = speech_batch_list[speech_idx]
+ speech_idx += 1
if n_widths > 1 and mat_shared_input:
per_width = args.batch_size // n_widths
@@ -734,28 +738,30 @@ def train(args):
mel_b = _tile_sp(mel_batch)
sp_tgt_in_b = _tile_sp(sp_tgt_in)
sp_tgt_out_b = _tile_sp(sp_tgt_out)
+ sp_lm_b = _tile_sp(sp_lm)
else:
mel_b = shard_batch(mel_batch, num_devices)
sp_tgt_in_b = shard_batch(sp_tgt_in, num_devices)
sp_tgt_out_b = shard_batch(sp_tgt_out, num_devices)
+ sp_lm_b = shard_batch(sp_lm, num_devices)
- speech_ffn_mask = text_ffn_mask
rng, spec_rng = jax.random.split(rng)
spec_rngs = jax.random.split(spec_rng, num_devices)
if prune_mask is not None:
state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech_masked(
- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, prune_mask, speech_ffn_mask, spec_rngs,
+ state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, prune_mask, text_ffn_mask, spec_rngs, sp_lm_b,
)
else:
state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech(
- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, speech_ffn_mask, spec_rngs,
+ state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, text_ffn_mask, spec_rngs, sp_lm_b,
)
speech_loss_val = float(sp_loss[0])
speech_losses.append(speech_loss_val)
+ step_grad_norm = float(sp_grad_norm[0])
+ text_loss_val = text_losses[-1] if text_losses else float("nan")
global_step += 1
- # --- Gradual magnitude sparsification (epoch 1 only) ---
if epoch == weight_prune_epoch and not gradual_sparsify_done:
epoch_step += 1
current_sparsity = _cubic_sparsity_schedule(epoch_step, t_start, t_end, sparsity_ratio)
@@ -776,11 +782,10 @@ def train(args):
vl, vt = val_loss_fn(_eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])
total_loss += float(vl)
total_toks += float(vt)
- last_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))
+ last_val_ppl = float(math.exp(min(total_loss / max(total_toks, 1), 20)))
del _eval_params
- # --- tqdm postfix (fixed keys to prevent bar flickering) ---
postfix = {
"speech_loss": f"{speech_loss_val:.4f}" if speech_loss_val is not None else "-",
"text_loss": f"{text_loss_val:.4f}",
@@ -796,7 +801,7 @@ def train(args):
if use_wandb:
log_dict = {
"train/text_loss": text_loss_val,
- "train/grad_norm": float(grad_norm[0]),
+ "train/grad_norm": step_grad_norm,
"train/adam_lr": float(adam_schedule(global_step)),
"train/muon_lr": float(muon_schedule(global_step)),
"train/tokens_per_sec": tokens_per_batch / dt,
@@ -810,17 +815,14 @@ def train(args):
log_dict["val/text_ppl"] = last_val_ppl
wandb.log(log_dict)
- # --- Lock mask after epoch 1 gradual sparsification ---
if epoch == weight_prune_epoch and not gradual_sparsify_done:
gradual_sparsify_done = True
if prune_mask is None:
- # Edge case: epoch had fewer steps than t_start
ema_unr = jax_utils.unreplicate(ema_params)
mask = _make_prune_mask(ema_unr, sparsity_ratio, _GROUP_SIZE)
del ema_unr
prune_mask = jax_utils.replicate(mask)
del mask
- # Apply final mask to both params and EMA (EMA lags due to slow decay)
state = state.replace(
params=jax.tree.map(lambda w, m: w * m, state.params, prune_mask))
ema_params = jax.tree.map(lambda w, m: w * m, ema_params, prune_mask)
@@ -833,7 +835,7 @@ def train(args):
epoch_avg_loss = sum(text_losses) / len(text_losses) if text_losses else float("nan")
final_loss = text_losses[-1] if text_losses else float("nan")
- final_ppl = math.exp(final_loss) if not math.isnan(final_loss) else float("nan")
+ final_ppl = math.exp(min(final_loss, 20)) if not math.isnan(final_loss) else float("nan")
eval_params = jax_utils.unreplicate(ema_params)
val_causal = make_causal_mask(args.max_dec_len)
@@ -842,13 +844,15 @@ def train(args):
vl, vt = val_loss_fn(eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])
total_loss += float(vl)
total_toks += float(vt)
- last_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))
+ last_val_ppl = float(math.exp(min(total_loss / max(total_toks, 1), 20)))
speech_val_ppl = None
- if speech_vl_fn is not None and val_speech_mel is not None:
+ if speech_vl_fn is not None and val_speech_audio_arrays is not None:
sp_total_loss, sp_total_toks = 0.0, 0.0
- for sp_batch in get_speech_batches(val_speech_mel, val_speech_dec_in, val_speech_dec_tgt, args.batch_size, shuffle=False):
- vl, vt = speech_vl_fn(eval_params, sp_batch[0], sp_batch[1], sp_batch[2], val_causal)
+ for sp_batch in get_speech_batches(val_speech_audio_arrays, val_speech_dec_in, val_speech_dec_tgt, args.batch_size,
+ shuffle=False, loss_mask=val_speech_loss_mask,
+ n_mels=n_mels, max_mel_len=max_mel_len, augmenter=None):
+ vl, vt = speech_vl_fn(eval_params, sp_batch[0], sp_batch[1], sp_batch[2], val_causal, sp_batch[3])
sp_total_loss += float(vl)
sp_total_toks += float(vt)
speech_val_loss = sp_total_loss / max(sp_total_toks, 1)
@@ -860,10 +864,9 @@ def train(args):
vl, vt = val_loss_fn(q_params, vb[0], vb[1], vb[2], val_causal, vb[3])
q_total_loss += float(vl)
q_total_toks += float(vt)
- quant_val_ppl = float(math.exp(q_total_loss / max(q_total_toks, 1)))
- del q_params # free device memory
+ quant_val_ppl = float(math.exp(min(q_total_loss / max(q_total_toks, 1), 20)))
+ del q_params
- # Matryoshka per-factor evaluation (reuse val_loss_fn's apply_fn)
mat_results = {}
if _MAT_FACTORS:
apply_fn = jax_utils.unreplicate(state).apply_fn
@@ -880,9 +883,8 @@ def train(args):
mat_results[factor] = (mat_ppl, mat_params, ff_w)
del apply_fn
- # Sparsity + checkpoint — move to CPU, reuse single copy
params_np = jax.tree.map(np.array, eval_params)
- del eval_params # free device memory
+ del eval_params
total_params = sum(x.size for x in jax.tree.leaves(params_np))
near_zero = sum(int(np.sum(np.abs(x) < 1e-6)) for x in jax.tree.leaves(params_np))
sparsity = near_zero / total_params * 100
@@ -893,33 +895,27 @@ def train(args):
with open(ckpt_path, "wb") as f:
pickle.dump({"params": params_np, "config": config.__dict__}, f)
- from .test import measure_throughput, benchmark_tool_calls, compute_wer
- from .run import transcribe
- from .data import _decode_audio
+ from .test import measure_throughput, benchmark_tool_calls
+ from .run import generate_from_audio
eval_params_jnp = jax.tree.map(jnp.array, params_np)
- del params_np # free CPU memory
+ del params_np
model = EncoderDecoderTransformer(config)
tp = measure_throughput(model, eval_params_jnp, tokenizer, num_runs=5)
tc_metrics = benchmark_tool_calls(model, eval_params_jnp, tokenizer, num_samples=20, max_gen_len=128)
- speech_wer = None
- transcribe_samples = []
- if not no_speech and val_speech_mel is not None:
- speech_val_ds = load_librispeech("validation.clean", max_samples=3)
- hyps, refs = [], []
- for i, example in enumerate(speech_val_ds):
- audio, sr = _decode_audio(example["audio"])
- ref_text = example["text"].strip().lower()
- hyp_text = transcribe(
+ voice_tc_samples = []
+ if not no_speech and val_speech_audio_arrays is not None:
+ val_pairs = load_tool_call_audio("validation", max_samples=3)
+ for i, pair in enumerate(val_pairs):
+ audio = pair["audio_array"]
+ sr = pair["sampling_rate"]
+ pred_text = generate_from_audio(
model, eval_params_jnp, tokenizer, audio, sr=sr,
- max_gen_len=64, temperature=0.0, seed=i, stream=False,
- ).strip().lower()
- hyps.append(hyp_text)
- refs.append(ref_text)
- transcribe_samples.append((ref_text, hyp_text))
- speech_wer = compute_wer(hyps, refs)
+ tools=pair["tools"], max_gen_len=128, seed=i, stream=False,
+ ).strip()
+ voice_tc_samples.append((pair["query"][:80], pair["answers"][:120], pred_text[:120]))
- del eval_params_jnp # free device memory
+ del eval_params_jnp
print(f"\n ─────────────────────────────────────")
print(f" Epoch {epoch + 1}/{args.epochs}")
@@ -928,9 +924,8 @@ def train(args):
print(f" Final loss {final_loss:>12.4f}")
print(f" Train ppl {final_ppl:>12.2f}")
print(f" TC val ppl {last_val_ppl:>12.2f}")
- if speech_wer is not None:
- print(f" Speech WER {speech_wer:>12.4f}")
- epoch_avg_speech = sum(speech_losses) / len(speech_losses) if speech_losses else float("nan")
+ if speech_losses:
+ epoch_avg_speech = sum(speech_losses) / len(speech_losses)
print(f" Avg speech loss{epoch_avg_speech:>12.4f}")
if speech_val_ppl is not None:
print(f" Speech val ppl {speech_val_ppl:>12.2f}")
@@ -959,12 +954,14 @@ def train(args):
print(f" R: {ref}")
print(f" P: {pred}")
print()
- if transcribe_samples:
+ if voice_tc_samples:
print(f" ─────────────────────────────────────")
- print(f" Transcribe (WER={speech_wer:.4f}):")
- for ref, hyp in transcribe_samples:
- print(f" <transcribe> ref: {ref[:80]}")
- print(f" hyp: {hyp[:80]}")
+ print(f" Voice-to-Tool-Call samples:")
+ for query, ref, pred in voice_tc_samples:
+ print(f" Q: {query}")
+ print(f" R: {ref}")
+ print(f" P: {pred}")
+ print()
print(f" ─────────────────────────────────────")
print(f" Checkpoint: {ckpt_path}")
print(f" ─────────────────────────────────────\n")
@@ -982,8 +979,6 @@ def train(args):
log_dict["epoch/tc_name_f1"] = tc_metrics["name_f1"]
log_dict["epoch/tc_call_f1"] = tc_metrics["call_f1"]
log_dict["epoch/tc_json_parse_rate"] = tc_metrics["json_parse_rate"]
- if speech_wer is not None:
- log_dict["epoch/speech_wer"] = speech_wer
if speech_losses:
log_dict["epoch/avg_speech_loss"] = sum(speech_losses) / len(speech_losses)
if speech_val_ppl is not None: