Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

tool call fixes

HenryNdubuakud03d21f3dfd9
diff --git a/README.md b/README.md
index 875a9c3..1b4aa48 100644
--- a/README.md
+++ b/README.md
@@ -15,8 +15,8 @@
└──────┬───────┘ │ Mat loss at factors │
┌┴──────────┐ │ + INT4 QAT (g=32) │
│ Softmax │ │ + z-loss + slot div │
- └─────┬─────┘ │ 1:1 text/speech │
- ┌─────┬─────┴─────┬─────┐ │ alternation │
+ └─────┬─────┘ │ text-only training │
+ ┌─────┬─────┴─────┬─────┐ │ │
│ @E[:d_ff/f] for each │ └──────────┬───────────┘
│ f in mat_factors │ │
└─────┬─────┬─────┬─────┘ ┌──────────┴───────────┐
@@ -39,13 +39,13 @@
│ │ │└───────────────┘│ ┌──────────┴───────────┐
│ ┌──────────┐ │ └────────┬────────┘ │ Checkpoint │
│ │Pack: │ │ ┌─────┴─────┐ │ full params, can │
- │ │ S←X Attn │ │ │ Embedding │ ← shared │ export mat slices │
+ │ │ S←X Attn │ │ │ Embedding │ ← shared │ export mat slices │
│ │ RoPE keys│ │ └─────┬─────┘ └──────────────────────┘
│ ├──────────┤ │ ┌───────┴───────┐
│ │Mix: │ │ │[EOS]<tool_call>│
- │ │ MLP-Mixer│ │ │ + tools │
- │ │ on S │ │ │ + answer │
- │ ├──────────┤ │ └───────────────┘
+ │ │ MLP-Mixer│ │ │ + answer │
+ │ │ on S │ │ └───────────────┘
+ │ ├──────────┤ │
│ │Local: │ │
│ │Gated FFN │ │
│ │ on X │ │
@@ -69,7 +69,7 @@
d=512 (default) / 1536 (--full) · GQA · QK-norm
SentencePiece BPE (8192) · gated dReLU · ZCRMSNorm · RoPE
strided DW conv · mat factors · INT4 QAT · Muon + AdamW
- text + speech encoder · <tool_call> task routing
+ text + speech encoder · <tool_call> task routing (speech used in inference only)
Data Pipeline (needle tokenize → needle train)
───────────────────────────────────────────────
@@ -77,34 +77,26 @@
┌─────────────────────────────────────────────────────────────┐
│ needle tokenize │
│ │
- │ Unified dataset (GCS / local) │
+ │ Local dataset (data/tool_calls_unified/) │
│ │ │
- │ ├──── Text ──────────────── Speech ─────────┐ │
- │ ▼ ▼ │
- │ ┌──────────────┐ ┌────────────────┐ │
- │ │ SentencePiece│ │ HF Audio col │ │
- │ │ BPE tokenize │ │ decode (16kHz) │ │
- │ │ (8192 vocab) │ └───────┬────────┘ │
- │ └──────┬───────┘ │ │
- │ │ ▼ │
- │ ▼ ┌────────────────┐ │
- │ ┌──────────────────┐ │ Log-mel spec │ │
- │ │ enc_inputs.npy │ │ 80 bins, 25ms │ │
- │ │ dec_inputs.npy │ │ window, 10ms │ │
- │ │ dec_targets.npy │ │ hop (~100 f/s) │ │
- │ │ loss_mask.npy │ └───────┬────────┘ │
- │ │ kept_idx.npy │ │ │
- │ └──────┬───────────┘ ▼ │
- │ │ ┌────────────────┐ │
- │ │ │ mels.npy │ │
- │ │ │ (N,T,80) f32 │ │
- │ │ │ writable mmap │ │
- │ │ └───────┬────────┘ │
- │ │ │ │
- │ └────────────┬───────────────────────────┘ │
- │ ▼ │
- │ {split}_metadata.json │
- │ .data_cache/ + GCS upload │
+ │ ▼ │
+ │ ┌──────────────┐ │
+ │ │ SentencePiece│ │
+ │ │ BPE tokenize │ │
+ │ │ (8192 vocab) │ │
+ │ └──────┬───────┘ │
+ │ │ │
+ │ ▼ │
+ │ ┌──────────────────┐ │
+ │ │ enc_inputs.npy │ │
+ │ │ dec_inputs.npy │ │
+ │ │ dec_targets.npy │ │
+ │ │ loss_mask.npy │ │
+ │ │ kept_idx.npy │ │
+ │ └──────┬───────────┘ │
+ │ ▼ │
+ │ {split}_metadata.json │
+ │ .data_cache/ │
└─────────────────────────────────────────────────────────────┘
│
▼
@@ -112,21 +104,15 @@
│ needle train │
│ │
│ load_prepared_data(mmap=True) │
- │ load_prepared_mels(mmap=True) │
│ │ │
│ ▼ │
- │ ┌──────────────────────┐ ┌──────────────────────┐ │
- │ │ PrefetchIterator │ │ PrefetchIterator │ │
- │ │ text batches (4) │ │ speech batches (4) │ │
- │ │ mmap → per-batch idx │ │ mmap → per-batch idx │ │
- │ └──────────┬───────────┘ └──────────┬───────────┘ │
- │ └───────────┬──────────────┘ │
- │ ▼ │
- │ 1:1 text/speech │
- │ alternation │
- │ │
- │ ~42MB RAM (8 prefetched batches) │
- │ SpecAugment for speech regularization │
+ │ ┌──────────────────────┐ │
+ │ │ PrefetchIterator │ │
+ │ │ text batches (4) │ │
+ │ │ mmap → per-batch idx │ │
+ │ └──────────┬───────────┘ │
+ │ ▼ │
+ │ text-only tool-call training │
└─────────────────────────────────────────────────────────────┘
```
@@ -142,20 +128,19 @@ needle [command]
┌───────────────────────────────────────────────────────────────────┐
│ │
│ train │
- │ --full Use full 1B config (~1.17B params) │
- │ --epochs INT Training epochs (default: 1) │
+ │ --epochs INT Training epochs (default: 10) │
│ --batch-size INT Batch size (default: 32) │
│ --lr FLOAT AdamW learning rate (default: 3e-4) │
│ --muon-lr FLOAT Muon learning rate (default: 0.02) │
│ --d-model INT Model dim (default: 512) │
- │ --num-heads INT Attention heads (default: 8) │
- │ --num-kv-heads INT KV heads for GQA (default: num-heads)│
+ │ --num-heads INT Attention heads (default: 16) │
+ │ --num-kv-heads INT KV heads for GQA (default: 8) │
│ --num-layers INT Encoder layers (default: 4) │
│ --num-dec-layers INT Decoder layers (default: 4) │
- │ --max-enc-len INT Max encoder seq len (default: 256) │
- │ --max-dec-len INT Max decoder seq len (default: 1024) │
+ │ --max-enc-len INT Max encoder seq len (default: 512) │
+ │ --max-dec-len INT Max decoder seq len (default: 512) │
│ --max-samples INT Training samples (default: all) │
- │ --mat-factors INT [...] FFN shrink factors (default: 2 4 8) │
+ │ --mat-factors INT [...] FFN shrink factors (default: 2 4) │
│ --sparsity-ratio FLOAT Block prune ratio (default: 0.0) │
│ --group-size INT Quant/prune group size (default: 32) │
│ --prune-interval INT Steps between mask updates (def: 100)│
@@ -168,18 +153,11 @@ needle [command]
│ --checkpoint PATH Resume from checkpoint │
│ --checkpoint-dir DIR Checkpoint directory │
│ --seed INT Random seed (default: 42) │
- │ --no-speech Disable speech (text-only training) │
- │ --max-mel-len INT Max mel frames (default: 1024) │
- │ --n-mels INT Mel frequency bins (default: 80) │
- │ --max-speech-samples INT Max voice-tool-call samples │
│ │
│ tokenize │
│ --max-samples INT Limit samples per split (dev/test) │
- │ --cleanup Delete local cache after GCS upload │
- │ --n-mels INT Mel frequency bins (default: 80) │
- │ --max-mel-len INT Max mel frames (default: 1024) │
- │ --max-enc-len INT Max encoder seq len (default: 256) │
- │ --max-dec-len INT Max decoder seq len (default: 1024) │
+ │ --max-enc-len INT Max encoder seq len (default: 512) │
+ │ --max-dec-len INT Max decoder seq len (default: 512) │
│ │
│ run │
│ --checkpoint PATH Path to model checkpoint (required) │
@@ -189,16 +167,15 @@ needle [command]
│ --max-len INT Max tokens to generate (default: 512) │
│ --seed INT Random seed (default: 0) │
│ │
- │ test │
+ │ eval │
│ --checkpoint PATH Path to model checkpoint (required) │
│ --batch-size INT Batch size (default: 32) │
│ --max-eval-samples INT Evaluation samples (default: 1000) │
- │ --max-enc-len INT Max encoder length (default: 256) │
- │ --max-dec-len INT Max decoder length (default: 1024) │
+ │ --max-enc-len INT Max encoder length (default: 512) │
+ │ --max-dec-len INT Max decoder length (default: 512) │
│ --max-gen-len INT Max generation length (default: 512) │
│ --throughput-runs INT Throughput runs (default: 10) │
│ --tool-call-samples INT Tool-call eval samples (default: 200) │
- │ --voice-tc-samples INT Voice-TC eval samples (default: 50) │
│ │
│ evaluate │
│ --checkpoint PATH Path to model checkpoint (required) │
@@ -236,20 +213,10 @@ needle [command]
──────────────────────────────────────────
Text Tool-call pairs
(query, tools, answers)
- from GCS unified dataset
- Audio TTS-generated speech
- (GCP Cloud TTS voices)
- 16kHz WAV, HF Audio col
+ ~57k examples (local)
+ xlam-60k + glaive-v2
──────────────────────────────────────────
- 100M params — 10M multimodal triples (5 epochs)
- ──────────────────────────────────────────
- Triples 10M (text / voice / image → JSON tool calls)
- Tokens/triple ~300 avg (text ~180, voice ~390, image ~336)
- Total tokens 10M × 300 × 5 epochs ≈ 15B
- Total FLOPs 6 × 100M × 15B ≈ 9e18
- MFU (100M) ~15% → ~138 effective TFLOPS/chip
-
┌────────────────────┬──────────┬──────────┬──────────┐
│ │ v6e-8 │ v6e-16 │ v6e-32 │
├────────────────────┼──────────┼──────────┼──────────┤
@@ -271,33 +238,6 @@ needle [command]
- Setup gcloud 4: else, set the project with `gcloud config set project needle-488623`
- setup gcloud 5: run `gcloud help` and read carefully
-## TPU Guide
-
-```
-needle tpu [command]
-
- ┌───────────────────────────────────────────────────────────────────┐
- │ │
- │ create NAME Create TPU (auto-finds zone) │
- │ --type STR Accelerator type (default: v6e-8) │
- │ --version STR TPU OS (auto-detected from --type) │
- │ │
- │ connect NAME SSH config + first connect (auto-zone) │
- │ claude NAME Install Claude Code on instance │
- │ │
- │ stop NAME Stop instance (keeps disk) │
- │ start NAME Restart a stopped instance │
- │ delete NAME Delete instance (prompts confirmation) │
- │ list List all TPU instances │
- │ │
- │ --zone ZONE Override auto-detected zone (optional) │
- │ │
- └───────────────────────────────────────────────────────────────────┘
-
- Quota increases:
- https://console.cloud.google.com/iam-admin/quotas?project=needle-488623
-```
-
## Example Workflow
```
@@ -322,10 +262,14 @@ needle tpu [command]
5. Install needle (follow instruction to setup wandb)
source ./setup
-6. Use needle as you normally would locally, like training
+6. Login to Hugging Face (required for gated datasets like xlam-60k)
+ huggingface-cli login
+ (paste your HF token — get one at https://huggingface.co/settings/tokens)
+
+7. Use needle as you normally would locally, like training
needle train --wandb
-7. Use tmux for long training runs (survives SSH disconnects)
+8. Use tmux for long training runs (survives SSH disconnects)
tmux new -s train # start a named session
needle train --wandb # run training inside it
Ctrl+B, then D # detach (keeps running)
@@ -333,9 +277,9 @@ needle tpu [command]
--- back on your Mac ---
-8. Stop when done (saves disk, stops billing)
+9. Stop when done (saves disk, stops billing)
needle tpu stop my-experiment
-9. (Optional) Delete instance when no longer needed
+10. (Optional) Delete instance when no longer needed
needle tpu delete my-experiment
```
diff --git a/scripts/build_dataset.py b/scripts/build_dataset.py
index 89b8033..1e8ded3 100644
--- a/scripts/build_dataset.py
+++ b/scripts/build_dataset.py
@@ -1,31 +1,19 @@
#!/usr/bin/env python3
-"""Unified dataset builder: text + TTS audio -> HuggingFace Dataset on GCS.
-
-Combines the text pipeline (tools_data.py) with TTS audio generation into
-one script that produces a single HF Dataset with an Audio column.
-
-When --gcs-upload is provided, processes in batches: each batch is synthesized,
-saved as a shard, uploaded to GCS, and deleted locally — keeping disk usage bounded.
+"""Build text-only tool-call dataset and save locally.
Usage:
- python scripts/build_dataset.py --max-samples 1000 --workers 16
- python scripts/build_dataset.py --gcs-upload gs://cactus-dataset/tool_calls --batch-size 500
+ python scripts/build_dataset.py
+ python scripts/build_dataset.py --max-samples 1000
+ python scripts/build_dataset.py --output data/tool_calls_unified
"""
import argparse
-import io
+import json
import logging
import os
-import random
-import shutil
-import subprocess
import sys
-import time
-import wave
-from concurrent.futures import ThreadPoolExecutor
-from datasets import Audio, Dataset, load_from_disk
-from tqdm import tqdm
+import numpy as np
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from tools_data import load_and_combine
@@ -33,524 +21,107 @@ from tools_data import load_and_combine
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
logger = logging.getLogger(__name__)
-_VOICE_TEMPLATES = [
- ("en-GB", "en-GB-Wavenet-B", "MALE", True, False),
- ("en-GB", "en-GB-Wavenet-A", "FEMALE", True, False),
- ("en-AU", "en-AU-Wavenet-B", "MALE", True, False),
- ("en-AU", "en-AU-Wavenet-C", "FEMALE", True, False),
- ("en-US", "en-US-Wavenet-D", "MALE", True, False),
- ("en-IN", "en-IN-Wavenet-A", "FEMALE", True, False),
- ("en-IN", "en-IN-Wavenet-B", "MALE", True, False),
- ("en-IN", "en-IN-Wavenet-C", "MALE", True, False),
- ("en-GB", "en-GB-Wavenet-D", "MALE", True, False),
- ("en-AU", "en-AU-Wavenet-D", "MALE", True, False),
- ("en-US", "en-US-Wavenet-E", "FEMALE", True, True),
- ("en-GB", "en-GB-Wavenet-A", "FEMALE", True, True),
- ("en-IN", "en-IN-Wavenet-A", "FEMALE", True, True),
-]
-
-_VOICE_TYPES = ["Wavenet", "Neural2"]
-
-VOICES = _VOICE_TEMPLATES
-
-
-def _get_voice(idx):
- """Return (lang, voice_name, gender_str, configurable, child) for a sample.
-
- Cycles through voice types (Wavenet->Neural2) via idx % 2,
- then picks a voice template via (idx // 3) % len(_VOICE_TEMPLATES).
- """
- vtype = _VOICE_TYPES[idx % len(_VOICE_TYPES)]
- template = _VOICE_TEMPLATES[(idx // 3) % len(_VOICE_TEMPLATES)]
- lang, base_name, gender, configurable, child = template
- voice_name = base_name.replace("Wavenet", vtype)
- return lang, voice_name, gender, configurable, child
-
-
-def _pcm_to_wav_bytes(pcm_bytes: bytes, sample_rate: int) -> bytes:
- """Convert raw LINEAR16 PCM bytes to a WAV byte buffer."""
- buf = io.BytesIO()
- with wave.open(buf, "wb") as wf:
- wf.setnchannels(1)
- wf.setsampwidth(2)
- wf.setframerate(sample_rate)
- wf.writeframes(pcm_bytes)
- return buf.getvalue()
-
-
-def _synthesize_one(client, idx, query, max_retries=5):
- """Synthesize TTS audio for a single query via GCP Cloud TTS.
-
- Returns:
- Tuple of (idx, wav_bytes, voice_name, error_msg).
- wav_bytes is None on failure; error_msg is None on success.
- """
- from google.cloud import texttospeech
- from google.api_core.exceptions import ResourceExhausted, ServiceUnavailable
-
- lang, voice_name, ssml_gender, configurable, child = _get_voice(idx)
- gender = getattr(texttospeech.SsmlVoiceGender, ssml_gender)
-
- synthesis_input = texttospeech.SynthesisInput(text=query)
- voice_params = texttospeech.VoiceSelectionParams(
- language_code=lang,
- name=voice_name,
- ssml_gender=gender,
- )
-
- if configurable:
- rng = random.Random(idx)
- if child:
- pitch = rng.uniform(4.0, 8.0)
- rate = rng.uniform(1.0, 1.15)
- else:
- pitch = rng.uniform(-3.0, 3.0)
- rate = rng.uniform(0.9, 1.1)
- audio_config = texttospeech.AudioConfig(
- audio_encoding=texttospeech.AudioEncoding.LINEAR16,
- sample_rate_hertz=16000,
- pitch=pitch,
- speaking_rate=rate,
- volume_gain_db=rng.uniform(-1.5, 1.5),
- )
- else:
- audio_config = texttospeech.AudioConfig(
- audio_encoding=texttospeech.AudioEncoding.LINEAR16,
- sample_rate_hertz=16000,
- )
-
- for attempt in range(max_retries):
- try:
- response = client.synthesize_speech(
- input=synthesis_input,
- voice=voice_params,
- audio_config=audio_config,
- )
- wav_bytes = _pcm_to_wav_bytes(response.audio_content, 16000)
- return idx, wav_bytes, voice_name, None
- except (ResourceExhausted, ServiceUnavailable):
- wait = min(2 ** attempt, 30)
- time.sleep(wait)
- except Exception as e:
- return idx, None, voice_name, str(e)
-
- return idx, None, voice_name, "max retries exceeded"
-
-def _synthesize_rows(batch_ds, audio_data, voice_names, workers, rate_limit,
- cache_dir, global_offset=0):
- """Run TTS synthesis for rows where audio_data[i] is None.
-
- Uses global_offset + local_index for voice selection so that voice
- assignment is deterministic regardless of batch boundaries.
- WAV files are written to cache_dir/{local_idx}.wav.
- """
- needs = [i for i in range(len(batch_ds)) if audio_data[i] is None]
- if not needs:
- return
-
- from google.cloud import texttospeech
-
- os.makedirs(cache_dir, exist_ok=True)
-
- items = [(i, global_offset + i, batch_ds[i]["query"]) for i in needs]
- logger.info(f"Synthesizing TTS for {len(items)} rows with {workers} workers...")
-
- def _worker(item):
- local_idx, global_idx, query = item
- client = texttospeech.TextToSpeechClient()
- _, wav_bytes, voice_name, err = _synthesize_one(client, global_idx, query)
- return local_idx, wav_bytes, voice_name, err
-
- success, failed = 0, 0
- delay = 1.0 / rate_limit if rate_limit > 0 else 0
- total = len(items)
-
- with ThreadPoolExecutor(max_workers=workers) as pool:
- pending = {}
- submitted = 0
- qi = iter(items)
-
- for _ in range(min(workers * 2, total)):
- q = next(qi)
- pending[pool.submit(_worker, q)] = q
- submitted += 1
-
- with tqdm(total=total, desc="TTS") as pbar:
- while pending:
- done_set = {f for f in list(pending) if f.done()}
- if not done_set:
- time.sleep(0.05)
- continue
- for fut in done_set:
- local_idx, wav_bytes, voice_name, err = fut.result()
- if wav_bytes is not None:
- wav_path = os.path.join(cache_dir, f"{local_idx}.wav")
- with open(wav_path, "wb") as f:
- f.write(wav_bytes)
- audio_data[local_idx] = wav_path
- voice_names[local_idx] = voice_name
- success += 1
- else:
- failed += 1
- logger.warning(f"Failed idx={global_offset + local_idx}: {err}")
- pbar.update(1)
- del pending[fut]
-
- if submitted < total:
- q = next(qi)
- pending[pool.submit(_worker, q)] = q
- submitted += 1
- time.sleep(delay)
-
- logger.info(f"TTS done: {success} generated, {failed} failed")
-
-
-def _backfill_voice_names(voice_names, global_offset=0):
- """Assign default voice names to any rows that don't have one."""
- for i in range(len(voice_names)):
- if not voice_names[i]:
- _, voice_name, _, _, _ = VOICES[(global_offset + i) % len(VOICES)]
- voice_names[i] = voice_name
-
-def _build_arrow(batch_ds, audio_data, voice_names, arrow_path):
- """Build an HF Dataset from a batch and write a single .arrow file.
-
- Uses save_to_disk into a temp dir so that Audio columns are properly
- serialized (WAV bytes embedded), then moves the generated .arrow file.
- """
- import tempfile
-
- n = len(batch_ds)
- audio_column = []
- for i in range(n):
- path = audio_data[i]
- if path is not None and os.path.exists(path):
- audio_column.append(path)
- else:
- audio_column.append(None)
-
- ds_dict = {
- "query": [batch_ds[i]["query"] for i in range(n)],
- "audio": audio_column,
- "tools": [batch_ds[i]["tools"] for i in range(n)],
- "answers": [batch_ds[i]["answers"] for i in range(n)],
- "source": [batch_ds[i]["source"] for i in range(n)],
- "voice": voice_names[:n],
- }
-
- ds = Dataset.from_dict(ds_dict)
- ds = ds.cast_column("audio", Audio(sampling_rate=16000))
-
- with tempfile.TemporaryDirectory() as tmpdir:
- ds.save_to_disk(tmpdir)
- arrow_files = sorted(f for f in os.listdir(tmpdir) if f.endswith(".arrow"))
- if len(arrow_files) == 1:
- shutil.move(os.path.join(tmpdir, arrow_files[0]), arrow_path)
- else:
- import pyarrow as pa
- tables = []
- for af in arrow_files:
- with open(os.path.join(tmpdir, af), "rb") as fh:
- reader = pa.ipc.open_stream(fh)
- tables.append(reader.read_all())
- merged = pa.concat_tables(tables)
- with pa.OSFile(arrow_path, "wb") as sink:
- writer = pa.ipc.new_stream(sink, merged.schema)
- writer.write_table(merged)
- writer.close()
-
- logger.info(f"Built {arrow_path} ({n} rows)")
- info = ds.info
- del ds, ds_dict, audio_column
- return info
-
-
-def _arrow_exists_on_gcs(gcs_base, arrow_name):
- """Check if an arrow file already exists on GCS."""
- gcs_path = gcs_base + "/" + arrow_name
- result = subprocess.run(
- ["gcloud", "storage", "ls", gcs_path],
- capture_output=True, text=True,
- )
- return result.returncode == 0 and result.stdout.strip()
-
-
-def _upload_file_to_gcs(local_path, gcs_dest):
- """Upload a single file to GCS."""
- logger.info(f"Uploading {os.path.basename(local_path)} -> {gcs_dest}")
- result = subprocess.run(
- ["gcloud", "storage", "cp", local_path, gcs_dest],
- text=True,
- )
- if result.returncode != 0:
- logger.error(f"Upload failed for {local_path}")
- sys.exit(1)
-
-
-def _upload_metadata_to_gcs(gcs_base, num_batches, dataset_info):
- """Generate and upload state.json + dataset_info.json so load_from_disk works."""
- import json
- import tempfile
-
- arrow_files = [
- {"filename": f"data-{i:05d}-of-{num_batches:05d}.arrow"}
- for i in range(num_batches)
- ]
-
- state = {
- "_data_files": arrow_files,
- "_fingerprint": "batch_upload",
- "_format_columns": None,
- "_format_kwargs": {},
- "_format_type": None,
- "_output_all_columns": False,
- "_split": None,
- }
-
- with tempfile.TemporaryDirectory() as tmpdir:
- state_path = os.path.join(tmpdir, "state.json")
- with open(state_path, "w") as f:
- json.dump(state, f)
- _upload_file_to_gcs(state_path, gcs_base + "/state.json")
- info_path = os.path.join(tmpdir, "dataset_info.json")
- dataset_info.write_to_directory(tmpdir)
- _upload_file_to_gcs(info_path, gcs_base + "/dataset_info.json")
-
-
-def _process_batched(combined, batch_size, workers, rate_limit, output_path, gcs_upload):
- """Process dataset in batches: one arrow per batch, flat on GCS.
-
- GCS layout:
- gs://bucket/path/data-00000-of-00020.arrow
- gs://bucket/path/data-00001-of-00020.arrow
- ...
- gs://bucket/path/state.json
- gs://bucket/path/dataset_info.json
-
- load_from_disk("gs://bucket/path") works directly.
- """
- n = len(combined)
- num_batches = (n + batch_size - 1) // batch_size
- total_rows = 0
- dataset_info = None
-
- logger.info(f"Processing {n} rows in {num_batches} batches of {batch_size}...")
-
- for batch_idx in range(num_batches):
- start = batch_idx * batch_size
- end = min(start + batch_size, n)
- arrow_name = f"data-{batch_idx:05d}-of-{num_batches:05d}.arrow"
-
- # Resume: skip arrows already on GCS
- if _arrow_exists_on_gcs(gcs_upload, arrow_name):
- logger.info(f"{arrow_name} already on GCS, skipping (rows {start}-{end - 1})")
- total_rows += (end - start)
- continue
-
- logger.info(f"=== Batch {batch_idx + 1}/{num_batches}: {arrow_name} (rows {start}-{end - 1}) ===")
- batch = combined.select(range(start, end))
- batch_len = len(batch)
-
- audio_data = [None] * batch_len
- voice_names = [""] * batch_len
- cache_dir = os.path.join(output_path, f".cache_{batch_idx:05d}")
-
- existing_cache = os.path.join(output_path, ".audio_cache")
- if os.path.isdir(existing_cache):
- recovered = 0
- for i in range(batch_len):
- wav_path = os.path.join(existing_cache, f"{start + i}.wav")
- if os.path.exists(wav_path):
- audio_data[i] = wav_path
- recovered += 1
- if recovered:
- logger.info(f" Recovered {recovered}/{batch_len} WAVs from existing cache")
-
- _synthesize_rows(batch, audio_data, voice_names, workers, rate_limit,
- cache_dir, global_offset=start)
- _backfill_voice_names(voice_names, global_offset=start)
-
- os.makedirs(output_path, exist_ok=True)
- arrow_path = os.path.join(output_path, arrow_name)
- info = _build_arrow(batch, audio_data, voice_names, arrow_path)
- if dataset_info is None:
- dataset_info = info
-
- _upload_file_to_gcs(arrow_path, gcs_upload + "/" + arrow_name)
- os.remove(arrow_path)
- shutil.rmtree(cache_dir, ignore_errors=True)
-
- existing_cache = os.path.join(output_path, ".audio_cache")
- if os.path.isdir(existing_cache):
- for i in range(batch_len):
- wav_path = os.path.join(existing_cache, f"{start + i}.wav")
- if os.path.exists(wav_path):
- os.remove(wav_path)
-
- total_rows += batch_len
- logger.info(f"Progress: {total_rows}/{n} rows uploaded")
-
- if dataset_info is not None:
- _upload_metadata_to_gcs(gcs_upload, num_batches, dataset_info)
-
- return total_rows
-
-
-def _load_text_dataset(output_path, max_samples):
- """Load the text dataset and optionally resume from an existing build."""
- existing_ds = None
- if os.path.exists(output_path):
- try:
- logger.info(f"Found existing dataset at {output_path}, loading for resume...")
- existing_ds = load_from_disk(output_path)
- logger.info(f" Existing dataset: {len(existing_ds)} rows")
- except (FileNotFoundError, Exception) as e:
- logger.warning(f"Could not load existing dataset (will use audio cache only): {e}")
+def main():
+ parser = argparse.ArgumentParser(description="Build text-only tool-call dataset")
+ parser.add_argument("--max-samples", type=int, default=None, help="Max samples to include")
+ parser.add_argument("--output", type=str, default="data/tool_calls_unified", help="Output directory")
+ args = parser.parse_args()
logger.info("Loading and combining text datasets...")
combined = load_and_combine()
- if max_samples:
- combined = combined.select(range(min(max_samples, len(combined))))
- logger.info(f"Text dataset: {len(combined)} rows")
-
- return combined, existing_ds
-
-
-def _find_rows_needing_tts(combined, existing_ds, output_path):
- """Identify which rows lack audio."""
- n = len(combined)
- needs_tts = set(range(n))
-
- cache_dir = os.path.join(output_path, ".audio_cache")
- if os.path.isdir(cache_dir):
- cached = 0
- for fname in os.listdir(cache_dir):
- if fname.endswith(".wav"):
- try:
- idx = int(fname[:-4])
- if idx < n:
- needs_tts.discard(idx)
- cached += 1
- except ValueError:
- pass
- if cached:
- logger.info(f" {cached} rows found in audio cache")
-
- if existing_ds is not None and len(existing_ds) == n and "audio" in existing_ds.column_names:
- audio_column = existing_ds.data.column("audio")
- for i in range(n):
- if i in needs_tts and audio_column[i].as_py() is not None:
- needs_tts.discard(i)
-
- logger.info(f" {n - len(needs_tts)} rows already have audio, {len(needs_tts)} need TTS")
- return needs_tts
-
+ if args.max_samples:
+ combined = combined.select(range(min(args.max_samples, len(combined))))
-def _recover_existing_audio(existing_ds, n, needs_tts, output_path):
- """Recover audio file paths from the cache dir and existing dataset."""
- audio_data = [None] * n
- voice_names = [""] * n
+ logger.info(f"Dataset: {len(combined)} rows")
- cache_dir = os.path.join(output_path, ".audio_cache")
- if os.path.isdir(cache_dir):
- for i in range(n):
- if i not in needs_tts:
- wav_path = os.path.join(cache_dir, f"{i}.wav")
- if os.path.exists(wav_path):
- audio_data[i] = wav_path
+ os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
+ combined.save_to_disk(args.output)
+ logger.info(f"Saved {len(combined)} rows to {args.output}/")
- if existing_ds is not None and len(existing_ds) == n and "audio" in existing_ds.column_names:
- audio_column = existing_ds.data.column("audio")
- voice_col = existing_ds["voice"] if "voice" in existing_ds.column_names else None
- for i in range(n):
- if i not in needs_tts and audio_data[i] is None:
- raw = audio_column[i].as_py()
- if isinstance(raw, dict) and raw.get("bytes"):
- wav_path = os.path.join(cache_dir, f"{i}.wav")
- os.makedirs(cache_dir, exist_ok=True)
- with open(wav_path, "wb") as f:
- f.write(raw["bytes"])
- audio_data[i] = wav_path
- elif isinstance(raw, dict) and raw.get("path") and os.path.exists(raw["path"]):
- audio_data[i] = raw["path"]
- if voice_col is not None and i not in needs_tts:
- voice_names[i] = voice_col[i] or ""
+ print_summary(combined)
- return audio_data, voice_names
+def print_summary(ds, n_samples=5):
+ """Print dataset statistics and sample examples."""
+ n = len(ds)
-def _process_local(combined, workers, rate_limit, output_path):
- """Original local-only processing: synthesize all, build one dataset."""
- existing_ds = None
- if os.path.exists(output_path):
+ # Parse tools/answers counts
+ n_tools, n_calls = [], []
+ n_with_calls, n_no_calls = 0, 0
+ query_lens, tools_lens, answers_lens = [], [], []
+ for ex in ds:
try:
- existing_ds = load_from_disk(output_path)
- except Exception:
- pass
-
- needs_tts = _find_rows_needing_tts(combined, existing_ds, output_path)
- audio_data, voice_names = _recover_existing_audio(
- existing_ds, len(combined), needs_tts, output_path
- )
-
- cache_dir = os.path.join(output_path, ".audio_cache")
- _synthesize_rows(combined, audio_data, voice_names, workers, rate_limit, cache_dir)
- _backfill_voice_names(voice_names)
-
- n = len(combined)
- os.makedirs(output_path, exist_ok=True)
- arrow_path = os.path.join(output_path, "data-00000-of-00001.arrow")
- info = _build_arrow(combined, audio_data, voice_names, arrow_path)
-
- # Write metadata so load_from_disk works
- import json
- state = {
- "_data_files": [{"filename": "data-00000-of-00001.arrow"}],
- "_fingerprint": "local_build",
- "_format_columns": None,
- "_format_kwargs": {},
- "_format_type": None,
- "_output_all_columns": False,
- "_split": None,
- }
- with open(os.path.join(output_path, "state.json"), "w") as f:
- json.dump(state, f)
- info.write_to_directory(output_path)
- logger.info(f"Saved {n} rows to {output_path}/")
- return n
-
-def main():
- parser = argparse.ArgumentParser(description="Build unified tool-call dataset with TTS audio")
- parser.add_argument("--max-samples", type=int, default=None, help="Max samples to process")
- parser.add_argument("--workers", type=int, default=16, help="TTS concurrent workers (default: 16)")
- parser.add_argument("--output", type=str, default="data/tool_calls_unified", help="Output directory")
- parser.add_argument("--rate-limit", type=float, default=30.0,
- help="Max TTS requests per second (default: 30, quota is 2x1000/min)")
- parser.add_argument("--gcs-upload", type=str, default=None,
- help="GCS path to upload dataset (e.g. gs://cactus-dataset/tool_calls)")
- parser.add_argument("--batch-size", type=int, default=5000,
- help="Rows per batch when using --gcs-upload (default: 500)")
- args = parser.parse_args()
-
- output_path = args.output
-
- logger.info("Loading and combining text datasets...")
- combined = load_and_combine()
- if args.max_samples:
- combined = combined.select(range(min(args.max_samples, len(combined))))
- logger.info(f"Text dataset: {len(combined)} rows")
-
- if args.gcs_upload:
- total_rows = _process_batched(
- combined, args.batch_size, args.workers, args.rate_limit,
- output_path, args.gcs_upload,
- )
- else:
- total_rows = _process_local(combined, args.workers, args.rate_limit, output_path)
-
- logger.info(f"Pipeline complete: {total_rows} rows")
+ tools = json.loads(ex["tools"])
+ except (json.JSONDecodeError, TypeError):
+ tools = []
+ try:
+ answers = json.loads(ex["answers"])
+ except (json.JSONDecodeError, TypeError):
+ answers = []
+ n_tools.append(len(tools))
+ n_calls.append(len(answers))
+ if answers:
+ n_with_calls += 1
+ else:
+ n_no_calls += 1
+ query_lens.append(len(ex["query"]))
+ tools_lens.append(len(ex["tools"]))
+ answers_lens.append(len(ex["answers"]))
+
+ n_tools = np.array(n_tools)
+ n_calls = np.array(n_calls)
+ query_lens = np.array(query_lens)
+ tools_lens = np.array(tools_lens)
+ answers_lens = np.array(answers_lens)
+
+ def _stats(arr):
+ return f"min={arr.min()}, max={arr.max()}, mean={arr.mean():.1f}, median={np.median(arr):.0f}"
+
+ print(f"\n{'=' * 60}")
+ print(f"DATASET SUMMARY")
+ print(f"{'=' * 60}")
+ print(f" Total examples: {n:,}")
+ print(f" With tool calls: {n_with_calls:,} ({n_with_calls/n*100:.1f}%)")
+ print(f" No-call (empty): {n_no_calls:,} ({n_no_calls/n*100:.1f}%)")
+ print()
+ print(f" Available tools/sample: {_stats(n_tools)}")
+ print(f" Tool calls/sample: {_stats(n_calls)}")
+ print()
+ print(f" Query length (chars): {_stats(query_lens)}")
+ print(f" Tools length (chars): {_stats(tools_lens)}")
+ print(f" Answers length (chars): {_stats(answers_lens)}")
+
+ # Samples with tool calls
+ with_calls_idx = [i for i in range(n) if n_calls[i] > 0]
+ no_calls_idx = [i for i in range(n) if n_calls[i] == 0]
+
+ rng = np.random.RandomState(42)
+
+ if with_calls_idx:
+ sample_idx = rng.choice(with_calls_idx, size=min(n_samples, len(with_calls_idx)), replace=False)
+ print(f"\n{'─' * 60}")
+ print(f"SAMPLES WITH TOOL CALLS ({len(sample_idx)})")
+ print(f"{'─' * 60}")
+ for i, idx in enumerate(sample_idx):
+ ex = ds[int(idx)]
+ print(f"\n [{i+1}] Query: {ex['query']}")
+ print(f" Tools: {ex['tools']}")
+ print(f" Answers: {ex['answers']}")
+
+ if no_calls_idx:
+ sample_idx = rng.choice(no_calls_idx, size=min(n_samples, len(no_calls_idx)), replace=False)
+ print(f"\n{'─' * 60}")
+ print(f"SAMPLES WITHOUT TOOL CALLS ({len(sample_idx)})")
+ print(f"{'─' * 60}")
+ for i, idx in enumerate(sample_idx):
+ ex = ds[int(idx)]
+ print(f"\n [{i+1}] Query: {ex['query']}")
+ print(f" Tools: {ex['tools']}")
+ print(f" Answers: {ex['answers']}")
+
+ print(f"\n{'=' * 60}\n")
if __name__ == "__main__":
diff --git a/scripts/tools_data.py b/scripts/tools_data.py
index e287abc..0f8d541 100755
--- a/scripts/tools_data.py
+++ b/scripts/tools_data.py
@@ -1,33 +1,22 @@
"""
Unified tool-calling dataset builder.
-Loads four HuggingFace tool-calling datasets and converts them all into the
-xlam-function-calling-60k schema:
+Loads HuggingFace tool-calling datasets and converts them into a common schema:
query : str – the user's natural-language request
answers : str – JSON list of {"name": ..., "arguments": {...}} tool calls
tools : str – JSON list of available tool definitions
source : str – which dataset the example came from
-Datasets and their native formats:
+Datasets:
1. Salesforce/xlam-function-calling-60k
Already in the target schema (query, answers, tools).
-2. Salesforce/APIGen-MT-5k
- Multi-turn conversations with human/gpt/function_call/observation roles.
- We extract the first user message as the query and collect all function_call
- messages (excluding internal "think" calls) as answers.
-
-3. glaiveai/glaive-function-calling-v2
+2. glaiveai/glaive-function-calling-v2
Tool definitions embedded in the system prompt; chat is a flat string with
USER:/A: role markers and <functioncall> tags. Examples with empty answers
(answers=[]) are kept so the model learns when NOT to call functions.
-
-4. Team-ACE/ToolACE
- Tool definitions as a JSON array in the system prompt; function calls use a
- bracket syntax [FuncName(key=val, ...)] where names may contain spaces. Only
- examples with at least one parsed function call are kept.
"""
import json
@@ -39,9 +28,7 @@ from datasets import load_dataset, Dataset, concatenate_datasets
DATASETS = {
"xlam-function-calling-60k": "Salesforce/xlam-function-calling-60k",
- "APIGen-MT-5k": "Salesforce/APIGen-MT-5k",
"glaive-function-calling": "glaiveai/glaive-function-calling-v2",
- "ToolACE": "Team-ACE/ToolACE",
}
@@ -49,34 +36,6 @@ def convert_xlam(dataset):
return dataset.select_columns(["query", "answers", "tools"])
-def convert_apigen_mt(dataset):
- rows = []
- for ex in dataset:
- convs = ex["conversations"]
- tools = ex["tools"]
-
- query = None
- answers = []
- for msg in convs:
- if msg["from"] == "human" and query is None:
- query = msg["value"]
- elif msg["from"] == "function_call":
- try:
- fc = json.loads(msg["value"])
- if fc.get("name") != "think":
- answers.append(fc)
- except json.JSONDecodeError:
- pass
-
- if query:
- rows.append({
- "query": query,
- "answers": json.dumps(answers),
- "tools": tools,
- })
- return Dataset.from_list(rows)
-
-
def convert_glaive(dataset):
rows = []
for ex in dataset:
@@ -149,109 +108,9 @@ def _parse_glaive_chat(chat):
return query, answers
-def convert_toolace(dataset):
- rows = []
- for ex in dataset:
- tools = _parse_toolace_tools(ex["system"])
- convs = ex["conversations"]
-
- query = None
- answers = []
- for msg in convs:
- if msg["from"] == "user" and query is None:
- query = msg["value"]
- elif msg["from"] == "assistant" and query is not None:
- calls = _parse_toolace_calls(msg["value"])
- if calls:
- answers.extend(calls)
- break
-
- if query and answers:
- rows.append({
- "query": query,
- "answers": json.dumps(answers),
- "tools": json.dumps(tools),
- })
- return Dataset.from_list(rows)
-
-
-def _parse_toolace_tools(system_text):
- """Extract tool list from ToolACE system prompt (JSON array after 'invoke:')."""
- match = re.search(r'invoke:\s*(\[)', system_text)
- if not match:
- return []
- start = match.start(1)
- depth = 0
- for i in range(start, len(system_text)):
- if system_text[i] == '[':
- depth += 1
- elif system_text[i] == ']':
- depth -= 1
- if depth == 0:
- try:
- return json.loads(system_text[start:i + 1])
- except json.JSONDecodeError:
- return []
- return []
-
-
-def _parse_toolace_calls(text):
- """Parse [FuncName(key=val, ...)] calls. Function names may contain spaces."""
- answers = []
- bracket_match = re.search(r'\[(.+)\]', text, re.DOTALL)
- if not bracket_match:
- return answers
-
- inner = bracket_match.group(1)
- for m in re.finditer(r'([\w][\w ]*?\w|\w)\(([^)]*)\)', inner):
- name = m.group(1).strip()
- args_str = m.group(2)
- arguments = {}
- for arg_m in re.finditer(
- r'(\w+)\s*=\s*(?:"((?:[^"\\]|\\.)*)"|\'((?:[^\'\\]|\\.)*)\'|(\[[^\]]*\])|([^,)]+))',
- args_str,
- ):
- key = arg_m.group(1)
- val = next(
- (g for g in (arg_m.group(2), arg_m.group(3), arg_m.group(4), arg_m.group(5)) if g is not None),
- None,
- )
- if val is not None:
- val = val.strip()
- try:
- val = json.loads(val)
- except (json.JSONDecodeError, TypeError):
- pass
- arguments[key] = val
- answers.append({"name": name, "arguments": arguments})
- return answers
-
-
-def compact_tools(tools_json):
- """Strip verbose schema descriptions, keep only function name + param name/type."""
- try:
- tools = json.loads(tools_json)
- except (json.JSONDecodeError, TypeError):
- return tools_json
- compact = []
- for t in tools:
- params = {}
- raw_params = t.get("parameters", {})
- props = raw_params.get("properties", raw_params) if isinstance(raw_params, dict) else {}
- for pname, pdef in props.items():
- if isinstance(pdef, dict):
- params[pname] = pdef.get("type", "string")
- else:
- params[pname] = str(pdef)
- compact.append({"name": t["name"], "parameters": params})
- return json.dumps(compact)
-
-
CONVERTERS = {
"xlam-function-calling-60k": convert_xlam,
- "APIGen-MT-5k": convert_apigen_mt,
"glaive-function-calling": convert_glaive,
- "ToolACE": convert_toolace,
}
_PLACEHOLDER_VALUES = {
@@ -421,18 +280,22 @@ def load_and_combine():
combined = concatenate_datasets(parts)
+ # Drop no-call examples with empty tools (trivial negatives)
+ before_empty = len(combined)
+ combined = combined.filter(
+ lambda ex: json.loads(ex["answers"]) != [] or json.loads(ex["tools"]) != [],
+ desc="Removing empty-tools no-call examples",
+ )
+ empty_dropped = before_empty - len(combined)
+ if empty_dropped:
+ print(f"\nRemoved {empty_dropped} no-call examples with empty tools")
+
before_dedup = len(combined)
combined = _deduplicate(combined)
dedup_dropped = before_dedup - len(combined)
if dedup_dropped:
print(f"\nDeduplicated: removed {dedup_dropped} duplicate queries")
- print(f"\nCompacting tool schemas...")
- combined = combined.map(
- lambda ex: {"tools": compact_tools(ex["tools"])},
- desc="Compacting tools",
- )
-
empty_count = sum(1 for ex in combined if json.loads(ex["answers"]) == [])
print(f"\n{'='*60}")
print(f"Combined dataset: {len(combined)} examples ({empty_count} with empty answers = no-call examples)")
diff --git a/src/cli.py b/src/cli.py
index 15a909e..1ada00e 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -12,9 +12,8 @@ def main():
sub = parser.add_subparsers(dest="command")
p = sub.add_parser("train", add_help=False)
- p.add_argument("--full", action="store_true")
p.add_argument("--checkpoint", type=str, default=None)
- p.add_argument("--epochs", type=int, default=3)
+ p.add_argument("--epochs", type=int, default=10)
p.add_argument("--batch-size", type=int, default=32)
p.add_argument("--lr", type=float, default=3e-4)
p.add_argument("--muon-lr", type=float, default=0.02)
@@ -23,15 +22,15 @@ def main():
p.add_argument("--num-kv-heads", type=int, default=8)
p.add_argument("--num-layers", type=int, default=4)
p.add_argument("--num-dec-layers", type=int, default=4)
- p.add_argument("--max-enc-len", type=int, default=256)
- p.add_argument("--max-dec-len", type=int, default=1024)
+ p.add_argument("--max-enc-len", type=int, default=1024)
+ p.add_argument("--max-dec-len", type=int, default=512)
p.add_argument("--max-samples", type=int, default=None)
p.add_argument("--warmup-ratio", type=float, default=0.05)
p.add_argument("--wandb", action="store_true")
p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])
p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
p.add_argument("--seed", type=int, default=42)
- p.add_argument("--eval-every", type=int, default=1000)
+ p.add_argument("--eval-every", type=int, default=100)
p.add_argument("--max-eval-samples", type=int, default=None)
p.add_argument("--sparsity-ratio", type=float, default=0.0)
p.add_argument("--group-size", type=int, default=32)
@@ -42,36 +41,21 @@ def main():
p.add_argument("--prune-end-frac", type=float, default=0.67,
help="Fraction of epoch at which pruning finishes and mask locks (default: 0.67)")
p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])
- p.add_argument("--num-memory-slots", type=int, default=64)
- p.add_argument("--mat-factors", type=int, nargs="*", default=[2, 4, 8],
- help="Matryoshka FFN shrink factors, e.g. 2=half width (default: 2 4 8)")
+ p.add_argument("--num-memory-slots", type=int, default=128)
+ p.add_argument("--mat-factors", type=int, nargs="*", default=[2, 4],
+ help="Matryoshka FFN shrink factors, e.g. 2=half width (default: 2 4)")
p.add_argument("--mat-shared-input", action="store_true",
help="Each unique input is repeated across all mat widths (default: unique input per width)")
- p.add_argument("--dropout", type=float, default=0.0,
+ p.add_argument("--dropout", type=float, default=0.05,
help="Dropout rate for residual connections (default: 0.1)")
- p.add_argument("--no-speech", action="store_true", help="Disable speech training (text-only)")
- p.add_argument("--max-mel-len", type=int, default=1024,
- help="Max mel spectrogram frames (default: 1024)")
- p.add_argument("--n-mels", type=int, default=80,
- help="Number of mel frequency bins (default: 80)")
- p.add_argument("--max-speech-samples", type=int, default=None,
- help="Max voice-tool-call training samples (default: all)")
p = sub.add_parser("tokenize", add_help=False)
p.add_argument("--max-samples", type=int, default=None,
help="Limit samples per split (for dev/test)")
- p.add_argument("--cleanup", action="store_true",
- help="Delete local .data_cache/ after GCS upload")
- p.add_argument("--n-mels", type=int, default=80,
- help="Number of mel frequency bins (default: 80)")
- p.add_argument("--max-mel-len", type=int, default=1024,
- help="Max mel spectrogram frames (default: 1024)")
- p.add_argument("--max-enc-len", type=int, default=256,
- help="Max encoder sequence length (default: 256)")
- p.add_argument("--max-dec-len", type=int, default=1024,
- help="Max decoder sequence length (default: 1024)")
- p.add_argument("--batch-size", type=int, default=None,
- help="Process in batches of this size, uploading shards to GCS incrementally")
+ p.add_argument("--max-enc-len", type=int, default=1024,
+ help="Max encoder sequence length (default: 1024)")
+ p.add_argument("--max-dec-len", type=int, default=512,
+ help="Max decoder sequence length (default: 512)")
p = sub.add_parser("run", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
@@ -81,17 +65,15 @@ def main():
p.add_argument("--max-len", type=int, default=512)
p.add_argument("--seed", type=int, default=0)
- p = sub.add_parser("test", add_help=False)
+ p = sub.add_parser("eval", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
p.add_argument("--batch-size", type=int, default=32)
p.add_argument("--max-eval-samples", type=int, default=1000)
- p.add_argument("--max-enc-len", type=int, default=256)
- p.add_argument("--max-dec-len", type=int, default=1024)
+ p.add_argument("--max-enc-len", type=int, default=1024)
+ p.add_argument("--max-dec-len", type=int, default=512)
p.add_argument("--max-gen-len", type=int, default=512)
p.add_argument("--tool-call-samples", type=int, default=200,
help="Samples for tool-call accuracy eval (default: 200)")
- p.add_argument("--voice-tc-samples", type=int, default=50,
- help="Samples for voice-to-tool-call eval (default: 50)")
p.add_argument("--throughput-runs", type=int, default=10)
p = sub.add_parser("evaluate", add_help=False)
@@ -141,22 +123,14 @@ def main():
from .tokenize_data import tokenize
tokenize(args)
elif args.command == "train":
- if getattr(args, "full", False):
- args.d_model = 1536
- args.num_heads = 24
- args.num_kv_heads = 8
- args.num_layers = 12
- args.num_dec_layers = 4
- args.num_memory_slots = 128
- args.mat_factors = [2, 3, 4, 8, 16]
from .train import train
train(args)
elif args.command == "run":
from .run import main as run_main
run_main(args)
- elif args.command == "test":
- from .test import main as test_main
- test_main(args)
+ elif args.command == "eval":
+ from .eval import main as eval_main_fn
+ eval_main_fn(args)
elif args.command == "evaluate":
from .evaluate import main as eval_main
eval_main(args)
diff --git a/src/data.py b/src/data.py
index e14fd7d..ffc5c31 100644
--- a/src/data.py
+++ b/src/data.py
@@ -3,7 +3,6 @@ import json as _json
import multiprocessing as mp
import os
import queue
-import subprocess
import threading
os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"
os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"
@@ -14,7 +13,7 @@ logging.getLogger("transformers").setLevel(logging.ERROR)
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
import numpy as np
-from datasets import Audio, Dataset, concatenate_datasets, load_from_disk
+from datasets import Audio, Dataset, load_from_disk
from tqdm import tqdm
import sentencepiece as spm
@@ -23,13 +22,8 @@ _DISK_CACHE_DIR = os.path.join(_PROJECT_ROOT, ".data_cache")
TOKENIZER_DIR = os.path.join(_PROJECT_ROOT, "tokenizer")
TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")
LOCAL_UNIFIED_DIR = os.path.join(_PROJECT_ROOT, "data", "tool_calls_unified")
-GCS_DATASET_PATH = "gs://cactus-dataset/tool_calls"
-JSONL_DATASET_PATHS = [
- "gs://needle-datasets-bucket/datasets/xlam/openai_jsonl_v1/xlam_openai_jsonl/train",
- "gs://needle-datasets-bucket/datasets/toucan/openai_jsonl_v1/toucan_openai_jsonl/train",
-]
-_MIN_SHM_BYTES = 200 * 1024**3
+_MIN_SHM_BYTES = 200 * 1024**3
def _pick_cache_dir():
@@ -53,67 +47,17 @@ CACHE_DIR = _pick_cache_dir()
PAD_ID = 0
EOS_ID = 1
-BOS_ID = 2
+BOS_ID = 2
UNK_ID = 3
TOOL_CALL_ID = 4
TRANSCRIBE_ID = 5
+TOOLS_ID = 6
_unified_dataset_cache = None
-def _load_jsonl_shards(gcs_paths):
- rows = []
- for gcs_path in gcs_paths:
- try:
- result = subprocess.run(
- ["gcloud", "storage", "ls", f"{gcs_path}/*.jsonl"],
- capture_output=True,
- text=True,
- check=True,
- )
- shard_uris = [line.strip() for line in result.stdout.splitlines() if line.strip()]
- except Exception as e:
- print(f"Warning: could not list shards at {gcs_path}: {e}")
- continue
-
- for uri in shard_uris:
- fname = os.path.basename(uri)
- local_path = os.path.join(CACHE_DIR, fname)
- try:
- subprocess.run(
- ["gcloud", "storage", "cp", uri, local_path],
- capture_output=True,
- check=True,
- )
- except Exception as e:
- print(f"Warning: could not download {uri}: {e}")
- continue
-
- with open(local_path, "r", encoding="utf-8") as f:
- for line in f:
- line = line.strip()
- if not line:
- continue
- try:
- record = _json.loads(line)
- except Exception:
- continue
- messages = record.get("messages", [])
- query = messages[0]["content"] if messages else ""
- tool_calls = messages[1].get("tool_calls") if len(messages) > 1 else None
- rows.append({
- "query": query,
- "tools": _json.dumps(record.get("tools", [])),
- "answers": _json.dumps(tool_calls if tool_calls is not None else []),
- })
-
- if not rows:
- return None
- return Dataset.from_list(rows)
-
-
def _load_unified_dataset():
- """Load the unified dataset from GCS (via gcsfs) or local fallback.
+ """Load the unified dataset from local disk.
Caches the result in memory after first load.
Uses soundfile as the audio decoding backend.
@@ -130,30 +74,16 @@ def _load_unified_dataset():
print(f"Loaded unified dataset from {LOCAL_UNIFIED_DIR} ({len(ds)} rows)")
_unified_dataset_cache = _set_audio_backend(ds)
return _unified_dataset_cache
- except Exception:
- print(f"Local dataset at {LOCAL_UNIFIED_DIR} is incomplete, trying GCS...")
-
- try:
- ds = load_from_disk(GCS_DATASET_PATH)
- print(f"Loaded unified dataset from GCS ({len(ds)} rows)")
- ds = _set_audio_backend(ds)
- except Exception as e:
- raise FileNotFoundError(
- f"Unified dataset not found. Run scripts/build_dataset.py first, "
- f"or ensure GCS path {GCS_DATASET_PATH} is accessible (pip install gcsfs). "
- f"GCS error: {e}"
- )
-
- try:
- jsonl_ds = _load_jsonl_shards(JSONL_DATASET_PATHS)
- if jsonl_ds is not None:
- ds = concatenate_datasets([ds, jsonl_ds])
- print(f"Concatenated JSONL shards: total {len(ds)} rows")
- except Exception as e:
- print(f"Warning: failed to load JSONL shards, continuing with Arrow only: {e}")
+ except Exception as e:
+ raise FileNotFoundError(
+ f"Failed to load dataset from {LOCAL_UNIFIED_DIR}: {e}. "
+ f"Run 'python scripts/build_dataset.py' first."
+ )
- _unified_dataset_cache = ds
- return _unified_dataset_cache
+ raise FileNotFoundError(
+ f"Unified dataset not found at {LOCAL_UNIFIED_DIR}. "
+ f"Run 'python scripts/build_dataset.py' first."
+ )
def _set_audio_backend(ds):
@@ -193,6 +123,10 @@ class NeedleTokenizer:
def transcribe_token_id(self):
return TRANSCRIBE_ID
+ @property
+ def tools_token_id(self):
+ return TOOLS_ID
+
@property
def vocab_size(self):
return self.sp.GetPieceSize()
@@ -264,7 +198,7 @@ def train_tokenizer(vocab_size=8192, max_samples=None, force=False):
eos_id=EOS_ID,
bos_id=BOS_ID,
unk_id=UNK_ID,
- user_defined_symbols=["<tool_call>", "<transcribe>"],
+ user_defined_symbols=["<tool_call>", "<transcribe>", "<tools>"],
byte_fallback=True,
normalization_rule_name="identity",
num_threads=os.cpu_count(),
@@ -277,96 +211,13 @@ def train_tokenizer(vocab_size=8192, max_samples=None, force=False):
return model_path
-GCS_TOKENIZER_PATH = "gs://cactus-dataset/tokenizer/"
-
-
-def upload_tokenizer_to_gcs():
- """Upload local tokenizer files to GCS."""
- import subprocess
- import glob as globmod
- tok_files = globmod.glob(os.path.join(TOKENIZER_DIR, "*"))
- if tok_files:
- subprocess.run(
- ["gcloud", "storage", "cp"] + tok_files + [GCS_TOKENIZER_PATH],
- capture_output=True, text=True,
- )
- print(f"Uploaded tokenizer to {GCS_TOKENIZER_PATH}")
-
-
def get_tokenizer(max_samples=None):
model_path = TOKENIZER_PREFIX + ".model"
if not os.path.exists(model_path):
- # Try downloading from GCS first
- if _download_tokenizer_from_gcs():
- return NeedleTokenizer(model_path)
train_tokenizer(max_samples=max_samples)
return NeedleTokenizer(model_path)
-def _download_tokenizer_from_gcs():
- """Download tokenizer files from GCS. Returns True on success."""
- import subprocess
- os.makedirs(TOKENIZER_DIR, exist_ok=True)
- result = subprocess.run(
- ["gcloud", "storage", "cp", "-r", GCS_TOKENIZER_PATH + "*", TOKENIZER_DIR + "/"],
- capture_output=True, text=True,
- )
- model_path = TOKENIZER_PREFIX + ".model"
- if result.returncode == 0 and os.path.exists(model_path):
- print(f"Downloaded tokenizer from {GCS_TOKENIZER_PATH}")
- return True
- return False
-
-
-GCS_CACHE_PATH = "gs://cactus-dataset/cache"
-
-
-def _gcs_cache_download(cache_id, suffixes):
- """Try downloading cached files from GCS. Returns True if all found."""
- import subprocess
- os.makedirs(CACHE_DIR, exist_ok=True)
- cache_path = os.path.join(CACHE_DIR, cache_id)
- srcs = [f"{GCS_CACHE_PATH}/{cache_id}{s}" for s in suffixes]
- subprocess.run(
- ["gcloud", "storage", "cp"] + srcs + [CACHE_DIR + "/"],
- capture_output=True, text=True,
- )
- return all(os.path.exists(cache_path + s) for s in suffixes)
-
-
-def _gcs_cache_upload(cache_id, suffixes):
- """Upload cached files to GCS."""
- import subprocess
- cache_path = os.path.join(CACHE_DIR, cache_id)
- files = [cache_path + s for s in suffixes if os.path.exists(cache_path + s)]
- if files:
- subprocess.run(
- ["gcloud", "storage", "cp"] + files + [GCS_CACHE_PATH + "/"],
- capture_output=True, text=True,
- )
-
-
-def _gcs_download_shards(cache_id, n_shards, shard_suffixes):
- """Download sharded .npy files from GCS. Returns True if all found."""
- import subprocess
- os.makedirs(CACHE_DIR, exist_ok=True)
- srcs = []
- for suffix in shard_suffixes:
- for i in range(n_shards):
- srcs.append(f"{GCS_CACHE_PATH}/{cache_id}{suffix}_{i:05d}.npy")
- if not srcs:
- return True
- subprocess.run(
- ["gcloud", "storage", "cp"] + srcs + [CACHE_DIR + "/"],
- capture_output=True, text=True,
- )
- for suffix in shard_suffixes:
- for i in range(n_shards):
- if not os.path.exists(os.path.join(CACHE_DIR, f"{cache_id}{suffix}_{i:05d}.npy")):
- return False
- return True
-
-
def load_tool_calls(split="train", max_samples=None, return_global_indices=False):
"""Load tool-calling dataset, splitting 90/10 for train/val.
@@ -410,117 +261,66 @@ def _cache_key(prefix, n_samples, max_enc_len, max_dec_len):
return hashlib.md5(key.encode()).hexdigest()[:12]
-def _mel_cache_key(prefix, n_samples, n_mels, max_mel_len):
- tok_hash = _tokenizer_hash()
- key = f"mel_{prefix}_{tok_hash}_{n_samples}_{n_mels}_{max_mel_len}"
- return hashlib.md5(key.encode()).hexdigest()[:12]
-
-
-def _save_cache_metadata(split, text_cache_id, mel_cache_id, n_samples,
- max_enc_len, max_dec_len, n_mels, max_mel_len):
- """Save metadata JSON for a split, upload to GCS."""
+def _save_cache_metadata(split, text_cache_id, n_samples, max_enc_len, max_dec_len):
+ """Save metadata JSON for a split locally."""
os.makedirs(CACHE_DIR, exist_ok=True)
meta = {
"split": split,
"text_cache_id": text_cache_id,
- "mel_cache_id": mel_cache_id,
"n_samples": n_samples,
"max_enc_len": max_enc_len,
"max_dec_len": max_dec_len,
- "n_mels": n_mels,
- "max_mel_len": max_mel_len,
}
meta_path = os.path.join(CACHE_DIR, f"{split}_metadata.json")
with open(meta_path, "w") as f:
_json.dump(meta, f)
- _gcs_cache_upload(f"{split}_metadata", [".json"])
def _load_cache_metadata(split):
- """Load metadata JSON from local or GCS. Returns dict or None."""
+ """Load metadata JSON from local cache. Returns dict or None."""
meta_path = os.path.join(CACHE_DIR, f"{split}_metadata.json")
- if os.path.exists(meta_path):
- with open(meta_path) as f:
- return _json.load(f)
-
- import subprocess
- os.makedirs(CACHE_DIR, exist_ok=True)
- gcs_path = f"{GCS_CACHE_PATH}/{split}_metadata.json"
- subprocess.run(
- ["gcloud", "storage", "cp", gcs_path, meta_path],
- capture_output=True, text=True,
- )
if os.path.exists(meta_path):
with open(meta_path) as f:
return _json.load(f)
return None
-def _tools_to_names(tools_json):
- try:
- tools = _json.loads(tools_json) if isinstance(tools_json, str) else tools_json
- names = [t["function"]["name"] for t in tools if t.get("function", {}).get("name")]
- return _json.dumps(names)
- except Exception:
- return tools_json
-
-
-def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024, batch_size=None):
+def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=1024, max_dec_len=512):
"""Prepare tool-call encoder-decoder pairs with <tool_call> task token.
- Encoder input: tokenize(query), truncated to max_enc_len.
- Decoder input: [BOS, <tool_call>, tools_tokens..., answer_tokens...]
- Decoder target: [<tool_call>, tools_tokens..., answer_tokens..., EOS]
- Loss mask: 1 only on answer + EOS positions (not tools prefix or padding).
+ Encoder input: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.
+ Decoder input: [EOS, <tool_call>, answer_tokens..., PAD...]
+ Decoder target: [<tool_call>, answer_tokens..., EOS, PAD...]
+ Loss mask: 1 on answer + trailing EOS positions only.
- When batch_size is set, produces per-batch shard files uploaded to GCS
- incrementally. Returns (None, None, None, None, kept_indices).
- When batch_size is None, returns (enc_inputs, dec_inputs, dec_targets,
- loss_mask, kept_indices) as before.
+ Returns (enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices).
"""
cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len)
cache_path = os.path.join(CACHE_DIR, cache_id)
- if batch_size is None:
- tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]
-
- def _load_tc_cache():
- return (
- np.load(cache_path + "_enc.npy"),
- np.load(cache_path + "_dec_in.npy"),
- np.load(cache_path + "_dec_tgt.npy"),
- np.load(cache_path + "_loss_mask.npy"),
- np.load(cache_path + "_kept_idx.npy"),
- )
-
- if os.path.exists(cache_path + "_enc.npy"):
- print(f"Loading cached tool-call data ({cache_id})...")
- return _load_tc_cache()
-
- if _gcs_cache_download(cache_id, tc_suffixes):
- print(f"Downloaded tool-call cache from GCS ({cache_id})...")
- return _load_tc_cache()
+ tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]
- if batch_size is not None:
- manifest_path = cache_path + "_manifest.json"
- if os.path.exists(manifest_path):
- kept_indices = np.load(cache_path + "_kept_idx.npy")
- print(f"Loading cached sharded tool-call data ({cache_id})...")
- return None, None, None, None, kept_indices
+ def _load_tc_cache():
+ return (
+ np.load(cache_path + "_enc.npy"),
+ np.load(cache_path + "_dec_in.npy"),
+ np.load(cache_path + "_dec_tgt.npy"),
+ np.load(cache_path + "_loss_mask.npy"),
+ np.load(cache_path + "_kept_idx.npy"),
+ )
- if _gcs_cache_download(cache_id, ["_manifest.json", "_kept_idx.npy"]):
- if os.path.exists(manifest_path):
- kept_indices = np.load(cache_path + "_kept_idx.npy")
- print(f"Downloaded sharded tool-call cache from GCS ({cache_id})...")
- return None, None, None, None, kept_indices
+ if os.path.exists(cache_path + "_enc.npy"):
+ print(f"Loading cached tool-call data ({cache_id})...")
+ return _load_tc_cache()
pad_id = tokenizer.pad_token_id
eos_id = tokenizer.eos_token_id
tool_call_id = tokenizer.tool_call_token_id
+ tools_sep_id = tokenizer.tools_token_id
enc_texts = [ex["query"] for ex in ds]
- tools_texts = [_tools_to_names(ex["tools"]) for ex in ds]
+ tools_texts = [ex["tools"] for ex in ds]
ans_texts = [ex["answers"] for ex in ds]
num_workers = min(os.cpu_count() or 1, 8)
@@ -537,7 +337,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024, ba
tools_chunks = [tools_texts[i:i + chunk_size] for i in range(0, len(tools_texts), chunk_size)]
print(f"Tokenizing tools ({num_workers} workers)...")
with mp.Pool(num_workers, initializer=_init_worker,
- initargs=(model_path, max_dec_len - 2)) as pool:
+ initargs=(model_path, max_enc_len)) as pool:
tools_results = pool.map(_tokenize_chunk, tools_chunks)
all_tools_tokens = [tok for chunk in tools_results for tok in chunk]
@@ -552,113 +352,41 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024, ba
def _fill_sample(j, e_tok, t_tok, a_tok, enc_arr, dec_in_arr, dec_tgt_arr, lm_arr):
"""Fill arrays at position j for one sample. Returns False if skipped."""
- prefix_len = 2 + len(t_tok)
- total_dec = prefix_len + len(a_tok) + 1
- if total_dec > max_dec_len:
- available_for_tools = max_dec_len - 2 - len(a_tok) - 1
- if available_for_tools < 1:
- return False
- t_tok = t_tok[:available_for_tools]
- prefix_len = 2 + len(t_tok)
-
- el = len(e_tok)
- enc_arr[j, :el] = e_tok
-
+ al = len(a_tok)
+ # Decoder needs: [EOS, <tool_call>, answer..., EOS] = 2 + al + 1
+ if 2 + al + 1 > max_dec_len:
+ return False
+
+ # Encoder: [query..., <tools>, tools...] truncated to max_enc_len
+ # Reserve 1 slot for <tools> separator + at least 1 token for tools
+ max_query = max_enc_len - 1 - 1 # room for <tools> + at least 1 tools token
+ if len(e_tok) > max_query:
+ e_tok = e_tok[:max_query]
+ remaining = max_enc_len - len(e_tok) - 1 # 1 for <tools>
+ t_trunc = t_tok[:remaining]
+
+ enc_seq = e_tok + [tools_sep_id] + t_trunc
+ el = len(enc_seq)
+ enc_arr[j, :el] = enc_seq
+
+ # Decoder input: [EOS, <tool_call>, answer...]
dec_in_arr[j, 0] = eos_id
dec_in_arr[j, 1] = tool_call_id
- tl = len(t_tok)
- if tl > 0:
- dec_in_arr[j, 2:2 + tl] = t_tok
- al = len(a_tok)
if al > 0:
- dec_in_arr[j, prefix_len:prefix_len + al] = a_tok
+ dec_in_arr[j, 2:2 + al] = a_tok
+ # Decoder target: [<tool_call>, answer..., EOS]
dec_tgt_arr[j, 0] = tool_call_id
- if tl > 0:
- dec_tgt_arr[j, 1:1 + tl] = t_tok
if al > 0:
- dec_tgt_arr[j, prefix_len - 1:prefix_len - 1 + al] = a_tok
- dec_tgt_arr[j, prefix_len - 1 + al] = eos_id
+ dec_tgt_arr[j, 1:1 + al] = a_tok
+ dec_tgt_arr[j, 1 + al] = eos_id
- lm_arr[j, prefix_len - 1:prefix_len - 1 + al + 1] = 1.0
+ # Loss mask: answer tokens + trailing EOS
+ lm_arr[j, 1:1 + al + 1] = 1.0
return True
os.makedirs(CACHE_DIR, exist_ok=True)
- if batch_size is not None:
- all_kept_indices = []
- shard_counts = []
- shard_idx = 0
- total_skipped = 0
-
- for batch_start in range(0, n, batch_size):
- batch_end = min(batch_start + batch_size, n)
- batch_n = batch_end - batch_start
-
- enc_batch = np.full((batch_n, max_enc_len), pad_id, dtype=np.int32)
- dec_in_batch = np.full((batch_n, max_dec_len), pad_id, dtype=np.int32)
- dec_tgt_batch = np.full((batch_n, max_dec_len), pad_id, dtype=np.int32)
- loss_batch = np.zeros((batch_n, max_dec_len), dtype=np.float32)
-
- skipped = 0
- for j in range(batch_n):
- i = batch_start + j
- if not _fill_sample(j, all_enc_tokens[i], all_tools_tokens[i],
- all_ans_tokens[i], enc_batch, dec_in_batch,
- dec_tgt_batch, loss_batch):
- skipped += 1
-
- if skipped > 0:
- keep = enc_batch[:, 0] != pad_id
- kept_in_batch = np.where(keep)[0]
- enc_batch = enc_batch[keep]
- dec_in_batch = dec_in_batch[keep]
- dec_tgt_batch = dec_tgt_batch[keep]
- loss_batch = loss_batch[keep]
- all_kept_indices.extend((batch_start + kept_in_batch).tolist())
- total_skipped += skipped
- else:
- all_kept_indices.extend(range(batch_start, batch_end))
-
- if len(enc_batch) == 0:
- continue
-
- shard_suffixes = []
- for suffix, arr in [("_enc", enc_batch), ("_dec_in", dec_in_batch),
- ("_dec_tgt", dec_tgt_batch), ("_loss_mask", loss_batch)]:
- shard_suffix = f"{suffix}_{shard_idx:05d}.npy"
- np.save(os.path.join(CACHE_DIR, f"{cache_id}{shard_suffix}"), arr)
- shard_suffixes.append(shard_suffix)
-
- _gcs_cache_upload(cache_id, shard_suffixes)
- for s in shard_suffixes:
- fpath = os.path.join(CACHE_DIR, f"{cache_id}{s}")
- if os.path.exists(fpath):
- os.remove(fpath)
-
- shard_counts.append(len(enc_batch))
- shard_idx += 1
-
- if total_skipped > 0:
- print(f" Skipped {total_skipped} examples (too long for max_dec_len={max_dec_len})")
-
- kept_indices = np.array(all_kept_indices, dtype=np.int64)
- np.save(cache_path + "_kept_idx.npy", kept_indices)
-
- manifest = {
- "n_shards": shard_idx,
- "shard_counts": shard_counts,
- "total_kept": len(kept_indices),
- }
- manifest_path = cache_path + "_manifest.json"
- with open(manifest_path, "w") as f:
- _json.dump(manifest, f)
-
- _gcs_cache_upload(cache_id, ["_manifest.json", "_kept_idx.npy"])
- print(f"Cached {len(kept_indices):,} tool-call pairs in {shard_idx} shards ({cache_id})")
-
- return None, None, None, None, kept_indices
-
enc_inputs = np.full((n, max_enc_len), pad_id, dtype=np.int32)
dec_inputs = np.full((n, max_dec_len), pad_id, dtype=np.int32)
dec_targets = np.full((n, max_dec_len), pad_id, dtype=np.int32)
@@ -687,8 +415,6 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024, ba
np.save(cache_path + "_dec_tgt.npy", dec_targets)
np.save(cache_path + "_loss_mask.npy", loss_mask)
np.save(cache_path + "_kept_idx.npy", kept_indices)
- tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]
- _gcs_cache_upload(cache_id, tc_suffixes)
print(f"Cached {len(enc_inputs):,} tool-call pairs to {CACHE_DIR}/{cache_id}")
return enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices
@@ -727,107 +453,34 @@ def load_tool_call_audio(split="train", max_samples=None):
return indices
-_GCS_SHARD_ROWS = 5000
-_GCS_N_SHARDS = 19
-_shard_cache = {}
-
-
-def _load_shard(shard_idx):
- """Load a single arrow shard, from local or GCS. Caches in memory."""
- if shard_idx in _shard_cache:
- return _shard_cache[shard_idx]
-
- fname = f"data-{shard_idx:05d}-of-{_GCS_N_SHARDS:05d}.arrow"
-
- local_path = os.path.join(LOCAL_UNIFIED_DIR, fname)
- if os.path.exists(local_path):
- import pyarrow as pa
- source = pa.memory_map(local_path, "r")
- try:
- reader = pa.ipc.open_file(source)
- except pa.lib.ArrowInvalid:
- source.seek(0)
- reader = pa.ipc.open_stream(source)
- tbl = reader.read_all()
- from datasets import Dataset
- ds = Dataset(tbl)
- ds = _set_audio_backend(ds)
- _shard_cache[shard_idx] = ds
- return ds
-
- import subprocess
- shm_dir = os.path.join(CACHE_DIR, "arrow_shards")
- os.makedirs(shm_dir, exist_ok=True)
- dest = os.path.join(shm_dir, fname)
- if not os.path.exists(dest):
- gcs_src = f"{GCS_DATASET_PATH}/{fname}"
- subprocess.run(
- ["gcloud", "storage", "cp", gcs_src, dest],
- capture_output=True, text=True,
- )
- if not os.path.exists(dest):
- raise FileNotFoundError(f"Failed to download shard {fname} from GCS")
-
- import pyarrow as pa
- source = pa.memory_map(dest, "r")
- try:
- reader = pa.ipc.open_file(source)
- except pa.lib.ArrowInvalid:
- source.seek(0)
- reader = pa.ipc.open_stream(source)
- tbl = reader.read_all()
- from datasets import Dataset
- ds = Dataset(tbl)
- ds = _set_audio_backend(ds)
- _shard_cache[shard_idx] = ds
- return ds
-
-
-def _load_example_by_index(idx):
- """Load a single example by global index, downloading only the needed shard."""
- shard_idx = idx // _GCS_SHARD_ROWS
- local_idx = idx % _GCS_SHARD_ROWS
- shard = _load_shard(shard_idx)
- if local_idx >= len(shard):
- raise IndexError(f"Index {idx} out of range (shard {shard_idx} has {len(shard)} rows)")
- return shard[local_idx]
-
-
-def load_audio_for_index(idx):
- """Load and decode audio for a single dataset index.
+def load_example_with_audio(idx):
+ """Load a dataset example by global index.
- Returns (audio_array, sampling_rate) or (None, None) if no audio.
+ Returns dict with {query, answers, tools, audio_array, sampling_rate}.
+ Audio fields are None for text-only datasets.
"""
- import io
- import soundfile as sf
+ ds = _load_unified_dataset()
+ if idx < 0 or idx >= len(ds):
+ raise IndexError(f"Index {idx} out of range (dataset has {len(ds)} rows)")
+ ex = ds[idx]
- ex = _load_example_by_index(idx)
+ audio, sr = None, None
audio_val = ex.get("audio")
- if audio_val is None:
- return None, None
-
- raw_bytes = None
- if isinstance(audio_val, dict):
- raw_bytes = audio_val.get("bytes")
- elif isinstance(audio_val, bytes):
- raw_bytes = audio_val
- if raw_bytes is None:
- return None, None
-
- audio_array, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")
- if audio_array.ndim > 1:
- audio_array = audio_array.mean(axis=1)
- return audio_array.astype(np.float32), sr
+ if audio_val is not None:
+ import io
+ import soundfile as sf
+ raw_bytes = None
+ if isinstance(audio_val, dict):
+ raw_bytes = audio_val.get("bytes")
+ elif isinstance(audio_val, bytes):
+ raw_bytes = audio_val
+ if raw_bytes is not None:
+ audio, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")
+ if audio.ndim > 1:
+ audio = audio.mean(axis=1)
+ audio = audio.astype(np.float32)
-def load_example_with_audio(idx):
- """Load a dataset example with decoded audio for eval use.
-
- Downloads only the needed arrow shard (~1-4GB) instead of the full dataset (~35GB).
- Returns dict with {query, answers, tools, audio_array, sampling_rate}.
- """
- ex = _load_example_by_index(idx)
- audio, sr = load_audio_for_index(idx)
return {
"query": ex["query"],
"answers": ex["answers"],
@@ -840,7 +493,7 @@ def load_example_with_audio(idx):
def compute_mel_spectrogram(audio, sr=16000, n_mels=80, n_fft=400, hop_length=160):
"""Compute log-mel spectrogram using numpy/scipy. Returns (T_mel, n_mels) float32.
- 25ms window (n_fft=400 at 16kHz), 10ms hop (hop_length=160) → ~100 frames/sec.
+ 25ms window (n_fft=400 at 16kHz), 10ms hop (hop_length=160) -> ~100 frames/sec.
"""
from scipy.signal import windows as scipy_windows
from scipy.fft import rfft
@@ -882,105 +535,6 @@ def compute_mel_spectrogram(audio, sr=16000, n_mels=80, n_fft=400, hop_length=16
return log_mel.astype(np.float32)
-def precompute_mels(kept_indices, n_mels=80, max_mel_len=1024, cache_id_prefix="", batch_size=None):
- """Precompute mel spectrograms.
-
- When batch_size is None: write a single (N, max_mel_len, n_mels) memmap file.
- When batch_size is set: produce per-batch shard .npy files, upload each to GCS.
- Returns the cache_id.
- """
- n_samples = len(kept_indices)
- cache_id = _mel_cache_key(cache_id_prefix, n_samples, n_mels, max_mel_len)
- cache_path = os.path.join(CACHE_DIR, cache_id)
-
- if batch_size is None:
- mel_file = cache_path + "_mels.npy"
-
- if os.path.exists(mel_file):
- print(f" Mel cache already exists ({cache_id})")
- return cache_id
-
- if _gcs_cache_download(cache_id, ["_mels.npy"]):
- print(f" Downloaded mel cache from GCS ({cache_id})")
- return cache_id
-
- os.makedirs(CACHE_DIR, exist_ok=True)
- shape = (n_samples, max_mel_len, n_mels)
- fp = np.lib.format.open_memmap(mel_file, mode='w+', dtype=np.float32, shape=shape)
-
- print(f" Precomputing {n_samples} mel spectrograms (n_mels={n_mels}, max_mel_len={max_mel_len})...")
- for i, idx in enumerate(tqdm(kept_indices, desc=" Computing mels")):
- audio, sr = load_audio_for_index(int(idx))
- if audio is None:
- continue
- mel = compute_mel_spectrogram(audio, sr=sr, n_mels=n_mels)
- if mel.shape[0] > max_mel_len:
- mel = mel[:max_mel_len]
- t = mel.shape[0]
- fp[i, :t, :] = mel
-
- del fp
- _gcs_cache_upload(cache_id, ["_mels.npy"])
- print(f" Cached mel spectrograms to {CACHE_DIR}/{cache_id}")
- return cache_id
-
- manifest_path = cache_path + "_mel_manifest.json"
-
- if os.path.exists(manifest_path):
- print(f" Mel shard cache already exists ({cache_id})")
- return cache_id
-
- if _gcs_cache_download(cache_id, ["_mel_manifest.json"]):
- if os.path.exists(manifest_path):
- print(f" Downloaded mel shard manifest from GCS ({cache_id})")
- return cache_id
-
- os.makedirs(CACHE_DIR, exist_ok=True)
- shard_counts = []
- shard_idx = 0
-
- print(f" Precomputing {n_samples} mel spectrograms in shards of {batch_size}...")
- for batch_start in range(0, n_samples, batch_size):
- batch_end = min(batch_start + batch_size, n_samples)
- batch_indices = kept_indices[batch_start:batch_end]
- batch_n = len(batch_indices)
-
- mel_batch = np.zeros((batch_n, max_mel_len, n_mels), dtype=np.float32)
-
- for j, idx in enumerate(tqdm(batch_indices, desc=f" Shard {shard_idx}", leave=False)):
- audio, sr = load_audio_for_index(int(idx))
- if audio is None:
- continue
- mel = compute_mel_spectrogram(audio, sr=sr, n_mels=n_mels)
- if mel.shape[0] > max_mel_len:
- mel = mel[:max_mel_len]
- t = mel.shape[0]
- mel_batch[j, :t, :] = mel
-
- shard_suffix = f"_mels_{shard_idx:05d}.npy"
- shard_path = os.path.join(CACHE_DIR, f"{cache_id}{shard_suffix}")
- np.save(shard_path, mel_batch)
-
- _gcs_cache_upload(cache_id, [shard_suffix])
- if os.path.exists(shard_path):
- os.remove(shard_path)
-
- shard_counts.append(batch_n)
- shard_idx += 1
-
- manifest = {
- "n_shards": shard_idx,
- "shard_counts": shard_counts,
- "total_samples": n_samples,
- }
- with open(manifest_path, "w") as f:
- _json.dump(manifest, f)
-
- _gcs_cache_upload(cache_id, ["_mel_manifest.json"])
- print(f" Cached mel spectrograms in {shard_idx} shards ({cache_id})")
- return cache_id
-
-
class ShardedMmapArray:
"""Array-like wrapper over multiple .npy shard files with mmap support.
@@ -1039,12 +593,11 @@ class ShardedMmapArray:
def load_prepared_data(split, mmap=False):
"""Load pre-tokenized .npy files. If mmap=True, returns memory-mapped arrays.
- Supports both single-file and sharded (manifest-based) caches.
- Tries local cache first, then GCS download. Raises FileNotFoundError
- if not found (no fallback tokenization — run 'needle tokenize' first).
+ Tries local cache only. Raises FileNotFoundError if not found
+ (run 'needle tokenize' first).
Returns dict with keys: enc_inputs, dec_inputs, dec_targets, loss_mask,
- kept_indices, mel_cache_id.
+ kept_indices.
"""
meta = _load_cache_metadata(split)
if meta is None:
@@ -1054,52 +607,12 @@ def load_prepared_data(split, mmap=False):
text_cache_id = meta["text_cache_id"]
cache_path = os.path.join(CACHE_DIR, text_cache_id)
- manifest_path = cache_path + "_manifest.json"
-
- if not os.path.exists(manifest_path):
- _gcs_cache_download(text_cache_id, ["_manifest.json"])
-
- if os.path.exists(manifest_path):
- with open(manifest_path) as f:
- manifest = _json.load(f)
- n_shards = manifest["n_shards"]
-
- if not os.path.exists(cache_path + "_kept_idx.npy"):
- _gcs_cache_download(text_cache_id, ["_kept_idx.npy"])
-
- shard_suffixes = ["_enc", "_dec_in", "_dec_tgt", "_loss_mask"]
- _gcs_download_shards(text_cache_id, n_shards, shard_suffixes)
-
- def _shard_paths(suffix):
- return [os.path.join(CACHE_DIR, f"{text_cache_id}{suffix}_{i:05d}.npy")
- for i in range(n_shards)]
-
- mmap_mode = "r" if mmap else None
- if mmap:
- result = {
- "enc_inputs": ShardedMmapArray(_shard_paths("_enc"), mmap_mode="r"),
- "dec_inputs": ShardedMmapArray(_shard_paths("_dec_in"), mmap_mode="r"),
- "dec_targets": ShardedMmapArray(_shard_paths("_dec_tgt"), mmap_mode="r"),
- "loss_mask": ShardedMmapArray(_shard_paths("_loss_mask"), mmap_mode="r"),
- }
- else:
- result = {
- "enc_inputs": np.concatenate([np.load(p) for p in _shard_paths("_enc")]),
- "dec_inputs": np.concatenate([np.load(p) for p in _shard_paths("_dec_in")]),
- "dec_targets": np.concatenate([np.load(p) for p in _shard_paths("_dec_tgt")]),
- "loss_mask": np.concatenate([np.load(p) for p in _shard_paths("_loss_mask")]),
- }
-
- result["kept_indices"] = np.load(cache_path + "_kept_idx.npy", mmap_mode=mmap_mode)
- result["mel_cache_id"] = meta.get("mel_cache_id")
- return result
tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]
if not os.path.exists(cache_path + "_enc.npy"):
- if not _gcs_cache_download(text_cache_id, tc_suffixes):
- raise FileNotFoundError(
- f"Text cache '{text_cache_id}' not found. Run 'needle tokenize' first."
- )
+ raise FileNotFoundError(
+ f"Text cache '{text_cache_id}' not found. Run 'needle tokenize' first."
+ )
mmap_mode = "r" if mmap else None
return {
@@ -1108,140 +621,9 @@ def load_prepared_data(split, mmap=False):
"dec_targets": np.load(cache_path + "_dec_tgt.npy", mmap_mode=mmap_mode),
"loss_mask": np.load(cache_path + "_loss_mask.npy", mmap_mode=mmap_mode),
"kept_indices": np.load(cache_path + "_kept_idx.npy", mmap_mode=mmap_mode),
- "mel_cache_id": meta.get("mel_cache_id"),
}
-def load_prepared_mels(mel_cache_id, mmap=False):
- """Load precomputed mel .npy file(s), optionally memory-mapped.
-
- Supports both single-file and sharded (manifest-based) caches.
- """
- cache_path = os.path.join(CACHE_DIR, mel_cache_id)
- manifest_path = cache_path + "_mel_manifest.json"
-
- if not os.path.exists(manifest_path):
- _gcs_cache_download(mel_cache_id, ["_mel_manifest.json"])
-
- if os.path.exists(manifest_path):
- with open(manifest_path) as f:
- manifest = _json.load(f)
- n_shards = manifest["n_shards"]
-
- _gcs_download_shards(mel_cache_id, n_shards, ["_mels"])
-
- shard_paths = [os.path.join(CACHE_DIR, f"{mel_cache_id}_mels_{i:05d}.npy")
- for i in range(n_shards)]
-
- if mmap:
- return ShardedMmapArray(shard_paths, mmap_mode="r")
- else:
- return np.concatenate([np.load(p) for p in shard_paths])
-
- mel_file = cache_path + "_mels.npy"
- if not os.path.exists(mel_file):
- if not _gcs_cache_download(mel_cache_id, ["_mels.npy"]):
- raise FileNotFoundError(
- f"Mel cache '{mel_cache_id}' not found. Run 'needle tokenize' first."
- )
-
- mmap_mode = "r" if mmap else None
- return np.load(mel_file, mmap_mode=mmap_mode)
-
-
-def build_audio_augmenter(sr=16000):
- """Build an audiomentations augmentation pipeline for training.
-
- Returns an augmenter callable or None if audiomentations is unavailable.
- """
- try:
- import audiomentations as A
- except ImportError:
- print(" WARNING: audiomentations not installed — no waveform augmentation")
- return None
-
- return A.Compose([
- A.AddGaussianSNR(min_snr_db=10.0, max_snr_db=35.0, p=0.5),
- A.AddGaussianNoise(min_amplitude=0.001, max_amplitude=0.015, p=0.3),
- A.TimeStretch(min_rate=0.9, max_rate=1.1, p=0.3),
- A.PitchShift(min_semitones=-2, max_semitones=2, p=0.3),
- A.Gain(min_gain_db=-6, max_gain_db=6, p=0.4),
- A.LowPassFilter(min_cutoff_freq=3000, max_cutoff_freq=7500, p=0.2),
- A.HighPassFilter(min_cutoff_freq=50, max_cutoff_freq=400, p=0.2),
- A.ClippingDistortion(min_percentile_threshold=0, max_percentile_threshold=5, p=0.1),
- ])
-
-
-def _load_mel_batch(audio_arrays, n_mels, max_mel_len, augmenter=None, sr=16000):
- """Compute mel spectrograms for a batch of audio arrays.
-
- If augmenter is provided, applies waveform augmentation before mel computation.
- """
- mels = []
- for audio in audio_arrays:
- audio = np.array(audio, dtype=np.float32)
- if audio.ndim > 1:
- audio = audio.mean(axis=1)
-
- if augmenter is not None:
- audio = augmenter(samples=audio, sample_rate=sr)
-
- mel = compute_mel_spectrogram(audio, sr=sr, n_mels=n_mels)
-
- if mel.shape[0] > max_mel_len:
- mel = mel[:max_mel_len]
- elif mel.shape[0] < max_mel_len:
- pad_len = max_mel_len - mel.shape[0]
- mel = np.pad(mel, ((0, pad_len), (0, 0)))
-
- mels.append(mel)
-
- return np.stack(mels).astype(np.float32)
-
-
-def _load_audio_batch(ds_indices):
- """Load and decode audio for a batch of dataset indices."""
- import io
- import soundfile as sf
-
- ds = _load_unified_dataset()
- arrays = []
- for idx in ds_indices:
- ex = ds[int(idx)]
- audio_val = ex.get("audio")
- raw_bytes = None
- if isinstance(audio_val, dict):
- raw_bytes = audio_val.get("bytes")
- elif isinstance(audio_val, bytes):
- raw_bytes = audio_val
- if raw_bytes is None:
- arrays.append(np.zeros(16000, dtype=np.float32))
- continue
- audio_array, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")
- if audio_array.ndim > 1:
- audio_array = audio_array.mean(axis=1)
- arrays.append(audio_array.astype(np.float32))
- return arrays
-
-
-def get_speech_batches(mel_data, dec_inputs, dec_targets, batch_size,
- shuffle=True, loss_mask=None):
- """Yield speech batches from precomputed mel data.
-
- mel_data: array of shape (N, max_mel_len, n_mels), possibly memory-mapped.
- Uses per-batch fancy indexing to avoid copying full arrays.
- """
- n = len(mel_data)
- indices = np.random.permutation(n) if shuffle else np.arange(n)
-
- for i in range(0, n - batch_size + 1, batch_size):
- idx = indices[i : i + batch_size]
- batch = (np.array(mel_data[idx]), np.array(dec_inputs[idx]), np.array(dec_targets[idx]))
- if loss_mask is not None:
- batch = batch + (np.array(loss_mask[idx]),)
- yield batch
-
-
class PrefetchIterator:
"""Generic prefetch wrapper: runs any batch-generating callable in a background thread."""
diff --git a/src/test.py b/src/eval.py
similarity index 65%
rename from src/test.py
rename to src/eval.py
index 93502d2..a21ea5a 100644
--- a/src/test.py
+++ b/src/eval.py
@@ -7,7 +7,7 @@ import jax.numpy as jnp
import numpy as np
import optax
-from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs, load_tool_call_audio, load_example_with_audio
+from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs
from .model import (
EncoderDecoderTransformer,
TransformerConfig,
@@ -168,112 +168,18 @@ def compute_wer(hypotheses, references):
return total_edits / max(total_ref_words, 1)
-def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=512):
+def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=512, max_enc_len=512):
"""Generate tool-call predictions and compute structured metrics."""
import json
- from .run import generate
+ from .run import generate_batch
from .data import load_tool_calls
ds = load_tool_calls("validation", max_samples=num_samples)
- total = 0
- exact_match = 0
- tp_names = 0 # predicted name in reference
- fp_names = 0 # predicted name not in reference
- fn_names = 0 # reference name not predicted
- tp_calls = 0
- fp_calls = 0
- fn_calls = 0
- json_parse_errors = 0
- empty_ref = 0
- empty_pred = 0
-
- samples = []
-
- for i, ex in enumerate(ds):
- ref_text = ex["answers"]
-
- pred_text = generate(
- model, params, tokenizer, ex["query"],
- tools=ex["tools"], max_gen_len=max_gen_len, seed=i, stream=False,
- ).strip()
-
- try:
- ref_calls = json.loads(ref_text)
- except (json.JSONDecodeError, TypeError):
- ref_calls = []
-
- try:
- pred_calls = json.loads(pred_text)
- if not isinstance(pred_calls, list):
- pred_calls = [pred_calls] if isinstance(pred_calls, dict) else []
- except (json.JSONDecodeError, TypeError):
- json_parse_errors += 1
- pred_calls = []
-
- total += 1
- if i < 10:
- samples.append((ex["query"][:80], ref_text[:120], pred_text[:120]))
-
- if len(ref_calls) == 0:
- empty_ref += 1
- if len(pred_calls) == 0:
- empty_pred += 1
-
- if json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):
- exact_match += 1
-
- ref_names = [c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c]
- pred_names = [c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c]
- ref_name_set = set(ref_names)
- pred_name_set = set(pred_names)
- tp_names += len(pred_name_set & ref_name_set)
- fp_names += len(pred_name_set - ref_name_set)
- fn_names += len(ref_name_set - pred_name_set)
-
- def call_key(c):
- if not isinstance(c, dict):
- return None
- return json.dumps({"name": c.get("name"), "arguments": c.get("arguments")}, sort_keys=True)
-
- ref_keys = set(call_key(c) for c in ref_calls)
- pred_keys = set(call_key(c) for c in pred_calls)
- ref_keys.discard(None)
- pred_keys.discard(None)
- tp_calls += len(pred_keys & ref_keys)
- fp_calls += len(pred_keys - ref_keys)
- fn_calls += len(ref_keys - pred_keys)
-
- name_precision = tp_names / max(tp_names + fp_names, 1)
- name_recall = tp_names / max(tp_names + fn_names, 1)
- name_f1 = 2 * name_precision * name_recall / max(name_precision + name_recall, 1e-9)
-
- call_precision = tp_calls / max(tp_calls + fp_calls, 1)
- call_recall = tp_calls / max(tp_calls + fn_calls, 1)
- call_f1 = 2 * call_precision * call_recall / max(call_precision + call_recall, 1e-9)
-
- return {
- "num_samples": total,
- "exact_match": exact_match / max(total, 1),
- "json_parse_rate": 1.0 - json_parse_errors / max(total, 1),
- "name_precision": name_precision,
- "name_recall": name_recall,
- "name_f1": name_f1,
- "call_precision": call_precision,
- "call_recall": call_recall,
- "call_f1": call_f1,
- "empty_ref_pct": empty_ref / max(total, 1),
- "empty_pred_pct": empty_pred / max(total, 1),
- "samples": samples,
- }
-
-
-def benchmark_voice_tool_calls(model, params, tokenizer, num_samples=100, max_gen_len=512):
- """Generate tool-call predictions from audio and compute structured metrics."""
- import json
- from .run import generate_from_audio
-
- indices = load_tool_call_audio("validation", max_samples=num_samples)
+ # Batch-generate all predictions at once
+ queries = [ex["query"] for ex in ds]
+ tools_list = [ex["tools"] for ex in ds]
+ all_preds = generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=max_gen_len, max_enc_len=max_enc_len)
total = 0
exact_match = 0
@@ -288,16 +194,14 @@ def benchmark_voice_tool_calls(model, params, tokenizer, num_samples=100, max_ge
empty_pred = 0
samples = []
- for i, ds_idx in enumerate(indices):
- pair = load_example_with_audio(ds_idx)
- if pair["audio_array"] is None:
- continue
- ref_text = pair["answers"]
+ def call_key(c):
+ if not isinstance(c, dict):
+ return None
+ return json.dumps({"name": c.get("name"), "arguments": c.get("arguments")}, sort_keys=True)
- pred_text = generate_from_audio(
- model, params, tokenizer, pair["audio_array"], sr=pair["sampling_rate"],
- tools=pair["tools"], max_gen_len=max_gen_len, seed=i, stream=False,
- ).strip()
+ for i, (ex, pred_text) in enumerate(zip(ds, all_preds)):
+ ref_text = ex["answers"]
+ pred_text = pred_text.strip()
try:
ref_calls = json.loads(ref_text)
@@ -314,7 +218,7 @@ def benchmark_voice_tool_calls(model, params, tokenizer, num_samples=100, max_ge
total += 1
if i < 10:
- samples.append((pair["query"][:80], ref_text[:120], pred_text[:120]))
+ samples.append((ex["query"][:80], ref_text[:120], pred_text[:120]))
if len(ref_calls) == 0:
empty_ref += 1
@@ -324,23 +228,14 @@ def benchmark_voice_tool_calls(model, params, tokenizer, num_samples=100, max_ge
if json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):
exact_match += 1
- ref_names = [c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c]
- pred_names = [c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c]
- ref_name_set = set(ref_names)
- pred_name_set = set(pred_names)
+ ref_name_set = {c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c}
+ pred_name_set = {c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c}
tp_names += len(pred_name_set & ref_name_set)
fp_names += len(pred_name_set - ref_name_set)
fn_names += len(ref_name_set - pred_name_set)
- def call_key(c):
- if not isinstance(c, dict):
- return None
- return json.dumps({"name": c.get("name"), "arguments": c.get("arguments")}, sort_keys=True)
-
- ref_keys = set(call_key(c) for c in ref_calls)
- pred_keys = set(call_key(c) for c in pred_calls)
- ref_keys.discard(None)
- pred_keys.discard(None)
+ ref_keys = {call_key(c) for c in ref_calls} - {None}
+ pred_keys = {call_key(c) for c in pred_calls} - {None}
tp_calls += len(pred_keys & ref_keys)
fp_calls += len(pred_keys - ref_keys)
fn_calls += len(ref_keys - pred_keys)
@@ -390,7 +285,7 @@ def main(args):
tc = None
if tc_samples > 0:
print(f"\nevaluating tool-call accuracy ({tc_samples} samples)...")
- tc = benchmark_tool_calls(model, params, tokenizer, num_samples=tc_samples, max_gen_len=args.max_gen_len)
+ tc = benchmark_tool_calls(model, params, tokenizer, num_samples=tc_samples, max_gen_len=args.max_gen_len, max_enc_len=args.max_enc_len)
print(f"\n ─────────────────────────────────────")
print(f" Tool-Call Metrics")
@@ -417,30 +312,6 @@ def main(args):
print(f" P: {pred}")
print()
- voice_tc_samples = getattr(args, "voice_tc_samples", 50)
- if voice_tc_samples > 0:
- print(f"\nevaluating voice-to-tool-call ({voice_tc_samples} samples)...")
- vtc = benchmark_voice_tool_calls(model, params, tokenizer, num_samples=voice_tc_samples, max_gen_len=args.max_gen_len)
- print(f"\n ─── Voice-Tool-Call Metrics ─────────")
- print(f" JSON parse rate {vtc['json_parse_rate']:>10.1%}")
- print(f" Exact match {vtc['exact_match']:>10.1%}")
- print(f" ─── Function Name ───────────────────")
- print(f" Precision {vtc['name_precision']:>10.3f}")
- print(f" Recall {vtc['name_recall']:>10.3f}")
- print(f" F1 {vtc['name_f1']:>10.3f}")
- print(f" ─── Full Call (name+args) ───────────")
- print(f" Precision {vtc['call_precision']:>10.3f}")
- print(f" Recall {vtc['call_recall']:>10.3f}")
- print(f" F1 {vtc['call_f1']:>10.3f}")
- print(f" ─────────────────────────────────────")
- if vtc["samples"]:
- print(f"\n samples:")
- for query, ref, pred in vtc["samples"][:5]:
- print(f" Q: {query}")
- print(f" R: {ref}")
- print(f" P: {pred}")
- print()
-
print(f"\nmeasuring throughput ({args.throughput_runs} runs)...")
throughput = measure_throughput(model, params, tokenizer, num_runs=args.throughput_runs)
print(f"avg tokens: {throughput['avg_tokens_generated']:.1f}")
@@ -456,12 +327,9 @@ def parse_args():
parser.add_argument("--max-enc-len", type=int, default=256)
parser.add_argument("--max-dec-len", type=int, default=1024)
parser.add_argument("--max-gen-len", type=int, default=512)
- parser.add_argument("--temperature", type=float, default=0.8)
parser.add_argument("--throughput-runs", type=int, default=10)
parser.add_argument("--tool-call-samples", type=int, default=200,
help="Samples for tool-call accuracy eval (default: 200)")
- parser.add_argument("--voice-tc-samples", type=int, default=50,
- help="Samples for voice-to-tool-call eval (default: 50)")
return parser.parse_args()
diff --git a/src/run.py b/src/run.py
index f029414..7bb18ef 100644
--- a/src/run.py
+++ b/src/run.py
@@ -48,13 +48,26 @@ def load_checkpoint(path):
return params, config
-def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=0, stream=True, task_token_id=None):
+def _build_encoder_input(tokenizer, query, tools, max_enc_len=1024):
+ """Build encoder input: [query..., <tools>, tools...] truncated to max_enc_len."""
+ tools_sep_id = tokenizer.tools_token_id
+ q_toks = tokenizer.encode(query)
+ t_toks = tokenizer.encode(tools)
+ max_query = max_enc_len - 2
+ if len(q_toks) > max_query:
+ q_toks = q_toks[:max_query]
+ remaining = max_enc_len - len(q_toks) - 1
+ t_toks = t_toks[:remaining]
+ return q_toks + [tools_sep_id] + t_toks
+
+
+def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, max_enc_len=1024, seed=0, stream=True, task_token_id=None):
"""Generate tool-call output.
- Encoder: query only.
- Decoder: prefilled with [EOS, <tool_call>, tools_tokens...], then greedy decode.
+ Encoder: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.
+ Decoder: prefilled with [EOS, <tool_call>], then greedy decode.
"""
- enc_tokens = tokenizer.encode(query)
+ enc_tokens = _build_encoder_input(tokenizer, query, tools, max_enc_len)
enc_input = jnp.array([enc_tokens])
pad_id = tokenizer.pad_token_id
@@ -66,13 +79,12 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=
{"params": params}, enc_input, src_mask=src_mask, method="encode"
)
- tools_tokens = tokenizer.encode(tools)
- prefix = [eos_id, tool_call_id] + tools_tokens
- prefix_len = min(len(prefix), max_gen_len)
+ prefix = [eos_id, tool_call_id]
+ prefix_len = 2
dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
- for j, tok in enumerate(prefix[:max_gen_len]):
- dec_buffer = dec_buffer.at[0, j].set(tok)
+ dec_buffer = dec_buffer.at[0, 0].set(eos_id)
+ dec_buffer = dec_buffer.at[0, 1].set(tool_call_id)
decode_fn = _get_decode_fn(model, max_gen_len)
@@ -106,6 +118,68 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=
return tokenizer.decode(generated_tokens)
+def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=512, max_enc_len=1024):
+ """Batch-generate tool-call outputs for multiple examples at once.
+
+ Encoder: [query_tokens..., <tools>, tools_tokens...] per example, truncated to max_enc_len.
+ Decoder: uniform prefix [EOS, <tool_call>] for all examples.
+
+ Returns a list of decoded strings, one per example.
+ """
+ B = len(queries)
+ pad_id = tokenizer.pad_token_id
+ eos_id = tokenizer.eos_token_id
+ tool_call_id = tokenizer.tool_call_token_id
+
+ # --- Encode: [query..., <tools>, tools...] per example, truncated ---
+ enc_token_lists = []
+ for q, t in zip(queries, tools_list):
+ enc_token_lists.append(_build_encoder_input(tokenizer, q, t, max_enc_len))
+ max_enc = max(len(toks) for toks in enc_token_lists)
+ enc_input = np.full((B, max_enc), pad_id, dtype=np.int32)
+ for i, toks in enumerate(enc_token_lists):
+ enc_input[i, :len(toks)] = toks
+ enc_input = jnp.array(enc_input)
+ src_mask = make_padding_mask(enc_input, pad_id)
+
+ encoder_out = model.apply(
+ {"params": params}, enc_input, src_mask=src_mask, method="encode"
+ )
+
+ # --- Decoder prefix: uniform [EOS, <tool_call>] (prefix_len=2) ---
+ prefix_len = 2
+ dec_buffer = np.full((B, max_gen_len), pad_id, dtype=np.int32)
+ dec_buffer[:, 0] = eos_id
+ dec_buffer[:, 1] = tool_call_id
+ dec_buffer = jnp.array(dec_buffer)
+
+ decode_fn = _get_decode_fn(model, max_gen_len)
+
+ # --- Autoregressive decoding ---
+ finished = [False] * B
+ gen_tokens = [[] for _ in range(B)]
+
+ logits = decode_fn(params, dec_buffer, encoder_out)
+
+ for pos in range(prefix_len - 1, max_gen_len - 1):
+ for i in range(B):
+ if finished[i]:
+ continue
+ next_token = int(jnp.argmax(logits[i, pos]))
+ if next_token == eos_id:
+ finished[i] = True
+ continue
+ gen_tokens[i].append(next_token)
+ dec_buffer = dec_buffer.at[i, pos + 1].set(next_token)
+
+ if all(finished):
+ break
+
+ logits = decode_fn(params, dec_buffer, encoder_out)
+
+ return [tokenizer.decode(toks) for toks in gen_tokens]
+
+
def load_audio(path, target_sr=16000):
"""Load an audio file and resample to target_sr. Returns (audio_array, sr)."""
import soundfile as sf
diff --git a/src/tokenize_data.py b/src/tokenize_data.py
index 04a6b5b..1158123 100644
--- a/src/tokenize_data.py
+++ b/src/tokenize_data.py
@@ -1,45 +1,28 @@
"""Standalone tokenization pipeline: train tokenizer and pre-tokenize all data.
-Trains the SentencePiece tokenizer and tokenizes train + val splits for both
-text and voice data, caching everything on GCS. Running fresh overwrites all
-existing tokenizer + caches.
+Trains the SentencePiece tokenizer and tokenizes train + val splits,
+caching everything locally.
Usage:
needle tokenize # full run
needle tokenize --max-samples 1000 # dev/test
- needle tokenize --cleanup # delete local cache after GCS upload
"""
import os
import shutil
-import subprocess
from .data import (
CACHE_DIR,
- GCS_CACHE_PATH,
- GCS_TOKENIZER_PATH,
TOKENIZER_DIR,
_cache_key,
_save_cache_metadata,
get_tokenizer,
load_tool_calls,
- precompute_mels,
prepare_tool_call_pairs,
train_tokenizer,
- upload_tokenizer_to_gcs,
)
-def _clear_gcs_caches():
- """Remove existing GCS cache and tokenizer files."""
- for path in [GCS_CACHE_PATH + "/*", GCS_TOKENIZER_PATH + "*"]:
- print(f"Clearing {path} ...")
- subprocess.run(
- ["gcloud", "storage", "rm", "-r", path],
- capture_output=True, text=True,
- )
-
-
def _clear_local_caches():
"""Remove local .data_cache/ and tokenizer/ directories."""
for d in [CACHE_DIR, TOKENIZER_DIR]:
@@ -50,21 +33,16 @@ def _clear_local_caches():
def tokenize(args):
print("=== Clearing existing caches ===")
- _clear_gcs_caches()
_clear_local_caches()
print("\n=== Training tokenizer ===")
train_tokenizer(max_samples=args.max_samples, force=True)
- upload_tokenizer_to_gcs()
tokenizer = get_tokenizer()
- print("\n=== Tokenizing text data + precomputing mels ===")
- max_enc_len = getattr(args, "max_enc_len", 256)
- max_dec_len = getattr(args, "max_dec_len", 1024)
- n_mels = getattr(args, "n_mels", 80)
- max_mel_len = getattr(args, "max_mel_len", 1024)
- batch_size = getattr(args, "batch_size", 5000)
+ print("\n=== Tokenizing text data ===")
+ max_enc_len = getattr(args, "max_enc_len", 1024)
+ max_dec_len = getattr(args, "max_dec_len", 512)
for split in ("train", "val"):
print(f"\n--- {split} split ---")
@@ -75,21 +53,10 @@ def tokenize(args):
)
_, _, _, _, kept_indices = prepare_tool_call_pairs(
ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len,
- batch_size=batch_size,
)
text_cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len)
- mel_cache_id = precompute_mels(
- global_indices[kept_indices], n_mels=n_mels, max_mel_len=max_mel_len,
- cache_id_prefix=split, batch_size=batch_size,
- )
-
- _save_cache_metadata(split, text_cache_id, mel_cache_id, len(kept_indices),
- max_enc_len, max_dec_len, n_mels, max_mel_len)
-
- if args.cleanup and os.path.exists(CACHE_DIR):
- print(f"\n=== Cleaning up {CACHE_DIR}/ ===")
- shutil.rmtree(CACHE_DIR)
- print("Done.")
+ _save_cache_metadata(split, text_cache_id, len(kept_indices),
+ max_enc_len, max_dec_len)
print("\n=== Tokenization pipeline complete ===")
diff --git a/src/train.py b/src/train.py
index f7bfdab..797c607 100644
--- a/src/train.py
+++ b/src/train.py
@@ -1,4 +1,3 @@
-import argparse
import math
import os
import pickle
@@ -14,9 +13,8 @@ from flax import jax_utils
from flax.training import train_state
from .data import (
- get_batches, get_tokenizer, get_speech_batches,
- load_prepared_data, load_prepared_mels,
- load_example_with_audio,
+ get_batches, get_tokenizer,
+ load_prepared_data, load_tool_calls,
PrefetchIterator, count_batches,
)
from .model import (
@@ -24,7 +22,6 @@ from .model import (
TransformerConfig,
make_causal_mask,
make_padding_mask,
- make_mel_padding_mask,
)
def _newton_schulz(G, steps=5):
@@ -251,28 +248,6 @@ def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rn
return ce_loss + z_loss + div_loss
-def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask):
- pad_id = 0
- src_mask = make_mel_padding_mask(mel)
- tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
- spec_rng, drop_rng = jax.random.split(rng)
- logits, slot_div = state.apply_fn(
- {"params": _quantize_params(params, group_size=_GROUP_SIZE)},
- mel, tgt_in, src_mask=src_mask, tgt_mask=tgt_mask,
- ffn_mask=ffn_mask,
- deterministic=False,
- method="forward_speech_masked",
- rngs={"specaugment": spec_rng, "dropout": drop_rng},
- )
- logits_f32 = logits.astype(jnp.float32)
- mask = loss_mask
- ce_loss = jnp.sum(
- optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask
- ) / jnp.maximum(jnp.sum(mask), 1.0)
- z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)
- div_loss = 1e-4 * slot_div
- return ce_loss + z_loss + div_loss
-
def _make_ffn_mask(batch_size, d_ff, mat_ff_widths):
"""Build (batch, d_ff) prefix mask: batch split equally across widths.
@@ -321,33 +296,6 @@ def _train_step_text_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask
return state, ema_params, loss, grad_norm
-def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask):
- ema_decay = 0.999
- loss, grads = jax.value_and_grad(
- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)
- )(state.params)
- grads = jax.lax.pmean(grads, axis_name="batch")
- loss = jax.lax.pmean(loss, axis_name="batch")
- grad_norm = optax.global_norm(grads)
- state = state.apply_gradients(grads=grads)
- ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, state.params)
- return state, ema_params, loss, grad_norm
-
-
-def _train_step_speech_masked(state, ema_params, mel, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng, loss_mask):
- """Speech training step with fused prune mask application."""
- ema_decay = 0.999
- loss, grads = jax.value_and_grad(
- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)
- )(state.params)
- grads = jax.lax.pmean(grads, axis_name="batch")
- loss = jax.lax.pmean(loss, axis_name="batch")
- grad_norm = optax.global_norm(grads)
- state = state.apply_gradients(grads=grads)
- masked_params = jax.tree.map(lambda w, m: w * m, state.params, prune_mask)
- state = state.replace(params=masked_params)
- ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, masked_params)
- return state, ema_params, loss, grad_norm
def _make_p_train_step():
@@ -358,14 +306,6 @@ def _make_p_train_step_masked():
return jax.pmap(_train_step_text_masked, axis_name="batch", donate_argnums=(0, 1))
-def _make_p_train_step_speech():
- return jax.pmap(_train_step_speech, axis_name="batch", donate_argnums=(0, 1))
-
-
-def _make_p_train_step_speech_masked():
- return jax.pmap(_train_step_speech_masked, axis_name="batch", donate_argnums=(0, 1))
-
-
def _make_val_loss_fn(apply_fn):
@jax.jit
def val_loss_batch(params, src, tgt_in, tgt_out, causal_mask, loss_mask):
@@ -400,24 +340,6 @@ def _make_mat_val_loss_fn(apply_fn, ff_width):
return val_loss_batch
-def _make_speech_val_loss_fn(apply_fn):
- @jax.jit
- def val_loss_batch(params, mel, tgt_in, tgt_out, causal_mask, loss_mask):
- pad_id = 0
- src_mask = make_mel_padding_mask(mel)
- tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
- logits, _, _ = apply_fn(
- {"params": params}, mel, tgt_in,
- src_mask=src_mask, tgt_mask=tgt_mask,
- deterministic=True,
- method="forward_speech_with_aux",
- )
- loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)
- mask = loss_mask
- return jnp.sum(loss * mask), jnp.sum(mask)
- return val_loss_batch
-
-
def _estimate_mat_params(config, matryoshka_factor):
"""Estimate parameter count of a sub-model at a given matryoshka factor.
@@ -450,9 +372,6 @@ def shard_batch(batch, num_devices):
def train(args):
num_devices = jax.local_device_count()
- no_speech = getattr(args, "no_speech", False)
- n_mels = getattr(args, "n_mels", 80)
- max_mel_len = getattr(args, "max_mel_len", 1024)
use_wandb = getattr(args, "wandb", False)
if use_wandb:
@@ -460,19 +379,13 @@ def train(args):
if wandb.run is None:
wandb.init(project="needle-v1", config=vars(args))
- total_data_steps = 4 if not no_speech else 3
- step_idx = 0
-
- step_idx += 1
- print(f"\n[{step_idx}/{total_data_steps}] Detecting devices...")
+ print(f"\n[1/3] Detecting devices...")
print(f" {num_devices} device(s) for data-parallel training")
- step_idx += 1
- print(f"\n[{step_idx}/{total_data_steps}] Loading tokenizer...")
+ print(f"\n[2/3] Loading tokenizer...")
tokenizer = get_tokenizer(max_samples=args.max_samples)
- step_idx += 1
- print(f"\n[{step_idx}/{total_data_steps}] Loading prepared data from disk (mmap)...")
+ print(f"\n[3/3] Loading prepared data from disk (mmap)...")
train_data = load_prepared_data("train", mmap=True)
val_data = load_prepared_data("val", mmap=True)
enc_inputs = train_data["enc_inputs"]
@@ -485,15 +398,6 @@ def train(args):
val_loss_mask = val_data["loss_mask"]
print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs (memory-mapped)")
- train_mels = None
- val_mels = None
- if not no_speech:
- step_idx += 1
- print(f"\n[{step_idx}/{total_data_steps}] Loading precomputed mel spectrograms (mmap)...")
- train_mels = load_prepared_mels(train_data["mel_cache_id"], mmap=True)
- val_mels = load_prepared_mels(val_data["mel_cache_id"], mmap=True)
- print(f" {len(train_mels):,} train / {len(val_mels):,} val mel spectrograms (memory-mapped)")
-
effective_batch_size = args.batch_size * num_devices
resume_checkpoint = getattr(args, "checkpoint", None)
@@ -516,8 +420,6 @@ def train(args):
dtype=args.dtype,
activation=getattr(args, "activation", "drelu"),
num_memory_slots=getattr(args, "num_memory_slots", 64),
- n_mels=n_mels,
- dropout_rate=getattr(args, "dropout", 0.1),
)
global _GROUP_SIZE, _MAT_FACTORS, _MAT_FF_WIDTHS, _D_FF
@@ -533,8 +435,6 @@ def train(args):
n_widths = 1 + len(_MAT_FF_WIDTHS) if _MAT_FF_WIDTHS else 1
p_train_step = _make_p_train_step()
p_train_step_masked = _make_p_train_step_masked()
- p_train_step_speech = _make_p_train_step_speech()
- p_train_step_speech_masked = _make_p_train_step_speech_masked()
np.random.seed(args.seed)
rng = jax.random.PRNGKey(args.seed)
@@ -543,11 +443,7 @@ def train(args):
mat_shared_input = getattr(args, "mat_shared_input", False)
unique_batch_size = effective_batch_size // n_widths if (mat_shared_input and n_widths > 1) else effective_batch_size
text_batches_per_epoch = count_batches(len(enc_inputs), unique_batch_size)
- if not no_speech and train_mels is not None:
- speech_batches_per_epoch = text_batches_per_epoch
- else:
- speech_batches_per_epoch = 0
- num_batches = text_batches_per_epoch + speech_batches_per_epoch
+ num_batches = text_batches_per_epoch
total_steps = num_batches * args.epochs
warmup_steps = max(1, int(total_steps * args.warmup_ratio))
@@ -555,7 +451,6 @@ def train(args):
muon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(num_devices)
state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps)
val_loss_fn = _make_val_loss_fn(state.apply_fn)
- speech_vl_fn = _make_speech_val_loss_fn(state.apply_fn) if not no_speech else None
if resume_checkpoint:
state = state.replace(params=ckpt_params)
@@ -577,13 +472,6 @@ def train(args):
print(f" Memory slots {config.num_memory_slots:>12}")
print(f" Activation {config.activation:>12}")
print(f" Dtype {config.dtype:>12}")
- print(f" Dropout {config.dropout_rate:>12}")
- if not no_speech and train_mels is not None:
- print(f" Speech {speech_batches_per_epoch} batches/epoch")
- print(f" n_mels {n_mels:>12}")
- print(f" max_mel_len {max_mel_len:>12}")
- else:
- print(f" Speech disabled")
print(f" ─────────────────────────────────────")
print(f" Devices {num_devices:>12}")
print(f" Batch {args.batch_size:>7} x {num_devices} = {effective_batch_size}")
@@ -639,113 +527,50 @@ def train(args):
epoch_step = 0
text_losses = []
- speech_losses = []
text_batch_iter = PrefetchIterator(
lambda: get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size,
loss_mask=train_loss_mask),
prefetch=4,
)
- speech_batch_iter = None
- if not no_speech and train_mels is not None:
- speech_batch_iter = PrefetchIterator(
- lambda: get_speech_batches(train_mels, dec_inputs, dec_targets, unique_batch_size,
- loss_mask=train_loss_mask),
- prefetch=4,
- )
-
- steps_this_epoch = text_batches_per_epoch + speech_batches_per_epoch
- text_idx = 0
- speech_idx = 0
- speech_loss_val = None
- pbar = tqdm(range(steps_this_epoch), desc=f"Epoch {epoch + 1}/{args.epochs}")
+ pbar = tqdm(range(text_batches_per_epoch), desc=f"Epoch {epoch + 1}/{args.epochs}")
for step_i in pbar:
t0 = time.perf_counter()
- do_speech = (step_i % 2 == 1) and speech_idx < speech_batches_per_epoch
- do_text = not do_speech and text_idx < text_batches_per_epoch
- if not do_speech and not do_text:
- if text_idx < text_batches_per_epoch:
- do_text = True
- elif speech_idx < speech_batches_per_epoch:
- do_speech = True
- else:
- break
-
- step_grad_norm = None
-
- if do_text:
- src, tgt_in, tgt_out, lm = next(text_batch_iter)
- text_idx += 1
-
- if n_widths > 1 and mat_shared_input:
- per_width = args.batch_size // n_widths
- def _tile_for_mat(arr):
- s = arr.reshape(num_devices, per_width, *arr.shape[1:])
- return np.tile(s, (1, n_widths) + (1,) * (arr.ndim - 1))
- src_b = _tile_for_mat(src)
- tgt_in_b = _tile_for_mat(tgt_in)
- tgt_out_b = _tile_for_mat(tgt_out)
- lm_b = _tile_for_mat(lm)
- else:
- src_b = shard_batch(src, num_devices)
- tgt_in_b = shard_batch(tgt_in, num_devices)
- tgt_out_b = shard_batch(tgt_out, num_devices)
- lm_b = shard_batch(lm, num_devices)
-
- rng, text_rng = jax.random.split(rng)
- text_rngs = jax.random.split(text_rng, num_devices)
-
- if prune_mask is not None:
- state, ema_params, loss, grad_norm = p_train_step_masked(
- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, text_rngs, lm_b,
- )
- else:
- state, ema_params, loss, grad_norm = p_train_step(
- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, text_rngs, lm_b,
- )
-
- text_loss_val = float(loss[0])
- text_losses.append(text_loss_val)
- step_grad_norm = float(grad_norm[0])
- global_step += 1
-
+ src, tgt_in, tgt_out, lm = next(text_batch_iter)
+
+ if n_widths > 1 and mat_shared_input:
+ per_width = args.batch_size // n_widths
+ def _tile_for_mat(arr):
+ s = arr.reshape(num_devices, per_width, *arr.shape[1:])
+ return np.tile(s, (1, n_widths) + (1,) * (arr.ndim - 1))
+ src_b = _tile_for_mat(src)
+ tgt_in_b = _tile_for_mat(tgt_in)
+ tgt_out_b = _tile_for_mat(tgt_out)
+ lm_b = _tile_for_mat(lm)
else:
- mel_batch, sp_tgt_in, sp_tgt_out, sp_lm = next(speech_batch_iter)
- speech_idx += 1
-
- if n_widths > 1 and mat_shared_input:
- per_width = args.batch_size // n_widths
- def _tile_sp(arr):
- s = arr.reshape(num_devices, per_width, *arr.shape[1:])
- return np.tile(s, (1, n_widths) + (1,) * (arr.ndim - 1))
- mel_b = _tile_sp(mel_batch)
- sp_tgt_in_b = _tile_sp(sp_tgt_in)
- sp_tgt_out_b = _tile_sp(sp_tgt_out)
- sp_lm_b = _tile_sp(sp_lm)
- else:
- mel_b = shard_batch(mel_batch, num_devices)
- sp_tgt_in_b = shard_batch(sp_tgt_in, num_devices)
- sp_tgt_out_b = shard_batch(sp_tgt_out, num_devices)
- sp_lm_b = shard_batch(sp_lm, num_devices)
-
- rng, spec_rng = jax.random.split(rng)
- spec_rngs = jax.random.split(spec_rng, num_devices)
-
- if prune_mask is not None:
- state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech_masked(
- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, prune_mask, text_ffn_mask, spec_rngs, sp_lm_b,
- )
- else:
- state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech(
- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, text_ffn_mask, spec_rngs, sp_lm_b,
- )
- speech_loss_val = float(sp_loss[0])
- speech_losses.append(speech_loss_val)
- step_grad_norm = float(sp_grad_norm[0])
- text_loss_val = text_losses[-1] if text_losses else float("nan")
- global_step += 1
+ src_b = shard_batch(src, num_devices)
+ tgt_in_b = shard_batch(tgt_in, num_devices)
+ tgt_out_b = shard_batch(tgt_out, num_devices)
+ lm_b = shard_batch(lm, num_devices)
+
+ rng, text_rng = jax.random.split(rng)
+ text_rngs = jax.random.split(text_rng, num_devices)
+
+ if prune_mask is not None:
+ state, ema_params, loss, grad_norm = p_train_step_masked(
+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, text_rngs, lm_b,
+ )
+ else:
+ state, ema_params, loss, grad_norm = p_train_step(
+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, text_rngs, lm_b,
+ )
+
+ text_loss_val = float(loss[0])
+ text_losses.append(text_loss_val)
+ step_grad_norm = float(grad_norm[0])
+ global_step += 1
if epoch == weight_prune_epoch and not gradual_sparsify_done:
epoch_step += 1
@@ -772,7 +597,6 @@ def train(args):
del _eval_params
postfix = {
- "speech_loss": f"{speech_loss_val:.4f}" if speech_loss_val is not None else "-",
"text_loss": f"{text_loss_val:.4f}",
"text_ppl": f"{last_val_ppl:.2f}" if last_val_ppl is not None else "?",
}
@@ -792,8 +616,6 @@ def train(args):
"train/tokens_per_sec": tokens_per_batch / dt,
"train/step": global_step,
}
- if speech_loss_val is not None:
- log_dict["train/speech_loss"] = speech_loss_val
if epoch == weight_prune_epoch and not gradual_sparsify_done:
log_dict["train/scheduled_sparsity"] = current_sparsity
if global_step % eval_every == 0 or global_step == total_steps:
@@ -801,8 +623,6 @@ def train(args):
wandb.log(log_dict)
text_batch_iter.close()
- if speech_batch_iter is not None:
- speech_batch_iter.close()
if epoch == weight_prune_epoch and not gradual_sparsify_done:
gradual_sparsify_done = True
@@ -863,17 +683,6 @@ def train(args):
mat_results[f] = (float(math.exp(min(avg, 20))),
_estimate_mat_params(config, f), config.d_ff // f)
- speech_val_ppl = None
- if speech_vl_fn is not None and val_mels is not None:
- sp_total_loss, sp_total_toks = 0.0, 0.0
- for sp_batch in get_speech_batches(val_mels, val_dec_in, val_dec_tgt, args.batch_size,
- shuffle=False, loss_mask=val_loss_mask):
- vl, vt = speech_vl_fn(eval_params, sp_batch[0], sp_batch[1], sp_batch[2], val_causal, sp_batch[3])
- sp_total_loss += float(vl)
- sp_total_toks += float(vt)
- speech_val_loss = sp_total_loss / max(sp_total_toks, 1)
- speech_val_ppl = float(math.exp(min(speech_val_loss, 20)))
-
params_np = jax.tree.map(np.array, eval_params)
total_params = sum(x.size for x in jax.tree.leaves(params_np))
near_zero = sum(int(np.sum(np.abs(x) < 1e-6)) for x in jax.tree.leaves(params_np))
@@ -885,12 +694,12 @@ def train(args):
pickle.dump({"params": params_np, "config": config.__dict__}, f)
del params_np
- from .test import measure_throughput
- from .run import generate, generate_from_audio
+ from .eval import measure_throughput
+ from .run import generate_batch
tp = measure_throughput(eval_model, eval_params, tokenizer, num_runs=5)
- _val_start = len(enc_inputs)
val_kept = val_data["kept_indices"]
+ val_ds = load_tool_calls("val", max_samples=args.max_samples)
# Pick 5 display samples (shuffled for diversity): 4 with tool calls, 1 without
sample_rng = np.random.RandomState(epoch + 7)
@@ -899,40 +708,17 @@ def train(args):
for k in sample_pool:
if len(display_with) >= 4 and len(display_without) >= 1:
break
- ds_idx = int(val_kept[k]) + _val_start
- pair = load_example_with_audio(ds_idx)
- is_empty = pair["answers"].strip() in ("", "[]")
+ local_idx = int(val_kept[k])
+ ex = val_ds[local_idx]
+ is_empty = ex["answers"].strip() in ("", "[]")
if not is_empty and len(display_with) < 4:
- display_with.append((ds_idx, pair))
+ display_with.append(ex)
elif is_empty and len(display_without) < 1:
- display_without.append((ds_idx, pair))
+ display_without.append(ex)
display_pairs = display_with + display_without
- unified_samples = []
- for i, (ds_idx, pair) in enumerate(display_pairs):
- text_pred = generate(
- eval_model, eval_params, tokenizer, pair["query"],
- tools=pair["tools"], max_gen_len=args.max_dec_len, seed=i, stream=False,
- ).strip()
-
- voice_pred = None
- if not no_speech and pair["audio_array"] is not None:
- voice_pred = generate_from_audio(
- eval_model, eval_params, tokenizer, pair["audio_array"], sr=pair["sampling_rate"],
- tools=pair["tools"], max_gen_len=args.max_dec_len, seed=i, stream=False,
- ).strip()
-
- unified_samples.append({
- "query": pair["query"],
- "tools": pair["tools"],
- "ref": pair["answers"],
- "text": text_pred,
- "voice": voice_pred,
- })
-
+ # Collect all eval examples, then batch-generate everything at once
import json as _json_mod
- tc_n, tc_exact, tc_name_tp, tc_name_fp, tc_name_fn = 0, 0, 0, 0, 0
- tc_call_tp, tc_call_fp, tc_call_fn, tc_parse_err = 0, 0, 0, 0
tc_with_n = 25
tc_without_n = 5
tc_rng = np.random.RandomState(epoch + 42)
@@ -941,26 +727,51 @@ def train(args):
for k in tc_pool:
if len(tc_with) >= tc_with_n and len(tc_without) >= tc_without_n:
break
- ds_idx = int(val_kept[k]) + _val_start
- pair = load_example_with_audio(ds_idx)
- ref_text = pair["answers"].strip()
+ local_idx = int(val_kept[k])
+ ex = val_ds[local_idx]
+ ref_text = ex["answers"].strip()
is_empty = ref_text in ("", "[]")
if not is_empty and len(tc_with) < tc_with_n:
- tc_with.append((ds_idx, pair))
+ tc_with.append(ex)
elif is_empty and len(tc_without) < tc_without_n:
- tc_without.append((ds_idx, pair))
+ tc_without.append(ex)
tc_eval_pairs = tc_with + tc_without
+ # Batch-generate: display samples + tc eval samples in one pass
+ all_eval_examples = display_pairs + tc_eval_pairs
+ eval_gen_len = min(args.max_dec_len, 512)
+ all_preds = generate_batch(
+ eval_model, eval_params, tokenizer,
+ [ex["query"] for ex in all_eval_examples],
+ [ex["tools"] for ex in all_eval_examples],
+ max_gen_len=eval_gen_len,
+ max_enc_len=args.max_enc_len,
+ )
+
+ # Split results back
+ display_preds = all_preds[:len(display_pairs)]
+ tc_preds = all_preds[len(display_pairs):]
+
+ unified_samples = []
+ for ex, pred in zip(display_pairs, display_preds):
+ unified_samples.append({
+ "query": ex["query"],
+ "tools": ex["tools"],
+ "ref": ex["answers"],
+ "text": pred.strip(),
+ })
+
+ tc_n, tc_exact, tc_name_tp, tc_name_fp, tc_name_fn = 0, 0, 0, 0, 0
+ tc_call_tp, tc_call_fp, tc_call_fn, tc_parse_err = 0, 0, 0, 0
+ tc_args_correct, tc_args_total = 0, 0
+
def _call_key(c):
if not isinstance(c, dict): return None
return _json_mod.dumps({"name": c.get("name"), "arguments": c.get("arguments")}, sort_keys=True)
- for i, (ds_idx, pair) in enumerate(tc_eval_pairs):
- ref_text = pair["answers"].strip()
- pred_text = generate(
- eval_model, eval_params, tokenizer, pair["query"],
- tools=pair["tools"], max_gen_len=args.max_dec_len, seed=ds_idx, stream=False,
- ).strip()
+ for ex, pred_text in zip(tc_eval_pairs, tc_preds):
+ ref_text = ex["answers"].strip()
+ pred_text = pred_text.strip()
try:
ref_calls = _json_mod.loads(ref_text)
except (ValueError, TypeError):
@@ -986,6 +797,18 @@ def train(args):
tc_call_fp += len(pk - rk)
tc_call_fn += len(rk - pk)
+ # Argument correctness: for name-matched calls, check if arguments match
+ ref_by_name = {}
+ for c in ref_calls:
+ if isinstance(c, dict) and "name" in c:
+ ref_by_name.setdefault(c["name"], []).append(c.get("arguments", {}))
+ for c in pred_calls:
+ if isinstance(c, dict) and "name" in c and c["name"] in ref_by_name:
+ tc_args_total += 1
+ pred_args = _json_mod.dumps(c.get("arguments", {}), sort_keys=True)
+ if any(pred_args == _json_mod.dumps(ra, sort_keys=True) for ra in ref_by_name[c["name"]]):
+ tc_args_correct += 1
+
tc_metrics = {}
if tc_n > 0:
tc_metrics["parse_rate"] = 1.0 - tc_parse_err / tc_n
@@ -996,19 +819,15 @@ def train(args):
cp_ = tc_call_tp + tc_call_fp
cr_ = tc_call_tp + tc_call_fn
tc_metrics["call_f1"] = 2 * tc_call_tp / max(cp_ + cr_, 1)
+ tc_metrics["args_acc"] = tc_args_correct / max(tc_args_total, 1)
del eval_params
- final_speech_loss = speech_losses[-1] if speech_losses else None
print(f"\n ─────────────────────────────────────")
print(f" Epoch {epoch + 1}/{args.epochs}")
print(f" ─────────────────────────────────────")
print(f" Text loss {final_loss:>12.4f}")
print(f" Text val ppl {last_val_ppl:>12.2f}")
- if final_speech_loss is not None:
- print(f" Speech loss {final_speech_loss:>12.4f}")
- if speech_val_ppl is not None:
- print(f" Speech val ppl {speech_val_ppl:>12.2f}")
print(f" Quant val ppl {quant_val_ppl:>12.2f} (INT4 g{_GROUP_SIZE})")
print(f" Sparsity {sparsity:>11.2f}% ({near_zero:,}/{total_params:,})")
if mat_results:
@@ -1022,9 +841,10 @@ def train(args):
if tc_metrics:
print(f" ─── Tool-Call Accuracy ({tc_n} samples) ──")
print(f" JSON parse {tc_metrics['parse_rate']:>10.1%}")
+ print(f" Name F1 {tc_metrics['name_f1']:>10.1%}")
+ print(f" Args acc {tc_metrics['args_acc']:>10.1%}")
+ print(f" Call F1 {tc_metrics['call_f1']:>10.1%}")
print(f" Exact match {tc_metrics['exact_match']:>10.1%}")
- print(f" Name F1 {tc_metrics['name_f1']:>10.3f}")
- print(f" Call F1 {tc_metrics['call_f1']:>10.3f}")
print(f" ─────────────────────────────────────")
print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")
print(f" Latency {tp['avg_latency_s']:>11.3f}s")
@@ -1038,8 +858,6 @@ def train(args):
print(f" Tools: {tools_short}")
print(f" Ref: {s['ref'][:200] or '[]'}")
print(f" Text: {s['text'][:200] or '(empty)'}")
- if s["voice"] is not None:
- print(f" Voice: {s['voice'][:200] or '(empty)'}")
if j < len(unified_samples) - 1:
print()
print(f" ─────────────────────────────────────")
@@ -1054,10 +872,6 @@ def train(args):
"epoch/weight_sparsity": sparsity,
"epoch": epoch + 1,
}
- if final_speech_loss is not None:
- log_dict["epoch/speech_loss"] = final_speech_loss
- if speech_val_ppl is not None:
- log_dict["epoch/speech_val_ppl"] = speech_val_ppl
for factor, (mat_ppl, mat_params, _) in mat_results.items():
log_dict[f"epoch/mat_ppl_{factor}x"] = mat_ppl
log_dict[f"epoch/mat_params_{factor}x"] = mat_params
@@ -1065,6 +879,7 @@ def train(args):
log_dict["epoch/tc_parse_rate"] = tc_metrics["parse_rate"]
log_dict["epoch/tc_exact_match"] = tc_metrics["exact_match"]
log_dict["epoch/tc_name_f1"] = tc_metrics["name_f1"]
+ log_dict["epoch/tc_args_acc"] = tc_metrics["args_acc"]
log_dict["epoch/tc_call_f1"] = tc_metrics["call_f1"]
wandb.log(log_dict)