Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
tool call fixes
HenryNdubuakud03d21f3dfd9
diff --git a/README.md b/README.mdindex 875a9c3..1b4aa48 100644--- a/README.md+++ b/README.md@@ -15,8 +15,8 @@└──────┬───────┘ │ Mat loss at factors │┌┴──────────┐ │ + INT4 QAT (g=32) ││ Softmax │ │ + z-loss + slot div │- └─────┬─────┘ │ 1:1 text/speech │- ┌─────┬─────┴─────┬─────┐ │ alternation │+ └─────┬─────┘ │ text-only training │+ ┌─────┬─────┴─────┬─────┐ │ ││ @E[:d_ff/f] for each │ └──────────┬───────────┘│ f in mat_factors │ │└─────┬─────┬─────┬─────┘ ┌──────────┴───────────┐@@ -39,13 +39,13 @@│ │ │└───────────────┘│ ┌──────────┴───────────┐│ ┌──────────┐ │ └────────┬────────┘ │ Checkpoint ││ │Pack: │ │ ┌─────┴─────┐ │ full params, can │- │ │ S←X Attn │ │ │ Embedding │ ← shared │ export mat slices │+ │ │ S←X Attn │ │ │ Embedding │ ← shared │ export mat slices ││ │ RoPE keys│ │ └─────┬─────┘ └──────────────────────┘│ ├──────────┤ │ ┌───────┴───────┐│ │Mix: │ │ │[EOS]<tool_call>│- │ │ MLP-Mixer│ │ │ + tools │- │ │ on S │ │ │ + answer │- │ ├──────────┤ │ └───────────────┘+ │ │ MLP-Mixer│ │ │ + answer │+ │ │ on S │ │ └───────────────┘+ │ ├──────────┤ ││ │Local: │ ││ │Gated FFN │ ││ │ on X │ │@@ -69,7 +69,7 @@d=512 (default) / 1536 (--full) · GQA · QK-normSentencePiece BPE (8192) · gated dReLU · ZCRMSNorm · RoPEstrided DW conv · mat factors · INT4 QAT · Muon + AdamW- text + speech encoder · <tool_call> task routing+ text + speech encoder · <tool_call> task routing (speech used in inference only)Data Pipeline (needle tokenize → needle train)───────────────────────────────────────────────@@ -77,34 +77,26 @@┌─────────────────────────────────────────────────────────────┐│ needle tokenize ││ │- │ Unified dataset (GCS / local) │+ │ Local dataset (data/tool_calls_unified/) ││ │ │- │ ├──── Text ──────────────── Speech ─────────┐ │- │ ▼ ▼ │- │ ┌──────────────┐ ┌────────────────┐ │- │ │ SentencePiece│ │ HF Audio col │ │- │ │ BPE tokenize │ │ decode (16kHz) │ │- │ │ (8192 vocab) │ └───────┬────────┘ │- │ └──────┬───────┘ │ │- │ │ ▼ │- │ ▼ ┌────────────────┐ │- │ ┌──────────────────┐ │ Log-mel spec │ │- │ │ enc_inputs.npy │ │ 80 bins, 25ms │ │- │ │ dec_inputs.npy │ │ window, 10ms │ │- │ │ dec_targets.npy │ │ hop (~100 f/s) │ │- │ │ loss_mask.npy │ └───────┬────────┘ │- │ │ kept_idx.npy │ │ │- │ └──────┬───────────┘ ▼ │- │ │ ┌────────────────┐ │- │ │ │ mels.npy │ │- │ │ │ (N,T,80) f32 │ │- │ │ │ writable mmap │ │- │ │ └───────┬────────┘ │- │ │ │ │- │ └────────────┬───────────────────────────┘ │- │ ▼ │- │ {split}_metadata.json │- │ .data_cache/ + GCS upload │+ │ ▼ │+ │ ┌──────────────┐ │+ │ │ SentencePiece│ │+ │ │ BPE tokenize │ │+ │ │ (8192 vocab) │ │+ │ └──────┬───────┘ │+ │ │ │+ │ ▼ │+ │ ┌──────────────────┐ │+ │ │ enc_inputs.npy │ │+ │ │ dec_inputs.npy │ │+ │ │ dec_targets.npy │ │+ │ │ loss_mask.npy │ │+ │ │ kept_idx.npy │ │+ │ └──────┬───────────┘ │+ │ ▼ │+ │ {split}_metadata.json │+ │ .data_cache/ │└─────────────────────────────────────────────────────────────┘│▼@@ -112,21 +104,15 @@│ needle train ││ ││ load_prepared_data(mmap=True) │- │ load_prepared_mels(mmap=True) ││ │ ││ ▼ │- │ ┌──────────────────────┐ ┌──────────────────────┐ │- │ │ PrefetchIterator │ │ PrefetchIterator │ │- │ │ text batches (4) │ │ speech batches (4) │ │- │ │ mmap → per-batch idx │ │ mmap → per-batch idx │ │- │ └──────────┬───────────┘ └──────────┬───────────┘ │- │ └───────────┬──────────────┘ │- │ ▼ │- │ 1:1 text/speech │- │ alternation │- │ │- │ ~42MB RAM (8 prefetched batches) │- │ SpecAugment for speech regularization │+ │ ┌──────────────────────┐ │+ │ │ PrefetchIterator │ │+ │ │ text batches (4) │ │+ │ │ mmap → per-batch idx │ │+ │ └──────────┬───────────┘ │+ │ ▼ │+ │ text-only tool-call training │└─────────────────────────────────────────────────────────────┘```@@ -142,20 +128,19 @@ needle [command]┌───────────────────────────────────────────────────────────────────┐│ ││ train │- │ --full Use full 1B config (~1.17B params) │- │ --epochs INT Training epochs (default: 1) │+ │ --epochs INT Training epochs (default: 10) ││ --batch-size INT Batch size (default: 32) ││ --lr FLOAT AdamW learning rate (default: 3e-4) ││ --muon-lr FLOAT Muon learning rate (default: 0.02) ││ --d-model INT Model dim (default: 512) │- │ --num-heads INT Attention heads (default: 8) │- │ --num-kv-heads INT KV heads for GQA (default: num-heads)│+ │ --num-heads INT Attention heads (default: 16) │+ │ --num-kv-heads INT KV heads for GQA (default: 8) ││ --num-layers INT Encoder layers (default: 4) ││ --num-dec-layers INT Decoder layers (default: 4) │- │ --max-enc-len INT Max encoder seq len (default: 256) │- │ --max-dec-len INT Max decoder seq len (default: 1024) │+ │ --max-enc-len INT Max encoder seq len (default: 512) │+ │ --max-dec-len INT Max decoder seq len (default: 512) ││ --max-samples INT Training samples (default: all) │- │ --mat-factors INT [...] FFN shrink factors (default: 2 4 8) │+ │ --mat-factors INT [...] FFN shrink factors (default: 2 4) ││ --sparsity-ratio FLOAT Block prune ratio (default: 0.0) ││ --group-size INT Quant/prune group size (default: 32) ││ --prune-interval INT Steps between mask updates (def: 100)│@@ -168,18 +153,11 @@ needle [command]│ --checkpoint PATH Resume from checkpoint ││ --checkpoint-dir DIR Checkpoint directory ││ --seed INT Random seed (default: 42) │- │ --no-speech Disable speech (text-only training) │- │ --max-mel-len INT Max mel frames (default: 1024) │- │ --n-mels INT Mel frequency bins (default: 80) │- │ --max-speech-samples INT Max voice-tool-call samples ││ ││ tokenize ││ --max-samples INT Limit samples per split (dev/test) │- │ --cleanup Delete local cache after GCS upload │- │ --n-mels INT Mel frequency bins (default: 80) │- │ --max-mel-len INT Max mel frames (default: 1024) │- │ --max-enc-len INT Max encoder seq len (default: 256) │- │ --max-dec-len INT Max decoder seq len (default: 1024) │+ │ --max-enc-len INT Max encoder seq len (default: 512) │+ │ --max-dec-len INT Max decoder seq len (default: 512) ││ ││ run ││ --checkpoint PATH Path to model checkpoint (required) │@@ -189,16 +167,15 @@ needle [command]│ --max-len INT Max tokens to generate (default: 512) ││ --seed INT Random seed (default: 0) ││ │- │ test │+ │ eval ││ --checkpoint PATH Path to model checkpoint (required) ││ --batch-size INT Batch size (default: 32) ││ --max-eval-samples INT Evaluation samples (default: 1000) │- │ --max-enc-len INT Max encoder length (default: 256) │- │ --max-dec-len INT Max decoder length (default: 1024) │+ │ --max-enc-len INT Max encoder length (default: 512) │+ │ --max-dec-len INT Max decoder length (default: 512) ││ --max-gen-len INT Max generation length (default: 512) ││ --throughput-runs INT Throughput runs (default: 10) ││ --tool-call-samples INT Tool-call eval samples (default: 200) │- │ --voice-tc-samples INT Voice-TC eval samples (default: 50) ││ ││ evaluate ││ --checkpoint PATH Path to model checkpoint (required) │@@ -236,20 +213,10 @@ needle [command]──────────────────────────────────────────Text Tool-call pairs(query, tools, answers)- from GCS unified dataset- Audio TTS-generated speech- (GCP Cloud TTS voices)- 16kHz WAV, HF Audio col+ ~57k examples (local)+ xlam-60k + glaive-v2──────────────────────────────────────────- 100M params — 10M multimodal triples (5 epochs)- ──────────────────────────────────────────- Triples 10M (text / voice / image → JSON tool calls)- Tokens/triple ~300 avg (text ~180, voice ~390, image ~336)- Total tokens 10M × 300 × 5 epochs ≈ 15B- Total FLOPs 6 × 100M × 15B ≈ 9e18- MFU (100M) ~15% → ~138 effective TFLOPS/chip-┌────────────────────┬──────────┬──────────┬──────────┐│ │ v6e-8 │ v6e-16 │ v6e-32 │├────────────────────┼──────────┼──────────┼──────────┤@@ -271,33 +238,6 @@ needle [command]- Setup gcloud 4: else, set the project with `gcloud config set project needle-488623`- setup gcloud 5: run `gcloud help` and read carefully-## TPU Guide--```-needle tpu [command]-- ┌───────────────────────────────────────────────────────────────────┐- │ │- │ create NAME Create TPU (auto-finds zone) │- │ --type STR Accelerator type (default: v6e-8) │- │ --version STR TPU OS (auto-detected from --type) │- │ │- │ connect NAME SSH config + first connect (auto-zone) │- │ claude NAME Install Claude Code on instance │- │ │- │ stop NAME Stop instance (keeps disk) │- │ start NAME Restart a stopped instance │- │ delete NAME Delete instance (prompts confirmation) │- │ list List all TPU instances │- │ │- │ --zone ZONE Override auto-detected zone (optional) │- │ │- └───────────────────────────────────────────────────────────────────┘-- Quota increases:- https://console.cloud.google.com/iam-admin/quotas?project=needle-488623-```-## Example Workflow```@@ -322,10 +262,14 @@ needle tpu [command]5. Install needle (follow instruction to setup wandb)source ./setup-6. Use needle as you normally would locally, like training+6. Login to Hugging Face (required for gated datasets like xlam-60k)+ huggingface-cli login+ (paste your HF token — get one at https://huggingface.co/settings/tokens)++7. Use needle as you normally would locally, like trainingneedle train --wandb-7. Use tmux for long training runs (survives SSH disconnects)+8. Use tmux for long training runs (survives SSH disconnects)tmux new -s train # start a named sessionneedle train --wandb # run training inside itCtrl+B, then D # detach (keeps running)@@ -333,9 +277,9 @@ needle tpu [command]--- back on your Mac ----8. Stop when done (saves disk, stops billing)+9. Stop when done (saves disk, stops billing)needle tpu stop my-experiment-9. (Optional) Delete instance when no longer needed+10. (Optional) Delete instance when no longer neededneedle tpu delete my-experiment```diff --git a/scripts/build_dataset.py b/scripts/build_dataset.pyindex 89b8033..1e8ded3 100644--- a/scripts/build_dataset.py+++ b/scripts/build_dataset.py@@ -1,31 +1,19 @@#!/usr/bin/env python3-"""Unified dataset builder: text + TTS audio -> HuggingFace Dataset on GCS.--Combines the text pipeline (tools_data.py) with TTS audio generation into-one script that produces a single HF Dataset with an Audio column.--When --gcs-upload is provided, processes in batches: each batch is synthesized,-saved as a shard, uploaded to GCS, and deleted locally — keeping disk usage bounded.+"""Build text-only tool-call dataset and save locally.Usage:- python scripts/build_dataset.py --max-samples 1000 --workers 16- python scripts/build_dataset.py --gcs-upload gs://cactus-dataset/tool_calls --batch-size 500+ python scripts/build_dataset.py+ python scripts/build_dataset.py --max-samples 1000+ python scripts/build_dataset.py --output data/tool_calls_unified"""import argparse-import io+import jsonimport loggingimport os-import random-import shutil-import subprocessimport sys-import time-import wave-from concurrent.futures import ThreadPoolExecutor-from datasets import Audio, Dataset, load_from_disk-from tqdm import tqdm+import numpy as npsys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))from tools_data import load_and_combine@@ -33,524 +21,107 @@ from tools_data import load_and_combinelogging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")logger = logging.getLogger(__name__)-_VOICE_TEMPLATES = [- ("en-GB", "en-GB-Wavenet-B", "MALE", True, False),- ("en-GB", "en-GB-Wavenet-A", "FEMALE", True, False),- ("en-AU", "en-AU-Wavenet-B", "MALE", True, False),- ("en-AU", "en-AU-Wavenet-C", "FEMALE", True, False),- ("en-US", "en-US-Wavenet-D", "MALE", True, False),- ("en-IN", "en-IN-Wavenet-A", "FEMALE", True, False),- ("en-IN", "en-IN-Wavenet-B", "MALE", True, False),- ("en-IN", "en-IN-Wavenet-C", "MALE", True, False),- ("en-GB", "en-GB-Wavenet-D", "MALE", True, False),- ("en-AU", "en-AU-Wavenet-D", "MALE", True, False),- ("en-US", "en-US-Wavenet-E", "FEMALE", True, True),- ("en-GB", "en-GB-Wavenet-A", "FEMALE", True, True),- ("en-IN", "en-IN-Wavenet-A", "FEMALE", True, True),-]--_VOICE_TYPES = ["Wavenet", "Neural2"]--VOICES = _VOICE_TEMPLATES---def _get_voice(idx):- """Return (lang, voice_name, gender_str, configurable, child) for a sample.-- Cycles through voice types (Wavenet->Neural2) via idx % 2,- then picks a voice template via (idx // 3) % len(_VOICE_TEMPLATES).- """- vtype = _VOICE_TYPES[idx % len(_VOICE_TYPES)]- template = _VOICE_TEMPLATES[(idx // 3) % len(_VOICE_TEMPLATES)]- lang, base_name, gender, configurable, child = template- voice_name = base_name.replace("Wavenet", vtype)- return lang, voice_name, gender, configurable, child---def _pcm_to_wav_bytes(pcm_bytes: bytes, sample_rate: int) -> bytes:- """Convert raw LINEAR16 PCM bytes to a WAV byte buffer."""- buf = io.BytesIO()- with wave.open(buf, "wb") as wf:- wf.setnchannels(1)- wf.setsampwidth(2)- wf.setframerate(sample_rate)- wf.writeframes(pcm_bytes)- return buf.getvalue()---def _synthesize_one(client, idx, query, max_retries=5):- """Synthesize TTS audio for a single query via GCP Cloud TTS.-- Returns:- Tuple of (idx, wav_bytes, voice_name, error_msg).- wav_bytes is None on failure; error_msg is None on success.- """- from google.cloud import texttospeech- from google.api_core.exceptions import ResourceExhausted, ServiceUnavailable-- lang, voice_name, ssml_gender, configurable, child = _get_voice(idx)- gender = getattr(texttospeech.SsmlVoiceGender, ssml_gender)-- synthesis_input = texttospeech.SynthesisInput(text=query)- voice_params = texttospeech.VoiceSelectionParams(- language_code=lang,- name=voice_name,- ssml_gender=gender,- )-- if configurable:- rng = random.Random(idx)- if child:- pitch = rng.uniform(4.0, 8.0)- rate = rng.uniform(1.0, 1.15)- else:- pitch = rng.uniform(-3.0, 3.0)- rate = rng.uniform(0.9, 1.1)- audio_config = texttospeech.AudioConfig(- audio_encoding=texttospeech.AudioEncoding.LINEAR16,- sample_rate_hertz=16000,- pitch=pitch,- speaking_rate=rate,- volume_gain_db=rng.uniform(-1.5, 1.5),- )- else:- audio_config = texttospeech.AudioConfig(- audio_encoding=texttospeech.AudioEncoding.LINEAR16,- sample_rate_hertz=16000,- )-- for attempt in range(max_retries):- try:- response = client.synthesize_speech(- input=synthesis_input,- voice=voice_params,- audio_config=audio_config,- )- wav_bytes = _pcm_to_wav_bytes(response.audio_content, 16000)- return idx, wav_bytes, voice_name, None- except (ResourceExhausted, ServiceUnavailable):- wait = min(2 ** attempt, 30)- time.sleep(wait)- except Exception as e:- return idx, None, voice_name, str(e)-- return idx, None, voice_name, "max retries exceeded"--def _synthesize_rows(batch_ds, audio_data, voice_names, workers, rate_limit,- cache_dir, global_offset=0):- """Run TTS synthesis for rows where audio_data[i] is None.-- Uses global_offset + local_index for voice selection so that voice- assignment is deterministic regardless of batch boundaries.- WAV files are written to cache_dir/{local_idx}.wav.- """- needs = [i for i in range(len(batch_ds)) if audio_data[i] is None]- if not needs:- return-- from google.cloud import texttospeech-- os.makedirs(cache_dir, exist_ok=True)-- items = [(i, global_offset + i, batch_ds[i]["query"]) for i in needs]- logger.info(f"Synthesizing TTS for {len(items)} rows with {workers} workers...")-- def _worker(item):- local_idx, global_idx, query = item- client = texttospeech.TextToSpeechClient()- _, wav_bytes, voice_name, err = _synthesize_one(client, global_idx, query)- return local_idx, wav_bytes, voice_name, err-- success, failed = 0, 0- delay = 1.0 / rate_limit if rate_limit > 0 else 0- total = len(items)-- with ThreadPoolExecutor(max_workers=workers) as pool:- pending = {}- submitted = 0- qi = iter(items)-- for _ in range(min(workers * 2, total)):- q = next(qi)- pending[pool.submit(_worker, q)] = q- submitted += 1-- with tqdm(total=total, desc="TTS") as pbar:- while pending:- done_set = {f for f in list(pending) if f.done()}- if not done_set:- time.sleep(0.05)- continue- for fut in done_set:- local_idx, wav_bytes, voice_name, err = fut.result()- if wav_bytes is not None:- wav_path = os.path.join(cache_dir, f"{local_idx}.wav")- with open(wav_path, "wb") as f:- f.write(wav_bytes)- audio_data[local_idx] = wav_path- voice_names[local_idx] = voice_name- success += 1- else:- failed += 1- logger.warning(f"Failed idx={global_offset + local_idx}: {err}")- pbar.update(1)- del pending[fut]-- if submitted < total:- q = next(qi)- pending[pool.submit(_worker, q)] = q- submitted += 1- time.sleep(delay)-- logger.info(f"TTS done: {success} generated, {failed} failed")---def _backfill_voice_names(voice_names, global_offset=0):- """Assign default voice names to any rows that don't have one."""- for i in range(len(voice_names)):- if not voice_names[i]:- _, voice_name, _, _, _ = VOICES[(global_offset + i) % len(VOICES)]- voice_names[i] = voice_name--def _build_arrow(batch_ds, audio_data, voice_names, arrow_path):- """Build an HF Dataset from a batch and write a single .arrow file.-- Uses save_to_disk into a temp dir so that Audio columns are properly- serialized (WAV bytes embedded), then moves the generated .arrow file.- """- import tempfile-- n = len(batch_ds)- audio_column = []- for i in range(n):- path = audio_data[i]- if path is not None and os.path.exists(path):- audio_column.append(path)- else:- audio_column.append(None)-- ds_dict = {- "query": [batch_ds[i]["query"] for i in range(n)],- "audio": audio_column,- "tools": [batch_ds[i]["tools"] for i in range(n)],- "answers": [batch_ds[i]["answers"] for i in range(n)],- "source": [batch_ds[i]["source"] for i in range(n)],- "voice": voice_names[:n],- }-- ds = Dataset.from_dict(ds_dict)- ds = ds.cast_column("audio", Audio(sampling_rate=16000))-- with tempfile.TemporaryDirectory() as tmpdir:- ds.save_to_disk(tmpdir)- arrow_files = sorted(f for f in os.listdir(tmpdir) if f.endswith(".arrow"))- if len(arrow_files) == 1:- shutil.move(os.path.join(tmpdir, arrow_files[0]), arrow_path)- else:- import pyarrow as pa- tables = []- for af in arrow_files:- with open(os.path.join(tmpdir, af), "rb") as fh:- reader = pa.ipc.open_stream(fh)- tables.append(reader.read_all())- merged = pa.concat_tables(tables)- with pa.OSFile(arrow_path, "wb") as sink:- writer = pa.ipc.new_stream(sink, merged.schema)- writer.write_table(merged)- writer.close()-- logger.info(f"Built {arrow_path} ({n} rows)")- info = ds.info- del ds, ds_dict, audio_column- return info---def _arrow_exists_on_gcs(gcs_base, arrow_name):- """Check if an arrow file already exists on GCS."""- gcs_path = gcs_base + "/" + arrow_name- result = subprocess.run(- ["gcloud", "storage", "ls", gcs_path],- capture_output=True, text=True,- )- return result.returncode == 0 and result.stdout.strip()---def _upload_file_to_gcs(local_path, gcs_dest):- """Upload a single file to GCS."""- logger.info(f"Uploading {os.path.basename(local_path)} -> {gcs_dest}")- result = subprocess.run(- ["gcloud", "storage", "cp", local_path, gcs_dest],- text=True,- )- if result.returncode != 0:- logger.error(f"Upload failed for {local_path}")- sys.exit(1)---def _upload_metadata_to_gcs(gcs_base, num_batches, dataset_info):- """Generate and upload state.json + dataset_info.json so load_from_disk works."""- import json- import tempfile-- arrow_files = [- {"filename": f"data-{i:05d}-of-{num_batches:05d}.arrow"}- for i in range(num_batches)- ]-- state = {- "_data_files": arrow_files,- "_fingerprint": "batch_upload",- "_format_columns": None,- "_format_kwargs": {},- "_format_type": None,- "_output_all_columns": False,- "_split": None,- }-- with tempfile.TemporaryDirectory() as tmpdir:- state_path = os.path.join(tmpdir, "state.json")- with open(state_path, "w") as f:- json.dump(state, f)- _upload_file_to_gcs(state_path, gcs_base + "/state.json")- info_path = os.path.join(tmpdir, "dataset_info.json")- dataset_info.write_to_directory(tmpdir)- _upload_file_to_gcs(info_path, gcs_base + "/dataset_info.json")---def _process_batched(combined, batch_size, workers, rate_limit, output_path, gcs_upload):- """Process dataset in batches: one arrow per batch, flat on GCS.-- GCS layout:- gs://bucket/path/data-00000-of-00020.arrow- gs://bucket/path/data-00001-of-00020.arrow- ...- gs://bucket/path/state.json- gs://bucket/path/dataset_info.json-- load_from_disk("gs://bucket/path") works directly.- """- n = len(combined)- num_batches = (n + batch_size - 1) // batch_size- total_rows = 0- dataset_info = None-- logger.info(f"Processing {n} rows in {num_batches} batches of {batch_size}...")-- for batch_idx in range(num_batches):- start = batch_idx * batch_size- end = min(start + batch_size, n)- arrow_name = f"data-{batch_idx:05d}-of-{num_batches:05d}.arrow"-- # Resume: skip arrows already on GCS- if _arrow_exists_on_gcs(gcs_upload, arrow_name):- logger.info(f"{arrow_name} already on GCS, skipping (rows {start}-{end - 1})")- total_rows += (end - start)- continue-- logger.info(f"=== Batch {batch_idx + 1}/{num_batches}: {arrow_name} (rows {start}-{end - 1}) ===")- batch = combined.select(range(start, end))- batch_len = len(batch)-- audio_data = [None] * batch_len- voice_names = [""] * batch_len- cache_dir = os.path.join(output_path, f".cache_{batch_idx:05d}")-- existing_cache = os.path.join(output_path, ".audio_cache")- if os.path.isdir(existing_cache):- recovered = 0- for i in range(batch_len):- wav_path = os.path.join(existing_cache, f"{start + i}.wav")- if os.path.exists(wav_path):- audio_data[i] = wav_path- recovered += 1- if recovered:- logger.info(f" Recovered {recovered}/{batch_len} WAVs from existing cache")-- _synthesize_rows(batch, audio_data, voice_names, workers, rate_limit,- cache_dir, global_offset=start)- _backfill_voice_names(voice_names, global_offset=start)-- os.makedirs(output_path, exist_ok=True)- arrow_path = os.path.join(output_path, arrow_name)- info = _build_arrow(batch, audio_data, voice_names, arrow_path)- if dataset_info is None:- dataset_info = info-- _upload_file_to_gcs(arrow_path, gcs_upload + "/" + arrow_name)- os.remove(arrow_path)- shutil.rmtree(cache_dir, ignore_errors=True)-- existing_cache = os.path.join(output_path, ".audio_cache")- if os.path.isdir(existing_cache):- for i in range(batch_len):- wav_path = os.path.join(existing_cache, f"{start + i}.wav")- if os.path.exists(wav_path):- os.remove(wav_path)-- total_rows += batch_len- logger.info(f"Progress: {total_rows}/{n} rows uploaded")-- if dataset_info is not None:- _upload_metadata_to_gcs(gcs_upload, num_batches, dataset_info)-- return total_rows---def _load_text_dataset(output_path, max_samples):- """Load the text dataset and optionally resume from an existing build."""- existing_ds = None- if os.path.exists(output_path):- try:- logger.info(f"Found existing dataset at {output_path}, loading for resume...")- existing_ds = load_from_disk(output_path)- logger.info(f" Existing dataset: {len(existing_ds)} rows")- except (FileNotFoundError, Exception) as e:- logger.warning(f"Could not load existing dataset (will use audio cache only): {e}")+def main():+ parser = argparse.ArgumentParser(description="Build text-only tool-call dataset")+ parser.add_argument("--max-samples", type=int, default=None, help="Max samples to include")+ parser.add_argument("--output", type=str, default="data/tool_calls_unified", help="Output directory")+ args = parser.parse_args()logger.info("Loading and combining text datasets...")combined = load_and_combine()- if max_samples:- combined = combined.select(range(min(max_samples, len(combined))))- logger.info(f"Text dataset: {len(combined)} rows")-- return combined, existing_ds---def _find_rows_needing_tts(combined, existing_ds, output_path):- """Identify which rows lack audio."""- n = len(combined)- needs_tts = set(range(n))-- cache_dir = os.path.join(output_path, ".audio_cache")- if os.path.isdir(cache_dir):- cached = 0- for fname in os.listdir(cache_dir):- if fname.endswith(".wav"):- try:- idx = int(fname[:-4])- if idx < n:- needs_tts.discard(idx)- cached += 1- except ValueError:- pass- if cached:- logger.info(f" {cached} rows found in audio cache")-- if existing_ds is not None and len(existing_ds) == n and "audio" in existing_ds.column_names:- audio_column = existing_ds.data.column("audio")- for i in range(n):- if i in needs_tts and audio_column[i].as_py() is not None:- needs_tts.discard(i)-- logger.info(f" {n - len(needs_tts)} rows already have audio, {len(needs_tts)} need TTS")- return needs_tts-+ if args.max_samples:+ combined = combined.select(range(min(args.max_samples, len(combined))))-def _recover_existing_audio(existing_ds, n, needs_tts, output_path):- """Recover audio file paths from the cache dir and existing dataset."""- audio_data = [None] * n- voice_names = [""] * n+ logger.info(f"Dataset: {len(combined)} rows")- cache_dir = os.path.join(output_path, ".audio_cache")- if os.path.isdir(cache_dir):- for i in range(n):- if i not in needs_tts:- wav_path = os.path.join(cache_dir, f"{i}.wav")- if os.path.exists(wav_path):- audio_data[i] = wav_path+ os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)+ combined.save_to_disk(args.output)+ logger.info(f"Saved {len(combined)} rows to {args.output}/")- if existing_ds is not None and len(existing_ds) == n and "audio" in existing_ds.column_names:- audio_column = existing_ds.data.column("audio")- voice_col = existing_ds["voice"] if "voice" in existing_ds.column_names else None- for i in range(n):- if i not in needs_tts and audio_data[i] is None:- raw = audio_column[i].as_py()- if isinstance(raw, dict) and raw.get("bytes"):- wav_path = os.path.join(cache_dir, f"{i}.wav")- os.makedirs(cache_dir, exist_ok=True)- with open(wav_path, "wb") as f:- f.write(raw["bytes"])- audio_data[i] = wav_path- elif isinstance(raw, dict) and raw.get("path") and os.path.exists(raw["path"]):- audio_data[i] = raw["path"]- if voice_col is not None and i not in needs_tts:- voice_names[i] = voice_col[i] or ""+ print_summary(combined)- return audio_data, voice_names+def print_summary(ds, n_samples=5):+ """Print dataset statistics and sample examples."""+ n = len(ds)-def _process_local(combined, workers, rate_limit, output_path):- """Original local-only processing: synthesize all, build one dataset."""- existing_ds = None- if os.path.exists(output_path):+ # Parse tools/answers counts+ n_tools, n_calls = [], []+ n_with_calls, n_no_calls = 0, 0+ query_lens, tools_lens, answers_lens = [], [], []+ for ex in ds:try:- existing_ds = load_from_disk(output_path)- except Exception:- pass-- needs_tts = _find_rows_needing_tts(combined, existing_ds, output_path)- audio_data, voice_names = _recover_existing_audio(- existing_ds, len(combined), needs_tts, output_path- )-- cache_dir = os.path.join(output_path, ".audio_cache")- _synthesize_rows(combined, audio_data, voice_names, workers, rate_limit, cache_dir)- _backfill_voice_names(voice_names)-- n = len(combined)- os.makedirs(output_path, exist_ok=True)- arrow_path = os.path.join(output_path, "data-00000-of-00001.arrow")- info = _build_arrow(combined, audio_data, voice_names, arrow_path)-- # Write metadata so load_from_disk works- import json- state = {- "_data_files": [{"filename": "data-00000-of-00001.arrow"}],- "_fingerprint": "local_build",- "_format_columns": None,- "_format_kwargs": {},- "_format_type": None,- "_output_all_columns": False,- "_split": None,- }- with open(os.path.join(output_path, "state.json"), "w") as f:- json.dump(state, f)- info.write_to_directory(output_path)- logger.info(f"Saved {n} rows to {output_path}/")- return n--def main():- parser = argparse.ArgumentParser(description="Build unified tool-call dataset with TTS audio")- parser.add_argument("--max-samples", type=int, default=None, help="Max samples to process")- parser.add_argument("--workers", type=int, default=16, help="TTS concurrent workers (default: 16)")- parser.add_argument("--output", type=str, default="data/tool_calls_unified", help="Output directory")- parser.add_argument("--rate-limit", type=float, default=30.0,- help="Max TTS requests per second (default: 30, quota is 2x1000/min)")- parser.add_argument("--gcs-upload", type=str, default=None,- help="GCS path to upload dataset (e.g. gs://cactus-dataset/tool_calls)")- parser.add_argument("--batch-size", type=int, default=5000,- help="Rows per batch when using --gcs-upload (default: 500)")- args = parser.parse_args()-- output_path = args.output-- logger.info("Loading and combining text datasets...")- combined = load_and_combine()- if args.max_samples:- combined = combined.select(range(min(args.max_samples, len(combined))))- logger.info(f"Text dataset: {len(combined)} rows")-- if args.gcs_upload:- total_rows = _process_batched(- combined, args.batch_size, args.workers, args.rate_limit,- output_path, args.gcs_upload,- )- else:- total_rows = _process_local(combined, args.workers, args.rate_limit, output_path)-- logger.info(f"Pipeline complete: {total_rows} rows")+ tools = json.loads(ex["tools"])+ except (json.JSONDecodeError, TypeError):+ tools = []+ try:+ answers = json.loads(ex["answers"])+ except (json.JSONDecodeError, TypeError):+ answers = []+ n_tools.append(len(tools))+ n_calls.append(len(answers))+ if answers:+ n_with_calls += 1+ else:+ n_no_calls += 1+ query_lens.append(len(ex["query"]))+ tools_lens.append(len(ex["tools"]))+ answers_lens.append(len(ex["answers"]))++ n_tools = np.array(n_tools)+ n_calls = np.array(n_calls)+ query_lens = np.array(query_lens)+ tools_lens = np.array(tools_lens)+ answers_lens = np.array(answers_lens)++ def _stats(arr):+ return f"min={arr.min()}, max={arr.max()}, mean={arr.mean():.1f}, median={np.median(arr):.0f}"++ print(f"\n{'=' * 60}")+ print(f"DATASET SUMMARY")+ print(f"{'=' * 60}")+ print(f" Total examples: {n:,}")+ print(f" With tool calls: {n_with_calls:,} ({n_with_calls/n*100:.1f}%)")+ print(f" No-call (empty): {n_no_calls:,} ({n_no_calls/n*100:.1f}%)")+ print()+ print(f" Available tools/sample: {_stats(n_tools)}")+ print(f" Tool calls/sample: {_stats(n_calls)}")+ print()+ print(f" Query length (chars): {_stats(query_lens)}")+ print(f" Tools length (chars): {_stats(tools_lens)}")+ print(f" Answers length (chars): {_stats(answers_lens)}")++ # Samples with tool calls+ with_calls_idx = [i for i in range(n) if n_calls[i] > 0]+ no_calls_idx = [i for i in range(n) if n_calls[i] == 0]++ rng = np.random.RandomState(42)++ if with_calls_idx:+ sample_idx = rng.choice(with_calls_idx, size=min(n_samples, len(with_calls_idx)), replace=False)+ print(f"\n{'─' * 60}")+ print(f"SAMPLES WITH TOOL CALLS ({len(sample_idx)})")+ print(f"{'─' * 60}")+ for i, idx in enumerate(sample_idx):+ ex = ds[int(idx)]+ print(f"\n [{i+1}] Query: {ex['query']}")+ print(f" Tools: {ex['tools']}")+ print(f" Answers: {ex['answers']}")++ if no_calls_idx:+ sample_idx = rng.choice(no_calls_idx, size=min(n_samples, len(no_calls_idx)), replace=False)+ print(f"\n{'─' * 60}")+ print(f"SAMPLES WITHOUT TOOL CALLS ({len(sample_idx)})")+ print(f"{'─' * 60}")+ for i, idx in enumerate(sample_idx):+ ex = ds[int(idx)]+ print(f"\n [{i+1}] Query: {ex['query']}")+ print(f" Tools: {ex['tools']}")+ print(f" Answers: {ex['answers']}")++ print(f"\n{'=' * 60}\n")if __name__ == "__main__":diff --git a/scripts/tools_data.py b/scripts/tools_data.pyindex e287abc..0f8d541 100755--- a/scripts/tools_data.py+++ b/scripts/tools_data.py@@ -1,33 +1,22 @@"""Unified tool-calling dataset builder.-Loads four HuggingFace tool-calling datasets and converts them all into the-xlam-function-calling-60k schema:+Loads HuggingFace tool-calling datasets and converts them into a common schema:query : str – the user's natural-language requestanswers : str – JSON list of {"name": ..., "arguments": {...}} tool callstools : str – JSON list of available tool definitionssource : str – which dataset the example came from-Datasets and their native formats:+Datasets:1. Salesforce/xlam-function-calling-60kAlready in the target schema (query, answers, tools).-2. Salesforce/APIGen-MT-5k- Multi-turn conversations with human/gpt/function_call/observation roles.- We extract the first user message as the query and collect all function_call- messages (excluding internal "think" calls) as answers.--3. glaiveai/glaive-function-calling-v2+2. glaiveai/glaive-function-calling-v2Tool definitions embedded in the system prompt; chat is a flat string withUSER:/A: role markers and <functioncall> tags. Examples with empty answers(answers=[]) are kept so the model learns when NOT to call functions.--4. Team-ACE/ToolACE- Tool definitions as a JSON array in the system prompt; function calls use a- bracket syntax [FuncName(key=val, ...)] where names may contain spaces. Only- examples with at least one parsed function call are kept."""import json@@ -39,9 +28,7 @@ from datasets import load_dataset, Dataset, concatenate_datasetsDATASETS = {"xlam-function-calling-60k": "Salesforce/xlam-function-calling-60k",- "APIGen-MT-5k": "Salesforce/APIGen-MT-5k","glaive-function-calling": "glaiveai/glaive-function-calling-v2",- "ToolACE": "Team-ACE/ToolACE",}@@ -49,34 +36,6 @@ def convert_xlam(dataset):return dataset.select_columns(["query", "answers", "tools"])-def convert_apigen_mt(dataset):- rows = []- for ex in dataset:- convs = ex["conversations"]- tools = ex["tools"]-- query = None- answers = []- for msg in convs:- if msg["from"] == "human" and query is None:- query = msg["value"]- elif msg["from"] == "function_call":- try:- fc = json.loads(msg["value"])- if fc.get("name") != "think":- answers.append(fc)- except json.JSONDecodeError:- pass-- if query:- rows.append({- "query": query,- "answers": json.dumps(answers),- "tools": tools,- })- return Dataset.from_list(rows)--def convert_glaive(dataset):rows = []for ex in dataset:@@ -149,109 +108,9 @@ def _parse_glaive_chat(chat):return query, answers-def convert_toolace(dataset):- rows = []- for ex in dataset:- tools = _parse_toolace_tools(ex["system"])- convs = ex["conversations"]-- query = None- answers = []- for msg in convs:- if msg["from"] == "user" and query is None:- query = msg["value"]- elif msg["from"] == "assistant" and query is not None:- calls = _parse_toolace_calls(msg["value"])- if calls:- answers.extend(calls)- break-- if query and answers:- rows.append({- "query": query,- "answers": json.dumps(answers),- "tools": json.dumps(tools),- })- return Dataset.from_list(rows)---def _parse_toolace_tools(system_text):- """Extract tool list from ToolACE system prompt (JSON array after 'invoke:')."""- match = re.search(r'invoke:\s*(\[)', system_text)- if not match:- return []- start = match.start(1)- depth = 0- for i in range(start, len(system_text)):- if system_text[i] == '[':- depth += 1- elif system_text[i] == ']':- depth -= 1- if depth == 0:- try:- return json.loads(system_text[start:i + 1])- except json.JSONDecodeError:- return []- return []---def _parse_toolace_calls(text):- """Parse [FuncName(key=val, ...)] calls. Function names may contain spaces."""- answers = []- bracket_match = re.search(r'\[(.+)\]', text, re.DOTALL)- if not bracket_match:- return answers-- inner = bracket_match.group(1)- for m in re.finditer(r'([\w][\w ]*?\w|\w)\(([^)]*)\)', inner):- name = m.group(1).strip()- args_str = m.group(2)- arguments = {}- for arg_m in re.finditer(- r'(\w+)\s*=\s*(?:"((?:[^"\\]|\\.)*)"|\'((?:[^\'\\]|\\.)*)\'|(\[[^\]]*\])|([^,)]+))',- args_str,- ):- key = arg_m.group(1)- val = next(- (g for g in (arg_m.group(2), arg_m.group(3), arg_m.group(4), arg_m.group(5)) if g is not None),- None,- )- if val is not None:- val = val.strip()- try:- val = json.loads(val)- except (json.JSONDecodeError, TypeError):- pass- arguments[key] = val- answers.append({"name": name, "arguments": arguments})- return answers---def compact_tools(tools_json):- """Strip verbose schema descriptions, keep only function name + param name/type."""- try:- tools = json.loads(tools_json)- except (json.JSONDecodeError, TypeError):- return tools_json- compact = []- for t in tools:- params = {}- raw_params = t.get("parameters", {})- props = raw_params.get("properties", raw_params) if isinstance(raw_params, dict) else {}- for pname, pdef in props.items():- if isinstance(pdef, dict):- params[pname] = pdef.get("type", "string")- else:- params[pname] = str(pdef)- compact.append({"name": t["name"], "parameters": params})- return json.dumps(compact)--CONVERTERS = {"xlam-function-calling-60k": convert_xlam,- "APIGen-MT-5k": convert_apigen_mt,"glaive-function-calling": convert_glaive,- "ToolACE": convert_toolace,}_PLACEHOLDER_VALUES = {@@ -421,18 +280,22 @@ def load_and_combine():combined = concatenate_datasets(parts)+ # Drop no-call examples with empty tools (trivial negatives)+ before_empty = len(combined)+ combined = combined.filter(+ lambda ex: json.loads(ex["answers"]) != [] or json.loads(ex["tools"]) != [],+ desc="Removing empty-tools no-call examples",+ )+ empty_dropped = before_empty - len(combined)+ if empty_dropped:+ print(f"\nRemoved {empty_dropped} no-call examples with empty tools")+before_dedup = len(combined)combined = _deduplicate(combined)dedup_dropped = before_dedup - len(combined)if dedup_dropped:print(f"\nDeduplicated: removed {dedup_dropped} duplicate queries")- print(f"\nCompacting tool schemas...")- combined = combined.map(- lambda ex: {"tools": compact_tools(ex["tools"])},- desc="Compacting tools",- )-empty_count = sum(1 for ex in combined if json.loads(ex["answers"]) == [])print(f"\n{'='*60}")print(f"Combined dataset: {len(combined)} examples ({empty_count} with empty answers = no-call examples)")diff --git a/src/cli.py b/src/cli.pyindex 15a909e..1ada00e 100644--- a/src/cli.py+++ b/src/cli.py@@ -12,9 +12,8 @@ def main():sub = parser.add_subparsers(dest="command")p = sub.add_parser("train", add_help=False)- p.add_argument("--full", action="store_true")p.add_argument("--checkpoint", type=str, default=None)- p.add_argument("--epochs", type=int, default=3)+ p.add_argument("--epochs", type=int, default=10)p.add_argument("--batch-size", type=int, default=32)p.add_argument("--lr", type=float, default=3e-4)p.add_argument("--muon-lr", type=float, default=0.02)@@ -23,15 +22,15 @@ def main():p.add_argument("--num-kv-heads", type=int, default=8)p.add_argument("--num-layers", type=int, default=4)p.add_argument("--num-dec-layers", type=int, default=4)- p.add_argument("--max-enc-len", type=int, default=256)- p.add_argument("--max-dec-len", type=int, default=1024)+ p.add_argument("--max-enc-len", type=int, default=1024)+ p.add_argument("--max-dec-len", type=int, default=512)p.add_argument("--max-samples", type=int, default=None)p.add_argument("--warmup-ratio", type=float, default=0.05)p.add_argument("--wandb", action="store_true")p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])p.add_argument("--checkpoint-dir", type=str, default="checkpoints")p.add_argument("--seed", type=int, default=42)- p.add_argument("--eval-every", type=int, default=1000)+ p.add_argument("--eval-every", type=int, default=100)p.add_argument("--max-eval-samples", type=int, default=None)p.add_argument("--sparsity-ratio", type=float, default=0.0)p.add_argument("--group-size", type=int, default=32)@@ -42,36 +41,21 @@ def main():p.add_argument("--prune-end-frac", type=float, default=0.67,help="Fraction of epoch at which pruning finishes and mask locks (default: 0.67)")p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])- p.add_argument("--num-memory-slots", type=int, default=64)- p.add_argument("--mat-factors", type=int, nargs="*", default=[2, 4, 8],- help="Matryoshka FFN shrink factors, e.g. 2=half width (default: 2 4 8)")+ p.add_argument("--num-memory-slots", type=int, default=128)+ p.add_argument("--mat-factors", type=int, nargs="*", default=[2, 4],+ help="Matryoshka FFN shrink factors, e.g. 2=half width (default: 2 4)")p.add_argument("--mat-shared-input", action="store_true",help="Each unique input is repeated across all mat widths (default: unique input per width)")- p.add_argument("--dropout", type=float, default=0.0,+ p.add_argument("--dropout", type=float, default=0.05,help="Dropout rate for residual connections (default: 0.1)")- p.add_argument("--no-speech", action="store_true", help="Disable speech training (text-only)")- p.add_argument("--max-mel-len", type=int, default=1024,- help="Max mel spectrogram frames (default: 1024)")- p.add_argument("--n-mels", type=int, default=80,- help="Number of mel frequency bins (default: 80)")- p.add_argument("--max-speech-samples", type=int, default=None,- help="Max voice-tool-call training samples (default: all)")p = sub.add_parser("tokenize", add_help=False)p.add_argument("--max-samples", type=int, default=None,help="Limit samples per split (for dev/test)")- p.add_argument("--cleanup", action="store_true",- help="Delete local .data_cache/ after GCS upload")- p.add_argument("--n-mels", type=int, default=80,- help="Number of mel frequency bins (default: 80)")- p.add_argument("--max-mel-len", type=int, default=1024,- help="Max mel spectrogram frames (default: 1024)")- p.add_argument("--max-enc-len", type=int, default=256,- help="Max encoder sequence length (default: 256)")- p.add_argument("--max-dec-len", type=int, default=1024,- help="Max decoder sequence length (default: 1024)")- p.add_argument("--batch-size", type=int, default=None,- help="Process in batches of this size, uploading shards to GCS incrementally")+ p.add_argument("--max-enc-len", type=int, default=1024,+ help="Max encoder sequence length (default: 1024)")+ p.add_argument("--max-dec-len", type=int, default=512,+ help="Max decoder sequence length (default: 512)")p = sub.add_parser("run", add_help=False)p.add_argument("--checkpoint", type=str, required=True)@@ -81,17 +65,15 @@ def main():p.add_argument("--max-len", type=int, default=512)p.add_argument("--seed", type=int, default=0)- p = sub.add_parser("test", add_help=False)+ p = sub.add_parser("eval", add_help=False)p.add_argument("--checkpoint", type=str, required=True)p.add_argument("--batch-size", type=int, default=32)p.add_argument("--max-eval-samples", type=int, default=1000)- p.add_argument("--max-enc-len", type=int, default=256)- p.add_argument("--max-dec-len", type=int, default=1024)+ p.add_argument("--max-enc-len", type=int, default=1024)+ p.add_argument("--max-dec-len", type=int, default=512)p.add_argument("--max-gen-len", type=int, default=512)p.add_argument("--tool-call-samples", type=int, default=200,help="Samples for tool-call accuracy eval (default: 200)")- p.add_argument("--voice-tc-samples", type=int, default=50,- help="Samples for voice-to-tool-call eval (default: 50)")p.add_argument("--throughput-runs", type=int, default=10)p = sub.add_parser("evaluate", add_help=False)@@ -141,22 +123,14 @@ def main():from .tokenize_data import tokenizetokenize(args)elif args.command == "train":- if getattr(args, "full", False):- args.d_model = 1536- args.num_heads = 24- args.num_kv_heads = 8- args.num_layers = 12- args.num_dec_layers = 4- args.num_memory_slots = 128- args.mat_factors = [2, 3, 4, 8, 16]from .train import traintrain(args)elif args.command == "run":from .run import main as run_mainrun_main(args)- elif args.command == "test":- from .test import main as test_main- test_main(args)+ elif args.command == "eval":+ from .eval import main as eval_main_fn+ eval_main_fn(args)elif args.command == "evaluate":from .evaluate import main as eval_maineval_main(args)diff --git a/src/data.py b/src/data.pyindex e14fd7d..ffc5c31 100644--- a/src/data.py+++ b/src/data.py@@ -3,7 +3,6 @@ import json as _jsonimport multiprocessing as mpimport osimport queue-import subprocessimport threadingos.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"@@ -14,7 +13,7 @@ logging.getLogger("transformers").setLevel(logging.ERROR)logging.getLogger("huggingface_hub").setLevel(logging.ERROR)import numpy as np-from datasets import Audio, Dataset, concatenate_datasets, load_from_disk+from datasets import Audio, Dataset, load_from_diskfrom tqdm import tqdmimport sentencepiece as spm@@ -23,13 +22,8 @@ _DISK_CACHE_DIR = os.path.join(_PROJECT_ROOT, ".data_cache")TOKENIZER_DIR = os.path.join(_PROJECT_ROOT, "tokenizer")TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")LOCAL_UNIFIED_DIR = os.path.join(_PROJECT_ROOT, "data", "tool_calls_unified")-GCS_DATASET_PATH = "gs://cactus-dataset/tool_calls"-JSONL_DATASET_PATHS = [- "gs://needle-datasets-bucket/datasets/xlam/openai_jsonl_v1/xlam_openai_jsonl/train",- "gs://needle-datasets-bucket/datasets/toucan/openai_jsonl_v1/toucan_openai_jsonl/train",-]-_MIN_SHM_BYTES = 200 * 1024**3+_MIN_SHM_BYTES = 200 * 1024**3def _pick_cache_dir():@@ -53,67 +47,17 @@ CACHE_DIR = _pick_cache_dir()PAD_ID = 0EOS_ID = 1-BOS_ID = 2+BOS_ID = 2UNK_ID = 3TOOL_CALL_ID = 4TRANSCRIBE_ID = 5+TOOLS_ID = 6_unified_dataset_cache = None-def _load_jsonl_shards(gcs_paths):- rows = []- for gcs_path in gcs_paths:- try:- result = subprocess.run(- ["gcloud", "storage", "ls", f"{gcs_path}/*.jsonl"],- capture_output=True,- text=True,- check=True,- )- shard_uris = [line.strip() for line in result.stdout.splitlines() if line.strip()]- except Exception as e:- print(f"Warning: could not list shards at {gcs_path}: {e}")- continue-- for uri in shard_uris:- fname = os.path.basename(uri)- local_path = os.path.join(CACHE_DIR, fname)- try:- subprocess.run(- ["gcloud", "storage", "cp", uri, local_path],- capture_output=True,- check=True,- )- except Exception as e:- print(f"Warning: could not download {uri}: {e}")- continue-- with open(local_path, "r", encoding="utf-8") as f:- for line in f:- line = line.strip()- if not line:- continue- try:- record = _json.loads(line)- except Exception:- continue- messages = record.get("messages", [])- query = messages[0]["content"] if messages else ""- tool_calls = messages[1].get("tool_calls") if len(messages) > 1 else None- rows.append({- "query": query,- "tools": _json.dumps(record.get("tools", [])),- "answers": _json.dumps(tool_calls if tool_calls is not None else []),- })-- if not rows:- return None- return Dataset.from_list(rows)--def _load_unified_dataset():- """Load the unified dataset from GCS (via gcsfs) or local fallback.+ """Load the unified dataset from local disk.Caches the result in memory after first load.Uses soundfile as the audio decoding backend.@@ -130,30 +74,16 @@ def _load_unified_dataset():print(f"Loaded unified dataset from {LOCAL_UNIFIED_DIR} ({len(ds)} rows)")_unified_dataset_cache = _set_audio_backend(ds)return _unified_dataset_cache- except Exception:- print(f"Local dataset at {LOCAL_UNIFIED_DIR} is incomplete, trying GCS...")-- try:- ds = load_from_disk(GCS_DATASET_PATH)- print(f"Loaded unified dataset from GCS ({len(ds)} rows)")- ds = _set_audio_backend(ds)- except Exception as e:- raise FileNotFoundError(- f"Unified dataset not found. Run scripts/build_dataset.py first, "- f"or ensure GCS path {GCS_DATASET_PATH} is accessible (pip install gcsfs). "- f"GCS error: {e}"- )-- try:- jsonl_ds = _load_jsonl_shards(JSONL_DATASET_PATHS)- if jsonl_ds is not None:- ds = concatenate_datasets([ds, jsonl_ds])- print(f"Concatenated JSONL shards: total {len(ds)} rows")- except Exception as e:- print(f"Warning: failed to load JSONL shards, continuing with Arrow only: {e}")+ except Exception as e:+ raise FileNotFoundError(+ f"Failed to load dataset from {LOCAL_UNIFIED_DIR}: {e}. "+ f"Run 'python scripts/build_dataset.py' first."+ )- _unified_dataset_cache = ds- return _unified_dataset_cache+ raise FileNotFoundError(+ f"Unified dataset not found at {LOCAL_UNIFIED_DIR}. "+ f"Run 'python scripts/build_dataset.py' first."+ )def _set_audio_backend(ds):@@ -193,6 +123,10 @@ class NeedleTokenizer:def transcribe_token_id(self):return TRANSCRIBE_ID+ @property+ def tools_token_id(self):+ return TOOLS_ID+@propertydef vocab_size(self):return self.sp.GetPieceSize()@@ -264,7 +198,7 @@ def train_tokenizer(vocab_size=8192, max_samples=None, force=False):eos_id=EOS_ID,bos_id=BOS_ID,unk_id=UNK_ID,- user_defined_symbols=["<tool_call>", "<transcribe>"],+ user_defined_symbols=["<tool_call>", "<transcribe>", "<tools>"],byte_fallback=True,normalization_rule_name="identity",num_threads=os.cpu_count(),@@ -277,96 +211,13 @@ def train_tokenizer(vocab_size=8192, max_samples=None, force=False):return model_path-GCS_TOKENIZER_PATH = "gs://cactus-dataset/tokenizer/"---def upload_tokenizer_to_gcs():- """Upload local tokenizer files to GCS."""- import subprocess- import glob as globmod- tok_files = globmod.glob(os.path.join(TOKENIZER_DIR, "*"))- if tok_files:- subprocess.run(- ["gcloud", "storage", "cp"] + tok_files + [GCS_TOKENIZER_PATH],- capture_output=True, text=True,- )- print(f"Uploaded tokenizer to {GCS_TOKENIZER_PATH}")--def get_tokenizer(max_samples=None):model_path = TOKENIZER_PREFIX + ".model"if not os.path.exists(model_path):- # Try downloading from GCS first- if _download_tokenizer_from_gcs():- return NeedleTokenizer(model_path)train_tokenizer(max_samples=max_samples)return NeedleTokenizer(model_path)-def _download_tokenizer_from_gcs():- """Download tokenizer files from GCS. Returns True on success."""- import subprocess- os.makedirs(TOKENIZER_DIR, exist_ok=True)- result = subprocess.run(- ["gcloud", "storage", "cp", "-r", GCS_TOKENIZER_PATH + "*", TOKENIZER_DIR + "/"],- capture_output=True, text=True,- )- model_path = TOKENIZER_PREFIX + ".model"- if result.returncode == 0 and os.path.exists(model_path):- print(f"Downloaded tokenizer from {GCS_TOKENIZER_PATH}")- return True- return False---GCS_CACHE_PATH = "gs://cactus-dataset/cache"---def _gcs_cache_download(cache_id, suffixes):- """Try downloading cached files from GCS. Returns True if all found."""- import subprocess- os.makedirs(CACHE_DIR, exist_ok=True)- cache_path = os.path.join(CACHE_DIR, cache_id)- srcs = [f"{GCS_CACHE_PATH}/{cache_id}{s}" for s in suffixes]- subprocess.run(- ["gcloud", "storage", "cp"] + srcs + [CACHE_DIR + "/"],- capture_output=True, text=True,- )- return all(os.path.exists(cache_path + s) for s in suffixes)---def _gcs_cache_upload(cache_id, suffixes):- """Upload cached files to GCS."""- import subprocess- cache_path = os.path.join(CACHE_DIR, cache_id)- files = [cache_path + s for s in suffixes if os.path.exists(cache_path + s)]- if files:- subprocess.run(- ["gcloud", "storage", "cp"] + files + [GCS_CACHE_PATH + "/"],- capture_output=True, text=True,- )---def _gcs_download_shards(cache_id, n_shards, shard_suffixes):- """Download sharded .npy files from GCS. Returns True if all found."""- import subprocess- os.makedirs(CACHE_DIR, exist_ok=True)- srcs = []- for suffix in shard_suffixes:- for i in range(n_shards):- srcs.append(f"{GCS_CACHE_PATH}/{cache_id}{suffix}_{i:05d}.npy")- if not srcs:- return True- subprocess.run(- ["gcloud", "storage", "cp"] + srcs + [CACHE_DIR + "/"],- capture_output=True, text=True,- )- for suffix in shard_suffixes:- for i in range(n_shards):- if not os.path.exists(os.path.join(CACHE_DIR, f"{cache_id}{suffix}_{i:05d}.npy")):- return False- return True--def load_tool_calls(split="train", max_samples=None, return_global_indices=False):"""Load tool-calling dataset, splitting 90/10 for train/val.@@ -410,117 +261,66 @@ def _cache_key(prefix, n_samples, max_enc_len, max_dec_len):return hashlib.md5(key.encode()).hexdigest()[:12]-def _mel_cache_key(prefix, n_samples, n_mels, max_mel_len):- tok_hash = _tokenizer_hash()- key = f"mel_{prefix}_{tok_hash}_{n_samples}_{n_mels}_{max_mel_len}"- return hashlib.md5(key.encode()).hexdigest()[:12]---def _save_cache_metadata(split, text_cache_id, mel_cache_id, n_samples,- max_enc_len, max_dec_len, n_mels, max_mel_len):- """Save metadata JSON for a split, upload to GCS."""+def _save_cache_metadata(split, text_cache_id, n_samples, max_enc_len, max_dec_len):+ """Save metadata JSON for a split locally."""os.makedirs(CACHE_DIR, exist_ok=True)meta = {"split": split,"text_cache_id": text_cache_id,- "mel_cache_id": mel_cache_id,"n_samples": n_samples,"max_enc_len": max_enc_len,"max_dec_len": max_dec_len,- "n_mels": n_mels,- "max_mel_len": max_mel_len,}meta_path = os.path.join(CACHE_DIR, f"{split}_metadata.json")with open(meta_path, "w") as f:_json.dump(meta, f)- _gcs_cache_upload(f"{split}_metadata", [".json"])def _load_cache_metadata(split):- """Load metadata JSON from local or GCS. Returns dict or None."""+ """Load metadata JSON from local cache. Returns dict or None."""meta_path = os.path.join(CACHE_DIR, f"{split}_metadata.json")- if os.path.exists(meta_path):- with open(meta_path) as f:- return _json.load(f)-- import subprocess- os.makedirs(CACHE_DIR, exist_ok=True)- gcs_path = f"{GCS_CACHE_PATH}/{split}_metadata.json"- subprocess.run(- ["gcloud", "storage", "cp", gcs_path, meta_path],- capture_output=True, text=True,- )if os.path.exists(meta_path):with open(meta_path) as f:return _json.load(f)return None-def _tools_to_names(tools_json):- try:- tools = _json.loads(tools_json) if isinstance(tools_json, str) else tools_json- names = [t["function"]["name"] for t in tools if t.get("function", {}).get("name")]- return _json.dumps(names)- except Exception:- return tools_json---def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024, batch_size=None):+def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=1024, max_dec_len=512):"""Prepare tool-call encoder-decoder pairs with <tool_call> task token.- Encoder input: tokenize(query), truncated to max_enc_len.- Decoder input: [BOS, <tool_call>, tools_tokens..., answer_tokens...]- Decoder target: [<tool_call>, tools_tokens..., answer_tokens..., EOS]- Loss mask: 1 only on answer + EOS positions (not tools prefix or padding).+ Encoder input: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.+ Decoder input: [EOS, <tool_call>, answer_tokens..., PAD...]+ Decoder target: [<tool_call>, answer_tokens..., EOS, PAD...]+ Loss mask: 1 on answer + trailing EOS positions only.- When batch_size is set, produces per-batch shard files uploaded to GCS- incrementally. Returns (None, None, None, None, kept_indices).- When batch_size is None, returns (enc_inputs, dec_inputs, dec_targets,- loss_mask, kept_indices) as before.+ Returns (enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices)."""cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len)cache_path = os.path.join(CACHE_DIR, cache_id)- if batch_size is None:- tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]-- def _load_tc_cache():- return (- np.load(cache_path + "_enc.npy"),- np.load(cache_path + "_dec_in.npy"),- np.load(cache_path + "_dec_tgt.npy"),- np.load(cache_path + "_loss_mask.npy"),- np.load(cache_path + "_kept_idx.npy"),- )-- if os.path.exists(cache_path + "_enc.npy"):- print(f"Loading cached tool-call data ({cache_id})...")- return _load_tc_cache()-- if _gcs_cache_download(cache_id, tc_suffixes):- print(f"Downloaded tool-call cache from GCS ({cache_id})...")- return _load_tc_cache()+ tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]- if batch_size is not None:- manifest_path = cache_path + "_manifest.json"- if os.path.exists(manifest_path):- kept_indices = np.load(cache_path + "_kept_idx.npy")- print(f"Loading cached sharded tool-call data ({cache_id})...")- return None, None, None, None, kept_indices+ def _load_tc_cache():+ return (+ np.load(cache_path + "_enc.npy"),+ np.load(cache_path + "_dec_in.npy"),+ np.load(cache_path + "_dec_tgt.npy"),+ np.load(cache_path + "_loss_mask.npy"),+ np.load(cache_path + "_kept_idx.npy"),+ )- if _gcs_cache_download(cache_id, ["_manifest.json", "_kept_idx.npy"]):- if os.path.exists(manifest_path):- kept_indices = np.load(cache_path + "_kept_idx.npy")- print(f"Downloaded sharded tool-call cache from GCS ({cache_id})...")- return None, None, None, None, kept_indices+ if os.path.exists(cache_path + "_enc.npy"):+ print(f"Loading cached tool-call data ({cache_id})...")+ return _load_tc_cache()pad_id = tokenizer.pad_token_ideos_id = tokenizer.eos_token_idtool_call_id = tokenizer.tool_call_token_id+ tools_sep_id = tokenizer.tools_token_idenc_texts = [ex["query"] for ex in ds]- tools_texts = [_tools_to_names(ex["tools"]) for ex in ds]+ tools_texts = [ex["tools"] for ex in ds]ans_texts = [ex["answers"] for ex in ds]num_workers = min(os.cpu_count() or 1, 8)@@ -537,7 +337,7 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024, batools_chunks = [tools_texts[i:i + chunk_size] for i in range(0, len(tools_texts), chunk_size)]print(f"Tokenizing tools ({num_workers} workers)...")with mp.Pool(num_workers, initializer=_init_worker,- initargs=(model_path, max_dec_len - 2)) as pool:+ initargs=(model_path, max_enc_len)) as pool:tools_results = pool.map(_tokenize_chunk, tools_chunks)all_tools_tokens = [tok for chunk in tools_results for tok in chunk]@@ -552,113 +352,41 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024, badef _fill_sample(j, e_tok, t_tok, a_tok, enc_arr, dec_in_arr, dec_tgt_arr, lm_arr):"""Fill arrays at position j for one sample. Returns False if skipped."""- prefix_len = 2 + len(t_tok)- total_dec = prefix_len + len(a_tok) + 1- if total_dec > max_dec_len:- available_for_tools = max_dec_len - 2 - len(a_tok) - 1- if available_for_tools < 1:- return False- t_tok = t_tok[:available_for_tools]- prefix_len = 2 + len(t_tok)-- el = len(e_tok)- enc_arr[j, :el] = e_tok-+ al = len(a_tok)+ # Decoder needs: [EOS, <tool_call>, answer..., EOS] = 2 + al + 1+ if 2 + al + 1 > max_dec_len:+ return False++ # Encoder: [query..., <tools>, tools...] truncated to max_enc_len+ # Reserve 1 slot for <tools> separator + at least 1 token for tools+ max_query = max_enc_len - 1 - 1 # room for <tools> + at least 1 tools token+ if len(e_tok) > max_query:+ e_tok = e_tok[:max_query]+ remaining = max_enc_len - len(e_tok) - 1 # 1 for <tools>+ t_trunc = t_tok[:remaining]++ enc_seq = e_tok + [tools_sep_id] + t_trunc+ el = len(enc_seq)+ enc_arr[j, :el] = enc_seq++ # Decoder input: [EOS, <tool_call>, answer...]dec_in_arr[j, 0] = eos_iddec_in_arr[j, 1] = tool_call_id- tl = len(t_tok)- if tl > 0:- dec_in_arr[j, 2:2 + tl] = t_tok- al = len(a_tok)if al > 0:- dec_in_arr[j, prefix_len:prefix_len + al] = a_tok+ dec_in_arr[j, 2:2 + al] = a_tok+ # Decoder target: [<tool_call>, answer..., EOS]dec_tgt_arr[j, 0] = tool_call_id- if tl > 0:- dec_tgt_arr[j, 1:1 + tl] = t_tokif al > 0:- dec_tgt_arr[j, prefix_len - 1:prefix_len - 1 + al] = a_tok- dec_tgt_arr[j, prefix_len - 1 + al] = eos_id+ dec_tgt_arr[j, 1:1 + al] = a_tok+ dec_tgt_arr[j, 1 + al] = eos_id- lm_arr[j, prefix_len - 1:prefix_len - 1 + al + 1] = 1.0+ # Loss mask: answer tokens + trailing EOS+ lm_arr[j, 1:1 + al + 1] = 1.0return Trueos.makedirs(CACHE_DIR, exist_ok=True)- if batch_size is not None:- all_kept_indices = []- shard_counts = []- shard_idx = 0- total_skipped = 0-- for batch_start in range(0, n, batch_size):- batch_end = min(batch_start + batch_size, n)- batch_n = batch_end - batch_start-- enc_batch = np.full((batch_n, max_enc_len), pad_id, dtype=np.int32)- dec_in_batch = np.full((batch_n, max_dec_len), pad_id, dtype=np.int32)- dec_tgt_batch = np.full((batch_n, max_dec_len), pad_id, dtype=np.int32)- loss_batch = np.zeros((batch_n, max_dec_len), dtype=np.float32)-- skipped = 0- for j in range(batch_n):- i = batch_start + j- if not _fill_sample(j, all_enc_tokens[i], all_tools_tokens[i],- all_ans_tokens[i], enc_batch, dec_in_batch,- dec_tgt_batch, loss_batch):- skipped += 1-- if skipped > 0:- keep = enc_batch[:, 0] != pad_id- kept_in_batch = np.where(keep)[0]- enc_batch = enc_batch[keep]- dec_in_batch = dec_in_batch[keep]- dec_tgt_batch = dec_tgt_batch[keep]- loss_batch = loss_batch[keep]- all_kept_indices.extend((batch_start + kept_in_batch).tolist())- total_skipped += skipped- else:- all_kept_indices.extend(range(batch_start, batch_end))-- if len(enc_batch) == 0:- continue-- shard_suffixes = []- for suffix, arr in [("_enc", enc_batch), ("_dec_in", dec_in_batch),- ("_dec_tgt", dec_tgt_batch), ("_loss_mask", loss_batch)]:- shard_suffix = f"{suffix}_{shard_idx:05d}.npy"- np.save(os.path.join(CACHE_DIR, f"{cache_id}{shard_suffix}"), arr)- shard_suffixes.append(shard_suffix)-- _gcs_cache_upload(cache_id, shard_suffixes)- for s in shard_suffixes:- fpath = os.path.join(CACHE_DIR, f"{cache_id}{s}")- if os.path.exists(fpath):- os.remove(fpath)-- shard_counts.append(len(enc_batch))- shard_idx += 1-- if total_skipped > 0:- print(f" Skipped {total_skipped} examples (too long for max_dec_len={max_dec_len})")-- kept_indices = np.array(all_kept_indices, dtype=np.int64)- np.save(cache_path + "_kept_idx.npy", kept_indices)-- manifest = {- "n_shards": shard_idx,- "shard_counts": shard_counts,- "total_kept": len(kept_indices),- }- manifest_path = cache_path + "_manifest.json"- with open(manifest_path, "w") as f:- _json.dump(manifest, f)-- _gcs_cache_upload(cache_id, ["_manifest.json", "_kept_idx.npy"])- print(f"Cached {len(kept_indices):,} tool-call pairs in {shard_idx} shards ({cache_id})")-- return None, None, None, None, kept_indices-enc_inputs = np.full((n, max_enc_len), pad_id, dtype=np.int32)dec_inputs = np.full((n, max_dec_len), pad_id, dtype=np.int32)dec_targets = np.full((n, max_dec_len), pad_id, dtype=np.int32)@@ -687,8 +415,6 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024, banp.save(cache_path + "_dec_tgt.npy", dec_targets)np.save(cache_path + "_loss_mask.npy", loss_mask)np.save(cache_path + "_kept_idx.npy", kept_indices)- tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]- _gcs_cache_upload(cache_id, tc_suffixes)print(f"Cached {len(enc_inputs):,} tool-call pairs to {CACHE_DIR}/{cache_id}")return enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices@@ -727,107 +453,34 @@ def load_tool_call_audio(split="train", max_samples=None):return indices-_GCS_SHARD_ROWS = 5000-_GCS_N_SHARDS = 19-_shard_cache = {}---def _load_shard(shard_idx):- """Load a single arrow shard, from local or GCS. Caches in memory."""- if shard_idx in _shard_cache:- return _shard_cache[shard_idx]-- fname = f"data-{shard_idx:05d}-of-{_GCS_N_SHARDS:05d}.arrow"-- local_path = os.path.join(LOCAL_UNIFIED_DIR, fname)- if os.path.exists(local_path):- import pyarrow as pa- source = pa.memory_map(local_path, "r")- try:- reader = pa.ipc.open_file(source)- except pa.lib.ArrowInvalid:- source.seek(0)- reader = pa.ipc.open_stream(source)- tbl = reader.read_all()- from datasets import Dataset- ds = Dataset(tbl)- ds = _set_audio_backend(ds)- _shard_cache[shard_idx] = ds- return ds-- import subprocess- shm_dir = os.path.join(CACHE_DIR, "arrow_shards")- os.makedirs(shm_dir, exist_ok=True)- dest = os.path.join(shm_dir, fname)- if not os.path.exists(dest):- gcs_src = f"{GCS_DATASET_PATH}/{fname}"- subprocess.run(- ["gcloud", "storage", "cp", gcs_src, dest],- capture_output=True, text=True,- )- if not os.path.exists(dest):- raise FileNotFoundError(f"Failed to download shard {fname} from GCS")-- import pyarrow as pa- source = pa.memory_map(dest, "r")- try:- reader = pa.ipc.open_file(source)- except pa.lib.ArrowInvalid:- source.seek(0)- reader = pa.ipc.open_stream(source)- tbl = reader.read_all()- from datasets import Dataset- ds = Dataset(tbl)- ds = _set_audio_backend(ds)- _shard_cache[shard_idx] = ds- return ds---def _load_example_by_index(idx):- """Load a single example by global index, downloading only the needed shard."""- shard_idx = idx // _GCS_SHARD_ROWS- local_idx = idx % _GCS_SHARD_ROWS- shard = _load_shard(shard_idx)- if local_idx >= len(shard):- raise IndexError(f"Index {idx} out of range (shard {shard_idx} has {len(shard)} rows)")- return shard[local_idx]---def load_audio_for_index(idx):- """Load and decode audio for a single dataset index.+def load_example_with_audio(idx):+ """Load a dataset example by global index.- Returns (audio_array, sampling_rate) or (None, None) if no audio.+ Returns dict with {query, answers, tools, audio_array, sampling_rate}.+ Audio fields are None for text-only datasets."""- import io- import soundfile as sf+ ds = _load_unified_dataset()+ if idx < 0 or idx >= len(ds):+ raise IndexError(f"Index {idx} out of range (dataset has {len(ds)} rows)")+ ex = ds[idx]- ex = _load_example_by_index(idx)+ audio, sr = None, Noneaudio_val = ex.get("audio")- if audio_val is None:- return None, None-- raw_bytes = None- if isinstance(audio_val, dict):- raw_bytes = audio_val.get("bytes")- elif isinstance(audio_val, bytes):- raw_bytes = audio_val- if raw_bytes is None:- return None, None-- audio_array, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")- if audio_array.ndim > 1:- audio_array = audio_array.mean(axis=1)- return audio_array.astype(np.float32), sr+ if audio_val is not None:+ import io+ import soundfile as sf+ raw_bytes = None+ if isinstance(audio_val, dict):+ raw_bytes = audio_val.get("bytes")+ elif isinstance(audio_val, bytes):+ raw_bytes = audio_val+ if raw_bytes is not None:+ audio, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")+ if audio.ndim > 1:+ audio = audio.mean(axis=1)+ audio = audio.astype(np.float32)-def load_example_with_audio(idx):- """Load a dataset example with decoded audio for eval use.-- Downloads only the needed arrow shard (~1-4GB) instead of the full dataset (~35GB).- Returns dict with {query, answers, tools, audio_array, sampling_rate}.- """- ex = _load_example_by_index(idx)- audio, sr = load_audio_for_index(idx)return {"query": ex["query"],"answers": ex["answers"],@@ -840,7 +493,7 @@ def load_example_with_audio(idx):def compute_mel_spectrogram(audio, sr=16000, n_mels=80, n_fft=400, hop_length=160):"""Compute log-mel spectrogram using numpy/scipy. Returns (T_mel, n_mels) float32.- 25ms window (n_fft=400 at 16kHz), 10ms hop (hop_length=160) → ~100 frames/sec.+ 25ms window (n_fft=400 at 16kHz), 10ms hop (hop_length=160) -> ~100 frames/sec."""from scipy.signal import windows as scipy_windowsfrom scipy.fft import rfft@@ -882,105 +535,6 @@ def compute_mel_spectrogram(audio, sr=16000, n_mels=80, n_fft=400, hop_length=16return log_mel.astype(np.float32)-def precompute_mels(kept_indices, n_mels=80, max_mel_len=1024, cache_id_prefix="", batch_size=None):- """Precompute mel spectrograms.-- When batch_size is None: write a single (N, max_mel_len, n_mels) memmap file.- When batch_size is set: produce per-batch shard .npy files, upload each to GCS.- Returns the cache_id.- """- n_samples = len(kept_indices)- cache_id = _mel_cache_key(cache_id_prefix, n_samples, n_mels, max_mel_len)- cache_path = os.path.join(CACHE_DIR, cache_id)-- if batch_size is None:- mel_file = cache_path + "_mels.npy"-- if os.path.exists(mel_file):- print(f" Mel cache already exists ({cache_id})")- return cache_id-- if _gcs_cache_download(cache_id, ["_mels.npy"]):- print(f" Downloaded mel cache from GCS ({cache_id})")- return cache_id-- os.makedirs(CACHE_DIR, exist_ok=True)- shape = (n_samples, max_mel_len, n_mels)- fp = np.lib.format.open_memmap(mel_file, mode='w+', dtype=np.float32, shape=shape)-- print(f" Precomputing {n_samples} mel spectrograms (n_mels={n_mels}, max_mel_len={max_mel_len})...")- for i, idx in enumerate(tqdm(kept_indices, desc=" Computing mels")):- audio, sr = load_audio_for_index(int(idx))- if audio is None:- continue- mel = compute_mel_spectrogram(audio, sr=sr, n_mels=n_mels)- if mel.shape[0] > max_mel_len:- mel = mel[:max_mel_len]- t = mel.shape[0]- fp[i, :t, :] = mel-- del fp- _gcs_cache_upload(cache_id, ["_mels.npy"])- print(f" Cached mel spectrograms to {CACHE_DIR}/{cache_id}")- return cache_id-- manifest_path = cache_path + "_mel_manifest.json"-- if os.path.exists(manifest_path):- print(f" Mel shard cache already exists ({cache_id})")- return cache_id-- if _gcs_cache_download(cache_id, ["_mel_manifest.json"]):- if os.path.exists(manifest_path):- print(f" Downloaded mel shard manifest from GCS ({cache_id})")- return cache_id-- os.makedirs(CACHE_DIR, exist_ok=True)- shard_counts = []- shard_idx = 0-- print(f" Precomputing {n_samples} mel spectrograms in shards of {batch_size}...")- for batch_start in range(0, n_samples, batch_size):- batch_end = min(batch_start + batch_size, n_samples)- batch_indices = kept_indices[batch_start:batch_end]- batch_n = len(batch_indices)-- mel_batch = np.zeros((batch_n, max_mel_len, n_mels), dtype=np.float32)-- for j, idx in enumerate(tqdm(batch_indices, desc=f" Shard {shard_idx}", leave=False)):- audio, sr = load_audio_for_index(int(idx))- if audio is None:- continue- mel = compute_mel_spectrogram(audio, sr=sr, n_mels=n_mels)- if mel.shape[0] > max_mel_len:- mel = mel[:max_mel_len]- t = mel.shape[0]- mel_batch[j, :t, :] = mel-- shard_suffix = f"_mels_{shard_idx:05d}.npy"- shard_path = os.path.join(CACHE_DIR, f"{cache_id}{shard_suffix}")- np.save(shard_path, mel_batch)-- _gcs_cache_upload(cache_id, [shard_suffix])- if os.path.exists(shard_path):- os.remove(shard_path)-- shard_counts.append(batch_n)- shard_idx += 1-- manifest = {- "n_shards": shard_idx,- "shard_counts": shard_counts,- "total_samples": n_samples,- }- with open(manifest_path, "w") as f:- _json.dump(manifest, f)-- _gcs_cache_upload(cache_id, ["_mel_manifest.json"])- print(f" Cached mel spectrograms in {shard_idx} shards ({cache_id})")- return cache_id--class ShardedMmapArray:"""Array-like wrapper over multiple .npy shard files with mmap support.@@ -1039,12 +593,11 @@ class ShardedMmapArray:def load_prepared_data(split, mmap=False):"""Load pre-tokenized .npy files. If mmap=True, returns memory-mapped arrays.- Supports both single-file and sharded (manifest-based) caches.- Tries local cache first, then GCS download. Raises FileNotFoundError- if not found (no fallback tokenization — run 'needle tokenize' first).+ Tries local cache only. Raises FileNotFoundError if not found+ (run 'needle tokenize' first).Returns dict with keys: enc_inputs, dec_inputs, dec_targets, loss_mask,- kept_indices, mel_cache_id.+ kept_indices."""meta = _load_cache_metadata(split)if meta is None:@@ -1054,52 +607,12 @@ def load_prepared_data(split, mmap=False):text_cache_id = meta["text_cache_id"]cache_path = os.path.join(CACHE_DIR, text_cache_id)- manifest_path = cache_path + "_manifest.json"-- if not os.path.exists(manifest_path):- _gcs_cache_download(text_cache_id, ["_manifest.json"])-- if os.path.exists(manifest_path):- with open(manifest_path) as f:- manifest = _json.load(f)- n_shards = manifest["n_shards"]-- if not os.path.exists(cache_path + "_kept_idx.npy"):- _gcs_cache_download(text_cache_id, ["_kept_idx.npy"])-- shard_suffixes = ["_enc", "_dec_in", "_dec_tgt", "_loss_mask"]- _gcs_download_shards(text_cache_id, n_shards, shard_suffixes)-- def _shard_paths(suffix):- return [os.path.join(CACHE_DIR, f"{text_cache_id}{suffix}_{i:05d}.npy")- for i in range(n_shards)]-- mmap_mode = "r" if mmap else None- if mmap:- result = {- "enc_inputs": ShardedMmapArray(_shard_paths("_enc"), mmap_mode="r"),- "dec_inputs": ShardedMmapArray(_shard_paths("_dec_in"), mmap_mode="r"),- "dec_targets": ShardedMmapArray(_shard_paths("_dec_tgt"), mmap_mode="r"),- "loss_mask": ShardedMmapArray(_shard_paths("_loss_mask"), mmap_mode="r"),- }- else:- result = {- "enc_inputs": np.concatenate([np.load(p) for p in _shard_paths("_enc")]),- "dec_inputs": np.concatenate([np.load(p) for p in _shard_paths("_dec_in")]),- "dec_targets": np.concatenate([np.load(p) for p in _shard_paths("_dec_tgt")]),- "loss_mask": np.concatenate([np.load(p) for p in _shard_paths("_loss_mask")]),- }-- result["kept_indices"] = np.load(cache_path + "_kept_idx.npy", mmap_mode=mmap_mode)- result["mel_cache_id"] = meta.get("mel_cache_id")- return resulttc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]if not os.path.exists(cache_path + "_enc.npy"):- if not _gcs_cache_download(text_cache_id, tc_suffixes):- raise FileNotFoundError(- f"Text cache '{text_cache_id}' not found. Run 'needle tokenize' first."- )+ raise FileNotFoundError(+ f"Text cache '{text_cache_id}' not found. Run 'needle tokenize' first."+ )mmap_mode = "r" if mmap else Nonereturn {@@ -1108,140 +621,9 @@ def load_prepared_data(split, mmap=False):"dec_targets": np.load(cache_path + "_dec_tgt.npy", mmap_mode=mmap_mode),"loss_mask": np.load(cache_path + "_loss_mask.npy", mmap_mode=mmap_mode),"kept_indices": np.load(cache_path + "_kept_idx.npy", mmap_mode=mmap_mode),- "mel_cache_id": meta.get("mel_cache_id"),}-def load_prepared_mels(mel_cache_id, mmap=False):- """Load precomputed mel .npy file(s), optionally memory-mapped.-- Supports both single-file and sharded (manifest-based) caches.- """- cache_path = os.path.join(CACHE_DIR, mel_cache_id)- manifest_path = cache_path + "_mel_manifest.json"-- if not os.path.exists(manifest_path):- _gcs_cache_download(mel_cache_id, ["_mel_manifest.json"])-- if os.path.exists(manifest_path):- with open(manifest_path) as f:- manifest = _json.load(f)- n_shards = manifest["n_shards"]-- _gcs_download_shards(mel_cache_id, n_shards, ["_mels"])-- shard_paths = [os.path.join(CACHE_DIR, f"{mel_cache_id}_mels_{i:05d}.npy")- for i in range(n_shards)]-- if mmap:- return ShardedMmapArray(shard_paths, mmap_mode="r")- else:- return np.concatenate([np.load(p) for p in shard_paths])-- mel_file = cache_path + "_mels.npy"- if not os.path.exists(mel_file):- if not _gcs_cache_download(mel_cache_id, ["_mels.npy"]):- raise FileNotFoundError(- f"Mel cache '{mel_cache_id}' not found. Run 'needle tokenize' first."- )-- mmap_mode = "r" if mmap else None- return np.load(mel_file, mmap_mode=mmap_mode)---def build_audio_augmenter(sr=16000):- """Build an audiomentations augmentation pipeline for training.-- Returns an augmenter callable or None if audiomentations is unavailable.- """- try:- import audiomentations as A- except ImportError:- print(" WARNING: audiomentations not installed — no waveform augmentation")- return None-- return A.Compose([- A.AddGaussianSNR(min_snr_db=10.0, max_snr_db=35.0, p=0.5),- A.AddGaussianNoise(min_amplitude=0.001, max_amplitude=0.015, p=0.3),- A.TimeStretch(min_rate=0.9, max_rate=1.1, p=0.3),- A.PitchShift(min_semitones=-2, max_semitones=2, p=0.3),- A.Gain(min_gain_db=-6, max_gain_db=6, p=0.4),- A.LowPassFilter(min_cutoff_freq=3000, max_cutoff_freq=7500, p=0.2),- A.HighPassFilter(min_cutoff_freq=50, max_cutoff_freq=400, p=0.2),- A.ClippingDistortion(min_percentile_threshold=0, max_percentile_threshold=5, p=0.1),- ])---def _load_mel_batch(audio_arrays, n_mels, max_mel_len, augmenter=None, sr=16000):- """Compute mel spectrograms for a batch of audio arrays.-- If augmenter is provided, applies waveform augmentation before mel computation.- """- mels = []- for audio in audio_arrays:- audio = np.array(audio, dtype=np.float32)- if audio.ndim > 1:- audio = audio.mean(axis=1)-- if augmenter is not None:- audio = augmenter(samples=audio, sample_rate=sr)-- mel = compute_mel_spectrogram(audio, sr=sr, n_mels=n_mels)-- if mel.shape[0] > max_mel_len:- mel = mel[:max_mel_len]- elif mel.shape[0] < max_mel_len:- pad_len = max_mel_len - mel.shape[0]- mel = np.pad(mel, ((0, pad_len), (0, 0)))-- mels.append(mel)-- return np.stack(mels).astype(np.float32)---def _load_audio_batch(ds_indices):- """Load and decode audio for a batch of dataset indices."""- import io- import soundfile as sf-- ds = _load_unified_dataset()- arrays = []- for idx in ds_indices:- ex = ds[int(idx)]- audio_val = ex.get("audio")- raw_bytes = None- if isinstance(audio_val, dict):- raw_bytes = audio_val.get("bytes")- elif isinstance(audio_val, bytes):- raw_bytes = audio_val- if raw_bytes is None:- arrays.append(np.zeros(16000, dtype=np.float32))- continue- audio_array, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")- if audio_array.ndim > 1:- audio_array = audio_array.mean(axis=1)- arrays.append(audio_array.astype(np.float32))- return arrays---def get_speech_batches(mel_data, dec_inputs, dec_targets, batch_size,- shuffle=True, loss_mask=None):- """Yield speech batches from precomputed mel data.-- mel_data: array of shape (N, max_mel_len, n_mels), possibly memory-mapped.- Uses per-batch fancy indexing to avoid copying full arrays.- """- n = len(mel_data)- indices = np.random.permutation(n) if shuffle else np.arange(n)-- for i in range(0, n - batch_size + 1, batch_size):- idx = indices[i : i + batch_size]- batch = (np.array(mel_data[idx]), np.array(dec_inputs[idx]), np.array(dec_targets[idx]))- if loss_mask is not None:- batch = batch + (np.array(loss_mask[idx]),)- yield batch--class PrefetchIterator:"""Generic prefetch wrapper: runs any batch-generating callable in a background thread."""diff --git a/src/test.py b/src/eval.pysimilarity index 65%rename from src/test.pyrename to src/eval.pyindex 93502d2..a21ea5a 100644--- a/src/test.py+++ b/src/eval.py@@ -7,7 +7,7 @@ import jax.numpy as jnpimport numpy as npimport optax-from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs, load_tool_call_audio, load_example_with_audio+from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairsfrom .model import (EncoderDecoderTransformer,TransformerConfig,@@ -168,112 +168,18 @@ def compute_wer(hypotheses, references):return total_edits / max(total_ref_words, 1)-def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=512):+def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=512, max_enc_len=512):"""Generate tool-call predictions and compute structured metrics."""import json- from .run import generate+ from .run import generate_batchfrom .data import load_tool_callsds = load_tool_calls("validation", max_samples=num_samples)- total = 0- exact_match = 0- tp_names = 0 # predicted name in reference- fp_names = 0 # predicted name not in reference- fn_names = 0 # reference name not predicted- tp_calls = 0- fp_calls = 0- fn_calls = 0- json_parse_errors = 0- empty_ref = 0- empty_pred = 0-- samples = []-- for i, ex in enumerate(ds):- ref_text = ex["answers"]-- pred_text = generate(- model, params, tokenizer, ex["query"],- tools=ex["tools"], max_gen_len=max_gen_len, seed=i, stream=False,- ).strip()-- try:- ref_calls = json.loads(ref_text)- except (json.JSONDecodeError, TypeError):- ref_calls = []-- try:- pred_calls = json.loads(pred_text)- if not isinstance(pred_calls, list):- pred_calls = [pred_calls] if isinstance(pred_calls, dict) else []- except (json.JSONDecodeError, TypeError):- json_parse_errors += 1- pred_calls = []-- total += 1- if i < 10:- samples.append((ex["query"][:80], ref_text[:120], pred_text[:120]))-- if len(ref_calls) == 0:- empty_ref += 1- if len(pred_calls) == 0:- empty_pred += 1-- if json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):- exact_match += 1-- ref_names = [c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c]- pred_names = [c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c]- ref_name_set = set(ref_names)- pred_name_set = set(pred_names)- tp_names += len(pred_name_set & ref_name_set)- fp_names += len(pred_name_set - ref_name_set)- fn_names += len(ref_name_set - pred_name_set)-- def call_key(c):- if not isinstance(c, dict):- return None- return json.dumps({"name": c.get("name"), "arguments": c.get("arguments")}, sort_keys=True)-- ref_keys = set(call_key(c) for c in ref_calls)- pred_keys = set(call_key(c) for c in pred_calls)- ref_keys.discard(None)- pred_keys.discard(None)- tp_calls += len(pred_keys & ref_keys)- fp_calls += len(pred_keys - ref_keys)- fn_calls += len(ref_keys - pred_keys)-- name_precision = tp_names / max(tp_names + fp_names, 1)- name_recall = tp_names / max(tp_names + fn_names, 1)- name_f1 = 2 * name_precision * name_recall / max(name_precision + name_recall, 1e-9)-- call_precision = tp_calls / max(tp_calls + fp_calls, 1)- call_recall = tp_calls / max(tp_calls + fn_calls, 1)- call_f1 = 2 * call_precision * call_recall / max(call_precision + call_recall, 1e-9)-- return {- "num_samples": total,- "exact_match": exact_match / max(total, 1),- "json_parse_rate": 1.0 - json_parse_errors / max(total, 1),- "name_precision": name_precision,- "name_recall": name_recall,- "name_f1": name_f1,- "call_precision": call_precision,- "call_recall": call_recall,- "call_f1": call_f1,- "empty_ref_pct": empty_ref / max(total, 1),- "empty_pred_pct": empty_pred / max(total, 1),- "samples": samples,- }---def benchmark_voice_tool_calls(model, params, tokenizer, num_samples=100, max_gen_len=512):- """Generate tool-call predictions from audio and compute structured metrics."""- import json- from .run import generate_from_audio-- indices = load_tool_call_audio("validation", max_samples=num_samples)+ # Batch-generate all predictions at once+ queries = [ex["query"] for ex in ds]+ tools_list = [ex["tools"] for ex in ds]+ all_preds = generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=max_gen_len, max_enc_len=max_enc_len)total = 0exact_match = 0@@ -288,16 +194,14 @@ def benchmark_voice_tool_calls(model, params, tokenizer, num_samples=100, max_geempty_pred = 0samples = []- for i, ds_idx in enumerate(indices):- pair = load_example_with_audio(ds_idx)- if pair["audio_array"] is None:- continue- ref_text = pair["answers"]+ def call_key(c):+ if not isinstance(c, dict):+ return None+ return json.dumps({"name": c.get("name"), "arguments": c.get("arguments")}, sort_keys=True)- pred_text = generate_from_audio(- model, params, tokenizer, pair["audio_array"], sr=pair["sampling_rate"],- tools=pair["tools"], max_gen_len=max_gen_len, seed=i, stream=False,- ).strip()+ for i, (ex, pred_text) in enumerate(zip(ds, all_preds)):+ ref_text = ex["answers"]+ pred_text = pred_text.strip()try:ref_calls = json.loads(ref_text)@@ -314,7 +218,7 @@ def benchmark_voice_tool_calls(model, params, tokenizer, num_samples=100, max_getotal += 1if i < 10:- samples.append((pair["query"][:80], ref_text[:120], pred_text[:120]))+ samples.append((ex["query"][:80], ref_text[:120], pred_text[:120]))if len(ref_calls) == 0:empty_ref += 1@@ -324,23 +228,14 @@ def benchmark_voice_tool_calls(model, params, tokenizer, num_samples=100, max_geif json.dumps(pred_calls, sort_keys=True) == json.dumps(ref_calls, sort_keys=True):exact_match += 1- ref_names = [c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c]- pred_names = [c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c]- ref_name_set = set(ref_names)- pred_name_set = set(pred_names)+ ref_name_set = {c["name"] for c in ref_calls if isinstance(c, dict) and "name" in c}+ pred_name_set = {c["name"] for c in pred_calls if isinstance(c, dict) and "name" in c}tp_names += len(pred_name_set & ref_name_set)fp_names += len(pred_name_set - ref_name_set)fn_names += len(ref_name_set - pred_name_set)- def call_key(c):- if not isinstance(c, dict):- return None- return json.dumps({"name": c.get("name"), "arguments": c.get("arguments")}, sort_keys=True)-- ref_keys = set(call_key(c) for c in ref_calls)- pred_keys = set(call_key(c) for c in pred_calls)- ref_keys.discard(None)- pred_keys.discard(None)+ ref_keys = {call_key(c) for c in ref_calls} - {None}+ pred_keys = {call_key(c) for c in pred_calls} - {None}tp_calls += len(pred_keys & ref_keys)fp_calls += len(pred_keys - ref_keys)fn_calls += len(ref_keys - pred_keys)@@ -390,7 +285,7 @@ def main(args):tc = Noneif tc_samples > 0:print(f"\nevaluating tool-call accuracy ({tc_samples} samples)...")- tc = benchmark_tool_calls(model, params, tokenizer, num_samples=tc_samples, max_gen_len=args.max_gen_len)+ tc = benchmark_tool_calls(model, params, tokenizer, num_samples=tc_samples, max_gen_len=args.max_gen_len, max_enc_len=args.max_enc_len)print(f"\n ─────────────────────────────────────")print(f" Tool-Call Metrics")@@ -417,30 +312,6 @@ def main(args):print(f" P: {pred}")print()- voice_tc_samples = getattr(args, "voice_tc_samples", 50)- if voice_tc_samples > 0:- print(f"\nevaluating voice-to-tool-call ({voice_tc_samples} samples)...")- vtc = benchmark_voice_tool_calls(model, params, tokenizer, num_samples=voice_tc_samples, max_gen_len=args.max_gen_len)- print(f"\n ─── Voice-Tool-Call Metrics ─────────")- print(f" JSON parse rate {vtc['json_parse_rate']:>10.1%}")- print(f" Exact match {vtc['exact_match']:>10.1%}")- print(f" ─── Function Name ───────────────────")- print(f" Precision {vtc['name_precision']:>10.3f}")- print(f" Recall {vtc['name_recall']:>10.3f}")- print(f" F1 {vtc['name_f1']:>10.3f}")- print(f" ─── Full Call (name+args) ───────────")- print(f" Precision {vtc['call_precision']:>10.3f}")- print(f" Recall {vtc['call_recall']:>10.3f}")- print(f" F1 {vtc['call_f1']:>10.3f}")- print(f" ─────────────────────────────────────")- if vtc["samples"]:- print(f"\n samples:")- for query, ref, pred in vtc["samples"][:5]:- print(f" Q: {query}")- print(f" R: {ref}")- print(f" P: {pred}")- print()-print(f"\nmeasuring throughput ({args.throughput_runs} runs)...")throughput = measure_throughput(model, params, tokenizer, num_runs=args.throughput_runs)print(f"avg tokens: {throughput['avg_tokens_generated']:.1f}")@@ -456,12 +327,9 @@ def parse_args():parser.add_argument("--max-enc-len", type=int, default=256)parser.add_argument("--max-dec-len", type=int, default=1024)parser.add_argument("--max-gen-len", type=int, default=512)- parser.add_argument("--temperature", type=float, default=0.8)parser.add_argument("--throughput-runs", type=int, default=10)parser.add_argument("--tool-call-samples", type=int, default=200,help="Samples for tool-call accuracy eval (default: 200)")- parser.add_argument("--voice-tc-samples", type=int, default=50,- help="Samples for voice-to-tool-call eval (default: 50)")return parser.parse_args()diff --git a/src/run.py b/src/run.pyindex f029414..7bb18ef 100644--- a/src/run.py+++ b/src/run.py@@ -48,13 +48,26 @@ def load_checkpoint(path):return params, config-def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=0, stream=True, task_token_id=None):+def _build_encoder_input(tokenizer, query, tools, max_enc_len=1024):+ """Build encoder input: [query..., <tools>, tools...] truncated to max_enc_len."""+ tools_sep_id = tokenizer.tools_token_id+ q_toks = tokenizer.encode(query)+ t_toks = tokenizer.encode(tools)+ max_query = max_enc_len - 2+ if len(q_toks) > max_query:+ q_toks = q_toks[:max_query]+ remaining = max_enc_len - len(q_toks) - 1+ t_toks = t_toks[:remaining]+ return q_toks + [tools_sep_id] + t_toks+++def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, max_enc_len=1024, seed=0, stream=True, task_token_id=None):"""Generate tool-call output.- Encoder: query only.- Decoder: prefilled with [EOS, <tool_call>, tools_tokens...], then greedy decode.+ Encoder: [query_tokens..., <tools>, tools_tokens...] truncated to max_enc_len.+ Decoder: prefilled with [EOS, <tool_call>], then greedy decode."""- enc_tokens = tokenizer.encode(query)+ enc_tokens = _build_encoder_input(tokenizer, query, tools, max_enc_len)enc_input = jnp.array([enc_tokens])pad_id = tokenizer.pad_token_id@@ -66,13 +79,12 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed={"params": params}, enc_input, src_mask=src_mask, method="encode")- tools_tokens = tokenizer.encode(tools)- prefix = [eos_id, tool_call_id] + tools_tokens- prefix_len = min(len(prefix), max_gen_len)+ prefix = [eos_id, tool_call_id]+ prefix_len = 2dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)- for j, tok in enumerate(prefix[:max_gen_len]):- dec_buffer = dec_buffer.at[0, j].set(tok)+ dec_buffer = dec_buffer.at[0, 0].set(eos_id)+ dec_buffer = dec_buffer.at[0, 1].set(tool_call_id)decode_fn = _get_decode_fn(model, max_gen_len)@@ -106,6 +118,68 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=return tokenizer.decode(generated_tokens)+def generate_batch(model, params, tokenizer, queries, tools_list, max_gen_len=512, max_enc_len=1024):+ """Batch-generate tool-call outputs for multiple examples at once.++ Encoder: [query_tokens..., <tools>, tools_tokens...] per example, truncated to max_enc_len.+ Decoder: uniform prefix [EOS, <tool_call>] for all examples.++ Returns a list of decoded strings, one per example.+ """+ B = len(queries)+ pad_id = tokenizer.pad_token_id+ eos_id = tokenizer.eos_token_id+ tool_call_id = tokenizer.tool_call_token_id++ # --- Encode: [query..., <tools>, tools...] per example, truncated ---+ enc_token_lists = []+ for q, t in zip(queries, tools_list):+ enc_token_lists.append(_build_encoder_input(tokenizer, q, t, max_enc_len))+ max_enc = max(len(toks) for toks in enc_token_lists)+ enc_input = np.full((B, max_enc), pad_id, dtype=np.int32)+ for i, toks in enumerate(enc_token_lists):+ enc_input[i, :len(toks)] = toks+ enc_input = jnp.array(enc_input)+ src_mask = make_padding_mask(enc_input, pad_id)++ encoder_out = model.apply(+ {"params": params}, enc_input, src_mask=src_mask, method="encode"+ )++ # --- Decoder prefix: uniform [EOS, <tool_call>] (prefix_len=2) ---+ prefix_len = 2+ dec_buffer = np.full((B, max_gen_len), pad_id, dtype=np.int32)+ dec_buffer[:, 0] = eos_id+ dec_buffer[:, 1] = tool_call_id+ dec_buffer = jnp.array(dec_buffer)++ decode_fn = _get_decode_fn(model, max_gen_len)++ # --- Autoregressive decoding ---+ finished = [False] * B+ gen_tokens = [[] for _ in range(B)]++ logits = decode_fn(params, dec_buffer, encoder_out)++ for pos in range(prefix_len - 1, max_gen_len - 1):+ for i in range(B):+ if finished[i]:+ continue+ next_token = int(jnp.argmax(logits[i, pos]))+ if next_token == eos_id:+ finished[i] = True+ continue+ gen_tokens[i].append(next_token)+ dec_buffer = dec_buffer.at[i, pos + 1].set(next_token)++ if all(finished):+ break++ logits = decode_fn(params, dec_buffer, encoder_out)++ return [tokenizer.decode(toks) for toks in gen_tokens]++def load_audio(path, target_sr=16000):"""Load an audio file and resample to target_sr. Returns (audio_array, sr)."""import soundfile as sfdiff --git a/src/tokenize_data.py b/src/tokenize_data.pyindex 04a6b5b..1158123 100644--- a/src/tokenize_data.py+++ b/src/tokenize_data.py@@ -1,45 +1,28 @@"""Standalone tokenization pipeline: train tokenizer and pre-tokenize all data.-Trains the SentencePiece tokenizer and tokenizes train + val splits for both-text and voice data, caching everything on GCS. Running fresh overwrites all-existing tokenizer + caches.+Trains the SentencePiece tokenizer and tokenizes train + val splits,+caching everything locally.Usage:needle tokenize # full runneedle tokenize --max-samples 1000 # dev/test- needle tokenize --cleanup # delete local cache after GCS upload"""import osimport shutil-import subprocessfrom .data import (CACHE_DIR,- GCS_CACHE_PATH,- GCS_TOKENIZER_PATH,TOKENIZER_DIR,_cache_key,_save_cache_metadata,get_tokenizer,load_tool_calls,- precompute_mels,prepare_tool_call_pairs,train_tokenizer,- upload_tokenizer_to_gcs,)-def _clear_gcs_caches():- """Remove existing GCS cache and tokenizer files."""- for path in [GCS_CACHE_PATH + "/*", GCS_TOKENIZER_PATH + "*"]:- print(f"Clearing {path} ...")- subprocess.run(- ["gcloud", "storage", "rm", "-r", path],- capture_output=True, text=True,- )--def _clear_local_caches():"""Remove local .data_cache/ and tokenizer/ directories."""for d in [CACHE_DIR, TOKENIZER_DIR]:@@ -50,21 +33,16 @@ def _clear_local_caches():def tokenize(args):print("=== Clearing existing caches ===")- _clear_gcs_caches()_clear_local_caches()print("\n=== Training tokenizer ===")train_tokenizer(max_samples=args.max_samples, force=True)- upload_tokenizer_to_gcs()tokenizer = get_tokenizer()- print("\n=== Tokenizing text data + precomputing mels ===")- max_enc_len = getattr(args, "max_enc_len", 256)- max_dec_len = getattr(args, "max_dec_len", 1024)- n_mels = getattr(args, "n_mels", 80)- max_mel_len = getattr(args, "max_mel_len", 1024)- batch_size = getattr(args, "batch_size", 5000)+ print("\n=== Tokenizing text data ===")+ max_enc_len = getattr(args, "max_enc_len", 1024)+ max_dec_len = getattr(args, "max_dec_len", 512)for split in ("train", "val"):print(f"\n--- {split} split ---")@@ -75,21 +53,10 @@ def tokenize(args):)_, _, _, _, kept_indices = prepare_tool_call_pairs(ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len,- batch_size=batch_size,)text_cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len)- mel_cache_id = precompute_mels(- global_indices[kept_indices], n_mels=n_mels, max_mel_len=max_mel_len,- cache_id_prefix=split, batch_size=batch_size,- )-- _save_cache_metadata(split, text_cache_id, mel_cache_id, len(kept_indices),- max_enc_len, max_dec_len, n_mels, max_mel_len)-- if args.cleanup and os.path.exists(CACHE_DIR):- print(f"\n=== Cleaning up {CACHE_DIR}/ ===")- shutil.rmtree(CACHE_DIR)- print("Done.")+ _save_cache_metadata(split, text_cache_id, len(kept_indices),+ max_enc_len, max_dec_len)print("\n=== Tokenization pipeline complete ===")diff --git a/src/train.py b/src/train.pyindex f7bfdab..797c607 100644--- a/src/train.py+++ b/src/train.py@@ -1,4 +1,3 @@-import argparseimport mathimport osimport pickle@@ -14,9 +13,8 @@ from flax import jax_utilsfrom flax.training import train_statefrom .data import (- get_batches, get_tokenizer, get_speech_batches,- load_prepared_data, load_prepared_mels,- load_example_with_audio,+ get_batches, get_tokenizer,+ load_prepared_data, load_tool_calls,PrefetchIterator, count_batches,)from .model import (@@ -24,7 +22,6 @@ from .model import (TransformerConfig,make_causal_mask,make_padding_mask,- make_mel_padding_mask,)def _newton_schulz(G, steps=5):@@ -251,28 +248,6 @@ def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rnreturn ce_loss + z_loss + div_loss-def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask):- pad_id = 0- src_mask = make_mel_padding_mask(mel)- tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)- spec_rng, drop_rng = jax.random.split(rng)- logits, slot_div = state.apply_fn(- {"params": _quantize_params(params, group_size=_GROUP_SIZE)},- mel, tgt_in, src_mask=src_mask, tgt_mask=tgt_mask,- ffn_mask=ffn_mask,- deterministic=False,- method="forward_speech_masked",- rngs={"specaugment": spec_rng, "dropout": drop_rng},- )- logits_f32 = logits.astype(jnp.float32)- mask = loss_mask- ce_loss = jnp.sum(- optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask- ) / jnp.maximum(jnp.sum(mask), 1.0)- z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)- div_loss = 1e-4 * slot_div- return ce_loss + z_loss + div_loss-def _make_ffn_mask(batch_size, d_ff, mat_ff_widths):"""Build (batch, d_ff) prefix mask: batch split equally across widths.@@ -321,33 +296,6 @@ def _train_step_text_masked(state, ema_params, src, tgt_in, tgt_out, causal_maskreturn state, ema_params, loss, grad_norm-def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask):- ema_decay = 0.999- loss, grads = jax.value_and_grad(- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)- )(state.params)- grads = jax.lax.pmean(grads, axis_name="batch")- loss = jax.lax.pmean(loss, axis_name="batch")- grad_norm = optax.global_norm(grads)- state = state.apply_gradients(grads=grads)- ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, state.params)- return state, ema_params, loss, grad_norm---def _train_step_speech_masked(state, ema_params, mel, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng, loss_mask):- """Speech training step with fused prune mask application."""- ema_decay = 0.999- loss, grads = jax.value_and_grad(- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask)- )(state.params)- grads = jax.lax.pmean(grads, axis_name="batch")- loss = jax.lax.pmean(loss, axis_name="batch")- grad_norm = optax.global_norm(grads)- state = state.apply_gradients(grads=grads)- masked_params = jax.tree.map(lambda w, m: w * m, state.params, prune_mask)- state = state.replace(params=masked_params)- ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, masked_params)- return state, ema_params, loss, grad_normdef _make_p_train_step():@@ -358,14 +306,6 @@ def _make_p_train_step_masked():return jax.pmap(_train_step_text_masked, axis_name="batch", donate_argnums=(0, 1))-def _make_p_train_step_speech():- return jax.pmap(_train_step_speech, axis_name="batch", donate_argnums=(0, 1))---def _make_p_train_step_speech_masked():- return jax.pmap(_train_step_speech_masked, axis_name="batch", donate_argnums=(0, 1))--def _make_val_loss_fn(apply_fn):@jax.jitdef val_loss_batch(params, src, tgt_in, tgt_out, causal_mask, loss_mask):@@ -400,24 +340,6 @@ def _make_mat_val_loss_fn(apply_fn, ff_width):return val_loss_batch-def _make_speech_val_loss_fn(apply_fn):- @jax.jit- def val_loss_batch(params, mel, tgt_in, tgt_out, causal_mask, loss_mask):- pad_id = 0- src_mask = make_mel_padding_mask(mel)- tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)- logits, _, _ = apply_fn(- {"params": params}, mel, tgt_in,- src_mask=src_mask, tgt_mask=tgt_mask,- deterministic=True,- method="forward_speech_with_aux",- )- loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)- mask = loss_mask- return jnp.sum(loss * mask), jnp.sum(mask)- return val_loss_batch--def _estimate_mat_params(config, matryoshka_factor):"""Estimate parameter count of a sub-model at a given matryoshka factor.@@ -450,9 +372,6 @@ def shard_batch(batch, num_devices):def train(args):num_devices = jax.local_device_count()- no_speech = getattr(args, "no_speech", False)- n_mels = getattr(args, "n_mels", 80)- max_mel_len = getattr(args, "max_mel_len", 1024)use_wandb = getattr(args, "wandb", False)if use_wandb:@@ -460,19 +379,13 @@ def train(args):if wandb.run is None:wandb.init(project="needle-v1", config=vars(args))- total_data_steps = 4 if not no_speech else 3- step_idx = 0-- step_idx += 1- print(f"\n[{step_idx}/{total_data_steps}] Detecting devices...")+ print(f"\n[1/3] Detecting devices...")print(f" {num_devices} device(s) for data-parallel training")- step_idx += 1- print(f"\n[{step_idx}/{total_data_steps}] Loading tokenizer...")+ print(f"\n[2/3] Loading tokenizer...")tokenizer = get_tokenizer(max_samples=args.max_samples)- step_idx += 1- print(f"\n[{step_idx}/{total_data_steps}] Loading prepared data from disk (mmap)...")+ print(f"\n[3/3] Loading prepared data from disk (mmap)...")train_data = load_prepared_data("train", mmap=True)val_data = load_prepared_data("val", mmap=True)enc_inputs = train_data["enc_inputs"]@@ -485,15 +398,6 @@ def train(args):val_loss_mask = val_data["loss_mask"]print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs (memory-mapped)")- train_mels = None- val_mels = None- if not no_speech:- step_idx += 1- print(f"\n[{step_idx}/{total_data_steps}] Loading precomputed mel spectrograms (mmap)...")- train_mels = load_prepared_mels(train_data["mel_cache_id"], mmap=True)- val_mels = load_prepared_mels(val_data["mel_cache_id"], mmap=True)- print(f" {len(train_mels):,} train / {len(val_mels):,} val mel spectrograms (memory-mapped)")-effective_batch_size = args.batch_size * num_devicesresume_checkpoint = getattr(args, "checkpoint", None)@@ -516,8 +420,6 @@ def train(args):dtype=args.dtype,activation=getattr(args, "activation", "drelu"),num_memory_slots=getattr(args, "num_memory_slots", 64),- n_mels=n_mels,- dropout_rate=getattr(args, "dropout", 0.1),)global _GROUP_SIZE, _MAT_FACTORS, _MAT_FF_WIDTHS, _D_FF@@ -533,8 +435,6 @@ def train(args):n_widths = 1 + len(_MAT_FF_WIDTHS) if _MAT_FF_WIDTHS else 1p_train_step = _make_p_train_step()p_train_step_masked = _make_p_train_step_masked()- p_train_step_speech = _make_p_train_step_speech()- p_train_step_speech_masked = _make_p_train_step_speech_masked()np.random.seed(args.seed)rng = jax.random.PRNGKey(args.seed)@@ -543,11 +443,7 @@ def train(args):mat_shared_input = getattr(args, "mat_shared_input", False)unique_batch_size = effective_batch_size // n_widths if (mat_shared_input and n_widths > 1) else effective_batch_sizetext_batches_per_epoch = count_batches(len(enc_inputs), unique_batch_size)- if not no_speech and train_mels is not None:- speech_batches_per_epoch = text_batches_per_epoch- else:- speech_batches_per_epoch = 0- num_batches = text_batches_per_epoch + speech_batches_per_epoch+ num_batches = text_batches_per_epochtotal_steps = num_batches * args.epochswarmup_steps = max(1, int(total_steps * args.warmup_ratio))@@ -555,7 +451,6 @@ def train(args):muon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(num_devices)state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps)val_loss_fn = _make_val_loss_fn(state.apply_fn)- speech_vl_fn = _make_speech_val_loss_fn(state.apply_fn) if not no_speech else Noneif resume_checkpoint:state = state.replace(params=ckpt_params)@@ -577,13 +472,6 @@ def train(args):print(f" Memory slots {config.num_memory_slots:>12}")print(f" Activation {config.activation:>12}")print(f" Dtype {config.dtype:>12}")- print(f" Dropout {config.dropout_rate:>12}")- if not no_speech and train_mels is not None:- print(f" Speech {speech_batches_per_epoch} batches/epoch")- print(f" n_mels {n_mels:>12}")- print(f" max_mel_len {max_mel_len:>12}")- else:- print(f" Speech disabled")print(f" ─────────────────────────────────────")print(f" Devices {num_devices:>12}")print(f" Batch {args.batch_size:>7} x {num_devices} = {effective_batch_size}")@@ -639,113 +527,50 @@ def train(args):epoch_step = 0text_losses = []- speech_losses = []text_batch_iter = PrefetchIterator(lambda: get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size,loss_mask=train_loss_mask),prefetch=4,)- speech_batch_iter = None- if not no_speech and train_mels is not None:- speech_batch_iter = PrefetchIterator(- lambda: get_speech_batches(train_mels, dec_inputs, dec_targets, unique_batch_size,- loss_mask=train_loss_mask),- prefetch=4,- )-- steps_this_epoch = text_batches_per_epoch + speech_batches_per_epoch- text_idx = 0- speech_idx = 0- speech_loss_val = None- pbar = tqdm(range(steps_this_epoch), desc=f"Epoch {epoch + 1}/{args.epochs}")+ pbar = tqdm(range(text_batches_per_epoch), desc=f"Epoch {epoch + 1}/{args.epochs}")for step_i in pbar:t0 = time.perf_counter()- do_speech = (step_i % 2 == 1) and speech_idx < speech_batches_per_epoch- do_text = not do_speech and text_idx < text_batches_per_epoch- if not do_speech and not do_text:- if text_idx < text_batches_per_epoch:- do_text = True- elif speech_idx < speech_batches_per_epoch:- do_speech = True- else:- break-- step_grad_norm = None-- if do_text:- src, tgt_in, tgt_out, lm = next(text_batch_iter)- text_idx += 1-- if n_widths > 1 and mat_shared_input:- per_width = args.batch_size // n_widths- def _tile_for_mat(arr):- s = arr.reshape(num_devices, per_width, *arr.shape[1:])- return np.tile(s, (1, n_widths) + (1,) * (arr.ndim - 1))- src_b = _tile_for_mat(src)- tgt_in_b = _tile_for_mat(tgt_in)- tgt_out_b = _tile_for_mat(tgt_out)- lm_b = _tile_for_mat(lm)- else:- src_b = shard_batch(src, num_devices)- tgt_in_b = shard_batch(tgt_in, num_devices)- tgt_out_b = shard_batch(tgt_out, num_devices)- lm_b = shard_batch(lm, num_devices)-- rng, text_rng = jax.random.split(rng)- text_rngs = jax.random.split(text_rng, num_devices)-- if prune_mask is not None:- state, ema_params, loss, grad_norm = p_train_step_masked(- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, text_rngs, lm_b,- )- else:- state, ema_params, loss, grad_norm = p_train_step(- state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, text_rngs, lm_b,- )-- text_loss_val = float(loss[0])- text_losses.append(text_loss_val)- step_grad_norm = float(grad_norm[0])- global_step += 1-+ src, tgt_in, tgt_out, lm = next(text_batch_iter)++ if n_widths > 1 and mat_shared_input:+ per_width = args.batch_size // n_widths+ def _tile_for_mat(arr):+ s = arr.reshape(num_devices, per_width, *arr.shape[1:])+ return np.tile(s, (1, n_widths) + (1,) * (arr.ndim - 1))+ src_b = _tile_for_mat(src)+ tgt_in_b = _tile_for_mat(tgt_in)+ tgt_out_b = _tile_for_mat(tgt_out)+ lm_b = _tile_for_mat(lm)else:- mel_batch, sp_tgt_in, sp_tgt_out, sp_lm = next(speech_batch_iter)- speech_idx += 1-- if n_widths > 1 and mat_shared_input:- per_width = args.batch_size // n_widths- def _tile_sp(arr):- s = arr.reshape(num_devices, per_width, *arr.shape[1:])- return np.tile(s, (1, n_widths) + (1,) * (arr.ndim - 1))- mel_b = _tile_sp(mel_batch)- sp_tgt_in_b = _tile_sp(sp_tgt_in)- sp_tgt_out_b = _tile_sp(sp_tgt_out)- sp_lm_b = _tile_sp(sp_lm)- else:- mel_b = shard_batch(mel_batch, num_devices)- sp_tgt_in_b = shard_batch(sp_tgt_in, num_devices)- sp_tgt_out_b = shard_batch(sp_tgt_out, num_devices)- sp_lm_b = shard_batch(sp_lm, num_devices)-- rng, spec_rng = jax.random.split(rng)- spec_rngs = jax.random.split(spec_rng, num_devices)-- if prune_mask is not None:- state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech_masked(- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, prune_mask, text_ffn_mask, spec_rngs, sp_lm_b,- )- else:- state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech(- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, text_ffn_mask, spec_rngs, sp_lm_b,- )- speech_loss_val = float(sp_loss[0])- speech_losses.append(speech_loss_val)- step_grad_norm = float(sp_grad_norm[0])- text_loss_val = text_losses[-1] if text_losses else float("nan")- global_step += 1+ src_b = shard_batch(src, num_devices)+ tgt_in_b = shard_batch(tgt_in, num_devices)+ tgt_out_b = shard_batch(tgt_out, num_devices)+ lm_b = shard_batch(lm, num_devices)++ rng, text_rng = jax.random.split(rng)+ text_rngs = jax.random.split(text_rng, num_devices)++ if prune_mask is not None:+ state, ema_params, loss, grad_norm = p_train_step_masked(+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, prune_mask, text_ffn_mask, text_rngs, lm_b,+ )+ else:+ state, ema_params, loss, grad_norm = p_train_step(+ state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, text_rngs, lm_b,+ )++ text_loss_val = float(loss[0])+ text_losses.append(text_loss_val)+ step_grad_norm = float(grad_norm[0])+ global_step += 1if epoch == weight_prune_epoch and not gradual_sparsify_done:epoch_step += 1@@ -772,7 +597,6 @@ def train(args):del _eval_paramspostfix = {- "speech_loss": f"{speech_loss_val:.4f}" if speech_loss_val is not None else "-","text_loss": f"{text_loss_val:.4f}","text_ppl": f"{last_val_ppl:.2f}" if last_val_ppl is not None else "?",}@@ -792,8 +616,6 @@ def train(args):"train/tokens_per_sec": tokens_per_batch / dt,"train/step": global_step,}- if speech_loss_val is not None:- log_dict["train/speech_loss"] = speech_loss_valif epoch == weight_prune_epoch and not gradual_sparsify_done:log_dict["train/scheduled_sparsity"] = current_sparsityif global_step % eval_every == 0 or global_step == total_steps:@@ -801,8 +623,6 @@ def train(args):wandb.log(log_dict)text_batch_iter.close()- if speech_batch_iter is not None:- speech_batch_iter.close()if epoch == weight_prune_epoch and not gradual_sparsify_done:gradual_sparsify_done = True@@ -863,17 +683,6 @@ def train(args):mat_results[f] = (float(math.exp(min(avg, 20))),_estimate_mat_params(config, f), config.d_ff // f)- speech_val_ppl = None- if speech_vl_fn is not None and val_mels is not None:- sp_total_loss, sp_total_toks = 0.0, 0.0- for sp_batch in get_speech_batches(val_mels, val_dec_in, val_dec_tgt, args.batch_size,- shuffle=False, loss_mask=val_loss_mask):- vl, vt = speech_vl_fn(eval_params, sp_batch[0], sp_batch[1], sp_batch[2], val_causal, sp_batch[3])- sp_total_loss += float(vl)- sp_total_toks += float(vt)- speech_val_loss = sp_total_loss / max(sp_total_toks, 1)- speech_val_ppl = float(math.exp(min(speech_val_loss, 20)))-params_np = jax.tree.map(np.array, eval_params)total_params = sum(x.size for x in jax.tree.leaves(params_np))near_zero = sum(int(np.sum(np.abs(x) < 1e-6)) for x in jax.tree.leaves(params_np))@@ -885,12 +694,12 @@ def train(args):pickle.dump({"params": params_np, "config": config.__dict__}, f)del params_np- from .test import measure_throughput- from .run import generate, generate_from_audio+ from .eval import measure_throughput+ from .run import generate_batchtp = measure_throughput(eval_model, eval_params, tokenizer, num_runs=5)- _val_start = len(enc_inputs)val_kept = val_data["kept_indices"]+ val_ds = load_tool_calls("val", max_samples=args.max_samples)# Pick 5 display samples (shuffled for diversity): 4 with tool calls, 1 withoutsample_rng = np.random.RandomState(epoch + 7)@@ -899,40 +708,17 @@ def train(args):for k in sample_pool:if len(display_with) >= 4 and len(display_without) >= 1:break- ds_idx = int(val_kept[k]) + _val_start- pair = load_example_with_audio(ds_idx)- is_empty = pair["answers"].strip() in ("", "[]")+ local_idx = int(val_kept[k])+ ex = val_ds[local_idx]+ is_empty = ex["answers"].strip() in ("", "[]")if not is_empty and len(display_with) < 4:- display_with.append((ds_idx, pair))+ display_with.append(ex)elif is_empty and len(display_without) < 1:- display_without.append((ds_idx, pair))+ display_without.append(ex)display_pairs = display_with + display_without- unified_samples = []- for i, (ds_idx, pair) in enumerate(display_pairs):- text_pred = generate(- eval_model, eval_params, tokenizer, pair["query"],- tools=pair["tools"], max_gen_len=args.max_dec_len, seed=i, stream=False,- ).strip()-- voice_pred = None- if not no_speech and pair["audio_array"] is not None:- voice_pred = generate_from_audio(- eval_model, eval_params, tokenizer, pair["audio_array"], sr=pair["sampling_rate"],- tools=pair["tools"], max_gen_len=args.max_dec_len, seed=i, stream=False,- ).strip()-- unified_samples.append({- "query": pair["query"],- "tools": pair["tools"],- "ref": pair["answers"],- "text": text_pred,- "voice": voice_pred,- })-+ # Collect all eval examples, then batch-generate everything at onceimport json as _json_mod- tc_n, tc_exact, tc_name_tp, tc_name_fp, tc_name_fn = 0, 0, 0, 0, 0- tc_call_tp, tc_call_fp, tc_call_fn, tc_parse_err = 0, 0, 0, 0tc_with_n = 25tc_without_n = 5tc_rng = np.random.RandomState(epoch + 42)@@ -941,26 +727,51 @@ def train(args):for k in tc_pool:if len(tc_with) >= tc_with_n and len(tc_without) >= tc_without_n:break- ds_idx = int(val_kept[k]) + _val_start- pair = load_example_with_audio(ds_idx)- ref_text = pair["answers"].strip()+ local_idx = int(val_kept[k])+ ex = val_ds[local_idx]+ ref_text = ex["answers"].strip()is_empty = ref_text in ("", "[]")if not is_empty and len(tc_with) < tc_with_n:- tc_with.append((ds_idx, pair))+ tc_with.append(ex)elif is_empty and len(tc_without) < tc_without_n:- tc_without.append((ds_idx, pair))+ tc_without.append(ex)tc_eval_pairs = tc_with + tc_without+ # Batch-generate: display samples + tc eval samples in one pass+ all_eval_examples = display_pairs + tc_eval_pairs+ eval_gen_len = min(args.max_dec_len, 512)+ all_preds = generate_batch(+ eval_model, eval_params, tokenizer,+ [ex["query"] for ex in all_eval_examples],+ [ex["tools"] for ex in all_eval_examples],+ max_gen_len=eval_gen_len,+ max_enc_len=args.max_enc_len,+ )++ # Split results back+ display_preds = all_preds[:len(display_pairs)]+ tc_preds = all_preds[len(display_pairs):]++ unified_samples = []+ for ex, pred in zip(display_pairs, display_preds):+ unified_samples.append({+ "query": ex["query"],+ "tools": ex["tools"],+ "ref": ex["answers"],+ "text": pred.strip(),+ })++ tc_n, tc_exact, tc_name_tp, tc_name_fp, tc_name_fn = 0, 0, 0, 0, 0+ tc_call_tp, tc_call_fp, tc_call_fn, tc_parse_err = 0, 0, 0, 0+ tc_args_correct, tc_args_total = 0, 0+def _call_key(c):if not isinstance(c, dict): return Nonereturn _json_mod.dumps({"name": c.get("name"), "arguments": c.get("arguments")}, sort_keys=True)- for i, (ds_idx, pair) in enumerate(tc_eval_pairs):- ref_text = pair["answers"].strip()- pred_text = generate(- eval_model, eval_params, tokenizer, pair["query"],- tools=pair["tools"], max_gen_len=args.max_dec_len, seed=ds_idx, stream=False,- ).strip()+ for ex, pred_text in zip(tc_eval_pairs, tc_preds):+ ref_text = ex["answers"].strip()+ pred_text = pred_text.strip()try:ref_calls = _json_mod.loads(ref_text)except (ValueError, TypeError):@@ -986,6 +797,18 @@ def train(args):tc_call_fp += len(pk - rk)tc_call_fn += len(rk - pk)+ # Argument correctness: for name-matched calls, check if arguments match+ ref_by_name = {}+ for c in ref_calls:+ if isinstance(c, dict) and "name" in c:+ ref_by_name.setdefault(c["name"], []).append(c.get("arguments", {}))+ for c in pred_calls:+ if isinstance(c, dict) and "name" in c and c["name"] in ref_by_name:+ tc_args_total += 1+ pred_args = _json_mod.dumps(c.get("arguments", {}), sort_keys=True)+ if any(pred_args == _json_mod.dumps(ra, sort_keys=True) for ra in ref_by_name[c["name"]]):+ tc_args_correct += 1+tc_metrics = {}if tc_n > 0:tc_metrics["parse_rate"] = 1.0 - tc_parse_err / tc_n@@ -996,19 +819,15 @@ def train(args):cp_ = tc_call_tp + tc_call_fpcr_ = tc_call_tp + tc_call_fntc_metrics["call_f1"] = 2 * tc_call_tp / max(cp_ + cr_, 1)+ tc_metrics["args_acc"] = tc_args_correct / max(tc_args_total, 1)del eval_params- final_speech_loss = speech_losses[-1] if speech_losses else Noneprint(f"\n ─────────────────────────────────────")print(f" Epoch {epoch + 1}/{args.epochs}")print(f" ─────────────────────────────────────")print(f" Text loss {final_loss:>12.4f}")print(f" Text val ppl {last_val_ppl:>12.2f}")- if final_speech_loss is not None:- print(f" Speech loss {final_speech_loss:>12.4f}")- if speech_val_ppl is not None:- print(f" Speech val ppl {speech_val_ppl:>12.2f}")print(f" Quant val ppl {quant_val_ppl:>12.2f} (INT4 g{_GROUP_SIZE})")print(f" Sparsity {sparsity:>11.2f}% ({near_zero:,}/{total_params:,})")if mat_results:@@ -1022,9 +841,10 @@ def train(args):if tc_metrics:print(f" ─── Tool-Call Accuracy ({tc_n} samples) ──")print(f" JSON parse {tc_metrics['parse_rate']:>10.1%}")+ print(f" Name F1 {tc_metrics['name_f1']:>10.1%}")+ print(f" Args acc {tc_metrics['args_acc']:>10.1%}")+ print(f" Call F1 {tc_metrics['call_f1']:>10.1%}")print(f" Exact match {tc_metrics['exact_match']:>10.1%}")- print(f" Name F1 {tc_metrics['name_f1']:>10.3f}")- print(f" Call F1 {tc_metrics['call_f1']:>10.3f}")print(f" ─────────────────────────────────────")print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")print(f" Latency {tp['avg_latency_s']:>11.3f}s")@@ -1038,8 +858,6 @@ def train(args):print(f" Tools: {tools_short}")print(f" Ref: {s['ref'][:200] or '[]'}")print(f" Text: {s['text'][:200] or '(empty)'}")- if s["voice"] is not None:- print(f" Voice: {s['voice'][:200] or '(empty)'}")if j < len(unified_samples) - 1:print()print(f" ─────────────────────────────────────")@@ -1054,10 +872,6 @@ def train(args):"epoch/weight_sparsity": sparsity,"epoch": epoch + 1,}- if final_speech_loss is not None:- log_dict["epoch/speech_loss"] = final_speech_loss- if speech_val_ppl is not None:- log_dict["epoch/speech_val_ppl"] = speech_val_pplfor factor, (mat_ppl, mat_params, _) in mat_results.items():log_dict[f"epoch/mat_ppl_{factor}x"] = mat_ppllog_dict[f"epoch/mat_params_{factor}x"] = mat_params@@ -1065,6 +879,7 @@ def train(args):log_dict["epoch/tc_parse_rate"] = tc_metrics["parse_rate"]log_dict["epoch/tc_exact_match"] = tc_metrics["exact_match"]log_dict["epoch/tc_name_f1"] = tc_metrics["name_f1"]+ log_dict["epoch/tc_args_acc"] = tc_metrics["args_acc"]log_dict["epoch/tc_call_f1"] = tc_metrics["call_f1"]wandb.log(log_dict)