Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Dataset setup complete

HenryNdubuaku07b2178f49d3
diff --git a/README.md b/README.md
index 38b549b..875a9c3 100644
--- a/README.md
+++ b/README.md
@@ -71,55 +71,63 @@
strided DW conv · mat factors · INT4 QAT · Muon + AdamW
text + speech encoder · <tool_call> task routing
- Data Preprocessing
- ──────────────────
-
- Text (Tool-Call Dataset) Speech (Voice-Tool-Call)
- ──────────────────────── ────────────────────────
-
- Unified dataset Unified dataset
- (GCS / local) (GCS / local)
- │ │
- ▼ ▼
- ┌──────────────┐ ┌────────────────┐
- │ SentencePiece│ │ HF Audio col │
- │ BPE tokenize │ │ decode (16kHz) │
- │ (8192 vocab) │ └───────┬────────┘
- └──────┬───────┘ │
- │ ▼
- ▼ ┌────────────────┐
- ┌──────────────┐ │ Augmentation │
- │ Encoder: │ │ (noise, pitch, │
- │ query tokens │ │ stretch, gain)│
- └──────┬───────┘ └───────┬────────┘
- │ │
- ▼ ▼
- ┌──────────────────┐ ┌────────────────┐
- │ Decoder: │ │ Log-mel spec │
- │ [EOS]<tool_call> │ │ 80 bins, 25ms │
- │ + tools tokens │ │ window, 10ms │
- │ + answer tokens │ │ hop (~100 f/s) │
- │ → dec_in │ └───────┬────────┘
- ├──────────────────┤ │
- │ <tool_call> │ ▼
- │ + tools tokens │ ┌────────────────┐
- │ + answer tokens │ │ Pad/trunc to │
- │ + [EOS] │ │ max_mel_len │
- │ → dec_tgt │ └───────┬────────┘
- ├──────────────────┤ │
- │ Loss mask: 1.0 │ ▼
- │ on answer + EOS │ ┌────────────────┐
- │ only (not tools) │ │ Same decoder │
- └──────────────────┘ │ format as text │
- └────────────────┘
- Both cached as
- .npy in .data_cache/ Mel computed on-the-fly
- (augment before mel)
- ┌──────────────────────┐
- │ Interleaved batches: │
- │ 1:1 text / speech │ Eval: voice-to-tool-call on
- │ alternation │ 3 val samples per epoch
- └──────────────────────┘
+ Data Pipeline (needle tokenize → needle train)
+ ───────────────────────────────────────────────
+
+ ┌─────────────────────────────────────────────────────────────┐
+ │ needle tokenize │
+ │ │
+ │ Unified dataset (GCS / local) │
+ │ │ │
+ │ ├──── Text ──────────────── Speech ─────────┐ │
+ │ ▼ ▼ │
+ │ ┌──────────────┐ ┌────────────────┐ │
+ │ │ SentencePiece│ │ HF Audio col │ │
+ │ │ BPE tokenize │ │ decode (16kHz) │ │
+ │ │ (8192 vocab) │ └───────┬────────┘ │
+ │ └──────┬───────┘ │ │
+ │ │ ▼ │
+ │ ▼ ┌────────────────┐ │
+ │ ┌──────────────────┐ │ Log-mel spec │ │
+ │ │ enc_inputs.npy │ │ 80 bins, 25ms │ │
+ │ │ dec_inputs.npy │ │ window, 10ms │ │
+ │ │ dec_targets.npy │ │ hop (~100 f/s) │ │
+ │ │ loss_mask.npy │ └───────┬────────┘ │
+ │ │ kept_idx.npy │ │ │
+ │ └──────┬───────────┘ ▼ │
+ │ │ ┌────────────────┐ │
+ │ │ │ mels.npy │ │
+ │ │ │ (N,T,80) f32 │ │
+ │ │ │ writable mmap │ │
+ │ │ └───────┬────────┘ │
+ │ │ │ │
+ │ └────────────┬───────────────────────────┘ │
+ │ ▼ │
+ │ {split}_metadata.json │
+ │ .data_cache/ + GCS upload │
+ └─────────────────────────────────────────────────────────────┘
+ │
+ ▼
+ ┌─────────────────────────────────────────────────────────────┐
+ │ needle train │
+ │ │
+ │ load_prepared_data(mmap=True) │
+ │ load_prepared_mels(mmap=True) │
+ │ │ │
+ │ ▼ │
+ │ ┌──────────────────────┐ ┌──────────────────────┐ │
+ │ │ PrefetchIterator │ │ PrefetchIterator │ │
+ │ │ text batches (4) │ │ speech batches (4) │ │
+ │ │ mmap → per-batch idx │ │ mmap → per-batch idx │ │
+ │ └──────────┬───────────┘ └──────────┬───────────┘ │
+ │ └───────────┬──────────────┘ │
+ │ ▼ │
+ │ 1:1 text/speech │
+ │ alternation │
+ │ │
+ │ ~42MB RAM (8 prefetched batches) │
+ │ SpecAugment for speech regularization │
+ └─────────────────────────────────────────────────────────────┘
```
## Usage
@@ -165,6 +173,14 @@ needle [command]
│ --n-mels INT Mel frequency bins (default: 80) │
│ --max-speech-samples INT Max voice-tool-call samples │
│ │
+ │ tokenize │
+ │ --max-samples INT Limit samples per split (dev/test) │
+ │ --cleanup Delete local cache after GCS upload │
+ │ --n-mels INT Mel frequency bins (default: 80) │
+ │ --max-mel-len INT Max mel frames (default: 1024) │
+ │ --max-enc-len INT Max encoder seq len (default: 256) │
+ │ --max-dec-len INT Max decoder seq len (default: 1024) │
+ │ │
│ run │
│ --checkpoint PATH Path to model checkpoint (required) │
│ --query STR Query text for tool-call generation │
diff --git a/scripts/build_dataset.py b/scripts/build_dataset.py
index 72f9385..c057bcb 100644
--- a/scripts/build_dataset.py
+++ b/scripts/build_dataset.py
@@ -144,11 +144,11 @@ def _load_text_dataset(output_path, max_samples):
return combined, existing_ds
-def _find_rows_needing_tts(combined, existing_ds):
- """Identify which rows lack audio by inspecting the raw Arrow column.
+def _find_rows_needing_tts(combined, existing_ds, output_path):
+ """Identify which rows lack audio by checking the cache dir and existing dataset.
- Accesses the underlying Arrow table directly to avoid triggering
- HF audio decoding (which requires torchcodec).
+ First checks {output_path}/.audio_cache/ for WAV files from prior runs,
+ then falls back to inspecting the Arrow column in existing_ds.
Returns:
Set of row indices that need TTS synthesis.
@@ -156,57 +156,89 @@ def _find_rows_needing_tts(combined, existing_ds):
n = len(combined)
needs_tts = set(range(n))
+ cache_dir = os.path.join(output_path, ".audio_cache")
+ if os.path.isdir(cache_dir):
+ cached = 0
+ for fname in os.listdir(cache_dir):
+ if fname.endswith(".wav"):
+ try:
+ idx = int(fname[:-4])
+ if idx < n:
+ needs_tts.discard(idx)
+ cached += 1
+ except ValueError:
+ pass
+ if cached:
+ logger.info(f" {cached} rows found in audio cache")
+
if existing_ds is not None and len(existing_ds) == n and "audio" in existing_ds.column_names:
audio_column = existing_ds.data.column("audio")
for i in range(n):
- if audio_column[i].as_py() is not None:
+ if i in needs_tts and audio_column[i].as_py() is not None:
needs_tts.discard(i)
- logger.info(f" {n - len(needs_tts)} rows already have audio, {len(needs_tts)} need TTS")
+ logger.info(f" {n - len(needs_tts)} rows already have audio, {len(needs_tts)} need TTS")
return needs_tts
-def _recover_existing_audio(existing_ds, n, needs_tts):
- """Extract audio bytes and voice names from a previously saved dataset.
+def _recover_existing_audio(existing_ds, n, needs_tts, output_path):
+ """Recover audio file paths from the cache dir and existing dataset.
- Reads raw Arrow data to avoid triggering audio decoding.
+ Stores file paths (not bytes) to keep memory usage low.
Returns:
Tuple of (audio_data list, voice_names list).
+ audio_data entries are file paths (str) or None.
"""
audio_data = [None] * n
voice_names = [""] * n
+ cache_dir = os.path.join(output_path, ".audio_cache")
+ if os.path.isdir(cache_dir):
+ for i in range(n):
+ if i not in needs_tts:
+ wav_path = os.path.join(cache_dir, f"{i}.wav")
+ if os.path.exists(wav_path):
+ audio_data[i] = wav_path
+
if existing_ds is not None and len(existing_ds) == n and "audio" in existing_ds.column_names:
audio_column = existing_ds.data.column("audio")
voice_col = existing_ds["voice"] if "voice" in existing_ds.column_names else None
for i in range(n):
- if i not in needs_tts:
+ if i not in needs_tts and audio_data[i] is None:
raw = audio_column[i].as_py()
if isinstance(raw, dict) and raw.get("bytes"):
- audio_data[i] = raw["bytes"]
- elif isinstance(raw, dict) and raw.get("path"):
- with open(raw["path"], "rb") as f:
- audio_data[i] = f.read()
- if voice_col is not None:
- voice_names[i] = voice_col[i] or ""
+ wav_path = os.path.join(cache_dir, f"{i}.wav")
+ os.makedirs(cache_dir, exist_ok=True)
+ with open(wav_path, "wb") as f:
+ f.write(raw["bytes"])
+ audio_data[i] = wav_path
+ elif isinstance(raw, dict) and raw.get("path") and os.path.exists(raw["path"]):
+ audio_data[i] = raw["path"]
+ if voice_col is not None and i not in needs_tts:
+ voice_names[i] = voice_col[i] or ""
return audio_data, voice_names
-def _synthesize_missing(combined, needs_tts, audio_data, voice_names, workers, rate_limit):
- """Run TTS synthesis for all rows that lack audio.
+def _synthesize_missing(combined, needs_tts, audio_data, voice_names, workers, rate_limit, output_path):
+ """Run TTS synthesis, writing each WAV to disk immediately.
- Uses a thread pool with rate-limited submission to stay within
- GCP Cloud TTS API quotas.
+ Audio bytes are written to {output_path}/.audio_cache/{idx}.wav as they
+ arrive, keeping RAM usage constant regardless of dataset size. The
+ audio_data list stores file paths (not bytes) for later assembly.
"""
if not needs_tts:
return
from google.cloud import texttospeech
+ cache_dir = os.path.join(output_path, ".audio_cache")
+ os.makedirs(cache_dir, exist_ok=True)
+
queries_to_synth = [(i, combined[i]["query"]) for i in sorted(needs_tts)]
logger.info(f"Synthesizing TTS for {len(queries_to_synth)} rows with {workers} workers...")
+ logger.info(f" Audio cache: {cache_dir}/")
def _worker(item):
idx, query = item
@@ -224,7 +256,10 @@ def _synthesize_missing(combined, needs_tts, audio_data, voice_names, workers, r
for future in as_completed(futures):
idx, wav_bytes, voice_name, err = future.result()
if wav_bytes is not None:
- audio_data[idx] = wav_bytes
+ wav_path = os.path.join(cache_dir, f"{idx}.wav")
+ with open(wav_path, "wb") as f:
+ f.write(wav_bytes)
+ audio_data[idx] = wav_path
voice_names[idx] = voice_name
success += 1
else:
@@ -244,16 +279,28 @@ def _backfill_voice_names(voice_names):
def _build_and_save(combined, audio_data, voice_names, output_path):
- """Assemble the unified HF Dataset and save to disk.
+ """Assemble the unified HF Dataset from file paths and save to disk.
+
+ audio_data entries are file paths (str) or None. Each file is read
+ one at a time during assembly to avoid loading all audio into RAM.
Returns:
The saved Dataset.
"""
n = len(combined)
- logger.info("Building unified HF Dataset...")
+ has_audio = sum(1 for x in audio_data if x is not None)
+ logger.info(f"Building unified HF Dataset... ({has_audio}/{n} with audio)")
+
+ audio_column = []
+ for path in audio_data:
+ if path is not None and os.path.exists(path):
+ audio_column.append(path)
+ else:
+ audio_column.append(None)
+
ds_dict = {
"query": [combined[i]["query"] for i in range(n)],
- "audio": audio_data,
+ "audio": audio_column,
"tools": [combined[i]["tools"] for i in range(n)],
"answers": [combined[i]["answers"] for i in range(n)],
"source": [combined[i]["source"] for i in range(n)],
@@ -269,21 +316,13 @@ def _build_and_save(combined, audio_data, voice_names, output_path):
return unified
-def _train_and_save_tokenizer(max_samples):
- """Train a SentencePiece tokenizer on the unified dataset."""
- from src.data import train_tokenizer, TOKENIZER_PREFIX
- logger.info("Training tokenizer on unified dataset...")
- model_path = train_tokenizer(max_samples=max_samples)
- logger.info(f"Tokenizer ready at {model_path}")
-
-
def _upload_to_gcs(output_path, gcs_upload):
- """Upload the dataset and tokenizer to GCS via gcloud CLI."""
- from src.data import TOKENIZER_PREFIX
-
+ """Upload the dataset to GCS via gcloud CLI."""
logger.info(f"Uploading dataset to {gcs_upload}...")
+ import glob as globmod
+ files = globmod.glob(os.path.join(output_path, "*"))
result = subprocess.run(
- ["gcloud", "storage", "cp", "-r", output_path, gcs_upload],
+ ["gcloud", "storage", "cp"] + files + [gcs_upload + "/"],
capture_output=True, text=True,
)
if result.returncode == 0:
@@ -292,20 +331,6 @@ def _upload_to_gcs(output_path, gcs_upload):
logger.error(f"Dataset upload failed: {result.stderr}")
sys.exit(1)
- gcs_bucket = gcs_upload.rsplit("/", 1)[0]
- gcs_tokenizer_path = f"{gcs_bucket}/tokenizer/"
- logger.info(f"Uploading tokenizer to {gcs_tokenizer_path}...")
- tokenizer_dir = os.path.dirname(TOKENIZER_PREFIX)
- result = subprocess.run(
- ["gcloud", "storage", "cp", "-r", tokenizer_dir, gcs_tokenizer_path],
- capture_output=True, text=True,
- )
- if result.returncode == 0:
- logger.info(f"Tokenizer upload complete: {gcs_tokenizer_path}")
- else:
- logger.error(f"Tokenizer upload failed: {result.stderr}")
- sys.exit(1)
-
def main():
"""Entry point for the unified dataset build pipeline."""
@@ -320,12 +345,11 @@ def main():
args = parser.parse_args()
combined, existing_ds = _load_text_dataset(args.output, args.max_samples)
- needs_tts = _find_rows_needing_tts(combined, existing_ds)
- audio_data, voice_names = _recover_existing_audio(existing_ds, len(combined), needs_tts)
- _synthesize_missing(combined, needs_tts, audio_data, voice_names, args.workers, args.rate_limit)
+ needs_tts = _find_rows_needing_tts(combined, existing_ds, args.output)
+ audio_data, voice_names = _recover_existing_audio(existing_ds, len(combined), needs_tts, args.output)
+ _synthesize_missing(combined, needs_tts, audio_data, voice_names, args.workers, args.rate_limit, args.output)
_backfill_voice_names(voice_names)
_build_and_save(combined, audio_data, voice_names, args.output)
- _train_and_save_tokenizer(args.max_samples)
if args.gcs_upload:
_upload_to_gcs(args.output, args.gcs_upload)
diff --git a/src/cli.py b/src/cli.py
index 928ae95..ff83a7f 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -55,6 +55,20 @@ def main():
p.add_argument("--max-speech-samples", type=int, default=None,
help="Max voice-tool-call training samples (default: all)")
+ p = sub.add_parser("tokenize", add_help=False)
+ p.add_argument("--max-samples", type=int, default=None,
+ help="Limit samples per split (for dev/test)")
+ p.add_argument("--cleanup", action="store_true",
+ help="Delete local .data_cache/ after GCS upload")
+ p.add_argument("--n-mels", type=int, default=80,
+ help="Number of mel frequency bins (default: 80)")
+ p.add_argument("--max-mel-len", type=int, default=1024,
+ help="Max mel spectrogram frames (default: 1024)")
+ p.add_argument("--max-enc-len", type=int, default=256,
+ help="Max encoder sequence length (default: 256)")
+ p.add_argument("--max-dec-len", type=int, default=1024,
+ help="Max decoder sequence length (default: 1024)")
+
p = sub.add_parser("run", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
p.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")
@@ -119,7 +133,10 @@ def main():
print(HELP)
sys.exit(0)
- if args.command == "train":
+ if args.command == "tokenize":
+ from .tokenize_data import tokenize
+ tokenize(args)
+ elif args.command == "train":
if getattr(args, "full", False):
args.d_model = 1536
args.num_heads = 24
diff --git a/src/data.py b/src/data.py
index a09d62f..3353430 100644
--- a/src/data.py
+++ b/src/data.py
@@ -1,6 +1,9 @@
import hashlib
+import json as _json
import multiprocessing as mp
import os
+import queue
+import threading
os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"
os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"
os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "1"
@@ -40,29 +43,26 @@ def _load_unified_dataset():
if _unified_dataset_cache is not None:
return _unified_dataset_cache
- # Try GCS first
- try:
- ds = load_from_disk(GCS_DATASET_PATH)
- print(f"Loaded unified dataset from GCS ({len(ds)} rows)")
- # Cache locally for faster subsequent loads
- if not os.path.exists(LOCAL_UNIFIED_DIR):
- ds.save_to_disk(LOCAL_UNIFIED_DIR)
- print(f" Cached locally to {LOCAL_UNIFIED_DIR}")
+ if os.path.exists(LOCAL_UNIFIED_DIR) and any(
+ f.endswith(".arrow") for f in os.listdir(LOCAL_UNIFIED_DIR)
+ ):
+ ds = load_from_disk(LOCAL_UNIFIED_DIR)
+ print(f"Loaded unified dataset from {LOCAL_UNIFIED_DIR} ({len(ds)} rows)")
_unified_dataset_cache = _set_audio_backend(ds)
return _unified_dataset_cache
- except Exception:
- pass
- # Fallback to local
- if os.path.exists(LOCAL_UNIFIED_DIR):
- ds = load_from_disk(LOCAL_UNIFIED_DIR)
- print(f"Loaded unified dataset from {LOCAL_UNIFIED_DIR} ({len(ds)} rows)")
+ try:
+ ds = load_from_disk(GCS_DATASET_PATH)
+ print(f"Loaded unified dataset from GCS ({len(ds)} rows)")
_unified_dataset_cache = _set_audio_backend(ds)
return _unified_dataset_cache
+ except Exception as e:
+ gcs_err = e
raise FileNotFoundError(
f"Unified dataset not found. Run scripts/build_dataset.py first, "
- f"or ensure GCS path {GCS_DATASET_PATH} is accessible (pip install gcsfs)."
+ f"or ensure GCS path {GCS_DATASET_PATH} is accessible (pip install gcsfs). "
+ f"GCS error: {gcs_err}"
)
@@ -142,10 +142,10 @@ def _tokenize_chunk(texts):
return [_worker_sp.Encode(t, out_type=int)[:_worker_max_len] for t in texts]
-def train_tokenizer(vocab_size=8192, max_samples=None):
+def train_tokenizer(vocab_size=8192, max_samples=None, force=False):
"""Train a SentencePiece BPE tokenizer on tool-calling corpus."""
model_path = TOKENIZER_PREFIX + ".model"
- if os.path.exists(model_path):
+ if os.path.exists(model_path) and not force:
print(f"Tokenizer already exists at {model_path}")
return model_path
@@ -190,6 +190,19 @@ def train_tokenizer(vocab_size=8192, max_samples=None):
GCS_TOKENIZER_PATH = "gs://cactus-dataset/tokenizer/"
+def upload_tokenizer_to_gcs():
+ """Upload local tokenizer files to GCS."""
+ import subprocess
+ import glob as globmod
+ tok_files = globmod.glob(os.path.join(TOKENIZER_DIR, "*"))
+ if tok_files:
+ subprocess.run(
+ ["gcloud", "storage", "cp"] + tok_files + [GCS_TOKENIZER_PATH],
+ capture_output=True, text=True,
+ )
+ print(f"Uploaded tokenizer to {GCS_TOKENIZER_PATH}")
+
+
def get_tokenizer(max_samples=None):
model_path = TOKENIZER_PREFIX + ".model"
if not os.path.exists(model_path):
@@ -215,6 +228,34 @@ def _download_tokenizer_from_gcs():
return False
+GCS_CACHE_PATH = "gs://cactus-dataset/cache"
+
+
+def _gcs_cache_download(cache_id, suffixes):
+ """Try downloading cached files from GCS. Returns True if all found."""
+ import subprocess
+ os.makedirs(CACHE_DIR, exist_ok=True)
+ cache_path = os.path.join(CACHE_DIR, cache_id)
+ srcs = [f"{GCS_CACHE_PATH}/{cache_id}{s}" for s in suffixes]
+ subprocess.run(
+ ["gcloud", "storage", "cp"] + srcs + [CACHE_DIR + "/"],
+ capture_output=True, text=True,
+ )
+ return all(os.path.exists(cache_path + s) for s in suffixes)
+
+
+def _gcs_cache_upload(cache_id, suffixes):
+ """Upload cached files to GCS."""
+ import subprocess
+ cache_path = os.path.join(CACHE_DIR, cache_id)
+ files = [cache_path + s for s in suffixes if os.path.exists(cache_path + s)]
+ if files:
+ subprocess.run(
+ ["gcloud", "storage", "cp"] + files + [GCS_CACHE_PATH + "/"],
+ capture_output=True, text=True,
+ )
+
+
def load_tool_calls(split="train", max_samples=None):
"""Load tool-calling dataset, splitting 90/10 for train/val."""
ds = _load_unified_dataset()
@@ -243,6 +284,52 @@ def _cache_key(prefix, n_samples, max_enc_len, max_dec_len):
return hashlib.md5(key.encode()).hexdigest()[:12]
+def _mel_cache_key(prefix, n_samples, n_mels, max_mel_len):
+ tok_hash = _tokenizer_hash()
+ key = f"mel_{prefix}_{tok_hash}_{n_samples}_{n_mels}_{max_mel_len}"
+ return hashlib.md5(key.encode()).hexdigest()[:12]
+
+
+def _save_cache_metadata(split, text_cache_id, mel_cache_id, n_samples,
+ max_enc_len, max_dec_len, n_mels, max_mel_len):
+ """Save metadata JSON for a split, upload to GCS."""
+ os.makedirs(CACHE_DIR, exist_ok=True)
+ meta = {
+ "split": split,
+ "text_cache_id": text_cache_id,
+ "mel_cache_id": mel_cache_id,
+ "n_samples": n_samples,
+ "max_enc_len": max_enc_len,
+ "max_dec_len": max_dec_len,
+ "n_mels": n_mels,
+ "max_mel_len": max_mel_len,
+ }
+ meta_path = os.path.join(CACHE_DIR, f"{split}_metadata.json")
+ with open(meta_path, "w") as f:
+ _json.dump(meta, f)
+ _gcs_cache_upload(f"{split}_metadata", [".json"])
+
+
+def _load_cache_metadata(split):
+ """Load metadata JSON from local or GCS. Returns dict or None."""
+ meta_path = os.path.join(CACHE_DIR, f"{split}_metadata.json")
+ if os.path.exists(meta_path):
+ with open(meta_path) as f:
+ return _json.load(f)
+ # Try GCS
+ import subprocess
+ os.makedirs(CACHE_DIR, exist_ok=True)
+ gcs_path = f"{GCS_CACHE_PATH}/{split}_metadata.json"
+ subprocess.run(
+ ["gcloud", "storage", "cp", gcs_path, meta_path],
+ capture_output=True, text=True,
+ )
+ if os.path.exists(meta_path):
+ with open(meta_path) as f:
+ return _json.load(f)
+ return None
+
+
def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):
"""Prepare tool-call encoder-decoder pairs with <tool_call> task token.
@@ -256,15 +343,25 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):
cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len)
cache_path = os.path.join(CACHE_DIR, cache_id)
- if os.path.exists(cache_path + "_enc.npy"):
- print(f"Loading cached tool-call data ({cache_id})...")
+ tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]
+
+ def _load_tc_cache():
return (
np.load(cache_path + "_enc.npy"),
np.load(cache_path + "_dec_in.npy"),
np.load(cache_path + "_dec_tgt.npy"),
np.load(cache_path + "_loss_mask.npy"),
+ np.load(cache_path + "_kept_idx.npy"),
)
+ if os.path.exists(cache_path + "_enc.npy"):
+ print(f"Loading cached tool-call data ({cache_id})...")
+ return _load_tc_cache()
+
+ if _gcs_cache_download(cache_id, tc_suffixes):
+ print(f"Downloaded tool-call cache from GCS ({cache_id})...")
+ return _load_tc_cache()
+
pad_id = tokenizer.pad_token_id
eos_id = tokenizer.eos_token_id
tool_call_id = tokenizer.tool_call_token_id
@@ -277,7 +374,6 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):
model_path = TOKENIZER_PREFIX + ".model"
chunk_size = max(1, len(enc_texts) // (num_workers * 4))
- # Tokenize queries (encoder)
enc_chunks = [enc_texts[i:i + chunk_size] for i in range(0, len(enc_texts), chunk_size)]
print(f"Tokenizing encoder inputs ({num_workers} workers)...")
with mp.Pool(num_workers, initializer=_init_worker,
@@ -285,7 +381,6 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):
enc_results = pool.map(_tokenize_chunk, enc_chunks)
all_enc_tokens = [tok for chunk in enc_results for tok in chunk]
- # Tokenize tools (decoder prefix) — reserve 2 for BOS + <tool_call>, rest for tools + answer
tools_chunks = [tools_texts[i:i + chunk_size] for i in range(0, len(tools_texts), chunk_size)]
print(f"Tokenizing tools ({num_workers} workers)...")
with mp.Pool(num_workers, initializer=_init_worker,
@@ -293,7 +388,6 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):
tools_results = pool.map(_tokenize_chunk, tools_chunks)
all_tools_tokens = [tok for chunk in tools_results for tok in chunk]
- # Tokenize answers (decoder generation target)
ans_chunks = [ans_texts[i:i + chunk_size] for i in range(0, len(ans_texts), chunk_size)]
print(f"Tokenizing answers ({num_workers} workers)...")
with mp.Pool(num_workers, initializer=_init_worker,
@@ -313,13 +407,10 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):
t_tok = all_tools_tokens[i]
a_tok = all_ans_tokens[i]
- # Prefix length: BOS + <tool_call> + tools_tokens
prefix_len = 2 + len(t_tok)
- # Total decoder length: prefix + answer + EOS
total_dec = prefix_len + len(a_tok) + 1
if total_dec > max_dec_len:
- # Truncate tools to fit answer + EOS
available_for_tools = max_dec_len - 2 - len(a_tok) - 1
if available_for_tools < 1:
skipped += 1
@@ -348,96 +439,107 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):
dec_targets[i, prefix_len - 1:prefix_len - 1 + al] = a_tok
dec_targets[i, prefix_len - 1 + al] = eos_id
- # Loss mask: 1 on answer positions + EOS only
loss_mask[i, prefix_len - 1:prefix_len - 1 + al + 1] = 1.0
if skipped > 0:
- # Remove skipped rows (all-pad)
keep = enc_inputs[:, 0] != pad_id
+ kept_indices = np.where(keep)[0].astype(np.int64)
enc_inputs = enc_inputs[keep]
dec_inputs = dec_inputs[keep]
dec_targets = dec_targets[keep]
loss_mask = loss_mask[keep]
print(f" Skipped {skipped} examples (too long for max_dec_len={max_dec_len})")
+ else:
+ kept_indices = np.arange(n, dtype=np.int64)
- # Save cache
os.makedirs(CACHE_DIR, exist_ok=True)
np.save(cache_path + "_enc.npy", enc_inputs)
np.save(cache_path + "_dec_in.npy", dec_inputs)
np.save(cache_path + "_dec_tgt.npy", dec_targets)
np.save(cache_path + "_loss_mask.npy", loss_mask)
+ np.save(cache_path + "_kept_idx.npy", kept_indices)
+ _gcs_cache_upload(cache_id, tc_suffixes)
print(f"Cached {len(enc_inputs):,} tool-call pairs to {CACHE_DIR}/{cache_id}")
- return enc_inputs, dec_inputs, dec_targets, loss_mask
+ return enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices
def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True, loss_mask=None):
n = len(enc_inputs)
- if shuffle:
- perm = np.random.permutation(n)
- enc_inputs = enc_inputs[perm]
- dec_inputs = dec_inputs[perm]
- dec_targets = dec_targets[perm]
- if loss_mask is not None:
- loss_mask = loss_mask[perm]
-
+ indices = np.random.permutation(n) if shuffle else np.arange(n)
for i in range(0, n - batch_size + 1, batch_size):
- batch = (enc_inputs[i : i + batch_size], dec_inputs[i : i + batch_size], dec_targets[i : i + batch_size])
+ idx = indices[i : i + batch_size]
+ batch = (np.array(enc_inputs[idx]), np.array(dec_inputs[idx]), np.array(dec_targets[idx]))
if loss_mask is not None:
- batch = batch + (loss_mask[i : i + batch_size],)
+ batch = batch + (np.array(loss_mask[idx]),)
yield batch
def load_tool_call_audio(split="train", max_samples=None):
- """Load tool-call dataset entries that have TTS audio.
-
- Applies the same 90/10 split as load_tool_calls, then filters to entries
- where the audio column is not None.
+ """Return dataset-global indices for the given split.
- Returns list of dicts: {query, answers, tools, audio_bytes, sampling_rate}.
+ Applies the same 90/10 split as load_tool_calls. Audio is NOT loaded into memory.
"""
ds = _load_unified_dataset()
n = len(ds)
if split in ("validation", "val", "test"):
- indices = list(range(int(n * 0.9), n))
+ start, end = int(n * 0.9), n
elif split == "train":
- indices = list(range(int(n * 0.9)))
+ start, end = 0, int(n * 0.9)
else:
- indices = list(range(n))
+ start, end = 0, n
+
+ indices = list(range(start, end))
+ if max_samples:
+ indices = indices[:max_samples]
+ return indices
+
+def load_audio_for_index(idx):
+ """Load and decode audio for a single dataset index.
+
+ Returns (audio_array, sampling_rate) or (None, None) if no audio.
+ """
import io
import soundfile as sf
- pairs = []
- for idx in indices:
- ex = ds[idx]
- audio_val = ex.get("audio")
- if audio_val is None:
- continue
- # With decode=False, audio_val is {"bytes": b"...", "path": ...}
- raw_bytes = None
- if isinstance(audio_val, dict):
- raw_bytes = audio_val.get("bytes")
- elif isinstance(audio_val, bytes):
- raw_bytes = audio_val
- if raw_bytes is None:
- continue
- audio_array, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")
- if audio_array.ndim > 1:
- audio_array = audio_array.mean(axis=1)
- pairs.append({
- "query": ex["query"],
- "answers": ex["answers"],
- "tools": ex["tools"],
- "audio_array": audio_array.astype(np.float32),
- "sampling_rate": sr,
- })
- if max_samples and len(pairs) >= max_samples:
- break
+ ds = _load_unified_dataset()
+ ex = ds[idx]
+ audio_val = ex.get("audio")
+ if audio_val is None:
+ return None, None
+
+ raw_bytes = None
+ if isinstance(audio_val, dict):
+ raw_bytes = audio_val.get("bytes")
+ elif isinstance(audio_val, bytes):
+ raw_bytes = audio_val
+ if raw_bytes is None:
+ return None, None
+
+ audio_array, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")
+ if audio_array.ndim > 1:
+ audio_array = audio_array.mean(axis=1)
+ return audio_array.astype(np.float32), sr
- return pairs
+
+def load_example_with_audio(idx):
+ """Load a dataset example with decoded audio for eval use.
+
+ Returns dict with {query, answers, tools, audio_array, sampling_rate}.
+ """
+ ds = _load_unified_dataset()
+ ex = ds[idx]
+ audio, sr = load_audio_for_index(idx)
+ return {
+ "query": ex["query"],
+ "answers": ex["answers"],
+ "tools": ex["tools"],
+ "audio_array": audio,
+ "sampling_rate": sr,
+ }
def compute_mel_spectrogram(audio, sr=16000, n_mels=80, n_fft=400, hop_length=160):
@@ -488,10 +590,99 @@ def compute_mel_spectrogram(audio, sr=16000, n_mels=80, n_fft=400, hop_length=16
return log_mel.astype(np.float32)
-def _speech_cache_key(n_samples, n_mels, max_mel_len, max_dec_len):
- tok_hash = _tokenizer_hash()
- key = f"speech_{tok_hash}_{n_samples}_{n_mels}_{max_mel_len}_{max_dec_len}"
- return hashlib.md5(key.encode()).hexdigest()[:12]
+def precompute_mels(kept_indices, n_mels=80, max_mel_len=1024, cache_id_prefix=""):
+ """Precompute mel spectrograms -> write to disk via writable memmap.
+
+ Creates {cache_id}_mels.npy of shape (N, max_mel_len, n_mels) float32.
+ Processes one sample at a time, writing to the memmap in-place.
+ Uploads to GCS when done. Returns the cache_id.
+ """
+ n_samples = len(kept_indices)
+ cache_id = _mel_cache_key(cache_id_prefix, n_samples, n_mels, max_mel_len)
+ cache_path = os.path.join(CACHE_DIR, cache_id)
+ mel_file = cache_path + "_mels.npy"
+
+ if os.path.exists(mel_file):
+ print(f" Mel cache already exists ({cache_id})")
+ return cache_id
+
+ if _gcs_cache_download(cache_id, ["_mels.npy"]):
+ print(f" Downloaded mel cache from GCS ({cache_id})")
+ return cache_id
+
+ os.makedirs(CACHE_DIR, exist_ok=True)
+ shape = (n_samples, max_mel_len, n_mels)
+ # Create the file with header so np.load(mmap_mode=...) works later
+ fp = np.lib.format.open_memmap(mel_file, mode='w+', dtype=np.float32, shape=shape)
+
+ print(f" Precomputing {n_samples} mel spectrograms (n_mels={n_mels}, max_mel_len={max_mel_len})...")
+ for i, idx in enumerate(tqdm(kept_indices, desc=" Computing mels")):
+ audio, sr = load_audio_for_index(int(idx))
+ if audio is None:
+ # Leave zeros for missing audio
+ continue
+ mel = compute_mel_spectrogram(audio, sr=sr, n_mels=n_mels)
+ if mel.shape[0] > max_mel_len:
+ mel = mel[:max_mel_len]
+ t = mel.shape[0]
+ fp[i, :t, :] = mel
+
+ del fp # flush to disk
+ _gcs_cache_upload(cache_id, ["_mels.npy"])
+ print(f" Cached mel spectrograms to {CACHE_DIR}/{cache_id}")
+ return cache_id
+
+
+def load_prepared_data(split, mmap=False):
+ """Load pre-tokenized .npy files. If mmap=True, returns memory-mapped arrays.
+
+ Tries local cache first, then GCS download. Raises FileNotFoundError
+ if not found (no fallback tokenization — run 'needle tokenize' first).
+
+ Returns dict with keys: enc_inputs, dec_inputs, dec_targets, loss_mask,
+ kept_indices, mel_cache_id.
+ """
+ meta = _load_cache_metadata(split)
+ if meta is None:
+ raise FileNotFoundError(
+ f"No prepared data found for split '{split}'. Run 'needle tokenize' first."
+ )
+
+ text_cache_id = meta["text_cache_id"]
+ cache_path = os.path.join(CACHE_DIR, text_cache_id)
+ tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]
+
+ # Try local, then GCS
+ if not os.path.exists(cache_path + "_enc.npy"):
+ if not _gcs_cache_download(text_cache_id, tc_suffixes):
+ raise FileNotFoundError(
+ f"Text cache '{text_cache_id}' not found. Run 'needle tokenize' first."
+ )
+
+ mmap_mode = "r" if mmap else None
+ return {
+ "enc_inputs": np.load(cache_path + "_enc.npy", mmap_mode=mmap_mode),
+ "dec_inputs": np.load(cache_path + "_dec_in.npy", mmap_mode=mmap_mode),
+ "dec_targets": np.load(cache_path + "_dec_tgt.npy", mmap_mode=mmap_mode),
+ "loss_mask": np.load(cache_path + "_loss_mask.npy", mmap_mode=mmap_mode),
+ "kept_indices": np.load(cache_path + "_kept_idx.npy", mmap_mode=mmap_mode),
+ "mel_cache_id": meta.get("mel_cache_id"),
+ }
+
+
+def load_prepared_mels(mel_cache_id, mmap=False):
+ """Load precomputed mel .npy file, optionally memory-mapped."""
+ cache_path = os.path.join(CACHE_DIR, mel_cache_id)
+ mel_file = cache_path + "_mels.npy"
+
+ if not os.path.exists(mel_file):
+ if not _gcs_cache_download(mel_cache_id, ["_mels.npy"]):
+ raise FileNotFoundError(
+ f"Mel cache '{mel_cache_id}' not found. Run 'needle tokenize' first."
+ )
+
+ mmap_mode = "r" if mmap else None
+ return np.load(mel_file, mmap_mode=mmap_mode)
def build_audio_augmenter(sr=16000):
@@ -517,99 +708,6 @@ def build_audio_augmenter(sr=16000):
])
-def prepare_voice_tool_call_pairs(pairs, tokenizer, n_mels=80, max_mel_len=1024, max_dec_len=1024):
- """Prepare voice-tool-call decoder arrays with <tool_call> task token.
-
- Caches only decoder tokens + loss mask + audio arrays (NOT mels).
- Mel spectrograms are computed on-the-fly by get_speech_batches to allow
- online waveform augmentation during training.
-
- Returns (audio_arrays, dec_inputs, dec_targets, loss_mask).
- """
-
- cache_id = _speech_cache_key(len(pairs), n_mels, max_mel_len, max_dec_len)
- cache_path = os.path.join(CACHE_DIR, cache_id)
- arrays_file = cache_path + "_arrays.npy"
- if os.path.exists(arrays_file):
- print(f"Loading cached voice-tool-call data ({cache_id})...")
- return (
- np.load(arrays_file, allow_pickle=True),
- np.load(cache_path + "_dec_in.npy"),
- np.load(cache_path + "_dec_tgt.npy"),
- np.load(cache_path + "_loss_mask.npy"),
- )
-
- pad_id = tokenizer.pad_token_id
- eos_id = tokenizer.eos_token_id
- tool_call_id = tokenizer.tool_call_token_id
-
- audio_arrays = []
- dec_inputs_list = []
- dec_targets_list = []
- loss_mask_list = []
-
- skipped = 0
- for pair in tqdm(pairs, desc="Processing voice-tool-call"):
- t_tok = tokenizer.encode(pair["tools"])[:max_dec_len - 2]
- a_tok = tokenizer.encode(pair["answers"])
-
- prefix_len = 2 + len(t_tok)
- total_dec = prefix_len + len(a_tok) + 1
-
- if total_dec > max_dec_len:
- available_for_tools = max_dec_len - 2 - len(a_tok) - 1
- if available_for_tools < 1:
- skipped += 1
- continue
- t_tok = t_tok[:available_for_tools]
- prefix_len = 2 + len(t_tok)
-
- dec_in = np.full(max_dec_len, pad_id, dtype=np.int32)
- dec_in[0] = eos_id
- dec_in[1] = tool_call_id
- tl = len(t_tok)
- if tl > 0:
- dec_in[2:2 + tl] = t_tok
- al = len(a_tok)
- if al > 0:
- dec_in[prefix_len:prefix_len + al] = a_tok
-
- # Decoder target: [<tool_call>, tools..., answer..., EOS]
- dec_tgt = np.full(max_dec_len, pad_id, dtype=np.int32)
- dec_tgt[0] = tool_call_id
- if tl > 0:
- dec_tgt[1:1 + tl] = t_tok
- if al > 0:
- dec_tgt[prefix_len - 1:prefix_len - 1 + al] = a_tok
- dec_tgt[prefix_len - 1 + al] = eos_id
-
- # Loss mask: 1 on answer positions + EOS only
- lm = np.zeros(max_dec_len, dtype=np.float32)
- lm[prefix_len - 1:prefix_len - 1 + al + 1] = 1.0
-
- audio_arrays.append(pair["audio_array"])
- dec_inputs_list.append(dec_in)
- dec_targets_list.append(dec_tgt)
- loss_mask_list.append(lm)
-
- if skipped > 0:
- print(f" Skipped {skipped} examples (too long for max_dec_len={max_dec_len})")
-
- audio_arrays = np.array(audio_arrays, dtype=object)
- dec_inputs = np.stack(dec_inputs_list)
- dec_targets = np.stack(dec_targets_list)
- loss_mask = np.stack(loss_mask_list)
-
- os.makedirs(CACHE_DIR, exist_ok=True)
- np.save(arrays_file, audio_arrays)
- np.save(cache_path + "_dec_in.npy", dec_inputs)
- np.save(cache_path + "_dec_tgt.npy", dec_targets)
- np.save(cache_path + "_loss_mask.npy", loss_mask)
- print(f"Cached {len(audio_arrays):,} voice-tool-call pairs to {CACHE_DIR}/{cache_id}")
-
- return audio_arrays, dec_inputs, dec_targets, loss_mask
-
-
def _load_mel_batch(audio_arrays, n_mels, max_mel_len, augmenter=None, sr=16000):
"""Compute mel spectrograms for a batch of audio arrays.
@@ -637,25 +735,91 @@ def _load_mel_batch(audio_arrays, n_mels, max_mel_len, augmenter=None, sr=16000)
return np.stack(mels).astype(np.float32)
-def get_speech_batches(audio_arrays, dec_inputs, dec_targets, batch_size,
- shuffle=True, loss_mask=None, n_mels=80, max_mel_len=1024,
- augmenter=None):
- """Yield speech batches with on-the-fly mel computation and optional augmentation.
+def _load_audio_batch(ds_indices):
+ """Load and decode audio for a batch of dataset indices."""
+ import io
+ import soundfile as sf
- Args:
- audio_arrays: array of audio waveform arrays (numpy float32)
- dec_inputs, dec_targets, loss_mask: pre-tokenized decoder arrays
- augmenter: audiomentations Compose pipeline (None = no augmentation)
+ ds = _load_unified_dataset()
+ arrays = []
+ for idx in ds_indices:
+ ex = ds[int(idx)]
+ audio_val = ex.get("audio")
+ raw_bytes = None
+ if isinstance(audio_val, dict):
+ raw_bytes = audio_val.get("bytes")
+ elif isinstance(audio_val, bytes):
+ raw_bytes = audio_val
+ if raw_bytes is None:
+ arrays.append(np.zeros(16000, dtype=np.float32))
+ continue
+ audio_array, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")
+ if audio_array.ndim > 1:
+ audio_array = audio_array.mean(axis=1)
+ arrays.append(audio_array.astype(np.float32))
+ return arrays
+
+
+def get_speech_batches(mel_data, dec_inputs, dec_targets, batch_size,
+ shuffle=True, loss_mask=None):
+ """Yield speech batches from precomputed mel data.
+
+ mel_data: array of shape (N, max_mel_len, n_mels), possibly memory-mapped.
+ Uses per-batch fancy indexing to avoid copying full arrays.
"""
- n = len(audio_arrays)
+ n = len(mel_data)
indices = np.random.permutation(n) if shuffle else np.arange(n)
for i in range(0, n - batch_size + 1, batch_size):
- batch_idx = indices[i : i + batch_size]
- batch_audio = audio_arrays[batch_idx]
- mel_batch = _load_mel_batch(batch_audio, n_mels, max_mel_len, augmenter=augmenter)
-
- batch = (mel_batch, dec_inputs[batch_idx], dec_targets[batch_idx])
+ idx = indices[i : i + batch_size]
+ batch = (np.array(mel_data[idx]), np.array(dec_inputs[idx]), np.array(dec_targets[idx]))
if loss_mask is not None:
- batch = batch + (loss_mask[batch_idx],)
+ batch = batch + (np.array(loss_mask[idx]),)
yield batch
+
+
+class PrefetchIterator:
+ """Generic prefetch wrapper: runs any batch-generating callable in a background thread."""
+
+ def __init__(self, generator_fn, prefetch=4):
+ """generator_fn: callable that returns an iterable of batches."""
+ self._queue = queue.Queue(maxsize=prefetch)
+ self._stop = threading.Event()
+ self._generator_fn = generator_fn
+ self._thread = threading.Thread(target=self._produce, daemon=True)
+ self._thread.start()
+
+ def _produce(self):
+ try:
+ for batch in self._generator_fn():
+ if self._stop.is_set():
+ return
+ self._queue.put(batch)
+ self._queue.put(None) # sentinel
+ except Exception as e:
+ self._queue.put(e)
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ item = self._queue.get()
+ if item is None:
+ raise StopIteration
+ if isinstance(item, Exception):
+ raise item
+ return item
+
+ def close(self):
+ self._stop.set()
+ while not self._queue.empty():
+ try:
+ self._queue.get_nowait()
+ except queue.Empty:
+ break
+ self._thread.join(timeout=5)
+
+
+def count_batches(n_samples, batch_size):
+ """Return the number of full batches for a dataset of n_samples."""
+ return n_samples // batch_size
diff --git a/src/run.py b/src/run.py
index a9a00ba..eb3cb9b 100644
--- a/src/run.py
+++ b/src/run.py
@@ -16,6 +16,30 @@ from .model import (
)
+_decode_fn_cache = {}
+
+
+def _get_decode_fn(model, max_gen_len):
+ """Return a JIT-compiled decode function, cached by (model, max_gen_len).
+
+ params is an explicit argument (not closed over) so the same compiled
+ function can be reused across calls with different params.
+ """
+ key = (id(model), max_gen_len)
+ if key not in _decode_fn_cache:
+ tgt_mask = make_causal_mask(max_gen_len)
+
+ @jax.jit
+ def decode_step(params, dec_buffer, encoder_out):
+ return model.apply(
+ {"params": params}, dec_buffer, encoder_out,
+ self_mask=tgt_mask, method="decode",
+ )
+
+ _decode_fn_cache[key] = decode_step
+ return _decode_fn_cache[key]
+
+
def load_checkpoint(path):
with open(path, "rb") as f:
data = pickle.load(f)
@@ -42,28 +66,15 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=
{"params": params}, enc_input, src_mask=src_mask, method="encode"
)
- # Build decoder prefix: [BOS, <tool_call>, tools_tokens...]
tools_tokens = tokenizer.encode(tools)
prefix = [eos_id, tool_call_id] + tools_tokens
- prefix_len = len(prefix)
+ prefix_len = min(len(prefix), max_gen_len)
- # Fixed-size buffer so JAX compiles once
- tgt_mask = make_causal_mask(max_gen_len)
dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
- # Fill prefix
for j, tok in enumerate(prefix[:max_gen_len]):
dec_buffer = dec_buffer.at[0, j].set(tok)
- @jax.jit
- def decode_step(dec_buffer, encoder_out):
- logits = model.apply(
- {"params": params},
- dec_buffer,
- encoder_out,
- self_mask=tgt_mask,
- method="decode",
- )
- return logits
+ decode_fn = _get_decode_fn(model, max_gen_len)
generated_tokens = []
@@ -71,7 +82,7 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=
sys.stdout.write(f"\n")
sys.stdout.flush()
- logits = decode_step(dec_buffer, encoder_out)
+ logits = decode_fn(params, dec_buffer, encoder_out)
for i in range(prefix_len - 1, max_gen_len - 1):
next_logits = logits[0, i]
@@ -87,7 +98,7 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=
sys.stdout.write(tokenizer.decode([next_token]))
sys.stdout.flush()
- logits = decode_step(dec_buffer, encoder_out)
+ logits = decode_fn(params, dec_buffer, encoder_out)
if stream:
sys.stdout.write("\n")
@@ -99,10 +110,8 @@ def load_audio(path, target_sr=16000):
"""Load an audio file and resample to target_sr. Returns (audio_array, sr)."""
import soundfile as sf
audio, sr = sf.read(path, dtype="float32")
- # Convert stereo to mono
if audio.ndim > 1:
audio = audio.mean(axis=1)
- # Resample if needed
if sr != target_sr:
from scipy.signal import resample
num_samples = int(len(audio) * target_sr / sr)
@@ -130,26 +139,15 @@ def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="
{"params": params}, mel_input, src_mask=src_mask, deterministic=True, method="encode_speech"
)
- # Build decoder prefix: [BOS, <tool_call>, tools_tokens...]
tools_tokens = tokenizer.encode(tools)
prefix = [eos_id, tool_call_id] + tools_tokens
- prefix_len = len(prefix)
+ prefix_len = min(len(prefix), max_gen_len)
- tgt_mask = make_causal_mask(max_gen_len)
dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
for j, tok in enumerate(prefix[:max_gen_len]):
dec_buffer = dec_buffer.at[0, j].set(tok)
- @jax.jit
- def decode_step(dec_buffer, encoder_out):
- logits = model.apply(
- {"params": params},
- dec_buffer,
- encoder_out,
- self_mask=tgt_mask,
- method="decode",
- )
- return logits
+ decode_fn = _get_decode_fn(model, max_gen_len)
generated_tokens = []
@@ -157,7 +155,7 @@ def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="
sys.stdout.write("\n")
sys.stdout.flush()
- logits = decode_step(dec_buffer, encoder_out)
+ logits = decode_fn(params, dec_buffer, encoder_out)
for i in range(prefix_len - 1, max_gen_len - 1):
next_logits = logits[0, i]
@@ -173,7 +171,7 @@ def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="
sys.stdout.write(tokenizer.decode([next_token]))
sys.stdout.flush()
- logits = decode_step(dec_buffer, encoder_out)
+ logits = decode_fn(params, dec_buffer, encoder_out)
if stream:
sys.stdout.write("\n")
@@ -191,7 +189,6 @@ def main(args):
param_count = sum(x.size for x in jax.tree.leaves(params))
print(f"Model parameters: {param_count:,}")
- # --- Voice-to-tool-call mode ---
audio_files = getattr(args, "audio", None)
if audio_files:
tools = getattr(args, "tools", None) or "[]"
@@ -212,7 +209,6 @@ def main(args):
)
return
- # --- Tool-call generation mode ---
query = getattr(args, "query", None)
tools = getattr(args, "tools", None) or "[]"
diff --git a/src/test.py b/src/test.py
index b454a97..93502d2 100644
--- a/src/test.py
+++ b/src/test.py
@@ -7,14 +7,14 @@ import jax.numpy as jnp
import numpy as np
import optax
-from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs, load_tool_call_audio
+from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs, load_tool_call_audio, load_example_with_audio
from .model import (
EncoderDecoderTransformer,
TransformerConfig,
make_causal_mask,
make_padding_mask,
)
-from .run import load_checkpoint
+from .run import load_checkpoint, _get_decode_fn
def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batch_size, pad_id, loss_mask=None):
@@ -54,22 +54,15 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is th
eos_id = tokenizer.eos_token_id
src_mask = make_padding_mask(enc_input, pad_id)
- tgt_mask = make_causal_mask(max_gen_len)
-
- @jax.jit
- def decode_step(dec_buffer, encoder_out):
- logits = model.apply(
- {"params": params}, dec_buffer, encoder_out,
- self_mask=tgt_mask, method="decode",
- )
- return logits
+ decode_fn = _get_decode_fn(model, max_gen_len)
+ # Warmup JIT
encoder_out = model.apply(
{"params": params}, enc_input, src_mask=src_mask, method="encode"
)
dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
dec_buffer = dec_buffer.at[0, 0].set(eos_id)
- decode_step(dec_buffer, encoder_out)
+ decode_fn(params, dec_buffer, encoder_out)
tokens_generated = []
latencies = []
@@ -83,7 +76,7 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is th
encoder_out = model.apply(
{"params": params}, enc_input, src_mask=src_mask, method="encode"
)
- logits = decode_step(dec_buffer, encoder_out)
+ logits = decode_fn(params, dec_buffer, encoder_out)
num_tokens = 0
for i in range(max_gen_len - 1):
@@ -95,7 +88,7 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is th
num_tokens += 1
dec_buffer = dec_buffer.at[0, i + 1].set(next_token)
- logits = decode_step(dec_buffer, encoder_out)
+ logits = decode_fn(params, dec_buffer, encoder_out)
elapsed = time.perf_counter() - start
tokens_generated.append(num_tokens)
@@ -280,7 +273,7 @@ def benchmark_voice_tool_calls(model, params, tokenizer, num_samples=100, max_ge
import json
from .run import generate_from_audio
- pairs = load_tool_call_audio("validation", max_samples=num_samples)
+ indices = load_tool_call_audio("validation", max_samples=num_samples)
total = 0
exact_match = 0
@@ -295,13 +288,14 @@ def benchmark_voice_tool_calls(model, params, tokenizer, num_samples=100, max_ge
empty_pred = 0
samples = []
- for i, pair in enumerate(pairs):
+ for i, ds_idx in enumerate(indices):
+ pair = load_example_with_audio(ds_idx)
+ if pair["audio_array"] is None:
+ continue
ref_text = pair["answers"]
- audio = pair["audio_array"]
- sr = pair["sampling_rate"]
pred_text = generate_from_audio(
- model, params, tokenizer, audio, sr=sr,
+ model, params, tokenizer, pair["audio_array"], sr=pair["sampling_rate"],
tools=pair["tools"], max_gen_len=max_gen_len, seed=i, stream=False,
).strip()
@@ -387,7 +381,7 @@ def main(args):
print(f"\nevaluating tool-call perplexity ({args.max_eval_samples} samples)...")
ds = load_tool_calls("validation", max_samples=args.max_eval_samples)
- enc_inputs, dec_inputs, dec_targets, loss_mask_arr = prepare_tool_call_pairs(
+ enc_inputs, dec_inputs, dec_targets, loss_mask_arr, _ = prepare_tool_call_pairs(
ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
)
ppl = compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, args.batch_size, config.pad_token_id, loss_mask=loss_mask_arr)
diff --git a/src/tokenize_data.py b/src/tokenize_data.py
new file mode 100644
index 0000000..fccde4e
--- /dev/null
+++ b/src/tokenize_data.py
@@ -0,0 +1,96 @@
+"""Standalone tokenization pipeline: train tokenizer and pre-tokenize all data.
+
+Trains the SentencePiece tokenizer and tokenizes train + val splits for both
+text and voice data, caching everything on GCS. Running fresh overwrites all
+existing tokenizer + caches.
+
+Usage:
+ needle tokenize # full run
+ needle tokenize --max-samples 1000 # dev/test
+ needle tokenize --cleanup # delete local cache after GCS upload
+"""
+
+import os
+import shutil
+import subprocess
+
+from .data import (
+ CACHE_DIR,
+ GCS_CACHE_PATH,
+ GCS_TOKENIZER_PATH,
+ TOKENIZER_DIR,
+ _cache_key,
+ _save_cache_metadata,
+ get_tokenizer,
+ load_tool_calls,
+ precompute_mels,
+ prepare_tool_call_pairs,
+ train_tokenizer,
+ upload_tokenizer_to_gcs,
+)
+
+
+def _clear_gcs_caches():
+ """Remove existing GCS cache and tokenizer files."""
+ for path in [GCS_CACHE_PATH + "/*", GCS_TOKENIZER_PATH + "*"]:
+ print(f"Clearing {path} ...")
+ subprocess.run(
+ ["gcloud", "storage", "rm", "-r", path],
+ capture_output=True, text=True,
+ )
+
+
+def _clear_local_caches():
+ """Remove local .data_cache/ and tokenizer/ directories."""
+ for d in [CACHE_DIR, TOKENIZER_DIR]:
+ if os.path.exists(d):
+ print(f"Removing {d}/ ...")
+ shutil.rmtree(d)
+
+
+def tokenize(args):
+ # 1. Clear existing caches
+ print("=== Clearing existing caches ===")
+ _clear_gcs_caches()
+ _clear_local_caches()
+
+ # 2. Train tokenizer and upload to GCS
+ print("\n=== Training tokenizer ===")
+ train_tokenizer(max_samples=args.max_samples, force=True)
+ upload_tokenizer_to_gcs()
+
+ # 3. Load tokenizer
+ tokenizer = get_tokenizer()
+
+ # 4. Tokenize text data + precompute mels (train + val)
+ print("\n=== Tokenizing text data + precomputing mels ===")
+ max_enc_len = getattr(args, "max_enc_len", 256)
+ max_dec_len = getattr(args, "max_dec_len", 1024)
+ n_mels = getattr(args, "n_mels", 80)
+ max_mel_len = getattr(args, "max_mel_len", 1024)
+
+ for split in ("train", "val"):
+ print(f"\n--- {split} split ---")
+ ds = load_tool_calls(split=split, max_samples=args.max_samples)
+ _, _, _, _, kept_indices = prepare_tool_call_pairs(
+ ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len
+ )
+ text_cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len)
+
+ # Precompute mel spectrograms
+ mel_cache_id = precompute_mels(
+ kept_indices, n_mels=n_mels, max_mel_len=max_mel_len,
+ cache_id_prefix=split,
+ )
+
+ # Save metadata for train.py to find caches
+ _save_cache_metadata(split, text_cache_id, mel_cache_id, len(kept_indices),
+ max_enc_len, max_dec_len, n_mels, max_mel_len)
+
+ # 5. Optionally clean up local cache
+ if args.cleanup and os.path.exists(CACHE_DIR):
+ print(f"\n=== Cleaning up {CACHE_DIR}/ ===")
+ shutil.rmtree(CACHE_DIR)
+ print("Done.")
+
+ print("\n=== Tokenization pipeline complete ===")
diff --git a/src/train.py b/src/train.py
index 9e14141..35bfa57 100644
--- a/src/train.py
+++ b/src/train.py
@@ -14,9 +14,10 @@ from flax import jax_utils
from flax.training import train_state
from .data import (
- get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs,
- load_tool_call_audio, prepare_voice_tool_call_pairs, get_speech_batches,
- build_audio_augmenter,
+ get_batches, get_tokenizer, get_speech_batches,
+ load_prepared_data, load_prepared_mels,
+ load_example_with_audio,
+ PrefetchIterator, count_batches,
)
from .model import (
EncoderDecoderTransformer,
@@ -456,7 +457,7 @@ def train(args):
if wandb.run is None:
wandb.init(project="needle-v1", config=vars(args))
- total_data_steps = 6 if not no_speech else 4
+ total_data_steps = 4 if not no_speech else 3
step_idx = 0
step_idx += 1
@@ -468,53 +469,27 @@ def train(args):
tokenizer = get_tokenizer(max_samples=args.max_samples)
step_idx += 1
- print(f"\n[{step_idx}/{total_data_steps}] Loading tool-calling dataset...")
- ds = load_tool_calls("train", max_samples=args.max_samples)
- val_ds = load_tool_calls("validation", max_samples=getattr(args, "max_eval_samples", None))
-
- step_idx += 1
- print(f"\n[{step_idx}/{total_data_steps}] Preparing tool-call pairs...")
- enc_inputs, dec_inputs, dec_targets, train_loss_mask = prepare_tool_call_pairs(
- ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
- )
- val_enc, val_dec_in, val_dec_tgt, val_loss_mask = prepare_tool_call_pairs(
- val_ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
- )
- val_enc = np.array(val_enc)
- val_dec_in = np.array(val_dec_in)
- val_dec_tgt = np.array(val_dec_tgt)
- val_loss_mask = np.array(val_loss_mask)
- print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs")
-
- speech_audio_arrays = speech_dec_in = speech_dec_tgt = speech_loss_mask_arr = None
- val_speech_audio_arrays = val_speech_dec_in = val_speech_dec_tgt = val_speech_loss_mask = None
- audio_augmenter = None
+ print(f"\n[{step_idx}/{total_data_steps}] Loading prepared data from disk (mmap)...")
+ train_data = load_prepared_data("train", mmap=True)
+ val_data = load_prepared_data("val", mmap=True)
+ enc_inputs = train_data["enc_inputs"]
+ dec_inputs = train_data["dec_inputs"]
+ dec_targets = train_data["dec_targets"]
+ train_loss_mask = train_data["loss_mask"]
+ val_enc = val_data["enc_inputs"]
+ val_dec_in = val_data["dec_inputs"]
+ val_dec_tgt = val_data["dec_targets"]
+ val_loss_mask = val_data["loss_mask"]
+ print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs (memory-mapped)")
+
+ train_mels = None
+ val_mels = None
if not no_speech:
step_idx += 1
- max_speech_samples = getattr(args, "max_speech_samples", None)
- print(f"\n[{step_idx}/{total_data_steps}] Loading voice-tool-call audio...")
- speech_pairs = load_tool_call_audio("train", max_samples=max_speech_samples)
- speech_val_pairs = load_tool_call_audio("validation", max_samples=getattr(args, "max_eval_samples", None))
-
- step_idx += 1
- print(f"\n[{step_idx}/{total_data_steps}] Preparing voice-tool-call pairs...")
- speech_audio_arrays, speech_dec_in, speech_dec_tgt, speech_loss_mask_arr = prepare_voice_tool_call_pairs(
- speech_pairs, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len
- )
- val_speech_audio_arrays, val_speech_dec_in, val_speech_dec_tgt, val_speech_loss_mask = prepare_voice_tool_call_pairs(
- speech_val_pairs, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len
- )
- speech_dec_in = np.array(speech_dec_in)
- speech_dec_tgt = np.array(speech_dec_tgt)
- speech_loss_mask_arr = np.array(speech_loss_mask_arr)
- val_speech_dec_in = np.array(val_speech_dec_in)
- val_speech_dec_tgt = np.array(val_speech_dec_tgt)
- val_speech_loss_mask = np.array(val_speech_loss_mask)
- print(f" {len(speech_audio_arrays):,} train / {len(val_speech_audio_arrays):,} val voice-tool-call pairs")
-
- audio_augmenter = build_audio_augmenter(sr=16000)
- if audio_augmenter is not None:
- print(f" Audio augmentation: enabled ({len(audio_augmenter.transforms)} transforms)")
+ print(f"\n[{step_idx}/{total_data_steps}] Loading precomputed mel spectrograms (mmap)...")
+ train_mels = load_prepared_mels(train_data["mel_cache_id"], mmap=True)
+ val_mels = load_prepared_mels(val_data["mel_cache_id"], mmap=True)
+ print(f" {len(train_mels):,} train / {len(val_mels):,} val mel spectrograms (memory-mapped)")
effective_batch_size = args.batch_size * num_devices
@@ -563,9 +538,9 @@ def train(args):
mat_shared_input = getattr(args, "mat_shared_input", False)
unique_batch_size = effective_batch_size // n_widths if (mat_shared_input and n_widths > 1) else effective_batch_size
- text_batches_per_epoch = len(enc_inputs) // unique_batch_size
- if not no_speech and speech_audio_arrays is not None:
- speech_batches_per_epoch = len(speech_audio_arrays) // unique_batch_size
+ text_batches_per_epoch = count_batches(len(enc_inputs), unique_batch_size)
+ if not no_speech and train_mels is not None:
+ speech_batches_per_epoch = text_batches_per_epoch
else:
speech_batches_per_epoch = 0
num_batches = text_batches_per_epoch + speech_batches_per_epoch
@@ -598,7 +573,7 @@ def train(args):
print(f" Memory slots {config.num_memory_slots:>12}")
print(f" Activation {config.activation:>12}")
print(f" Dtype {config.dtype:>12}")
- if not no_speech:
+ if not no_speech and train_mels is not None:
print(f" Speech {speech_batches_per_epoch} batches/epoch")
print(f" n_mels {n_mels:>12}")
print(f" max_mel_len {max_mel_len:>12}")
@@ -637,9 +612,8 @@ def train(args):
muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps)
tokens_per_batch = effective_batch_size * (args.max_enc_len + args.max_dec_len)
- enc_inputs = np.array(enc_inputs)
- dec_inputs = np.array(dec_inputs)
- dec_targets = np.array(dec_targets)
+ # Create model instance once (JIT caches persist across epochs)
+ eval_model = EncoderDecoderTransformer(config)
last_val_ppl = None
sparsity_ratio = getattr(args, "sparsity_ratio", 0.0)
@@ -662,39 +636,43 @@ def train(args):
text_losses = []
speech_losses = []
- text_batches = list(get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size, loss_mask=train_loss_mask))
-
- speech_batch_list = []
- if not no_speech and speech_audio_arrays is not None:
- speech_batch_list = list(
- get_speech_batches(speech_audio_arrays, speech_dec_in, speech_dec_tgt, unique_batch_size,
- loss_mask=speech_loss_mask_arr, n_mels=n_mels, max_mel_len=max_mel_len,
- augmenter=audio_augmenter)
+ text_batch_iter = PrefetchIterator(
+ lambda: get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size,
+ loss_mask=train_loss_mask),
+ prefetch=4,
+ )
+
+ speech_batch_iter = None
+ if not no_speech and train_mels is not None:
+ speech_batch_iter = PrefetchIterator(
+ lambda: get_speech_batches(train_mels, dec_inputs, dec_targets, unique_batch_size,
+ loss_mask=train_loss_mask),
+ prefetch=4,
)
- steps_this_epoch = len(text_batches) + len(speech_batch_list)
+ steps_this_epoch = text_batches_per_epoch + speech_batches_per_epoch
text_idx = 0
speech_idx = 0
+ speech_loss_val = None
pbar = tqdm(range(steps_this_epoch), desc=f"Epoch {epoch + 1}/{args.epochs}")
for step_i in pbar:
t0 = time.perf_counter()
- do_speech = (step_i % 2 == 1) and speech_idx < len(speech_batch_list)
- do_text = not do_speech and text_idx < len(text_batches)
+ do_speech = (step_i % 2 == 1) and speech_idx < speech_batches_per_epoch
+ do_text = not do_speech and text_idx < text_batches_per_epoch
if not do_speech and not do_text:
- if text_idx < len(text_batches):
+ if text_idx < text_batches_per_epoch:
do_text = True
- elif speech_idx < len(speech_batch_list):
+ elif speech_idx < speech_batches_per_epoch:
do_speech = True
else:
break
- speech_loss_val = None
step_grad_norm = None
if do_text:
- src, tgt_in, tgt_out, lm = text_batches[text_idx]
+ src, tgt_in, tgt_out, lm = next(text_batch_iter)
text_idx += 1
if n_widths > 1 and mat_shared_input:
@@ -727,7 +705,7 @@ def train(args):
global_step += 1
else:
- mel_batch, sp_tgt_in, sp_tgt_out, sp_lm = speech_batch_list[speech_idx]
+ mel_batch, sp_tgt_in, sp_tgt_out, sp_lm = next(speech_batch_iter)
speech_idx += 1
if n_widths > 1 and mat_shared_input:
@@ -815,6 +793,10 @@ def train(args):
log_dict["val/text_ppl"] = last_val_ppl
wandb.log(log_dict)
+ text_batch_iter.close()
+ if speech_batch_iter is not None:
+ speech_batch_iter.close()
+
if epoch == weight_prune_epoch and not gradual_sparsify_done:
gradual_sparsify_done = True
if prune_mask is None:
@@ -839,101 +821,114 @@ def train(args):
eval_params = jax_utils.unreplicate(ema_params)
val_causal = make_causal_mask(args.max_dec_len)
- total_loss, total_toks = 0.0, 0.0
- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask):
- vl, vt = val_loss_fn(eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])
- total_loss += float(vl)
- total_toks += float(vt)
- last_val_ppl = float(math.exp(min(total_loss / max(total_toks, 1), 20)))
+ q_params = _quantize_params(eval_params, group_size=_GROUP_SIZE)
+ mat_vl_fns = {}
+ if _MAT_FACTORS:
+ _apply_fn = jax_utils.unreplicate(state).apply_fn
+ mat_vl_fns = {f: _make_mat_val_loss_fn(_apply_fn, fw)
+ for f, fw in zip(_MAT_FACTORS, _MAT_FF_WIDTHS)}
+ del _apply_fn
+
+ full_loss, full_toks = 0.0, 0.0
+ q_loss, q_toks = 0.0, 0.0
+ mat_accum = {f: [0.0, 0.0] for f in _MAT_FACTORS}
+
+ for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size,
+ shuffle=False, loss_mask=val_loss_mask):
+ src, dec_in, dec_tgt, lm = vb[0], vb[1], vb[2], vb[3]
+ vl, vt = val_loss_fn(eval_params, src, dec_in, dec_tgt, val_causal, lm)
+ full_loss += float(vl); full_toks += float(vt)
+ vl, vt = val_loss_fn(q_params, src, dec_in, dec_tgt, val_causal, lm)
+ q_loss += float(vl); q_toks += float(vt)
+ for f, fn in mat_vl_fns.items():
+ vl, vt = fn(eval_params, src, dec_in, dec_tgt, val_causal, lm)
+ mat_accum[f][0] += float(vl)
+ mat_accum[f][1] += float(vt)
+
+ last_val_ppl = float(math.exp(min(full_loss / max(full_toks, 1), 20)))
+ quant_val_ppl = float(math.exp(min(q_loss / max(q_toks, 1), 20)))
+ del q_params
+
+ mat_results = {}
+ for f in _MAT_FACTORS:
+ avg = mat_accum[f][0] / max(mat_accum[f][1], 1)
+ mat_results[f] = (float(math.exp(min(avg, 20))),
+ _estimate_mat_params(config, f), config.d_ff // f)
+
+ # --- Speech val PPL (separate pass — different data) ---
speech_val_ppl = None
- if speech_vl_fn is not None and val_speech_audio_arrays is not None:
+ if speech_vl_fn is not None and val_mels is not None:
sp_total_loss, sp_total_toks = 0.0, 0.0
- for sp_batch in get_speech_batches(val_speech_audio_arrays, val_speech_dec_in, val_speech_dec_tgt, args.batch_size,
- shuffle=False, loss_mask=val_speech_loss_mask,
- n_mels=n_mels, max_mel_len=max_mel_len, augmenter=None):
+ for sp_batch in get_speech_batches(val_mels, val_dec_in, val_dec_tgt, args.batch_size,
+ shuffle=False, loss_mask=val_loss_mask):
vl, vt = speech_vl_fn(eval_params, sp_batch[0], sp_batch[1], sp_batch[2], val_causal, sp_batch[3])
sp_total_loss += float(vl)
sp_total_toks += float(vt)
speech_val_loss = sp_total_loss / max(sp_total_toks, 1)
speech_val_ppl = float(math.exp(min(speech_val_loss, 20)))
- q_params = _quantize_params(eval_params, group_size=_GROUP_SIZE)
- q_total_loss, q_total_toks = 0.0, 0.0
- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask):
- vl, vt = val_loss_fn(q_params, vb[0], vb[1], vb[2], val_causal, vb[3])
- q_total_loss += float(vl)
- q_total_toks += float(vt)
- quant_val_ppl = float(math.exp(min(q_total_loss / max(q_total_toks, 1), 20)))
- del q_params
-
- mat_results = {}
- if _MAT_FACTORS:
- apply_fn = jax_utils.unreplicate(state).apply_fn
- for factor, ff_w in zip(_MAT_FACTORS, _MAT_FF_WIDTHS):
- mat_vl_fn = _make_mat_val_loss_fn(apply_fn, ff_w)
- mat_total_loss, mat_total_toks = 0.0, 0.0
- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask):
- vl, vt = mat_vl_fn(eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])
- mat_total_loss += float(vl)
- mat_total_toks += float(vt)
- avg_loss = mat_total_loss / max(mat_total_toks, 1)
- mat_ppl = float(math.exp(min(avg_loss, 20)))
- mat_params = _estimate_mat_params(config, factor)
- mat_results[factor] = (mat_ppl, mat_params, ff_w)
- del apply_fn
-
params_np = jax.tree.map(np.array, eval_params)
- del eval_params
total_params = sum(x.size for x in jax.tree.leaves(params_np))
near_zero = sum(int(np.sum(np.abs(x) < 1e-6)) for x in jax.tree.leaves(params_np))
sparsity = near_zero / total_params * 100
ckpt_name = f"needle_{args.num_layers}_{args.d_model}_{global_step}.pkl"
ckpt_path = os.path.join(args.checkpoint_dir, ckpt_name)
-
with open(ckpt_path, "wb") as f:
pickle.dump({"params": params_np, "config": config.__dict__}, f)
-
- from .test import measure_throughput, benchmark_tool_calls
- from .run import generate_from_audio
- eval_params_jnp = jax.tree.map(jnp.array, params_np)
- del params_np
- model = EncoderDecoderTransformer(config)
- tp = measure_throughput(model, eval_params_jnp, tokenizer, num_runs=5)
- tc_metrics = benchmark_tool_calls(model, eval_params_jnp, tokenizer, num_samples=20, max_gen_len=128)
-
- voice_tc_samples = []
- if not no_speech and val_speech_audio_arrays is not None:
- val_pairs = load_tool_call_audio("validation", max_samples=3)
- for i, pair in enumerate(val_pairs):
- audio = pair["audio_array"]
- sr = pair["sampling_rate"]
- pred_text = generate_from_audio(
- model, eval_params_jnp, tokenizer, audio, sr=sr,
- tools=pair["tools"], max_gen_len=128, seed=i, stream=False,
- ).strip()
- voice_tc_samples.append((pair["query"][:80], pair["answers"][:120], pred_text[:120]))
-
- del eval_params_jnp
-
+ del params_np
+
+ from .test import measure_throughput
+ from .run import generate, generate_from_audio
+ tp = measure_throughput(eval_model, eval_params, tokenizer, num_runs=5)
+
+ from .data import _load_unified_dataset
+ _ds_full = _load_unified_dataset()
+ _val_start = int(len(_ds_full) * 0.9)
+ val_kept = val_data["kept_indices"]
+ n_eval_samples = min(3, len(val_kept))
+ eval_indices = val_kept[:n_eval_samples] + _val_start
+
+ unified_samples = []
+ for i, ds_idx in enumerate(eval_indices):
+ pair = load_example_with_audio(int(ds_idx))
+ query = pair["query"][:80]
+ ref = pair["answers"][:120]
+
+ # Text prediction
+ text_pred = generate(
+ eval_model, eval_params, tokenizer, pair["query"],
+ tools=pair["tools"], max_gen_len=args.max_dec_len, seed=i, stream=False,
+ ).strip()[:120]
+
+ # Voice prediction
+ voice_pred = None
+ if not no_speech and pair["audio_array"] is not None:
+ voice_pred = generate_from_audio(
+ eval_model, eval_params, tokenizer, pair["audio_array"], sr=pair["sampling_rate"],
+ tools=pair["tools"], max_gen_len=args.max_dec_len, seed=i, stream=False,
+ ).strip()[:120]
+
+ unified_samples.append((query, ref, text_pred, voice_pred))
+
+ del eval_params
+
+ final_speech_loss = speech_losses[-1] if speech_losses else None
print(f"\n ─────────────────────────────────────")
print(f" Epoch {epoch + 1}/{args.epochs}")
print(f" ─────────────────────────────────────")
- print(f" Avg loss {epoch_avg_loss:>12.4f}")
- print(f" Final loss {final_loss:>12.4f}")
- print(f" Train ppl {final_ppl:>12.2f}")
- print(f" TC val ppl {last_val_ppl:>12.2f}")
- if speech_losses:
- epoch_avg_speech = sum(speech_losses) / len(speech_losses)
- print(f" Avg speech loss{epoch_avg_speech:>12.4f}")
+ print(f" Text loss {final_loss:>12.4f}")
+ print(f" Text val ppl {last_val_ppl:>12.2f}")
+ if final_speech_loss is not None:
+ print(f" Speech loss {final_speech_loss:>12.4f}")
if speech_val_ppl is not None:
print(f" Speech val ppl {speech_val_ppl:>12.2f}")
print(f" Quant val ppl {quant_val_ppl:>12.2f} (INT4 g{_GROUP_SIZE})")
print(f" Sparsity {sparsity:>11.2f}% ({near_zero:,}/{total_params:,})")
if mat_results:
print(f" ─────────────────────────────────────")
- print(f" Matryoshka exportable sub-models (FFN interior slicing):")
+ print(f" Matryoshka sub-models:")
print(f" {'factor':>6} {'d_ff':>6} {'val ppl':>10} {'params':>12}")
print(f" {'1x':>6} {config.d_ff:>6} {last_val_ppl:>10.2f} {total_params:>12,} (full)")
for factor in sorted(mat_results.keys()):
@@ -942,25 +937,14 @@ def train(args):
print(f" ─────────────────────────────────────")
print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")
print(f" Latency {tp['avg_latency_s']:>11.3f}s")
- print(f" ─── Tool-Call Metrics ───────────────")
- print(f" JSON parse {tc_metrics['json_parse_rate']:>10.1%}")
- print(f" Exact match {tc_metrics['exact_match']:>10.1%}")
- print(f" Name F1 {tc_metrics['name_f1']:>10.3f} (P={tc_metrics['name_precision']:.3f} R={tc_metrics['name_recall']:.3f})")
- print(f" Call F1 {tc_metrics['call_f1']:>10.3f} (P={tc_metrics['call_precision']:.3f} R={tc_metrics['call_recall']:.3f})")
- if tc_metrics["samples"]:
- print(f" ─────────────────────────────────────")
- for query, ref, pred in tc_metrics["samples"][:3]:
- print(f" Q: {query}")
- print(f" R: {ref}")
- print(f" P: {pred}")
- print()
- if voice_tc_samples:
- print(f" ─────────────────────────────────────")
- print(f" Voice-to-Tool-Call samples:")
- for query, ref, pred in voice_tc_samples:
+ if unified_samples:
+ print(f" ─── Samples ────────────────────────")
+ for query, ref, text_pred, voice_pred in unified_samples:
print(f" Q: {query}")
print(f" R: {ref}")
- print(f" P: {pred}")
+ print(f" T: {text_pred or '(empty)'}")
+ if voice_pred is not None:
+ print(f" V: {voice_pred or '(empty)'}")
print()
print(f" ─────────────────────────────────────")
print(f" Checkpoint: {ckpt_path}")
@@ -968,19 +952,14 @@ def train(args):
if use_wandb:
log_dict = {
- "epoch/avg_text_loss": epoch_avg_loss,
- "epoch/final_text_loss": final_loss,
+ "epoch/text_loss": final_loss,
"epoch/text_val_ppl": last_val_ppl,
"epoch/quant_val_ppl": quant_val_ppl,
"epoch/weight_sparsity": sparsity,
"epoch": epoch + 1,
}
- log_dict["epoch/tc_exact_match"] = tc_metrics["exact_match"]
- log_dict["epoch/tc_name_f1"] = tc_metrics["name_f1"]
- log_dict["epoch/tc_call_f1"] = tc_metrics["call_f1"]
- log_dict["epoch/tc_json_parse_rate"] = tc_metrics["json_parse_rate"]
- if speech_losses:
- log_dict["epoch/avg_speech_loss"] = sum(speech_losses) / len(speech_losses)
+ if final_speech_loss is not None:
+ log_dict["epoch/speech_loss"] = final_speech_loss
if speech_val_ppl is not None:
log_dict["epoch/speech_val_ppl"] = speech_val_ppl
for factor, (mat_ppl, mat_params, _) in mat_results.items():