Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Dataset setup complete
HenryNdubuaku07b2178f49d3
diff --git a/README.md b/README.mdindex 38b549b..875a9c3 100644--- a/README.md+++ b/README.md@@ -71,55 +71,63 @@strided DW conv · mat factors · INT4 QAT · Muon + AdamWtext + speech encoder · <tool_call> task routing- Data Preprocessing- ──────────────────-- Text (Tool-Call Dataset) Speech (Voice-Tool-Call)- ──────────────────────── ────────────────────────-- Unified dataset Unified dataset- (GCS / local) (GCS / local)- │ │- ▼ ▼- ┌──────────────┐ ┌────────────────┐- │ SentencePiece│ │ HF Audio col │- │ BPE tokenize │ │ decode (16kHz) │- │ (8192 vocab) │ └───────┬────────┘- └──────┬───────┘ │- │ ▼- ▼ ┌────────────────┐- ┌──────────────┐ │ Augmentation │- │ Encoder: │ │ (noise, pitch, │- │ query tokens │ │ stretch, gain)│- └──────┬───────┘ └───────┬────────┘- │ │- ▼ ▼- ┌──────────────────┐ ┌────────────────┐- │ Decoder: │ │ Log-mel spec │- │ [EOS]<tool_call> │ │ 80 bins, 25ms │- │ + tools tokens │ │ window, 10ms │- │ + answer tokens │ │ hop (~100 f/s) │- │ → dec_in │ └───────┬────────┘- ├──────────────────┤ │- │ <tool_call> │ ▼- │ + tools tokens │ ┌────────────────┐- │ + answer tokens │ │ Pad/trunc to │- │ + [EOS] │ │ max_mel_len │- │ → dec_tgt │ └───────┬────────┘- ├──────────────────┤ │- │ Loss mask: 1.0 │ ▼- │ on answer + EOS │ ┌────────────────┐- │ only (not tools) │ │ Same decoder │- └──────────────────┘ │ format as text │- └────────────────┘- Both cached as- .npy in .data_cache/ Mel computed on-the-fly- (augment before mel)- ┌──────────────────────┐- │ Interleaved batches: │- │ 1:1 text / speech │ Eval: voice-to-tool-call on- │ alternation │ 3 val samples per epoch- └──────────────────────┘+ Data Pipeline (needle tokenize → needle train)+ ───────────────────────────────────────────────++ ┌─────────────────────────────────────────────────────────────┐+ │ needle tokenize │+ │ │+ │ Unified dataset (GCS / local) │+ │ │ │+ │ ├──── Text ──────────────── Speech ─────────┐ │+ │ ▼ ▼ │+ │ ┌──────────────┐ ┌────────────────┐ │+ │ │ SentencePiece│ │ HF Audio col │ │+ │ │ BPE tokenize │ │ decode (16kHz) │ │+ │ │ (8192 vocab) │ └───────┬────────┘ │+ │ └──────┬───────┘ │ │+ │ │ ▼ │+ │ ▼ ┌────────────────┐ │+ │ ┌──────────────────┐ │ Log-mel spec │ │+ │ │ enc_inputs.npy │ │ 80 bins, 25ms │ │+ │ │ dec_inputs.npy │ │ window, 10ms │ │+ │ │ dec_targets.npy │ │ hop (~100 f/s) │ │+ │ │ loss_mask.npy │ └───────┬────────┘ │+ │ │ kept_idx.npy │ │ │+ │ └──────┬───────────┘ ▼ │+ │ │ ┌────────────────┐ │+ │ │ │ mels.npy │ │+ │ │ │ (N,T,80) f32 │ │+ │ │ │ writable mmap │ │+ │ │ └───────┬────────┘ │+ │ │ │ │+ │ └────────────┬───────────────────────────┘ │+ │ ▼ │+ │ {split}_metadata.json │+ │ .data_cache/ + GCS upload │+ └─────────────────────────────────────────────────────────────┘+ │+ ▼+ ┌─────────────────────────────────────────────────────────────┐+ │ needle train │+ │ │+ │ load_prepared_data(mmap=True) │+ │ load_prepared_mels(mmap=True) │+ │ │ │+ │ ▼ │+ │ ┌──────────────────────┐ ┌──────────────────────┐ │+ │ │ PrefetchIterator │ │ PrefetchIterator │ │+ │ │ text batches (4) │ │ speech batches (4) │ │+ │ │ mmap → per-batch idx │ │ mmap → per-batch idx │ │+ │ └──────────┬───────────┘ └──────────┬───────────┘ │+ │ └───────────┬──────────────┘ │+ │ ▼ │+ │ 1:1 text/speech │+ │ alternation │+ │ │+ │ ~42MB RAM (8 prefetched batches) │+ │ SpecAugment for speech regularization │+ └─────────────────────────────────────────────────────────────┘```## Usage@@ -165,6 +173,14 @@ needle [command]│ --n-mels INT Mel frequency bins (default: 80) ││ --max-speech-samples INT Max voice-tool-call samples ││ │+ │ tokenize │+ │ --max-samples INT Limit samples per split (dev/test) │+ │ --cleanup Delete local cache after GCS upload │+ │ --n-mels INT Mel frequency bins (default: 80) │+ │ --max-mel-len INT Max mel frames (default: 1024) │+ │ --max-enc-len INT Max encoder seq len (default: 256) │+ │ --max-dec-len INT Max decoder seq len (default: 1024) │+ │ ││ run ││ --checkpoint PATH Path to model checkpoint (required) ││ --query STR Query text for tool-call generation │diff --git a/scripts/build_dataset.py b/scripts/build_dataset.pyindex 72f9385..c057bcb 100644--- a/scripts/build_dataset.py+++ b/scripts/build_dataset.py@@ -144,11 +144,11 @@ def _load_text_dataset(output_path, max_samples):return combined, existing_ds-def _find_rows_needing_tts(combined, existing_ds):- """Identify which rows lack audio by inspecting the raw Arrow column.+def _find_rows_needing_tts(combined, existing_ds, output_path):+ """Identify which rows lack audio by checking the cache dir and existing dataset.- Accesses the underlying Arrow table directly to avoid triggering- HF audio decoding (which requires torchcodec).+ First checks {output_path}/.audio_cache/ for WAV files from prior runs,+ then falls back to inspecting the Arrow column in existing_ds.Returns:Set of row indices that need TTS synthesis.@@ -156,57 +156,89 @@ def _find_rows_needing_tts(combined, existing_ds):n = len(combined)needs_tts = set(range(n))+ cache_dir = os.path.join(output_path, ".audio_cache")+ if os.path.isdir(cache_dir):+ cached = 0+ for fname in os.listdir(cache_dir):+ if fname.endswith(".wav"):+ try:+ idx = int(fname[:-4])+ if idx < n:+ needs_tts.discard(idx)+ cached += 1+ except ValueError:+ pass+ if cached:+ logger.info(f" {cached} rows found in audio cache")+if existing_ds is not None and len(existing_ds) == n and "audio" in existing_ds.column_names:audio_column = existing_ds.data.column("audio")for i in range(n):- if audio_column[i].as_py() is not None:+ if i in needs_tts and audio_column[i].as_py() is not None:needs_tts.discard(i)- logger.info(f" {n - len(needs_tts)} rows already have audio, {len(needs_tts)} need TTS")+ logger.info(f" {n - len(needs_tts)} rows already have audio, {len(needs_tts)} need TTS")return needs_tts-def _recover_existing_audio(existing_ds, n, needs_tts):- """Extract audio bytes and voice names from a previously saved dataset.+def _recover_existing_audio(existing_ds, n, needs_tts, output_path):+ """Recover audio file paths from the cache dir and existing dataset.- Reads raw Arrow data to avoid triggering audio decoding.+ Stores file paths (not bytes) to keep memory usage low.Returns:Tuple of (audio_data list, voice_names list).+ audio_data entries are file paths (str) or None."""audio_data = [None] * nvoice_names = [""] * n+ cache_dir = os.path.join(output_path, ".audio_cache")+ if os.path.isdir(cache_dir):+ for i in range(n):+ if i not in needs_tts:+ wav_path = os.path.join(cache_dir, f"{i}.wav")+ if os.path.exists(wav_path):+ audio_data[i] = wav_path+if existing_ds is not None and len(existing_ds) == n and "audio" in existing_ds.column_names:audio_column = existing_ds.data.column("audio")voice_col = existing_ds["voice"] if "voice" in existing_ds.column_names else Nonefor i in range(n):- if i not in needs_tts:+ if i not in needs_tts and audio_data[i] is None:raw = audio_column[i].as_py()if isinstance(raw, dict) and raw.get("bytes"):- audio_data[i] = raw["bytes"]- elif isinstance(raw, dict) and raw.get("path"):- with open(raw["path"], "rb") as f:- audio_data[i] = f.read()- if voice_col is not None:- voice_names[i] = voice_col[i] or ""+ wav_path = os.path.join(cache_dir, f"{i}.wav")+ os.makedirs(cache_dir, exist_ok=True)+ with open(wav_path, "wb") as f:+ f.write(raw["bytes"])+ audio_data[i] = wav_path+ elif isinstance(raw, dict) and raw.get("path") and os.path.exists(raw["path"]):+ audio_data[i] = raw["path"]+ if voice_col is not None and i not in needs_tts:+ voice_names[i] = voice_col[i] or ""return audio_data, voice_names-def _synthesize_missing(combined, needs_tts, audio_data, voice_names, workers, rate_limit):- """Run TTS synthesis for all rows that lack audio.+def _synthesize_missing(combined, needs_tts, audio_data, voice_names, workers, rate_limit, output_path):+ """Run TTS synthesis, writing each WAV to disk immediately.- Uses a thread pool with rate-limited submission to stay within- GCP Cloud TTS API quotas.+ Audio bytes are written to {output_path}/.audio_cache/{idx}.wav as they+ arrive, keeping RAM usage constant regardless of dataset size. The+ audio_data list stores file paths (not bytes) for later assembly."""if not needs_tts:returnfrom google.cloud import texttospeech+ cache_dir = os.path.join(output_path, ".audio_cache")+ os.makedirs(cache_dir, exist_ok=True)+queries_to_synth = [(i, combined[i]["query"]) for i in sorted(needs_tts)]logger.info(f"Synthesizing TTS for {len(queries_to_synth)} rows with {workers} workers...")+ logger.info(f" Audio cache: {cache_dir}/")def _worker(item):idx, query = item@@ -224,7 +256,10 @@ def _synthesize_missing(combined, needs_tts, audio_data, voice_names, workers, rfor future in as_completed(futures):idx, wav_bytes, voice_name, err = future.result()if wav_bytes is not None:- audio_data[idx] = wav_bytes+ wav_path = os.path.join(cache_dir, f"{idx}.wav")+ with open(wav_path, "wb") as f:+ f.write(wav_bytes)+ audio_data[idx] = wav_pathvoice_names[idx] = voice_namesuccess += 1else:@@ -244,16 +279,28 @@ def _backfill_voice_names(voice_names):def _build_and_save(combined, audio_data, voice_names, output_path):- """Assemble the unified HF Dataset and save to disk.+ """Assemble the unified HF Dataset from file paths and save to disk.++ audio_data entries are file paths (str) or None. Each file is read+ one at a time during assembly to avoid loading all audio into RAM.Returns:The saved Dataset."""n = len(combined)- logger.info("Building unified HF Dataset...")+ has_audio = sum(1 for x in audio_data if x is not None)+ logger.info(f"Building unified HF Dataset... ({has_audio}/{n} with audio)")++ audio_column = []+ for path in audio_data:+ if path is not None and os.path.exists(path):+ audio_column.append(path)+ else:+ audio_column.append(None)+ds_dict = {"query": [combined[i]["query"] for i in range(n)],- "audio": audio_data,+ "audio": audio_column,"tools": [combined[i]["tools"] for i in range(n)],"answers": [combined[i]["answers"] for i in range(n)],"source": [combined[i]["source"] for i in range(n)],@@ -269,21 +316,13 @@ def _build_and_save(combined, audio_data, voice_names, output_path):return unified-def _train_and_save_tokenizer(max_samples):- """Train a SentencePiece tokenizer on the unified dataset."""- from src.data import train_tokenizer, TOKENIZER_PREFIX- logger.info("Training tokenizer on unified dataset...")- model_path = train_tokenizer(max_samples=max_samples)- logger.info(f"Tokenizer ready at {model_path}")--def _upload_to_gcs(output_path, gcs_upload):- """Upload the dataset and tokenizer to GCS via gcloud CLI."""- from src.data import TOKENIZER_PREFIX-+ """Upload the dataset to GCS via gcloud CLI."""logger.info(f"Uploading dataset to {gcs_upload}...")+ import glob as globmod+ files = globmod.glob(os.path.join(output_path, "*"))result = subprocess.run(- ["gcloud", "storage", "cp", "-r", output_path, gcs_upload],+ ["gcloud", "storage", "cp"] + files + [gcs_upload + "/"],capture_output=True, text=True,)if result.returncode == 0:@@ -292,20 +331,6 @@ def _upload_to_gcs(output_path, gcs_upload):logger.error(f"Dataset upload failed: {result.stderr}")sys.exit(1)- gcs_bucket = gcs_upload.rsplit("/", 1)[0]- gcs_tokenizer_path = f"{gcs_bucket}/tokenizer/"- logger.info(f"Uploading tokenizer to {gcs_tokenizer_path}...")- tokenizer_dir = os.path.dirname(TOKENIZER_PREFIX)- result = subprocess.run(- ["gcloud", "storage", "cp", "-r", tokenizer_dir, gcs_tokenizer_path],- capture_output=True, text=True,- )- if result.returncode == 0:- logger.info(f"Tokenizer upload complete: {gcs_tokenizer_path}")- else:- logger.error(f"Tokenizer upload failed: {result.stderr}")- sys.exit(1)-def main():"""Entry point for the unified dataset build pipeline."""@@ -320,12 +345,11 @@ def main():args = parser.parse_args()combined, existing_ds = _load_text_dataset(args.output, args.max_samples)- needs_tts = _find_rows_needing_tts(combined, existing_ds)- audio_data, voice_names = _recover_existing_audio(existing_ds, len(combined), needs_tts)- _synthesize_missing(combined, needs_tts, audio_data, voice_names, args.workers, args.rate_limit)+ needs_tts = _find_rows_needing_tts(combined, existing_ds, args.output)+ audio_data, voice_names = _recover_existing_audio(existing_ds, len(combined), needs_tts, args.output)+ _synthesize_missing(combined, needs_tts, audio_data, voice_names, args.workers, args.rate_limit, args.output)_backfill_voice_names(voice_names)_build_and_save(combined, audio_data, voice_names, args.output)- _train_and_save_tokenizer(args.max_samples)if args.gcs_upload:_upload_to_gcs(args.output, args.gcs_upload)diff --git a/src/cli.py b/src/cli.pyindex 928ae95..ff83a7f 100644--- a/src/cli.py+++ b/src/cli.py@@ -55,6 +55,20 @@ def main():p.add_argument("--max-speech-samples", type=int, default=None,help="Max voice-tool-call training samples (default: all)")+ p = sub.add_parser("tokenize", add_help=False)+ p.add_argument("--max-samples", type=int, default=None,+ help="Limit samples per split (for dev/test)")+ p.add_argument("--cleanup", action="store_true",+ help="Delete local .data_cache/ after GCS upload")+ p.add_argument("--n-mels", type=int, default=80,+ help="Number of mel frequency bins (default: 80)")+ p.add_argument("--max-mel-len", type=int, default=1024,+ help="Max mel spectrogram frames (default: 1024)")+ p.add_argument("--max-enc-len", type=int, default=256,+ help="Max encoder sequence length (default: 256)")+ p.add_argument("--max-dec-len", type=int, default=1024,+ help="Max decoder sequence length (default: 1024)")+p = sub.add_parser("run", add_help=False)p.add_argument("--checkpoint", type=str, required=True)p.add_argument("--query", type=str, default=None, help="Query text for tool-call generation")@@ -119,7 +133,10 @@ def main():print(HELP)sys.exit(0)- if args.command == "train":+ if args.command == "tokenize":+ from .tokenize_data import tokenize+ tokenize(args)+ elif args.command == "train":if getattr(args, "full", False):args.d_model = 1536args.num_heads = 24diff --git a/src/data.py b/src/data.pyindex a09d62f..3353430 100644--- a/src/data.py+++ b/src/data.py@@ -1,6 +1,9 @@import hashlib+import json as _jsonimport multiprocessing as mpimport os+import queue+import threadingos.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "1"@@ -40,29 +43,26 @@ def _load_unified_dataset():if _unified_dataset_cache is not None:return _unified_dataset_cache- # Try GCS first- try:- ds = load_from_disk(GCS_DATASET_PATH)- print(f"Loaded unified dataset from GCS ({len(ds)} rows)")- # Cache locally for faster subsequent loads- if not os.path.exists(LOCAL_UNIFIED_DIR):- ds.save_to_disk(LOCAL_UNIFIED_DIR)- print(f" Cached locally to {LOCAL_UNIFIED_DIR}")+ if os.path.exists(LOCAL_UNIFIED_DIR) and any(+ f.endswith(".arrow") for f in os.listdir(LOCAL_UNIFIED_DIR)+ ):+ ds = load_from_disk(LOCAL_UNIFIED_DIR)+ print(f"Loaded unified dataset from {LOCAL_UNIFIED_DIR} ({len(ds)} rows)")_unified_dataset_cache = _set_audio_backend(ds)return _unified_dataset_cache- except Exception:- pass- # Fallback to local- if os.path.exists(LOCAL_UNIFIED_DIR):- ds = load_from_disk(LOCAL_UNIFIED_DIR)- print(f"Loaded unified dataset from {LOCAL_UNIFIED_DIR} ({len(ds)} rows)")+ try:+ ds = load_from_disk(GCS_DATASET_PATH)+ print(f"Loaded unified dataset from GCS ({len(ds)} rows)")_unified_dataset_cache = _set_audio_backend(ds)return _unified_dataset_cache+ except Exception as e:+ gcs_err = eraise FileNotFoundError(f"Unified dataset not found. Run scripts/build_dataset.py first, "- f"or ensure GCS path {GCS_DATASET_PATH} is accessible (pip install gcsfs)."+ f"or ensure GCS path {GCS_DATASET_PATH} is accessible (pip install gcsfs). "+ f"GCS error: {gcs_err}")@@ -142,10 +142,10 @@ def _tokenize_chunk(texts):return [_worker_sp.Encode(t, out_type=int)[:_worker_max_len] for t in texts]-def train_tokenizer(vocab_size=8192, max_samples=None):+def train_tokenizer(vocab_size=8192, max_samples=None, force=False):"""Train a SentencePiece BPE tokenizer on tool-calling corpus."""model_path = TOKENIZER_PREFIX + ".model"- if os.path.exists(model_path):+ if os.path.exists(model_path) and not force:print(f"Tokenizer already exists at {model_path}")return model_path@@ -190,6 +190,19 @@ def train_tokenizer(vocab_size=8192, max_samples=None):GCS_TOKENIZER_PATH = "gs://cactus-dataset/tokenizer/"+def upload_tokenizer_to_gcs():+ """Upload local tokenizer files to GCS."""+ import subprocess+ import glob as globmod+ tok_files = globmod.glob(os.path.join(TOKENIZER_DIR, "*"))+ if tok_files:+ subprocess.run(+ ["gcloud", "storage", "cp"] + tok_files + [GCS_TOKENIZER_PATH],+ capture_output=True, text=True,+ )+ print(f"Uploaded tokenizer to {GCS_TOKENIZER_PATH}")++def get_tokenizer(max_samples=None):model_path = TOKENIZER_PREFIX + ".model"if not os.path.exists(model_path):@@ -215,6 +228,34 @@ def _download_tokenizer_from_gcs():return False+GCS_CACHE_PATH = "gs://cactus-dataset/cache"+++def _gcs_cache_download(cache_id, suffixes):+ """Try downloading cached files from GCS. Returns True if all found."""+ import subprocess+ os.makedirs(CACHE_DIR, exist_ok=True)+ cache_path = os.path.join(CACHE_DIR, cache_id)+ srcs = [f"{GCS_CACHE_PATH}/{cache_id}{s}" for s in suffixes]+ subprocess.run(+ ["gcloud", "storage", "cp"] + srcs + [CACHE_DIR + "/"],+ capture_output=True, text=True,+ )+ return all(os.path.exists(cache_path + s) for s in suffixes)+++def _gcs_cache_upload(cache_id, suffixes):+ """Upload cached files to GCS."""+ import subprocess+ cache_path = os.path.join(CACHE_DIR, cache_id)+ files = [cache_path + s for s in suffixes if os.path.exists(cache_path + s)]+ if files:+ subprocess.run(+ ["gcloud", "storage", "cp"] + files + [GCS_CACHE_PATH + "/"],+ capture_output=True, text=True,+ )++def load_tool_calls(split="train", max_samples=None):"""Load tool-calling dataset, splitting 90/10 for train/val."""ds = _load_unified_dataset()@@ -243,6 +284,52 @@ def _cache_key(prefix, n_samples, max_enc_len, max_dec_len):return hashlib.md5(key.encode()).hexdigest()[:12]+def _mel_cache_key(prefix, n_samples, n_mels, max_mel_len):+ tok_hash = _tokenizer_hash()+ key = f"mel_{prefix}_{tok_hash}_{n_samples}_{n_mels}_{max_mel_len}"+ return hashlib.md5(key.encode()).hexdigest()[:12]+++def _save_cache_metadata(split, text_cache_id, mel_cache_id, n_samples,+ max_enc_len, max_dec_len, n_mels, max_mel_len):+ """Save metadata JSON for a split, upload to GCS."""+ os.makedirs(CACHE_DIR, exist_ok=True)+ meta = {+ "split": split,+ "text_cache_id": text_cache_id,+ "mel_cache_id": mel_cache_id,+ "n_samples": n_samples,+ "max_enc_len": max_enc_len,+ "max_dec_len": max_dec_len,+ "n_mels": n_mels,+ "max_mel_len": max_mel_len,+ }+ meta_path = os.path.join(CACHE_DIR, f"{split}_metadata.json")+ with open(meta_path, "w") as f:+ _json.dump(meta, f)+ _gcs_cache_upload(f"{split}_metadata", [".json"])+++def _load_cache_metadata(split):+ """Load metadata JSON from local or GCS. Returns dict or None."""+ meta_path = os.path.join(CACHE_DIR, f"{split}_metadata.json")+ if os.path.exists(meta_path):+ with open(meta_path) as f:+ return _json.load(f)+ # Try GCS+ import subprocess+ os.makedirs(CACHE_DIR, exist_ok=True)+ gcs_path = f"{GCS_CACHE_PATH}/{split}_metadata.json"+ subprocess.run(+ ["gcloud", "storage", "cp", gcs_path, meta_path],+ capture_output=True, text=True,+ )+ if os.path.exists(meta_path):+ with open(meta_path) as f:+ return _json.load(f)+ return None++def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):"""Prepare tool-call encoder-decoder pairs with <tool_call> task token.@@ -256,15 +343,25 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len)cache_path = os.path.join(CACHE_DIR, cache_id)- if os.path.exists(cache_path + "_enc.npy"):- print(f"Loading cached tool-call data ({cache_id})...")+ tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]++ def _load_tc_cache():return (np.load(cache_path + "_enc.npy"),np.load(cache_path + "_dec_in.npy"),np.load(cache_path + "_dec_tgt.npy"),np.load(cache_path + "_loss_mask.npy"),+ np.load(cache_path + "_kept_idx.npy"),)+ if os.path.exists(cache_path + "_enc.npy"):+ print(f"Loading cached tool-call data ({cache_id})...")+ return _load_tc_cache()++ if _gcs_cache_download(cache_id, tc_suffixes):+ print(f"Downloaded tool-call cache from GCS ({cache_id})...")+ return _load_tc_cache()+pad_id = tokenizer.pad_token_ideos_id = tokenizer.eos_token_idtool_call_id = tokenizer.tool_call_token_id@@ -277,7 +374,6 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):model_path = TOKENIZER_PREFIX + ".model"chunk_size = max(1, len(enc_texts) // (num_workers * 4))- # Tokenize queries (encoder)enc_chunks = [enc_texts[i:i + chunk_size] for i in range(0, len(enc_texts), chunk_size)]print(f"Tokenizing encoder inputs ({num_workers} workers)...")with mp.Pool(num_workers, initializer=_init_worker,@@ -285,7 +381,6 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):enc_results = pool.map(_tokenize_chunk, enc_chunks)all_enc_tokens = [tok for chunk in enc_results for tok in chunk]- # Tokenize tools (decoder prefix) — reserve 2 for BOS + <tool_call>, rest for tools + answertools_chunks = [tools_texts[i:i + chunk_size] for i in range(0, len(tools_texts), chunk_size)]print(f"Tokenizing tools ({num_workers} workers)...")with mp.Pool(num_workers, initializer=_init_worker,@@ -293,7 +388,6 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):tools_results = pool.map(_tokenize_chunk, tools_chunks)all_tools_tokens = [tok for chunk in tools_results for tok in chunk]- # Tokenize answers (decoder generation target)ans_chunks = [ans_texts[i:i + chunk_size] for i in range(0, len(ans_texts), chunk_size)]print(f"Tokenizing answers ({num_workers} workers)...")with mp.Pool(num_workers, initializer=_init_worker,@@ -313,13 +407,10 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):t_tok = all_tools_tokens[i]a_tok = all_ans_tokens[i]- # Prefix length: BOS + <tool_call> + tools_tokensprefix_len = 2 + len(t_tok)- # Total decoder length: prefix + answer + EOStotal_dec = prefix_len + len(a_tok) + 1if total_dec > max_dec_len:- # Truncate tools to fit answer + EOSavailable_for_tools = max_dec_len - 2 - len(a_tok) - 1if available_for_tools < 1:skipped += 1@@ -348,96 +439,107 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=256, max_dec_len=1024):dec_targets[i, prefix_len - 1:prefix_len - 1 + al] = a_tokdec_targets[i, prefix_len - 1 + al] = eos_id- # Loss mask: 1 on answer positions + EOS onlyloss_mask[i, prefix_len - 1:prefix_len - 1 + al + 1] = 1.0if skipped > 0:- # Remove skipped rows (all-pad)keep = enc_inputs[:, 0] != pad_id+ kept_indices = np.where(keep)[0].astype(np.int64)enc_inputs = enc_inputs[keep]dec_inputs = dec_inputs[keep]dec_targets = dec_targets[keep]loss_mask = loss_mask[keep]print(f" Skipped {skipped} examples (too long for max_dec_len={max_dec_len})")+ else:+ kept_indices = np.arange(n, dtype=np.int64)- # Save cacheos.makedirs(CACHE_DIR, exist_ok=True)np.save(cache_path + "_enc.npy", enc_inputs)np.save(cache_path + "_dec_in.npy", dec_inputs)np.save(cache_path + "_dec_tgt.npy", dec_targets)np.save(cache_path + "_loss_mask.npy", loss_mask)+ np.save(cache_path + "_kept_idx.npy", kept_indices)+ _gcs_cache_upload(cache_id, tc_suffixes)print(f"Cached {len(enc_inputs):,} tool-call pairs to {CACHE_DIR}/{cache_id}")- return enc_inputs, dec_inputs, dec_targets, loss_mask+ return enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indicesdef get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True, loss_mask=None):n = len(enc_inputs)- if shuffle:- perm = np.random.permutation(n)- enc_inputs = enc_inputs[perm]- dec_inputs = dec_inputs[perm]- dec_targets = dec_targets[perm]- if loss_mask is not None:- loss_mask = loss_mask[perm]-+ indices = np.random.permutation(n) if shuffle else np.arange(n)for i in range(0, n - batch_size + 1, batch_size):- batch = (enc_inputs[i : i + batch_size], dec_inputs[i : i + batch_size], dec_targets[i : i + batch_size])+ idx = indices[i : i + batch_size]+ batch = (np.array(enc_inputs[idx]), np.array(dec_inputs[idx]), np.array(dec_targets[idx]))if loss_mask is not None:- batch = batch + (loss_mask[i : i + batch_size],)+ batch = batch + (np.array(loss_mask[idx]),)yield batchdef load_tool_call_audio(split="train", max_samples=None):- """Load tool-call dataset entries that have TTS audio.-- Applies the same 90/10 split as load_tool_calls, then filters to entries- where the audio column is not None.+ """Return dataset-global indices for the given split.- Returns list of dicts: {query, answers, tools, audio_bytes, sampling_rate}.+ Applies the same 90/10 split as load_tool_calls. Audio is NOT loaded into memory."""ds = _load_unified_dataset()n = len(ds)if split in ("validation", "val", "test"):- indices = list(range(int(n * 0.9), n))+ start, end = int(n * 0.9), nelif split == "train":- indices = list(range(int(n * 0.9)))+ start, end = 0, int(n * 0.9)else:- indices = list(range(n))+ start, end = 0, n++ indices = list(range(start, end))+ if max_samples:+ indices = indices[:max_samples]+ return indices++def load_audio_for_index(idx):+ """Load and decode audio for a single dataset index.++ Returns (audio_array, sampling_rate) or (None, None) if no audio.+ """import ioimport soundfile as sf- pairs = []- for idx in indices:- ex = ds[idx]- audio_val = ex.get("audio")- if audio_val is None:- continue- # With decode=False, audio_val is {"bytes": b"...", "path": ...}- raw_bytes = None- if isinstance(audio_val, dict):- raw_bytes = audio_val.get("bytes")- elif isinstance(audio_val, bytes):- raw_bytes = audio_val- if raw_bytes is None:- continue- audio_array, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")- if audio_array.ndim > 1:- audio_array = audio_array.mean(axis=1)- pairs.append({- "query": ex["query"],- "answers": ex["answers"],- "tools": ex["tools"],- "audio_array": audio_array.astype(np.float32),- "sampling_rate": sr,- })- if max_samples and len(pairs) >= max_samples:- break+ ds = _load_unified_dataset()+ ex = ds[idx]+ audio_val = ex.get("audio")+ if audio_val is None:+ return None, None++ raw_bytes = None+ if isinstance(audio_val, dict):+ raw_bytes = audio_val.get("bytes")+ elif isinstance(audio_val, bytes):+ raw_bytes = audio_val+ if raw_bytes is None:+ return None, None++ audio_array, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")+ if audio_array.ndim > 1:+ audio_array = audio_array.mean(axis=1)+ return audio_array.astype(np.float32), sr- return pairs++def load_example_with_audio(idx):+ """Load a dataset example with decoded audio for eval use.++ Returns dict with {query, answers, tools, audio_array, sampling_rate}.+ """+ ds = _load_unified_dataset()+ ex = ds[idx]+ audio, sr = load_audio_for_index(idx)+ return {+ "query": ex["query"],+ "answers": ex["answers"],+ "tools": ex["tools"],+ "audio_array": audio,+ "sampling_rate": sr,+ }def compute_mel_spectrogram(audio, sr=16000, n_mels=80, n_fft=400, hop_length=160):@@ -488,10 +590,99 @@ def compute_mel_spectrogram(audio, sr=16000, n_mels=80, n_fft=400, hop_length=16return log_mel.astype(np.float32)-def _speech_cache_key(n_samples, n_mels, max_mel_len, max_dec_len):- tok_hash = _tokenizer_hash()- key = f"speech_{tok_hash}_{n_samples}_{n_mels}_{max_mel_len}_{max_dec_len}"- return hashlib.md5(key.encode()).hexdigest()[:12]+def precompute_mels(kept_indices, n_mels=80, max_mel_len=1024, cache_id_prefix=""):+ """Precompute mel spectrograms -> write to disk via writable memmap.++ Creates {cache_id}_mels.npy of shape (N, max_mel_len, n_mels) float32.+ Processes one sample at a time, writing to the memmap in-place.+ Uploads to GCS when done. Returns the cache_id.+ """+ n_samples = len(kept_indices)+ cache_id = _mel_cache_key(cache_id_prefix, n_samples, n_mels, max_mel_len)+ cache_path = os.path.join(CACHE_DIR, cache_id)+ mel_file = cache_path + "_mels.npy"++ if os.path.exists(mel_file):+ print(f" Mel cache already exists ({cache_id})")+ return cache_id++ if _gcs_cache_download(cache_id, ["_mels.npy"]):+ print(f" Downloaded mel cache from GCS ({cache_id})")+ return cache_id++ os.makedirs(CACHE_DIR, exist_ok=True)+ shape = (n_samples, max_mel_len, n_mels)+ # Create the file with header so np.load(mmap_mode=...) works later+ fp = np.lib.format.open_memmap(mel_file, mode='w+', dtype=np.float32, shape=shape)++ print(f" Precomputing {n_samples} mel spectrograms (n_mels={n_mels}, max_mel_len={max_mel_len})...")+ for i, idx in enumerate(tqdm(kept_indices, desc=" Computing mels")):+ audio, sr = load_audio_for_index(int(idx))+ if audio is None:+ # Leave zeros for missing audio+ continue+ mel = compute_mel_spectrogram(audio, sr=sr, n_mels=n_mels)+ if mel.shape[0] > max_mel_len:+ mel = mel[:max_mel_len]+ t = mel.shape[0]+ fp[i, :t, :] = mel++ del fp # flush to disk+ _gcs_cache_upload(cache_id, ["_mels.npy"])+ print(f" Cached mel spectrograms to {CACHE_DIR}/{cache_id}")+ return cache_id+++def load_prepared_data(split, mmap=False):+ """Load pre-tokenized .npy files. If mmap=True, returns memory-mapped arrays.++ Tries local cache first, then GCS download. Raises FileNotFoundError+ if not found (no fallback tokenization — run 'needle tokenize' first).++ Returns dict with keys: enc_inputs, dec_inputs, dec_targets, loss_mask,+ kept_indices, mel_cache_id.+ """+ meta = _load_cache_metadata(split)+ if meta is None:+ raise FileNotFoundError(+ f"No prepared data found for split '{split}'. Run 'needle tokenize' first."+ )++ text_cache_id = meta["text_cache_id"]+ cache_path = os.path.join(CACHE_DIR, text_cache_id)+ tc_suffixes = ["_enc.npy", "_dec_in.npy", "_dec_tgt.npy", "_loss_mask.npy", "_kept_idx.npy"]++ # Try local, then GCS+ if not os.path.exists(cache_path + "_enc.npy"):+ if not _gcs_cache_download(text_cache_id, tc_suffixes):+ raise FileNotFoundError(+ f"Text cache '{text_cache_id}' not found. Run 'needle tokenize' first."+ )++ mmap_mode = "r" if mmap else None+ return {+ "enc_inputs": np.load(cache_path + "_enc.npy", mmap_mode=mmap_mode),+ "dec_inputs": np.load(cache_path + "_dec_in.npy", mmap_mode=mmap_mode),+ "dec_targets": np.load(cache_path + "_dec_tgt.npy", mmap_mode=mmap_mode),+ "loss_mask": np.load(cache_path + "_loss_mask.npy", mmap_mode=mmap_mode),+ "kept_indices": np.load(cache_path + "_kept_idx.npy", mmap_mode=mmap_mode),+ "mel_cache_id": meta.get("mel_cache_id"),+ }+++def load_prepared_mels(mel_cache_id, mmap=False):+ """Load precomputed mel .npy file, optionally memory-mapped."""+ cache_path = os.path.join(CACHE_DIR, mel_cache_id)+ mel_file = cache_path + "_mels.npy"++ if not os.path.exists(mel_file):+ if not _gcs_cache_download(mel_cache_id, ["_mels.npy"]):+ raise FileNotFoundError(+ f"Mel cache '{mel_cache_id}' not found. Run 'needle tokenize' first."+ )++ mmap_mode = "r" if mmap else None+ return np.load(mel_file, mmap_mode=mmap_mode)def build_audio_augmenter(sr=16000):@@ -517,99 +708,6 @@ def build_audio_augmenter(sr=16000):])-def prepare_voice_tool_call_pairs(pairs, tokenizer, n_mels=80, max_mel_len=1024, max_dec_len=1024):- """Prepare voice-tool-call decoder arrays with <tool_call> task token.-- Caches only decoder tokens + loss mask + audio arrays (NOT mels).- Mel spectrograms are computed on-the-fly by get_speech_batches to allow- online waveform augmentation during training.-- Returns (audio_arrays, dec_inputs, dec_targets, loss_mask).- """-- cache_id = _speech_cache_key(len(pairs), n_mels, max_mel_len, max_dec_len)- cache_path = os.path.join(CACHE_DIR, cache_id)- arrays_file = cache_path + "_arrays.npy"- if os.path.exists(arrays_file):- print(f"Loading cached voice-tool-call data ({cache_id})...")- return (- np.load(arrays_file, allow_pickle=True),- np.load(cache_path + "_dec_in.npy"),- np.load(cache_path + "_dec_tgt.npy"),- np.load(cache_path + "_loss_mask.npy"),- )-- pad_id = tokenizer.pad_token_id- eos_id = tokenizer.eos_token_id- tool_call_id = tokenizer.tool_call_token_id-- audio_arrays = []- dec_inputs_list = []- dec_targets_list = []- loss_mask_list = []-- skipped = 0- for pair in tqdm(pairs, desc="Processing voice-tool-call"):- t_tok = tokenizer.encode(pair["tools"])[:max_dec_len - 2]- a_tok = tokenizer.encode(pair["answers"])-- prefix_len = 2 + len(t_tok)- total_dec = prefix_len + len(a_tok) + 1-- if total_dec > max_dec_len:- available_for_tools = max_dec_len - 2 - len(a_tok) - 1- if available_for_tools < 1:- skipped += 1- continue- t_tok = t_tok[:available_for_tools]- prefix_len = 2 + len(t_tok)-- dec_in = np.full(max_dec_len, pad_id, dtype=np.int32)- dec_in[0] = eos_id- dec_in[1] = tool_call_id- tl = len(t_tok)- if tl > 0:- dec_in[2:2 + tl] = t_tok- al = len(a_tok)- if al > 0:- dec_in[prefix_len:prefix_len + al] = a_tok-- # Decoder target: [<tool_call>, tools..., answer..., EOS]- dec_tgt = np.full(max_dec_len, pad_id, dtype=np.int32)- dec_tgt[0] = tool_call_id- if tl > 0:- dec_tgt[1:1 + tl] = t_tok- if al > 0:- dec_tgt[prefix_len - 1:prefix_len - 1 + al] = a_tok- dec_tgt[prefix_len - 1 + al] = eos_id-- # Loss mask: 1 on answer positions + EOS only- lm = np.zeros(max_dec_len, dtype=np.float32)- lm[prefix_len - 1:prefix_len - 1 + al + 1] = 1.0-- audio_arrays.append(pair["audio_array"])- dec_inputs_list.append(dec_in)- dec_targets_list.append(dec_tgt)- loss_mask_list.append(lm)-- if skipped > 0:- print(f" Skipped {skipped} examples (too long for max_dec_len={max_dec_len})")-- audio_arrays = np.array(audio_arrays, dtype=object)- dec_inputs = np.stack(dec_inputs_list)- dec_targets = np.stack(dec_targets_list)- loss_mask = np.stack(loss_mask_list)-- os.makedirs(CACHE_DIR, exist_ok=True)- np.save(arrays_file, audio_arrays)- np.save(cache_path + "_dec_in.npy", dec_inputs)- np.save(cache_path + "_dec_tgt.npy", dec_targets)- np.save(cache_path + "_loss_mask.npy", loss_mask)- print(f"Cached {len(audio_arrays):,} voice-tool-call pairs to {CACHE_DIR}/{cache_id}")-- return audio_arrays, dec_inputs, dec_targets, loss_mask--def _load_mel_batch(audio_arrays, n_mels, max_mel_len, augmenter=None, sr=16000):"""Compute mel spectrograms for a batch of audio arrays.@@ -637,25 +735,91 @@ def _load_mel_batch(audio_arrays, n_mels, max_mel_len, augmenter=None, sr=16000)return np.stack(mels).astype(np.float32)-def get_speech_batches(audio_arrays, dec_inputs, dec_targets, batch_size,- shuffle=True, loss_mask=None, n_mels=80, max_mel_len=1024,- augmenter=None):- """Yield speech batches with on-the-fly mel computation and optional augmentation.+def _load_audio_batch(ds_indices):+ """Load and decode audio for a batch of dataset indices."""+ import io+ import soundfile as sf- Args:- audio_arrays: array of audio waveform arrays (numpy float32)- dec_inputs, dec_targets, loss_mask: pre-tokenized decoder arrays- augmenter: audiomentations Compose pipeline (None = no augmentation)+ ds = _load_unified_dataset()+ arrays = []+ for idx in ds_indices:+ ex = ds[int(idx)]+ audio_val = ex.get("audio")+ raw_bytes = None+ if isinstance(audio_val, dict):+ raw_bytes = audio_val.get("bytes")+ elif isinstance(audio_val, bytes):+ raw_bytes = audio_val+ if raw_bytes is None:+ arrays.append(np.zeros(16000, dtype=np.float32))+ continue+ audio_array, sr = sf.read(io.BytesIO(raw_bytes), dtype="float32")+ if audio_array.ndim > 1:+ audio_array = audio_array.mean(axis=1)+ arrays.append(audio_array.astype(np.float32))+ return arrays+++def get_speech_batches(mel_data, dec_inputs, dec_targets, batch_size,+ shuffle=True, loss_mask=None):+ """Yield speech batches from precomputed mel data.++ mel_data: array of shape (N, max_mel_len, n_mels), possibly memory-mapped.+ Uses per-batch fancy indexing to avoid copying full arrays."""- n = len(audio_arrays)+ n = len(mel_data)indices = np.random.permutation(n) if shuffle else np.arange(n)for i in range(0, n - batch_size + 1, batch_size):- batch_idx = indices[i : i + batch_size]- batch_audio = audio_arrays[batch_idx]- mel_batch = _load_mel_batch(batch_audio, n_mels, max_mel_len, augmenter=augmenter)-- batch = (mel_batch, dec_inputs[batch_idx], dec_targets[batch_idx])+ idx = indices[i : i + batch_size]+ batch = (np.array(mel_data[idx]), np.array(dec_inputs[idx]), np.array(dec_targets[idx]))if loss_mask is not None:- batch = batch + (loss_mask[batch_idx],)+ batch = batch + (np.array(loss_mask[idx]),)yield batch+++class PrefetchIterator:+ """Generic prefetch wrapper: runs any batch-generating callable in a background thread."""++ def __init__(self, generator_fn, prefetch=4):+ """generator_fn: callable that returns an iterable of batches."""+ self._queue = queue.Queue(maxsize=prefetch)+ self._stop = threading.Event()+ self._generator_fn = generator_fn+ self._thread = threading.Thread(target=self._produce, daemon=True)+ self._thread.start()++ def _produce(self):+ try:+ for batch in self._generator_fn():+ if self._stop.is_set():+ return+ self._queue.put(batch)+ self._queue.put(None) # sentinel+ except Exception as e:+ self._queue.put(e)++ def __iter__(self):+ return self++ def __next__(self):+ item = self._queue.get()+ if item is None:+ raise StopIteration+ if isinstance(item, Exception):+ raise item+ return item++ def close(self):+ self._stop.set()+ while not self._queue.empty():+ try:+ self._queue.get_nowait()+ except queue.Empty:+ break+ self._thread.join(timeout=5)+++def count_batches(n_samples, batch_size):+ """Return the number of full batches for a dataset of n_samples."""+ return n_samples // batch_sizediff --git a/src/run.py b/src/run.pyindex a9a00ba..eb3cb9b 100644--- a/src/run.py+++ b/src/run.py@@ -16,6 +16,30 @@ from .model import ()+_decode_fn_cache = {}+++def _get_decode_fn(model, max_gen_len):+ """Return a JIT-compiled decode function, cached by (model, max_gen_len).++ params is an explicit argument (not closed over) so the same compiled+ function can be reused across calls with different params.+ """+ key = (id(model), max_gen_len)+ if key not in _decode_fn_cache:+ tgt_mask = make_causal_mask(max_gen_len)++ @jax.jit+ def decode_step(params, dec_buffer, encoder_out):+ return model.apply(+ {"params": params}, dec_buffer, encoder_out,+ self_mask=tgt_mask, method="decode",+ )++ _decode_fn_cache[key] = decode_step+ return _decode_fn_cache[key]++def load_checkpoint(path):with open(path, "rb") as f:data = pickle.load(f)@@ -42,28 +66,15 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed={"params": params}, enc_input, src_mask=src_mask, method="encode")- # Build decoder prefix: [BOS, <tool_call>, tools_tokens...]tools_tokens = tokenizer.encode(tools)prefix = [eos_id, tool_call_id] + tools_tokens- prefix_len = len(prefix)+ prefix_len = min(len(prefix), max_gen_len)- # Fixed-size buffer so JAX compiles once- tgt_mask = make_causal_mask(max_gen_len)dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)- # Fill prefixfor j, tok in enumerate(prefix[:max_gen_len]):dec_buffer = dec_buffer.at[0, j].set(tok)- @jax.jit- def decode_step(dec_buffer, encoder_out):- logits = model.apply(- {"params": params},- dec_buffer,- encoder_out,- self_mask=tgt_mask,- method="decode",- )- return logits+ decode_fn = _get_decode_fn(model, max_gen_len)generated_tokens = []@@ -71,7 +82,7 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=sys.stdout.write(f"\n")sys.stdout.flush()- logits = decode_step(dec_buffer, encoder_out)+ logits = decode_fn(params, dec_buffer, encoder_out)for i in range(prefix_len - 1, max_gen_len - 1):next_logits = logits[0, i]@@ -87,7 +98,7 @@ def generate(model, params, tokenizer, query, tools="[]", max_gen_len=512, seed=sys.stdout.write(tokenizer.decode([next_token]))sys.stdout.flush()- logits = decode_step(dec_buffer, encoder_out)+ logits = decode_fn(params, dec_buffer, encoder_out)if stream:sys.stdout.write("\n")@@ -99,10 +110,8 @@ def load_audio(path, target_sr=16000):"""Load an audio file and resample to target_sr. Returns (audio_array, sr)."""import soundfile as sfaudio, sr = sf.read(path, dtype="float32")- # Convert stereo to monoif audio.ndim > 1:audio = audio.mean(axis=1)- # Resample if neededif sr != target_sr:from scipy.signal import resamplenum_samples = int(len(audio) * target_sr / sr)@@ -130,26 +139,15 @@ def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="{"params": params}, mel_input, src_mask=src_mask, deterministic=True, method="encode_speech")- # Build decoder prefix: [BOS, <tool_call>, tools_tokens...]tools_tokens = tokenizer.encode(tools)prefix = [eos_id, tool_call_id] + tools_tokens- prefix_len = len(prefix)+ prefix_len = min(len(prefix), max_gen_len)- tgt_mask = make_causal_mask(max_gen_len)dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)for j, tok in enumerate(prefix[:max_gen_len]):dec_buffer = dec_buffer.at[0, j].set(tok)- @jax.jit- def decode_step(dec_buffer, encoder_out):- logits = model.apply(- {"params": params},- dec_buffer,- encoder_out,- self_mask=tgt_mask,- method="decode",- )- return logits+ decode_fn = _get_decode_fn(model, max_gen_len)generated_tokens = []@@ -157,7 +155,7 @@ def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="sys.stdout.write("\n")sys.stdout.flush()- logits = decode_step(dec_buffer, encoder_out)+ logits = decode_fn(params, dec_buffer, encoder_out)for i in range(prefix_len - 1, max_gen_len - 1):next_logits = logits[0, i]@@ -173,7 +171,7 @@ def generate_from_audio(model, params, tokenizer, audio_array, sr=16000, tools="sys.stdout.write(tokenizer.decode([next_token]))sys.stdout.flush()- logits = decode_step(dec_buffer, encoder_out)+ logits = decode_fn(params, dec_buffer, encoder_out)if stream:sys.stdout.write("\n")@@ -191,7 +189,6 @@ def main(args):param_count = sum(x.size for x in jax.tree.leaves(params))print(f"Model parameters: {param_count:,}")- # --- Voice-to-tool-call mode ---audio_files = getattr(args, "audio", None)if audio_files:tools = getattr(args, "tools", None) or "[]"@@ -212,7 +209,6 @@ def main(args):)return- # --- Tool-call generation mode ---query = getattr(args, "query", None)tools = getattr(args, "tools", None) or "[]"diff --git a/src/test.py b/src/test.pyindex b454a97..93502d2 100644--- a/src/test.py+++ b/src/test.py@@ -7,14 +7,14 @@ import jax.numpy as jnpimport numpy as npimport optax-from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs, load_tool_call_audio+from .data import get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs, load_tool_call_audio, load_example_with_audiofrom .model import (EncoderDecoderTransformer,TransformerConfig,make_causal_mask,make_padding_mask,)-from .run import load_checkpoint+from .run import load_checkpoint, _get_decode_fndef compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batch_size, pad_id, loss_mask=None):@@ -54,22 +54,15 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is theos_id = tokenizer.eos_token_idsrc_mask = make_padding_mask(enc_input, pad_id)- tgt_mask = make_causal_mask(max_gen_len)-- @jax.jit- def decode_step(dec_buffer, encoder_out):- logits = model.apply(- {"params": params}, dec_buffer, encoder_out,- self_mask=tgt_mask, method="decode",- )- return logits+ decode_fn = _get_decode_fn(model, max_gen_len)+ # Warmup JITencoder_out = model.apply({"params": params}, enc_input, src_mask=src_mask, method="encode")dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)dec_buffer = dec_buffer.at[0, 0].set(eos_id)- decode_step(dec_buffer, encoder_out)+ decode_fn(params, dec_buffer, encoder_out)tokens_generated = []latencies = []@@ -83,7 +76,7 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is thencoder_out = model.apply({"params": params}, enc_input, src_mask=src_mask, method="encode")- logits = decode_step(dec_buffer, encoder_out)+ logits = decode_fn(params, dec_buffer, encoder_out)num_tokens = 0for i in range(max_gen_len - 1):@@ -95,7 +88,7 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt='What is thnum_tokens += 1dec_buffer = dec_buffer.at[0, i + 1].set(next_token)- logits = decode_step(dec_buffer, encoder_out)+ logits = decode_fn(params, dec_buffer, encoder_out)elapsed = time.perf_counter() - starttokens_generated.append(num_tokens)@@ -280,7 +273,7 @@ def benchmark_voice_tool_calls(model, params, tokenizer, num_samples=100, max_geimport jsonfrom .run import generate_from_audio- pairs = load_tool_call_audio("validation", max_samples=num_samples)+ indices = load_tool_call_audio("validation", max_samples=num_samples)total = 0exact_match = 0@@ -295,13 +288,14 @@ def benchmark_voice_tool_calls(model, params, tokenizer, num_samples=100, max_geempty_pred = 0samples = []- for i, pair in enumerate(pairs):+ for i, ds_idx in enumerate(indices):+ pair = load_example_with_audio(ds_idx)+ if pair["audio_array"] is None:+ continueref_text = pair["answers"]- audio = pair["audio_array"]- sr = pair["sampling_rate"]pred_text = generate_from_audio(- model, params, tokenizer, audio, sr=sr,+ model, params, tokenizer, pair["audio_array"], sr=pair["sampling_rate"],tools=pair["tools"], max_gen_len=max_gen_len, seed=i, stream=False,).strip()@@ -387,7 +381,7 @@ def main(args):print(f"\nevaluating tool-call perplexity ({args.max_eval_samples} samples)...")ds = load_tool_calls("validation", max_samples=args.max_eval_samples)- enc_inputs, dec_inputs, dec_targets, loss_mask_arr = prepare_tool_call_pairs(+ enc_inputs, dec_inputs, dec_targets, loss_mask_arr, _ = prepare_tool_call_pairs(ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len)ppl = compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, args.batch_size, config.pad_token_id, loss_mask=loss_mask_arr)diff --git a/src/tokenize_data.py b/src/tokenize_data.pynew file mode 100644index 0000000..fccde4e--- /dev/null+++ b/src/tokenize_data.py@@ -0,0 +1,96 @@+"""Standalone tokenization pipeline: train tokenizer and pre-tokenize all data.++Trains the SentencePiece tokenizer and tokenizes train + val splits for both+text and voice data, caching everything on GCS. Running fresh overwrites all+existing tokenizer + caches.++Usage:+ needle tokenize # full run+ needle tokenize --max-samples 1000 # dev/test+ needle tokenize --cleanup # delete local cache after GCS upload+"""++import os+import shutil+import subprocess++from .data import (+ CACHE_DIR,+ GCS_CACHE_PATH,+ GCS_TOKENIZER_PATH,+ TOKENIZER_DIR,+ _cache_key,+ _save_cache_metadata,+ get_tokenizer,+ load_tool_calls,+ precompute_mels,+ prepare_tool_call_pairs,+ train_tokenizer,+ upload_tokenizer_to_gcs,+)+++def _clear_gcs_caches():+ """Remove existing GCS cache and tokenizer files."""+ for path in [GCS_CACHE_PATH + "/*", GCS_TOKENIZER_PATH + "*"]:+ print(f"Clearing {path} ...")+ subprocess.run(+ ["gcloud", "storage", "rm", "-r", path],+ capture_output=True, text=True,+ )+++def _clear_local_caches():+ """Remove local .data_cache/ and tokenizer/ directories."""+ for d in [CACHE_DIR, TOKENIZER_DIR]:+ if os.path.exists(d):+ print(f"Removing {d}/ ...")+ shutil.rmtree(d)+++def tokenize(args):+ # 1. Clear existing caches+ print("=== Clearing existing caches ===")+ _clear_gcs_caches()+ _clear_local_caches()++ # 2. Train tokenizer and upload to GCS+ print("\n=== Training tokenizer ===")+ train_tokenizer(max_samples=args.max_samples, force=True)+ upload_tokenizer_to_gcs()++ # 3. Load tokenizer+ tokenizer = get_tokenizer()++ # 4. Tokenize text data + precompute mels (train + val)+ print("\n=== Tokenizing text data + precomputing mels ===")+ max_enc_len = getattr(args, "max_enc_len", 256)+ max_dec_len = getattr(args, "max_dec_len", 1024)+ n_mels = getattr(args, "n_mels", 80)+ max_mel_len = getattr(args, "max_mel_len", 1024)++ for split in ("train", "val"):+ print(f"\n--- {split} split ---")+ ds = load_tool_calls(split=split, max_samples=args.max_samples)+ _, _, _, _, kept_indices = prepare_tool_call_pairs(+ ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len+ )+ text_cache_id = _cache_key("toolcall", len(ds), max_enc_len, max_dec_len)++ # Precompute mel spectrograms+ mel_cache_id = precompute_mels(+ kept_indices, n_mels=n_mels, max_mel_len=max_mel_len,+ cache_id_prefix=split,+ )++ # Save metadata for train.py to find caches+ _save_cache_metadata(split, text_cache_id, mel_cache_id, len(kept_indices),+ max_enc_len, max_dec_len, n_mels, max_mel_len)++ # 5. Optionally clean up local cache+ if args.cleanup and os.path.exists(CACHE_DIR):+ print(f"\n=== Cleaning up {CACHE_DIR}/ ===")+ shutil.rmtree(CACHE_DIR)+ print("Done.")++ print("\n=== Tokenization pipeline complete ===")diff --git a/src/train.py b/src/train.pyindex 9e14141..35bfa57 100644--- a/src/train.py+++ b/src/train.py@@ -14,9 +14,10 @@ from flax import jax_utilsfrom flax.training import train_statefrom .data import (- get_batches, get_tokenizer, load_tool_calls, prepare_tool_call_pairs,- load_tool_call_audio, prepare_voice_tool_call_pairs, get_speech_batches,- build_audio_augmenter,+ get_batches, get_tokenizer, get_speech_batches,+ load_prepared_data, load_prepared_mels,+ load_example_with_audio,+ PrefetchIterator, count_batches,)from .model import (EncoderDecoderTransformer,@@ -456,7 +457,7 @@ def train(args):if wandb.run is None:wandb.init(project="needle-v1", config=vars(args))- total_data_steps = 6 if not no_speech else 4+ total_data_steps = 4 if not no_speech else 3step_idx = 0step_idx += 1@@ -468,53 +469,27 @@ def train(args):tokenizer = get_tokenizer(max_samples=args.max_samples)step_idx += 1- print(f"\n[{step_idx}/{total_data_steps}] Loading tool-calling dataset...")- ds = load_tool_calls("train", max_samples=args.max_samples)- val_ds = load_tool_calls("validation", max_samples=getattr(args, "max_eval_samples", None))-- step_idx += 1- print(f"\n[{step_idx}/{total_data_steps}] Preparing tool-call pairs...")- enc_inputs, dec_inputs, dec_targets, train_loss_mask = prepare_tool_call_pairs(- ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len- )- val_enc, val_dec_in, val_dec_tgt, val_loss_mask = prepare_tool_call_pairs(- val_ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len- )- val_enc = np.array(val_enc)- val_dec_in = np.array(val_dec_in)- val_dec_tgt = np.array(val_dec_tgt)- val_loss_mask = np.array(val_loss_mask)- print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs")-- speech_audio_arrays = speech_dec_in = speech_dec_tgt = speech_loss_mask_arr = None- val_speech_audio_arrays = val_speech_dec_in = val_speech_dec_tgt = val_speech_loss_mask = None- audio_augmenter = None+ print(f"\n[{step_idx}/{total_data_steps}] Loading prepared data from disk (mmap)...")+ train_data = load_prepared_data("train", mmap=True)+ val_data = load_prepared_data("val", mmap=True)+ enc_inputs = train_data["enc_inputs"]+ dec_inputs = train_data["dec_inputs"]+ dec_targets = train_data["dec_targets"]+ train_loss_mask = train_data["loss_mask"]+ val_enc = val_data["enc_inputs"]+ val_dec_in = val_data["dec_inputs"]+ val_dec_tgt = val_data["dec_targets"]+ val_loss_mask = val_data["loss_mask"]+ print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs (memory-mapped)")++ train_mels = None+ val_mels = Noneif not no_speech:step_idx += 1- max_speech_samples = getattr(args, "max_speech_samples", None)- print(f"\n[{step_idx}/{total_data_steps}] Loading voice-tool-call audio...")- speech_pairs = load_tool_call_audio("train", max_samples=max_speech_samples)- speech_val_pairs = load_tool_call_audio("validation", max_samples=getattr(args, "max_eval_samples", None))-- step_idx += 1- print(f"\n[{step_idx}/{total_data_steps}] Preparing voice-tool-call pairs...")- speech_audio_arrays, speech_dec_in, speech_dec_tgt, speech_loss_mask_arr = prepare_voice_tool_call_pairs(- speech_pairs, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len- )- val_speech_audio_arrays, val_speech_dec_in, val_speech_dec_tgt, val_speech_loss_mask = prepare_voice_tool_call_pairs(- speech_val_pairs, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len- )- speech_dec_in = np.array(speech_dec_in)- speech_dec_tgt = np.array(speech_dec_tgt)- speech_loss_mask_arr = np.array(speech_loss_mask_arr)- val_speech_dec_in = np.array(val_speech_dec_in)- val_speech_dec_tgt = np.array(val_speech_dec_tgt)- val_speech_loss_mask = np.array(val_speech_loss_mask)- print(f" {len(speech_audio_arrays):,} train / {len(val_speech_audio_arrays):,} val voice-tool-call pairs")-- audio_augmenter = build_audio_augmenter(sr=16000)- if audio_augmenter is not None:- print(f" Audio augmentation: enabled ({len(audio_augmenter.transforms)} transforms)")+ print(f"\n[{step_idx}/{total_data_steps}] Loading precomputed mel spectrograms (mmap)...")+ train_mels = load_prepared_mels(train_data["mel_cache_id"], mmap=True)+ val_mels = load_prepared_mels(val_data["mel_cache_id"], mmap=True)+ print(f" {len(train_mels):,} train / {len(val_mels):,} val mel spectrograms (memory-mapped)")effective_batch_size = args.batch_size * num_devices@@ -563,9 +538,9 @@ def train(args):mat_shared_input = getattr(args, "mat_shared_input", False)unique_batch_size = effective_batch_size // n_widths if (mat_shared_input and n_widths > 1) else effective_batch_size- text_batches_per_epoch = len(enc_inputs) // unique_batch_size- if not no_speech and speech_audio_arrays is not None:- speech_batches_per_epoch = len(speech_audio_arrays) // unique_batch_size+ text_batches_per_epoch = count_batches(len(enc_inputs), unique_batch_size)+ if not no_speech and train_mels is not None:+ speech_batches_per_epoch = text_batches_per_epochelse:speech_batches_per_epoch = 0num_batches = text_batches_per_epoch + speech_batches_per_epoch@@ -598,7 +573,7 @@ def train(args):print(f" Memory slots {config.num_memory_slots:>12}")print(f" Activation {config.activation:>12}")print(f" Dtype {config.dtype:>12}")- if not no_speech:+ if not no_speech and train_mels is not None:print(f" Speech {speech_batches_per_epoch} batches/epoch")print(f" n_mels {n_mels:>12}")print(f" max_mel_len {max_mel_len:>12}")@@ -637,9 +612,8 @@ def train(args):muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps)tokens_per_batch = effective_batch_size * (args.max_enc_len + args.max_dec_len)- enc_inputs = np.array(enc_inputs)- dec_inputs = np.array(dec_inputs)- dec_targets = np.array(dec_targets)+ # Create model instance once (JIT caches persist across epochs)+ eval_model = EncoderDecoderTransformer(config)last_val_ppl = Nonesparsity_ratio = getattr(args, "sparsity_ratio", 0.0)@@ -662,39 +636,43 @@ def train(args):text_losses = []speech_losses = []- text_batches = list(get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size, loss_mask=train_loss_mask))-- speech_batch_list = []- if not no_speech and speech_audio_arrays is not None:- speech_batch_list = list(- get_speech_batches(speech_audio_arrays, speech_dec_in, speech_dec_tgt, unique_batch_size,- loss_mask=speech_loss_mask_arr, n_mels=n_mels, max_mel_len=max_mel_len,- augmenter=audio_augmenter)+ text_batch_iter = PrefetchIterator(+ lambda: get_batches(enc_inputs, dec_inputs, dec_targets, unique_batch_size,+ loss_mask=train_loss_mask),+ prefetch=4,+ )++ speech_batch_iter = None+ if not no_speech and train_mels is not None:+ speech_batch_iter = PrefetchIterator(+ lambda: get_speech_batches(train_mels, dec_inputs, dec_targets, unique_batch_size,+ loss_mask=train_loss_mask),+ prefetch=4,)- steps_this_epoch = len(text_batches) + len(speech_batch_list)+ steps_this_epoch = text_batches_per_epoch + speech_batches_per_epochtext_idx = 0speech_idx = 0+ speech_loss_val = Nonepbar = tqdm(range(steps_this_epoch), desc=f"Epoch {epoch + 1}/{args.epochs}")for step_i in pbar:t0 = time.perf_counter()- do_speech = (step_i % 2 == 1) and speech_idx < len(speech_batch_list)- do_text = not do_speech and text_idx < len(text_batches)+ do_speech = (step_i % 2 == 1) and speech_idx < speech_batches_per_epoch+ do_text = not do_speech and text_idx < text_batches_per_epochif not do_speech and not do_text:- if text_idx < len(text_batches):+ if text_idx < text_batches_per_epoch:do_text = True- elif speech_idx < len(speech_batch_list):+ elif speech_idx < speech_batches_per_epoch:do_speech = Trueelse:break- speech_loss_val = Nonestep_grad_norm = Noneif do_text:- src, tgt_in, tgt_out, lm = text_batches[text_idx]+ src, tgt_in, tgt_out, lm = next(text_batch_iter)text_idx += 1if n_widths > 1 and mat_shared_input:@@ -727,7 +705,7 @@ def train(args):global_step += 1else:- mel_batch, sp_tgt_in, sp_tgt_out, sp_lm = speech_batch_list[speech_idx]+ mel_batch, sp_tgt_in, sp_tgt_out, sp_lm = next(speech_batch_iter)speech_idx += 1if n_widths > 1 and mat_shared_input:@@ -815,6 +793,10 @@ def train(args):log_dict["val/text_ppl"] = last_val_pplwandb.log(log_dict)+ text_batch_iter.close()+ if speech_batch_iter is not None:+ speech_batch_iter.close()+if epoch == weight_prune_epoch and not gradual_sparsify_done:gradual_sparsify_done = Trueif prune_mask is None:@@ -839,101 +821,114 @@ def train(args):eval_params = jax_utils.unreplicate(ema_params)val_causal = make_causal_mask(args.max_dec_len)- total_loss, total_toks = 0.0, 0.0- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask):- vl, vt = val_loss_fn(eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])- total_loss += float(vl)- total_toks += float(vt)- last_val_ppl = float(math.exp(min(total_loss / max(total_toks, 1), 20)))+ q_params = _quantize_params(eval_params, group_size=_GROUP_SIZE)+ mat_vl_fns = {}+ if _MAT_FACTORS:+ _apply_fn = jax_utils.unreplicate(state).apply_fn+ mat_vl_fns = {f: _make_mat_val_loss_fn(_apply_fn, fw)+ for f, fw in zip(_MAT_FACTORS, _MAT_FF_WIDTHS)}+ del _apply_fn++ full_loss, full_toks = 0.0, 0.0+ q_loss, q_toks = 0.0, 0.0+ mat_accum = {f: [0.0, 0.0] for f in _MAT_FACTORS}++ for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size,+ shuffle=False, loss_mask=val_loss_mask):+ src, dec_in, dec_tgt, lm = vb[0], vb[1], vb[2], vb[3]+ vl, vt = val_loss_fn(eval_params, src, dec_in, dec_tgt, val_causal, lm)+ full_loss += float(vl); full_toks += float(vt)+ vl, vt = val_loss_fn(q_params, src, dec_in, dec_tgt, val_causal, lm)+ q_loss += float(vl); q_toks += float(vt)+ for f, fn in mat_vl_fns.items():+ vl, vt = fn(eval_params, src, dec_in, dec_tgt, val_causal, lm)+ mat_accum[f][0] += float(vl)+ mat_accum[f][1] += float(vt)++ last_val_ppl = float(math.exp(min(full_loss / max(full_toks, 1), 20)))+ quant_val_ppl = float(math.exp(min(q_loss / max(q_toks, 1), 20)))+ del q_params++ mat_results = {}+ for f in _MAT_FACTORS:+ avg = mat_accum[f][0] / max(mat_accum[f][1], 1)+ mat_results[f] = (float(math.exp(min(avg, 20))),+ _estimate_mat_params(config, f), config.d_ff // f)++ # --- Speech val PPL (separate pass — different data) ---speech_val_ppl = None- if speech_vl_fn is not None and val_speech_audio_arrays is not None:+ if speech_vl_fn is not None and val_mels is not None:sp_total_loss, sp_total_toks = 0.0, 0.0- for sp_batch in get_speech_batches(val_speech_audio_arrays, val_speech_dec_in, val_speech_dec_tgt, args.batch_size,- shuffle=False, loss_mask=val_speech_loss_mask,- n_mels=n_mels, max_mel_len=max_mel_len, augmenter=None):+ for sp_batch in get_speech_batches(val_mels, val_dec_in, val_dec_tgt, args.batch_size,+ shuffle=False, loss_mask=val_loss_mask):vl, vt = speech_vl_fn(eval_params, sp_batch[0], sp_batch[1], sp_batch[2], val_causal, sp_batch[3])sp_total_loss += float(vl)sp_total_toks += float(vt)speech_val_loss = sp_total_loss / max(sp_total_toks, 1)speech_val_ppl = float(math.exp(min(speech_val_loss, 20)))- q_params = _quantize_params(eval_params, group_size=_GROUP_SIZE)- q_total_loss, q_total_toks = 0.0, 0.0- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask):- vl, vt = val_loss_fn(q_params, vb[0], vb[1], vb[2], val_causal, vb[3])- q_total_loss += float(vl)- q_total_toks += float(vt)- quant_val_ppl = float(math.exp(min(q_total_loss / max(q_total_toks, 1), 20)))- del q_params-- mat_results = {}- if _MAT_FACTORS:- apply_fn = jax_utils.unreplicate(state).apply_fn- for factor, ff_w in zip(_MAT_FACTORS, _MAT_FF_WIDTHS):- mat_vl_fn = _make_mat_val_loss_fn(apply_fn, ff_w)- mat_total_loss, mat_total_toks = 0.0, 0.0- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False, loss_mask=val_loss_mask):- vl, vt = mat_vl_fn(eval_params, vb[0], vb[1], vb[2], val_causal, vb[3])- mat_total_loss += float(vl)- mat_total_toks += float(vt)- avg_loss = mat_total_loss / max(mat_total_toks, 1)- mat_ppl = float(math.exp(min(avg_loss, 20)))- mat_params = _estimate_mat_params(config, factor)- mat_results[factor] = (mat_ppl, mat_params, ff_w)- del apply_fn-params_np = jax.tree.map(np.array, eval_params)- del eval_paramstotal_params = sum(x.size for x in jax.tree.leaves(params_np))near_zero = sum(int(np.sum(np.abs(x) < 1e-6)) for x in jax.tree.leaves(params_np))sparsity = near_zero / total_params * 100ckpt_name = f"needle_{args.num_layers}_{args.d_model}_{global_step}.pkl"ckpt_path = os.path.join(args.checkpoint_dir, ckpt_name)-with open(ckpt_path, "wb") as f:pickle.dump({"params": params_np, "config": config.__dict__}, f)-- from .test import measure_throughput, benchmark_tool_calls- from .run import generate_from_audio- eval_params_jnp = jax.tree.map(jnp.array, params_np)- del params_np- model = EncoderDecoderTransformer(config)- tp = measure_throughput(model, eval_params_jnp, tokenizer, num_runs=5)- tc_metrics = benchmark_tool_calls(model, eval_params_jnp, tokenizer, num_samples=20, max_gen_len=128)-- voice_tc_samples = []- if not no_speech and val_speech_audio_arrays is not None:- val_pairs = load_tool_call_audio("validation", max_samples=3)- for i, pair in enumerate(val_pairs):- audio = pair["audio_array"]- sr = pair["sampling_rate"]- pred_text = generate_from_audio(- model, eval_params_jnp, tokenizer, audio, sr=sr,- tools=pair["tools"], max_gen_len=128, seed=i, stream=False,- ).strip()- voice_tc_samples.append((pair["query"][:80], pair["answers"][:120], pred_text[:120]))-- del eval_params_jnp-+ del params_np++ from .test import measure_throughput+ from .run import generate, generate_from_audio+ tp = measure_throughput(eval_model, eval_params, tokenizer, num_runs=5)++ from .data import _load_unified_dataset+ _ds_full = _load_unified_dataset()+ _val_start = int(len(_ds_full) * 0.9)+ val_kept = val_data["kept_indices"]+ n_eval_samples = min(3, len(val_kept))+ eval_indices = val_kept[:n_eval_samples] + _val_start++ unified_samples = []+ for i, ds_idx in enumerate(eval_indices):+ pair = load_example_with_audio(int(ds_idx))+ query = pair["query"][:80]+ ref = pair["answers"][:120]++ # Text prediction+ text_pred = generate(+ eval_model, eval_params, tokenizer, pair["query"],+ tools=pair["tools"], max_gen_len=args.max_dec_len, seed=i, stream=False,+ ).strip()[:120]++ # Voice prediction+ voice_pred = None+ if not no_speech and pair["audio_array"] is not None:+ voice_pred = generate_from_audio(+ eval_model, eval_params, tokenizer, pair["audio_array"], sr=pair["sampling_rate"],+ tools=pair["tools"], max_gen_len=args.max_dec_len, seed=i, stream=False,+ ).strip()[:120]++ unified_samples.append((query, ref, text_pred, voice_pred))++ del eval_params++ final_speech_loss = speech_losses[-1] if speech_losses else Noneprint(f"\n ─────────────────────────────────────")print(f" Epoch {epoch + 1}/{args.epochs}")print(f" ─────────────────────────────────────")- print(f" Avg loss {epoch_avg_loss:>12.4f}")- print(f" Final loss {final_loss:>12.4f}")- print(f" Train ppl {final_ppl:>12.2f}")- print(f" TC val ppl {last_val_ppl:>12.2f}")- if speech_losses:- epoch_avg_speech = sum(speech_losses) / len(speech_losses)- print(f" Avg speech loss{epoch_avg_speech:>12.4f}")+ print(f" Text loss {final_loss:>12.4f}")+ print(f" Text val ppl {last_val_ppl:>12.2f}")+ if final_speech_loss is not None:+ print(f" Speech loss {final_speech_loss:>12.4f}")if speech_val_ppl is not None:print(f" Speech val ppl {speech_val_ppl:>12.2f}")print(f" Quant val ppl {quant_val_ppl:>12.2f} (INT4 g{_GROUP_SIZE})")print(f" Sparsity {sparsity:>11.2f}% ({near_zero:,}/{total_params:,})")if mat_results:print(f" ─────────────────────────────────────")- print(f" Matryoshka exportable sub-models (FFN interior slicing):")+ print(f" Matryoshka sub-models:")print(f" {'factor':>6} {'d_ff':>6} {'val ppl':>10} {'params':>12}")print(f" {'1x':>6} {config.d_ff:>6} {last_val_ppl:>10.2f} {total_params:>12,} (full)")for factor in sorted(mat_results.keys()):@@ -942,25 +937,14 @@ def train(args):print(f" ─────────────────────────────────────")print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")print(f" Latency {tp['avg_latency_s']:>11.3f}s")- print(f" ─── Tool-Call Metrics ───────────────")- print(f" JSON parse {tc_metrics['json_parse_rate']:>10.1%}")- print(f" Exact match {tc_metrics['exact_match']:>10.1%}")- print(f" Name F1 {tc_metrics['name_f1']:>10.3f} (P={tc_metrics['name_precision']:.3f} R={tc_metrics['name_recall']:.3f})")- print(f" Call F1 {tc_metrics['call_f1']:>10.3f} (P={tc_metrics['call_precision']:.3f} R={tc_metrics['call_recall']:.3f})")- if tc_metrics["samples"]:- print(f" ─────────────────────────────────────")- for query, ref, pred in tc_metrics["samples"][:3]:- print(f" Q: {query}")- print(f" R: {ref}")- print(f" P: {pred}")- print()- if voice_tc_samples:- print(f" ─────────────────────────────────────")- print(f" Voice-to-Tool-Call samples:")- for query, ref, pred in voice_tc_samples:+ if unified_samples:+ print(f" ─── Samples ────────────────────────")+ for query, ref, text_pred, voice_pred in unified_samples:print(f" Q: {query}")print(f" R: {ref}")- print(f" P: {pred}")+ print(f" T: {text_pred or '(empty)'}")+ if voice_pred is not None:+ print(f" V: {voice_pred or '(empty)'}")print()print(f" ─────────────────────────────────────")print(f" Checkpoint: {ckpt_path}")@@ -968,19 +952,14 @@ def train(args):if use_wandb:log_dict = {- "epoch/avg_text_loss": epoch_avg_loss,- "epoch/final_text_loss": final_loss,+ "epoch/text_loss": final_loss,"epoch/text_val_ppl": last_val_ppl,"epoch/quant_val_ppl": quant_val_ppl,"epoch/weight_sparsity": sparsity,"epoch": epoch + 1,}- log_dict["epoch/tc_exact_match"] = tc_metrics["exact_match"]- log_dict["epoch/tc_name_f1"] = tc_metrics["name_f1"]- log_dict["epoch/tc_call_f1"] = tc_metrics["call_f1"]- log_dict["epoch/tc_json_parse_rate"] = tc_metrics["json_parse_rate"]- if speech_losses:- log_dict["epoch/avg_speech_loss"] = sum(speech_losses) / len(speech_losses)+ if final_speech_loss is not None:+ log_dict["epoch/speech_loss"] = final_speech_lossif speech_val_ppl is not None:log_dict["epoch/speech_val_ppl"] = speech_val_pplfor factor, (mat_ppl, mat_params, _) in mat_results.items():