Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Add speech
HenryNdubuaku56e88e1adf60
diff --git a/README.md b/README.mdindex 692f384..85dfa9d 100644--- a/README.md+++ b/README.md@@ -11,7 +11,7 @@──────────── ─────────────────┌──────────────┐ ┌──────────────┐- │ Transcription│ │ Tool JSON │ ┌──────────────────────┐+ │ Transcription│ │ Story │ ┌──────────────────────┐└──────┬───────┘ └──────┬───────┘ │ MRL loss at dims │└───────┬─────────┘ │ + INT4 QAT (g=32) │┌───┴───────┐ │ + z-loss + slot div │@@ -27,13 +27,13 @@┌─────┴─────┐ └──────────┬───────────┘│ ZCRMSNorm │ │└─────┬─────┘ ┌──────────┴───────────┐- ┌────────┴────────┐ │ EMA params (β=0.999) │- │ Decoder x N_dec │ └──────────┬───────────┘- │┌───────────────┐│ │- ││ Masked Self ││ ┌──────────┴───────────┐- ││ Attn + RoPE ││ │ Sparsification │- │├───────────────┤│ │ cubic schedule │- ┌──────────────┐ S ││ Cross ││ │ mask every N steps │+ ┌────────┴────────┐ │ Sparsification │+ │ Decoder x N_dec │ │ cubic schedule │+ │┌───────────────┐│ │ mask every N steps │+ ││ Masked Self ││ └──────────┬───────────┘+ ││ Attn + RoPE ││ │+ │├───────────────┤│ ┌──────────┴───────────┐+ ┌──────────────┐ S ││ Cross ││ │ EMA params (β=0.999) ││ MemoryMixer │────────▶ Attention ││ └──────────┬───────────┘│ Encoder │ │├───────────────┤│ ││ x N_enc │ ││ Gated FFN ││ ┌──────────┴───────────┐@@ -44,7 +44,7 @@│ │ RoPE keys│ │ └─────┬─────┘│ ├──────────┤ │ ┌───────┴───────┐│ │Mix: │ │ │ <transcribe> │- │ │ MLP-Mixer│ │ │ or <tools> │+ │ │ MLP-Mixer│ │ │ or <story> ││ │ on S │ │ │ + target ││ ├──────────┤ │ └───────────────┘│ │Local: │ │@@ -56,18 +56,64 @@│ DW Conv ↓2 │ stride-2 depthwise-separable│ S ∈ (M, d) │└──────┬───────┘- ┌──────┴───────┐- │ Embedding │ ← shared- └──────┬───────┘- ┌────┴─────────┐- │ Text/Speech │- │ Input │- └──────────────┘+ │+ ┌────┴──────┐ ┌──────────────┐+ │ Embedding │ │ Mel Proj │+ │ (text) │ │ (speech) │+ └────┬──────┘ └──────┬───────┘+ │ │+ ┌────┴──────┐ ┌──────┴───────┐+ │ Text │ │ Audio │+ │ tokens │ │ mel spec │+ └───────────┘ └──────────────┘d=512 (default) / 1536 (--full) · GQA · QK-normSentencePiece BPE (8192) · gated dReLU · ZCRMSNorm · RoPEstrided DW conv · MRL dims · INT4 QAT · Muon + AdamW- text + speech encoder · <transcribe> / <tools> task routing+ text + speech encoder · <transcribe> / <story> task routing++ Data Preprocessing+ ──────────────────++ Text (TinyStories) Speech (LibriSpeech-100)+ ────────────────── ────────────────────────++ TinyStories LibriSpeech-100+ │ │+ ▼ ▼+ ┌──────────────┐ ┌────────────────┐+ │ SentencePiece│ │ soundfile │+ │ BPE tokenize │ │ decode (16kHz) │+ │ (8192 vocab) │ └───────┬────────┘+ └──────┬───────┘ │+ │ ▼+ ▼ ┌────────────────┐+ ┌──────────────┐ │ Log-mel spec │+ │ Split at 30% │ │ 80 bins, 25ms │+ │ enc / dec │ │ window, 10ms │+ └──┬───────┬───┘ │ hop (~100 f/s) │+ │ │ └───────┬────────┘+ ▼ ▼ │+ ┌─────┐ ┌──────────────┐ ▼+ │ enc │ │ [BOS]<story> │ ┌────────────────┐+ │ ids │ │ + tokens │ │ Pad/trunc to │+ └─────┘ │ → dec_in │ │ max_mel_len │+ ├──────────────┤ └───────┬────────┘+ │ <story> │ │+ │ + tokens │ ▼+ │ + [EOS] │ ┌────────────────┐+ │ → dec_tgt │ │ [BOS]<transcr> │+ └──────────────┘ │ + tokens │+ │ → dec_in │+ Both cached as ├────────────────┤+ .npy in .data_cache/ │ <transcr> │+ │ + tokens │+ ┌──────────────────────┐ │ + [EOS] │+ │ Interleaved batches: │ │ → dec_tgt │+ │ 1 speech every N │ └────────────────┘+ │ text steps │+ │ (--speech-every N) │ Eval: WER on 3 val samples+ └──────────────────────┘ at end of each epoch```## Usage@@ -108,10 +154,16 @@ needle [command]│ --checkpoint PATH Resume from checkpoint ││ --checkpoint-dir DIR Checkpoint directory ││ --seed INT Random seed (default: 42) │+ │ --no-speech Disable speech (text-only training) │+ │ --speech-every INT Speech step every N text (default: 3) │+ │ --max-mel-len INT Max mel frames (default: 1024) │+ │ --n-mels INT Mel frequency bins (default: 80) │+ │ --max-speech-samples INT Max LibriSpeech samples ││ ││ run ││ --checkpoint PATH Path to model checkpoint (required) ││ --prompts STR [...] One or more prompts to continue │+ │ --audio PATH [...] Audio files to transcribe ││ --max-len INT Max tokens to generate (default: 128) ││ --temperature FLOAT Sampling temperature (default: 0.8) ││ --seed INT Random seed (default: 0) │diff --git a/pyproject.toml b/pyproject.tomlindex 13ebd8e..229c96a 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -11,6 +11,8 @@ dependencies = ["transformers","wandb","sentencepiece",+ "soundfile",+ "scipy",][project.optional-dependencies]diff --git a/src/cli.py b/src/cli.pyindex 037cbb6..e912af0 100644--- a/src/cli.py+++ b/src/cli.py@@ -36,10 +36,16 @@ HELP = """│ --checkpoint PATH Resume from checkpoint ││ --checkpoint-dir DIR Checkpoint directory ││ --seed INT Random seed (default: 42) │+ │ --no-speech Disable speech (text-only training) │+ │ --speech-every INT Speech step every N text (default: 3) │+ │ --max-mel-len INT Max mel frames (default: 1024) │+ │ --n-mels INT Mel frequency bins (default: 80) │+ │ --max-speech-samples INT Max LibriSpeech samples ││ ││ run ││ --checkpoint PATH Path to model checkpoint (required) ││ --prompts STR [...] One or more prompts to continue │+ │ --audio PATH [...] Audio files to transcribe ││ --max-len INT Max tokens to generate (default: 128) ││ --temperature FLOAT Sampling temperature (default: 0.8) ││ --seed INT Random seed (default: 0) │@@ -114,10 +120,20 @@ def main():p.add_argument("--num-memory-slots", type=int, default=64)p.add_argument("--mrl-dims", type=int, nargs="*", default=[256, 128, 64],help="MRL dimension targets (default: 256 128 64)")+ p.add_argument("--no-speech", action="store_true", help="Disable speech training (text-only)")+ p.add_argument("--speech-every", type=int, default=3,+ help="Do one speech step every N text steps (default: 3)")+ p.add_argument("--max-mel-len", type=int, default=1024,+ help="Max mel spectrogram frames (default: 1024)")+ p.add_argument("--n-mels", type=int, default=80,+ help="Number of mel frequency bins (default: 80)")+ p.add_argument("--max-speech-samples", type=int, default=None,+ help="Max LibriSpeech training samples (default: all)")p = sub.add_parser("run", add_help=False)p.add_argument("--checkpoint", type=str, required=True)p.add_argument("--prompts", type=str, nargs="*")+ p.add_argument("--audio", type=str, nargs="*", help="Audio file paths to transcribe")p.add_argument("--max-len", type=int, default=128)p.add_argument("--temperature", type=float, default=0.8)p.add_argument("--seed", type=int, default=0)diff --git a/src/data.py b/src/data.pyindex 9cadde8..b89a420 100644--- a/src/data.py+++ b/src/data.py@@ -10,7 +10,7 @@ logging.getLogger("transformers").setLevel(logging.ERROR)logging.getLogger("huggingface_hub").setLevel(logging.ERROR)import numpy as np-from datasets import load_dataset+from datasets import Audio, load_datasetfrom tqdm import tqdmimport sentencepiece as spm@@ -21,6 +21,9 @@ TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")PAD_ID = 0EOS_ID = 1BOS_ID = 2+UNK_ID = 3+STORY_ID = 4+TRANSCRIBE_ID = 5class NeedleTokenizer:@@ -42,6 +45,14 @@ class NeedleTokenizer:def bos_token_id(self):return BOS_ID+ @property+ def story_token_id(self):+ return STORY_ID++ @property+ def transcribe_token_id(self):+ return TRANSCRIBE_ID+@propertydef vocab_size(self):return self.sp.GetPieceSize()@@ -111,7 +122,8 @@ def train_tokenizer(vocab_size=8192, max_samples=None):pad_id=PAD_ID,eos_id=EOS_ID,bos_id=BOS_ID,- unk_id=3,+ unk_id=UNK_ID,+ user_defined_symbols=["<story>", "<transcribe>"],byte_fallback=True,normalization_rule_name="identity",num_threads=os.cpu_count(),@@ -147,18 +159,19 @@ def _tokenizer_hash():return "none"-def _cache_key(n_samples, max_enc_len, max_dec_len):+def _cache_key(prefix, n_samples, max_enc_len, max_dec_len):tok_hash = _tokenizer_hash()- key = f"tinystories_{tok_hash}_{n_samples}_{max_enc_len}_{max_dec_len}"+ key = f"{prefix}_{tok_hash}_{n_samples}_{max_enc_len}_{max_dec_len}"return hashlib.md5(key.encode()).hexdigest()[:12]-def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):+def prepare_text_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):+ """Prepare text encoder-decoder pairs with <story> task token."""- cache_id = _cache_key(len(ds), max_enc_len, max_dec_len)+ cache_id = _cache_key("text", len(ds), max_enc_len, max_dec_len)cache_path = os.path.join(CACHE_DIR, cache_id)if os.path.exists(cache_path + "_enc.npy"):- print(f"Loading cached data ({cache_id})...")+ print(f"Loading cached text data ({cache_id})...")return (np.load(cache_path + "_enc.npy"),np.load(cache_path + "_dec_in.npy"),@@ -167,6 +180,7 @@ def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=12pad_id = tokenizer.pad_token_idbos_id = tokenizer.eos_token_id+ story_id = tokenizer.story_token_idmax_total = max_enc_len + max_dec_len# --- Parallel tokenization ---@@ -189,7 +203,8 @@ def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=12split_points = np.maximum(2, (lengths[valid_indices] * split_ratio).astype(np.int32))enc_lens = np.minimum(split_points, max_enc_len)- dec_lens = np.minimum(lengths[valid_indices] - split_points, max_dec_len)+ # Account for <story> token taking one slot in decoder+ dec_lens = np.minimum(lengths[valid_indices] - split_points, max_dec_len - 1)keep = dec_lens >= 2valid_indices = valid_indices[keep]@@ -207,20 +222,29 @@ def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=12):tokens = all_tokens[idx]enc_inputs[i, :el] = tokens[:sp][:max_enc_len]+ # Decoder input: [BOS, <story>, tok1, tok2, ...]dec_inputs[i, 0] = bos_id- dec_inputs[i, 1:dl] = tokens[sp:][:dl - 1]- dec_targets[i, :dl] = tokens[sp:][:max_dec_len]+ dec_inputs[i, 1] = story_id+ dec_inputs[i, 2:dl + 1] = tokens[sp:][:dl - 1]+ # Decoder target: [<story>, tok1, tok2, ..., EOS]+ dec_targets[i, 0] = story_id+ dec_targets[i, 1:dl + 1] = tokens[sp:][:dl]# Save cacheos.makedirs(CACHE_DIR, exist_ok=True)np.save(cache_path + "_enc.npy", enc_inputs)np.save(cache_path + "_dec_in.npy", dec_inputs)np.save(cache_path + "_dec_tgt.npy", dec_targets)- print(f"Cached {n_keep:,} pairs to {CACHE_DIR}/{cache_id}")+ print(f"Cached {n_keep:,} text pairs to {CACHE_DIR}/{cache_id}")return enc_inputs, dec_inputs, dec_targets+# Backward compatibility alias+def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):+ return prepare_text_pairs(ds, tokenizer, max_enc_len, max_dec_len, split_ratio)++def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True):n = len(enc_inputs)if shuffle:@@ -231,3 +255,194 @@ def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True):for i in range(0, n - batch_size + 1, batch_size):yield enc_inputs[i : i + batch_size], dec_inputs[i : i + batch_size], dec_targets[i : i + batch_size]+++get_text_batches = get_batches+++# ─── Speech pipeline ───────────────────────────────────────────────────────────++def load_librispeech(split="train", max_samples=None):+ """Load LibriSpeech-100 from HuggingFace (SPRINGLab/LibriSpeech-100).++ The dataset has a single 'train' split, so we partition it:+ - train / train.clean.100: first 90%+ - validation.clean / test.clean / validation / test: last 10%++ Audio decoding is disabled (to avoid torchcodec/torch dependency);+ prepare_speech_pairs decodes audio bytes via soundfile instead.+ """+ ds = load_dataset("SPRINGLab/LibriSpeech-100", split="train")+ ds = ds.cast_column("audio", Audio(decode=False))+ n = len(ds)+ if split in ("validation.clean", "validation", "test.clean", "test"):+ ds = ds.select(range(int(n * 0.9), n))+ elif split.startswith("train"):+ ds = ds.select(range(int(n * 0.9)))+ if max_samples:+ ds = ds.select(range(min(max_samples, len(ds))))+ return ds+++def _decode_audio(audio_data, target_sr=16000):+ """Decode raw audio bytes from a datasets Audio(decode=False) field using soundfile."""+ import io+ import soundfile as sf++ audio_bytes = audio_data["bytes"]+ if audio_bytes is None:+ path = audio_data["path"]+ audio, sr = sf.read(path, dtype="float32")+ else:+ audio, sr = sf.read(io.BytesIO(audio_bytes), dtype="float32")++ # Convert stereo to mono+ if audio.ndim > 1:+ audio = audio.mean(axis=1)++ # Resample if needed+ if sr != target_sr:+ from scipy.signal import resample+ num_samples = int(len(audio) * target_sr / sr)+ audio = resample(audio, num_samples).astype(np.float32)+ sr = target_sr++ return audio.astype(np.float32), sr+++def compute_mel_spectrogram(audio, sr=16000, n_mels=80, n_fft=400, hop_length=160):+ """Compute log-mel spectrogram using numpy/scipy. Returns (T_mel, n_mels) float32.++ 25ms window (n_fft=400 at 16kHz), 10ms hop (hop_length=160) → ~100 frames/sec.+ """+ from scipy.signal import windows as scipy_windows+ from scipy.fft import rfft++ # Pre-emphasis+ audio = np.append(audio[0], audio[1:] - 0.97 * audio[:-1])++ # STFT+ window = scipy_windows.hann(n_fft, sym=False).astype(np.float32)+ num_frames = 1 + (len(audio) - n_fft) // hop_length+ if num_frames <= 0:+ return np.zeros((1, n_mels), dtype=np.float32)++ frames = np.lib.stride_tricks.as_strided(+ audio,+ shape=(num_frames, n_fft),+ strides=(audio.strides[0] * hop_length, audio.strides[0]),+ ).copy()+ frames = frames * window+ spectrum = np.abs(rfft(frames, n=n_fft, axis=-1)) ** 2++ # Mel filterbank+ fmin, fmax = 0.0, sr / 2.0+ mel_low = 2595.0 * np.log10(1.0 + fmin / 700.0)+ mel_high = 2595.0 * np.log10(1.0 + fmax / 700.0)+ mel_points = np.linspace(mel_low, mel_high, n_mels + 2)+ hz_points = 700.0 * (10.0 ** (mel_points / 2595.0) - 1.0)+ bin_points = np.floor((n_fft + 1) * hz_points / sr).astype(np.int32)++ filterbank = np.zeros((n_mels, n_fft // 2 + 1), dtype=np.float32)+ for m in range(n_mels):+ f_left, f_center, f_right = bin_points[m], bin_points[m + 1], bin_points[m + 2]+ for k in range(f_left, f_center):+ if f_center > f_left:+ filterbank[m, k] = (k - f_left) / (f_center - f_left)+ for k in range(f_center, f_right):+ if f_right > f_center:+ filterbank[m, k] = (f_right - k) / (f_right - f_center)++ mel_spec = spectrum @ filterbank.T+ log_mel = np.log(np.maximum(mel_spec, 1e-10))+ return log_mel.astype(np.float32)+++def _speech_cache_key(n_samples, n_mels, max_mel_len, max_dec_len):+ tok_hash = _tokenizer_hash()+ key = f"speech_{tok_hash}_{n_samples}_{n_mels}_{max_mel_len}_{max_dec_len}"+ return hashlib.md5(key.encode()).hexdigest()[:12]+++def prepare_speech_pairs(ds, tokenizer, n_mels=80, max_mel_len=1024, max_dec_len=128):+ """Prepare speech encoder-decoder pairs with <transcribe> task token."""++ cache_id = _speech_cache_key(len(ds), n_mels, max_mel_len, max_dec_len)+ cache_path = os.path.join(CACHE_DIR, cache_id)+ if os.path.exists(cache_path + "_mel.npy"):+ print(f"Loading cached speech data ({cache_id})...")+ return (+ np.load(cache_path + "_mel.npy"),+ np.load(cache_path + "_dec_in.npy"),+ np.load(cache_path + "_dec_tgt.npy"),+ )++ pad_id = tokenizer.pad_token_id+ bos_id = tokenizer.eos_token_id+ eos_id = tokenizer.eos_token_id+ transcribe_id = tokenizer.transcribe_token_id++ mel_features = []+ dec_inputs_list = []+ dec_targets_list = []++ for example in tqdm(ds, desc="Processing speech"):+ audio, sr = _decode_audio(example["audio"])++ mel = compute_mel_spectrogram(audio, sr=sr, n_mels=n_mels)++ # Pad or truncate mel to max_mel_len+ if mel.shape[0] > max_mel_len:+ mel = mel[:max_mel_len]+ elif mel.shape[0] < max_mel_len:+ pad_len = max_mel_len - mel.shape[0]+ mel = np.pad(mel, ((0, pad_len), (0, 0)))++ # Tokenize transcript+ text = example["text"].strip().lower()+ tokens = tokenizer.encode(text)+ # Leave room for <transcribe> prefix and EOS+ tokens = tokens[:max_dec_len - 2]++ # Decoder input: [BOS, <transcribe>, tok1, tok2, ...]+ dec_in = np.full(max_dec_len, pad_id, dtype=np.int32)+ dec_in[0] = bos_id+ dec_in[1] = transcribe_id+ n_tok = len(tokens)+ if n_tok > 0:+ dec_in[2:2 + n_tok] = tokens++ # Decoder target: [<transcribe>, tok1, tok2, ..., EOS]+ dec_tgt = np.full(max_dec_len, pad_id, dtype=np.int32)+ dec_tgt[0] = transcribe_id+ if n_tok > 0:+ dec_tgt[1:1 + n_tok] = tokens+ dec_tgt[1 + n_tok] = eos_id++ mel_features.append(mel)+ dec_inputs_list.append(dec_in)+ dec_targets_list.append(dec_tgt)++ mel_features = np.stack(mel_features).astype(np.float32)+ dec_inputs = np.stack(dec_inputs_list)+ dec_targets = np.stack(dec_targets_list)++ os.makedirs(CACHE_DIR, exist_ok=True)+ np.save(cache_path + "_mel.npy", mel_features)+ np.save(cache_path + "_dec_in.npy", dec_inputs)+ np.save(cache_path + "_dec_tgt.npy", dec_targets)+ print(f"Cached {len(mel_features):,} speech pairs to {CACHE_DIR}/{cache_id}")++ return mel_features, dec_inputs, dec_targets+++def get_speech_batches(mel_features, dec_inputs, dec_targets, batch_size, shuffle=True):+ n = len(mel_features)+ if shuffle:+ perm = np.random.permutation(n)+ mel_features = mel_features[perm]+ dec_inputs = dec_inputs[perm]+ dec_targets = dec_targets[perm]++ for i in range(0, n - batch_size + 1, batch_size):+ yield mel_features[i : i + batch_size], dec_inputs[i : i + batch_size], dec_targets[i : i + batch_size]diff --git a/src/model.py b/src/model.pyindex af99328..15bd222 100644--- a/src/model.py+++ b/src/model.py@@ -44,6 +44,7 @@ class TransformerConfig:dtype: str = "bfloat16"activation: str = "drelu"num_memory_slots: int = 64+ n_mels: int = 80@propertydef jax_dtype(self):@@ -303,6 +304,17 @@ class Decoder(nn.Module):return x+class MelProjection(nn.Module):+ """Project mel spectrogram features to model dimension."""+ d_model: int+ dtype: jnp.dtype = jnp.bfloat16++ @nn.compact+ def __call__(self, mel):+ return nn.Dense(self.d_model, dtype=self.dtype, use_bias=True,+ kernel_init=default_init(), name="proj")(mel)++class EncoderDecoderTransformer(nn.Module):"""Encoder-decoder transformer with shared embeddings, tied output, RoPE, and bfloat16."""config: TransformerConfig@@ -310,6 +322,7 @@ class EncoderDecoderTransformer(nn.Module):def setup(self):self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))self.embed_scale = math.sqrt(self.config.d_model)+ self.mel_proj = MelProjection(self.config.d_model, self.config.jax_dtype)self.encoder = MemoryMixerEncoder(self.config)self.decoder = Decoder(self.config)@@ -317,11 +330,20 @@ class EncoderDecoderTransformer(nn.Module):head_dim = self.config.d_model // self.config.num_headsreturn precompute_rope_freqs(head_dim, seq_len, self.config.rope_theta)- def encode(self, src, src_mask=None):+ def encode_text(self, src, src_mask=None):x = self.embedding(src) * self.embed_scalerope = self._rope(src.shape[1])return self.encoder(x, mask=src_mask, rope=rope)+ def encode(self, src, src_mask=None):+ """Backward-compatible alias for encode_text."""+ return self.encode_text(src, src_mask=src_mask)++ def encode_speech(self, mel, src_mask=None):+ x = self.mel_proj(mel) * self.embed_scale+ rope = self._rope(mel.shape[1])+ return self.encoder(x, mask=src_mask, rope=rope)+def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None):x = self.embedding(tgt) * self.embed_scalerope = self._rope(tgt.shape[1])@@ -332,19 +354,19 @@ class EncoderDecoderTransformer(nn.Module):return logitsdef __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):- encoder_out = self.encode(src, src_mask=src_mask)+ encoder_out = self.encode_text(src, src_mask=src_mask)logits = self.decode(tgt, encoder_out, self_mask=tgt_mask)return logits- def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mrl_dims=None):- encoder_out = self.encode(src, src_mask=src_mask)+ def _decode_with_aux(self, encoder_out, tgt, tgt_mask=None, mrl_dims=None):+ """Shared decode logic returning logits, slot_div, mrl_logits."""x = self.embedding(tgt) * self.embed_scalerope = self._rope(tgt.shape[1])x = self.decoder(x, encoder_out, self_mask=tgt_mask, cross_mask=None, rope=rope)x_f32 = x.astype(jnp.float32)- emb = self.embedding.embedding+ emb = self.embedding.embeddinglogits = x_f32 @ emb.Tmrl_logits = []@@ -359,6 +381,25 @@ class EncoderDecoderTransformer(nn.Module):slot_div = (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]return logits, slot_div, mrl_logits+ def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mrl_dims=None):+ encoder_out = self.encode_text(src, src_mask=src_mask)+ return self._decode_with_aux(encoder_out, tgt, tgt_mask=tgt_mask, mrl_dims=mrl_dims)++ def forward_speech_with_aux(self, mel, tgt, src_mask=None, tgt_mask=None, mrl_dims=None):+ encoder_out = self.encode_speech(mel, src_mask=src_mask)+ return self._decode_with_aux(encoder_out, tgt, tgt_mask=tgt_mask, mrl_dims=mrl_dims)++ def init_all(self, src, tgt, mel):+ """Dummy forward through both text and speech pathways to initialize all params."""+ src_mask = make_padding_mask(src, self.config.pad_token_id)+ tgt_mask = make_causal_mask(tgt.shape[1]) & make_padding_mask(tgt, self.config.pad_token_id)+ text_out = self.encode_text(src, src_mask=src_mask)+ mel_mask = make_mel_padding_mask(mel)+ speech_out = self.encode_speech(mel, src_mask=mel_mask)+ _ = self._decode_with_aux(text_out, tgt, tgt_mask=tgt_mask)+ _ = self._decode_with_aux(speech_out, tgt, tgt_mask=tgt_mask)+ return jnp.zeros(())+def make_causal_mask(seq_len):mask = jnp.tril(jnp.ones((seq_len, seq_len), dtype=jnp.bool_))@@ -368,3 +409,9 @@ def make_causal_mask(seq_len):def make_padding_mask(tokens, pad_token_id):mask = tokens != pad_token_idreturn mask[:, None, None, :]+++def make_mel_padding_mask(mel):+ """Create padding mask from mel spectrogram: non-zero frames are valid."""+ mask = jnp.any(mel != 0, axis=-1) # (B, T)+ return mask[:, None, None, :] # (B, 1, 1, T)diff --git a/src/run.py b/src/run.pyindex df60452..f4490f7 100644--- a/src/run.py+++ b/src/run.py@@ -4,13 +4,15 @@ import sysimport jaximport jax.numpy as jnp+import numpy as np-from .data import get_tokenizer+from .data import get_tokenizer, compute_mel_spectrogramfrom .model import (EncoderDecoderTransformer,TransformerConfig,make_causal_mask,make_padding_mask,+ make_mel_padding_mask,)@@ -84,6 +86,87 @@ def generate(model, params, tokenizer, prompt, max_gen_len=128, temperature=0.8,return tokenizer.decode(generated_tokens)+def load_audio(path, target_sr=16000):+ """Load an audio file and resample to target_sr. Returns (audio_array, sr)."""+ import soundfile as sf+ audio, sr = sf.read(path, dtype="float32")+ # Convert stereo to mono+ if audio.ndim > 1:+ audio = audio.mean(axis=1)+ # Resample if needed+ if sr != target_sr:+ from scipy.signal import resample+ num_samples = int(len(audio) * target_sr / sr)+ audio = resample(audio, num_samples).astype(np.float32)+ sr = target_sr+ return audio, sr+++def transcribe(model, params, tokenizer, audio_array, sr=16000, max_gen_len=128, temperature=0.8, seed=0, stream=True):+ """Transcribe audio using the speech encoder pathway."""+ n_mels = model.config.n_mels+ mel = compute_mel_spectrogram(audio_array, sr=sr, n_mels=n_mels)+ mel_input = jnp.array(mel)[None, :, :] # (1, T_mel, n_mels)++ pad_id = tokenizer.pad_token_id+ eos_id = tokenizer.eos_token_id+ transcribe_id = tokenizer.transcribe_token_id++ src_mask = make_mel_padding_mask(mel_input)+ encoder_out = model.apply(+ {"params": params}, mel_input, src_mask=src_mask, method="encode_speech"+ )++ tgt_mask = make_causal_mask(max_gen_len)+ dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)+ dec_buffer = dec_buffer.at[0, 0].set(eos_id) # BOS = EOS in this tokenizer+ dec_buffer = dec_buffer.at[0, 1].set(transcribe_id)++ @jax.jit+ def decode_step(dec_buffer, encoder_out, src_mask):+ logits = model.apply(+ {"params": params},+ dec_buffer,+ encoder_out,+ self_mask=tgt_mask,+ cross_mask=src_mask,+ method="decode",+ )+ return logits++ rng = jax.random.PRNGKey(seed)+ generated_tokens = []++ if stream:+ sys.stdout.write("\n")+ sys.stdout.flush()++ logits = decode_step(dec_buffer, encoder_out, src_mask)++ # Start generating from position 2 (after BOS + <transcribe>)+ for i in range(1, max_gen_len - 1):+ next_logits = logits[0, i] / temperature+ rng, sample_rng = jax.random.split(rng)+ next_token = jax.random.categorical(sample_rng, next_logits).item()++ if next_token == eos_id:+ break++ generated_tokens.append(next_token)+ dec_buffer = dec_buffer.at[0, i + 1].set(next_token)++ if stream:+ sys.stdout.write(tokenizer.decode([next_token]))+ sys.stdout.flush()++ logits = decode_step(dec_buffer, encoder_out, src_mask)++ if stream:+ sys.stdout.write("\n")++ return tokenizer.decode(generated_tokens)++def main(args):print(f"Loading checkpoint: {args.checkpoint}")params, config = load_checkpoint(args.checkpoint)@@ -94,6 +177,26 @@ def main(args):param_count = sum(x.size for x in jax.tree.leaves(params))print(f"Model parameters: {param_count:,}")+ # --- Audio transcription mode ---+ audio_files = getattr(args, "audio", None)+ if audio_files:+ for i, audio_path in enumerate(audio_files):+ print(f"\nTranscribing: {audio_path}")+ audio, sr = load_audio(audio_path)+ transcribe(+ model,+ params,+ tokenizer,+ audio,+ sr=sr,+ max_gen_len=args.max_len,+ temperature=args.temperature,+ seed=args.seed + i,+ stream=True,+ )+ return++ # --- Text generation mode ---prompts = args.prompts or ["Once upon a time", "The little dog", "She was very happy because"]for i, prompt in enumerate(prompts):diff --git a/src/test.py b/src/test.pyindex 86d3598..93e45e5 100644--- a/src/test.py+++ b/src/test.py@@ -8,7 +8,7 @@ import jax.numpy as jnpimport numpy as npimport optax-from .data import get_batches, get_tokenizer, load_tinystories, prepare_encoder_decoder_pairs+from .data import get_batches, get_tokenizer, load_tinystories, prepare_text_pairs, load_librispeechfrom .model import (EncoderDecoderTransformer,TransformerConfig,@@ -150,6 +150,64 @@ def benchmark_generation_quality(model, params, tokenizer, prompts, max_gen_len=}+def compute_wer(hypotheses, references):+ """Compute word error rate using edit distance."""+ total_edits = 0+ total_ref_words = 0++ for hyp, ref in zip(hypotheses, references):+ hyp_words = hyp.lower().split()+ ref_words = ref.lower().split()+ n = len(ref_words)+ m = len(hyp_words)++ # DP edit distance+ d = [[0] * (m + 1) for _ in range(n + 1)]+ for i in range(n + 1):+ d[i][0] = i+ for j in range(m + 1):+ d[0][j] = j+ for i in range(1, n + 1):+ for j in range(1, m + 1):+ if ref_words[i - 1] == hyp_words[j - 1]:+ d[i][j] = d[i - 1][j - 1]+ else:+ d[i][j] = 1 + min(d[i - 1][j], d[i][j - 1], d[i - 1][j - 1])++ total_edits += d[n][m]+ total_ref_words += n++ return total_edits / max(total_ref_words, 1)+++def benchmark_asr(model, params, tokenizer, num_samples=100):+ """Transcribe LibriSpeech test-clean samples and report WER."""+ from .run import transcribe+ from .data import load_librispeech++ ds = load_librispeech("test.clean", max_samples=num_samples)++ hypotheses = []+ references = []++ for example in ds:+ audio_data = example["audio"]+ audio = np.array(audio_data["array"], dtype=np.float32)+ sr = audio_data["sampling_rate"]+ ref_text = example["text"].strip().lower()++ hyp_text = transcribe(+ model, params, tokenizer, audio, sr=sr,+ max_gen_len=128, temperature=0.0, seed=0, stream=False,+ ).strip().lower()++ hypotheses.append(hyp_text)+ references.append(ref_text)++ wer = compute_wer(hypotheses, references)+ return {"wer": wer, "num_samples": len(hypotheses)}++def main(args):params, config = load_checkpoint(args.checkpoint)model = EncoderDecoderTransformer(config)@@ -160,13 +218,19 @@ def main(args):print(f"parameters: {param_count:,}")print(f"config: d={config.d_model}, heads={config.num_heads}, layers={config.num_encoder_layers}/{config.num_decoder_layers}")- print(f"\nevaluating perplexity ({args.max_eval_samples} samples)...")+ print(f"\nevaluating text perplexity ({args.max_eval_samples} samples)...")ds = load_tinystories("validation", max_samples=args.max_eval_samples)- enc_inputs, dec_inputs, dec_targets = prepare_encoder_decoder_pairs(+ enc_inputs, dec_inputs, dec_targets = prepare_text_pairs(ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len)ppl = compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, args.batch_size, config.pad_token_id)- print(f"perplexity: {ppl:.2f}")+ print(f"text ppl: {ppl:.2f}")++ asr_samples = getattr(args, "asr_samples", 50)+ if asr_samples > 0:+ print(f"\nevaluating ASR WER ({asr_samples} samples)...")+ asr_results = benchmark_asr(model, params, tokenizer, num_samples=asr_samples)+ print(f"ASR WER: {asr_results['wer']:.4f} ({asr_results['num_samples']} samples)")print(f"\nmeasuring throughput ({args.throughput_runs} runs)...")throughput = measure_throughput(model, params, tokenizer, num_runs=args.throughput_runs)diff --git a/src/train.py b/src/train.pyindex c61f852..c5651fa 100644--- a/src/train.py+++ b/src/train.py@@ -13,12 +13,16 @@ from tqdm import tqdmfrom flax import jax_utilsfrom flax.training import train_state-from .data import get_batches, get_tokenizer, load_tinystories, prepare_encoder_decoder_pairs+from .data import (+ get_batches, get_tokenizer, load_tinystories, prepare_text_pairs,+ load_librispeech, prepare_speech_pairs, get_speech_batches,+)from .model import (EncoderDecoderTransformer,TransformerConfig,make_causal_mask,make_padding_mask,+ make_mel_padding_mask,)def _newton_schulz(G, steps=5):@@ -99,10 +103,13 @@ def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_rng, init_rng = jax.random.split(rng)dummy_src = jnp.ones((1, 128), dtype=jnp.int32)dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)+ dummy_mel = jnp.ones((1, 128, config.n_mels), dtype=jnp.float32)variables = model.init({"params": init_rng},dummy_src,dummy_tgt,+ dummy_mel,+ method="init_all",)adam_schedule = _wsd_schedule(learning_rate, total_steps, warmup_steps)@@ -220,23 +227,9 @@ _GROUP_SIZE = 32_MRL_DIMS = ()-def _loss_fn(state, params, src, tgt_in, tgt_out, causal_mask):+def _compute_loss(logits, slot_div, mrl_logits, tgt_out):+ """Shared CE + MRL + z-loss + slot-div computation."""pad_id = 0- src_mask = make_padding_mask(src, pad_id)- tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)- cross_mask = src_mask-- logits, slot_div, mrl_logits = state.apply_fn(- {"params": _quantize_params(params, group_size=_GROUP_SIZE)},- src,- tgt_in,- src_mask=src_mask,- tgt_mask=tgt_mask,- cross_mask=cross_mask,- mrl_dims=_MRL_DIMS if _MRL_DIMS else None,- method="forward_with_aux",- )-logits_f32 = logits.astype(jnp.float32)mask = (tgt_out != pad_id).astype(jnp.float32)ce_loss = jnp.sum(@@ -259,10 +252,74 @@ def _loss_fn(state, params, src, tgt_in, tgt_out, causal_mask):return total_loss + z_loss + div_loss-def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):+def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask):+ pad_id = 0+ src_mask = make_padding_mask(src, pad_id)+ tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)++ logits, slot_div, mrl_logits = state.apply_fn(+ {"params": _quantize_params(params, group_size=_GROUP_SIZE)},+ src,+ tgt_in,+ src_mask=src_mask,+ tgt_mask=tgt_mask,+ cross_mask=src_mask,+ mrl_dims=_MRL_DIMS if _MRL_DIMS else None,+ method="forward_with_aux",+ )+ return _compute_loss(logits, slot_div, mrl_logits, tgt_out)+++def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask):+ pad_id = 0+ src_mask = make_mel_padding_mask(mel)+ tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)++ logits, slot_div, mrl_logits = state.apply_fn(+ {"params": _quantize_params(params, group_size=_GROUP_SIZE)},+ mel,+ tgt_in,+ src_mask=src_mask,+ tgt_mask=tgt_mask,+ mrl_dims=_MRL_DIMS if _MRL_DIMS else None,+ method="forward_speech_with_aux",+ )+ return _compute_loss(logits, slot_div, mrl_logits, tgt_out)+++def _train_step_text(state, ema_params, src, tgt_in, tgt_out, causal_mask):+ ema_decay = 0.999+ loss, grads = jax.value_and_grad(+ lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask)+ )(state.params)+ grads = jax.lax.pmean(grads, axis_name="batch")+ loss = jax.lax.pmean(loss, axis_name="batch")+ grad_norm = optax.global_norm(grads)+ state = state.apply_gradients(grads=grads)+ ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, state.params)+ return state, ema_params, loss, grad_norm+++def _train_step_text_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask):+ """Text training step with fused mask application."""+ ema_decay = 0.999+ loss, grads = jax.value_and_grad(+ lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask)+ )(state.params)+ grads = jax.lax.pmean(grads, axis_name="batch")+ loss = jax.lax.pmean(loss, axis_name="batch")+ grad_norm = optax.global_norm(grads)+ state = state.apply_gradients(grads=grads)+ masked_params = jax.tree.map(lambda w, m: w * m, state.params, prune_mask)+ state = state.replace(params=masked_params)+ ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, masked_params)+ return state, ema_params, loss, grad_norm+++def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask):ema_decay = 0.999loss, grads = jax.value_and_grad(- lambda p: _loss_fn(state, p, src, tgt_in, tgt_out, causal_mask)+ lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask))(state.params)grads = jax.lax.pmean(grads, axis_name="batch")loss = jax.lax.pmean(loss, axis_name="batch")@@ -272,11 +329,11 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):return state, ema_params, loss, grad_norm-def _train_step_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask):- """Training step with fused mask application — avoids post-pmap copies."""+def _train_step_speech_masked(state, ema_params, mel, tgt_in, tgt_out, causal_mask, prune_mask):+ """Speech training step with fused mask application."""ema_decay = 0.999loss, grads = jax.value_and_grad(- lambda p: _loss_fn(state, p, src, tgt_in, tgt_out, causal_mask)+ lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask))(state.params)grads = jax.lax.pmean(grads, axis_name="batch")loss = jax.lax.pmean(loss, axis_name="batch")@@ -289,11 +346,19 @@ def _train_step_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, prudef _make_p_train_step():- return jax.pmap(_train_step, axis_name="batch", donate_argnums=(0, 1))+ return jax.pmap(_train_step_text, axis_name="batch", donate_argnums=(0, 1))def _make_p_train_step_masked():- return jax.pmap(_train_step_masked, axis_name="batch", donate_argnums=(0, 1))+ return jax.pmap(_train_step_text_masked, axis_name="batch", donate_argnums=(0, 1))+++def _make_p_train_step_speech():+ return jax.pmap(_train_step_speech, axis_name="batch", donate_argnums=(0, 1))+++def _make_p_train_step_speech_masked():+ return jax.pmap(_train_step_speech_masked, axis_name="batch", donate_argnums=(0, 1))def _make_val_loss_fn(apply_fn):@@ -332,6 +397,23 @@ def _make_mrl_val_loss_fn(apply_fn, mrl_dim):return val_loss_batch+def _make_speech_val_loss_fn(apply_fn):+ @jax.jit+ def val_loss_batch(params, mel, tgt_in, tgt_out, causal_mask):+ pad_id = 0+ src_mask = make_mel_padding_mask(mel)+ tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)+ logits, _, _ = apply_fn(+ {"params": params}, mel, tgt_in,+ src_mask=src_mask, tgt_mask=tgt_mask,+ method="forward_speech_with_aux",+ )+ loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)+ mask = (tgt_out != pad_id).astype(jnp.float32)+ return jnp.sum(loss * mask), jnp.sum(mask)+ return val_loss_batch++def _estimate_mrl_params(config, d_prime):"""Estimate parameter count of an exported sub-model at dimension d_prime.@@ -370,7 +452,13 @@ def shard_batch(batch, num_devices):def train(args):+ import itertools+num_devices = jax.local_device_count()+ no_speech = getattr(args, "no_speech", False)+ speech_every = getattr(args, "speech_every", 3)+ n_mels = getattr(args, "n_mels", 80)+ max_mel_len = getattr(args, "max_mel_len", 1024)use_wandb = getattr(args, "wandb", False)if use_wandb:@@ -378,27 +466,60 @@ def train(args):if wandb.run is None:wandb.init(project="needle-v1", config=vars(args))- print(f"\n[1/4] Detecting devices...")+ total_data_steps = 6 if not no_speech else 4+ step_idx = 0++ step_idx += 1+ print(f"\n[{step_idx}/{total_data_steps}] Detecting devices...")print(f" {num_devices} device(s) for data-parallel training")- print(f"\n[2/4] Loading tokenizer...")+ step_idx += 1+ print(f"\n[{step_idx}/{total_data_steps}] Loading tokenizer...")tokenizer = get_tokenizer(max_samples=args.max_samples)- print(f"\n[3/4] Loading TinyStories dataset...")+ step_idx += 1+ print(f"\n[{step_idx}/{total_data_steps}] Loading TinyStories dataset...")ds = load_tinystories("train", max_samples=args.max_samples)val_ds = load_tinystories("validation", max_samples=getattr(args, "max_eval_samples", None))- print(f"\n[4/4] Preparing encoder-decoder pairs...")- enc_inputs, dec_inputs, dec_targets = prepare_encoder_decoder_pairs(+ step_idx += 1+ print(f"\n[{step_idx}/{total_data_steps}] Preparing text pairs...")+ enc_inputs, dec_inputs, dec_targets = prepare_text_pairs(ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len)- val_enc, val_dec_in, val_dec_tgt = prepare_encoder_decoder_pairs(+ val_enc, val_dec_in, val_dec_tgt = prepare_text_pairs(val_ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len)val_enc = np.array(val_enc)val_dec_in = np.array(val_dec_in)val_dec_tgt = np.array(val_dec_tgt)- print(f" {len(enc_inputs):,} train / {len(val_enc):,} val pairs")+ print(f" {len(enc_inputs):,} train / {len(val_enc):,} val text pairs")++ # --- Speech data ---+ speech_mel = speech_dec_in = speech_dec_tgt = None+ val_speech_mel = val_speech_dec_in = val_speech_dec_tgt = None+ if not no_speech:+ step_idx += 1+ max_speech_samples = getattr(args, "max_speech_samples", None)+ print(f"\n[{step_idx}/{total_data_steps}] Loading LibriSpeech...")+ speech_ds = load_librispeech("train.clean.100", max_samples=max_speech_samples)+ speech_val_ds = load_librispeech("validation.clean", max_samples=getattr(args, "max_eval_samples", None))++ step_idx += 1+ print(f"\n[{step_idx}/{total_data_steps}] Preparing speech pairs...")+ speech_mel, speech_dec_in, speech_dec_tgt = prepare_speech_pairs(+ speech_ds, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len+ )+ val_speech_mel, val_speech_dec_in, val_speech_dec_tgt = prepare_speech_pairs(+ speech_val_ds, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len+ )+ speech_mel = np.array(speech_mel)+ speech_dec_in = np.array(speech_dec_in)+ speech_dec_tgt = np.array(speech_dec_tgt)+ val_speech_mel = np.array(val_speech_mel)+ val_speech_dec_in = np.array(val_speech_dec_in)+ val_speech_dec_tgt = np.array(val_speech_dec_tgt)+ print(f" {len(speech_mel):,} train / {len(val_speech_mel):,} val speech pairs")effective_batch_size = args.batch_size * num_devices@@ -422,6 +543,7 @@ def train(args):dtype=args.dtype,activation=getattr(args, "activation", "drelu"),num_memory_slots=getattr(args, "num_memory_slots", 64),+ n_mels=n_mels,)global _GROUP_SIZE, _MRL_DIMS@@ -433,12 +555,16 @@ def train(args):_MRL_DIMS = ()p_train_step = _make_p_train_step()p_train_step_masked = _make_p_train_step_masked()+ p_train_step_speech = _make_p_train_step_speech()+ p_train_step_speech_masked = _make_p_train_step_speech_masked()np.random.seed(args.seed)rng = jax.random.PRNGKey(args.seed)rng, init_rng = jax.random.split(rng)- num_batches = len(enc_inputs) // effective_batch_size+ text_batches_per_epoch = len(enc_inputs) // effective_batch_size+ speech_batches_per_epoch = text_batches_per_epoch // speech_every if (not no_speech and speech_mel is not None) else 0+ num_batches = text_batches_per_epoch + speech_batches_per_epochtotal_steps = num_batches * args.epochswarmup_steps = max(1, int(total_steps * args.warmup_ratio))@@ -467,6 +593,12 @@ def train(args):print(f" Memory slots {config.num_memory_slots:>12}")print(f" Activation {config.activation:>12}")print(f" Dtype {config.dtype:>12}")+ if not no_speech:+ print(f" Speech every {speech_every} text steps")+ print(f" n_mels {n_mels:>12}")+ print(f" max_mel_len {max_mel_len:>12}")+ else:+ print(f" Speech disabled")print(f" ─────────────────────────────────────")print(f" Devices {num_devices:>12}")print(f" Batch {args.batch_size:>7} x {num_devices} = {effective_batch_size}")@@ -511,11 +643,22 @@ def train(args):f"(steps {t_start}-{t_end}/{num_batches}, interval={prune_interval}, group_size={_GROUP_SIZE})")epoch_step = 0- losses = []- batches = get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size)- pbar = tqdm(batches, total=num_batches, desc=f"Epoch {epoch + 1}/{args.epochs}")+ text_losses = []+ speech_losses = []+ text_batches = get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size)++ # Speech batches cycle via itertools.cycle+ speech_iter = None+ if not no_speech and speech_mel is not None:+ speech_iter = iter(itertools.cycle(+ get_speech_batches(speech_mel, speech_dec_in, speech_dec_tgt, effective_batch_size)+ ))++ text_step_in_epoch = 0+ pbar = tqdm(text_batches, total=text_batches_per_epoch, desc=f"Epoch {epoch + 1}/{args.epochs}")for src, tgt_in, tgt_out in pbar:+ # --- Text step ---t0 = time.perf_counter()src_b = shard_batch(src, num_devices)tgt_in_b = shard_batch(tgt_in, num_devices)@@ -530,6 +673,31 @@ def train(args):state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask,)+ text_loss_val = float(loss[0])+ text_losses.append(text_loss_val)+ global_step += 1+ text_step_in_epoch += 1++ # --- Speech step (every speech_every text steps) ---+ speech_loss_val = None+ if speech_iter is not None and text_step_in_epoch % speech_every == 0:+ mel_batch, sp_tgt_in, sp_tgt_out = next(speech_iter)+ mel_b = shard_batch(mel_batch, num_devices)+ sp_tgt_in_b = shard_batch(sp_tgt_in, num_devices)+ sp_tgt_out_b = shard_batch(sp_tgt_out, num_devices)++ if prune_mask is not None:+ state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech_masked(+ state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, prune_mask,+ )+ else:+ state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech(+ state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask,+ )+ speech_loss_val = float(sp_loss[0])+ speech_losses.append(speech_loss_val)+ global_step += 1+# --- Gradual magnitude sparsification (epoch 1 only) ---if epoch == weight_prune_epoch and not gradual_sparsify_done:epoch_step += 1@@ -541,9 +709,6 @@ def train(args):prune_mask = jax_utils.replicate(mask)del mask- loss_val = float(loss[0])- losses.append(loss_val)- global_step += 1dt = time.perf_counter() - t0eval_every = getattr(args, "eval_every", 100)if global_step % eval_every == 0 or global_step == total_steps:@@ -554,28 +719,36 @@ def train(args):vl, vt = val_loss_fn(_eval_params, vb[0], vb[1], vb[2], val_causal)total_loss += float(vl)total_toks += float(vt)- del _eval_paramslast_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))- # --- tqdm postfix with sparsity during epoch 1 ---- postfix = {"loss": f"{loss_val:.4f}", "val_ppl": f"{last_val_ppl:.2f}" if last_val_ppl is not None else "?"}+ del _eval_params++ # --- tqdm postfix ---+ postfix = {+ "text_loss": f"{text_loss_val:.4f}",+ "text_ppl": f"{last_val_ppl:.2f}" if last_val_ppl is not None else "?",+ }+ if speech_loss_val is not None:+ postfix["speech_loss"] = f"{speech_loss_val:.4f}"if epoch == weight_prune_epoch and not gradual_sparsify_done:- postfix["sparsity"] = f"{current_sparsity*100:.1f}%"+ postfix["prune"] = f"{current_sparsity*100:.1f}%"pbar.set_postfix(**postfix)if use_wandb:log_dict = {- "train/loss": loss_val,+ "train/text_loss": text_loss_val,"train/grad_norm": float(grad_norm[0]),"train/adam_lr": float(adam_schedule(global_step)),"train/muon_lr": float(muon_schedule(global_step)),"train/tokens_per_sec": tokens_per_batch / dt,"train/step": global_step,}+ if speech_loss_val is not None:+ log_dict["train/speech_loss"] = speech_loss_valif epoch == weight_prune_epoch and not gradual_sparsify_done:log_dict["train/scheduled_sparsity"] = current_sparsityif global_step % eval_every == 0 or global_step == total_steps:- log_dict["val/ppl"] = last_val_ppl+ log_dict["val/text_ppl"] = last_val_pplwandb.log(log_dict)# --- Lock mask after epoch 1 gradual sparsification ---@@ -599,8 +772,8 @@ def train(args):print(f" Final sparsity: {zero_p/total_p*100:.2f}% ({zero_p:,}/{total_p:,} near-zero)")del final_pruned- epoch_avg_loss = sum(losses) / len(losses) if losses else float("nan")- final_loss = losses[-1] if losses else float("nan")+ epoch_avg_loss = sum(text_losses) / len(text_losses) if text_losses else float("nan")+ final_loss = text_losses[-1] if text_losses else float("nan")final_ppl = math.exp(final_loss) if not math.isnan(final_loss) else float("nan")eval_params = jax_utils.unreplicate(ema_params)@@ -651,13 +824,33 @@ def train(args):with open(ckpt_path, "wb") as f:pickle.dump({"params": params_np, "config": config.__dict__}, f)- from .test import measure_throughput, benchmark_generation_quality+ from .test import measure_throughput, benchmark_generation_quality, compute_wer+ from .run import generate, transcribe+ from .data import _decode_audioeval_params_jnp = jax.tree.map(jnp.array, params_np)del params_np # free CPU memorymodel = EncoderDecoderTransformer(config)tp = measure_throughput(model, eval_params_jnp, tokenizer, num_runs=5)prompts = ["Once upon a time", "The little dog", "She was very happy because"]quality = benchmark_generation_quality(model, eval_params_jnp, tokenizer, prompts, max_gen_len=64, temperature=0.8)++ speech_wer = None+ transcribe_samples = []+ if not no_speech and val_speech_mel is not None:+ speech_val_ds = load_librispeech("validation.clean", max_samples=3)+ hyps, refs = [], []+ for i, example in enumerate(speech_val_ds):+ audio, sr = _decode_audio(example["audio"])+ ref_text = example["text"].strip().lower()+ hyp_text = transcribe(+ model, eval_params_jnp, tokenizer, audio, sr=sr,+ max_gen_len=64, temperature=0.0, seed=i, stream=False,+ ).strip().lower()+ hyps.append(hyp_text)+ refs.append(ref_text)+ transcribe_samples.append((ref_text, hyp_text))+ speech_wer = compute_wer(hyps, refs)+del eval_params_jnp # free device memoryprint(f"\n ─────────────────────────────────────")@@ -666,7 +859,11 @@ def train(args):print(f" Avg loss {epoch_avg_loss:>12.4f}")print(f" Final loss {final_loss:>12.4f}")print(f" Train ppl {final_ppl:>12.2f}")- print(f" Val ppl {last_val_ppl:>12.2f}")+ print(f" Text val ppl {last_val_ppl:>12.2f}")+ if speech_wer is not None:+ print(f" Speech WER {speech_wer:>12.4f}")+ epoch_avg_speech = sum(speech_losses) / len(speech_losses) if speech_losses else float("nan")+ print(f" Avg speech loss{epoch_avg_speech:>12.4f}")print(f" Quant val ppl {quant_val_ppl:>12.2f} (INT4 g{_GROUP_SIZE})")print(f" Sparsity {sparsity:>11.2f}% ({near_zero:,}/{total_params:,})")if mrl_results:@@ -686,21 +883,31 @@ def train(args):print(f" Repetition {quality['bigram_repetition_rate']:>12.3f}")print(f" ─────────────────────────────────────")for prompt, gen in quality["generations"]:- print(f" [{prompt}]")- print(f" {gen[:100]}")+ gen_clean = gen.replace("<story>", "").replace("<transcribe>", "").strip()+ print(f" <story>[{prompt}] {gen_clean[:80]}")+ if transcribe_samples:+ print(f" ─────────────────────────────────────")+ print(f" Transcribe (WER={speech_wer:.4f}):")+ for ref, hyp in transcribe_samples:+ print(f" <transcribe> ref: {ref[:80]}")+ print(f" hyp: {hyp[:80]}")print(f" ─────────────────────────────────────")print(f" Checkpoint: {ckpt_path}")print(f" ─────────────────────────────────────\n")if use_wandb:log_dict = {- "epoch/avg_loss": epoch_avg_loss,- "epoch/final_loss": final_loss,- "epoch/val_ppl": last_val_ppl,+ "epoch/avg_text_loss": epoch_avg_loss,+ "epoch/final_text_loss": final_loss,+ "epoch/text_val_ppl": last_val_ppl,"epoch/quant_val_ppl": quant_val_ppl,"epoch/weight_sparsity": sparsity,"epoch": epoch + 1,}+ if speech_wer is not None:+ log_dict["epoch/speech_wer"] = speech_wer+ if speech_losses:+ log_dict["epoch/avg_speech_loss"] = sum(speech_losses) / len(speech_losses)for d_prime, (mrl_ppl, mrl_params) in mrl_results.items():log_dict[f"epoch/mrl_ppl_d{d_prime}"] = mrl_ppllog_dict[f"epoch/mrl_params_d{d_prime}"] = mrl_params