Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Add speech

HenryNdubuaku56e88e1adf60
diff --git a/README.md b/README.md
index 692f384..85dfa9d 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,7 @@
──────────── ─────────────────
┌──────────────┐ ┌──────────────┐
- │ Transcription│ │ Tool JSON │ ┌──────────────────────┐
+ │ Transcription│ │ Story │ ┌──────────────────────┐
└──────┬───────┘ └──────┬───────┘ │ MRL loss at dims │
└───────┬─────────┘ │ + INT4 QAT (g=32) │
┌───┴───────┐ │ + z-loss + slot div │
@@ -27,13 +27,13 @@
┌─────┴─────┐ └──────────┬───────────┘
│ ZCRMSNorm │ │
└─────┬─────┘ ┌──────────┴───────────┐
- ┌────────┴────────┐ │ EMA params (β=0.999) │
- │ Decoder x N_dec │ └──────────┬───────────┘
- │┌───────────────┐│ │
- ││ Masked Self ││ ┌──────────┴───────────┐
- ││ Attn + RoPE ││ │ Sparsification │
- │├───────────────┤│ │ cubic schedule │
- ┌──────────────┐ S ││ Cross ││ │ mask every N steps │
+ ┌────────┴────────┐ │ Sparsification │
+ │ Decoder x N_dec │ │ cubic schedule │
+ │┌───────────────┐│ │ mask every N steps │
+ ││ Masked Self ││ └──────────┬───────────┘
+ ││ Attn + RoPE ││ │
+ │├───────────────┤│ ┌──────────┴───────────┐
+ ┌──────────────┐ S ││ Cross ││ │ EMA params (β=0.999) │
│ MemoryMixer │────────▶ Attention ││ └──────────┬───────────┘
│ Encoder │ │├───────────────┤│ │
│ x N_enc │ ││ Gated FFN ││ ┌──────────┴───────────┐
@@ -44,7 +44,7 @@
│ │ RoPE keys│ │ └─────┬─────┘
│ ├──────────┤ │ ┌───────┴───────┐
│ │Mix: │ │ │ <transcribe> │
- │ │ MLP-Mixer│ │ │ or <tools> │
+ │ │ MLP-Mixer│ │ │ or <story> │
│ │ on S │ │ │ + target │
│ ├──────────┤ │ └───────────────┘
│ │Local: │ │
@@ -56,18 +56,64 @@
│ DW Conv ↓2 │ stride-2 depthwise-separable
│ S ∈ (M, d) │
└──────┬───────┘
- ┌──────┴───────┐
- │ Embedding │ ← shared
- └──────┬───────┘
- ┌────┴─────────┐
- │ Text/Speech │
- │ Input │
- └──────────────┘
+ │
+ ┌────┴──────┐ ┌──────────────┐
+ │ Embedding │ │ Mel Proj │
+ │ (text) │ │ (speech) │
+ └────┬──────┘ └──────┬───────┘
+ │ │
+ ┌────┴──────┐ ┌──────┴───────┐
+ │ Text │ │ Audio │
+ │ tokens │ │ mel spec │
+ └───────────┘ └──────────────┘
d=512 (default) / 1536 (--full) · GQA · QK-norm
SentencePiece BPE (8192) · gated dReLU · ZCRMSNorm · RoPE
strided DW conv · MRL dims · INT4 QAT · Muon + AdamW
- text + speech encoder · <transcribe> / <tools> task routing
+ text + speech encoder · <transcribe> / <story> task routing
+
+ Data Preprocessing
+ ──────────────────
+
+ Text (TinyStories) Speech (LibriSpeech-100)
+ ────────────────── ────────────────────────
+
+ TinyStories LibriSpeech-100
+ │ │
+ ▼ ▼
+ ┌──────────────┐ ┌────────────────┐
+ │ SentencePiece│ │ soundfile │
+ │ BPE tokenize │ │ decode (16kHz) │
+ │ (8192 vocab) │ └───────┬────────┘
+ └──────┬───────┘ │
+ │ ▼
+ ▼ ┌────────────────┐
+ ┌──────────────┐ │ Log-mel spec │
+ │ Split at 30% │ │ 80 bins, 25ms │
+ │ enc / dec │ │ window, 10ms │
+ └──┬───────┬───┘ │ hop (~100 f/s) │
+ │ │ └───────┬────────┘
+ ▼ ▼ │
+ ┌─────┐ ┌──────────────┐ ▼
+ │ enc │ │ [BOS]<story> │ ┌────────────────┐
+ │ ids │ │ + tokens │ │ Pad/trunc to │
+ └─────┘ │ → dec_in │ │ max_mel_len │
+ ├──────────────┤ └───────┬────────┘
+ │ <story> │ │
+ │ + tokens │ ▼
+ │ + [EOS] │ ┌────────────────┐
+ │ → dec_tgt │ │ [BOS]<transcr> │
+ └──────────────┘ │ + tokens │
+ │ → dec_in │
+ Both cached as ├────────────────┤
+ .npy in .data_cache/ │ <transcr> │
+ │ + tokens │
+ ┌──────────────────────┐ │ + [EOS] │
+ │ Interleaved batches: │ │ → dec_tgt │
+ │ 1 speech every N │ └────────────────┘
+ │ text steps │
+ │ (--speech-every N) │ Eval: WER on 3 val samples
+ └──────────────────────┘ at end of each epoch
```
## Usage
@@ -108,10 +154,16 @@ needle [command]
│ --checkpoint PATH Resume from checkpoint │
│ --checkpoint-dir DIR Checkpoint directory │
│ --seed INT Random seed (default: 42) │
+ │ --no-speech Disable speech (text-only training) │
+ │ --speech-every INT Speech step every N text (default: 3) │
+ │ --max-mel-len INT Max mel frames (default: 1024) │
+ │ --n-mels INT Mel frequency bins (default: 80) │
+ │ --max-speech-samples INT Max LibriSpeech samples │
│ │
│ run │
│ --checkpoint PATH Path to model checkpoint (required) │
│ --prompts STR [...] One or more prompts to continue │
+ │ --audio PATH [...] Audio files to transcribe │
│ --max-len INT Max tokens to generate (default: 128) │
│ --temperature FLOAT Sampling temperature (default: 0.8) │
│ --seed INT Random seed (default: 0) │
diff --git a/pyproject.toml b/pyproject.toml
index 13ebd8e..229c96a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -11,6 +11,8 @@ dependencies = [
"transformers",
"wandb",
"sentencepiece",
+ "soundfile",
+ "scipy",
]
[project.optional-dependencies]
diff --git a/src/cli.py b/src/cli.py
index 037cbb6..e912af0 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -36,10 +36,16 @@ HELP = """
│ --checkpoint PATH Resume from checkpoint │
│ --checkpoint-dir DIR Checkpoint directory │
│ --seed INT Random seed (default: 42) │
+ │ --no-speech Disable speech (text-only training) │
+ │ --speech-every INT Speech step every N text (default: 3) │
+ │ --max-mel-len INT Max mel frames (default: 1024) │
+ │ --n-mels INT Mel frequency bins (default: 80) │
+ │ --max-speech-samples INT Max LibriSpeech samples │
│ │
│ run │
│ --checkpoint PATH Path to model checkpoint (required) │
│ --prompts STR [...] One or more prompts to continue │
+ │ --audio PATH [...] Audio files to transcribe │
│ --max-len INT Max tokens to generate (default: 128) │
│ --temperature FLOAT Sampling temperature (default: 0.8) │
│ --seed INT Random seed (default: 0) │
@@ -114,10 +120,20 @@ def main():
p.add_argument("--num-memory-slots", type=int, default=64)
p.add_argument("--mrl-dims", type=int, nargs="*", default=[256, 128, 64],
help="MRL dimension targets (default: 256 128 64)")
+ p.add_argument("--no-speech", action="store_true", help="Disable speech training (text-only)")
+ p.add_argument("--speech-every", type=int, default=3,
+ help="Do one speech step every N text steps (default: 3)")
+ p.add_argument("--max-mel-len", type=int, default=1024,
+ help="Max mel spectrogram frames (default: 1024)")
+ p.add_argument("--n-mels", type=int, default=80,
+ help="Number of mel frequency bins (default: 80)")
+ p.add_argument("--max-speech-samples", type=int, default=None,
+ help="Max LibriSpeech training samples (default: all)")
p = sub.add_parser("run", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
p.add_argument("--prompts", type=str, nargs="*")
+ p.add_argument("--audio", type=str, nargs="*", help="Audio file paths to transcribe")
p.add_argument("--max-len", type=int, default=128)
p.add_argument("--temperature", type=float, default=0.8)
p.add_argument("--seed", type=int, default=0)
diff --git a/src/data.py b/src/data.py
index 9cadde8..b89a420 100644
--- a/src/data.py
+++ b/src/data.py
@@ -10,7 +10,7 @@ logging.getLogger("transformers").setLevel(logging.ERROR)
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
import numpy as np
-from datasets import load_dataset
+from datasets import Audio, load_dataset
from tqdm import tqdm
import sentencepiece as spm
@@ -21,6 +21,9 @@ TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")
PAD_ID = 0
EOS_ID = 1
BOS_ID = 2
+UNK_ID = 3
+STORY_ID = 4
+TRANSCRIBE_ID = 5
class NeedleTokenizer:
@@ -42,6 +45,14 @@ class NeedleTokenizer:
def bos_token_id(self):
return BOS_ID
+ @property
+ def story_token_id(self):
+ return STORY_ID
+
+ @property
+ def transcribe_token_id(self):
+ return TRANSCRIBE_ID
+
@property
def vocab_size(self):
return self.sp.GetPieceSize()
@@ -111,7 +122,8 @@ def train_tokenizer(vocab_size=8192, max_samples=None):
pad_id=PAD_ID,
eos_id=EOS_ID,
bos_id=BOS_ID,
- unk_id=3,
+ unk_id=UNK_ID,
+ user_defined_symbols=["<story>", "<transcribe>"],
byte_fallback=True,
normalization_rule_name="identity",
num_threads=os.cpu_count(),
@@ -147,18 +159,19 @@ def _tokenizer_hash():
return "none"
-def _cache_key(n_samples, max_enc_len, max_dec_len):
+def _cache_key(prefix, n_samples, max_enc_len, max_dec_len):
tok_hash = _tokenizer_hash()
- key = f"tinystories_{tok_hash}_{n_samples}_{max_enc_len}_{max_dec_len}"
+ key = f"{prefix}_{tok_hash}_{n_samples}_{max_enc_len}_{max_dec_len}"
return hashlib.md5(key.encode()).hexdigest()[:12]
-def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):
+def prepare_text_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):
+ """Prepare text encoder-decoder pairs with <story> task token."""
- cache_id = _cache_key(len(ds), max_enc_len, max_dec_len)
+ cache_id = _cache_key("text", len(ds), max_enc_len, max_dec_len)
cache_path = os.path.join(CACHE_DIR, cache_id)
if os.path.exists(cache_path + "_enc.npy"):
- print(f"Loading cached data ({cache_id})...")
+ print(f"Loading cached text data ({cache_id})...")
return (
np.load(cache_path + "_enc.npy"),
np.load(cache_path + "_dec_in.npy"),
@@ -167,6 +180,7 @@ def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=12
pad_id = tokenizer.pad_token_id
bos_id = tokenizer.eos_token_id
+ story_id = tokenizer.story_token_id
max_total = max_enc_len + max_dec_len
# --- Parallel tokenization ---
@@ -189,7 +203,8 @@ def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=12
split_points = np.maximum(2, (lengths[valid_indices] * split_ratio).astype(np.int32))
enc_lens = np.minimum(split_points, max_enc_len)
- dec_lens = np.minimum(lengths[valid_indices] - split_points, max_dec_len)
+ # Account for <story> token taking one slot in decoder
+ dec_lens = np.minimum(lengths[valid_indices] - split_points, max_dec_len - 1)
keep = dec_lens >= 2
valid_indices = valid_indices[keep]
@@ -207,20 +222,29 @@ def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=12
):
tokens = all_tokens[idx]
enc_inputs[i, :el] = tokens[:sp][:max_enc_len]
+ # Decoder input: [BOS, <story>, tok1, tok2, ...]
dec_inputs[i, 0] = bos_id
- dec_inputs[i, 1:dl] = tokens[sp:][:dl - 1]
- dec_targets[i, :dl] = tokens[sp:][:max_dec_len]
+ dec_inputs[i, 1] = story_id
+ dec_inputs[i, 2:dl + 1] = tokens[sp:][:dl - 1]
+ # Decoder target: [<story>, tok1, tok2, ..., EOS]
+ dec_targets[i, 0] = story_id
+ dec_targets[i, 1:dl + 1] = tokens[sp:][:dl]
# Save cache
os.makedirs(CACHE_DIR, exist_ok=True)
np.save(cache_path + "_enc.npy", enc_inputs)
np.save(cache_path + "_dec_in.npy", dec_inputs)
np.save(cache_path + "_dec_tgt.npy", dec_targets)
- print(f"Cached {n_keep:,} pairs to {CACHE_DIR}/{cache_id}")
+ print(f"Cached {n_keep:,} text pairs to {CACHE_DIR}/{cache_id}")
return enc_inputs, dec_inputs, dec_targets
+# Backward compatibility alias
+def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):
+ return prepare_text_pairs(ds, tokenizer, max_enc_len, max_dec_len, split_ratio)
+
+
def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True):
n = len(enc_inputs)
if shuffle:
@@ -231,3 +255,194 @@ def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True):
for i in range(0, n - batch_size + 1, batch_size):
yield enc_inputs[i : i + batch_size], dec_inputs[i : i + batch_size], dec_targets[i : i + batch_size]
+
+
+get_text_batches = get_batches
+
+
+# ─── Speech pipeline ───────────────────────────────────────────────────────────
+
+def load_librispeech(split="train", max_samples=None):
+ """Load LibriSpeech-100 from HuggingFace (SPRINGLab/LibriSpeech-100).
+
+ The dataset has a single 'train' split, so we partition it:
+ - train / train.clean.100: first 90%
+ - validation.clean / test.clean / validation / test: last 10%
+
+ Audio decoding is disabled (to avoid torchcodec/torch dependency);
+ prepare_speech_pairs decodes audio bytes via soundfile instead.
+ """
+ ds = load_dataset("SPRINGLab/LibriSpeech-100", split="train")
+ ds = ds.cast_column("audio", Audio(decode=False))
+ n = len(ds)
+ if split in ("validation.clean", "validation", "test.clean", "test"):
+ ds = ds.select(range(int(n * 0.9), n))
+ elif split.startswith("train"):
+ ds = ds.select(range(int(n * 0.9)))
+ if max_samples:
+ ds = ds.select(range(min(max_samples, len(ds))))
+ return ds
+
+
+def _decode_audio(audio_data, target_sr=16000):
+ """Decode raw audio bytes from a datasets Audio(decode=False) field using soundfile."""
+ import io
+ import soundfile as sf
+
+ audio_bytes = audio_data["bytes"]
+ if audio_bytes is None:
+ path = audio_data["path"]
+ audio, sr = sf.read(path, dtype="float32")
+ else:
+ audio, sr = sf.read(io.BytesIO(audio_bytes), dtype="float32")
+
+ # Convert stereo to mono
+ if audio.ndim > 1:
+ audio = audio.mean(axis=1)
+
+ # Resample if needed
+ if sr != target_sr:
+ from scipy.signal import resample
+ num_samples = int(len(audio) * target_sr / sr)
+ audio = resample(audio, num_samples).astype(np.float32)
+ sr = target_sr
+
+ return audio.astype(np.float32), sr
+
+
+def compute_mel_spectrogram(audio, sr=16000, n_mels=80, n_fft=400, hop_length=160):
+ """Compute log-mel spectrogram using numpy/scipy. Returns (T_mel, n_mels) float32.
+
+ 25ms window (n_fft=400 at 16kHz), 10ms hop (hop_length=160) → ~100 frames/sec.
+ """
+ from scipy.signal import windows as scipy_windows
+ from scipy.fft import rfft
+
+ # Pre-emphasis
+ audio = np.append(audio[0], audio[1:] - 0.97 * audio[:-1])
+
+ # STFT
+ window = scipy_windows.hann(n_fft, sym=False).astype(np.float32)
+ num_frames = 1 + (len(audio) - n_fft) // hop_length
+ if num_frames <= 0:
+ return np.zeros((1, n_mels), dtype=np.float32)
+
+ frames = np.lib.stride_tricks.as_strided(
+ audio,
+ shape=(num_frames, n_fft),
+ strides=(audio.strides[0] * hop_length, audio.strides[0]),
+ ).copy()
+ frames = frames * window
+ spectrum = np.abs(rfft(frames, n=n_fft, axis=-1)) ** 2
+
+ # Mel filterbank
+ fmin, fmax = 0.0, sr / 2.0
+ mel_low = 2595.0 * np.log10(1.0 + fmin / 700.0)
+ mel_high = 2595.0 * np.log10(1.0 + fmax / 700.0)
+ mel_points = np.linspace(mel_low, mel_high, n_mels + 2)
+ hz_points = 700.0 * (10.0 ** (mel_points / 2595.0) - 1.0)
+ bin_points = np.floor((n_fft + 1) * hz_points / sr).astype(np.int32)
+
+ filterbank = np.zeros((n_mels, n_fft // 2 + 1), dtype=np.float32)
+ for m in range(n_mels):
+ f_left, f_center, f_right = bin_points[m], bin_points[m + 1], bin_points[m + 2]
+ for k in range(f_left, f_center):
+ if f_center > f_left:
+ filterbank[m, k] = (k - f_left) / (f_center - f_left)
+ for k in range(f_center, f_right):
+ if f_right > f_center:
+ filterbank[m, k] = (f_right - k) / (f_right - f_center)
+
+ mel_spec = spectrum @ filterbank.T
+ log_mel = np.log(np.maximum(mel_spec, 1e-10))
+ return log_mel.astype(np.float32)
+
+
+def _speech_cache_key(n_samples, n_mels, max_mel_len, max_dec_len):
+ tok_hash = _tokenizer_hash()
+ key = f"speech_{tok_hash}_{n_samples}_{n_mels}_{max_mel_len}_{max_dec_len}"
+ return hashlib.md5(key.encode()).hexdigest()[:12]
+
+
+def prepare_speech_pairs(ds, tokenizer, n_mels=80, max_mel_len=1024, max_dec_len=128):
+ """Prepare speech encoder-decoder pairs with <transcribe> task token."""
+
+ cache_id = _speech_cache_key(len(ds), n_mels, max_mel_len, max_dec_len)
+ cache_path = os.path.join(CACHE_DIR, cache_id)
+ if os.path.exists(cache_path + "_mel.npy"):
+ print(f"Loading cached speech data ({cache_id})...")
+ return (
+ np.load(cache_path + "_mel.npy"),
+ np.load(cache_path + "_dec_in.npy"),
+ np.load(cache_path + "_dec_tgt.npy"),
+ )
+
+ pad_id = tokenizer.pad_token_id
+ bos_id = tokenizer.eos_token_id
+ eos_id = tokenizer.eos_token_id
+ transcribe_id = tokenizer.transcribe_token_id
+
+ mel_features = []
+ dec_inputs_list = []
+ dec_targets_list = []
+
+ for example in tqdm(ds, desc="Processing speech"):
+ audio, sr = _decode_audio(example["audio"])
+
+ mel = compute_mel_spectrogram(audio, sr=sr, n_mels=n_mels)
+
+ # Pad or truncate mel to max_mel_len
+ if mel.shape[0] > max_mel_len:
+ mel = mel[:max_mel_len]
+ elif mel.shape[0] < max_mel_len:
+ pad_len = max_mel_len - mel.shape[0]
+ mel = np.pad(mel, ((0, pad_len), (0, 0)))
+
+ # Tokenize transcript
+ text = example["text"].strip().lower()
+ tokens = tokenizer.encode(text)
+ # Leave room for <transcribe> prefix and EOS
+ tokens = tokens[:max_dec_len - 2]
+
+ # Decoder input: [BOS, <transcribe>, tok1, tok2, ...]
+ dec_in = np.full(max_dec_len, pad_id, dtype=np.int32)
+ dec_in[0] = bos_id
+ dec_in[1] = transcribe_id
+ n_tok = len(tokens)
+ if n_tok > 0:
+ dec_in[2:2 + n_tok] = tokens
+
+ # Decoder target: [<transcribe>, tok1, tok2, ..., EOS]
+ dec_tgt = np.full(max_dec_len, pad_id, dtype=np.int32)
+ dec_tgt[0] = transcribe_id
+ if n_tok > 0:
+ dec_tgt[1:1 + n_tok] = tokens
+ dec_tgt[1 + n_tok] = eos_id
+
+ mel_features.append(mel)
+ dec_inputs_list.append(dec_in)
+ dec_targets_list.append(dec_tgt)
+
+ mel_features = np.stack(mel_features).astype(np.float32)
+ dec_inputs = np.stack(dec_inputs_list)
+ dec_targets = np.stack(dec_targets_list)
+
+ os.makedirs(CACHE_DIR, exist_ok=True)
+ np.save(cache_path + "_mel.npy", mel_features)
+ np.save(cache_path + "_dec_in.npy", dec_inputs)
+ np.save(cache_path + "_dec_tgt.npy", dec_targets)
+ print(f"Cached {len(mel_features):,} speech pairs to {CACHE_DIR}/{cache_id}")
+
+ return mel_features, dec_inputs, dec_targets
+
+
+def get_speech_batches(mel_features, dec_inputs, dec_targets, batch_size, shuffle=True):
+ n = len(mel_features)
+ if shuffle:
+ perm = np.random.permutation(n)
+ mel_features = mel_features[perm]
+ dec_inputs = dec_inputs[perm]
+ dec_targets = dec_targets[perm]
+
+ for i in range(0, n - batch_size + 1, batch_size):
+ yield mel_features[i : i + batch_size], dec_inputs[i : i + batch_size], dec_targets[i : i + batch_size]
diff --git a/src/model.py b/src/model.py
index af99328..15bd222 100644
--- a/src/model.py
+++ b/src/model.py
@@ -44,6 +44,7 @@ class TransformerConfig:
dtype: str = "bfloat16"
activation: str = "drelu"
num_memory_slots: int = 64
+ n_mels: int = 80
@property
def jax_dtype(self):
@@ -303,6 +304,17 @@ class Decoder(nn.Module):
return x
+class MelProjection(nn.Module):
+ """Project mel spectrogram features to model dimension."""
+ d_model: int
+ dtype: jnp.dtype = jnp.bfloat16
+
+ @nn.compact
+ def __call__(self, mel):
+ return nn.Dense(self.d_model, dtype=self.dtype, use_bias=True,
+ kernel_init=default_init(), name="proj")(mel)
+
+
class EncoderDecoderTransformer(nn.Module):
"""Encoder-decoder transformer with shared embeddings, tied output, RoPE, and bfloat16."""
config: TransformerConfig
@@ -310,6 +322,7 @@ class EncoderDecoderTransformer(nn.Module):
def setup(self):
self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))
self.embed_scale = math.sqrt(self.config.d_model)
+ self.mel_proj = MelProjection(self.config.d_model, self.config.jax_dtype)
self.encoder = MemoryMixerEncoder(self.config)
self.decoder = Decoder(self.config)
@@ -317,11 +330,20 @@ class EncoderDecoderTransformer(nn.Module):
head_dim = self.config.d_model // self.config.num_heads
return precompute_rope_freqs(head_dim, seq_len, self.config.rope_theta)
- def encode(self, src, src_mask=None):
+ def encode_text(self, src, src_mask=None):
x = self.embedding(src) * self.embed_scale
rope = self._rope(src.shape[1])
return self.encoder(x, mask=src_mask, rope=rope)
+ def encode(self, src, src_mask=None):
+ """Backward-compatible alias for encode_text."""
+ return self.encode_text(src, src_mask=src_mask)
+
+ def encode_speech(self, mel, src_mask=None):
+ x = self.mel_proj(mel) * self.embed_scale
+ rope = self._rope(mel.shape[1])
+ return self.encoder(x, mask=src_mask, rope=rope)
+
def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None):
x = self.embedding(tgt) * self.embed_scale
rope = self._rope(tgt.shape[1])
@@ -332,19 +354,19 @@ class EncoderDecoderTransformer(nn.Module):
return logits
def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):
- encoder_out = self.encode(src, src_mask=src_mask)
+ encoder_out = self.encode_text(src, src_mask=src_mask)
logits = self.decode(
tgt, encoder_out, self_mask=tgt_mask
)
return logits
- def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mrl_dims=None):
- encoder_out = self.encode(src, src_mask=src_mask)
+ def _decode_with_aux(self, encoder_out, tgt, tgt_mask=None, mrl_dims=None):
+ """Shared decode logic returning logits, slot_div, mrl_logits."""
x = self.embedding(tgt) * self.embed_scale
rope = self._rope(tgt.shape[1])
x = self.decoder(x, encoder_out, self_mask=tgt_mask, cross_mask=None, rope=rope)
x_f32 = x.astype(jnp.float32)
- emb = self.embedding.embedding
+ emb = self.embedding.embedding
logits = x_f32 @ emb.T
mrl_logits = []
@@ -359,6 +381,25 @@ class EncoderDecoderTransformer(nn.Module):
slot_div = (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]
return logits, slot_div, mrl_logits
+ def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mrl_dims=None):
+ encoder_out = self.encode_text(src, src_mask=src_mask)
+ return self._decode_with_aux(encoder_out, tgt, tgt_mask=tgt_mask, mrl_dims=mrl_dims)
+
+ def forward_speech_with_aux(self, mel, tgt, src_mask=None, tgt_mask=None, mrl_dims=None):
+ encoder_out = self.encode_speech(mel, src_mask=src_mask)
+ return self._decode_with_aux(encoder_out, tgt, tgt_mask=tgt_mask, mrl_dims=mrl_dims)
+
+ def init_all(self, src, tgt, mel):
+ """Dummy forward through both text and speech pathways to initialize all params."""
+ src_mask = make_padding_mask(src, self.config.pad_token_id)
+ tgt_mask = make_causal_mask(tgt.shape[1]) & make_padding_mask(tgt, self.config.pad_token_id)
+ text_out = self.encode_text(src, src_mask=src_mask)
+ mel_mask = make_mel_padding_mask(mel)
+ speech_out = self.encode_speech(mel, src_mask=mel_mask)
+ _ = self._decode_with_aux(text_out, tgt, tgt_mask=tgt_mask)
+ _ = self._decode_with_aux(speech_out, tgt, tgt_mask=tgt_mask)
+ return jnp.zeros(())
+
def make_causal_mask(seq_len):
mask = jnp.tril(jnp.ones((seq_len, seq_len), dtype=jnp.bool_))
@@ -368,3 +409,9 @@ def make_causal_mask(seq_len):
def make_padding_mask(tokens, pad_token_id):
mask = tokens != pad_token_id
return mask[:, None, None, :]
+
+
+def make_mel_padding_mask(mel):
+ """Create padding mask from mel spectrogram: non-zero frames are valid."""
+ mask = jnp.any(mel != 0, axis=-1) # (B, T)
+ return mask[:, None, None, :] # (B, 1, 1, T)
diff --git a/src/run.py b/src/run.py
index df60452..f4490f7 100644
--- a/src/run.py
+++ b/src/run.py
@@ -4,13 +4,15 @@ import sys
import jax
import jax.numpy as jnp
+import numpy as np
-from .data import get_tokenizer
+from .data import get_tokenizer, compute_mel_spectrogram
from .model import (
EncoderDecoderTransformer,
TransformerConfig,
make_causal_mask,
make_padding_mask,
+ make_mel_padding_mask,
)
@@ -84,6 +86,87 @@ def generate(model, params, tokenizer, prompt, max_gen_len=128, temperature=0.8,
return tokenizer.decode(generated_tokens)
+def load_audio(path, target_sr=16000):
+ """Load an audio file and resample to target_sr. Returns (audio_array, sr)."""
+ import soundfile as sf
+ audio, sr = sf.read(path, dtype="float32")
+ # Convert stereo to mono
+ if audio.ndim > 1:
+ audio = audio.mean(axis=1)
+ # Resample if needed
+ if sr != target_sr:
+ from scipy.signal import resample
+ num_samples = int(len(audio) * target_sr / sr)
+ audio = resample(audio, num_samples).astype(np.float32)
+ sr = target_sr
+ return audio, sr
+
+
+def transcribe(model, params, tokenizer, audio_array, sr=16000, max_gen_len=128, temperature=0.8, seed=0, stream=True):
+ """Transcribe audio using the speech encoder pathway."""
+ n_mels = model.config.n_mels
+ mel = compute_mel_spectrogram(audio_array, sr=sr, n_mels=n_mels)
+ mel_input = jnp.array(mel)[None, :, :] # (1, T_mel, n_mels)
+
+ pad_id = tokenizer.pad_token_id
+ eos_id = tokenizer.eos_token_id
+ transcribe_id = tokenizer.transcribe_token_id
+
+ src_mask = make_mel_padding_mask(mel_input)
+ encoder_out = model.apply(
+ {"params": params}, mel_input, src_mask=src_mask, method="encode_speech"
+ )
+
+ tgt_mask = make_causal_mask(max_gen_len)
+ dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
+ dec_buffer = dec_buffer.at[0, 0].set(eos_id) # BOS = EOS in this tokenizer
+ dec_buffer = dec_buffer.at[0, 1].set(transcribe_id)
+
+ @jax.jit
+ def decode_step(dec_buffer, encoder_out, src_mask):
+ logits = model.apply(
+ {"params": params},
+ dec_buffer,
+ encoder_out,
+ self_mask=tgt_mask,
+ cross_mask=src_mask,
+ method="decode",
+ )
+ return logits
+
+ rng = jax.random.PRNGKey(seed)
+ generated_tokens = []
+
+ if stream:
+ sys.stdout.write("\n")
+ sys.stdout.flush()
+
+ logits = decode_step(dec_buffer, encoder_out, src_mask)
+
+ # Start generating from position 2 (after BOS + <transcribe>)
+ for i in range(1, max_gen_len - 1):
+ next_logits = logits[0, i] / temperature
+ rng, sample_rng = jax.random.split(rng)
+ next_token = jax.random.categorical(sample_rng, next_logits).item()
+
+ if next_token == eos_id:
+ break
+
+ generated_tokens.append(next_token)
+ dec_buffer = dec_buffer.at[0, i + 1].set(next_token)
+
+ if stream:
+ sys.stdout.write(tokenizer.decode([next_token]))
+ sys.stdout.flush()
+
+ logits = decode_step(dec_buffer, encoder_out, src_mask)
+
+ if stream:
+ sys.stdout.write("\n")
+
+ return tokenizer.decode(generated_tokens)
+
+
def main(args):
print(f"Loading checkpoint: {args.checkpoint}")
params, config = load_checkpoint(args.checkpoint)
@@ -94,6 +177,26 @@ def main(args):
param_count = sum(x.size for x in jax.tree.leaves(params))
print(f"Model parameters: {param_count:,}")
+ # --- Audio transcription mode ---
+ audio_files = getattr(args, "audio", None)
+ if audio_files:
+ for i, audio_path in enumerate(audio_files):
+ print(f"\nTranscribing: {audio_path}")
+ audio, sr = load_audio(audio_path)
+ transcribe(
+ model,
+ params,
+ tokenizer,
+ audio,
+ sr=sr,
+ max_gen_len=args.max_len,
+ temperature=args.temperature,
+ seed=args.seed + i,
+ stream=True,
+ )
+ return
+
+ # --- Text generation mode ---
prompts = args.prompts or ["Once upon a time", "The little dog", "She was very happy because"]
for i, prompt in enumerate(prompts):
diff --git a/src/test.py b/src/test.py
index 86d3598..93e45e5 100644
--- a/src/test.py
+++ b/src/test.py
@@ -8,7 +8,7 @@ import jax.numpy as jnp
import numpy as np
import optax
-from .data import get_batches, get_tokenizer, load_tinystories, prepare_encoder_decoder_pairs
+from .data import get_batches, get_tokenizer, load_tinystories, prepare_text_pairs, load_librispeech
from .model import (
EncoderDecoderTransformer,
TransformerConfig,
@@ -150,6 +150,64 @@ def benchmark_generation_quality(model, params, tokenizer, prompts, max_gen_len=
}
+def compute_wer(hypotheses, references):
+ """Compute word error rate using edit distance."""
+ total_edits = 0
+ total_ref_words = 0
+
+ for hyp, ref in zip(hypotheses, references):
+ hyp_words = hyp.lower().split()
+ ref_words = ref.lower().split()
+ n = len(ref_words)
+ m = len(hyp_words)
+
+ # DP edit distance
+ d = [[0] * (m + 1) for _ in range(n + 1)]
+ for i in range(n + 1):
+ d[i][0] = i
+ for j in range(m + 1):
+ d[0][j] = j
+ for i in range(1, n + 1):
+ for j in range(1, m + 1):
+ if ref_words[i - 1] == hyp_words[j - 1]:
+ d[i][j] = d[i - 1][j - 1]
+ else:
+ d[i][j] = 1 + min(d[i - 1][j], d[i][j - 1], d[i - 1][j - 1])
+
+ total_edits += d[n][m]
+ total_ref_words += n
+
+ return total_edits / max(total_ref_words, 1)
+
+
+def benchmark_asr(model, params, tokenizer, num_samples=100):
+ """Transcribe LibriSpeech test-clean samples and report WER."""
+ from .run import transcribe
+ from .data import load_librispeech
+
+ ds = load_librispeech("test.clean", max_samples=num_samples)
+
+ hypotheses = []
+ references = []
+
+ for example in ds:
+ audio_data = example["audio"]
+ audio = np.array(audio_data["array"], dtype=np.float32)
+ sr = audio_data["sampling_rate"]
+ ref_text = example["text"].strip().lower()
+
+ hyp_text = transcribe(
+ model, params, tokenizer, audio, sr=sr,
+ max_gen_len=128, temperature=0.0, seed=0, stream=False,
+ ).strip().lower()
+
+ hypotheses.append(hyp_text)
+ references.append(ref_text)
+
+ wer = compute_wer(hypotheses, references)
+ return {"wer": wer, "num_samples": len(hypotheses)}
+
+
def main(args):
params, config = load_checkpoint(args.checkpoint)
model = EncoderDecoderTransformer(config)
@@ -160,13 +218,19 @@ def main(args):
print(f"parameters: {param_count:,}")
print(f"config: d={config.d_model}, heads={config.num_heads}, layers={config.num_encoder_layers}/{config.num_decoder_layers}")
- print(f"\nevaluating perplexity ({args.max_eval_samples} samples)...")
+ print(f"\nevaluating text perplexity ({args.max_eval_samples} samples)...")
ds = load_tinystories("validation", max_samples=args.max_eval_samples)
- enc_inputs, dec_inputs, dec_targets = prepare_encoder_decoder_pairs(
+ enc_inputs, dec_inputs, dec_targets = prepare_text_pairs(
ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
)
ppl = compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, args.batch_size, config.pad_token_id)
- print(f"perplexity: {ppl:.2f}")
+ print(f"text ppl: {ppl:.2f}")
+
+ asr_samples = getattr(args, "asr_samples", 50)
+ if asr_samples > 0:
+ print(f"\nevaluating ASR WER ({asr_samples} samples)...")
+ asr_results = benchmark_asr(model, params, tokenizer, num_samples=asr_samples)
+ print(f"ASR WER: {asr_results['wer']:.4f} ({asr_results['num_samples']} samples)")
print(f"\nmeasuring throughput ({args.throughput_runs} runs)...")
throughput = measure_throughput(model, params, tokenizer, num_runs=args.throughput_runs)
diff --git a/src/train.py b/src/train.py
index c61f852..c5651fa 100644
--- a/src/train.py
+++ b/src/train.py
@@ -13,12 +13,16 @@ from tqdm import tqdm
from flax import jax_utils
from flax.training import train_state
-from .data import get_batches, get_tokenizer, load_tinystories, prepare_encoder_decoder_pairs
+from .data import (
+ get_batches, get_tokenizer, load_tinystories, prepare_text_pairs,
+ load_librispeech, prepare_speech_pairs, get_speech_batches,
+)
from .model import (
EncoderDecoderTransformer,
TransformerConfig,
make_causal_mask,
make_padding_mask,
+ make_mel_padding_mask,
)
def _newton_schulz(G, steps=5):
@@ -99,10 +103,13 @@ def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_
rng, init_rng = jax.random.split(rng)
dummy_src = jnp.ones((1, 128), dtype=jnp.int32)
dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)
+ dummy_mel = jnp.ones((1, 128, config.n_mels), dtype=jnp.float32)
variables = model.init(
{"params": init_rng},
dummy_src,
dummy_tgt,
+ dummy_mel,
+ method="init_all",
)
adam_schedule = _wsd_schedule(learning_rate, total_steps, warmup_steps)
@@ -220,23 +227,9 @@ _GROUP_SIZE = 32
_MRL_DIMS = ()
-def _loss_fn(state, params, src, tgt_in, tgt_out, causal_mask):
+def _compute_loss(logits, slot_div, mrl_logits, tgt_out):
+ """Shared CE + MRL + z-loss + slot-div computation."""
pad_id = 0
- src_mask = make_padding_mask(src, pad_id)
- tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
- cross_mask = src_mask
-
- logits, slot_div, mrl_logits = state.apply_fn(
- {"params": _quantize_params(params, group_size=_GROUP_SIZE)},
- src,
- tgt_in,
- src_mask=src_mask,
- tgt_mask=tgt_mask,
- cross_mask=cross_mask,
- mrl_dims=_MRL_DIMS if _MRL_DIMS else None,
- method="forward_with_aux",
- )
-
logits_f32 = logits.astype(jnp.float32)
mask = (tgt_out != pad_id).astype(jnp.float32)
ce_loss = jnp.sum(
@@ -259,10 +252,74 @@ def _loss_fn(state, params, src, tgt_in, tgt_out, causal_mask):
return total_loss + z_loss + div_loss
-def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):
+def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask):
+ pad_id = 0
+ src_mask = make_padding_mask(src, pad_id)
+ tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
+
+ logits, slot_div, mrl_logits = state.apply_fn(
+ {"params": _quantize_params(params, group_size=_GROUP_SIZE)},
+ src,
+ tgt_in,
+ src_mask=src_mask,
+ tgt_mask=tgt_mask,
+ cross_mask=src_mask,
+ mrl_dims=_MRL_DIMS if _MRL_DIMS else None,
+ method="forward_with_aux",
+ )
+ return _compute_loss(logits, slot_div, mrl_logits, tgt_out)
+
+
+def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask):
+ pad_id = 0
+ src_mask = make_mel_padding_mask(mel)
+ tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
+
+ logits, slot_div, mrl_logits = state.apply_fn(
+ {"params": _quantize_params(params, group_size=_GROUP_SIZE)},
+ mel,
+ tgt_in,
+ src_mask=src_mask,
+ tgt_mask=tgt_mask,
+ mrl_dims=_MRL_DIMS if _MRL_DIMS else None,
+ method="forward_speech_with_aux",
+ )
+ return _compute_loss(logits, slot_div, mrl_logits, tgt_out)
+
+
+def _train_step_text(state, ema_params, src, tgt_in, tgt_out, causal_mask):
+ ema_decay = 0.999
+ loss, grads = jax.value_and_grad(
+ lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask)
+ )(state.params)
+ grads = jax.lax.pmean(grads, axis_name="batch")
+ loss = jax.lax.pmean(loss, axis_name="batch")
+ grad_norm = optax.global_norm(grads)
+ state = state.apply_gradients(grads=grads)
+ ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, state.params)
+ return state, ema_params, loss, grad_norm
+
+
+def _train_step_text_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask):
+ """Text training step with fused mask application."""
+ ema_decay = 0.999
+ loss, grads = jax.value_and_grad(
+ lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask)
+ )(state.params)
+ grads = jax.lax.pmean(grads, axis_name="batch")
+ loss = jax.lax.pmean(loss, axis_name="batch")
+ grad_norm = optax.global_norm(grads)
+ state = state.apply_gradients(grads=grads)
+ masked_params = jax.tree.map(lambda w, m: w * m, state.params, prune_mask)
+ state = state.replace(params=masked_params)
+ ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, masked_params)
+ return state, ema_params, loss, grad_norm
+
+
+def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask):
ema_decay = 0.999
loss, grads = jax.value_and_grad(
- lambda p: _loss_fn(state, p, src, tgt_in, tgt_out, causal_mask)
+ lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask)
)(state.params)
grads = jax.lax.pmean(grads, axis_name="batch")
loss = jax.lax.pmean(loss, axis_name="batch")
@@ -272,11 +329,11 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):
return state, ema_params, loss, grad_norm
-def _train_step_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, prune_mask):
- """Training step with fused mask application — avoids post-pmap copies."""
+def _train_step_speech_masked(state, ema_params, mel, tgt_in, tgt_out, causal_mask, prune_mask):
+ """Speech training step with fused mask application."""
ema_decay = 0.999
loss, grads = jax.value_and_grad(
- lambda p: _loss_fn(state, p, src, tgt_in, tgt_out, causal_mask)
+ lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask)
)(state.params)
grads = jax.lax.pmean(grads, axis_name="batch")
loss = jax.lax.pmean(loss, axis_name="batch")
@@ -289,11 +346,19 @@ def _train_step_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask, pru
def _make_p_train_step():
- return jax.pmap(_train_step, axis_name="batch", donate_argnums=(0, 1))
+ return jax.pmap(_train_step_text, axis_name="batch", donate_argnums=(0, 1))
def _make_p_train_step_masked():
- return jax.pmap(_train_step_masked, axis_name="batch", donate_argnums=(0, 1))
+ return jax.pmap(_train_step_text_masked, axis_name="batch", donate_argnums=(0, 1))
+
+
+def _make_p_train_step_speech():
+ return jax.pmap(_train_step_speech, axis_name="batch", donate_argnums=(0, 1))
+
+
+def _make_p_train_step_speech_masked():
+ return jax.pmap(_train_step_speech_masked, axis_name="batch", donate_argnums=(0, 1))
def _make_val_loss_fn(apply_fn):
@@ -332,6 +397,23 @@ def _make_mrl_val_loss_fn(apply_fn, mrl_dim):
return val_loss_batch
+def _make_speech_val_loss_fn(apply_fn):
+ @jax.jit
+ def val_loss_batch(params, mel, tgt_in, tgt_out, causal_mask):
+ pad_id = 0
+ src_mask = make_mel_padding_mask(mel)
+ tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
+ logits, _, _ = apply_fn(
+ {"params": params}, mel, tgt_in,
+ src_mask=src_mask, tgt_mask=tgt_mask,
+ method="forward_speech_with_aux",
+ )
+ loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)
+ mask = (tgt_out != pad_id).astype(jnp.float32)
+ return jnp.sum(loss * mask), jnp.sum(mask)
+ return val_loss_batch
+
+
def _estimate_mrl_params(config, d_prime):
"""Estimate parameter count of an exported sub-model at dimension d_prime.
@@ -370,7 +452,13 @@ def shard_batch(batch, num_devices):
def train(args):
+ import itertools
+
num_devices = jax.local_device_count()
+ no_speech = getattr(args, "no_speech", False)
+ speech_every = getattr(args, "speech_every", 3)
+ n_mels = getattr(args, "n_mels", 80)
+ max_mel_len = getattr(args, "max_mel_len", 1024)
use_wandb = getattr(args, "wandb", False)
if use_wandb:
@@ -378,27 +466,60 @@ def train(args):
if wandb.run is None:
wandb.init(project="needle-v1", config=vars(args))
- print(f"\n[1/4] Detecting devices...")
+ total_data_steps = 6 if not no_speech else 4
+ step_idx = 0
+
+ step_idx += 1
+ print(f"\n[{step_idx}/{total_data_steps}] Detecting devices...")
print(f" {num_devices} device(s) for data-parallel training")
- print(f"\n[2/4] Loading tokenizer...")
+ step_idx += 1
+ print(f"\n[{step_idx}/{total_data_steps}] Loading tokenizer...")
tokenizer = get_tokenizer(max_samples=args.max_samples)
- print(f"\n[3/4] Loading TinyStories dataset...")
+ step_idx += 1
+ print(f"\n[{step_idx}/{total_data_steps}] Loading TinyStories dataset...")
ds = load_tinystories("train", max_samples=args.max_samples)
val_ds = load_tinystories("validation", max_samples=getattr(args, "max_eval_samples", None))
- print(f"\n[4/4] Preparing encoder-decoder pairs...")
- enc_inputs, dec_inputs, dec_targets = prepare_encoder_decoder_pairs(
+ step_idx += 1
+ print(f"\n[{step_idx}/{total_data_steps}] Preparing text pairs...")
+ enc_inputs, dec_inputs, dec_targets = prepare_text_pairs(
ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
)
- val_enc, val_dec_in, val_dec_tgt = prepare_encoder_decoder_pairs(
+ val_enc, val_dec_in, val_dec_tgt = prepare_text_pairs(
val_ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
)
val_enc = np.array(val_enc)
val_dec_in = np.array(val_dec_in)
val_dec_tgt = np.array(val_dec_tgt)
- print(f" {len(enc_inputs):,} train / {len(val_enc):,} val pairs")
+ print(f" {len(enc_inputs):,} train / {len(val_enc):,} val text pairs")
+
+ # --- Speech data ---
+ speech_mel = speech_dec_in = speech_dec_tgt = None
+ val_speech_mel = val_speech_dec_in = val_speech_dec_tgt = None
+ if not no_speech:
+ step_idx += 1
+ max_speech_samples = getattr(args, "max_speech_samples", None)
+ print(f"\n[{step_idx}/{total_data_steps}] Loading LibriSpeech...")
+ speech_ds = load_librispeech("train.clean.100", max_samples=max_speech_samples)
+ speech_val_ds = load_librispeech("validation.clean", max_samples=getattr(args, "max_eval_samples", None))
+
+ step_idx += 1
+ print(f"\n[{step_idx}/{total_data_steps}] Preparing speech pairs...")
+ speech_mel, speech_dec_in, speech_dec_tgt = prepare_speech_pairs(
+ speech_ds, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len
+ )
+ val_speech_mel, val_speech_dec_in, val_speech_dec_tgt = prepare_speech_pairs(
+ speech_val_ds, tokenizer, n_mels=n_mels, max_mel_len=max_mel_len, max_dec_len=args.max_dec_len
+ )
+ speech_mel = np.array(speech_mel)
+ speech_dec_in = np.array(speech_dec_in)
+ speech_dec_tgt = np.array(speech_dec_tgt)
+ val_speech_mel = np.array(val_speech_mel)
+ val_speech_dec_in = np.array(val_speech_dec_in)
+ val_speech_dec_tgt = np.array(val_speech_dec_tgt)
+ print(f" {len(speech_mel):,} train / {len(val_speech_mel):,} val speech pairs")
effective_batch_size = args.batch_size * num_devices
@@ -422,6 +543,7 @@ def train(args):
dtype=args.dtype,
activation=getattr(args, "activation", "drelu"),
num_memory_slots=getattr(args, "num_memory_slots", 64),
+ n_mels=n_mels,
)
global _GROUP_SIZE, _MRL_DIMS
@@ -433,12 +555,16 @@ def train(args):
_MRL_DIMS = ()
p_train_step = _make_p_train_step()
p_train_step_masked = _make_p_train_step_masked()
+ p_train_step_speech = _make_p_train_step_speech()
+ p_train_step_speech_masked = _make_p_train_step_speech_masked()
np.random.seed(args.seed)
rng = jax.random.PRNGKey(args.seed)
rng, init_rng = jax.random.split(rng)
- num_batches = len(enc_inputs) // effective_batch_size
+ text_batches_per_epoch = len(enc_inputs) // effective_batch_size
+ speech_batches_per_epoch = text_batches_per_epoch // speech_every if (not no_speech and speech_mel is not None) else 0
+ num_batches = text_batches_per_epoch + speech_batches_per_epoch
total_steps = num_batches * args.epochs
warmup_steps = max(1, int(total_steps * args.warmup_ratio))
@@ -467,6 +593,12 @@ def train(args):
print(f" Memory slots {config.num_memory_slots:>12}")
print(f" Activation {config.activation:>12}")
print(f" Dtype {config.dtype:>12}")
+ if not no_speech:
+ print(f" Speech every {speech_every} text steps")
+ print(f" n_mels {n_mels:>12}")
+ print(f" max_mel_len {max_mel_len:>12}")
+ else:
+ print(f" Speech disabled")
print(f" ─────────────────────────────────────")
print(f" Devices {num_devices:>12}")
print(f" Batch {args.batch_size:>7} x {num_devices} = {effective_batch_size}")
@@ -511,11 +643,22 @@ def train(args):
f"(steps {t_start}-{t_end}/{num_batches}, interval={prune_interval}, group_size={_GROUP_SIZE})")
epoch_step = 0
- losses = []
- batches = get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size)
- pbar = tqdm(batches, total=num_batches, desc=f"Epoch {epoch + 1}/{args.epochs}")
+ text_losses = []
+ speech_losses = []
+ text_batches = get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size)
+
+ # Speech batches cycle via itertools.cycle
+ speech_iter = None
+ if not no_speech and speech_mel is not None:
+ speech_iter = iter(itertools.cycle(
+ get_speech_batches(speech_mel, speech_dec_in, speech_dec_tgt, effective_batch_size)
+ ))
+
+ text_step_in_epoch = 0
+ pbar = tqdm(text_batches, total=text_batches_per_epoch, desc=f"Epoch {epoch + 1}/{args.epochs}")
for src, tgt_in, tgt_out in pbar:
+ # --- Text step ---
t0 = time.perf_counter()
src_b = shard_batch(src, num_devices)
tgt_in_b = shard_batch(tgt_in, num_devices)
@@ -530,6 +673,31 @@ def train(args):
state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask,
)
+ text_loss_val = float(loss[0])
+ text_losses.append(text_loss_val)
+ global_step += 1
+ text_step_in_epoch += 1
+
+ # --- Speech step (every speech_every text steps) ---
+ speech_loss_val = None
+ if speech_iter is not None and text_step_in_epoch % speech_every == 0:
+ mel_batch, sp_tgt_in, sp_tgt_out = next(speech_iter)
+ mel_b = shard_batch(mel_batch, num_devices)
+ sp_tgt_in_b = shard_batch(sp_tgt_in, num_devices)
+ sp_tgt_out_b = shard_batch(sp_tgt_out, num_devices)
+
+ if prune_mask is not None:
+ state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech_masked(
+ state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, prune_mask,
+ )
+ else:
+ state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech(
+ state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask,
+ )
+ speech_loss_val = float(sp_loss[0])
+ speech_losses.append(speech_loss_val)
+ global_step += 1
+
# --- Gradual magnitude sparsification (epoch 1 only) ---
if epoch == weight_prune_epoch and not gradual_sparsify_done:
epoch_step += 1
@@ -541,9 +709,6 @@ def train(args):
prune_mask = jax_utils.replicate(mask)
del mask
- loss_val = float(loss[0])
- losses.append(loss_val)
- global_step += 1
dt = time.perf_counter() - t0
eval_every = getattr(args, "eval_every", 100)
if global_step % eval_every == 0 or global_step == total_steps:
@@ -554,28 +719,36 @@ def train(args):
vl, vt = val_loss_fn(_eval_params, vb[0], vb[1], vb[2], val_causal)
total_loss += float(vl)
total_toks += float(vt)
- del _eval_params
last_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))
- # --- tqdm postfix with sparsity during epoch 1 ---
- postfix = {"loss": f"{loss_val:.4f}", "val_ppl": f"{last_val_ppl:.2f}" if last_val_ppl is not None else "?"}
+ del _eval_params
+
+ # --- tqdm postfix ---
+ postfix = {
+ "text_loss": f"{text_loss_val:.4f}",
+ "text_ppl": f"{last_val_ppl:.2f}" if last_val_ppl is not None else "?",
+ }
+ if speech_loss_val is not None:
+ postfix["speech_loss"] = f"{speech_loss_val:.4f}"
if epoch == weight_prune_epoch and not gradual_sparsify_done:
- postfix["sparsity"] = f"{current_sparsity*100:.1f}%"
+ postfix["prune"] = f"{current_sparsity*100:.1f}%"
pbar.set_postfix(**postfix)
if use_wandb:
log_dict = {
- "train/loss": loss_val,
+ "train/text_loss": text_loss_val,
"train/grad_norm": float(grad_norm[0]),
"train/adam_lr": float(adam_schedule(global_step)),
"train/muon_lr": float(muon_schedule(global_step)),
"train/tokens_per_sec": tokens_per_batch / dt,
"train/step": global_step,
}
+ if speech_loss_val is not None:
+ log_dict["train/speech_loss"] = speech_loss_val
if epoch == weight_prune_epoch and not gradual_sparsify_done:
log_dict["train/scheduled_sparsity"] = current_sparsity
if global_step % eval_every == 0 or global_step == total_steps:
- log_dict["val/ppl"] = last_val_ppl
+ log_dict["val/text_ppl"] = last_val_ppl
wandb.log(log_dict)
# --- Lock mask after epoch 1 gradual sparsification ---
@@ -599,8 +772,8 @@ def train(args):
print(f" Final sparsity: {zero_p/total_p*100:.2f}% ({zero_p:,}/{total_p:,} near-zero)")
del final_pruned
- epoch_avg_loss = sum(losses) / len(losses) if losses else float("nan")
- final_loss = losses[-1] if losses else float("nan")
+ epoch_avg_loss = sum(text_losses) / len(text_losses) if text_losses else float("nan")
+ final_loss = text_losses[-1] if text_losses else float("nan")
final_ppl = math.exp(final_loss) if not math.isnan(final_loss) else float("nan")
eval_params = jax_utils.unreplicate(ema_params)
@@ -651,13 +824,33 @@ def train(args):
with open(ckpt_path, "wb") as f:
pickle.dump({"params": params_np, "config": config.__dict__}, f)
- from .test import measure_throughput, benchmark_generation_quality
+ from .test import measure_throughput, benchmark_generation_quality, compute_wer
+ from .run import generate, transcribe
+ from .data import _decode_audio
eval_params_jnp = jax.tree.map(jnp.array, params_np)
del params_np # free CPU memory
model = EncoderDecoderTransformer(config)
tp = measure_throughput(model, eval_params_jnp, tokenizer, num_runs=5)
prompts = ["Once upon a time", "The little dog", "She was very happy because"]
quality = benchmark_generation_quality(model, eval_params_jnp, tokenizer, prompts, max_gen_len=64, temperature=0.8)
+
+ speech_wer = None
+ transcribe_samples = []
+ if not no_speech and val_speech_mel is not None:
+ speech_val_ds = load_librispeech("validation.clean", max_samples=3)
+ hyps, refs = [], []
+ for i, example in enumerate(speech_val_ds):
+ audio, sr = _decode_audio(example["audio"])
+ ref_text = example["text"].strip().lower()
+ hyp_text = transcribe(
+ model, eval_params_jnp, tokenizer, audio, sr=sr,
+ max_gen_len=64, temperature=0.0, seed=i, stream=False,
+ ).strip().lower()
+ hyps.append(hyp_text)
+ refs.append(ref_text)
+ transcribe_samples.append((ref_text, hyp_text))
+ speech_wer = compute_wer(hyps, refs)
+
del eval_params_jnp # free device memory
print(f"\n ─────────────────────────────────────")
@@ -666,7 +859,11 @@ def train(args):
print(f" Avg loss {epoch_avg_loss:>12.4f}")
print(f" Final loss {final_loss:>12.4f}")
print(f" Train ppl {final_ppl:>12.2f}")
- print(f" Val ppl {last_val_ppl:>12.2f}")
+ print(f" Text val ppl {last_val_ppl:>12.2f}")
+ if speech_wer is not None:
+ print(f" Speech WER {speech_wer:>12.4f}")
+ epoch_avg_speech = sum(speech_losses) / len(speech_losses) if speech_losses else float("nan")
+ print(f" Avg speech loss{epoch_avg_speech:>12.4f}")
print(f" Quant val ppl {quant_val_ppl:>12.2f} (INT4 g{_GROUP_SIZE})")
print(f" Sparsity {sparsity:>11.2f}% ({near_zero:,}/{total_params:,})")
if mrl_results:
@@ -686,21 +883,31 @@ def train(args):
print(f" Repetition {quality['bigram_repetition_rate']:>12.3f}")
print(f" ─────────────────────────────────────")
for prompt, gen in quality["generations"]:
- print(f" [{prompt}]")
- print(f" {gen[:100]}")
+ gen_clean = gen.replace("<story>", "").replace("<transcribe>", "").strip()
+ print(f" <story>[{prompt}] {gen_clean[:80]}")
+ if transcribe_samples:
+ print(f" ─────────────────────────────────────")
+ print(f" Transcribe (WER={speech_wer:.4f}):")
+ for ref, hyp in transcribe_samples:
+ print(f" <transcribe> ref: {ref[:80]}")
+ print(f" hyp: {hyp[:80]}")
print(f" ─────────────────────────────────────")
print(f" Checkpoint: {ckpt_path}")
print(f" ─────────────────────────────────────\n")
if use_wandb:
log_dict = {
- "epoch/avg_loss": epoch_avg_loss,
- "epoch/final_loss": final_loss,
- "epoch/val_ppl": last_val_ppl,
+ "epoch/avg_text_loss": epoch_avg_loss,
+ "epoch/final_text_loss": final_loss,
+ "epoch/text_val_ppl": last_val_ppl,
"epoch/quant_val_ppl": quant_val_ppl,
"epoch/weight_sparsity": sparsity,
"epoch": epoch + 1,
}
+ if speech_wer is not None:
+ log_dict["epoch/speech_wer"] = speech_wer
+ if speech_losses:
+ log_dict["epoch/avg_speech_loss"] = sum(speech_losses) / len(speech_losses)
for d_prime, (mrl_ppl, mrl_params) in mrl_results.items():
log_dict[f"epoch/mrl_ppl_d{d_prime}"] = mrl_ppl
log_dict[f"epoch/mrl_params_d{d_prime}"] = mrl_params