Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Merge branch 'main' into fixed-mrl
Noah Cylich7d8f27d4248e
diff --cc src/model.pyindex 74bf602,d29f084..7abbd21--- a/src/model.py+++ b/src/model.py@@@ -326,10 -385,25 +372,25 @@@ class EncoderDecoderTransformer(nn.Modu"""Backward-compatible alias for encode_text."""return self.encode_text(src, src_mask=src_mask)- def encode_speech(self, mel, src_mask=None, ffn_mask=None):- x = self.mel_proj(mel) * self.embed_scale- rope = self._rope(mel.shape[1])+ @staticmethod+ def _downsample_mask(mask, factor):+ """Downsample a padding mask by OR-pooling groups of `factor` frames."""+ if mask is None:+ return None+ # mask: (B, 1, 1, T)+ T = mask.shape[-1]+ pad = (factor - T % factor) % factor+ if pad > 0:+ mask = jnp.pad(mask, ((0, 0), (0, 0), (0, 0), (0, pad)))+ mask = mask.reshape(mask.shape[0], 1, 1, -1, factor).any(axis=-1)+ return mask+- def encode_speech(self, mel, src_mask=None, deterministic=True):++ def encode_speech(self, mel, src_mask=None, ffn_mask=None, deterministic=True):+ mel = self.spec_augment(mel, deterministic=deterministic)+ x = self.conv_subsample(mel) * self.embed_scale+ src_mask = self._downsample_mask(src_mask, 4)+ rope = self._rope(x.shape[1])- return self.encoder(x, mask=src_mask, rope=rope)+ return self.encoder(x, mask=src_mask, rope=rope, ffn_mask=ffn_mask)def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None):x = self.embedding(tgt) * self.embed_scale@@@ -358,71 -439,16 +419,71 @@@s = encoder_out.astype(jnp.float32)gram = jnp.matmul(s, s.transpose(0, 2, 1))diag_sq = jnp.sum(jnp.diagonal(gram, axis1=1, axis2=2) ** 2)- slot_div = (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]- return logits, slot_div, mrl_logits+ return (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]++ def forward_masked(self, src, tgt, src_mask=None, tgt_mask=None, ffn_mask=None):+ """Single forward with per-batch-item FFN masking. Returns (logits, slot_div)."""+ encoder_out = self.encode_text(src, src_mask=src_mask, ffn_mask=ffn_mask)+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, ffn_mask=ffn_mask)+ logits = x_f32 @ self.embedding.embedding.T+ slot_div = self._slot_diversity(encoder_out)+ return logits, slot_div+- def forward_speech_masked(self, mel, tgt, src_mask=None, tgt_mask=None, ffn_mask=None):++ def forward_speech_masked(self, mel, tgt, src_mask=None, tgt_mask=None, ffn_mask=None, deterministic=True):+ """Single speech forward with per-batch-item FFN masking. Returns (logits, slot_div)."""- encoder_out = self.encode_speech(mel, src_mask=src_mask, ffn_mask=ffn_mask)++ encoder_out = self.encode_speech(mel, src_mask=src_mask, ffn_mask=ffn_mask, deterministic=deterministic)+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, ffn_mask=ffn_mask)+ logits = x_f32 @ self.embedding.embedding.T+ slot_div = self._slot_diversity(encoder_out)+ return logits, slot_div++ def _make_eval_ffn_mask(self, ff_width, B, dtype):+ """Default prefix FFN mask for eval: first ff_width neurons active."""+ mask = (jnp.arange(self.config.d_ff) < ff_width).astype(dtype)+ return jnp.broadcast_to(mask[None, :], (B, self.config.d_ff))++ def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mat_ff_widths=None):+ """Eval-only: separate per-width forwards for reporting per-width PPL.++ mat_ff_widths: list of FFN widths to evaluate (e.g. [1024, 512, 256]).+ """+ emb = self.embedding.embedding+ B = src.shape[0]- def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mrl_dims=None):encoder_out = self.encode_text(src, src_mask=src_mask)- return self._decode_with_aux(encoder_out, tgt, tgt_mask=tgt_mask, mrl_dims=mrl_dims)+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask)+ logits = x_f32 @ emb.T+ slot_div = self._slot_diversity(encoder_out)++ mat_logits = []+ if mat_ff_widths is not None:+ for ff_w in mat_ff_widths:+ mask = self._make_eval_ffn_mask(ff_w, B, x_f32.dtype)+ enc_m = self.encode_text(src, src_mask=src_mask, ffn_mask=mask)+ x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, ffn_mask=mask)+ mat_logits.append(x_m @ emb.T)++ return logits, slot_div, mat_logits+- def forward_speech_with_aux(self, mel, tgt, src_mask=None, tgt_mask=None, mat_ff_widths=None):++ def forward_speech_with_aux(self, mel, tgt, src_mask=None, tgt_mask=None, mat_ff_widths=None, deterministic=True):+ """Eval-only: separate per-width speech forwards for reporting per-width PPL."""+ emb = self.embedding.embedding+ B = mel.shape[0]- encoder_out = self.encode_speech(mel, src_mask=src_mask)- def forward_speech_with_aux(self, mel, tgt, src_mask=None, tgt_mask=None, mrl_dims=None, deterministic=True):+ encoder_out = self.encode_speech(mel, src_mask=src_mask, deterministic=deterministic)- return self._decode_with_aux(encoder_out, tgt, tgt_mask=tgt_mask, mrl_dims=mrl_dims)+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask)+ logits = x_f32 @ emb.T+ slot_div = self._slot_diversity(encoder_out)++ mat_logits = []+ if mat_ff_widths is not None:+ for ff_w in mat_ff_widths:+ mask = self._make_eval_ffn_mask(ff_w, B, x_f32.dtype)- enc_m = self.encode_speech(mel, src_mask=src_mask, ffn_mask=mask)++ enc_m = self.encode_speech(mel, src_mask=src_mask, ffn_mask=mask, deterministic=deterministic)+ x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, ffn_mask=mask)+ mat_logits.append(x_m @ emb.T)++ return logits, slot_div, mat_logitsdef init_all(self, src, tgt, mel):"""Dummy forward through both text and speech pathways to initialize all params."""@@@ -430,9 -456,9 +491,9 @@@tgt_mask = make_causal_mask(tgt.shape[1]) & make_padding_mask(tgt, self.config.pad_token_id)text_out = self.encode_text(src, src_mask=src_mask)mel_mask = make_mel_padding_mask(mel)- speech_out = self.encode_speech(mel, src_mask=mel_mask)+ speech_out = self.encode_speech(mel, src_mask=mel_mask, deterministic=True)- _ = self._decode_with_aux(text_out, tgt, tgt_mask=tgt_mask)- _ = self._decode_with_aux(speech_out, tgt, tgt_mask=tgt_mask)+ _ = self._run_decoder(text_out, tgt, tgt_mask=tgt_mask)+ _ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask)return jnp.zeros(())diff --cc src/train.pyindex a4b1d79,59e0063..63ff961--- a/src/train.py+++ b/src/train.py@@@ -244,61 -235,64 +244,63 @@@ def _text_loss_fn(state, params, src, tce_loss = jnp.sum(optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask) / jnp.maximum(jnp.sum(mask), 1.0)-- total_loss = ce_loss- num_scales = 1- for mrl_log in mrl_logits:- mrl_log_f32 = mrl_log.astype(jnp.float32)- aux_loss = jnp.sum(- optax.softmax_cross_entropy_with_integer_labels(mrl_log_f32, tgt_out) * mask- ) / jnp.maximum(jnp.sum(mask), 1.0)- total_loss = total_loss + aux_loss- num_scales = num_scales + 1- total_loss = total_loss / num_scales-z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)div_loss = 1e-4 * slot_div- return total_loss + z_loss + div_loss---def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask):- pad_id = 0- src_mask = make_padding_mask(src, pad_id)- tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)-- logits, slot_div, mrl_logits = state.apply_fn(- {"params": _quantize_params(params, group_size=_GROUP_SIZE)},- src,- tgt_in,- src_mask=src_mask,- tgt_mask=tgt_mask,- cross_mask=src_mask,- mrl_dims=_MRL_DIMS if _MRL_DIMS else None,- method="forward_with_aux",- )- return _compute_loss(logits, slot_div, mrl_logits, tgt_out)+ return ce_loss + z_loss + div_loss- def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask, ffn_mask):-def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask, rng):++def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng):pad_id = 0src_mask = make_mel_padding_mask(mel)tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)-- logits, slot_div, mrl_logits = state.apply_fn(+ logits, slot_div = state.apply_fn({"params": _quantize_params(params, group_size=_GROUP_SIZE)},- mel,- tgt_in,- src_mask=src_mask,- tgt_mask=tgt_mask,- mrl_dims=_MRL_DIMS if _MRL_DIMS else None,+ mel, tgt_in, src_mask=src_mask, tgt_mask=tgt_mask,+ ffn_mask=ffn_mask,+ deterministic=False,- method="forward_speech_with_aux",+ method="forward_speech_masked",+ rngs={"specaugment": rng},)- return _compute_loss(logits, slot_div, mrl_logits, tgt_out)+ logits_f32 = logits.astype(jnp.float32)+ mask = (tgt_out != pad_id).astype(jnp.float32)+ ce_loss = jnp.sum(+ optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask+ ) / jnp.maximum(jnp.sum(mask), 1.0)+ z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)+ div_loss = 1e-4 * slot_div+ return ce_loss + z_loss + div_loss-def _train_step_text(state, ema_params, src, tgt_in, tgt_out, causal_mask):+def _make_ffn_mask(batch_size, d_ff, mat_ff_widths):+ """Build (batch, d_ff) prefix mask: batch split equally across widths.++ First quarter = full width (all ones), remaining quarters = mat widths.+ Each unique input is repeated once per width.+ """+ n_widths = 1 + len(mat_ff_widths) # full + mat widths+ per_width = batch_size // n_widths+ arange = jnp.arange(d_ff)+ rows = [jnp.ones((per_width, d_ff), dtype=jnp.bfloat16)] # full width+ for k in mat_ff_widths:+ rows.append((arange[None, :] < k).astype(jnp.bfloat16).repeat(per_width, axis=0))+ # Handle remainder (assign to full width)+ remainder = batch_size - per_width * n_widths+ if remainder > 0:+ rows.append(jnp.ones((remainder, d_ff), dtype=jnp.bfloat16))+ return jnp.concatenate(rows, axis=0)+++def _repeat_for_mat(arr, n_widths):+ """Repeat first batch/n_widths items n_widths times to fill batch."""+ per_width = arr.shape[0] // n_widths+ unique = arr[:per_width]+ return jnp.tile(unique, (n_widths,) + (1,) * (arr.ndim - 1))+++def _train_step_text(state, ema_params, src, tgt_in, tgt_out, causal_mask, ffn_mask):ema_decay = 0.999loss, grads = jax.value_and_grad(- lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask)+ lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask))(state.params)grads = jax.lax.pmean(grads, axis_name="batch")loss = jax.lax.pmean(loss, axis_name="batch")@@@ -324,10 -318,10 +326,10 @@@ def _train_step_text_masked(state, ema_return state, ema_params, loss, grad_norm- def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask, ffn_mask):-def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask, rng):++def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng):ema_decay = 0.999loss, grads = jax.value_and_grad(- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask)- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, rng)++ lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng))(state.params)grads = jax.lax.pmean(grads, axis_name="batch")loss = jax.lax.pmean(loss, axis_name="batch")@@@ -337,11 -331,11 +339,11 @@@return state, ema_params, loss, grad_norm- def _train_step_speech_masked(state, ema_params, mel, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask):-def _train_step_speech_masked(state, ema_params, mel, tgt_in, tgt_out, causal_mask, prune_mask, rng):- """Speech training step with fused mask application."""++def _train_step_speech_masked(state, ema_params, mel, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng):+ """Speech training step with fused prune mask application."""ema_decay = 0.999loss, grads = jax.value_and_grad(- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask)- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, rng)++ lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng))(state.params)grads = jax.lax.pmean(grads, axis_name="batch")loss = jax.lax.pmean(loss, axis_name="batch")@@@ -567,11 -569,14 +570,18 @@@ def train(args)rng = jax.random.PRNGKey(args.seed)rng, init_rng = jax.random.split(rng)- text_batches_per_epoch = len(enc_inputs) // effective_batch_size+ mat_shared_input = getattr(args, "mat_shared_input", False)+ # With shared input, each unique sample is repeated n_widths times,+ # so we fetch smaller batches but take more steps per epoch.+ unique_batch_size = effective_batch_size // n_widths if (mat_shared_input and n_widths > 1) else effective_batch_size+ text_batches_per_epoch = len(enc_inputs) // unique_batch_size+ if speech_every_override is not None:+ speech_every = speech_every_override+ elif not no_speech and speech_mel is not None:+ speech_batches_available = len(speech_mel) // effective_batch_size+ speech_every = max(1, text_batches_per_epoch // max(1, speech_batches_available))+ else:+ speech_every = 1speech_batches_per_epoch = text_batches_per_epoch // speech_every if (not no_speech and speech_mel is not None) else 0num_batches = text_batches_per_epoch + speech_batches_per_epochtotal_steps = num_batches * args.epochs@@@ -724,16 -701,16 +734,17 @@@sp_tgt_in_b = shard_batch(sp_tgt_in, num_devices)sp_tgt_out_b = shard_batch(sp_tgt_out, num_devices)- # Speech ffn_mask: same width structure, but mel has different seq dim- speech_ffn_mask = text_ffn_mask # same (devices, batch, d_ff) mask++ speech_ffn_mask = text_ffn_mask+ rng, spec_rng = jax.random.split(rng)+ spec_rngs = jax.random.split(spec_rng, num_devices)if prune_mask is not None:state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech_masked(- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, prune_mask, speech_ffn_mask,- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, prune_mask, spec_rngs,++ state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, prune_mask, speech_ffn_mask, spec_rngs,)else:state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech(- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, speech_ffn_mask,- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, spec_rngs,++ state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, speech_ffn_mask, spec_rngs,)speech_loss_val = float(sp_loss[0])speech_losses.append(speech_loss_val)