Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Merge branch 'main' into fixed-mrl

Noah Cylich7d8f27d4248e
diff --cc src/model.py
index 74bf602,d29f084..7abbd21
--- a/src/model.py
+++ b/src/model.py
@@@ -326,10 -385,25 +372,25 @@@ class EncoderDecoderTransformer(nn.Modu
"""Backward-compatible alias for encode_text."""
return self.encode_text(src, src_mask=src_mask)
- def encode_speech(self, mel, src_mask=None, ffn_mask=None):
- x = self.mel_proj(mel) * self.embed_scale
- rope = self._rope(mel.shape[1])
+ @staticmethod
+ def _downsample_mask(mask, factor):
+ """Downsample a padding mask by OR-pooling groups of `factor` frames."""
+ if mask is None:
+ return None
+ # mask: (B, 1, 1, T)
+ T = mask.shape[-1]
+ pad = (factor - T % factor) % factor
+ if pad > 0:
+ mask = jnp.pad(mask, ((0, 0), (0, 0), (0, 0), (0, pad)))
+ mask = mask.reshape(mask.shape[0], 1, 1, -1, factor).any(axis=-1)
+ return mask
+
- def encode_speech(self, mel, src_mask=None, deterministic=True):
++ def encode_speech(self, mel, src_mask=None, ffn_mask=None, deterministic=True):
+ mel = self.spec_augment(mel, deterministic=deterministic)
+ x = self.conv_subsample(mel) * self.embed_scale
+ src_mask = self._downsample_mask(src_mask, 4)
+ rope = self._rope(x.shape[1])
- return self.encoder(x, mask=src_mask, rope=rope)
+ return self.encoder(x, mask=src_mask, rope=rope, ffn_mask=ffn_mask)
def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None):
x = self.embedding(tgt) * self.embed_scale
@@@ -358,71 -439,16 +419,71 @@@
s = encoder_out.astype(jnp.float32)
gram = jnp.matmul(s, s.transpose(0, 2, 1))
diag_sq = jnp.sum(jnp.diagonal(gram, axis1=1, axis2=2) ** 2)
- slot_div = (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]
- return logits, slot_div, mrl_logits
+ return (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]
+
+ def forward_masked(self, src, tgt, src_mask=None, tgt_mask=None, ffn_mask=None):
+ """Single forward with per-batch-item FFN masking. Returns (logits, slot_div)."""
+ encoder_out = self.encode_text(src, src_mask=src_mask, ffn_mask=ffn_mask)
+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, ffn_mask=ffn_mask)
+ logits = x_f32 @ self.embedding.embedding.T
+ slot_div = self._slot_diversity(encoder_out)
+ return logits, slot_div
+
- def forward_speech_masked(self, mel, tgt, src_mask=None, tgt_mask=None, ffn_mask=None):
++ def forward_speech_masked(self, mel, tgt, src_mask=None, tgt_mask=None, ffn_mask=None, deterministic=True):
+ """Single speech forward with per-batch-item FFN masking. Returns (logits, slot_div)."""
- encoder_out = self.encode_speech(mel, src_mask=src_mask, ffn_mask=ffn_mask)
++ encoder_out = self.encode_speech(mel, src_mask=src_mask, ffn_mask=ffn_mask, deterministic=deterministic)
+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, ffn_mask=ffn_mask)
+ logits = x_f32 @ self.embedding.embedding.T
+ slot_div = self._slot_diversity(encoder_out)
+ return logits, slot_div
+
+ def _make_eval_ffn_mask(self, ff_width, B, dtype):
+ """Default prefix FFN mask for eval: first ff_width neurons active."""
+ mask = (jnp.arange(self.config.d_ff) < ff_width).astype(dtype)
+ return jnp.broadcast_to(mask[None, :], (B, self.config.d_ff))
+
+ def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mat_ff_widths=None):
+ """Eval-only: separate per-width forwards for reporting per-width PPL.
+
+ mat_ff_widths: list of FFN widths to evaluate (e.g. [1024, 512, 256]).
+ """
+ emb = self.embedding.embedding
+ B = src.shape[0]
- def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mrl_dims=None):
encoder_out = self.encode_text(src, src_mask=src_mask)
- return self._decode_with_aux(encoder_out, tgt, tgt_mask=tgt_mask, mrl_dims=mrl_dims)
+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask)
+ logits = x_f32 @ emb.T
+ slot_div = self._slot_diversity(encoder_out)
+
+ mat_logits = []
+ if mat_ff_widths is not None:
+ for ff_w in mat_ff_widths:
+ mask = self._make_eval_ffn_mask(ff_w, B, x_f32.dtype)
+ enc_m = self.encode_text(src, src_mask=src_mask, ffn_mask=mask)
+ x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, ffn_mask=mask)
+ mat_logits.append(x_m @ emb.T)
+
+ return logits, slot_div, mat_logits
+
- def forward_speech_with_aux(self, mel, tgt, src_mask=None, tgt_mask=None, mat_ff_widths=None):
++ def forward_speech_with_aux(self, mel, tgt, src_mask=None, tgt_mask=None, mat_ff_widths=None, deterministic=True):
+ """Eval-only: separate per-width speech forwards for reporting per-width PPL."""
+ emb = self.embedding.embedding
+ B = mel.shape[0]
- encoder_out = self.encode_speech(mel, src_mask=src_mask)
- def forward_speech_with_aux(self, mel, tgt, src_mask=None, tgt_mask=None, mrl_dims=None, deterministic=True):
+ encoder_out = self.encode_speech(mel, src_mask=src_mask, deterministic=deterministic)
- return self._decode_with_aux(encoder_out, tgt, tgt_mask=tgt_mask, mrl_dims=mrl_dims)
+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask)
+ logits = x_f32 @ emb.T
+ slot_div = self._slot_diversity(encoder_out)
+
+ mat_logits = []
+ if mat_ff_widths is not None:
+ for ff_w in mat_ff_widths:
+ mask = self._make_eval_ffn_mask(ff_w, B, x_f32.dtype)
- enc_m = self.encode_speech(mel, src_mask=src_mask, ffn_mask=mask)
++ enc_m = self.encode_speech(mel, src_mask=src_mask, ffn_mask=mask, deterministic=deterministic)
+ x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, ffn_mask=mask)
+ mat_logits.append(x_m @ emb.T)
+
+ return logits, slot_div, mat_logits
def init_all(self, src, tgt, mel):
"""Dummy forward through both text and speech pathways to initialize all params."""
@@@ -430,9 -456,9 +491,9 @@@
tgt_mask = make_causal_mask(tgt.shape[1]) & make_padding_mask(tgt, self.config.pad_token_id)
text_out = self.encode_text(src, src_mask=src_mask)
mel_mask = make_mel_padding_mask(mel)
- speech_out = self.encode_speech(mel, src_mask=mel_mask)
+ speech_out = self.encode_speech(mel, src_mask=mel_mask, deterministic=True)
- _ = self._decode_with_aux(text_out, tgt, tgt_mask=tgt_mask)
- _ = self._decode_with_aux(speech_out, tgt, tgt_mask=tgt_mask)
+ _ = self._run_decoder(text_out, tgt, tgt_mask=tgt_mask)
+ _ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask)
return jnp.zeros(())
diff --cc src/train.py
index a4b1d79,59e0063..63ff961
--- a/src/train.py
+++ b/src/train.py
@@@ -244,61 -235,64 +244,63 @@@ def _text_loss_fn(state, params, src, t
ce_loss = jnp.sum(
optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask
) / jnp.maximum(jnp.sum(mask), 1.0)
-
- total_loss = ce_loss
- num_scales = 1
- for mrl_log in mrl_logits:
- mrl_log_f32 = mrl_log.astype(jnp.float32)
- aux_loss = jnp.sum(
- optax.softmax_cross_entropy_with_integer_labels(mrl_log_f32, tgt_out) * mask
- ) / jnp.maximum(jnp.sum(mask), 1.0)
- total_loss = total_loss + aux_loss
- num_scales = num_scales + 1
- total_loss = total_loss / num_scales
-
z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)
div_loss = 1e-4 * slot_div
- return total_loss + z_loss + div_loss
-
-
-def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask):
- pad_id = 0
- src_mask = make_padding_mask(src, pad_id)
- tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
-
- logits, slot_div, mrl_logits = state.apply_fn(
- {"params": _quantize_params(params, group_size=_GROUP_SIZE)},
- src,
- tgt_in,
- src_mask=src_mask,
- tgt_mask=tgt_mask,
- cross_mask=src_mask,
- mrl_dims=_MRL_DIMS if _MRL_DIMS else None,
- method="forward_with_aux",
- )
- return _compute_loss(logits, slot_div, mrl_logits, tgt_out)
+ return ce_loss + z_loss + div_loss
- def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask, ffn_mask):
-def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask, rng):
++def _speech_loss_fn(state, params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng):
pad_id = 0
src_mask = make_mel_padding_mask(mel)
tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
-
- logits, slot_div, mrl_logits = state.apply_fn(
+ logits, slot_div = state.apply_fn(
{"params": _quantize_params(params, group_size=_GROUP_SIZE)},
- mel,
- tgt_in,
- src_mask=src_mask,
- tgt_mask=tgt_mask,
- mrl_dims=_MRL_DIMS if _MRL_DIMS else None,
+ mel, tgt_in, src_mask=src_mask, tgt_mask=tgt_mask,
+ ffn_mask=ffn_mask,
+ deterministic=False,
- method="forward_speech_with_aux",
+ method="forward_speech_masked",
+ rngs={"specaugment": rng},
)
- return _compute_loss(logits, slot_div, mrl_logits, tgt_out)
+ logits_f32 = logits.astype(jnp.float32)
+ mask = (tgt_out != pad_id).astype(jnp.float32)
+ ce_loss = jnp.sum(
+ optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask
+ ) / jnp.maximum(jnp.sum(mask), 1.0)
+ z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)
+ div_loss = 1e-4 * slot_div
+ return ce_loss + z_loss + div_loss
-def _train_step_text(state, ema_params, src, tgt_in, tgt_out, causal_mask):
+def _make_ffn_mask(batch_size, d_ff, mat_ff_widths):
+ """Build (batch, d_ff) prefix mask: batch split equally across widths.
+
+ First quarter = full width (all ones), remaining quarters = mat widths.
+ Each unique input is repeated once per width.
+ """
+ n_widths = 1 + len(mat_ff_widths) # full + mat widths
+ per_width = batch_size // n_widths
+ arange = jnp.arange(d_ff)
+ rows = [jnp.ones((per_width, d_ff), dtype=jnp.bfloat16)] # full width
+ for k in mat_ff_widths:
+ rows.append((arange[None, :] < k).astype(jnp.bfloat16).repeat(per_width, axis=0))
+ # Handle remainder (assign to full width)
+ remainder = batch_size - per_width * n_widths
+ if remainder > 0:
+ rows.append(jnp.ones((remainder, d_ff), dtype=jnp.bfloat16))
+ return jnp.concatenate(rows, axis=0)
+
+
+def _repeat_for_mat(arr, n_widths):
+ """Repeat first batch/n_widths items n_widths times to fill batch."""
+ per_width = arr.shape[0] // n_widths
+ unique = arr[:per_width]
+ return jnp.tile(unique, (n_widths,) + (1,) * (arr.ndim - 1))
+
+
+def _train_step_text(state, ema_params, src, tgt_in, tgt_out, causal_mask, ffn_mask):
ema_decay = 0.999
loss, grads = jax.value_and_grad(
- lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask)
+ lambda p: _text_loss_fn(state, p, src, tgt_in, tgt_out, causal_mask, ffn_mask)
)(state.params)
grads = jax.lax.pmean(grads, axis_name="batch")
loss = jax.lax.pmean(loss, axis_name="batch")
@@@ -324,10 -318,10 +326,10 @@@ def _train_step_text_masked(state, ema_
return state, ema_params, loss, grad_norm
- def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask, ffn_mask):
-def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask, rng):
++def _train_step_speech(state, ema_params, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng):
ema_decay = 0.999
loss, grads = jax.value_and_grad(
- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask)
- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, rng)
++ lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng)
)(state.params)
grads = jax.lax.pmean(grads, axis_name="batch")
loss = jax.lax.pmean(loss, axis_name="batch")
@@@ -337,11 -331,11 +339,11 @@@
return state, ema_params, loss, grad_norm
- def _train_step_speech_masked(state, ema_params, mel, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask):
-def _train_step_speech_masked(state, ema_params, mel, tgt_in, tgt_out, causal_mask, prune_mask, rng):
- """Speech training step with fused mask application."""
++def _train_step_speech_masked(state, ema_params, mel, tgt_in, tgt_out, causal_mask, prune_mask, ffn_mask, rng):
+ """Speech training step with fused prune mask application."""
ema_decay = 0.999
loss, grads = jax.value_and_grad(
- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask)
- lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, rng)
++ lambda p: _speech_loss_fn(state, p, mel, tgt_in, tgt_out, causal_mask, ffn_mask, rng)
)(state.params)
grads = jax.lax.pmean(grads, axis_name="batch")
loss = jax.lax.pmean(loss, axis_name="batch")
@@@ -567,11 -569,14 +570,18 @@@ def train(args)
rng = jax.random.PRNGKey(args.seed)
rng, init_rng = jax.random.split(rng)
- text_batches_per_epoch = len(enc_inputs) // effective_batch_size
+ mat_shared_input = getattr(args, "mat_shared_input", False)
+ # With shared input, each unique sample is repeated n_widths times,
+ # so we fetch smaller batches but take more steps per epoch.
+ unique_batch_size = effective_batch_size // n_widths if (mat_shared_input and n_widths > 1) else effective_batch_size
+ text_batches_per_epoch = len(enc_inputs) // unique_batch_size
+ if speech_every_override is not None:
+ speech_every = speech_every_override
+ elif not no_speech and speech_mel is not None:
+ speech_batches_available = len(speech_mel) // effective_batch_size
+ speech_every = max(1, text_batches_per_epoch // max(1, speech_batches_available))
+ else:
+ speech_every = 1
speech_batches_per_epoch = text_batches_per_epoch // speech_every if (not no_speech and speech_mel is not None) else 0
num_batches = text_batches_per_epoch + speech_batches_per_epoch
total_steps = num_batches * args.epochs
@@@ -724,16 -701,16 +734,17 @@@
sp_tgt_in_b = shard_batch(sp_tgt_in, num_devices)
sp_tgt_out_b = shard_batch(sp_tgt_out, num_devices)
- # Speech ffn_mask: same width structure, but mel has different seq dim
- speech_ffn_mask = text_ffn_mask # same (devices, batch, d_ff) mask
++ speech_ffn_mask = text_ffn_mask
+ rng, spec_rng = jax.random.split(rng)
+ spec_rngs = jax.random.split(spec_rng, num_devices)
if prune_mask is not None:
state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech_masked(
- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, prune_mask, speech_ffn_mask,
- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, prune_mask, spec_rngs,
++ state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, prune_mask, speech_ffn_mask, spec_rngs,
)
else:
state, ema_params, sp_loss, sp_grad_norm = p_train_step_speech(
- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, speech_ffn_mask,
- state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, spec_rngs,
++ state, ema_params, mel_b, sp_tgt_in_b, sp_tgt_out_b, causal_mask, speech_ffn_mask, spec_rngs,
)
speech_loss_val = float(sp_loss[0])
speech_losses.append(speech_loss_val)