Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
bugfix
HenryNdubuakudb0a95db418a
diff --git a/src/cli.py b/src/cli.pyindex 1f61186..66ec530 100644--- a/src/cli.py+++ b/src/cli.py@@ -19,7 +19,7 @@ def main():p.add_argument("--batch-size", type=int, default=32)p.add_argument("--lr", type=float, default=3e-4)p.add_argument("--muon-lr", type=float, default=0.02)- p.add_argument("--d-model", type=int, default=512)+ p.add_argument("--d-model", type=int, default=1024)p.add_argument("--num-heads", type=int, default=4)p.add_argument("--num-kv-heads", type=int, default=4)p.add_argument("--num-layers", type=int, default=8)diff --git a/src/model.py b/src/model.pyindex 522ff9b..df3f547 100644--- a/src/model.py+++ b/src/model.py@@ -538,7 +538,7 @@ class EncoderDecoderTransformer(nn.Module):mask = (jnp.arange(self.config.d_ff) < ff_width).astype(dtype)return jnp.broadcast_to(mask[None, :], (B, self.config.d_ff))- def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, mat_ff_widths=None):+ def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mat_ff_widths=None):"""Eval-only: separate per-width forwards for reporting per-width PPL.mat_ff_widths: list of FFN widths to evaluate (e.g. [1024, 512, 256]).@@ -547,7 +547,8 @@ class EncoderDecoderTransformer(nn.Module):B = src.shape[0]encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)- x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, cross_mask=enc_mask)+ cm = cross_mask if cross_mask is not None else enc_mask+ x_f32 = self._run_decoder(encoder_out, tgt, tgt_mask=tgt_mask, cross_mask=cm)logits = x_f32 @ emb.Tmat_logits = []@@ -555,7 +556,7 @@ class EncoderDecoderTransformer(nn.Module):for ff_w in mat_ff_widths:mask = self._make_eval_ffn_mask(ff_w, B, x_f32.dtype)enc_m, enc_m_mask = self.encode_text(src, src_mask=src_mask, ffn_mask=mask)- x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, cross_mask=enc_m_mask, ffn_mask=mask)+ x_m = self._run_decoder(enc_m, tgt, tgt_mask=tgt_mask, cross_mask=cm, ffn_mask=mask)mat_logits.append(x_m @ emb.T)return logits, 0.0, mat_logits