Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuakue13d985268af
diff --git a/src/cli.py b/src/cli.pyindex d108120..10459e3 100644--- a/src/cli.py+++ b/src/cli.py@@ -16,7 +16,7 @@ def main():p = sub.add_parser("train", add_help=False)p.add_argument("--checkpoint", type=str, default=None)p.add_argument("--epochs", type=int, default=1)- p.add_argument("--batch-size", type=int, default=32)+ p.add_argument("--batch-size", type=int, default=64)p.add_argument("--lr", type=float, default=3e-4)p.add_argument("--muon-lr", type=float, default=0.02)p.add_argument("--d-model", type=int, default=512)diff --git a/src/train.py b/src/train.pyindex 61e97e5..ee1ff3a 100644--- a/src/train.py+++ b/src/train.py@@ -316,9 +316,10 @@ def _text_loss_fn(state, params, src, tgt_in, tgt_out, ffn_mask, rng, loss_mask,)logits_f32 = logits.astype(jnp.float32)mask = loss_mask+ num_tokens = jnp.maximum(jnp.sum(mask > 0), 1.0)ce_loss = jnp.sum(optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask- ) / jnp.maximum(jnp.sum(mask), 1.0)+ ) / num_tokensz_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)return ce_loss + z_loss