Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuakue13d985268af
diff --git a/src/cli.py b/src/cli.py
index d108120..10459e3 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -16,7 +16,7 @@ def main():
p = sub.add_parser("train", add_help=False)
p.add_argument("--checkpoint", type=str, default=None)
p.add_argument("--epochs", type=int, default=1)
- p.add_argument("--batch-size", type=int, default=32)
+ p.add_argument("--batch-size", type=int, default=64)
p.add_argument("--lr", type=float, default=3e-4)
p.add_argument("--muon-lr", type=float, default=0.02)
p.add_argument("--d-model", type=int, default=512)
diff --git a/src/train.py b/src/train.py
index 61e97e5..ee1ff3a 100644
--- a/src/train.py
+++ b/src/train.py
@@ -316,9 +316,10 @@ def _text_loss_fn(state, params, src, tgt_in, tgt_out, ffn_mask, rng, loss_mask,
)
logits_f32 = logits.astype(jnp.float32)
mask = loss_mask
+ num_tokens = jnp.maximum(jnp.sum(mask > 0), 1.0)
ce_loss = jnp.sum(
optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask
- ) / jnp.maximum(jnp.sum(mask), 1.0)
+ ) / num_tokens
z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)
return ce_loss + z_loss