Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

FIx default precision

HenryNdubuakua03639f7232d
diff --git a/src/cli.py b/src/cli.py
index 789c9c2..2fc5116 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -80,7 +80,7 @@ TOY_CONFIG = {
"max_dec_len": 128,
"max_samples": 10000,
"warmup_ratio": 0.05,
- "dtype": "float32",
+ "dtype": "bfloat16",
"seed": 42,
}
@@ -97,7 +97,7 @@ BASE_CONFIG = {
"max_dec_len": 256,
"max_samples": 500000,
"warmup_ratio": 0.05,
- "dtype": "float32",
+ "dtype": "bfloat16",
"seed": 42,
}
diff --git a/src/train.py b/src/train.py
index ab2f0cf..1029914 100644
--- a/src/train.py
+++ b/src/train.py
@@ -149,9 +149,12 @@ def _train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):
rngs={"dropout": dropout_rng},
)
- loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)
+ logits_f32 = logits.astype(jnp.float32)
+ loss = optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out)
mask = (tgt_out != pad_id).astype(jnp.float32)
- return jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)
+ ce_loss = jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)
+ z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)
+ return ce_loss + z_loss
loss, grads = jax.value_and_grad(loss_fn)(state.params)
# Average gradients and loss across devices