Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
FIx default precision
HenryNdubuakua03639f7232d
diff --git a/src/cli.py b/src/cli.pyindex 789c9c2..2fc5116 100644--- a/src/cli.py+++ b/src/cli.py@@ -80,7 +80,7 @@ TOY_CONFIG = {"max_dec_len": 128,"max_samples": 10000,"warmup_ratio": 0.05,- "dtype": "float32",+ "dtype": "bfloat16","seed": 42,}@@ -97,7 +97,7 @@ BASE_CONFIG = {"max_dec_len": 256,"max_samples": 500000,"warmup_ratio": 0.05,- "dtype": "float32",+ "dtype": "bfloat16","seed": 42,}diff --git a/src/train.py b/src/train.pyindex ab2f0cf..1029914 100644--- a/src/train.py+++ b/src/train.py@@ -149,9 +149,12 @@ def _train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):rngs={"dropout": dropout_rng},)- loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)+ logits_f32 = logits.astype(jnp.float32)+ loss = optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out)mask = (tgt_out != pad_id).astype(jnp.float32)- return jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)+ ce_loss = jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)+ z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)+ return ce_loss + z_lossloss, grads = jax.value_and_grad(loss_fn)(state.params)# Average gradients and loss across devices