Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Bug fixes

HenryNdubuaku796a13ef6608
diff --git a/src/cli.py b/src/cli.py
index c9cfefc..8b2fbbe 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -54,7 +54,7 @@ HELP = """
│ │
│ tpu │
│ create NAME Create TPU (auto-finds zone) │
- │ --type STR Accelerator type (default: v5litepod-4) │
+ │ --type STR Accelerator (default: v5litepod-4) │
│ --version STR TPU OS (auto-detected from --type) │
│ connect NAME SSH config + connect (auto-zone) │
│ claude NAME Install Claude Code on instance │
@@ -135,7 +135,7 @@ def main():
p.add_argument("--dtype", type=str, default=None, choices=["float32", "bfloat16"])
p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
p.add_argument("--seed", type=int, default=None)
- p.add_argument("--eval-every", type=int, default=100)
+ p.add_argument("--eval-every", type=int, default=1000)
p.add_argument("--max-eval-samples", type=int, default=None)
p = sub.add_parser("run", add_help=False)
diff --git a/src/train.py b/src/train.py
index 6dee227..a91a5a4 100644
--- a/src/train.py
+++ b/src/train.py
@@ -282,6 +282,7 @@ def train(args):
dec_inputs = jnp.array(dec_inputs)
dec_targets = jnp.array(dec_targets)
+ last_val_ppl = None
for epoch in range(args.epochs):
losses = []
batches = get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size)
@@ -317,10 +318,8 @@ def train(args):
vl, vt = val_loss_fn(eval_params, vb[0], vb[1], vb[2], val_causal)
total_loss += float(vl)
total_toks += float(vt)
- val_ppl = float(math.exp(total_loss / max(total_toks, 1)))
- pbar.set_postfix(loss=f"{loss_val:.4f}", val_ppl=f"{val_ppl:.2f}")
- else:
- pbar.set_postfix(loss=f"{loss_val:.4f}")
+ last_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))
+ pbar.set_postfix(loss=f"{loss_val:.4f}", val_ppl=f"{last_val_ppl:.2f}" if last_val_ppl is not None else "?")
if use_wandb:
log_dict = {
@@ -332,7 +331,7 @@ def train(args):
"train/step": global_step,
}
if global_step % eval_every == 0 or global_step == total_steps:
- log_dict["val/ppl"] = val_ppl
+ log_dict["val/ppl"] = last_val_ppl
wandb.log(log_dict)
epoch_avg_loss = sum(losses) / len(losses) if losses else float("nan")