Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuakubd8982d368be
diff --git a/debug_nan.py b/debug_nan.py
new file mode 100644
index 0000000..f3338b8
--- /dev/null
+++ b/debug_nan.py
@@ -0,0 +1,104 @@
+"""Debug why train.py produces NaN from step 0. Run on a TPU worker."""
+import pickle
+import jax
+import jax.numpy as jnp
+import numpy as np
+import optax
+
+from src.model import (
+ EncoderDecoderTransformer, TransformerConfig,
+ make_packing_mask, make_causal_packing_mask, make_cross_packing_mask,
+)
+from src.data import load_prepared_data
+
+print("=" * 60)
+print("Loading checkpoint...")
+with open("checkpoints/needle_base.pkl", "rb") as f:
+ ckpt = pickle.load(f)
+config = TransformerConfig(**ckpt["config"])
+params = jax.tree.map(jnp.array, ckpt["params"])
+print(f" config: d={config.d_model} heads={config.num_heads} layers={config.num_encoder_layers}/{config.num_decoder_layers}")
+print(f" dtype={config.dtype} no_ff={config.no_feedforward}")
+
+print("Loading data...")
+data = load_prepared_data("train", mmap=True)
+bs = 4
+src = jnp.asarray(data["packed_enc"][:bs], dtype=jnp.int32)
+tgt_in = jnp.asarray(data["packed_dec_in"][:bs], dtype=jnp.int32)
+tgt_out = jnp.asarray(data["packed_dec_tgt"][:bs], dtype=jnp.int32)
+enc_seg = jnp.asarray(data["packed_enc_seg"][:bs], dtype=jnp.int32)
+dec_seg = jnp.asarray(data["packed_dec_seg"][:bs], dtype=jnp.int32)
+lm = jnp.asarray(data["packed_loss"][:bs], dtype=jnp.int32)
+
+print(f" src: shape={src.shape} range=[{int(src.min())}, {int(src.max())}]")
+print(f" tgt_in: shape={tgt_in.shape} range=[{int(tgt_in.min())}, {int(tgt_in.max())}]")
+print(f" tgt_out: shape={tgt_out.shape} range=[{int(tgt_out.min())}, {int(tgt_out.max())}]")
+print(f" enc_seg: range=[{int(enc_seg.min())}, {int(enc_seg.max())}] nnz_rows={int((enc_seg.sum(axis=-1) > 0).sum())}/{bs}")
+print(f" dec_seg: range=[{int(dec_seg.min())}, {int(dec_seg.max())}] nnz_rows={int((dec_seg.sum(axis=-1) > 0).sum())}/{bs}")
+print(f" lm: range=[{int(lm.min())}, {int(lm.max())}]")
+
+print("=" * 60)
+print("Checking params for NaN/Inf...")
+any_bad = False
+for path, leaf in jax.tree_util.tree_leaves_with_path(params):
+ n_bad = int(jnp.sum(~jnp.isfinite(leaf)))
+ if n_bad:
+ any_bad = True
+ print(f" BAD {path}: {n_bad} nonfinite")
+if not any_bad:
+ print(" all params finite")
+
+print("=" * 60)
+print("Building masks...")
+src_mask = make_packing_mask(enc_seg)
+tgt_mask = make_causal_packing_mask(dec_seg)
+cross_mask = make_cross_packing_mask(enc_seg, dec_seg)
+print(f" src_mask: {src_mask.shape} true_frac={float(src_mask.mean()):.3f}")
+print(f" tgt_mask: {tgt_mask.shape} true_frac={float(tgt_mask.mean()):.3f}")
+print(f" cross_mask: {cross_mask.shape} true_frac={float(cross_mask.mean()):.3f}")
+
+# Check if any attention row is all-False
+def all_false_rows(m):
+ # m: (B, 1, T_q, T_k)
+ return int((~m.any(axis=-1)).sum())
+print(f" src_mask all-False rows: {all_false_rows(src_mask)}")
+print(f" tgt_mask all-False rows: {all_false_rows(tgt_mask)}")
+print(f" cross_mask all-False rows: {all_false_rows(cross_mask)}")
+
+print("=" * 60)
+print("Running __call__ forward (no dropout)...")
+model = EncoderDecoderTransformer(config)
+logits = model.apply({"params": params}, src, tgt_in,
+ src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask)
+print(f" logits: shape={logits.shape} dtype={logits.dtype}")
+print(f" NaN={int(jnp.sum(jnp.isnan(logits)))} Inf={int(jnp.sum(jnp.isinf(logits)))}")
+print(f" min={float(jnp.min(logits)):+.3g} max={float(jnp.max(logits)):+.3g}")
+
+logits_f32 = logits.astype(jnp.float32)
+ce = optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out)
+print(f" CE: NaN={int(jnp.sum(jnp.isnan(ce)))} min={float(jnp.min(ce)):+.3g} max={float(jnp.max(ce)):+.3g}")
+padding = (dec_seg > 0).astype(jnp.float32)
+num_tok = jnp.sum(padding)
+ce_loss = jnp.sum(ce * padding) / jnp.maximum(num_tok, 1.0)
+print(f" masked CE loss: {float(ce_loss):.4f} (num_tok={float(num_tok):.0f})")
+
+# Check z_loss
+lse = jax.nn.logsumexp(logits_f32, axis=-1)
+print(f" logsumexp: NaN={int(jnp.sum(jnp.isnan(lse)))} min={float(jnp.min(lse)):+.3g} max={float(jnp.max(lse)):+.3g}")
+z_loss = 1e-4 * jnp.mean(lse ** 2)
+print(f" z_loss: {float(z_loss):.4f}")
+
+print("=" * 60)
+print("Running forward_masked (with dropout)...")
+ffn_mask = jnp.ones((bs, config.d_ff), dtype=config.jax_dtype)
+logits2, _ = model.apply(
+ {"params": params}, src, tgt_in,
+ src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
+ ffn_mask=ffn_mask, deterministic=False,
+ method="forward_masked", rngs={"dropout": jax.random.PRNGKey(0)},
+)
+print(f" logits: NaN={int(jnp.sum(jnp.isnan(logits2)))} Inf={int(jnp.sum(jnp.isinf(logits2)))}")
+print(f" min={float(jnp.min(logits2)):+.3g} max={float(jnp.max(logits2)):+.3g}")
+
+ce2 = optax.softmax_cross_entropy_with_integer_labels(logits2.astype(jnp.float32), tgt_out)
+print(f" CE: NaN={int(jnp.sum(jnp.isnan(ce2)))} min={float(jnp.min(ce2)):+.3g} max={float(jnp.max(ce2)):+.3g}")
diff --git a/src/model.py b/src/model.py
index 94e02c6..2e926c2 100644
--- a/src/model.py
+++ b/src/model.py
@@ -499,7 +499,11 @@ class EncoderDecoderTransformer(nn.Module):
pooled = self._mean_pool(encoder_out, enc_mask)
h = nn.relu(self.contrastive_hidden(pooled))
projected = self.contrastive_proj(h)
- return projected / jnp.maximum(jnp.linalg.norm(projected, axis=-1, keepdims=True), 1e-8)
+ # Safe L2 normalize: sqrt(sum² + eps²) has smooth gradient at origin,
+ # whereas max(norm, eps) NaNs in the backward pass when projected == 0
+ # (which happens after pretrain decays contrastive weights to ~0).
+ denom = jnp.sqrt(jnp.sum(projected.astype(jnp.float32) ** 2, axis=-1, keepdims=True) + 1e-12)
+ return (projected / denom.astype(projected.dtype))
def forward_contrastive(self, query_tokens, tool_tokens, deterministic=True):
"""Encode query and tool tokens, return (q_emb, t_emb, log_temp)."""
diff --git a/src/train.py b/src/train.py
index e9cc7f5..a6bfe4e 100644
--- a/src/train.py
+++ b/src/train.py
@@ -280,11 +280,23 @@ def train(args):
resume_checkpoint = getattr(args, "checkpoint", None)
if resume_checkpoint:
+ if not os.path.exists(resume_checkpoint):
+ print(f" Checkpoint not found locally, downloading from HF...", flush=True)
+ from huggingface_hub import hf_hub_download
+ local_dir = os.path.dirname(resume_checkpoint) or "checkpoints"
+ os.makedirs(local_dir, exist_ok=True)
+ resume_checkpoint = hf_hub_download(
+ repo_id="Cactus-Compute/checkpoints",
+ filename=os.path.basename(resume_checkpoint),
+ repo_type="model",
+ local_dir=local_dir,
+ )
+ print(f" Downloaded to {resume_checkpoint}", flush=True)
print(f"Resuming from checkpoint: {resume_checkpoint}")
with open(resume_checkpoint, "rb") as f:
ckpt_data = pickle.load(f)
- ckpt_params = jax.tree.map(jnp.array, ckpt_data["params"])
config = TransformerConfig(**ckpt_data["config"])
+ ckpt_params = jax.tree.map(jnp.array, ckpt_data["params"])
print(f" Config: d={config.d_model}, heads={config.num_heads}, layers={config.num_encoder_layers}/{config.num_decoder_layers}")
else:
config = TransformerConfig(
@@ -343,6 +355,12 @@ def train(args):
val_loss_fn = _make_val_loss_fn(state.apply_fn)
if resume_checkpoint:
+ # Cast each loaded leaf to the dtype of the freshly-initialized param
+ # (most weights -> bf16, scalars like log_temp stay fp32)
+ ckpt_params = jax.tree.map(
+ lambda ref, loaded: jnp.asarray(loaded, dtype=ref.dtype),
+ state.params, ckpt_params,
+ )
state = state.replace(params=ckpt_params)
print(f" Loaded checkpoint params into train state")
@@ -466,9 +484,9 @@ def train(args):
rng, text_rng = jax.random.split(rng)
text_rngs = jax.random.split(text_rng, num_devices)
- do_qat = jnp.broadcast_to(jnp.array(global_step % 100 == 0), (num_devices,))
+ do_qat = jnp.broadcast_to(jnp.array(global_step > 0 and global_step % 100 == 0), (num_devices,))
- do_contrastive = cl_batch_iter is not None and global_step % 1000 == 0
+ do_contrastive = cl_batch_iter is not None and global_step > 0 and global_step % 1000 == 0
cl_q_b, cl_t_b, cl_rngs = dummy_cl_tokens, dummy_cl_tokens, dummy_cl_rng
if do_contrastive:
try: