Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Training restart

HenryNdubuakuc6fd91f9610a
diff --git a/src/cli.py b/src/cli.py
index 52a8359..b3c1f3b 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -116,6 +116,7 @@ def main():
p = sub.add_parser("train", add_help=False)
p.add_argument("--toy", action="store_true")
+ p.add_argument("--checkpoint", type=str, default=None)
p.add_argument("--epochs", type=int, default=None)
p.add_argument("--batch-size", type=int, default=None)
p.add_argument("--lr", type=float, default=None)
@@ -133,12 +134,11 @@ def main():
p.add_argument("--seed", type=int, default=None)
p.add_argument("--eval-every", type=int, default=1000)
p.add_argument("--max-eval-samples", type=int, default=None)
- p.add_argument("--sparsity-ratio", type=float, default=0.33)
- p.add_argument("--layer-prune-ratio", type=float, default=0.33)
+ p.add_argument("--sparsity-ratio", type=float, default=0.0)
+ p.add_argument("--layer-prune-ratio", type=float, default=0.0)
p.add_argument("--group-size", type=int, default=32)
p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])
p.add_argument("--num-memory-slots", type=int, default=64)
- p.add_argument("--ortho-weight", type=float, default=1e-4)
p = sub.add_parser("run", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
diff --git a/src/model.py b/src/model.py
index 4f8d44c..da70255 100644
--- a/src/model.py
+++ b/src/model.py
@@ -138,100 +138,48 @@ class FeedForward(nn.Module):
return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)
-class MLPMixer(nn.Module):
- """MLP-Mixer operating on fixed-size memory slots (B, M, d)."""
- num_slots: int
- d_model: int
- d_ff: int
- dtype: jnp.dtype = jnp.bfloat16
- activation: str = "drelu"
-
- @nn.compact
- def __call__(self, s):
- residual = s
- s = ZCRMSNorm(dtype=self.dtype, name="token_mix_norm")(s)
- s = s.transpose(0, 2, 1)
- gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_gate")(s)
- up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_up")(s)
- if self.activation == "swiglu":
- s = nn.silu(gate) * up
- elif self.activation == "geglu":
- s = nn.gelu(gate) * up
- else:
- s = nn.relu(gate) * nn.relu(up)
- s = nn.Dense(self.num_slots, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_down")(s)
- s = s.transpose(0, 2, 1)
- s = s + residual
-
- residual = s
- s = ZCRMSNorm(dtype=self.dtype, name="channel_mix_norm")(s)
- gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_gate")(s)
- up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_up")(s)
- if self.activation == "swiglu":
- s = nn.silu(gate) * up
- elif self.activation == "geglu":
- s = nn.gelu(gate) * up
- else:
- s = nn.relu(gate) * nn.relu(up)
- s = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_down")(s)
- s = s + residual
-
- return s
-
-
-class MemoryMixerBlock(nn.Module):
- """Block 4a: Pack (cross-attn) → Mix (MLP-Mixer) → Local Update (MLP)."""
+class EncoderBlock(nn.Module):
num_heads: int
num_kv_heads: int
d_model: int
d_ff: int
- num_slots: int
num_layers: int
dtype: jnp.dtype = jnp.bfloat16
activation: str = "drelu"
@nn.compact
- def __call__(self, x, s, mask=None):
- s_norm = ZCRMSNorm(dtype=self.dtype, name="pack_s_norm")(s)
- x_norm = ZCRMSNorm(dtype=self.dtype, name="pack_x_norm")(x)
- s = s + MultiHeadAttention(
- self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="pack_attn"
- )(s_norm, x_norm, mask=mask)
-
- s = s + MLPMixer(self.num_slots, self.d_model, self.d_ff, self.dtype, self.activation, name="mixer")(
- ZCRMSNorm(dtype=self.dtype, name="mix_norm")(s)
+ def __call__(self, x, mask=None, rope=None):
+ residual = x
+ x = ZCRMSNorm(dtype=self.dtype)(x)
+ x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype)(
+ x, x, mask=mask, rope=rope
)
+ x = x + residual
residual = x
- x = ZCRMSNorm(dtype=self.dtype, name="local_norm")(x)
- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="local_ffn")(x)
+ x = ZCRMSNorm(dtype=self.dtype)(x)
+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x)
x = x + residual
- return x, s
+ return x
-class MemoryMixerEncoder(nn.Module):
- """Encoder using MemoryMixer blocks. Output is the final memory slots S."""
+class Encoder(nn.Module):
config: TransformerConfig
@nn.compact
- def __call__(self, x, mask=None):
+ def __call__(self, x, mask=None, rope=None):
cfg = self.config
dt = cfg.jax_dtype
x = x.astype(dt)
- s = self.param("memory_slots", jinit.normal(stddev=0.02), (1, cfg.num_memory_slots, cfg.d_model))
- s = jnp.broadcast_to(s.astype(dt), (x.shape[0], cfg.num_memory_slots, cfg.d_model))
-
for i in range(cfg.num_encoder_layers):
- x, s = MemoryMixerBlock(
- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,
- cfg.num_memory_slots, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"
- )(x, s, mask=mask)
-
- s = ZCRMSNorm(dtype=dt, name="final_norm")(s)
- return s
+ x = EncoderBlock(
+ cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"
+ )(x, mask=mask, rope=rope)
+ x = ZCRMSNorm(dtype=dt)(x)
+ return x
class DecoderBlock(nn.Module):
@@ -293,7 +241,7 @@ class EncoderDecoderTransformer(nn.Module):
def setup(self):
self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))
self.embed_scale = math.sqrt(self.config.d_model)
- self.encoder = MemoryMixerEncoder(self.config)
+ self.encoder = Encoder(self.config)
self.decoder = Decoder(self.config)
def _rope(self, seq_len):
@@ -302,13 +250,14 @@ class EncoderDecoderTransformer(nn.Module):
def encode(self, src, src_mask=None):
x = self.embedding(src) * self.embed_scale
- return self.encoder(x, mask=src_mask)
+ rope = self._rope(src.shape[1])
+ return self.encoder(x, mask=src_mask, rope=rope)
def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None):
x = self.embedding(tgt) * self.embed_scale
rope = self._rope(tgt.shape[1])
x = self.decoder(
- x, encoder_out, self_mask=self_mask, cross_mask=None, rope=rope
+ x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope
)
logits = x.astype(jnp.float32) @ self.embedding.embedding.T
return logits
@@ -316,7 +265,7 @@ class EncoderDecoderTransformer(nn.Module):
def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):
encoder_out = self.encode(src, src_mask=src_mask)
logits = self.decode(
- tgt, encoder_out, self_mask=tgt_mask
+ tgt, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask
)
return logits
diff --git a/src/train.py b/src/train.py
index b58307a..08b8aa7 100644
--- a/src/train.py
+++ b/src/train.py
@@ -282,29 +282,8 @@ def _quantize_params(params, group_size=32):
return jax.tree_util.tree_map_with_path(_maybe_quantize, params)
-# Module-level config used by _train_step (set before pmap in train())
+# Module-level group_size used by _train_step (set before pmap in train())
_GROUP_SIZE = 32
-_ORTHO_WEIGHT = 1e-4
-
-
-def _ortho_reg(params):
- """Orthogonal regularization: avg ||WᵀW - I||²_F over 2D kernels."""
- penalty = jnp.float32(0.0)
- count = 0
- for path, leaf in jax.tree_util.tree_leaves_with_path(params):
- name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])
- if name == "kernel" and leaf.ndim == 2:
- w = leaf.astype(jnp.float32)
- m, n = w.shape
- if m >= n:
- gram = w.T @ w
- eye = jnp.eye(n)
- else:
- gram = w @ w.T
- eye = jnp.eye(m)
- penalty = penalty + jnp.mean((gram - eye) ** 2)
- count += 1
- return penalty / max(count, 1)
def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):
@@ -330,8 +309,7 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):
mask = (tgt_out != pad_id).astype(jnp.float32)
ce_loss = jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)
z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)
- ortho_loss = _ORTHO_WEIGHT * _ortho_reg(params)
- return ce_loss + z_loss + ortho_loss
+ return ce_loss + z_loss
loss, grads = jax.value_and_grad(loss_fn)(state.params)
grads = jax.lax.pmean(grads, axis_name="batch")
@@ -401,22 +379,31 @@ def train(args):
# Effective batch size must be divisible by num_devices
effective_batch_size = args.batch_size * num_devices
- config = TransformerConfig(
- d_model=args.d_model,
- num_heads=args.num_heads,
- num_encoder_layers=args.num_layers,
- num_decoder_layers=args.num_layers,
- d_ff=args.d_model * 4,
- max_seq_len=max(args.max_enc_len, args.max_dec_len),
- dtype=args.dtype,
- activation=getattr(args, "activation", "drelu"),
- num_memory_slots=getattr(args, "num_memory_slots", 64),
- )
+ # Load config from checkpoint or build from CLI args
+ resume_checkpoint = getattr(args, "checkpoint", None)
+ if resume_checkpoint:
+ print(f"Resuming from checkpoint: {resume_checkpoint}")
+ with open(resume_checkpoint, "rb") as f:
+ ckpt_data = pickle.load(f)
+ ckpt_params = jax.tree.map(jnp.array, ckpt_data["params"])
+ config = TransformerConfig(**ckpt_data["config"])
+ print(f" Config: d={config.d_model}, heads={config.num_heads}, layers={config.num_encoder_layers}/{config.num_decoder_layers}")
+ else:
+ config = TransformerConfig(
+ d_model=args.d_model,
+ num_heads=args.num_heads,
+ num_encoder_layers=args.num_layers,
+ num_decoder_layers=args.num_layers,
+ d_ff=args.d_model * 4,
+ max_seq_len=max(args.max_enc_len, args.max_dec_len),
+ dtype=args.dtype,
+ activation=getattr(args, "activation", "drelu"),
+ num_memory_slots=getattr(args, "num_memory_slots", 64),
+ )
# Set module-level config (captured by _train_step closure)
- global _GROUP_SIZE, _ORTHO_WEIGHT
+ global _GROUP_SIZE
_GROUP_SIZE = getattr(args, "group_size", 32)
- _ORTHO_WEIGHT = getattr(args, "ortho_weight", 1e-4)
p_train_step = _make_p_train_step()
np.random.seed(args.seed)
@@ -432,6 +419,11 @@ def train(args):
state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps)
val_loss_fn = _make_val_loss_fn(state.apply_fn)
+ # Override with checkpoint params if resuming
+ if resume_checkpoint:
+ state = state.replace(params=ckpt_params)
+ print(f" Loaded checkpoint params into train state")
+
ema_params = jax.tree.map(jnp.copy, state.params)
state = jax_utils.replicate(state)
ema_params = jax_utils.replicate(ema_params)
@@ -640,6 +632,20 @@ def train(args):
pickle.dump({"params": params_np, "config": config.__dict__}, f)
print(f"Saved checkpoint: {ckpt_path}")
+ from .test import measure_throughput, benchmark_generation_quality
+ model = EncoderDecoderTransformer(config)
+
+ print(f"\n Throughput:")
+ tp = measure_throughput(model, eval_params, tokenizer, num_runs=5)
+ print(f" {tp['tokens_per_second']:.1f} tok/s, {tp['avg_latency_s']:.3f}s avg latency")
+
+ prompts = ["Once upon a time", "The little dog", "She was very happy because"]
+ print(f" Generation samples:")
+ quality = benchmark_generation_quality(model, eval_params, tokenizer, prompts, max_gen_len=64, temperature=0.8)
+ print(f" Avg length: {quality['avg_generation_length']:.1f} tok, repetition: {quality['bigram_repetition_rate']:.3f}")
+ for prompt, gen in quality["generations"]:
+ print(f" [{prompt}] {gen[:100]}")
+
if use_wandb:
wandb.finish()
print("\nTraining complete.")