Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Training restart
HenryNdubuakuc6fd91f9610a
diff --git a/src/cli.py b/src/cli.pyindex 52a8359..b3c1f3b 100644--- a/src/cli.py+++ b/src/cli.py@@ -116,6 +116,7 @@ def main():p = sub.add_parser("train", add_help=False)p.add_argument("--toy", action="store_true")+ p.add_argument("--checkpoint", type=str, default=None)p.add_argument("--epochs", type=int, default=None)p.add_argument("--batch-size", type=int, default=None)p.add_argument("--lr", type=float, default=None)@@ -133,12 +134,11 @@ def main():p.add_argument("--seed", type=int, default=None)p.add_argument("--eval-every", type=int, default=1000)p.add_argument("--max-eval-samples", type=int, default=None)- p.add_argument("--sparsity-ratio", type=float, default=0.33)- p.add_argument("--layer-prune-ratio", type=float, default=0.33)+ p.add_argument("--sparsity-ratio", type=float, default=0.0)+ p.add_argument("--layer-prune-ratio", type=float, default=0.0)p.add_argument("--group-size", type=int, default=32)p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])p.add_argument("--num-memory-slots", type=int, default=64)- p.add_argument("--ortho-weight", type=float, default=1e-4)p = sub.add_parser("run", add_help=False)p.add_argument("--checkpoint", type=str, required=True)diff --git a/src/model.py b/src/model.pyindex 4f8d44c..da70255 100644--- a/src/model.py+++ b/src/model.py@@ -138,100 +138,48 @@ class FeedForward(nn.Module):return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)-class MLPMixer(nn.Module):- """MLP-Mixer operating on fixed-size memory slots (B, M, d)."""- num_slots: int- d_model: int- d_ff: int- dtype: jnp.dtype = jnp.bfloat16- activation: str = "drelu"-- @nn.compact- def __call__(self, s):- residual = s- s = ZCRMSNorm(dtype=self.dtype, name="token_mix_norm")(s)- s = s.transpose(0, 2, 1)- gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_gate")(s)- up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_up")(s)- if self.activation == "swiglu":- s = nn.silu(gate) * up- elif self.activation == "geglu":- s = nn.gelu(gate) * up- else:- s = nn.relu(gate) * nn.relu(up)- s = nn.Dense(self.num_slots, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_down")(s)- s = s.transpose(0, 2, 1)- s = s + residual-- residual = s- s = ZCRMSNorm(dtype=self.dtype, name="channel_mix_norm")(s)- gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_gate")(s)- up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_up")(s)- if self.activation == "swiglu":- s = nn.silu(gate) * up- elif self.activation == "geglu":- s = nn.gelu(gate) * up- else:- s = nn.relu(gate) * nn.relu(up)- s = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_down")(s)- s = s + residual-- return s---class MemoryMixerBlock(nn.Module):- """Block 4a: Pack (cross-attn) → Mix (MLP-Mixer) → Local Update (MLP)."""+class EncoderBlock(nn.Module):num_heads: intnum_kv_heads: intd_model: intd_ff: int- num_slots: intnum_layers: intdtype: jnp.dtype = jnp.bfloat16activation: str = "drelu"@nn.compact- def __call__(self, x, s, mask=None):- s_norm = ZCRMSNorm(dtype=self.dtype, name="pack_s_norm")(s)- x_norm = ZCRMSNorm(dtype=self.dtype, name="pack_x_norm")(x)- s = s + MultiHeadAttention(- self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="pack_attn"- )(s_norm, x_norm, mask=mask)-- s = s + MLPMixer(self.num_slots, self.d_model, self.d_ff, self.dtype, self.activation, name="mixer")(- ZCRMSNorm(dtype=self.dtype, name="mix_norm")(s)+ def __call__(self, x, mask=None, rope=None):+ residual = x+ x = ZCRMSNorm(dtype=self.dtype)(x)+ x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype)(+ x, x, mask=mask, rope=rope)+ x = x + residualresidual = x- x = ZCRMSNorm(dtype=self.dtype, name="local_norm")(x)- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="local_ffn")(x)+ x = ZCRMSNorm(dtype=self.dtype)(x)+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x)x = x + residual- return x, s+ return x-class MemoryMixerEncoder(nn.Module):- """Encoder using MemoryMixer blocks. Output is the final memory slots S."""+class Encoder(nn.Module):config: TransformerConfig@nn.compact- def __call__(self, x, mask=None):+ def __call__(self, x, mask=None, rope=None):cfg = self.configdt = cfg.jax_dtypex = x.astype(dt)- s = self.param("memory_slots", jinit.normal(stddev=0.02), (1, cfg.num_memory_slots, cfg.d_model))- s = jnp.broadcast_to(s.astype(dt), (x.shape[0], cfg.num_memory_slots, cfg.d_model))-for i in range(cfg.num_encoder_layers):- x, s = MemoryMixerBlock(- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,- cfg.num_memory_slots, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"- )(x, s, mask=mask)-- s = ZCRMSNorm(dtype=dt, name="final_norm")(s)- return s+ x = EncoderBlock(+ cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"+ )(x, mask=mask, rope=rope)+ x = ZCRMSNorm(dtype=dt)(x)+ return xclass DecoderBlock(nn.Module):@@ -293,7 +241,7 @@ class EncoderDecoderTransformer(nn.Module):def setup(self):self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))self.embed_scale = math.sqrt(self.config.d_model)- self.encoder = MemoryMixerEncoder(self.config)+ self.encoder = Encoder(self.config)self.decoder = Decoder(self.config)def _rope(self, seq_len):@@ -302,13 +250,14 @@ class EncoderDecoderTransformer(nn.Module):def encode(self, src, src_mask=None):x = self.embedding(src) * self.embed_scale- return self.encoder(x, mask=src_mask)+ rope = self._rope(src.shape[1])+ return self.encoder(x, mask=src_mask, rope=rope)def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None):x = self.embedding(tgt) * self.embed_scalerope = self._rope(tgt.shape[1])x = self.decoder(- x, encoder_out, self_mask=self_mask, cross_mask=None, rope=rope+ x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope)logits = x.astype(jnp.float32) @ self.embedding.embedding.Treturn logits@@ -316,7 +265,7 @@ class EncoderDecoderTransformer(nn.Module):def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):encoder_out = self.encode(src, src_mask=src_mask)logits = self.decode(- tgt, encoder_out, self_mask=tgt_mask+ tgt, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask)return logitsdiff --git a/src/train.py b/src/train.pyindex b58307a..08b8aa7 100644--- a/src/train.py+++ b/src/train.py@@ -282,29 +282,8 @@ def _quantize_params(params, group_size=32):return jax.tree_util.tree_map_with_path(_maybe_quantize, params)-# Module-level config used by _train_step (set before pmap in train())+# Module-level group_size used by _train_step (set before pmap in train())_GROUP_SIZE = 32-_ORTHO_WEIGHT = 1e-4---def _ortho_reg(params):- """Orthogonal regularization: avg ||WᵀW - I||²_F over 2D kernels."""- penalty = jnp.float32(0.0)- count = 0- for path, leaf in jax.tree_util.tree_leaves_with_path(params):- name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])- if name == "kernel" and leaf.ndim == 2:- w = leaf.astype(jnp.float32)- m, n = w.shape- if m >= n:- gram = w.T @ w- eye = jnp.eye(n)- else:- gram = w @ w.T- eye = jnp.eye(m)- penalty = penalty + jnp.mean((gram - eye) ** 2)- count += 1- return penalty / max(count, 1)def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):@@ -330,8 +309,7 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):mask = (tgt_out != pad_id).astype(jnp.float32)ce_loss = jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)- ortho_loss = _ORTHO_WEIGHT * _ortho_reg(params)- return ce_loss + z_loss + ortho_loss+ return ce_loss + z_lossloss, grads = jax.value_and_grad(loss_fn)(state.params)grads = jax.lax.pmean(grads, axis_name="batch")@@ -401,22 +379,31 @@ def train(args):# Effective batch size must be divisible by num_deviceseffective_batch_size = args.batch_size * num_devices- config = TransformerConfig(- d_model=args.d_model,- num_heads=args.num_heads,- num_encoder_layers=args.num_layers,- num_decoder_layers=args.num_layers,- d_ff=args.d_model * 4,- max_seq_len=max(args.max_enc_len, args.max_dec_len),- dtype=args.dtype,- activation=getattr(args, "activation", "drelu"),- num_memory_slots=getattr(args, "num_memory_slots", 64),- )+ # Load config from checkpoint or build from CLI args+ resume_checkpoint = getattr(args, "checkpoint", None)+ if resume_checkpoint:+ print(f"Resuming from checkpoint: {resume_checkpoint}")+ with open(resume_checkpoint, "rb") as f:+ ckpt_data = pickle.load(f)+ ckpt_params = jax.tree.map(jnp.array, ckpt_data["params"])+ config = TransformerConfig(**ckpt_data["config"])+ print(f" Config: d={config.d_model}, heads={config.num_heads}, layers={config.num_encoder_layers}/{config.num_decoder_layers}")+ else:+ config = TransformerConfig(+ d_model=args.d_model,+ num_heads=args.num_heads,+ num_encoder_layers=args.num_layers,+ num_decoder_layers=args.num_layers,+ d_ff=args.d_model * 4,+ max_seq_len=max(args.max_enc_len, args.max_dec_len),+ dtype=args.dtype,+ activation=getattr(args, "activation", "drelu"),+ num_memory_slots=getattr(args, "num_memory_slots", 64),+ )# Set module-level config (captured by _train_step closure)- global _GROUP_SIZE, _ORTHO_WEIGHT+ global _GROUP_SIZE_GROUP_SIZE = getattr(args, "group_size", 32)- _ORTHO_WEIGHT = getattr(args, "ortho_weight", 1e-4)p_train_step = _make_p_train_step()np.random.seed(args.seed)@@ -432,6 +419,11 @@ def train(args):state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps)val_loss_fn = _make_val_loss_fn(state.apply_fn)+ # Override with checkpoint params if resuming+ if resume_checkpoint:+ state = state.replace(params=ckpt_params)+ print(f" Loaded checkpoint params into train state")+ema_params = jax.tree.map(jnp.copy, state.params)state = jax_utils.replicate(state)ema_params = jax_utils.replicate(ema_params)@@ -640,6 +632,20 @@ def train(args):pickle.dump({"params": params_np, "config": config.__dict__}, f)print(f"Saved checkpoint: {ckpt_path}")+ from .test import measure_throughput, benchmark_generation_quality+ model = EncoderDecoderTransformer(config)++ print(f"\n Throughput:")+ tp = measure_throughput(model, eval_params, tokenizer, num_runs=5)+ print(f" {tp['tokens_per_second']:.1f} tok/s, {tp['avg_latency_s']:.3f}s avg latency")++ prompts = ["Once upon a time", "The little dog", "She was very happy because"]+ print(f" Generation samples:")+ quality = benchmark_generation_quality(model, eval_params, tokenizer, prompts, max_gen_len=64, temperature=0.8)+ print(f" Avg length: {quality['avg_generation_length']:.1f} tok, repetition: {quality['bigram_repetition_rate']:.3f}")+ for prompt, gen in quality["generations"]:+ print(f" [{prompt}] {gen[:100]}")+if use_wandb:wandb.finish()print("\nTraining complete.")