Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
embedding scaling
HenryNdubuaku90d51ed77629
diff --git a/src/cli.py b/src/cli.pyindex 962879c..86ff0ff 100644--- a/src/cli.py+++ b/src/cli.py@@ -17,7 +17,6 @@ HELP = """│ --d-model INT Model dimension (default: 128) ││ --num-heads INT Attention heads (default: 4) ││ --num-layers INT Encoder/decoder layers (default: 2) │- │ --dropout FLOAT Dropout rate (default: 0.1) ││ --max-enc-len INT Max encoder seq length (default: 128) ││ --max-dec-len INT Max decoder seq length (default: 128) ││ --max-samples INT Training samples (default: 20000) │@@ -75,7 +74,6 @@ TOY_CONFIG = {"d_model": 128,"num_heads": 4,"num_layers": 2,- "dropout": 0.1,"max_enc_len": 128,"max_dec_len": 128,"max_samples": 10000,@@ -92,7 +90,6 @@ BASE_CONFIG = {"d_model": 256,"num_heads": 4,"num_layers": 4,- "dropout": 0.0,"max_enc_len": 256,"max_dec_len": 256,"max_samples": 500000,@@ -126,7 +123,6 @@ def main():p.add_argument("--d-model", type=int, default=None)p.add_argument("--num-heads", type=int, default=None)p.add_argument("--num-layers", type=int, default=None)- p.add_argument("--dropout", type=float, default=None)p.add_argument("--max-enc-len", type=int, default=None)p.add_argument("--max-dec-len", type=int, default=None)p.add_argument("--max-samples", type=int, default=None)@@ -141,6 +137,7 @@ def main():p.add_argument("--layer-prune-ratio", type=float, default=0.33)p.add_argument("--group-size", type=int, default=32)p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])+ p.add_argument("--grad-accum-steps", type=int, default=1)p = sub.add_parser("run", add_help=False)p.add_argument("--checkpoint", type=str, required=True)@@ -170,7 +167,6 @@ def main():p.add_argument("--d-model", type=int, default=128)p.add_argument("--num-heads", type=int, default=4)p.add_argument("--num-layers", type=int, default=2)- p.add_argument("--dropout", type=float, default=0.1)p.add_argument("--max-enc-len", type=int, default=128)p.add_argument("--max-dec-len", type=int, default=128)p.add_argument("--max-samples", type=int, default=20000)diff --git a/src/evaluate.py b/src/evaluate.pyindex df544c0..39a8a7c 100644--- a/src/evaluate.py+++ b/src/evaluate.py@@ -29,7 +29,7 @@ def _make_p_encode(model):"""Create a pmap'd encode function."""def _encode(params, src, src_mask):return model.apply(- {"params": params}, src, src_mask=src_mask, deterministic=True, method="encode",+ {"params": params}, src, src_mask=src_mask, method="encode",)return jax.pmap(_encode, axis_name="batch")@@ -39,7 +39,7 @@ def _make_p_decode(model):def _decode(params, dec_input, encoder_out, tgt_mask, cross_mask):return model.apply({"params": params}, dec_input, encoder_out,- self_mask=tgt_mask, cross_mask=cross_mask, deterministic=True, method="decode",+ self_mask=tgt_mask, cross_mask=cross_mask, method="decode",)return jax.pmap(_decode, axis_name="batch")@@ -50,8 +50,7 @@ def _make_p_forward(model):return model.apply({"params": params}, src, tgt_in,src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,- deterministic=True,- )+ )return jax.pmap(_forward, axis_name="batch")@@ -78,7 +77,7 @@ def score_sequence(model, params, enc_tokens, dec_tokens, pad_id, p_encode=None,else:p = params if num_devices <= 1 else jax_utils.unreplicate(params)encoder_out = model.apply(- {"params": p}, enc_input, src_mask=src_mask, deterministic=True, method="encode",+ {"params": p}, enc_input, src_mask=src_mask, method="encode",)dec_in = [pad_id] + list(dec_tokens[:-1])@@ -96,7 +95,7 @@ def score_sequence(model, params, enc_tokens, dec_tokens, pad_id, p_encode=None,p = params if num_devices <= 1 else jax_utils.unreplicate(params)logits = model.apply({"params": p}, dec_input, encoder_out,- self_mask=tgt_mask, cross_mask=cross_mask, deterministic=True, method="decode",+ self_mask=tgt_mask, cross_mask=cross_mask, method="decode",)[0]log_probs = jax.nn.log_softmax(logits if logits.ndim == 2 else logits[0])@@ -140,7 +139,7 @@ def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256,encoder_out = p_encode(params, enc_s, src_mask_s)[0:1]else:encoder_out = model.apply(- {"params": single_params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",+ {"params": single_params}, enc_input, src_mask=src_mask, method="encode",)dec_in = [pad_id] + list(dec_tokens[:-1])@@ -156,7 +155,7 @@ def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256,else:logits = model.apply({"params": single_params}, dec_input, encoder_out,- self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",+ self_mask=tgt_mask, cross_mask=src_mask, method="decode",)log_probs = jax.nn.log_softmax(logits[0] if logits.ndim == 3 else logits[0])@@ -207,7 +206,7 @@ def eval_lambada(model, params, tokenizer, max_samples=500,encoder_out = p_encode(params, enc_s, src_mask_s)[0:1]else:encoder_out = model.apply(- {"params": single_params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",+ {"params": single_params}, enc_input, src_mask=src_mask, method="encode",)dec_in = jnp.array([[pad_id]])@@ -222,7 +221,7 @@ def eval_lambada(model, params, tokenizer, max_samples=500,else:logits = model.apply({"params": single_params}, dec_in, encoder_out,- self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",+ self_mask=tgt_mask, cross_mask=src_mask, method="decode",)predicted = int(jnp.argmax(logits[0, 0] if logits.ndim == 3 else logits[0, 0]))diff --git a/src/model.py b/src/model.pyindex a445730..f4523c6 100644--- a/src/model.py+++ b/src/model.py@@ -39,7 +39,6 @@ class TransformerConfig:num_decoder_layers: int = 2d_ff: int = 512max_seq_len: int = 128- dropout_rate: float = 0.1pad_token_id: int = 0rope_theta: float = 10000.0dtype: str = "bfloat16"@@ -76,11 +75,10 @@ class MultiHeadAttention(nn.Module):num_kv_heads: intd_model: intnum_layers: int- dropout_rate: float = 0.1dtype: jnp.dtype = jnp.bfloat16@nn.compact- def __call__(self, q_input, kv_input, mask=None, rope=None, deterministic=True):+ def __call__(self, q_input, kv_input, mask=None, rope=None):head_dim = self.d_model // self.num_headskv_dim = self.num_kv_heads * head_dimB = q_input.shape[0]@@ -114,7 +112,6 @@ class MultiHeadAttention(nn.Module):attn_weights = jnp.where(mask, attn_weights, jnp.finfo(attn_weights.dtype).min)attn_weights = nn.softmax(attn_weights, axis=-1)- attn_weights = nn.Dropout(rate=self.dropout_rate)(attn_weights, deterministic=deterministic)out = jnp.matmul(attn_weights, v)out = out.transpose(0, 2, 1, 3).reshape(B, -1, self.d_model)@@ -125,12 +122,11 @@ class FeedForward(nn.Module):d_model: intd_ff: intnum_layers: int- dropout_rate: float = 0.1dtype: jnp.dtype = jnp.bfloat16activation: str = "drelu"@nn.compact- def __call__(self, x, deterministic=True):+ def __call__(self, x):gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="gate_proj")(x)up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="up_proj")(x)if self.activation == "swiglu":@@ -139,9 +135,7 @@ class FeedForward(nn.Module):x = nn.gelu(gate) * upelse: # drelux = nn.relu(gate) * nn.relu(up)- x = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)- x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)- return x+ return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)class EncoderBlock(nn.Module):@@ -150,23 +144,21 @@ class EncoderBlock(nn.Module):d_model: intd_ff: intnum_layers: int- dropout_rate: float = 0.1dtype: jnp.dtype = jnp.bfloat16activation: str = "drelu"@nn.compact- def __call__(self, x, mask=None, rope=None, deterministic=True):+ def __call__(self, x, mask=None, rope=None):residual = xx = ZCRMSNorm(dtype=self.dtype)(x)- x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dropout_rate, self.dtype)(- x, x, mask=mask, rope=rope, deterministic=deterministic+ x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype)(+ x, x, mask=mask, rope=rope)- x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)x = x + residualresidual = xx = ZCRMSNorm(dtype=self.dtype)(x)- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dropout_rate, self.dtype, self.activation)(x, deterministic=deterministic)+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x)x = x + residualreturn x@@ -178,33 +170,30 @@ class DecoderBlock(nn.Module):d_model: intd_ff: intnum_layers: int- dropout_rate: float = 0.1dtype: jnp.dtype = jnp.bfloat16activation: str = "drelu"@nn.compact- def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None, deterministic=True):+ def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None):# Self-attention with RoPEresidual = xx = ZCRMSNorm(dtype=self.dtype)(x)- x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dropout_rate, self.dtype, name="self_attn")(- x, x, mask=self_mask, rope=rope, deterministic=deterministic+ x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(+ x, x, mask=self_mask, rope=rope)- x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)x = x + residual# Cross-attention (no RoPE)residual = xx = ZCRMSNorm(dtype=self.dtype)(x)- x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dropout_rate, self.dtype, name="cross_attn")(- x, encoder_out, mask=cross_mask, deterministic=deterministic+ x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="cross_attn")(+ x, encoder_out, mask=cross_mask)- x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)x = x + residualresidual = xx = ZCRMSNorm(dtype=self.dtype)(x)- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dropout_rate, self.dtype, self.activation)(x, deterministic=deterministic)+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x)x = x + residualreturn x@@ -214,16 +203,15 @@ class Encoder(nn.Module):config: TransformerConfig@nn.compact- def __call__(self, x, mask=None, rope=None, deterministic=True):+ def __call__(self, x, mask=None, rope=None):cfg = self.configdt = cfg.jax_dtypex = x.astype(dt)- x = nn.Dropout(rate=cfg.dropout_rate)(x, deterministic=deterministic)for i in range(cfg.num_encoder_layers):x = EncoderBlock(- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, cfg.dropout_rate, dt, cfg.activation, name=f"block_{i}"- )(x, mask=mask, rope=rope, deterministic=deterministic)+ cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"+ )(x, mask=mask, rope=rope)x = ZCRMSNorm(dtype=dt)(x)return x@@ -233,16 +221,15 @@ class Decoder(nn.Module):config: TransformerConfig@nn.compact- def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None, deterministic=True):+ def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None):cfg = self.configdt = cfg.jax_dtypex = x.astype(dt)- x = nn.Dropout(rate=cfg.dropout_rate)(x, deterministic=deterministic)for i in range(cfg.num_decoder_layers):x = DecoderBlock(- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, cfg.dropout_rate, dt, cfg.activation, name=f"block_{i}"- )(x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope, deterministic=deterministic)+ cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"+ )(x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope)x = ZCRMSNorm(dtype=dt)(x)return x@@ -254,6 +241,7 @@ class EncoderDecoderTransformer(nn.Module):def setup(self):self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))+ self.embed_scale = math.sqrt(self.config.d_model)self.encoder = Encoder(self.config)self.decoder = Decoder(self.config)@@ -261,24 +249,24 @@ class EncoderDecoderTransformer(nn.Module):head_dim = self.config.d_model // self.config.num_headsreturn precompute_rope_freqs(head_dim, seq_len, self.config.rope_theta)- def encode(self, src, src_mask=None, deterministic=True):- x = self.embedding(src)+ def encode(self, src, src_mask=None):+ x = self.embedding(src) * self.embed_scalerope = self._rope(src.shape[1])- return self.encoder(x, mask=src_mask, rope=rope, deterministic=deterministic)+ return self.encoder(x, mask=src_mask, rope=rope)- def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None, deterministic=True):- x = self.embedding(tgt)+ def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None):+ x = self.embedding(tgt) * self.embed_scalerope = self._rope(tgt.shape[1])x = self.decoder(- x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope, deterministic=deterministic+ x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope)logits = x.astype(jnp.float32) @ self.embedding.embedding.Treturn logits- def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, deterministic=True):- encoder_out = self.encode(src, src_mask=src_mask, deterministic=deterministic)+ def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):+ encoder_out = self.encode(src, src_mask=src_mask)logits = self.decode(- tgt, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask, deterministic=deterministic+ tgt, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask)return logitsdiff --git a/src/run.py b/src/run.pyindex b4a7bf2..df60452 100644--- a/src/run.py+++ b/src/run.py@@ -31,7 +31,7 @@ def generate(model, params, tokenizer, prompt, max_gen_len=128, temperature=0.8,src_mask = make_padding_mask(enc_input, pad_id)encoder_out = model.apply(- {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode"+ {"params": params}, enc_input, src_mask=src_mask, method="encode")# Fixed-size buffer so JAX compiles once@@ -47,8 +47,7 @@ def generate(model, params, tokenizer, prompt, max_gen_len=128, temperature=0.8,encoder_out,self_mask=tgt_mask,cross_mask=src_mask,- deterministic=True,- method="decode",+ method="decode",)return logitsdiff --git a/src/test.py b/src/test.pyindex 15d6c2c..86d3598 100644--- a/src/test.py+++ b/src/test.py@@ -39,8 +39,7 @@ def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batchlogits = model.apply({"params": params}, src, tgt_in,src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,- deterministic=True,- )+ )loss = optax.softmax_cross_entropy_with_integer_labels(logits, tgt_out)mask = (tgt_out != pad_id).astype(jnp.float32)@@ -64,13 +63,13 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt="Once upondef decode_step(dec_buffer, encoder_out, src_mask):logits = model.apply({"params": params}, dec_buffer, encoder_out,- self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",+ self_mask=tgt_mask, cross_mask=src_mask, method="decode",)return logits# Warmup — compile onceencoder_out = model.apply(- {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode"+ {"params": params}, enc_input, src_mask=src_mask, method="encode")dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)dec_buffer = dec_buffer.at[0, 0].set(eos_id)@@ -87,7 +86,7 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt="Once uponstart = time.perf_counter()encoder_out = model.apply(- {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode"+ {"params": params}, enc_input, src_mask=src_mask, method="encode")logits = decode_step(dec_buffer, encoder_out, src_mask)diff --git a/src/train.py b/src/train.pyindex 5b1f0c8..61bfcbd 100644--- a/src/train.py+++ b/src/train.py@@ -96,14 +96,13 @@ def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.15):def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps):model = EncoderDecoderTransformer(config)- rng, init_rng, dropout_rng = jax.random.split(rng, 3)+ rng, init_rng = jax.random.split(rng)dummy_src = jnp.ones((1, 128), dtype=jnp.int32)dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)variables = model.init(- {"params": init_rng, "dropout": dropout_rng},+ {"params": init_rng},dummy_src,dummy_tgt,- deterministic=False,)adam_schedule = _wsd_schedule(learning_rate, total_steps, warmup_steps)@@ -287,9 +286,8 @@ def _quantize_params(params, group_size=32):_GROUP_SIZE = 32-def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask, dropout_rng):+def _compute_grads(state, src, tgt_in, tgt_out, causal_mask):pad_id = 0- ema_decay = 0.999def loss_fn(params):src_mask = make_padding_mask(src, pad_id)@@ -303,8 +301,6 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask, dropout_rnsrc_mask=src_mask,tgt_mask=tgt_mask,cross_mask=cross_mask,- deterministic=False,- rngs={"dropout": dropout_rng},)logits_f32 = logits.astype(jnp.float32)@@ -317,14 +313,23 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask, dropout_rnloss, grads = jax.value_and_grad(loss_fn)(state.params)grads = jax.lax.pmean(grads, axis_name="batch")loss = jax.lax.pmean(loss, axis_name="batch")+ return loss, grads+++def _apply_grads(state, ema_params, grads):+ ema_decay = 0.999grad_norm = optax.global_norm(grads)state = state.apply_gradients(grads=grads)ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, state.params)- return state, ema_params, loss, grad_norm+ return state, ema_params, grad_norm-def _make_p_train_step():- return jax.pmap(_train_step, axis_name="batch", donate_argnums=(0, 1))+def _make_p_compute_grads():+ return jax.pmap(_compute_grads, axis_name="batch")+++def _make_p_apply_grads():+ return jax.pmap(_apply_grads, axis_name="batch", donate_argnums=(0, 1))def _make_val_loss_fn(apply_fn):@@ -336,7 +341,6 @@ def _make_val_loss_fn(apply_fn):logits = apply_fn({"params": params}, src, tgt_in,src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=src_mask,- deterministic=True,)loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)mask = (tgt_out != pad_id).astype(jnp.float32)@@ -390,7 +394,6 @@ def train(args):num_decoder_layers=args.num_layers,d_ff=args.d_model * 4,max_seq_len=max(args.max_enc_len, args.max_dec_len),- dropout_rate=args.dropout,dtype=args.dtype,activation=getattr(args, "activation", "drelu"),)@@ -398,14 +401,17 @@ def train(args):# Set group size for QAT and sparsification (captured by _train_step closure)global _GROUP_SIZE_GROUP_SIZE = getattr(args, "group_size", 32)- p_train_step = _make_p_train_step()+ grad_accum_steps = getattr(args, "grad_accum_steps", 1)+ p_compute_grads = _make_p_compute_grads()+ p_apply_grads = _make_p_apply_grads()np.random.seed(args.seed)rng = jax.random.PRNGKey(args.seed)rng, init_rng = jax.random.split(rng)num_batches = len(enc_inputs) // effective_batch_size- total_steps = num_batches * args.epochs+ optimizer_steps_per_epoch = num_batches // grad_accum_steps+ total_steps = optimizer_steps_per_epoch * args.epochswarmup_steps = max(1, int(total_steps * args.warmup_ratio))scaled_lr = args.lr * num_devices@@ -419,7 +425,8 @@ def train(args):param_count = sum(x.size for x in jax.tree.leaves(jax_utils.unreplicate(state).params))print(f"Model parameters: {param_count:,}")- print(f"Devices: {num_devices}, per-device batch: {args.batch_size}, effective batch: {effective_batch_size}")+ accum_batch = effective_batch_size * grad_accum_steps+ print(f"Devices: {num_devices}, per-device batch: {args.batch_size}, effective batch: {effective_batch_size}, grad accum: {grad_accum_steps}x = {accum_batch}")print(f"Adam LR: {args.lr} x {num_devices} = {scaled_lr}, Muon LR: {args.muon_lr} x sqrt({num_devices}) = {muon_lr:.4f}")decay_steps = max(1, int(total_steps * 0.15))stable_steps = total_steps - warmup_steps - decay_steps@@ -518,32 +525,46 @@ def train(args):print(f"Continuing training with {config.num_encoder_layers}+{config.num_decoder_layers} layers...\n")losses = []- batches = get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size)- pbar = tqdm(batches, total=num_batches, desc=f"Epoch {epoch + 1}/{args.epochs}")-- for src, tgt_in, tgt_out in pbar:- rng, dropout_rng = jax.random.split(rng)- dropout_rngs = jax.random.split(dropout_rng, num_devices)+ batches = list(get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size))+ pbar = tqdm(range(optimizer_steps_per_epoch), desc=f"Epoch {epoch + 1}/{args.epochs}")+ for opt_step in pbar:t0 = time.perf_counter()-- state, ema_params, loss, grad_norm = p_train_step(- state,- ema_params,- shard_batch(src, num_devices),- shard_batch(tgt_in, num_devices),- shard_batch(tgt_out, num_devices),- causal_mask,- dropout_rngs,- )+ accum_grads = None+ accum_loss = 0.0++ for micro in range(grad_accum_steps):+ batch_idx = opt_step * grad_accum_steps + micro+ if batch_idx >= len(batches):+ break+ src, tgt_in, tgt_out = batches[batch_idx]++ loss, grads = p_compute_grads(+ state,+ shard_batch(src, num_devices),+ shard_batch(tgt_in, num_devices),+ shard_batch(tgt_out, num_devices),+ causal_mask,+ )++ if accum_grads is None:+ accum_grads = grads+ else:+ accum_grads = jax.tree.map(lambda a, g: a + g, accum_grads, grads)+ accum_loss += float(loss[0])++ # Average accumulated gradients+ accum_grads = jax.tree.map(lambda g: g / grad_accum_steps, accum_grads)+ accum_loss /= grad_accum_steps++ state, ema_params, grad_norm = p_apply_grads(state, ema_params, accum_grads)# Re-apply prune mask to keep pruned weights at zeroif prune_mask is not None:state = state.replace(params=jax.tree.map(lambda w, m: w * m, state.params, prune_mask))ema_params = jax.tree.map(lambda w, m: w * m, ema_params, prune_mask)- loss_val = float(loss[0])- losses.append(loss_val)+ losses.append(accum_loss)global_step += 1dt = time.perf_counter() - t0eval_every = getattr(args, "eval_every", 100)@@ -556,15 +577,15 @@ def train(args):total_loss += float(vl)total_toks += float(vt)last_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))- pbar.set_postfix(loss=f"{loss_val:.4f}", val_ppl=f"{last_val_ppl:.2f}" if last_val_ppl is not None else "?")+ pbar.set_postfix(loss=f"{accum_loss:.4f}", val_ppl=f"{last_val_ppl:.2f}" if last_val_ppl is not None else "?")if use_wandb:log_dict = {- "train/loss": loss_val,+ "train/loss": accum_loss,"train/grad_norm": float(grad_norm[0]),"train/adam_lr": float(adam_schedule(global_step)),"train/muon_lr": float(muon_schedule(global_step)),- "train/tokens_per_sec": tokens_per_batch / dt,+ "train/tokens_per_sec": tokens_per_batch * grad_accum_steps / dt,"train/step": global_step,}if global_step % eval_every == 0 or global_step == total_steps:@@ -659,7 +680,6 @@ def parse_args():parser.add_argument("--d-model", type=int, default=128)parser.add_argument("--num-heads", type=int, default=4)parser.add_argument("--num-layers", type=int, default=2)- parser.add_argument("--dropout", type=float, default=0.1)parser.add_argument("--max-enc-len", type=int, default=128)parser.add_argument("--max-dec-len", type=int, default=128)parser.add_argument("--max-samples", type=int, default=10000)