Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

embedding scaling

HenryNdubuaku90d51ed77629
diff --git a/src/cli.py b/src/cli.py
index 962879c..86ff0ff 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -17,7 +17,6 @@ HELP = """
│ --d-model INT Model dimension (default: 128) │
│ --num-heads INT Attention heads (default: 4) │
│ --num-layers INT Encoder/decoder layers (default: 2) │
- │ --dropout FLOAT Dropout rate (default: 0.1) │
│ --max-enc-len INT Max encoder seq length (default: 128) │
│ --max-dec-len INT Max decoder seq length (default: 128) │
│ --max-samples INT Training samples (default: 20000) │
@@ -75,7 +74,6 @@ TOY_CONFIG = {
"d_model": 128,
"num_heads": 4,
"num_layers": 2,
- "dropout": 0.1,
"max_enc_len": 128,
"max_dec_len": 128,
"max_samples": 10000,
@@ -92,7 +90,6 @@ BASE_CONFIG = {
"d_model": 256,
"num_heads": 4,
"num_layers": 4,
- "dropout": 0.0,
"max_enc_len": 256,
"max_dec_len": 256,
"max_samples": 500000,
@@ -126,7 +123,6 @@ def main():
p.add_argument("--d-model", type=int, default=None)
p.add_argument("--num-heads", type=int, default=None)
p.add_argument("--num-layers", type=int, default=None)
- p.add_argument("--dropout", type=float, default=None)
p.add_argument("--max-enc-len", type=int, default=None)
p.add_argument("--max-dec-len", type=int, default=None)
p.add_argument("--max-samples", type=int, default=None)
@@ -141,6 +137,7 @@ def main():
p.add_argument("--layer-prune-ratio", type=float, default=0.33)
p.add_argument("--group-size", type=int, default=32)
p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])
+ p.add_argument("--grad-accum-steps", type=int, default=1)
p = sub.add_parser("run", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
@@ -170,7 +167,6 @@ def main():
p.add_argument("--d-model", type=int, default=128)
p.add_argument("--num-heads", type=int, default=4)
p.add_argument("--num-layers", type=int, default=2)
- p.add_argument("--dropout", type=float, default=0.1)
p.add_argument("--max-enc-len", type=int, default=128)
p.add_argument("--max-dec-len", type=int, default=128)
p.add_argument("--max-samples", type=int, default=20000)
diff --git a/src/evaluate.py b/src/evaluate.py
index df544c0..39a8a7c 100644
--- a/src/evaluate.py
+++ b/src/evaluate.py
@@ -29,7 +29,7 @@ def _make_p_encode(model):
"""Create a pmap'd encode function."""
def _encode(params, src, src_mask):
return model.apply(
- {"params": params}, src, src_mask=src_mask, deterministic=True, method="encode",
+ {"params": params}, src, src_mask=src_mask, method="encode",
)
return jax.pmap(_encode, axis_name="batch")
@@ -39,7 +39,7 @@ def _make_p_decode(model):
def _decode(params, dec_input, encoder_out, tgt_mask, cross_mask):
return model.apply(
{"params": params}, dec_input, encoder_out,
- self_mask=tgt_mask, cross_mask=cross_mask, deterministic=True, method="decode",
+ self_mask=tgt_mask, cross_mask=cross_mask, method="decode",
)
return jax.pmap(_decode, axis_name="batch")
@@ -50,8 +50,7 @@ def _make_p_forward(model):
return model.apply(
{"params": params}, src, tgt_in,
src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
- deterministic=True,
- )
+ )
return jax.pmap(_forward, axis_name="batch")
@@ -78,7 +77,7 @@ def score_sequence(model, params, enc_tokens, dec_tokens, pad_id, p_encode=None,
else:
p = params if num_devices <= 1 else jax_utils.unreplicate(params)
encoder_out = model.apply(
- {"params": p}, enc_input, src_mask=src_mask, deterministic=True, method="encode",
+ {"params": p}, enc_input, src_mask=src_mask, method="encode",
)
dec_in = [pad_id] + list(dec_tokens[:-1])
@@ -96,7 +95,7 @@ def score_sequence(model, params, enc_tokens, dec_tokens, pad_id, p_encode=None,
p = params if num_devices <= 1 else jax_utils.unreplicate(params)
logits = model.apply(
{"params": p}, dec_input, encoder_out,
- self_mask=tgt_mask, cross_mask=cross_mask, deterministic=True, method="decode",
+ self_mask=tgt_mask, cross_mask=cross_mask, method="decode",
)[0]
log_probs = jax.nn.log_softmax(logits if logits.ndim == 2 else logits[0])
@@ -140,7 +139,7 @@ def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256,
encoder_out = p_encode(params, enc_s, src_mask_s)[0:1]
else:
encoder_out = model.apply(
- {"params": single_params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",
+ {"params": single_params}, enc_input, src_mask=src_mask, method="encode",
)
dec_in = [pad_id] + list(dec_tokens[:-1])
@@ -156,7 +155,7 @@ def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256,
else:
logits = model.apply(
{"params": single_params}, dec_input, encoder_out,
- self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",
+ self_mask=tgt_mask, cross_mask=src_mask, method="decode",
)
log_probs = jax.nn.log_softmax(logits[0] if logits.ndim == 3 else logits[0])
@@ -207,7 +206,7 @@ def eval_lambada(model, params, tokenizer, max_samples=500,
encoder_out = p_encode(params, enc_s, src_mask_s)[0:1]
else:
encoder_out = model.apply(
- {"params": single_params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",
+ {"params": single_params}, enc_input, src_mask=src_mask, method="encode",
)
dec_in = jnp.array([[pad_id]])
@@ -222,7 +221,7 @@ def eval_lambada(model, params, tokenizer, max_samples=500,
else:
logits = model.apply(
{"params": single_params}, dec_in, encoder_out,
- self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",
+ self_mask=tgt_mask, cross_mask=src_mask, method="decode",
)
predicted = int(jnp.argmax(logits[0, 0] if logits.ndim == 3 else logits[0, 0]))
diff --git a/src/model.py b/src/model.py
index a445730..f4523c6 100644
--- a/src/model.py
+++ b/src/model.py
@@ -39,7 +39,6 @@ class TransformerConfig:
num_decoder_layers: int = 2
d_ff: int = 512
max_seq_len: int = 128
- dropout_rate: float = 0.1
pad_token_id: int = 0
rope_theta: float = 10000.0
dtype: str = "bfloat16"
@@ -76,11 +75,10 @@ class MultiHeadAttention(nn.Module):
num_kv_heads: int
d_model: int
num_layers: int
- dropout_rate: float = 0.1
dtype: jnp.dtype = jnp.bfloat16
@nn.compact
- def __call__(self, q_input, kv_input, mask=None, rope=None, deterministic=True):
+ def __call__(self, q_input, kv_input, mask=None, rope=None):
head_dim = self.d_model // self.num_heads
kv_dim = self.num_kv_heads * head_dim
B = q_input.shape[0]
@@ -114,7 +112,6 @@ class MultiHeadAttention(nn.Module):
attn_weights = jnp.where(mask, attn_weights, jnp.finfo(attn_weights.dtype).min)
attn_weights = nn.softmax(attn_weights, axis=-1)
- attn_weights = nn.Dropout(rate=self.dropout_rate)(attn_weights, deterministic=deterministic)
out = jnp.matmul(attn_weights, v)
out = out.transpose(0, 2, 1, 3).reshape(B, -1, self.d_model)
@@ -125,12 +122,11 @@ class FeedForward(nn.Module):
d_model: int
d_ff: int
num_layers: int
- dropout_rate: float = 0.1
dtype: jnp.dtype = jnp.bfloat16
activation: str = "drelu"
@nn.compact
- def __call__(self, x, deterministic=True):
+ def __call__(self, x):
gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="gate_proj")(x)
up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="up_proj")(x)
if self.activation == "swiglu":
@@ -139,9 +135,7 @@ class FeedForward(nn.Module):
x = nn.gelu(gate) * up
else: # drelu
x = nn.relu(gate) * nn.relu(up)
- x = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)
- x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)
- return x
+ return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)
class EncoderBlock(nn.Module):
@@ -150,23 +144,21 @@ class EncoderBlock(nn.Module):
d_model: int
d_ff: int
num_layers: int
- dropout_rate: float = 0.1
dtype: jnp.dtype = jnp.bfloat16
activation: str = "drelu"
@nn.compact
- def __call__(self, x, mask=None, rope=None, deterministic=True):
+ def __call__(self, x, mask=None, rope=None):
residual = x
x = ZCRMSNorm(dtype=self.dtype)(x)
- x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dropout_rate, self.dtype)(
- x, x, mask=mask, rope=rope, deterministic=deterministic
+ x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype)(
+ x, x, mask=mask, rope=rope
)
- x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)
x = x + residual
residual = x
x = ZCRMSNorm(dtype=self.dtype)(x)
- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dropout_rate, self.dtype, self.activation)(x, deterministic=deterministic)
+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x)
x = x + residual
return x
@@ -178,33 +170,30 @@ class DecoderBlock(nn.Module):
d_model: int
d_ff: int
num_layers: int
- dropout_rate: float = 0.1
dtype: jnp.dtype = jnp.bfloat16
activation: str = "drelu"
@nn.compact
- def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None, deterministic=True):
+ def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None):
# Self-attention with RoPE
residual = x
x = ZCRMSNorm(dtype=self.dtype)(x)
- x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dropout_rate, self.dtype, name="self_attn")(
- x, x, mask=self_mask, rope=rope, deterministic=deterministic
+ x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="self_attn")(
+ x, x, mask=self_mask, rope=rope
)
- x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)
x = x + residual
# Cross-attention (no RoPE)
residual = x
x = ZCRMSNorm(dtype=self.dtype)(x)
- x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dropout_rate, self.dtype, name="cross_attn")(
- x, encoder_out, mask=cross_mask, deterministic=deterministic
+ x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype, name="cross_attn")(
+ x, encoder_out, mask=cross_mask
)
- x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)
x = x + residual
residual = x
x = ZCRMSNorm(dtype=self.dtype)(x)
- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dropout_rate, self.dtype, self.activation)(x, deterministic=deterministic)
+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x)
x = x + residual
return x
@@ -214,16 +203,15 @@ class Encoder(nn.Module):
config: TransformerConfig
@nn.compact
- def __call__(self, x, mask=None, rope=None, deterministic=True):
+ def __call__(self, x, mask=None, rope=None):
cfg = self.config
dt = cfg.jax_dtype
x = x.astype(dt)
- x = nn.Dropout(rate=cfg.dropout_rate)(x, deterministic=deterministic)
for i in range(cfg.num_encoder_layers):
x = EncoderBlock(
- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, cfg.dropout_rate, dt, cfg.activation, name=f"block_{i}"
- )(x, mask=mask, rope=rope, deterministic=deterministic)
+ cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"
+ )(x, mask=mask, rope=rope)
x = ZCRMSNorm(dtype=dt)(x)
return x
@@ -233,16 +221,15 @@ class Decoder(nn.Module):
config: TransformerConfig
@nn.compact
- def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None, deterministic=True):
+ def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None):
cfg = self.config
dt = cfg.jax_dtype
x = x.astype(dt)
- x = nn.Dropout(rate=cfg.dropout_rate)(x, deterministic=deterministic)
for i in range(cfg.num_decoder_layers):
x = DecoderBlock(
- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, cfg.dropout_rate, dt, cfg.activation, name=f"block_{i}"
- )(x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope, deterministic=deterministic)
+ cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"
+ )(x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope)
x = ZCRMSNorm(dtype=dt)(x)
return x
@@ -254,6 +241,7 @@ class EncoderDecoderTransformer(nn.Module):
def setup(self):
self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))
+ self.embed_scale = math.sqrt(self.config.d_model)
self.encoder = Encoder(self.config)
self.decoder = Decoder(self.config)
@@ -261,24 +249,24 @@ class EncoderDecoderTransformer(nn.Module):
head_dim = self.config.d_model // self.config.num_heads
return precompute_rope_freqs(head_dim, seq_len, self.config.rope_theta)
- def encode(self, src, src_mask=None, deterministic=True):
- x = self.embedding(src)
+ def encode(self, src, src_mask=None):
+ x = self.embedding(src) * self.embed_scale
rope = self._rope(src.shape[1])
- return self.encoder(x, mask=src_mask, rope=rope, deterministic=deterministic)
+ return self.encoder(x, mask=src_mask, rope=rope)
- def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None, deterministic=True):
- x = self.embedding(tgt)
+ def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None):
+ x = self.embedding(tgt) * self.embed_scale
rope = self._rope(tgt.shape[1])
x = self.decoder(
- x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope, deterministic=deterministic
+ x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope
)
logits = x.astype(jnp.float32) @ self.embedding.embedding.T
return logits
- def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, deterministic=True):
- encoder_out = self.encode(src, src_mask=src_mask, deterministic=deterministic)
+ def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):
+ encoder_out = self.encode(src, src_mask=src_mask)
logits = self.decode(
- tgt, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask, deterministic=deterministic
+ tgt, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask
)
return logits
diff --git a/src/run.py b/src/run.py
index b4a7bf2..df60452 100644
--- a/src/run.py
+++ b/src/run.py
@@ -31,7 +31,7 @@ def generate(model, params, tokenizer, prompt, max_gen_len=128, temperature=0.8,
src_mask = make_padding_mask(enc_input, pad_id)
encoder_out = model.apply(
- {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode"
+ {"params": params}, enc_input, src_mask=src_mask, method="encode"
)
# Fixed-size buffer so JAX compiles once
@@ -47,8 +47,7 @@ def generate(model, params, tokenizer, prompt, max_gen_len=128, temperature=0.8,
encoder_out,
self_mask=tgt_mask,
cross_mask=src_mask,
- deterministic=True,
- method="decode",
+ method="decode",
)
return logits
diff --git a/src/test.py b/src/test.py
index 15d6c2c..86d3598 100644
--- a/src/test.py
+++ b/src/test.py
@@ -39,8 +39,7 @@ def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batch
logits = model.apply(
{"params": params}, src, tgt_in,
src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
- deterministic=True,
- )
+ )
loss = optax.softmax_cross_entropy_with_integer_labels(logits, tgt_out)
mask = (tgt_out != pad_id).astype(jnp.float32)
@@ -64,13 +63,13 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt="Once upon
def decode_step(dec_buffer, encoder_out, src_mask):
logits = model.apply(
{"params": params}, dec_buffer, encoder_out,
- self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",
+ self_mask=tgt_mask, cross_mask=src_mask, method="decode",
)
return logits
# Warmup — compile once
encoder_out = model.apply(
- {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode"
+ {"params": params}, enc_input, src_mask=src_mask, method="encode"
)
dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
dec_buffer = dec_buffer.at[0, 0].set(eos_id)
@@ -87,7 +86,7 @@ def measure_throughput(model, params, tokenizer, num_runs=10, prompt="Once upon
start = time.perf_counter()
encoder_out = model.apply(
- {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode"
+ {"params": params}, enc_input, src_mask=src_mask, method="encode"
)
logits = decode_step(dec_buffer, encoder_out, src_mask)
diff --git a/src/train.py b/src/train.py
index 5b1f0c8..61bfcbd 100644
--- a/src/train.py
+++ b/src/train.py
@@ -96,14 +96,13 @@ def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.15):
def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps):
model = EncoderDecoderTransformer(config)
- rng, init_rng, dropout_rng = jax.random.split(rng, 3)
+ rng, init_rng = jax.random.split(rng)
dummy_src = jnp.ones((1, 128), dtype=jnp.int32)
dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)
variables = model.init(
- {"params": init_rng, "dropout": dropout_rng},
+ {"params": init_rng},
dummy_src,
dummy_tgt,
- deterministic=False,
)
adam_schedule = _wsd_schedule(learning_rate, total_steps, warmup_steps)
@@ -287,9 +286,8 @@ def _quantize_params(params, group_size=32):
_GROUP_SIZE = 32
-def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask, dropout_rng):
+def _compute_grads(state, src, tgt_in, tgt_out, causal_mask):
pad_id = 0
- ema_decay = 0.999
def loss_fn(params):
src_mask = make_padding_mask(src, pad_id)
@@ -303,8 +301,6 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask, dropout_rn
src_mask=src_mask,
tgt_mask=tgt_mask,
cross_mask=cross_mask,
- deterministic=False,
- rngs={"dropout": dropout_rng},
)
logits_f32 = logits.astype(jnp.float32)
@@ -317,14 +313,23 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask, dropout_rn
loss, grads = jax.value_and_grad(loss_fn)(state.params)
grads = jax.lax.pmean(grads, axis_name="batch")
loss = jax.lax.pmean(loss, axis_name="batch")
+ return loss, grads
+
+
+def _apply_grads(state, ema_params, grads):
+ ema_decay = 0.999
grad_norm = optax.global_norm(grads)
state = state.apply_gradients(grads=grads)
ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, state.params)
- return state, ema_params, loss, grad_norm
+ return state, ema_params, grad_norm
-def _make_p_train_step():
- return jax.pmap(_train_step, axis_name="batch", donate_argnums=(0, 1))
+def _make_p_compute_grads():
+ return jax.pmap(_compute_grads, axis_name="batch")
+
+
+def _make_p_apply_grads():
+ return jax.pmap(_apply_grads, axis_name="batch", donate_argnums=(0, 1))
def _make_val_loss_fn(apply_fn):
@@ -336,7 +341,6 @@ def _make_val_loss_fn(apply_fn):
logits = apply_fn(
{"params": params}, src, tgt_in,
src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=src_mask,
- deterministic=True,
)
loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)
mask = (tgt_out != pad_id).astype(jnp.float32)
@@ -390,7 +394,6 @@ def train(args):
num_decoder_layers=args.num_layers,
d_ff=args.d_model * 4,
max_seq_len=max(args.max_enc_len, args.max_dec_len),
- dropout_rate=args.dropout,
dtype=args.dtype,
activation=getattr(args, "activation", "drelu"),
)
@@ -398,14 +401,17 @@ def train(args):
# Set group size for QAT and sparsification (captured by _train_step closure)
global _GROUP_SIZE
_GROUP_SIZE = getattr(args, "group_size", 32)
- p_train_step = _make_p_train_step()
+ grad_accum_steps = getattr(args, "grad_accum_steps", 1)
+ p_compute_grads = _make_p_compute_grads()
+ p_apply_grads = _make_p_apply_grads()
np.random.seed(args.seed)
rng = jax.random.PRNGKey(args.seed)
rng, init_rng = jax.random.split(rng)
num_batches = len(enc_inputs) // effective_batch_size
- total_steps = num_batches * args.epochs
+ optimizer_steps_per_epoch = num_batches // grad_accum_steps
+ total_steps = optimizer_steps_per_epoch * args.epochs
warmup_steps = max(1, int(total_steps * args.warmup_ratio))
scaled_lr = args.lr * num_devices
@@ -419,7 +425,8 @@ def train(args):
param_count = sum(x.size for x in jax.tree.leaves(jax_utils.unreplicate(state).params))
print(f"Model parameters: {param_count:,}")
- print(f"Devices: {num_devices}, per-device batch: {args.batch_size}, effective batch: {effective_batch_size}")
+ accum_batch = effective_batch_size * grad_accum_steps
+ print(f"Devices: {num_devices}, per-device batch: {args.batch_size}, effective batch: {effective_batch_size}, grad accum: {grad_accum_steps}x = {accum_batch}")
print(f"Adam LR: {args.lr} x {num_devices} = {scaled_lr}, Muon LR: {args.muon_lr} x sqrt({num_devices}) = {muon_lr:.4f}")
decay_steps = max(1, int(total_steps * 0.15))
stable_steps = total_steps - warmup_steps - decay_steps
@@ -518,32 +525,46 @@ def train(args):
print(f"Continuing training with {config.num_encoder_layers}+{config.num_decoder_layers} layers...\n")
losses = []
- batches = get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size)
- pbar = tqdm(batches, total=num_batches, desc=f"Epoch {epoch + 1}/{args.epochs}")
-
- for src, tgt_in, tgt_out in pbar:
- rng, dropout_rng = jax.random.split(rng)
- dropout_rngs = jax.random.split(dropout_rng, num_devices)
+ batches = list(get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size))
+ pbar = tqdm(range(optimizer_steps_per_epoch), desc=f"Epoch {epoch + 1}/{args.epochs}")
+ for opt_step in pbar:
t0 = time.perf_counter()
-
- state, ema_params, loss, grad_norm = p_train_step(
- state,
- ema_params,
- shard_batch(src, num_devices),
- shard_batch(tgt_in, num_devices),
- shard_batch(tgt_out, num_devices),
- causal_mask,
- dropout_rngs,
- )
+ accum_grads = None
+ accum_loss = 0.0
+
+ for micro in range(grad_accum_steps):
+ batch_idx = opt_step * grad_accum_steps + micro
+ if batch_idx >= len(batches):
+ break
+ src, tgt_in, tgt_out = batches[batch_idx]
+
+ loss, grads = p_compute_grads(
+ state,
+ shard_batch(src, num_devices),
+ shard_batch(tgt_in, num_devices),
+ shard_batch(tgt_out, num_devices),
+ causal_mask,
+ )
+
+ if accum_grads is None:
+ accum_grads = grads
+ else:
+ accum_grads = jax.tree.map(lambda a, g: a + g, accum_grads, grads)
+ accum_loss += float(loss[0])
+
+ # Average accumulated gradients
+ accum_grads = jax.tree.map(lambda g: g / grad_accum_steps, accum_grads)
+ accum_loss /= grad_accum_steps
+
+ state, ema_params, grad_norm = p_apply_grads(state, ema_params, accum_grads)
# Re-apply prune mask to keep pruned weights at zero
if prune_mask is not None:
state = state.replace(params=jax.tree.map(lambda w, m: w * m, state.params, prune_mask))
ema_params = jax.tree.map(lambda w, m: w * m, ema_params, prune_mask)
- loss_val = float(loss[0])
- losses.append(loss_val)
+ losses.append(accum_loss)
global_step += 1
dt = time.perf_counter() - t0
eval_every = getattr(args, "eval_every", 100)
@@ -556,15 +577,15 @@ def train(args):
total_loss += float(vl)
total_toks += float(vt)
last_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))
- pbar.set_postfix(loss=f"{loss_val:.4f}", val_ppl=f"{last_val_ppl:.2f}" if last_val_ppl is not None else "?")
+ pbar.set_postfix(loss=f"{accum_loss:.4f}", val_ppl=f"{last_val_ppl:.2f}" if last_val_ppl is not None else "?")
if use_wandb:
log_dict = {
- "train/loss": loss_val,
+ "train/loss": accum_loss,
"train/grad_norm": float(grad_norm[0]),
"train/adam_lr": float(adam_schedule(global_step)),
"train/muon_lr": float(muon_schedule(global_step)),
- "train/tokens_per_sec": tokens_per_batch / dt,
+ "train/tokens_per_sec": tokens_per_batch * grad_accum_steps / dt,
"train/step": global_step,
}
if global_step % eval_every == 0 or global_step == total_steps:
@@ -659,7 +680,6 @@ def parse_args():
parser.add_argument("--d-model", type=int, default=128)
parser.add_argument("--num-heads", type=int, default=4)
parser.add_argument("--num-layers", type=int, default=2)
- parser.add_argument("--dropout", type=float, default=0.1)
parser.add_argument("--max-enc-len", type=int, default=128)
parser.add_argument("--max-dec-len", type=int, default=128)
parser.add_argument("--max-samples", type=int, default=10000)