Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

small vocab size

HenryNdubuakua9334eeaebc4
diff --git a/.gitignore b/.gitignore
index 9c716c0..8ebc8b4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -88,3 +88,4 @@ __marimo__/
checkpoints/
wandb/
.data_cache/
+tokenizer/
\ No newline at end of file
diff --git a/experiements/architecture.txt b/experiements/architecture.txt
new file mode 100644
index 0000000..e69de29
diff --git a/experiements/tokenizer.txt b/experiements/tokenizer.txt
new file mode 100644
index 0000000..e69de29
diff --git a/pyproject.toml b/pyproject.toml
index aba3ce6..13ebd8e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,6 +10,7 @@ dependencies = [
"datasets",
"transformers",
"wandb",
+ "sentencepiece",
]
[project.optional-dependencies]
diff --git a/requirements.txt b/requirements.txt
index 0b2e3f0..4e3232f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,3 +6,4 @@ datasets
transformers
wandb
pyyaml
+sentencepiece
diff --git a/src/data.py b/src/data.py
index 1c1bb31..4e0f1ed 100644
--- a/src/data.py
+++ b/src/data.py
@@ -8,16 +8,105 @@ logging.getLogger("transformers").setLevel(logging.ERROR)
import numpy as np
from datasets import load_dataset
from tqdm import tqdm
-from transformers import AutoTokenizer
+import sentencepiece as spm
CACHE_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), ".data_cache")
+TOKENIZER_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tokenizer")
+TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")
+
+PAD_ID = 0
+EOS_ID = 1
+BOS_ID = 2
+
+
+class NeedleTokenizer:
+ """Wrapper around SentencePiece providing the interface the codebase expects."""
+
+ def __init__(self, model_path):
+ self.sp = spm.SentencePieceProcessor()
+ self.sp.Load(model_path)
+
+ @property
+ def pad_token_id(self):
+ return PAD_ID
+
+ @property
+ def eos_token_id(self):
+ return EOS_ID
+
+ @property
+ def bos_token_id(self):
+ return BOS_ID
+
+ @property
+ def vocab_size(self):
+ return self.sp.GetPieceSize()
+
+ def encode(self, text):
+ return self.sp.Encode(text, out_type=int)
+
+ def decode(self, ids):
+ if isinstance(ids, (list, tuple)) and len(ids) > 0 and isinstance(ids[0], (list, tuple)):
+ return [self.sp.Decode(seq) for seq in ids]
+ return self.sp.Decode(list(ids))
+
+ def __call__(self, texts, truncation=True, max_length=None, **kwargs):
+ all_ids = []
+ for text in texts:
+ ids = self.sp.Encode(text, out_type=int)
+ if truncation and max_length:
+ ids = ids[:max_length]
+ all_ids.append(ids)
+ return {"input_ids": all_ids}
+
+
+def train_tokenizer(vocab_size=8192, max_samples=100000):
+ """Train a SentencePiece BPE tokenizer on TinyStories."""
+ model_path = TOKENIZER_PREFIX + ".model"
+ if os.path.exists(model_path):
+ print(f"Tokenizer already exists at {model_path}")
+ return model_path
+
+ os.makedirs(TOKENIZER_DIR, exist_ok=True)
+
+ print(f"Training SentencePiece BPE tokenizer (vocab_size={vocab_size})...")
+ ds = load_dataset("roneneldan/TinyStories", split="train")
+ indices = range(min(max_samples, len(ds)))
+ ds_subset = ds.select(indices)
+
+ corpus_path = os.path.join(TOKENIZER_DIR, "corpus.txt")
+ with open(corpus_path, "w") as f:
+ for example in tqdm(ds_subset, desc="Writing corpus"):
+ text = example["text"].strip()
+ if text:
+ f.write(text + "\n")
+
+ spm.SentencePieceTrainer.Train(
+ input=corpus_path,
+ model_prefix=TOKENIZER_PREFIX,
+ vocab_size=vocab_size,
+ model_type="bpe",
+ pad_id=PAD_ID,
+ eos_id=EOS_ID,
+ bos_id=BOS_ID,
+ unk_id=3,
+ byte_fallback=True,
+ normalization_rule_name="identity",
+ num_threads=os.cpu_count(),
+ train_extremely_large_corpus=False,
+ minloglevel=2,
+ )
+
+ os.remove(corpus_path)
+ print(f"Tokenizer saved to {model_path}")
+ return model_path
def get_tokenizer():
- tokenizer = AutoTokenizer.from_pretrained("LiquidAI/LFM2-1.2B")
- if tokenizer.pad_token is None:
- tokenizer.pad_token = tokenizer.eos_token
- return tokenizer
+ model_path = TOKENIZER_PREFIX + ".model"
+ if not os.path.exists(model_path):
+ train_tokenizer()
+ return NeedleTokenizer(model_path)
def load_tinystories(split="train", max_samples=None):
@@ -27,13 +116,13 @@ def load_tinystories(split="train", max_samples=None):
return ds
-def _cache_key(n_samples, max_enc_len, max_dec_len, tokenizer_name="lfm2"):
+def _cache_key(n_samples, max_enc_len, max_dec_len, tokenizer_name="needle_spm"):
key = f"tinystories_{tokenizer_name}_{n_samples}_{max_enc_len}_{max_dec_len}"
return hashlib.md5(key.encode()).hexdigest()[:12]
def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):
-
+
cache_id = _cache_key(len(ds), max_enc_len, max_dec_len)
cache_path = os.path.join(CACHE_DIR, cache_id)
if os.path.exists(cache_path + "_enc.npy"):
@@ -52,7 +141,6 @@ def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=12
texts = list(ds["text"])
all_tokens = tokenizer(
texts, truncation=True, max_length=max_total,
- return_attention_mask=False, return_token_type_ids=False,
)["input_ids"]
n = len(all_tokens)
diff --git a/src/model.py b/src/model.py
index da70255..fe0c995 100644
--- a/src/model.py
+++ b/src/model.py
@@ -31,7 +31,7 @@ class ZCRMSNorm(nn.Module):
@dataclass
class TransformerConfig:
- vocab_size: int = 65536
+ vocab_size: int = 8192
d_model: int = 128
num_heads: int = 4
num_kv_heads: int = 2
@@ -77,6 +77,7 @@ class MultiHeadAttention(nn.Module):
d_model: int
num_layers: int
dtype: jnp.dtype = jnp.bfloat16
+ rope_keys_only: bool = False
@nn.compact
def __call__(self, q_input, kv_input, mask=None, rope=None):
@@ -102,7 +103,8 @@ class MultiHeadAttention(nn.Module):
if rope is not None:
cos, sin = rope
- q = apply_rope(q, cos, sin)
+ if not self.rope_keys_only:
+ q = apply_rope(q, cos, sin)
k = apply_rope(k, cos, sin)
scale = jnp.sqrt(jnp.float32(head_dim))
@@ -138,33 +140,81 @@ class FeedForward(nn.Module):
return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)
-class EncoderBlock(nn.Module):
+class MLPMixer(nn.Module):
+ """MLP-Mixer operating on fixed-size memory slots (B, M, d)."""
+ num_slots: int
+ d_model: int
+ d_ff: int
+ dtype: jnp.dtype = jnp.bfloat16
+ activation: str = "drelu"
+
+ @nn.compact
+ def __call__(self, s):
+ residual = s
+ s = ZCRMSNorm(dtype=self.dtype, name="token_mix_norm")(s)
+ s = s.transpose(0, 2, 1)
+ gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_gate")(s)
+ up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_up")(s)
+ if self.activation == "swiglu":
+ s = nn.silu(gate) * up
+ elif self.activation == "geglu":
+ s = nn.gelu(gate) * up
+ else:
+ s = nn.relu(gate) * nn.relu(up)
+ s = nn.Dense(self.num_slots, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_down")(s)
+ s = s.transpose(0, 2, 1)
+ s = s + residual
+
+ residual = s
+ s = ZCRMSNorm(dtype=self.dtype, name="channel_mix_norm")(s)
+ gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_gate")(s)
+ up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_up")(s)
+ if self.activation == "swiglu":
+ s = nn.silu(gate) * up
+ elif self.activation == "geglu":
+ s = nn.gelu(gate) * up
+ else:
+ s = nn.relu(gate) * nn.relu(up)
+ s = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_down")(s)
+ s = s + residual
+
+ return s
+
+
+class MemoryMixerBlock(nn.Module):
+ """Block 4a: Pack (cross-attn S←X with RoPE on keys) → Mix (MLP-Mixer) → Local Update (MLP)."""
num_heads: int
num_kv_heads: int
d_model: int
d_ff: int
+ num_slots: int
num_layers: int
dtype: jnp.dtype = jnp.bfloat16
activation: str = "drelu"
@nn.compact
- def __call__(self, x, mask=None, rope=None):
- residual = x
- x = ZCRMSNorm(dtype=self.dtype)(x)
- x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype)(
- x, x, mask=mask, rope=rope
+ def __call__(self, x, s, mask=None, rope=None):
+ s_norm = ZCRMSNorm(dtype=self.dtype, name="pack_s_norm")(s)
+ x_norm = ZCRMSNorm(dtype=self.dtype, name="pack_x_norm")(x)
+ s = s + MultiHeadAttention(
+ self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype,
+ rope_keys_only=True, name="pack_attn"
+ )(s_norm, x_norm, mask=mask, rope=rope)
+
+ s = s + MLPMixer(self.num_slots, self.d_model, self.d_ff, self.dtype, self.activation, name="mixer")(
+ ZCRMSNorm(dtype=self.dtype, name="mix_norm")(s)
)
- x = x + residual
residual = x
- x = ZCRMSNorm(dtype=self.dtype)(x)
- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x)
+ x = ZCRMSNorm(dtype=self.dtype, name="local_norm")(x)
+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="local_ffn")(x)
x = x + residual
- return x
+ return x, s
-class Encoder(nn.Module):
+class MemoryMixerEncoder(nn.Module):
+ """Encoder using MemoryMixer blocks. Output is the final memory slots S."""
config: TransformerConfig
@nn.compact
@@ -173,13 +223,19 @@ class Encoder(nn.Module):
dt = cfg.jax_dtype
x = x.astype(dt)
+ s_base = self.param("memory_slots", jinit.normal(stddev=0.02), (1, cfg.num_memory_slots, cfg.d_model))
+ x_pool = jnp.mean(x, axis=1) # (B, d)
+ s_bias = nn.Dense(cfg.d_model, dtype=dt, use_bias=False, kernel_init=jinit.zeros, name="slot_init")(x_pool)
+ s = jnp.broadcast_to(s_base.astype(dt), (x.shape[0], cfg.num_memory_slots, cfg.d_model)) + s_bias[:, None, :]
+
for i in range(cfg.num_encoder_layers):
- x = EncoderBlock(
- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"
- )(x, mask=mask, rope=rope)
+ x, s = MemoryMixerBlock(
+ cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,
+ cfg.num_memory_slots, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"
+ )(x, s, mask=mask, rope=rope)
- x = ZCRMSNorm(dtype=dt)(x)
- return x
+ s = ZCRMSNorm(dtype=dt, name="final_norm")(s)
+ return s
class DecoderBlock(nn.Module):
@@ -241,7 +297,7 @@ class EncoderDecoderTransformer(nn.Module):
def setup(self):
self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))
self.embed_scale = math.sqrt(self.config.d_model)
- self.encoder = Encoder(self.config)
+ self.encoder = MemoryMixerEncoder(self.config)
self.decoder = Decoder(self.config)
def _rope(self, seq_len):
@@ -257,7 +313,7 @@ class EncoderDecoderTransformer(nn.Module):
x = self.embedding(tgt) * self.embed_scale
rope = self._rope(tgt.shape[1])
x = self.decoder(
- x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope
+ x, encoder_out, self_mask=self_mask, cross_mask=None, rope=rope
)
logits = x.astype(jnp.float32) @ self.embedding.embedding.T
return logits
@@ -265,10 +321,19 @@ class EncoderDecoderTransformer(nn.Module):
def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):
encoder_out = self.encode(src, src_mask=src_mask)
logits = self.decode(
- tgt, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask
+ tgt, encoder_out, self_mask=tgt_mask
)
return logits
+ def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):
+ encoder_out = self.encode(src, src_mask=src_mask)
+ logits = self.decode(tgt, encoder_out, self_mask=tgt_mask)
+ s = encoder_out.astype(jnp.float32)
+ gram = jnp.matmul(s, s.transpose(0, 2, 1))
+ diag_sq = jnp.sum(jnp.diagonal(gram, axis1=1, axis2=2) ** 2)
+ slot_div = (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]
+ return logits, slot_div
+
def make_causal_mask(seq_len):
mask = jnp.tril(jnp.ones((seq_len, seq_len), dtype=jnp.bool_))
diff --git a/src/train.py b/src/train.py
index 08b8aa7..a422122 100644
--- a/src/train.py
+++ b/src/train.py
@@ -167,7 +167,8 @@ def _block_prune_params(params, prune_ratio=0.25, group_size=32):
and the bottom `prune_ratio` fraction (globally) are zeroed out.
"""
all_scores = []
- for _, leaf in jax.tree_util.tree_leaves_with_path(params):
+ for path, leaf in jax.tree_util.tree_leaves_with_path(params):
+ name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])
if leaf.ndim != 2:
continue
w = np.array(leaf)
@@ -186,6 +187,7 @@ def _block_prune_params(params, prune_ratio=0.25, group_size=32):
threshold = np.percentile(np.concatenate(all_scores), prune_ratio * 100)
def _prune_weight(path, leaf):
+ name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])
if leaf.ndim != 2:
return leaf
w = np.array(leaf)
@@ -222,17 +224,14 @@ def _layer_prune(params, config, layer_prune_ratio=0.25):
enc_scores = _score_blocks(p["encoder"], "encoder")
dec_scores = _score_blocks(p["decoder"], "decoder")
- # Pool all block scores, find global threshold
all_scores = list(enc_scores.values()) + list(dec_scores.values())
if not all_scores:
return params, config
- # Keep at least 1 encoder and 1 decoder block
num_enc = len(enc_scores)
num_dec = len(dec_scores)
total_to_remove = max(0, int(len(all_scores) * layer_prune_ratio))
- # Score and rank all blocks together
tagged = [(s, "encoder", k) for k, s in enc_scores.items()] + \
[(s, "decoder", k) for k, s in dec_scores.items()]
tagged.sort(key=lambda x: x[0])
@@ -282,7 +281,6 @@ def _quantize_params(params, group_size=32):
return jax.tree_util.tree_map_with_path(_maybe_quantize, params)
-# Module-level group_size used by _train_step (set before pmap in train())
_GROUP_SIZE = 32
@@ -295,13 +293,14 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):
tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
cross_mask = src_mask
- logits = state.apply_fn(
+ logits, slot_div = state.apply_fn(
{"params": _quantize_params(params, group_size=_GROUP_SIZE)},
src,
tgt_in,
src_mask=src_mask,
tgt_mask=tgt_mask,
cross_mask=cross_mask,
+ method="forward_with_aux",
)
logits_f32 = logits.astype(jnp.float32)
@@ -309,7 +308,8 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):
mask = (tgt_out != pad_id).astype(jnp.float32)
ce_loss = jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)
z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)
- return ce_loss + z_loss
+ div_loss = 1e-4 * slot_div
+ return ce_loss + z_loss + div_loss
loss, grads = jax.value_and_grad(loss_fn)(state.params)
grads = jax.lax.pmean(grads, axis_name="batch")
@@ -347,7 +347,6 @@ def shard_batch(batch, num_devices):
def train(args):
num_devices = jax.local_device_count()
- print(f"Detected {num_devices} device(s) for data-parallel training")
use_wandb = getattr(args, "wandb", False)
if use_wandb:
@@ -355,31 +354,30 @@ def train(args):
if wandb.run is None:
wandb.init(project="needle-v1", config=vars(args))
- print("Loading tokenizer...")
+ print(f"\n[1/4] Detecting devices...")
+ print(f" {num_devices} device(s) for data-parallel training")
+
+ print(f"\n[2/4] Loading tokenizer...")
tokenizer = get_tokenizer()
- print("Loading TinyStories dataset...")
+ print(f"\n[3/4] Loading TinyStories dataset...")
ds = load_tinystories("train", max_samples=args.max_samples)
val_ds = load_tinystories("validation", max_samples=getattr(args, "max_eval_samples", None))
- print("Preparing encoder-decoder pairs...")
+ print(f"\n[4/4] Preparing encoder-decoder pairs...")
enc_inputs, dec_inputs, dec_targets = prepare_encoder_decoder_pairs(
ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
)
- print(f"Prepared {len(enc_inputs)} training pairs")
-
val_enc, val_dec_in, val_dec_tgt = prepare_encoder_decoder_pairs(
val_ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
)
val_enc = jnp.array(val_enc)
val_dec_in = jnp.array(val_dec_in)
val_dec_tgt = jnp.array(val_dec_tgt)
- print(f"Prepared {len(val_enc)} validation pairs")
+ print(f" {len(enc_inputs):,} train / {len(val_enc):,} val pairs")
- # Effective batch size must be divisible by num_devices
effective_batch_size = args.batch_size * num_devices
- # Load config from checkpoint or build from CLI args
resume_checkpoint = getattr(args, "checkpoint", None)
if resume_checkpoint:
print(f"Resuming from checkpoint: {resume_checkpoint}")
@@ -401,7 +399,6 @@ def train(args):
num_memory_slots=getattr(args, "num_memory_slots", 64),
)
- # Set module-level config (captured by _train_step closure)
global _GROUP_SIZE
_GROUP_SIZE = getattr(args, "group_size", 32)
p_train_step = _make_p_train_step()
@@ -419,7 +416,6 @@ def train(args):
state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps)
val_loss_fn = _make_val_loss_fn(state.apply_fn)
- # Override with checkpoint params if resuming
if resume_checkpoint:
state = state.replace(params=ckpt_params)
print(f" Loaded checkpoint params into train state")
@@ -429,12 +425,26 @@ def train(args):
ema_params = jax_utils.replicate(ema_params)
param_count = sum(x.size for x in jax.tree.leaves(jax_utils.unreplicate(state).params))
- print(f"Model parameters: {param_count:,}")
- print(f"Devices: {num_devices}, per-device batch: {args.batch_size}, effective batch: {effective_batch_size}")
- print(f"Adam LR: {args.lr} x {num_devices} = {scaled_lr}, Muon LR: {args.muon_lr} x sqrt({num_devices}) = {muon_lr:.4f}")
decay_steps = max(1, int(total_steps * 0.15))
stable_steps = total_steps - warmup_steps - decay_steps
- print(f"LR schedule: warmup {warmup_steps}, stable {stable_steps}, decay {decay_steps} steps (WSD)")
+
+ print(f"\n ─────────────────────────────────────")
+ print(f" Parameters {param_count:>12,}")
+ print(f" d_model {config.d_model:>12}")
+ print(f" Heads {config.num_heads:>7} ({config.num_kv_heads} KV)")
+ print(f" Layers {config.num_encoder_layers:>7} enc / {config.num_decoder_layers} dec")
+ print(f" Memory slots {config.num_memory_slots:>12}")
+ print(f" Activation {config.activation:>12}")
+ print(f" Dtype {config.dtype:>12}")
+ print(f" ─────────────────────────────────────")
+ print(f" Devices {num_devices:>12}")
+ print(f" Batch {args.batch_size:>7} x {num_devices} = {effective_batch_size}")
+ print(f" Adam LR {args.lr:>7} x {num_devices} = {scaled_lr}")
+ print(f" Muon LR {args.muon_lr:>7.4f} -> {muon_lr:.4f}")
+ print(f" Schedule {warmup_steps}w / {stable_steps}s / {decay_steps}d (WSD)")
+ print(f" Total steps {total_steps:>12,}")
+ print(f" Epochs {args.epochs:>12}")
+ print(f" ─────────────────────────────────────\n")
os.makedirs(args.checkpoint_dir, exist_ok=True)
global_step = 0
@@ -456,15 +466,10 @@ def train(args):
layer_prune_ratio = getattr(args, "layer_prune_ratio", 0.0)
prune_mask = None
- # Determine which epoch each pruning type fires on:
- # both flags: group sparsify @ epoch 1, layer prune @ epoch 2
- # only --sparsity-ratio: group sparsify @ epoch 1
- # only --layer-prune: layer prune @ epoch 1
weight_prune_epoch = 1 if sparsity_ratio > 0 else -1
layer_prune_epoch = (2 if sparsity_ratio > 0 else 1) if layer_prune_ratio > 0 else -1
for epoch in range(args.epochs):
- # --- Weight block pruning ---
if epoch == weight_prune_epoch and prune_mask is None:
print(f"\nGroup-wise sparsification: {sparsity_ratio*100:.0f}% of blocks (group_size={_GROUP_SIZE})...")
pruned_params = _block_prune_params(jax_utils.unreplicate(ema_params), prune_ratio=sparsity_ratio, group_size=_GROUP_SIZE)
@@ -490,15 +495,13 @@ def train(args):
prune_mask = jax_utils.replicate(prune_mask)
state = jax_utils.replicate(state)
ema_params = jax_utils.replicate(ema_params)
- print(f"Continuing training with sparsity mask locked...\n")
+ print(f"\nContinuing training with sparsity mask locked...\n")
- # --- Layer pruning ---
if epoch == layer_prune_epoch:
print(f"\nLayer pruning: removing {layer_prune_ratio*100:.0f}% of layers...")
current_params = jax_utils.unreplicate(ema_params)
new_params, config = _layer_prune(current_params, config, layer_prune_ratio=layer_prune_ratio)
- # Rebuild train state with new config/model so tree types are consistent
rng, reinit_rng = jax.random.split(rng)
state = create_train_state(reinit_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps)
matched_params = jax.tree.map(lambda init_leaf, prune_leaf: prune_leaf, state.params, new_params)
@@ -508,7 +511,6 @@ def train(args):
param_count = sum(x.size for x in jax.tree.leaves(matched_params))
print(f" Parameters: {param_count:,}")
- # Eval val perplexity after layer pruning
val_loss_fn = _make_val_loss_fn(state.apply_fn)
val_causal = make_causal_mask(args.max_dec_len)
total_loss, total_toks = 0.0, 0.0
@@ -519,14 +521,13 @@ def train(args):
layer_prune_ppl = float(math.exp(total_loss / max(total_toks, 1)))
print(f" Post-layer-prune val ppl: {layer_prune_ppl:.2f} (was {last_val_ppl:.2f})" if last_val_ppl is not None else f" Post-layer-prune val ppl: {layer_prune_ppl:.2f}")
- # Rebuild prune mask if weight pruning was already applied
if prune_mask is not None:
prune_mask = jax.tree.map(lambda w: (jnp.abs(w) > 1e-8).astype(jnp.float32), matched_params)
prune_mask = jax_utils.replicate(prune_mask)
state = jax_utils.replicate(state)
ema_params = jax_utils.replicate(ema_params)
- print(f"Continuing training with {config.num_encoder_layers}+{config.num_decoder_layers} layers...\n")
+ print(f"\nContinuing training with {config.num_encoder_layers}+{config.num_decoder_layers} layers...\n")
losses = []
batches = get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size)
@@ -544,7 +545,6 @@ def train(args):
causal_mask,
)
- # Re-apply prune mask to keep pruned weights at zero
if prune_mask is not None:
state = state.replace(params=jax.tree.map(lambda w, m: w * m, state.params, prune_mask))
ema_params = jax.tree.map(lambda w, m: w * m, ema_params, prune_mask)
@@ -582,7 +582,6 @@ def train(args):
final_loss = losses[-1] if losses else float("nan")
final_ppl = math.exp(final_loss) if not math.isnan(final_loss) else float("nan")
- # Compute val perplexity at end of epoch
eval_params = jax_utils.unreplicate(ema_params)
val_causal = make_causal_mask(args.max_dec_len)
total_loss, total_toks = 0.0, 0.0
@@ -592,7 +591,6 @@ def train(args):
total_toks += float(vt)
last_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))
- # Quantized val perplexity (INT4 fake-quantized weights)
q_params = _quantize_params(eval_params, group_size=_GROUP_SIZE)
q_total_loss, q_total_toks = 0.0, 0.0
for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False):
@@ -606,45 +604,50 @@ def train(args):
near_zero = sum(int(np.sum(np.abs(x) < 1e-6)) for x in jax.tree.leaves(epoch_params))
sparsity = near_zero / total_params * 100
- print(f"\n Epoch {epoch + 1}/{args.epochs} report:")
- print(f" Avg loss: {epoch_avg_loss:.4f}")
- print(f" Final loss: {final_loss:.4f}")
- print(f" Perplexity: {final_ppl:.2f}")
- print(f" Val perplexity: {last_val_ppl:.2f}")
- print(f" Quant val ppl: {quant_val_ppl:.2f} (INT4 g{_GROUP_SIZE})")
- print(f" Weight sparsity: {sparsity:.2f}% ({near_zero:,}/{total_params:,} near-zero)")
-
- if use_wandb:
- wandb.log({
- "epoch/avg_loss": epoch_avg_loss,
- "epoch/final_loss": final_loss,
- "epoch/val_ppl": last_val_ppl,
- "epoch/quant_val_ppl": quant_val_ppl,
- "epoch/weight_sparsity": sparsity,
- "epoch": epoch + 1,
- })
-
ckpt_name = f"needle_{args.num_layers}_{args.d_model}_{global_step}.pkl"
ckpt_path = os.path.join(args.checkpoint_dir, ckpt_name)
params_np = jax.tree.map(np.array, jax_utils.unreplicate(ema_params))
with open(ckpt_path, "wb") as f:
pickle.dump({"params": params_np, "config": config.__dict__}, f)
- print(f"Saved checkpoint: {ckpt_path}")
from .test import measure_throughput, benchmark_generation_quality
model = EncoderDecoderTransformer(config)
-
- print(f"\n Throughput:")
tp = measure_throughput(model, eval_params, tokenizer, num_runs=5)
- print(f" {tp['tokens_per_second']:.1f} tok/s, {tp['avg_latency_s']:.3f}s avg latency")
-
prompts = ["Once upon a time", "The little dog", "She was very happy because"]
- print(f" Generation samples:")
quality = benchmark_generation_quality(model, eval_params, tokenizer, prompts, max_gen_len=64, temperature=0.8)
- print(f" Avg length: {quality['avg_generation_length']:.1f} tok, repetition: {quality['bigram_repetition_rate']:.3f}")
+
+ print(f"\n ─────────────────────────────────────")
+ print(f" Epoch {epoch + 1}/{args.epochs}")
+ print(f" ─────────────────────────────────────")
+ print(f" Avg loss {epoch_avg_loss:>12.4f}")
+ print(f" Final loss {final_loss:>12.4f}")
+ print(f" Train ppl {final_ppl:>12.2f}")
+ print(f" Val ppl {last_val_ppl:>12.2f}")
+ print(f" Quant val ppl {quant_val_ppl:>12.2f} (INT4 g{_GROUP_SIZE})")
+ print(f" Sparsity {sparsity:>11.2f}% ({near_zero:,}/{total_params:,})")
+ print(f" ─────────────────────────────────────")
+ print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")
+ print(f" Latency {tp['avg_latency_s']:>11.3f}s")
+ print(f" Gen length {quality['avg_generation_length']:>10.1f} tok")
+ print(f" Repetition {quality['bigram_repetition_rate']:>12.3f}")
+ print(f" ─────────────────────────────────────")
for prompt, gen in quality["generations"]:
- print(f" [{prompt}] {gen[:100]}")
+ print(f" [{prompt}]")
+ print(f" {gen[:100]}")
+ print(f" ─────────────────────────────────────")
+ print(f" Checkpoint: {ckpt_path}")
+ print(f" ─────────────────────────────────────\n")
+
+ if use_wandb:
+ wandb.log({
+ "epoch/avg_loss": epoch_avg_loss,
+ "epoch/final_loss": final_loss,
+ "epoch/val_ppl": last_val_ppl,
+ "epoch/quant_val_ppl": quant_val_ppl,
+ "epoch/weight_sparsity": sparsity,
+ "epoch": epoch + 1,
+ })
if use_wandb:
wandb.finish()