Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
small vocab size
HenryNdubuakua9334eeaebc4
diff --git a/.gitignore b/.gitignoreindex 9c716c0..8ebc8b4 100644--- a/.gitignore+++ b/.gitignore@@ -88,3 +88,4 @@ __marimo__/checkpoints/wandb/.data_cache/+tokenizer/\ No newline at end of filediff --git a/experiements/architecture.txt b/experiements/architecture.txtnew file mode 100644index 0000000..e69de29diff --git a/experiements/tokenizer.txt b/experiements/tokenizer.txtnew file mode 100644index 0000000..e69de29diff --git a/pyproject.toml b/pyproject.tomlindex aba3ce6..13ebd8e 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -10,6 +10,7 @@ dependencies = ["datasets","transformers","wandb",+ "sentencepiece",][project.optional-dependencies]diff --git a/requirements.txt b/requirements.txtindex 0b2e3f0..4e3232f 100644--- a/requirements.txt+++ b/requirements.txt@@ -6,3 +6,4 @@ datasetstransformerswandbpyyaml+sentencepiecediff --git a/src/data.py b/src/data.pyindex 1c1bb31..4e0f1ed 100644--- a/src/data.py+++ b/src/data.py@@ -8,16 +8,105 @@ logging.getLogger("transformers").setLevel(logging.ERROR)import numpy as npfrom datasets import load_datasetfrom tqdm import tqdm-from transformers import AutoTokenizer+import sentencepiece as spmCACHE_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), ".data_cache")+TOKENIZER_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tokenizer")+TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")++PAD_ID = 0+EOS_ID = 1+BOS_ID = 2+++class NeedleTokenizer:+ """Wrapper around SentencePiece providing the interface the codebase expects."""++ def __init__(self, model_path):+ self.sp = spm.SentencePieceProcessor()+ self.sp.Load(model_path)++ @property+ def pad_token_id(self):+ return PAD_ID++ @property+ def eos_token_id(self):+ return EOS_ID++ @property+ def bos_token_id(self):+ return BOS_ID++ @property+ def vocab_size(self):+ return self.sp.GetPieceSize()++ def encode(self, text):+ return self.sp.Encode(text, out_type=int)++ def decode(self, ids):+ if isinstance(ids, (list, tuple)) and len(ids) > 0 and isinstance(ids[0], (list, tuple)):+ return [self.sp.Decode(seq) for seq in ids]+ return self.sp.Decode(list(ids))++ def __call__(self, texts, truncation=True, max_length=None, **kwargs):+ all_ids = []+ for text in texts:+ ids = self.sp.Encode(text, out_type=int)+ if truncation and max_length:+ ids = ids[:max_length]+ all_ids.append(ids)+ return {"input_ids": all_ids}+++def train_tokenizer(vocab_size=8192, max_samples=100000):+ """Train a SentencePiece BPE tokenizer on TinyStories."""+ model_path = TOKENIZER_PREFIX + ".model"+ if os.path.exists(model_path):+ print(f"Tokenizer already exists at {model_path}")+ return model_path++ os.makedirs(TOKENIZER_DIR, exist_ok=True)++ print(f"Training SentencePiece BPE tokenizer (vocab_size={vocab_size})...")+ ds = load_dataset("roneneldan/TinyStories", split="train")+ indices = range(min(max_samples, len(ds)))+ ds_subset = ds.select(indices)++ corpus_path = os.path.join(TOKENIZER_DIR, "corpus.txt")+ with open(corpus_path, "w") as f:+ for example in tqdm(ds_subset, desc="Writing corpus"):+ text = example["text"].strip()+ if text:+ f.write(text + "\n")++ spm.SentencePieceTrainer.Train(+ input=corpus_path,+ model_prefix=TOKENIZER_PREFIX,+ vocab_size=vocab_size,+ model_type="bpe",+ pad_id=PAD_ID,+ eos_id=EOS_ID,+ bos_id=BOS_ID,+ unk_id=3,+ byte_fallback=True,+ normalization_rule_name="identity",+ num_threads=os.cpu_count(),+ train_extremely_large_corpus=False,+ minloglevel=2,+ )++ os.remove(corpus_path)+ print(f"Tokenizer saved to {model_path}")+ return model_pathdef get_tokenizer():- tokenizer = AutoTokenizer.from_pretrained("LiquidAI/LFM2-1.2B")- if tokenizer.pad_token is None:- tokenizer.pad_token = tokenizer.eos_token- return tokenizer+ model_path = TOKENIZER_PREFIX + ".model"+ if not os.path.exists(model_path):+ train_tokenizer()+ return NeedleTokenizer(model_path)def load_tinystories(split="train", max_samples=None):@@ -27,13 +116,13 @@ def load_tinystories(split="train", max_samples=None):return ds-def _cache_key(n_samples, max_enc_len, max_dec_len, tokenizer_name="lfm2"):+def _cache_key(n_samples, max_enc_len, max_dec_len, tokenizer_name="needle_spm"):key = f"tinystories_{tokenizer_name}_{n_samples}_{max_enc_len}_{max_dec_len}"return hashlib.md5(key.encode()).hexdigest()[:12]def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):-+cache_id = _cache_key(len(ds), max_enc_len, max_dec_len)cache_path = os.path.join(CACHE_DIR, cache_id)if os.path.exists(cache_path + "_enc.npy"):@@ -52,7 +141,6 @@ def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=12texts = list(ds["text"])all_tokens = tokenizer(texts, truncation=True, max_length=max_total,- return_attention_mask=False, return_token_type_ids=False,)["input_ids"]n = len(all_tokens)diff --git a/src/model.py b/src/model.pyindex da70255..fe0c995 100644--- a/src/model.py+++ b/src/model.py@@ -31,7 +31,7 @@ class ZCRMSNorm(nn.Module):@dataclassclass TransformerConfig:- vocab_size: int = 65536+ vocab_size: int = 8192d_model: int = 128num_heads: int = 4num_kv_heads: int = 2@@ -77,6 +77,7 @@ class MultiHeadAttention(nn.Module):d_model: intnum_layers: intdtype: jnp.dtype = jnp.bfloat16+ rope_keys_only: bool = False@nn.compactdef __call__(self, q_input, kv_input, mask=None, rope=None):@@ -102,7 +103,8 @@ class MultiHeadAttention(nn.Module):if rope is not None:cos, sin = rope- q = apply_rope(q, cos, sin)+ if not self.rope_keys_only:+ q = apply_rope(q, cos, sin)k = apply_rope(k, cos, sin)scale = jnp.sqrt(jnp.float32(head_dim))@@ -138,33 +140,81 @@ class FeedForward(nn.Module):return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)-class EncoderBlock(nn.Module):+class MLPMixer(nn.Module):+ """MLP-Mixer operating on fixed-size memory slots (B, M, d)."""+ num_slots: int+ d_model: int+ d_ff: int+ dtype: jnp.dtype = jnp.bfloat16+ activation: str = "drelu"++ @nn.compact+ def __call__(self, s):+ residual = s+ s = ZCRMSNorm(dtype=self.dtype, name="token_mix_norm")(s)+ s = s.transpose(0, 2, 1)+ gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_gate")(s)+ up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_up")(s)+ if self.activation == "swiglu":+ s = nn.silu(gate) * up+ elif self.activation == "geglu":+ s = nn.gelu(gate) * up+ else:+ s = nn.relu(gate) * nn.relu(up)+ s = nn.Dense(self.num_slots, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="token_mix_down")(s)+ s = s.transpose(0, 2, 1)+ s = s + residual++ residual = s+ s = ZCRMSNorm(dtype=self.dtype, name="channel_mix_norm")(s)+ gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_gate")(s)+ up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_up")(s)+ if self.activation == "swiglu":+ s = nn.silu(gate) * up+ elif self.activation == "geglu":+ s = nn.gelu(gate) * up+ else:+ s = nn.relu(gate) * nn.relu(up)+ s = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="channel_mix_down")(s)+ s = s + residual++ return s+++class MemoryMixerBlock(nn.Module):+ """Block 4a: Pack (cross-attn S←X with RoPE on keys) → Mix (MLP-Mixer) → Local Update (MLP)."""num_heads: intnum_kv_heads: intd_model: intd_ff: int+ num_slots: intnum_layers: intdtype: jnp.dtype = jnp.bfloat16activation: str = "drelu"@nn.compact- def __call__(self, x, mask=None, rope=None):- residual = x- x = ZCRMSNorm(dtype=self.dtype)(x)- x = MultiHeadAttention(self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype)(- x, x, mask=mask, rope=rope+ def __call__(self, x, s, mask=None, rope=None):+ s_norm = ZCRMSNorm(dtype=self.dtype, name="pack_s_norm")(s)+ x_norm = ZCRMSNorm(dtype=self.dtype, name="pack_x_norm")(x)+ s = s + MultiHeadAttention(+ self.num_heads, self.num_kv_heads, self.d_model, self.num_layers, self.dtype,+ rope_keys_only=True, name="pack_attn"+ )(s_norm, x_norm, mask=mask, rope=rope)++ s = s + MLPMixer(self.num_slots, self.d_model, self.d_ff, self.dtype, self.activation, name="mixer")(+ ZCRMSNorm(dtype=self.dtype, name="mix_norm")(s))- x = x + residualresidual = x- x = ZCRMSNorm(dtype=self.dtype)(x)- x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation)(x)+ x = ZCRMSNorm(dtype=self.dtype, name="local_norm")(x)+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dtype, self.activation, name="local_ffn")(x)x = x + residual- return x+ return x, s-class Encoder(nn.Module):+class MemoryMixerEncoder(nn.Module):+ """Encoder using MemoryMixer blocks. Output is the final memory slots S."""config: TransformerConfig@nn.compact@@ -173,13 +223,19 @@ class Encoder(nn.Module):dt = cfg.jax_dtypex = x.astype(dt)+ s_base = self.param("memory_slots", jinit.normal(stddev=0.02), (1, cfg.num_memory_slots, cfg.d_model))+ x_pool = jnp.mean(x, axis=1) # (B, d)+ s_bias = nn.Dense(cfg.d_model, dtype=dt, use_bias=False, kernel_init=jinit.zeros, name="slot_init")(x_pool)+ s = jnp.broadcast_to(s_base.astype(dt), (x.shape[0], cfg.num_memory_slots, cfg.d_model)) + s_bias[:, None, :]+for i in range(cfg.num_encoder_layers):- x = EncoderBlock(- cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"- )(x, mask=mask, rope=rope)+ x, s = MemoryMixerBlock(+ cfg.num_heads, cfg.num_kv_heads, cfg.d_model, cfg.d_ff,+ cfg.num_memory_slots, cfg.total_layers, dt, cfg.activation, name=f"block_{i}"+ )(x, s, mask=mask, rope=rope)- x = ZCRMSNorm(dtype=dt)(x)- return x+ s = ZCRMSNorm(dtype=dt, name="final_norm")(s)+ return sclass DecoderBlock(nn.Module):@@ -241,7 +297,7 @@ class EncoderDecoderTransformer(nn.Module):def setup(self):self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))self.embed_scale = math.sqrt(self.config.d_model)- self.encoder = Encoder(self.config)+ self.encoder = MemoryMixerEncoder(self.config)self.decoder = Decoder(self.config)def _rope(self, seq_len):@@ -257,7 +313,7 @@ class EncoderDecoderTransformer(nn.Module):x = self.embedding(tgt) * self.embed_scalerope = self._rope(tgt.shape[1])x = self.decoder(- x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope+ x, encoder_out, self_mask=self_mask, cross_mask=None, rope=rope)logits = x.astype(jnp.float32) @ self.embedding.embedding.Treturn logits@@ -265,10 +321,19 @@ class EncoderDecoderTransformer(nn.Module):def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):encoder_out = self.encode(src, src_mask=src_mask)logits = self.decode(- tgt, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask+ tgt, encoder_out, self_mask=tgt_mask)return logits+ def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):+ encoder_out = self.encode(src, src_mask=src_mask)+ logits = self.decode(tgt, encoder_out, self_mask=tgt_mask)+ s = encoder_out.astype(jnp.float32)+ gram = jnp.matmul(s, s.transpose(0, 2, 1))+ diag_sq = jnp.sum(jnp.diagonal(gram, axis1=1, axis2=2) ** 2)+ slot_div = (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]+ return logits, slot_div+def make_causal_mask(seq_len):mask = jnp.tril(jnp.ones((seq_len, seq_len), dtype=jnp.bool_))diff --git a/src/train.py b/src/train.pyindex 08b8aa7..a422122 100644--- a/src/train.py+++ b/src/train.py@@ -167,7 +167,8 @@ def _block_prune_params(params, prune_ratio=0.25, group_size=32):and the bottom `prune_ratio` fraction (globally) are zeroed out."""all_scores = []- for _, leaf in jax.tree_util.tree_leaves_with_path(params):+ for path, leaf in jax.tree_util.tree_leaves_with_path(params):+ name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])if leaf.ndim != 2:continuew = np.array(leaf)@@ -186,6 +187,7 @@ def _block_prune_params(params, prune_ratio=0.25, group_size=32):threshold = np.percentile(np.concatenate(all_scores), prune_ratio * 100)def _prune_weight(path, leaf):+ name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])if leaf.ndim != 2:return leafw = np.array(leaf)@@ -222,17 +224,14 @@ def _layer_prune(params, config, layer_prune_ratio=0.25):enc_scores = _score_blocks(p["encoder"], "encoder")dec_scores = _score_blocks(p["decoder"], "decoder")- # Pool all block scores, find global thresholdall_scores = list(enc_scores.values()) + list(dec_scores.values())if not all_scores:return params, config- # Keep at least 1 encoder and 1 decoder blocknum_enc = len(enc_scores)num_dec = len(dec_scores)total_to_remove = max(0, int(len(all_scores) * layer_prune_ratio))- # Score and rank all blocks togethertagged = [(s, "encoder", k) for k, s in enc_scores.items()] + \[(s, "decoder", k) for k, s in dec_scores.items()]tagged.sort(key=lambda x: x[0])@@ -282,7 +281,6 @@ def _quantize_params(params, group_size=32):return jax.tree_util.tree_map_with_path(_maybe_quantize, params)-# Module-level group_size used by _train_step (set before pmap in train())_GROUP_SIZE = 32@@ -295,13 +293,14 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)cross_mask = src_mask- logits = state.apply_fn(+ logits, slot_div = state.apply_fn({"params": _quantize_params(params, group_size=_GROUP_SIZE)},src,tgt_in,src_mask=src_mask,tgt_mask=tgt_mask,cross_mask=cross_mask,+ method="forward_with_aux",)logits_f32 = logits.astype(jnp.float32)@@ -309,7 +308,8 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):mask = (tgt_out != pad_id).astype(jnp.float32)ce_loss = jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)- return ce_loss + z_loss+ div_loss = 1e-4 * slot_div+ return ce_loss + z_loss + div_lossloss, grads = jax.value_and_grad(loss_fn)(state.params)grads = jax.lax.pmean(grads, axis_name="batch")@@ -347,7 +347,6 @@ def shard_batch(batch, num_devices):def train(args):num_devices = jax.local_device_count()- print(f"Detected {num_devices} device(s) for data-parallel training")use_wandb = getattr(args, "wandb", False)if use_wandb:@@ -355,31 +354,30 @@ def train(args):if wandb.run is None:wandb.init(project="needle-v1", config=vars(args))- print("Loading tokenizer...")+ print(f"\n[1/4] Detecting devices...")+ print(f" {num_devices} device(s) for data-parallel training")++ print(f"\n[2/4] Loading tokenizer...")tokenizer = get_tokenizer()- print("Loading TinyStories dataset...")+ print(f"\n[3/4] Loading TinyStories dataset...")ds = load_tinystories("train", max_samples=args.max_samples)val_ds = load_tinystories("validation", max_samples=getattr(args, "max_eval_samples", None))- print("Preparing encoder-decoder pairs...")+ print(f"\n[4/4] Preparing encoder-decoder pairs...")enc_inputs, dec_inputs, dec_targets = prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len)- print(f"Prepared {len(enc_inputs)} training pairs")-val_enc, val_dec_in, val_dec_tgt = prepare_encoder_decoder_pairs(val_ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len)val_enc = jnp.array(val_enc)val_dec_in = jnp.array(val_dec_in)val_dec_tgt = jnp.array(val_dec_tgt)- print(f"Prepared {len(val_enc)} validation pairs")+ print(f" {len(enc_inputs):,} train / {len(val_enc):,} val pairs")- # Effective batch size must be divisible by num_deviceseffective_batch_size = args.batch_size * num_devices- # Load config from checkpoint or build from CLI argsresume_checkpoint = getattr(args, "checkpoint", None)if resume_checkpoint:print(f"Resuming from checkpoint: {resume_checkpoint}")@@ -401,7 +399,6 @@ def train(args):num_memory_slots=getattr(args, "num_memory_slots", 64),)- # Set module-level config (captured by _train_step closure)global _GROUP_SIZE_GROUP_SIZE = getattr(args, "group_size", 32)p_train_step = _make_p_train_step()@@ -419,7 +416,6 @@ def train(args):state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps)val_loss_fn = _make_val_loss_fn(state.apply_fn)- # Override with checkpoint params if resumingif resume_checkpoint:state = state.replace(params=ckpt_params)print(f" Loaded checkpoint params into train state")@@ -429,12 +425,26 @@ def train(args):ema_params = jax_utils.replicate(ema_params)param_count = sum(x.size for x in jax.tree.leaves(jax_utils.unreplicate(state).params))- print(f"Model parameters: {param_count:,}")- print(f"Devices: {num_devices}, per-device batch: {args.batch_size}, effective batch: {effective_batch_size}")- print(f"Adam LR: {args.lr} x {num_devices} = {scaled_lr}, Muon LR: {args.muon_lr} x sqrt({num_devices}) = {muon_lr:.4f}")decay_steps = max(1, int(total_steps * 0.15))stable_steps = total_steps - warmup_steps - decay_steps- print(f"LR schedule: warmup {warmup_steps}, stable {stable_steps}, decay {decay_steps} steps (WSD)")++ print(f"\n ─────────────────────────────────────")+ print(f" Parameters {param_count:>12,}")+ print(f" d_model {config.d_model:>12}")+ print(f" Heads {config.num_heads:>7} ({config.num_kv_heads} KV)")+ print(f" Layers {config.num_encoder_layers:>7} enc / {config.num_decoder_layers} dec")+ print(f" Memory slots {config.num_memory_slots:>12}")+ print(f" Activation {config.activation:>12}")+ print(f" Dtype {config.dtype:>12}")+ print(f" ─────────────────────────────────────")+ print(f" Devices {num_devices:>12}")+ print(f" Batch {args.batch_size:>7} x {num_devices} = {effective_batch_size}")+ print(f" Adam LR {args.lr:>7} x {num_devices} = {scaled_lr}")+ print(f" Muon LR {args.muon_lr:>7.4f} -> {muon_lr:.4f}")+ print(f" Schedule {warmup_steps}w / {stable_steps}s / {decay_steps}d (WSD)")+ print(f" Total steps {total_steps:>12,}")+ print(f" Epochs {args.epochs:>12}")+ print(f" ─────────────────────────────────────\n")os.makedirs(args.checkpoint_dir, exist_ok=True)global_step = 0@@ -456,15 +466,10 @@ def train(args):layer_prune_ratio = getattr(args, "layer_prune_ratio", 0.0)prune_mask = None- # Determine which epoch each pruning type fires on:- # both flags: group sparsify @ epoch 1, layer prune @ epoch 2- # only --sparsity-ratio: group sparsify @ epoch 1- # only --layer-prune: layer prune @ epoch 1weight_prune_epoch = 1 if sparsity_ratio > 0 else -1layer_prune_epoch = (2 if sparsity_ratio > 0 else 1) if layer_prune_ratio > 0 else -1for epoch in range(args.epochs):- # --- Weight block pruning ---if epoch == weight_prune_epoch and prune_mask is None:print(f"\nGroup-wise sparsification: {sparsity_ratio*100:.0f}% of blocks (group_size={_GROUP_SIZE})...")pruned_params = _block_prune_params(jax_utils.unreplicate(ema_params), prune_ratio=sparsity_ratio, group_size=_GROUP_SIZE)@@ -490,15 +495,13 @@ def train(args):prune_mask = jax_utils.replicate(prune_mask)state = jax_utils.replicate(state)ema_params = jax_utils.replicate(ema_params)- print(f"Continuing training with sparsity mask locked...\n")+ print(f"\nContinuing training with sparsity mask locked...\n")- # --- Layer pruning ---if epoch == layer_prune_epoch:print(f"\nLayer pruning: removing {layer_prune_ratio*100:.0f}% of layers...")current_params = jax_utils.unreplicate(ema_params)new_params, config = _layer_prune(current_params, config, layer_prune_ratio=layer_prune_ratio)- # Rebuild train state with new config/model so tree types are consistentrng, reinit_rng = jax.random.split(rng)state = create_train_state(reinit_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps)matched_params = jax.tree.map(lambda init_leaf, prune_leaf: prune_leaf, state.params, new_params)@@ -508,7 +511,6 @@ def train(args):param_count = sum(x.size for x in jax.tree.leaves(matched_params))print(f" Parameters: {param_count:,}")- # Eval val perplexity after layer pruningval_loss_fn = _make_val_loss_fn(state.apply_fn)val_causal = make_causal_mask(args.max_dec_len)total_loss, total_toks = 0.0, 0.0@@ -519,14 +521,13 @@ def train(args):layer_prune_ppl = float(math.exp(total_loss / max(total_toks, 1)))print(f" Post-layer-prune val ppl: {layer_prune_ppl:.2f} (was {last_val_ppl:.2f})" if last_val_ppl is not None else f" Post-layer-prune val ppl: {layer_prune_ppl:.2f}")- # Rebuild prune mask if weight pruning was already appliedif prune_mask is not None:prune_mask = jax.tree.map(lambda w: (jnp.abs(w) > 1e-8).astype(jnp.float32), matched_params)prune_mask = jax_utils.replicate(prune_mask)state = jax_utils.replicate(state)ema_params = jax_utils.replicate(ema_params)- print(f"Continuing training with {config.num_encoder_layers}+{config.num_decoder_layers} layers...\n")+ print(f"\nContinuing training with {config.num_encoder_layers}+{config.num_decoder_layers} layers...\n")losses = []batches = get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size)@@ -544,7 +545,6 @@ def train(args):causal_mask,)- # Re-apply prune mask to keep pruned weights at zeroif prune_mask is not None:state = state.replace(params=jax.tree.map(lambda w, m: w * m, state.params, prune_mask))ema_params = jax.tree.map(lambda w, m: w * m, ema_params, prune_mask)@@ -582,7 +582,6 @@ def train(args):final_loss = losses[-1] if losses else float("nan")final_ppl = math.exp(final_loss) if not math.isnan(final_loss) else float("nan")- # Compute val perplexity at end of epocheval_params = jax_utils.unreplicate(ema_params)val_causal = make_causal_mask(args.max_dec_len)total_loss, total_toks = 0.0, 0.0@@ -592,7 +591,6 @@ def train(args):total_toks += float(vt)last_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))- # Quantized val perplexity (INT4 fake-quantized weights)q_params = _quantize_params(eval_params, group_size=_GROUP_SIZE)q_total_loss, q_total_toks = 0.0, 0.0for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False):@@ -606,45 +604,50 @@ def train(args):near_zero = sum(int(np.sum(np.abs(x) < 1e-6)) for x in jax.tree.leaves(epoch_params))sparsity = near_zero / total_params * 100- print(f"\n Epoch {epoch + 1}/{args.epochs} report:")- print(f" Avg loss: {epoch_avg_loss:.4f}")- print(f" Final loss: {final_loss:.4f}")- print(f" Perplexity: {final_ppl:.2f}")- print(f" Val perplexity: {last_val_ppl:.2f}")- print(f" Quant val ppl: {quant_val_ppl:.2f} (INT4 g{_GROUP_SIZE})")- print(f" Weight sparsity: {sparsity:.2f}% ({near_zero:,}/{total_params:,} near-zero)")-- if use_wandb:- wandb.log({- "epoch/avg_loss": epoch_avg_loss,- "epoch/final_loss": final_loss,- "epoch/val_ppl": last_val_ppl,- "epoch/quant_val_ppl": quant_val_ppl,- "epoch/weight_sparsity": sparsity,- "epoch": epoch + 1,- })-ckpt_name = f"needle_{args.num_layers}_{args.d_model}_{global_step}.pkl"ckpt_path = os.path.join(args.checkpoint_dir, ckpt_name)params_np = jax.tree.map(np.array, jax_utils.unreplicate(ema_params))with open(ckpt_path, "wb") as f:pickle.dump({"params": params_np, "config": config.__dict__}, f)- print(f"Saved checkpoint: {ckpt_path}")from .test import measure_throughput, benchmark_generation_qualitymodel = EncoderDecoderTransformer(config)-- print(f"\n Throughput:")tp = measure_throughput(model, eval_params, tokenizer, num_runs=5)- print(f" {tp['tokens_per_second']:.1f} tok/s, {tp['avg_latency_s']:.3f}s avg latency")-prompts = ["Once upon a time", "The little dog", "She was very happy because"]- print(f" Generation samples:")quality = benchmark_generation_quality(model, eval_params, tokenizer, prompts, max_gen_len=64, temperature=0.8)- print(f" Avg length: {quality['avg_generation_length']:.1f} tok, repetition: {quality['bigram_repetition_rate']:.3f}")++ print(f"\n ─────────────────────────────────────")+ print(f" Epoch {epoch + 1}/{args.epochs}")+ print(f" ─────────────────────────────────────")+ print(f" Avg loss {epoch_avg_loss:>12.4f}")+ print(f" Final loss {final_loss:>12.4f}")+ print(f" Train ppl {final_ppl:>12.2f}")+ print(f" Val ppl {last_val_ppl:>12.2f}")+ print(f" Quant val ppl {quant_val_ppl:>12.2f} (INT4 g{_GROUP_SIZE})")+ print(f" Sparsity {sparsity:>11.2f}% ({near_zero:,}/{total_params:,})")+ print(f" ─────────────────────────────────────")+ print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")+ print(f" Latency {tp['avg_latency_s']:>11.3f}s")+ print(f" Gen length {quality['avg_generation_length']:>10.1f} tok")+ print(f" Repetition {quality['bigram_repetition_rate']:>12.3f}")+ print(f" ─────────────────────────────────────")for prompt, gen in quality["generations"]:- print(f" [{prompt}] {gen[:100]}")+ print(f" [{prompt}]")+ print(f" {gen[:100]}")+ print(f" ─────────────────────────────────────")+ print(f" Checkpoint: {ckpt_path}")+ print(f" ─────────────────────────────────────\n")++ if use_wandb:+ wandb.log({+ "epoch/avg_loss": epoch_avg_loss,+ "epoch/final_loss": final_loss,+ "epoch/val_ppl": last_val_ppl,+ "epoch/quant_val_ppl": quant_val_ppl,+ "epoch/weight_sparsity": sparsity,+ "epoch": epoch + 1,+ })if use_wandb:wandb.finish()