Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
training
HenryNdubuakufd57d5f499a6
diff --git a/README.md b/README.mdindex 547d7bb..22ef593 100644--- a/README.md+++ b/README.md@@ -12,47 +12,54 @@┌─────────────┐│ Softmax │ ┌──────────────────────┐- └──────┬──────┘ │ Forward pass with │- ┌──────┴──────┐ │ INT4 fake-quantized │- │ Linear (T) │ ← tied │ weights (g=32, STE) │- └──────┬──────┘ └──────────┬───────────┘+ └──────┬──────┘ │ Forward with MRL │+ ┌─────┬──────┴──────┬─────┐ │ loss at dims │+ │ @E[:d'] for each d' │ │ {512,256,128,64} │+ │ in mrl_dims │ │ + INT4 QAT (g=32) │+ └─────┬──────┬──────┬─────┘ └──────────┬───────────┘┌──────┴──────┐ │- │ ZCRMSNorm │ ┌──────────┴───────────┐+ │ Linear (T) │ ← tied ┌──────────┴───────────┐└──────┬──────┘ │ Muon (2D kernels) │- ┌─────────┴─────────┐ │ AdamW (everything │- │ Decoder x N │ │ else) │- │ ┌───────────────┐ │ │ WSD LR schedule │- │ │ Masked Self │ │ └──────────┬───────────┘- │ │ Attn + RoPE │ │ │- │ ├───────────────┤ │ ┌──────────┴───────────┐- ┌──────────────┐ S │ │ Cross │ │ │ EMA params (β=0.999) │- │ MemoryMixer │─────────▶ Attention │ │ └──────────┬───────────┘- │ Encoder x N │ │ ├───────────────┤ │ │- │ │ │ │ Feed-Forward │ │ ┌──────────┴───────────┐- │ ┌──────────┐ │ │ │ (dReLU) │ │ │ Block Prune │- │ │Pack: │ │ │ └───────────────┘ │ │ after epoch 1 │- │ │ S←X Attn │ │ └─────────┬─────────┘ │ group magnitude │- │ ├──────────┤ │ ┌──────┴──────┐ │ lock sparsity mask │- │ │Mix: │ │ │ Embedding │ ← shared └──────────┬───────────┘- │ │ MLP-Mixer│ │ └──────┬──────┘ │- │ │ on S │ │ │ ┌──────────┴───────────┐- │ ├──────────┤ │ ┌─────┴─────┐ │ Layer Prune │- │ │Local: │ │ │ Decoder │ │ after epoch 2 │- │ │ FFN on X │ │ │ Input │ │ L1 block scoring │- │ └──────────┘ │ └───────────┘ │ keep ≥1 enc & dec │- │ S ∈ (M, d) │ └──────────┬───────────┘- └──────┬───────┘ │- ┌──────┴───────┐ ┌──────────┴───────────┐- │ Embedding │ ← shared │ Checkpoint │- └──────┬───────┘ │ sparse + INT4 │- │ └──────────────────────┘+ ┌──────┴──────┐ │ AdamW (everything │+ │ ZCRMSNorm │ │ else) │+ └──────┬──────┘ │ WSD LR schedule │+ ┌─────────┴─────────┐ └──────────┬───────────┘+ │ Decoder x N_dec │ │+ │ ┌───────────────┐ │ ┌──────────┴───────────┐+ │ │ Masked Self │ │ │ EMA params (β=0.999) │+ │ │ Attn + RoPE │ │ └──────────┬───────────┘+ │ ├───────────────┤ │ │+ ┌──────────────┐ S │ │ Cross │ │ ┌──────────┴───────────┐+ │ MemoryMixer │─────────▶ Attention │ │ │ Block Prune │+ │ Encoder │ │ ├───────────────┤ │ │ after epoch 1 │+ │ x N_enc │ │ │ Feed-Forward │ │ │ group magnitude │+ │ │ │ │ (dReLU) │ │ │ lock sparsity mask │+ │ ┌──────────┐ │ │ └───────────────┘ │ └──────────┬───────────┘+ │ │Pack: │ │ └─────────┬─────────┘ │+ │ │ S←X Attn │ │ ┌──────┴──────┐ ┌──────────┴───────────┐+ │ │ RoPE keys│ │ │ Embedding │ ← shared │ Layer Prune │+ │ ├──────────┤ │ └──────┬──────┘ │ after epoch 2 │+ │ │Mix: │ │ │ │ L1 block scoring │+ │ │ MLP-Mixer│ │ ┌─────┴─────┐ │ keep ≥1 enc & dec │+ │ │ on S │ │ │ Decoder │ └──────────┬───────────┘+ │ ├──────────┤ │ │ Input │ │+ │ │Local: │ │ └───────────┘ ┌──────────┴───────────┐+ │ │ FFN on X │ │ │ Checkpoint │+ │ └──────────┘ │ │ MRL + sparse + INT4 │+ │ S ∈ (M, d) │ └──────────────────────┘+ └──────┬───────┘+ ┌──────┴───────┐+ │ Embedding │ ← shared+ └──────┬───────┘+ │┌────┴────┐│ Encoder ││ Input │└─────────┘- d=128 · 4 heads · 2 KV heads · 64 memory slots- dReLU · ZCRMSNorm · RoPE · INT4 QAT · Muon+ d=max(mrl_dims) · 4 heads · 2 KV heads · 64 memory slots+ SentencePiece BPE (8192) · dReLU · ZCRMSNorm · RoPE+ Matryoshka dims · INT4 QAT · Muon + AdamW```## Usage@@ -67,45 +74,66 @@ needle [command]┌───────────────────────────────────────────────────────────────────┐│ ││ train │- │ --epochs INT Training epochs (default: 1) │- │ --batch-size INT Batch size (default: 32) │- │ --lr FLOAT AdamW learning rate (default: 3e-4) │- │ --muon-lr FLOAT Muon learning rate (default: 0.02) │- │ --d-model INT Model dimension (default: 128) │- │ --num-heads INT Attention heads (default: 4) │- │ --num-layers INT Encoder/decoder layers (default: 2) │- │ --max-enc-len INT Max encoder seq length (default: 128) │- │ --max-dec-len INT Max decoder seq length (default: 128) │- │ --max-samples INT Training samples (default: 20000) │- │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │- │ --wandb Enable W&B logging │- │ --checkpoint-dir DIR Checkpoint directory │- │ --seed INT Random seed (default: 42) │+ │ --toy Use toy config for quick iteration │+ │ --epochs INT Training epochs (default: 3) │+ │ --batch-size INT Batch size (default: 32) │+ │ --lr FLOAT AdamW learning rate (default: 3e-4) │+ │ --muon-lr FLOAT Muon learning rate (default: 0.02) │+ │ --d-model INT Model dim (default: max of mrl-dims) │+ │ --num-heads INT Attention heads (default: 4) │+ │ --num-layers INT Encoder layers (default: 12) │+ │ --num-dec-layers INT Decoder layers (default: 4) │+ │ --max-enc-len INT Max encoder seq length (default: 256) │+ │ --max-dec-len INT Max decoder seq length (default: 256) │+ │ --max-samples INT Training samples (default: 1000000) │+ │ --mrl-dims INT [...] MRL dim targets (default: 512 256 128) │+ │ --sparsity-ratio FLOAT Block prune ratio (default: 0.33) │+ │ --layer-prune-ratio FL Layer prune ratio (default: 0.0) │+ │ --group-size INT Quant/prune group size (default: 32) │+ │ --activation STR drelu|swiglu|geglu (default: drelu) │+ │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │+ │ --eval-every INT Val eval interval (default: 1000) │+ │ --wandb Enable W&B logging │+ │ --checkpoint PATH Resume from checkpoint │+ │ --checkpoint-dir DIR Checkpoint directory │+ │ --seed INT Random seed (default: 42) ││ ││ sweep │- │ --sweep-config PATH Sweep YAML config │- │ --project STR W&B project name (default: needle-v1) │- │ --count INT Number of trials (default: 20) │+ │ --sweep-config PATH Sweep YAML config │+ │ --project STR W&B project name (default: needle-v1) │+ │ --count INT Number of trials (default: 20) ││ ││ run │- │ --checkpoint PATH Path to model checkpoint (required) │- │ --prompts STR [...] One or more prompts to continue │- │ --max-len INT Max tokens to generate (default: 128) │- │ --temperature FLOAT Sampling temperature (default: 0.8) │- │ --seed INT Random seed (default: 0) │+ │ --checkpoint PATH Path to model checkpoint (required) │+ │ --prompts STR [...] One or more prompts to continue │+ │ --max-len INT Max tokens to generate (default: 128) │+ │ --temperature FLOAT Sampling temperature (default: 0.8) │+ │ --seed INT Random seed (default: 0) ││ ││ test │- │ --checkpoint PATH Path to model checkpoint (required) │- │ --batch-size INT Batch size (default: 32) │- │ --max-eval-samples INT Evaluation samples (default: 1000) │- │ --max-gen-len INT Max generation length (default: 128) │- │ --temperature FLOAT Sampling temperature (default: 0.8) │- │ --throughput-runs INT Throughput runs (default: 10) │+ │ --checkpoint PATH Path to model checkpoint (required) │+ │ --batch-size INT Batch size (default: 32) │+ │ --max-eval-samples INT Evaluation samples (default: 1000) │+ │ --max-gen-len INT Max generation length (default: 128) │+ │ --temperature FLOAT Sampling temperature (default: 0.8) │+ │ --throughput-runs INT Throughput runs (default: 10) ││ ││ evaluate │- │ --checkpoint PATH Path to model checkpoint (required) │- │ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │- │ --max-samples INT Samples per benchmark (default: 500) │+ │ --checkpoint PATH Path to model checkpoint (required) │+ │ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │+ │ --max-samples INT Samples per benchmark (default: 500) │+ │ │+ │ tpu │+ │ create NAME Create TPU (auto-finds zone) │+ │ --type STR Accelerator (default: v6e-4) │+ │ --version STR TPU OS (auto-detected from --type) │+ │ connect NAME SSH config + connect (auto-zone) │+ │ claude NAME Install Claude Code on instance │+ │ stop NAME Stop instance (auto-zone) │+ │ start NAME Start stopped instance (auto-zone) │+ │ delete NAME Delete instance (auto-zone) │+ │ list List all TPU instances │+ │ --zone ZONE Override auto-detected zone ││ │└───────────────────────────────────────────────────────────────────┘```@@ -113,37 +141,45 @@ needle [command]## TPU Factsheet```-┌────────────────────┬───────────────────┬──────────────────────┬────────────────────────────┐-│ │ v5e │ v5p │ v6e (Trillium) │-├────────────────────┼───────────────────┼──────────────────────┼────────────────────────────┤-│ Optimized for │ Train + inference │ Training (max perf) │ Train + inference │-│ HBM per chip │ 16 GB │ 95 GB │ 32 GB │-│ FLOPS (BF16) │ 197 TFLOPS │ 459 TFLOPS │ 918 TFLOPS │-│ HBM bandwidth │ 819 GB/s │ 2,765 GB/s │ 1,640 GB/s │-│ ICI bandwidth │ 1,600 Gbps │ 4,800 Gbps │ 3,584 Gbps │-│ On-demand/chip/hr │ $1.20 │ $4.20 │ $2.70 │-│ Spot/chip/hr │ $0.60 │ $2.10 │ $1.35 │-│ Perf per $ │ 1x (baseline) │ 0.5x │ 2x │-├────────────────────┼───────────────────┴──────────────────────┴────────────────────────────┤-│ │ │-│ DATASET │ │-│ Text │ 100B tokens │-│ Audio │ 200k × 20s = ~200M audio tokens + ~13M transcription tokens │-│ Effective total │ ~100.5B equivalent tokens (audio has ~2-3x encoder overhead) │-│ Storage │ ~5 GB audio (compressed) + ~400 GB text corpus │-│ │ │-├────────────────────┼───────────────────┬──────────────────────┬─────────────┬──────────────┤-│ 300M multimodal │ v5litepod-4 │ v6e-4 │ v6e-8 │ v6e-16 │-├────────────────────┼───────────────────┼──────────────────────┼─────────────┼──────────────┤-│ Chips │ 4 │ 4 │ 8 │ 16 │-│ Total HBM │ 64 GB │ 128 GB │ 256 GB │ 512 GB │-│ Est. time │ ~16-21 days │ ~4-5 days │ ~2-3 days │ ~1-1.5 days │-│ Spot $/hr │ $2.40 │ $5.40 │ $10.80 │ $21.60 │-│ Est. total cost │ ~$900-1,200 │ ~$550-700 │ ~$550-750 │ ~$550-750 │-└────────────────────┴───────────────────┴──────────────────────┴─────────────┴──────────────┘+ Trillium (v6e)+ ──────────────────────────────────────────+ HBM per chip 32 GB+ BF16 FLOPS 918 TFLOPS+ HBM bandwidth 1,640 GB/s+ ICI bandwidth 3,584 Gbps+ On-demand/chip/hr $2.70 (US regions)+ ──────────────────────────────────────────++ Dataset+ ──────────────────────────────────────────+ Text 100B tokens (~400 GB)+ Audio Emilia subset+ 200k × 20s clips (~1.1k hrs)+ ~200M audio tokens+ ~13M transcription tokens+ ~27 GB (from ~4.5 TB full dataset)+ ──────────────────────────────────────────++ 300M multimodal training estimates+ ──────────────────────────────────────────+ Total FLOPs 6 × 300M × 100B = 1.8e20+ MFU (300M model) ~15% → ~138 effective TFLOPS/chip+ Audio overhead <1% of total (negligible)++ ┌────────────────────┬──────────┬──────────┬──────────┐+ │ │ v6e-4 │ v6e-8 │ v6e-16 │+ ├────────────────────┼──────────┼──────────┼──────────┤+ │ Chips │ 4 │ 8 │ 16 │+ │ Total HBM │ 128 GB │ 256 GB │ 512 GB │+ │ Scaling eff. │ 1.0× │ 0.9× │ 0.8× │+ │ Eff. TFLOPS │ 551 │ 992 │ 1,766 │+ │ Est. time │ ~96h │ ~50h │ ~29h │+ │ On-demand $/hr │ $10.80 │ $21.60 │ $43.20 │+ │ Est. total cost │ ~$1,040 │ ~$1,080 │ ~$1,250 │+ └────────────────────┴──────────┴──────────┴──────────┘```-## Setup For TPU/GCP+## Setup For TPU/GCP- Setup gcloud 1: download the `macOS ARM` from [here](https://docs.cloud.google.com/sdk/docs/install-sdk) and uzip.- Setup gcloud 2: open terminal, cd to ypur downloads and run `./google-cloud-sdk/install.sh`@@ -159,8 +195,8 @@ needle tpu [command]┌───────────────────────────────────────────────────────────────────┐│ ││ create NAME Create TPU (auto-finds zone) │- │ --type STR Accelerator type (default: v5litepod-4) │- │ --version STR TPU OS (default: tpu-ubuntu2204-base) │+ │ --type STR Accelerator type (default: v6e-4) │+ │ --version STR TPU OS (auto-detected from --type) ││ ││ connect NAME SSH config + first connect (auto-zone) ││ claude NAME Install Claude Code on instance │@@ -208,4 +244,4 @@ needle tpu [command]7. (Optional) Delete instance when no longer neededneedle tpu delete my-experiment-```\ No newline at end of file+```diff --git a/experiements/datasets.txt b/experiements/datasets.txtnew file mode 100644index 0000000..e69de29diff --git a/experiements/pruning.txt b/experiements/pruning.txtindex a3ca264..59828d7 100644--- a/experiements/pruning.txt+++ b/experiements/pruning.txt@@ -1,161 +1,175 @@================================================================================-STRUCTURED COMPRESSION OF SMALL ENCODER-DECODER TRANSFORMERS:-JOINT GROUP-WISE SPARSIFICATION AND LAYER PRUNING+MULTI-AXIS STRUCTURED COMPRESSION OF SMALL ENCODER-DECODER TRANSFORMERS:+MATRYOSHKA DIMENSIONS, BLOCK SPARSITY, LAYER PRUNING, AND INT4 QAT================================================================================-Target venue: ICML 2026+Target venue: ICLR 2026-Thesis: Structured compression (group-aligned block sparsity + layer removal)-applied in a staged schedule during training yields Pareto-optimal accuracy--efficiency tradeoffs for small encoder-decoder models, matching or exceeding-post-hoc pruning at equivalent FLOPs reduction while preserving INT4-quantization compatibility.+Thesis: Four complementary compression axes — Matryoshka dimension pruning,+group-aligned block sparsity, layer removal, and INT4 quantization-aware+training — can be jointly applied during training in a staged schedule to+produce a single checkpoint that is deployable at multiple accuracy-efficiency+operating points. Training once at d_model=D with MRL loss at dimensions+{D, D/2, D/4, D/8, ...} yields a family of models from one training run,+each further compressible via sparsification and layer pruning.================================================================================SECTION 1 — EXPERIMENTAL AXES================================================================================Independent variables:- A. Sparsity ratio s ∈ {0.0, 0.10, 0.25, 0.40, 0.50}- B. Layer prune ratio l ∈ {0.0, 0.125, 0.25, 0.375, 0.50}- C. Group size g ∈ {16, 32, 64, 128}- D. Pruning schedule ∈ {staged, simultaneous, reverse-order}- E. FFN activation ∈ {dReLU, SwiGLU, GeGLU}- F. Model scale ∈ {BASE-256, LARGE-512}- G. QAT ∈ {off, int4-group32}--Dependent variables (measured):+ A. MRL deployment dim d' ∈ {1024, 512, 256, 128, 64}+ B. Sparsity ratio s ∈ {0.0, 0.10, 0.25, 0.33, 0.50}+ C. Layer prune ratio l ∈ {0.0, 0.125, 0.25, 0.375, 0.50}+ D. Group size g ∈ {16, 32, 64, 128}+ E. Pruning schedule ∈ {staged, simultaneous, reverse-order}+ F. FFN activation ∈ {dReLU, SwiGLU, GeGLU}+ G. Model scale ∈ {TOY-128, MAIN-1024}+ H. QAT ∈ {off, int4-group32}++Dependent variables (measured at EACH MRL dimension):- Validation perplexity (TinyStories)+ - Quantized validation perplexity (INT4)- WikiText-2 perplexity- LAMBADA accuracy- HellaSwag accuracy- ARC-Easy accuracy+ - Estimated parameter count at deployment dim- Throughput (tokens/sec on inference)- Bigram repetition rate (generation diversity)- - Parameter count (absolute and % of dense baseline)- - Effective FLOPs (params * density)- - Weight sparsity (% near-zero, measured at threshold < 1e-6)+ - Block sparsity (% of fully-zero groups, group_size=g)+ - Effective FLOPs (est. params × density)================================================================================SECTION 2 — MODEL CONFIGURATIONS================================================================================-BASE-256 (25.4M params, default):- d_model=256, num_heads=4, num_kv_heads=2, num_layers=4,- d_ff=1024, max_seq_len=256, vocab=65536, dtype=bfloat16+Architecture: MemoryMixer encoder + Transformer decoder, shared embedding,+GQA (grouped query attention), ZC-RMSNorm, RoPE, SentencePiece BPE (8192).-LARGE-512 (≈100M params, scaling study):- d_model=512, num_heads=8, num_kv_heads=4, num_layers=8,- d_ff=2048, max_seq_len=256, vocab=65536, dtype=bfloat16+TOY-128 (rapid iteration):+ mrl_dims=[128, 64], d_model=128, num_heads=4, num_kv_heads=2,+ num_enc_layers=2, num_dec_layers=2, d_ff=512,+ max_enc_len=128, max_dec_len=128, vocab=8192, dtype=bfloat16+ needle train --toy --seed {42,137,256}-Both trained on TinyStories (500K samples, encoder-decoder split at 30%).+MAIN-1024 (paper results):+ mrl_dims=[1024, 512, 256, 128, 64], d_model=1024, num_heads=4,+ num_kv_heads=2, num_enc_layers=4, num_dec_layers=2, d_ff=4096,+ max_enc_len=256, max_dec_len=256, vocab=8192, dtype=bfloat16+ needle train --seed {42,137,256}++Both trained on TinyStories (1M samples, encoder-decoder split at 30%).Optimizer: Muon (kernels) + AdamW (embeddings/norms), WSD schedule.+All experiments report metrics at each MRL dimension slice.================================================================================SECTION 3 — EXPERIMENT GROUPS================================================================================All experiments use 3 seeds (42, 137, 256) for statistical significance.-Report mean ± std on all metrics.+Report mean +/- std on all metrics. Every run automatically reports val PPL+and INT4-quantized val PPL at each MRL dimension slice.--------------------------------------------------------------------------------GROUP 1: Dense Baselines (Table 1)+GROUP 1: Dense Baselines + MRL Dimension Scaling (Table 1, Figure 1)--------------------------------------------------------------------------------Purpose: Establish upper-bound quality for each model scale.-- 1a. BASE-256, 3 epochs, no pruning, no QAT- needle train --epochs 3 --seed {42,137,256}+Purpose: Establish quality at each deployment dimension from a single training+run. This is the MRL baseline — no sparsification or layer pruning.- 1b. BASE-256, 3 epochs, no pruning, with QAT (int4-group32)- [QAT is always on in current code — this IS the baseline]+ 1a. MAIN-1024, 3 epochs, no pruning, MRL dims [1024,512,256,128,64]+ needle train --sparsity-ratio 0.0 --layer-prune-ratio 0.0 \+ --seed {42,137,256}- 1c. LARGE-512, 3 epochs, no pruning- needle train --epochs 3 --d-model 512 --num-heads 8 --num-layers 8 \- --max-enc-len 256 --max-dec-len 256 --seed {42,137,256}+ 1b. TOY-128, 3 epochs, no pruning, MRL dims [128,64]+ needle train --toy --sparsity-ratio 0.0 --layer-prune-ratio 0.0 \+ --seed {42,137,256}Deliverables:- - Val perplexity, benchmark scores, throughput, generation samples+ - Table: val PPL and quant PPL at each MRL dim (1024, 512, 256, 128, 64)+ - Line plot: deployment dim vs val PPL — the MRL scaling curve (Figure 1a)+ - Line plot: deployment dim vs estimated param count (Figure 1b)+ - Benchmark scores at each dim- These are the denominators for all relative comparisons--------------------------------------------------------------------------------GROUP 2: Group-Wise Sparsification Only (Table 2, Figure 1)+GROUP 2: MRL + Group-Wise Sparsification (Table 2, Figure 2)--------------------------------------------------------------------------------Purpose: Characterise accuracy-sparsity Pareto frontier for block pruning alone.-Schedule: Epoch 1 dense, epochs 2-3 with sparsity mask locked.+Purpose: Characterise how MRL dimension scaling interacts with block sparsity.+At smaller deployment dims, does sparsification hurt more or less?- For s in {0.10, 0.25, 0.40, 0.50}:- 2a. BASE-256, 3 epochs, --sparsity-ratio s- needle train --epochs 3 --sparsity-ratio {s} --seed {42,137,256}+Schedule: Epoch 1 dense, epochs 2-3 with sparsity mask locked.- For s in {0.10, 0.25, 0.40, 0.50}:- 2b. LARGE-512, 3 epochs, --sparsity-ratio s- needle train --epochs 3 --d-model 512 --num-heads 8 --num-layers 8 \- --sparsity-ratio {s} --seed {42,137,256}+ For s in {0.10, 0.25, 0.33, 0.50}:+ 2a. MAIN-1024, 3 epochs, --sparsity-ratio s+ needle train --sparsity-ratio {s} --layer-prune-ratio 0.0 \+ --seed {42,137,256}Deliverables:- - Pareto curve: sparsity % vs val perplexity (Figure 1a)- - Pareto curve: sparsity % vs LAMBADA accuracy (Figure 1b)- - Table: all metrics at each sparsity level- - Analysis: at what sparsity does quality degrade >5% relative to baseline?+ - 2D heatmap: (sparsity, MRL dim) → val PPL (Figure 2a)+ - Pareto curve per MRL dim: sparsity % vs val PPL (Figure 2b)+ - Table: all metrics at each (sparsity, dim) pair+ - Analysis: does sparsification degrade smaller MRL slices more?+ (Hypothesis: smaller slices are more sensitive because fewer dimensions+ carry the representation — each zeroed weight matters more)--------------------------------------------------------------------------------GROUP 3: Layer Pruning Only (Table 3, Figure 2)+GROUP 3: MRL + Layer Pruning (Table 3, Figure 3)--------------------------------------------------------------------------------Purpose: Characterise accuracy-compression frontier for layer removal alone.-Schedule: Epoch 1 dense, epochs 2-3 with reduced architecture.+Purpose: Characterise how MRL dimension scaling interacts with layer removal.- For l in {0.125, 0.25, 0.375, 0.50}:- 3a. BASE-256 (4+4 layers), 3 epochs, --layer-prune-ratio l- needle train --epochs 3 --layer-prune-ratio {l} --seed {42,137,256}- Expected surviving layers: 7, 6, 5, 4 (of 8 total)+Schedule: Epoch 1 dense, epochs 2-3 with reduced architecture.For l in {0.125, 0.25, 0.375, 0.50}:- 3b. LARGE-512 (8+8 layers), 3 epochs, --layer-prune-ratio l- needle train --epochs 3 --d-model 512 --num-heads 8 --num-layers 8 \- --layer-prune-ratio {l} --seed {42,137,256}- Expected surviving layers: 14, 12, 10, 8 (of 16 total)+ 3a. MAIN-1024 (4 enc + 2 dec layers), 3 epochs, --layer-prune-ratio l+ needle train --sparsity-ratio 0.0 --layer-prune-ratio {l} \+ --seed {42,137,256}+ Expected surviving layers: 5, 4-5, 4, 3 (of 6 total)Deliverables:- - Pareto curve: layer ratio vs val perplexity (Figure 2a)- - Pareto curve: parameter count vs LAMBADA accuracy (Figure 2b)- - Table: per-run encoder/decoder surviving layer counts+ - 2D heatmap: (layer ratio, MRL dim) → val PPL (Figure 3a)+ - Pareto curve: param count vs val PPL at each MRL dim (Figure 3b)+ - Per-run encoder/decoder surviving layer counts- Analysis: does the pruner prefer removing encoder or decoder layers?- (Hypothesis: encoder layers are pruned more aggressively since the- decoder carries more of the generation burden.)+ (Hypothesis: encoder MemoryMixer blocks are pruned more aggressively+ since the decoder carries more of the generation burden)--------------------------------------------------------------------------------GROUP 4: Joint Sparsification + Layer Pruning (Table 4, Figure 3)- *** CORE CONTRIBUTION ***+GROUP 4: Full Pipeline — MRL + Sparsification + Layer Pruning (Table 4, Fig 4)+ *** CORE CONTRIBUTION ***--------------------------------------------------------------------------------Purpose: Show that combining both methods outperforms either alone at matched-compression ratio. This is the central claim of the paper.+Purpose: Show that all three compression axes compose — a single checkpoint+trained with MRL loss, then sparsified and layer-pruned, produces a family+of models spanning 10-100x parameter range. This is the central claim.Schedule (staged):- Epoch 1: dense training- Epoch 2: apply group-wise sparsification, train with mask- Epoch 3: apply layer pruning, train with reduced architecture + mask- Epoch 4: recovery fine-tuning+ Epoch 1: dense training with MRL loss+ Epoch 2: apply group-wise sparsification, train with mask + MRL+ Epoch 3: apply layer pruning, train reduced architecture + mask + MRL+ (Optional epoch 4: recovery fine-tuning)For (s, l) in grid:{(0.10, 0.125), (0.10, 0.25),(0.25, 0.125), (0.25, 0.25), (0.25, 0.375),- (0.40, 0.125), (0.40, 0.25),+ (0.33, 0.125), (0.33, 0.25),(0.50, 0.25), (0.50, 0.50)}- 4a. BASE-256, 4 epochs- needle train --epochs 4 --sparsity-ratio {s} --layer-prune-ratio {l} \+ 4a. MAIN-1024, 3 epochs+ needle train --sparsity-ratio {s} --layer-prune-ratio {l} \--seed {42,137,256}- 4b. LARGE-512, select best 4 (s,l) pairs from 4a, 4 epochs- needle train --epochs 4 --d-model 512 --num-heads 8 --num-layers 8 \- --sparsity-ratio {s} --layer-prune-ratio {l} --seed {42,137,256}+ 4b. Select best 4 (s,l) pairs from 4a, run with 4 epochs (recovery)+ needle train --epochs 4 --sparsity-ratio {s} --layer-prune-ratio {l} \+ --seed {42,137,256}Deliverables:- - 2D heatmap: (s, l) → val perplexity (Figure 3a)- - 2D heatmap: (s, l) → effective parameter count (Figure 3b)- - Pareto overlay: joint vs sparsity-only vs layer-only at matched- compression (Figure 3c) — THIS IS THE KEY FIGURE- - Table: full metrics for all (s, l) pairs- - Analysis: multiplicative vs additive compression benefit+ - 3D surface: (s, l, MRL dim) → val PPL (Figure 4a, sliced by dim)+ - Pareto overlay at each MRL dim: full-pipeline vs sparsity-only vs+ layer-only at matched effective params (Figure 4b) — KEY FIGURE+ - Table: full metrics for all (s, l) pairs at each MRL dim+ - Compression spectrum: for best (s, l), show the range of models:+ d'=1024 (full) → d'=64 (smallest) with estimated params and PPL+ - Analysis: multiplicative compression — MRL dimension reduction ×+ sparsity × layer removal = total compression ratio-------------------------------------------------------------------------------GROUP 5: Schedule Ablation (Table 5)@@ -164,42 +178,42 @@ Purpose: Justify the staged schedule (sparsify first, then prune layers).Pick the best (s, l) pair from Group 4a. Run 3 schedules:- 5a. Staged (default): sparsify epoch 2, layer prune epoch 3, recover epoch 4- needle train --epochs 4 --sparsity-ratio {s} --layer-prune-ratio {l}+ 5a. Staged (default): sparsify epoch 2, layer prune epoch 3+ needle train --sparsity-ratio {s} --layer-prune-ratio {l}- 5b. Reverse: layer prune epoch 2, sparsify epoch 3, recover epoch 4+ 5b. Reverse: layer prune epoch 2, sparsify epoch 3[Requires code change to swap epoch triggers]- 5c. Simultaneous: both at epoch 2, recover epochs 3-4+ 5c. Simultaneous: both at epoch 2, recover epoch 3[Requires code change to apply both at same epoch]Deliverables:- - Table: val perplexity and benchmark scores for each schedule- - Analysis: why staged order matters (hypothesis: sparsification is- lower-risk, lets model adapt before the more destructive layer removal)+ - Table: val PPL at each MRL dim for each schedule+ - Analysis: does schedule affect small MRL dims differently from large?--------------------------------------------------------------------------------GROUP 6: Recovery Epoch Scaling (Figure 4)+GROUP 6: Recovery Epoch Scaling (Figure 5)-------------------------------------------------------------------------------Purpose: How many recovery epochs are needed after compression?Pick the best (s, l) pair from Group 4a. Vary total epochs:For E in {2, 3, 4, 5, 6}:- 6a. BASE-256, E epochs, --sparsity-ratio {s} --layer-prune-ratio {l}- (pruning always at epochs 2 and 3; recovery = E - 3 epochs)+ 6a. MAIN-1024, E epochs, --sparsity-ratio {s} --layer-prune-ratio {l}+ (pruning at epochs 2 and 3; recovery = E - 3 epochs)Deliverables:- - Line plot: recovery epochs vs val perplexity (Figure 4)- - Analysis: diminishing returns curve, recommend minimum recovery budget+ - Line plot per MRL dim: recovery epochs vs val PPL (Figure 5)+ - Analysis: do smaller MRL dims need more recovery?-------------------------------------------------------------------------------GROUP 7: QAT Interaction Study (Table 6)-------------------------------------------------------------------------------Purpose: Show that group-wise sparsification is compatible with INT4 QAT-because both operate on the same group_size=32 granularity.+because both operate on the same group_size=32 granularity. Measure at+each MRL dimension — INT4 quantization error at smaller dims may compound.- 7a. Dense + QAT (baseline, = Group 1b)+ 7a. Dense + QAT (baseline, = Group 1a)7b. Sparsified (s=0.25) + QAT7c. Layer-pruned (l=0.25) + QAT7d. Joint (s=0.25, l=0.25) + QAT@@ -208,158 +222,142 @@ because both operate on the same group_size=32 granularity.to not call _quantize_params.Deliverables:- - Table: QAT on vs off for each compression method- - Analysis: does QAT hurt more when combined with sparsification?- (Hypothesis: no, because zeroed groups have no quantization error)- - Effective compression ratio: sparsity + INT4 → bits per parameter+ - Table: (QAT on/off) × compression method × MRL dim → val PPL+ - Analysis: does QAT degrade more at smaller MRL dims?+ (Hypothesis: yes — fewer dimensions means each quantization error+ has proportionally larger impact on output quality)+ - Effective compression: MRL dim reduction × sparsity × INT4 → total+ bits per effective parameter-------------------------------------------------------------------------------GROUP 8: Group Size Ablation (Table 7, Figure 6)--------------------------------------------------------------------------------Purpose: Determine optimal granularity for block sparsification. Smaller groups-give finer-grained pruning decisions (better accuracy retention) but produce-less structured sparsity (harder to exploit in hardware). Larger groups are-more hardware-friendly but coarser. The group size also controls QAT-quantization granularity, so this directly tests the sparsity-quantization-co-design hypothesis.+Purpose: Determine optimal block granularity for sparsification.-Note: requires adding --group-size CLI flag and threading it through to-_block_prune_params and _fake_quantize_int4.--Fixed configuration: BASE-256, s=0.25, 3 epochs, QAT on.+Fixed: MAIN-1024, s=0.25, 3 epochs, QAT on.For g in {16, 32, 64, 128}:- 8a. Sparsification only (isolate group size effect on pruning)- needle train --epochs 3 --sparsity-ratio 0.25 --group-size {g} \- --seed {42,137,256}+ 8a. Sparsification only+ needle train --sparsity-ratio 0.25 --layer-prune-ratio 0.0 \+ --group-size {g} --seed {42,137,256}8b. Joint sparsification + layer pruning (best l from Group 4a)- needle train --epochs 4 --sparsity-ratio 0.25 --layer-prune-ratio {l} \+ needle train --sparsity-ratio 0.25 --layer-prune-ratio {l} \--group-size {g} --seed {42,137,256}-Additional comparison at high sparsity to stress-test granularity:-+ High sparsity stress test:For g in {16, 32, 64, 128}:8c. s=0.50, sparsification only- needle train --epochs 3 --sparsity-ratio 0.50 --group-size {g} \- --seed {42,137,256}+ needle train --sparsity-ratio 0.50 --layer-prune-ratio 0.0 \+ --group-size {g} --seed {42,137,256}Deliverables:- - Line plot: group size vs val perplexity at s=0.25 and s=0.50 (Figure 6a)- - Line plot: group size vs LAMBADA accuracy (Figure 6b)- - Table: all metrics at each group size for both sparsity levels- - Analysis of key tradeoffs:- (a) g=16: finest granularity, best accuracy, but 2× more scale factors- to store than g=32 and least structured pattern- (b) g=32: default, matches common INT4 quantization group size,- good balance of accuracy and structure- (c) g=64: coarser, each pruning decision covers 64 weights,- more aggressive but more hardware-friendly- (d) g=128: coarsest, each block is a full row for d_model=128,- approaches per-channel granularity, may degrade significantly- - Cross-reference with QAT: when group_size is shared between sparsification- and quantization, zeroed groups save both weight storage AND scale storage- (no scale needed for an all-zero group). Report effective bits-per-param- accounting for this: bpp = (1 - sparsity) × (4 + 32/g) where 4 is INT4- bits and 32/g is amortized scale overhead per weight.+ - Line plot per MRL dim: group size vs val PPL (Figure 6a)+ - Table: all metrics at each (group size, MRL dim)+ - Effective bits-per-param: bpp = (1 - sparsity) * (4 + 32/g)+ - Analysis: does optimal group size change with MRL deployment dim?--------------------------------------------------------------------------------GROUP 9: Activation Function × Pruning Interaction (Table 8, Figure 8)+GROUP 9: Activation Function x Pruning Interaction (Table 8, Figure 7)*** KEY FINDING ***--------------------------------------------------------------------------------Purpose: Test whether activation function choice determines pruning robustness.-Early observation: dReLU dramatically increases post-prune perplexity compared-to SwiGLU. This group systematically characterises the interaction.+Purpose: Test whether activation function choice determines pruning robustness+across MRL dimension slices.Activation definitions:- - dReLU: ReLU(gate) * ReLU(up) — naturally ~75% sparse activations- - SwiGLU: SiLU(gate) * up — smooth, dense activations- - GeGLU: GELU(gate) * up — smooth, nearly dense activations--Requires: --activation CLI flag wired to FeedForward module.+ - dReLU: ReLU(gate) * ReLU(up) — naturally ~75% sparse activations+ - SwiGLU: SiLU(gate) * up — smooth, dense activations+ - GeGLU: GELU(gate) * up — smooth, nearly dense activations9a. Dense baselines (no pruning) — isolate activation qualityFor act in {drelu, swiglu, geglu}:- needle train --epochs 3 --activation {act} --sparsity-ratio 0.0 \+ needle train --activation {act} --sparsity-ratio 0.0 \--layer-prune-ratio 0.0 --seed {42,137,256}9b. Sparsification only — test pruning robustness per activationFor act in {drelu, swiglu, geglu}, s in {0.25, 0.50}:- needle train --epochs 3 --activation {act} --sparsity-ratio {s} \- --seed {42,137,256}+ needle train --activation {act} --sparsity-ratio {s} \+ --layer-prune-ratio 0.0 --seed {42,137,256}- 9c. Joint sparsification + layer pruning — full compression+ 9c. Joint — full compressionFor act in {drelu, swiglu, geglu}:- needle train --epochs 4 --activation {act} --sparsity-ratio 0.25 \+ needle train --activation {act} --sparsity-ratio 0.25 \--layer-prune-ratio 0.25 --seed {42,137,256}- 9d. LARGE-512 validation on best and worst activation from 9b-9c- needle train --epochs 4 --d-model 512 --num-heads 8 --num-layers 8 \- --activation {act} --sparsity-ratio 0.25 --layer-prune-ratio 0.25 \- --seed {42,137,256}+Deliverables:+ - Grouped bar chart per MRL dim: activation × compression → val PPL (Fig 7a)+ - Relative degradation: (pruned_ppl / dense_ppl) per activation per dim+ (Figure 7b) — KEY CHART: pruning robustness not just raw quality+ - Analysis: does dReLU's natural activation sparsity help or hurt at+ smaller MRL dims where redundancy is already low?++-------------------------------------------------------------------------------+GROUP 10: MRL Dimension-Specific Analysis (Table 9, Figure 8)+ *** MRL-SPECIFIC CONTRIBUTION ***+-------------------------------------------------------------------------------+Purpose: Deep-dive into what the model learns at each MRL dimension slice.++Using best checkpoint from Group 4a:++ 10a. Representation similarity across dims+ - CKA (centered kernel alignment) between d'=1024 and each smaller d'+ - Measure: do smaller dims learn a subspace of the full representation?++ 10b. Per-dim generation quality+ - Generate from each MRL dim slice at inference time+ - Compare generation diversity, coherence, repetition rate++ 10c. MRL loss weighting ablation+ - Uniform weighting (current): L = (1/K) * sum(L_k)+ - Scaled weighting: L = sum(d_k/D * L_k) (larger dims weighted more)+ - Inverse weighting: L = sum(D/d_k * L_k) (smaller dims weighted more)Deliverables:- - Grouped bar chart: activation × {dense, s=0.25, s=0.50} → val ppl (Fig 8a)- - Table: full metrics for all activation × compression combos- - Relative degradation plot: (pruned_ppl / dense_ppl) per activation (Fig 8b)- — THIS IS THE KEY CHART: shows pruning *robustness* not just raw quality- - Analysis of competing hypotheses:- (a) dReLU hurts pruning because its ~75% activation sparsity means fewer- active paths compensate for zeroed weights — the network has less- redundancy to absorb pruning damage- (b) dReLU helps pruning because its natural sparsity pattern means weight- blocks that process mostly-zero activations are truly unimportant- and can be safely pruned (better signal for the pruning criterion)- (c) Smooth activations (SwiGLU/GeGLU) spread information across more- weights, creating redundancy that makes the model more robust to- weight removal but harder to compress efficiently- - Recommendation: which activation to use when pruning is planned vs not-- Additional metric for this group:- - Activation sparsity: measure % of zeros in FFN intermediate activations- for each activation function at each pruning level. Report as:- act_sparsity = E[fraction of zeros in ReLU(gate) * ReLU(up)]- This connects weight sparsity to activation sparsity.+ - CKA heatmap: dim × dim similarity (Figure 8a)+ - Table: generation quality metrics at each dim+ - Ablation table: MRL loss weighting strategy × dim → val PPL--------------------------------------------------------------------------------GROUP 10: Inference Efficiency Validation (Table 9, Figure 9)+GROUP 11: Inference Efficiency Validation (Table 10, Figure 9)--------------------------------------------------------------------------------Purpose: Translate theoretical compression into real speedup.+Purpose: Translate theoretical compression into real speedup across the+full spectrum of deployment options.-For each Pareto-optimal checkpoint from Groups 2-4:+For each Pareto-optimal checkpoint from Groups 2-4, at each MRL dim:needle test --checkpoint {path} --throughput-runs 50-Measure:+Measure at each deployment dim:- Tokens/sec (decode throughput)- Latency per generation (ms)- Peak memory (via JAX profiler)Deliverables:- - Scatter plot: compression ratio vs actual speedup (Figure 5)- - Table: all Pareto-optimal models with throughput + quality- - Analysis: does structured sparsity translate to real speedup?- (Note: dense JAX matmuls don't skip zeros — speedup comes from- layer removal. Sparsity benefit is in model size / memory only- until sparse kernel support is added.)+ - Scatter plot: effective params vs actual throughput (Figure 9a)+ - Table: all Pareto-optimal (s, l, d') with throughput + quality+ - The "menu" figure: one chart showing ALL deployment options from a+ single checkpoint, from largest (d'=1024, no pruning) to smallest+ (d'=64, s=0.50, l=0.50) — THIS IS THE PAPER'S HERO FIGURE================================================================================SECTION 4 — EVALUATION PROTOCOL================================================================================-For every checkpoint:+For every checkpoint, evaluate at EACH MRL dimension slice:- 1. Validation perplexity (TinyStories val split, full)+ 1. Validation perplexity (TinyStories val split)+ - Full-precision val PPL+ - INT4-quantized val PPL (at each MRL dim)needle test --checkpoint {path} --max-eval-samples 5000- 2. Benchmark suite+ 2. Benchmark suite (at full dim and smallest viable dim)needle evaluate --checkpoint {path} \--benchmarks wikitext2 lambada hellaswag arc_easy --max-samples 500- 3. Generation quality (qualitative + bigram diversity)+ 3. Generation quality at each MRL dimneedle test --checkpoint {path}- → Inspect 10 sample generations, report bigram repetition rate+ Report: bigram repetition rate, avg generation length, sample outputs- 4. Throughput measurement+ 4. Throughput at each MRL dimneedle test --checkpoint {path} --throughput-runs 50================================================================================@@ -367,50 +365,51 @@ SECTION 5 — EXPECTED TABLES AND FIGURES================================================================================Tables:- T1. Dense baselines (2 scales × {with,without} QAT)- T2. Group-wise sparsification sweep (2 scales × 4 ratios)- T3. Layer pruning sweep (2 scales × 4 ratios)- T4. Joint compression grid (9 pairs × 2 scales)- T5. Schedule ablation (3 schedules)- T6. QAT interaction (4 methods × {on,off})- T7. Group size ablation (4 sizes × 2 sparsity levels + joint)- T8. Activation × pruning interaction (3 activations × 4 compression levels)- T9. Inference efficiency (Pareto-optimal checkpoints)+ T1. Dense baselines + MRL scaling curve (val PPL at each dim)+ T2. MRL + sparsification (sparsity × dim → val PPL)+ T3. MRL + layer pruning (layer ratio × dim → val PPL)+ T4. Full pipeline grid (s × l at each MRL dim)+ T5. Schedule ablation (3 schedules × MRL dims)+ T6. QAT interaction (4 methods × {on,off} × MRL dims)+ T7. Group size ablation (4 sizes × MRL dims)+ T8. Activation × pruning interaction (3 act × compression × MRL dims)+ T9. MRL dimension analysis (CKA, generation quality, loss weighting)+ T10. Inference efficiency (Pareto-optimal checkpoints × MRL dims)Figures:- F1. Sparsity-only Pareto: sparsity % vs perplexity + LAMBADA- F2. Layer-prune-only Pareto: param count vs perplexity + LAMBADA- F3. Joint compression: (a) heatmap, (b) param heatmap, (c) Pareto overlay- F4. Recovery epoch scaling curve- F5. Compression ratio vs actual inference speedup- F6. Group size: (a) g vs perplexity at s=0.25/0.50, (b) g vs LAMBADA- F7. Effective bits-per-param vs quality across group sizes- F8. Activation × pruning: (a) grouped bars, (b) relative degradation ratio- F9. Activation sparsity vs weight sparsity interaction+ F1. MRL scaling curve: deployment dim vs PPL + param count+ F2. MRL + sparsity: (a) heatmap, (b) Pareto per dim+ F3. MRL + layer pruning: (a) heatmap, (b) Pareto per dim+ F4. Full pipeline: (a) 3D surface slices, (b) Pareto overlay — KEY FIGURE+ F5. Recovery epoch scaling per MRL dim+ F6. Group size: (a) g vs PPL per dim, (b) effective bpp+ F7. Activation × pruning: (a) grouped bars per dim, (b) degradation ratio+ F8. MRL representation analysis: (a) CKA heatmap, (b) gen quality vs dim+ F9. Hero figure: full deployment menu from single checkpoint================================================================================SECTION 6 — COMPUTE BUDGET================================================================================-Per-run cost (BASE-256, 4 epochs, 500K samples, 4× TPU v5 lite):- ~10 minutes training + ~2 minutes evaluation+Per-run cost (MAIN-1024, 3 epochs, 1M samples, 4x TPU v5 lite):+ ~15 minutes training + ~5 minutes MRL evaluation (5 dims × val + quant)Estimated total runs:Group 1: 2 configs × 3 seeds = 6 runs- Group 2: 4 ratios × 2 scales × 3 seeds = 24 runs- Group 3: 4 ratios × 2 scales × 3 seeds = 24 runs- Group 4: 9 pairs × 3 seeds + 4 pairs × 3 seeds (large) = 39 runs- Group 5: 3 schedules × 3 seeds = 9 runs- Group 6: 5 epoch counts × 3 seeds = 15 runs- Group 7: 4 methods × 2 (QAT on/off) × 3 seeds = 24 runs- Group 8: 4 sizes × 3 configs (s=.25, s=.50, joint) × 3 = 36 runs- Group 9: 3 act × (3 dense + 6 sparse + 3 joint) × 3 = 36 runs- + 2 act × 3 seeds (large validation) = 6 runs- Group 10: evaluation only (reuses checkpoints) = 0 runs- ─────────────────────────────────────────────────────────────────────- Total: 219 runs--At ~12 min/run: ~44 TPU-hours (very manageable on v5litepod-4)+ Group 2: 4 ratios × 3 seeds = 12 runs+ Group 3: 4 ratios × 3 seeds = 12 runs+ Group 4: 9 pairs × 3 seeds + 4 pairs × 3 seeds (recov)= 39 runs+ Group 5: 3 schedules × 3 seeds = 9 runs+ Group 6: 5 epoch counts × 3 seeds = 15 runs+ Group 7: 4 methods × 2 (QAT on/off) × 3 seeds = 24 runs+ Group 8: 4 sizes × 3 configs × 3 seeds = 36 runs+ Group 9: 3 act × (3 + 6 + 3) × 3 seeds = 36 runs+ Group 10: 3 weighting strategies × 3 seeds = 9 runs+ Group 11: evaluation only (reuses checkpoints) = 0 runs+ ────────────────────────────────────────────────────────────────────+ Total: 198 runs++At ~20 min/run: ~66 TPU-hours (manageable on v5litepod-4)================================================================================SECTION 7 — ABLATION CHECKLIST@@ -418,41 +417,76 @@ SECTION 7 — ABLATION CHECKLISTBefore submission, verify these are addressed:- [ ] Statistical significance: 3 seeds, report mean ± std- [ ] Matched comparisons: compare joint vs individual at SAME compression- [ ] Scaling behavior: BASE-256 and LARGE-512 show consistent trends+ [ ] Statistical significance: 3 seeds, report mean +/- std+ [ ] MRL scaling: smooth degradation curve from full dim to smallest+ [ ] MRL + compression composability: all three axes combine gracefully+ [ ] Matched comparisons: compare methods at SAME effective param count[ ] Schedule sensitivity: staged vs reverse vs simultaneous[ ] Recovery saturation: diminishing returns on fine-tuning epochs- [ ] QAT compatibility: sparsity + quantization are not antagonistic+ [ ] QAT compatibility: sparsity + quantization + MRL are not antagonistic[ ] Pruning bias: report which layers (enc vs dec) get pruned- [ ] Generation quality: not just perplexity, show actual text samples- [ ] Group size sensitivity: optimal g, and does it interact with model scale?- [ ] Activation × pruning: does dReLU help or hurt compression robustness?- [ ] Activation sparsity measured: connect weight sparsity to activation sparsity- [ ] Honest limitations: sparse matmuls don't accelerate on current JAX+ [ ] Generation quality at each MRL dim, not just perplexity+ [ ] Group size sensitivity: optimal g, interaction with MRL dim+ [ ] Activation × pruning × MRL dim: three-way interaction+ [ ] MRL representation analysis: CKA, subspace structure+ [ ] Honest limitations: sparse matmuls don't accelerate on current JAX;+ MRL dimension reduction requires custom kernel for real speedup+ [ ] Comparison to baselines: training K separate small models vs one MRL================================================================================SECTION 8 — EXECUTION ORDER================================================================================-Phase 1 (Day 1): Groups 1 + 2 + 3 (54 runs, ~11 hrs)- Run all baselines and single-method sweeps in parallel across seeds.- Immediate deliverable: Pareto curves for each method alone.+Phase 1 (Day 1): Groups 1 + 2 + 3 (30 runs, ~10 hrs)+ Baselines, sparsity-only, layer-prune-only, all with MRL evaluation.+ Deliverable: MRL scaling curves and single-axis Pareto frontiers.-Phase 2 (Day 2): Group 4 (39 runs, ~8 hrs)- Joint compression grid. This produces the key result.- Immediate deliverable: heatmaps and overlay Pareto.+Phase 2 (Day 2): Group 4 (39 runs, ~13 hrs)+ Full pipeline grid. This produces the key result.+ Deliverable: multi-axis Pareto and deployment menu.-Phase 3 (Day 3): Groups 5 + 6 + 7 + 8 (84 runs, ~17 hrs)- Ablations: schedule, recovery, QAT interaction, group size.- Immediate deliverable: complete ablation tables.+Phase 3 (Day 3): Groups 5 + 6 + 7 (48 runs, ~16 hrs)+ Schedule, recovery, QAT ablations.+ Deliverable: complete ablation tables.-Phase 4 (Day 4): Group 9 (42 runs, ~8 hrs)- Activation × pruning interaction study.- Immediate deliverable: activation robustness table + degradation plot.+Phase 4 (Day 4): Groups 8 + 9 (72 runs, ~24 hrs)+ Group size and activation interaction studies.+ Deliverable: granularity and activation robustness analysis.++Phase 5 (Day 5): Groups 10 + 11 + writing (9 runs + eval)+ MRL analysis, inference benchmarks, paper draft.+ Deliverable: hero figure, all tables populated.++================================================================================+SECTION 9 — PAPER NARRATIVE+================================================================================-Phase 5 (Day 5): Group 10 + writing- Run inference benchmarks on all Pareto-optimal checkpoints.- Begin paper draft with all tables and figures populated.+Framing: "Train once, deploy anywhere."++A single training run with Matryoshka loss produces a checkpoint that can be+deployed at any dimension slice. Each slice can be further compressed with+group-wise sparsification and layer pruning. The result is a deployment MENU:++ ┌─────────────────────────────────────────────────────────────────────┐+ │ Deployment option Est. params Val PPL Quant PPL │+ │ ───────────────────────────────────────────────────────────────── │+ │ d'=1024, full ~44M X.XX X.XX │+ │ d'=512, full ~11M X.XX X.XX │+ │ d'=512, s=0.33 ~7M X.XX X.XX │+ │ d'=256, full ~3M X.XX X.XX │+ │ d'=256, s=0.33 ~2M X.XX X.XX │+ │ d'=256, s=0.33, l=0.25 ~1.5M X.XX X.XX │+ │ d'=128, full ~0.8M X.XX X.XX │+ │ d'=128, s=0.33 ~0.5M X.XX X.XX │+ │ d'=64, full ~0.2M X.XX X.XX │+ │ d'=64, s=0.33, l=0.25 ~0.1M X.XX X.XX │+ └─────────────────────────────────────────────────────────────────────┘++Key claims:+ 1. MRL dimension pruning is nearly free — minimal degradation at 2x reduction+ 2. Sparsification and layer pruning compose multiplicatively with MRL+ 3. A single checkpoint spans 100x+ parameter range+ 4. INT4 QAT is compatible with all three compression axes+ 5. MemoryMixer encoder is more compressible than standard transformer encoder================================================================================diff --git a/src/cli.py b/src/cli.pyindex 98d56bf..ebde0da 100644--- a/src/cli.py+++ b/src/cli.py@@ -10,26 +10,34 @@ HELP = """│ ...the tiny model to rule them all... ││ ││ train │- │ --epochs INT Training epochs (default: 1) │+ │ --toy Use toy config for quick iteration │+ │ --epochs INT Training epochs (default: 3) ││ --batch-size INT Batch size (default: 32) ││ --lr FLOAT AdamW learning rate (default: 3e-4) ││ --muon-lr FLOAT Muon learning rate (default: 0.02) │- │ --d-model INT Model dimension (default: 128) │+ │ --d-model INT Model dim (default: max of mrl-dims) ││ --num-heads INT Attention heads (default: 4) │- │ --num-layers INT Encoder/decoder layers (default: 2) │- │ --max-enc-len INT Max encoder seq length (default: 128) │- │ --max-dec-len INT Max decoder seq length (default: 128) │- │ --max-samples INT Training samples (default: 20000) │+ │ --num-layers INT Encoder layers (default: 12) │+ │ --num-dec-layers INT Decoder layers (default: 4) │+ │ --max-enc-len INT Max encoder seq length (default: 256) │+ │ --max-dec-len INT Max decoder seq length (default: 256) │+ │ --max-samples INT Training samples (default: 1000000) │+ │ --mrl-dims INT [...] MRL dim targets (default: 512 256 ..) │+ │ --sparsity-ratio FLOAT Block prune ratio (default: 0.33) │+ │ --layer-prune-ratio FL Layer prune ratio (default: 0.0) │+ │ --group-size INT Quant/prune group size (default: 32) │+ │ --activation STR drelu|swiglu|geglu (default: drelu) │+ │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │+ │ --eval-every INT Val eval interval (default: 1000) │+ │ --wandb Enable W&B logging │+ │ --checkpoint PATH Resume from checkpoint ││ --checkpoint-dir DIR Checkpoint directory │- │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) ││ --seed INT Random seed (default: 42) ││ │- │ │- │ sweep │- │ --sweep-config PATH Sweep YAML config (default: sweep.yaml)│- │ --project STR W&B project name (default: needle-v1)│- │ --count INT Number of trials (default: 20) │- │ (also accepts all train flags as defaults for non-swept params)│+ │ sweep │+ │ --sweep-config PATH Sweep YAML config │+ │ --project STR W&B project name (default: needle-v1) │+ │ --count INT Number of trials (default: 20) ││ ││ run ││ --checkpoint PATH Path to model checkpoint (required) │@@ -53,7 +61,7 @@ HELP = """│ ││ tpu ││ create NAME Create TPU (auto-finds zone) │- │ --type STR Accelerator (default: v5litepod-4) │+ │ --type STR Accelerator (default: v6e-4) ││ --version STR TPU OS (auto-detected from --type) ││ connect NAME SSH config + connect (auto-zone) ││ claude NAME Install Claude Code on instance │@@ -67,45 +75,31 @@ HELP = """"""TOY_CONFIG = {- "epochs": 1,- "batch_size": 32,- "lr": 3e-4,- "muon_lr": 0.02,- "d_model": 128,"num_heads": 4,"num_layers": 2,"num_dec_layers": 2,"max_enc_len": 128,"max_dec_len": 128,"max_samples": 10000,- "warmup_ratio": 0.05,- "dtype": "bfloat16",- "seed": 42,}-BASE_CONFIG = {- "epochs": 1,- "batch_size": 32,- "lr": 3e-4,- "muon_lr": 0.02,- "d_model": 512,+MAIN_CONFIG = {"num_heads": 4,- "num_layers": 4,- "num_dec_layers": 2,+ "num_layers": 12,+ "num_dec_layers": 4,"max_enc_len": 256,"max_dec_len": 256,- "max_samples": 500000,- "warmup_ratio": 0.05,- "dtype": "bfloat16",- "seed": 42,+ "max_samples": 1000000,}def _apply_train_defaults(args):- config = TOY_CONFIG if args.toy else BASE_CONFIG+ config = TOY_CONFIG if args.toy else MAIN_CONFIGfor key, value in config.items():if getattr(args, key, None) is None:setattr(args, key, value)+ if getattr(args, "d_model", None) is None:+ args.d_model = max(args.mrl_dims)def main():@@ -119,10 +113,10 @@ def main():p = sub.add_parser("train", add_help=False)p.add_argument("--toy", action="store_true")p.add_argument("--checkpoint", type=str, default=None)- p.add_argument("--epochs", type=int, default=None)- p.add_argument("--batch-size", type=int, default=None)- p.add_argument("--lr", type=float, default=None)- p.add_argument("--muon-lr", type=float, default=None)+ p.add_argument("--epochs", type=int, default=3)+ p.add_argument("--batch-size", type=int, default=32)+ p.add_argument("--lr", type=float, default=3e-4)+ p.add_argument("--muon-lr", type=float, default=0.02)p.add_argument("--d-model", type=int, default=None)p.add_argument("--num-heads", type=int, default=None)p.add_argument("--num-layers", type=int, default=None)@@ -130,18 +124,20 @@ def main():p.add_argument("--max-enc-len", type=int, default=None)p.add_argument("--max-dec-len", type=int, default=None)p.add_argument("--max-samples", type=int, default=None)- p.add_argument("--warmup-ratio", type=float, default=None)+ p.add_argument("--warmup-ratio", type=float, default=0.05)p.add_argument("--wandb", action="store_true")- p.add_argument("--dtype", type=str, default=None, choices=["float32", "bfloat16"])+ p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])p.add_argument("--checkpoint-dir", type=str, default="checkpoints")- p.add_argument("--seed", type=int, default=None)+ p.add_argument("--seed", type=int, default=42)p.add_argument("--eval-every", type=int, default=1000)p.add_argument("--max-eval-samples", type=int, default=None)- p.add_argument("--sparsity-ratio", type=float, default=0.0)+ p.add_argument("--sparsity-ratio", type=float, default=0.33)p.add_argument("--layer-prune-ratio", type=float, default=0.0)p.add_argument("--group-size", type=int, default=32)p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])p.add_argument("--num-memory-slots", type=int, default=64)+ p.add_argument("--mrl-dims", type=int, nargs="*", default=[512, 256, 128, 64],+ help="MRL dimension pruning targets (default: 512 256 128 64)")p = sub.add_parser("run", add_help=False)p.add_argument("--checkpoint", type=str, required=True)@@ -165,7 +161,7 @@ def main():p.add_argument("--project", type=str, default="needle-v1")p.add_argument("--count", type=int, default=20)p.add_argument("--epochs", type=int, default=1)- p.add_argument("--batch-size", type=int, default=32)+ p.add_argument("--batch-size", type=int, default=16)p.add_argument("--lr", type=float, default=3e-4)p.add_argument("--muon-lr", type=float, default=0.02)p.add_argument("--d-model", type=int, default=128)@@ -190,7 +186,7 @@ def main():tp = tpu_sub.add_parser("create", add_help=False)tp.add_argument("name", type=str)- tp.add_argument("--type", dest="accel_type", type=str, default="v5litepod-4")+ tp.add_argument("--type", dest="accel_type", type=str, default="v6e-4")tp.add_argument("--version", type=str, default=None,help="Software version (auto-detected from --type if omitted)")diff --git a/src/data.py b/src/data.pyindex a2c7471..7c572b8 100644--- a/src/data.py+++ b/src/data.py@@ -63,7 +63,7 @@ class NeedleTokenizer:return {"input_ids": all_ids}-def train_tokenizer(vocab_size=8192, max_samples=100000):+def train_tokenizer(vocab_size=8192, max_samples=None):"""Train a SentencePiece BPE tokenizer on TinyStories."""model_path = TOKENIZER_PREFIX + ".model"if os.path.exists(model_path):@@ -72,14 +72,15 @@ def train_tokenizer(vocab_size=8192, max_samples=100000):os.makedirs(TOKENIZER_DIR, exist_ok=True)- print(f"Training SentencePiece BPE tokenizer (vocab_size={vocab_size})...")ds = load_dataset("roneneldan/TinyStories", split="train")- indices = range(min(max_samples, len(ds)))- ds_subset = ds.select(indices)+ if max_samples:+ ds = ds.select(range(min(max_samples, len(ds))))++ print(f"Training SentencePiece BPE tokenizer (vocab_size={vocab_size}, samples={len(ds):,})...")corpus_path = os.path.join(TOKENIZER_DIR, "corpus.txt")with open(corpus_path, "w") as f:- for example in tqdm(ds_subset, desc="Writing corpus"):+ for example in tqdm(ds, desc="Writing corpus"):text = example["text"].strip()if text:f.write(text + "\n")@@ -105,10 +106,10 @@ def train_tokenizer(vocab_size=8192, max_samples=100000):return model_path-def get_tokenizer():+def get_tokenizer(max_samples=None):model_path = TOKENIZER_PREFIX + ".model"if not os.path.exists(model_path):- train_tokenizer()+ train_tokenizer(max_samples=max_samples)return NeedleTokenizer(model_path)@@ -119,8 +120,18 @@ def load_tinystories(split="train", max_samples=None):return ds-def _cache_key(n_samples, max_enc_len, max_dec_len, tokenizer_name="needle_spm"):- key = f"tinystories_{tokenizer_name}_{n_samples}_{max_enc_len}_{max_dec_len}"+def _tokenizer_hash():+ """Hash the tokenizer model file to detect retraining."""+ model_path = TOKENIZER_PREFIX + ".model"+ if os.path.exists(model_path):+ with open(model_path, "rb") as f:+ return hashlib.md5(f.read()).hexdigest()[:8]+ return "none"+++def _cache_key(n_samples, max_enc_len, max_dec_len):+ tok_hash = _tokenizer_hash()+ key = f"tinystories_{tok_hash}_{n_samples}_{max_enc_len}_{max_dec_len}"return hashlib.md5(key.encode()).hexdigest()[:12]diff --git a/src/model.py b/src/model.pyindex fe0c995..a8140ba 100644--- a/src/model.py+++ b/src/model.py@@ -325,14 +325,26 @@ class EncoderDecoderTransformer(nn.Module):)return logits- def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):+ def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mrl_dims=None):encoder_out = self.encode(src, src_mask=src_mask)- logits = self.decode(tgt, encoder_out, self_mask=tgt_mask)+ x = self.embedding(tgt) * self.embed_scale+ rope = self._rope(tgt.shape[1])+ x = self.decoder(x, encoder_out, self_mask=tgt_mask, cross_mask=None, rope=rope)+ x_f32 = x.astype(jnp.float32)+ emb = self.embedding.embedding+ logits = x_f32 @ emb.T++ mrl_logits = []+ if mrl_dims is not None:+ for d in mrl_dims:+ if d < self.config.d_model:+ mrl_logits.append(x_f32[..., :d] @ emb[:, :d].T)+s = encoder_out.astype(jnp.float32)- gram = jnp.matmul(s, s.transpose(0, 2, 1))+ gram = jnp.matmul(s, s.transpose(0, 2, 1))diag_sq = jnp.sum(jnp.diagonal(gram, axis1=1, axis2=2) ** 2)slot_div = (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]- return logits, slot_div+ return logits, slot_div, mrl_logitsdef make_causal_mask(seq_len):diff --git a/src/tpu.py b/src/tpu.pyindex c4f286e..717d8ce 100644--- a/src/tpu.py+++ b/src/tpu.py@@ -7,12 +7,6 @@ import sysPROJECT = "needle-488623"VERSION_FOR_TYPE = {- "v2": "tpu-ubuntu2204-base",- "v3": "tpu-ubuntu2204-base",- "v4": "tpu-ubuntu2204-base",- "v5litepod": "v2-alpha-tpuv5-lite",- "v5e": "v2-alpha-tpuv5-lite",- "v5p": "v2-alpha-tpuv5","v6e": "v2-alpha-tpuv6e",}@@ -37,16 +31,15 @@ def _resolve_version(accel_type, explicit_version):return "tpu-ubuntu2204-base"+# Trillium (v6e) zones, ordered by on-demand price (cheapest first)+# us-east1/us-east5: $2.70/chip/hr+# europe-west4: $2.97/chip/hr+# asia-northeast1: $3.24/chip/hrZONES = [- "us-central1-a", "us-central1-b", "us-central1-f","us-east1-b", "us-east1-c", "us-east1-d","us-east5-a", "us-east5-b",- "us-south1-a", "us-south1-b",- "us-west1-a", "us-west1-b",- "us-west4-a","europe-west4-a","asia-northeast1-b",- "southamerica-west1-a",]TPU_HELP = """diff --git a/src/train.py b/src/train.pyindex ee9a098..9964749 100644--- a/src/train.py+++ b/src/train.py@@ -282,6 +282,7 @@ def _quantize_params(params, group_size=32):_GROUP_SIZE = 32+_MRL_DIMS = ()def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):@@ -293,23 +294,37 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)cross_mask = src_mask- logits, slot_div = state.apply_fn(+ logits, slot_div, mrl_logits = state.apply_fn({"params": _quantize_params(params, group_size=_GROUP_SIZE)},src,tgt_in,src_mask=src_mask,tgt_mask=tgt_mask,cross_mask=cross_mask,+ mrl_dims=_MRL_DIMS if _MRL_DIMS else None,method="forward_with_aux",)logits_f32 = logits.astype(jnp.float32)- loss = optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out)mask = (tgt_out != pad_id).astype(jnp.float32)- ce_loss = jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)+ ce_loss = jnp.sum(+ optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask+ ) / jnp.maximum(jnp.sum(mask), 1.0)++ total_loss = ce_loss+ num_scales = 1+ for mrl_log in mrl_logits:+ mrl_log_f32 = mrl_log.astype(jnp.float32)+ aux_loss = jnp.sum(+ optax.softmax_cross_entropy_with_integer_labels(mrl_log_f32, tgt_out) * mask+ ) / jnp.maximum(jnp.sum(mask), 1.0)+ total_loss = total_loss + aux_loss+ num_scales = num_scales + 1+ total_loss = total_loss / num_scales+z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)div_loss = 1e-4 * slot_div- return ce_loss + z_loss + div_loss+ return total_loss + z_loss + div_lossloss, grads = jax.value_and_grad(loss_fn)(state.params)grads = jax.lax.pmean(grads, axis_name="batch")@@ -340,6 +355,50 @@ def _make_val_loss_fn(apply_fn):return val_loss_batch+def _make_mrl_val_loss_fn(apply_fn, mrl_dim):+ """Val loss using truncated decoder hidden states and embedding at dimension mrl_dim."""+ @jax.jit+ def val_loss_batch(params, src, tgt_in, tgt_out, causal_mask):+ pad_id = 0+ src_mask = make_padding_mask(src, pad_id)+ tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)+ logits, _, mrl_logits = apply_fn(+ {"params": params}, src, tgt_in,+ src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=src_mask,+ mrl_dims=(mrl_dim,),+ method="forward_with_aux",+ )+ trunc_logits = mrl_logits[0].astype(jnp.float32)+ loss = optax.softmax_cross_entropy_with_integer_labels(trunc_logits, tgt_out)+ mask = (tgt_out != pad_id).astype(jnp.float32)+ return jnp.sum(loss * mask), jnp.sum(mask)+ return val_loss_batch+++def _estimate_mrl_params(config, d_prime):+ """Estimate parameter count if model were deployed at dimension d_prime."""+ d = d_prime+ v = config.vocab_size+ n_enc = config.num_encoder_layers+ n_dec = config.num_decoder_layers+ n_h = config.num_heads+ n_kv = config.num_kv_heads+ d_ff = d * 4+ m = config.num_memory_slots++ emb = v * d+ head_dim = d // n_h+ kv_dim = n_kv * head_dim+ attn = d * d + d * kv_dim * 2 + d * d+ ffn = d * d_ff * 3+ mixer_token = d * d_ff * 2 + d_ff * m+ mixer_channel = d * d_ff * 3+ enc_block = attn + ffn + mixer_token + mixer_channel+ dec_block = attn * 2 + ffn+ total = emb + n_enc * enc_block + n_dec * dec_block+ return int(total)++def shard_batch(batch, num_devices):"""Reshape a batch array so leading dim is (num_devices, per_device_batch, ...)."""return batch.reshape(num_devices, -1, *batch.shape[1:])@@ -358,7 +417,7 @@ def train(args):print(f" {num_devices} device(s) for data-parallel training")print(f"\n[2/4] Loading tokenizer...")- tokenizer = get_tokenizer()+ tokenizer = get_tokenizer(max_samples=args.max_samples)print(f"\n[3/4] Loading TinyStories dataset...")ds = load_tinystories("train", max_samples=args.max_samples)@@ -371,9 +430,9 @@ def train(args):val_enc, val_dec_in, val_dec_tgt = prepare_encoder_decoder_pairs(val_ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len)- val_enc = jnp.array(val_enc)- val_dec_in = jnp.array(val_dec_in)- val_dec_tgt = jnp.array(val_dec_tgt)+ val_enc = np.array(val_enc)+ val_dec_in = np.array(val_dec_in)+ val_dec_tgt = np.array(val_dec_tgt)print(f" {len(enc_inputs):,} train / {len(val_enc):,} val pairs")effective_batch_size = args.batch_size * num_devices@@ -399,8 +458,13 @@ def train(args):num_memory_slots=getattr(args, "num_memory_slots", 64),)- global _GROUP_SIZE+ global _GROUP_SIZE, _MRL_DIMS_GROUP_SIZE = getattr(args, "group_size", 32)+ mrl_dims_raw = getattr(args, "mrl_dims", None)+ if mrl_dims_raw:+ _MRL_DIMS = tuple(d for d in mrl_dims_raw if d < config.d_model)+ else:+ _MRL_DIMS = ()p_train_step = _make_p_train_step()np.random.seed(args.seed)@@ -457,9 +521,9 @@ def train(args):muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps)tokens_per_batch = effective_batch_size * (args.max_enc_len + args.max_dec_len)- enc_inputs = jnp.array(enc_inputs)- dec_inputs = jnp.array(dec_inputs)- dec_targets = jnp.array(dec_targets)+ enc_inputs = np.array(enc_inputs)+ dec_inputs = np.array(dec_inputs)+ dec_targets = np.array(dec_targets)last_val_ppl = Nonesparsity_ratio = getattr(args, "sparsity_ratio", 0.0)@@ -555,13 +619,14 @@ def train(args):dt = time.perf_counter() - t0eval_every = getattr(args, "eval_every", 100)if global_step % eval_every == 0 or global_step == total_steps:- eval_params = jax_utils.unreplicate(ema_params)+ _eval_params = jax_utils.unreplicate(ema_params)val_causal = make_causal_mask(args.max_dec_len)total_loss, total_toks = 0.0, 0.0for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False):- vl, vt = val_loss_fn(eval_params, vb[0], vb[1], vb[2], val_causal)+ vl, vt = val_loss_fn(_eval_params, vb[0], vb[1], vb[2], val_causal)total_loss += float(vl)total_toks += float(vt)+ del _eval_paramslast_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))pbar.set_postfix(loss=f"{loss_val:.4f}", val_ppl=f"{last_val_ppl:.2f}" if last_val_ppl is not None else "?")@@ -598,24 +663,46 @@ def train(args):q_total_loss += float(vl)q_total_toks += float(vt)quant_val_ppl = float(math.exp(q_total_loss / max(q_total_toks, 1)))-- epoch_params = jax.tree.map(np.array, eval_params)- total_params = sum(x.size for x in jax.tree.leaves(epoch_params))- near_zero = sum(int(np.sum(np.abs(x) < 1e-6)) for x in jax.tree.leaves(epoch_params))+ del q_params # free device memory++ # MRL per-dimension evaluation (reuse val_loss_fn's apply_fn)+ mrl_results = {}+ if _MRL_DIMS:+ apply_fn = jax_utils.unreplicate(state).apply_fn+ for d_prime in _MRL_DIMS:+ mrl_vl_fn = _make_mrl_val_loss_fn(apply_fn, d_prime)+ mrl_total_loss, mrl_total_toks = 0.0, 0.0+ for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False):+ vl, vt = mrl_vl_fn(eval_params, vb[0], vb[1], vb[2], val_causal)+ mrl_total_loss += float(vl)+ mrl_total_toks += float(vt)+ avg_loss = mrl_total_loss / max(mrl_total_toks, 1)+ mrl_ppl = float(math.exp(min(avg_loss, 20)))+ mrl_params = _estimate_mrl_params(config, d_prime)+ mrl_results[d_prime] = (mrl_ppl, mrl_params)+ del apply_fn++ # Sparsity + checkpoint — move to CPU, reuse single copy+ params_np = jax.tree.map(np.array, eval_params)+ del eval_params # free device memory+ total_params = sum(x.size for x in jax.tree.leaves(params_np))+ near_zero = sum(int(np.sum(np.abs(x) < 1e-6)) for x in jax.tree.leaves(params_np))sparsity = near_zero / total_params * 100ckpt_name = f"needle_{args.num_layers}_{args.d_model}_{global_step}.pkl"ckpt_path = os.path.join(args.checkpoint_dir, ckpt_name)- params_np = jax.tree.map(np.array, jax_utils.unreplicate(ema_params))with open(ckpt_path, "wb") as f:pickle.dump({"params": params_np, "config": config.__dict__}, f)from .test import measure_throughput, benchmark_generation_quality+ eval_params_jnp = jax.tree.map(jnp.array, params_np)+ del params_np # free CPU memorymodel = EncoderDecoderTransformer(config)- tp = measure_throughput(model, eval_params, tokenizer, num_runs=5)+ tp = measure_throughput(model, eval_params_jnp, tokenizer, num_runs=5)prompts = ["Once upon a time", "The little dog", "She was very happy because"]- quality = benchmark_generation_quality(model, eval_params, tokenizer, prompts, max_gen_len=64, temperature=0.8)+ quality = benchmark_generation_quality(model, eval_params_jnp, tokenizer, prompts, max_gen_len=64, temperature=0.8)+ del eval_params_jnp # free device memoryprint(f"\n ─────────────────────────────────────")print(f" Epoch {epoch + 1}/{args.epochs}")@@ -626,6 +713,14 @@ def train(args):print(f" Val ppl {last_val_ppl:>12.2f}")print(f" Quant val ppl {quant_val_ppl:>12.2f} (INT4 g{_GROUP_SIZE})")print(f" Sparsity {sparsity:>11.2f}% ({near_zero:,}/{total_params:,})")+ if mrl_results:+ print(f" ─────────────────────────────────────")+ print(f" MRL dimension pruning:")+ print(f" {'dim':>6} {'val ppl':>10} {'est. params':>12}")+ print(f" {config.d_model:>6} {last_val_ppl:>10.2f} {total_params:>12,} (full)")+ for d_prime in sorted(mrl_results.keys(), reverse=True):+ mrl_ppl, mrl_params = mrl_results[d_prime]+ print(f" {d_prime:>6} {mrl_ppl:>10.2f} {mrl_params:>12,}")print(f" ─────────────────────────────────────")print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")print(f" Latency {tp['avg_latency_s']:>11.3f}s")@@ -640,14 +735,18 @@ def train(args):print(f" ─────────────────────────────────────\n")if use_wandb:- wandb.log({+ log_dict = {"epoch/avg_loss": epoch_avg_loss,"epoch/final_loss": final_loss,"epoch/val_ppl": last_val_ppl,"epoch/quant_val_ppl": quant_val_ppl,"epoch/weight_sparsity": sparsity,"epoch": epoch + 1,- })+ }+ for d_prime, (mrl_ppl, mrl_params) in mrl_results.items():+ log_dict[f"epoch/mrl_ppl_d{d_prime}"] = mrl_ppl+ log_dict[f"epoch/mrl_params_d{d_prime}"] = mrl_params+ wandb.log(log_dict)if use_wandb:wandb.finish()