Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

training

HenryNdubuakufd57d5f499a6
diff --git a/README.md b/README.md
index 547d7bb..22ef593 100644
--- a/README.md
+++ b/README.md
@@ -12,47 +12,54 @@
┌─────────────┐
│ Softmax │ ┌──────────────────────┐
- └──────┬──────┘ │ Forward pass with │
- ┌──────┴──────┐ │ INT4 fake-quantized │
- │ Linear (T) │ ← tied │ weights (g=32, STE) │
- └──────┬──────┘ └──────────┬───────────┘
+ └──────┬──────┘ │ Forward with MRL │
+ ┌─────┬──────┴──────┬─────┐ │ loss at dims │
+ │ @E[:d'] for each d' │ │ {512,256,128,64} │
+ │ in mrl_dims │ │ + INT4 QAT (g=32) │
+ └─────┬──────┬──────┬─────┘ └──────────┬───────────┘
┌──────┴──────┐ │
- │ ZCRMSNorm │ ┌──────────┴───────────┐
+ │ Linear (T) │ ← tied ┌──────────┴───────────┐
└──────┬──────┘ │ Muon (2D kernels) │
- ┌─────────┴─────────┐ │ AdamW (everything │
- │ Decoder x N │ │ else) │
- │ ┌───────────────┐ │ │ WSD LR schedule │
- │ │ Masked Self │ │ └──────────┬───────────┘
- │ │ Attn + RoPE │ │ │
- │ ├───────────────┤ │ ┌──────────┴───────────┐
- ┌──────────────┐ S │ │ Cross │ │ │ EMA params (β=0.999) │
- │ MemoryMixer │─────────▶ Attention │ │ └──────────┬───────────┘
- │ Encoder x N │ │ ├───────────────┤ │ │
- │ │ │ │ Feed-Forward │ │ ┌──────────┴───────────┐
- │ ┌──────────┐ │ │ │ (dReLU) │ │ │ Block Prune │
- │ │Pack: │ │ │ └───────────────┘ │ │ after epoch 1 │
- │ │ S←X Attn │ │ └─────────┬─────────┘ │ group magnitude │
- │ ├──────────┤ │ ┌──────┴──────┐ │ lock sparsity mask │
- │ │Mix: │ │ │ Embedding │ ← shared └──────────┬───────────┘
- │ │ MLP-Mixer│ │ └──────┬──────┘ │
- │ │ on S │ │ │ ┌──────────┴───────────┐
- │ ├──────────┤ │ ┌─────┴─────┐ │ Layer Prune │
- │ │Local: │ │ │ Decoder │ │ after epoch 2 │
- │ │ FFN on X │ │ │ Input │ │ L1 block scoring │
- │ └──────────┘ │ └───────────┘ │ keep ≥1 enc & dec │
- │ S ∈ (M, d) │ └──────────┬───────────┘
- └──────┬───────┘ │
- ┌──────┴───────┐ ┌──────────┴───────────┐
- │ Embedding │ ← shared │ Checkpoint │
- └──────┬───────┘ │ sparse + INT4 │
- │ └──────────────────────┘
+ ┌──────┴──────┐ │ AdamW (everything │
+ │ ZCRMSNorm │ │ else) │
+ └──────┬──────┘ │ WSD LR schedule │
+ ┌─────────┴─────────┐ └──────────┬───────────┘
+ │ Decoder x N_dec │ │
+ │ ┌───────────────┐ │ ┌──────────┴───────────┐
+ │ │ Masked Self │ │ │ EMA params (β=0.999) │
+ │ │ Attn + RoPE │ │ └──────────┬───────────┘
+ │ ├───────────────┤ │ │
+ ┌──────────────┐ S │ │ Cross │ │ ┌──────────┴───────────┐
+ │ MemoryMixer │─────────▶ Attention │ │ │ Block Prune │
+ │ Encoder │ │ ├───────────────┤ │ │ after epoch 1 │
+ │ x N_enc │ │ │ Feed-Forward │ │ │ group magnitude │
+ │ │ │ │ (dReLU) │ │ │ lock sparsity mask │
+ │ ┌──────────┐ │ │ └───────────────┘ │ └──────────┬───────────┘
+ │ │Pack: │ │ └─────────┬─────────┘ │
+ │ │ S←X Attn │ │ ┌──────┴──────┐ ┌──────────┴───────────┐
+ │ │ RoPE keys│ │ │ Embedding │ ← shared │ Layer Prune │
+ │ ├──────────┤ │ └──────┬──────┘ │ after epoch 2 │
+ │ │Mix: │ │ │ │ L1 block scoring │
+ │ │ MLP-Mixer│ │ ┌─────┴─────┐ │ keep ≥1 enc & dec │
+ │ │ on S │ │ │ Decoder │ └──────────┬───────────┘
+ │ ├──────────┤ │ │ Input │ │
+ │ │Local: │ │ └───────────┘ ┌──────────┴───────────┐
+ │ │ FFN on X │ │ │ Checkpoint │
+ │ └──────────┘ │ │ MRL + sparse + INT4 │
+ │ S ∈ (M, d) │ └──────────────────────┘
+ └──────┬───────┘
+ ┌──────┴───────┐
+ │ Embedding │ ← shared
+ └──────┬───────┘
+ │
┌────┴────┐
│ Encoder │
│ Input │
└─────────┘
- d=128 · 4 heads · 2 KV heads · 64 memory slots
- dReLU · ZCRMSNorm · RoPE · INT4 QAT · Muon
+ d=max(mrl_dims) · 4 heads · 2 KV heads · 64 memory slots
+ SentencePiece BPE (8192) · dReLU · ZCRMSNorm · RoPE
+ Matryoshka dims · INT4 QAT · Muon + AdamW
```
## Usage
@@ -67,45 +74,66 @@ needle [command]
┌───────────────────────────────────────────────────────────────────┐
│ │
│ train │
- │ --epochs INT Training epochs (default: 1) │
- │ --batch-size INT Batch size (default: 32) │
- │ --lr FLOAT AdamW learning rate (default: 3e-4) │
- │ --muon-lr FLOAT Muon learning rate (default: 0.02) │
- │ --d-model INT Model dimension (default: 128) │
- │ --num-heads INT Attention heads (default: 4) │
- │ --num-layers INT Encoder/decoder layers (default: 2) │
- │ --max-enc-len INT Max encoder seq length (default: 128) │
- │ --max-dec-len INT Max decoder seq length (default: 128) │
- │ --max-samples INT Training samples (default: 20000) │
- │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │
- │ --wandb Enable W&B logging │
- │ --checkpoint-dir DIR Checkpoint directory │
- │ --seed INT Random seed (default: 42) │
+ │ --toy Use toy config for quick iteration │
+ │ --epochs INT Training epochs (default: 3) │
+ │ --batch-size INT Batch size (default: 32) │
+ │ --lr FLOAT AdamW learning rate (default: 3e-4) │
+ │ --muon-lr FLOAT Muon learning rate (default: 0.02) │
+ │ --d-model INT Model dim (default: max of mrl-dims) │
+ │ --num-heads INT Attention heads (default: 4) │
+ │ --num-layers INT Encoder layers (default: 12) │
+ │ --num-dec-layers INT Decoder layers (default: 4) │
+ │ --max-enc-len INT Max encoder seq length (default: 256) │
+ │ --max-dec-len INT Max decoder seq length (default: 256) │
+ │ --max-samples INT Training samples (default: 1000000) │
+ │ --mrl-dims INT [...] MRL dim targets (default: 512 256 128) │
+ │ --sparsity-ratio FLOAT Block prune ratio (default: 0.33) │
+ │ --layer-prune-ratio FL Layer prune ratio (default: 0.0) │
+ │ --group-size INT Quant/prune group size (default: 32) │
+ │ --activation STR drelu|swiglu|geglu (default: drelu) │
+ │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │
+ │ --eval-every INT Val eval interval (default: 1000) │
+ │ --wandb Enable W&B logging │
+ │ --checkpoint PATH Resume from checkpoint │
+ │ --checkpoint-dir DIR Checkpoint directory │
+ │ --seed INT Random seed (default: 42) │
│ │
│ sweep │
- │ --sweep-config PATH Sweep YAML config │
- │ --project STR W&B project name (default: needle-v1) │
- │ --count INT Number of trials (default: 20) │
+ │ --sweep-config PATH Sweep YAML config │
+ │ --project STR W&B project name (default: needle-v1) │
+ │ --count INT Number of trials (default: 20) │
│ │
│ run │
- │ --checkpoint PATH Path to model checkpoint (required) │
- │ --prompts STR [...] One or more prompts to continue │
- │ --max-len INT Max tokens to generate (default: 128) │
- │ --temperature FLOAT Sampling temperature (default: 0.8) │
- │ --seed INT Random seed (default: 0) │
+ │ --checkpoint PATH Path to model checkpoint (required) │
+ │ --prompts STR [...] One or more prompts to continue │
+ │ --max-len INT Max tokens to generate (default: 128) │
+ │ --temperature FLOAT Sampling temperature (default: 0.8) │
+ │ --seed INT Random seed (default: 0) │
│ │
│ test │
- │ --checkpoint PATH Path to model checkpoint (required) │
- │ --batch-size INT Batch size (default: 32) │
- │ --max-eval-samples INT Evaluation samples (default: 1000) │
- │ --max-gen-len INT Max generation length (default: 128) │
- │ --temperature FLOAT Sampling temperature (default: 0.8) │
- │ --throughput-runs INT Throughput runs (default: 10) │
+ │ --checkpoint PATH Path to model checkpoint (required) │
+ │ --batch-size INT Batch size (default: 32) │
+ │ --max-eval-samples INT Evaluation samples (default: 1000) │
+ │ --max-gen-len INT Max generation length (default: 128) │
+ │ --temperature FLOAT Sampling temperature (default: 0.8) │
+ │ --throughput-runs INT Throughput runs (default: 10) │
│ │
│ evaluate │
- │ --checkpoint PATH Path to model checkpoint (required) │
- │ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │
- │ --max-samples INT Samples per benchmark (default: 500) │
+ │ --checkpoint PATH Path to model checkpoint (required) │
+ │ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │
+ │ --max-samples INT Samples per benchmark (default: 500) │
+ │ │
+ │ tpu │
+ │ create NAME Create TPU (auto-finds zone) │
+ │ --type STR Accelerator (default: v6e-4) │
+ │ --version STR TPU OS (auto-detected from --type) │
+ │ connect NAME SSH config + connect (auto-zone) │
+ │ claude NAME Install Claude Code on instance │
+ │ stop NAME Stop instance (auto-zone) │
+ │ start NAME Start stopped instance (auto-zone) │
+ │ delete NAME Delete instance (auto-zone) │
+ │ list List all TPU instances │
+ │ --zone ZONE Override auto-detected zone │
│ │
└───────────────────────────────────────────────────────────────────┘
```
@@ -113,37 +141,45 @@ needle [command]
## TPU Factsheet
```
-┌────────────────────┬───────────────────┬──────────────────────┬────────────────────────────┐
-│ │ v5e │ v5p │ v6e (Trillium) │
-├────────────────────┼───────────────────┼──────────────────────┼────────────────────────────┤
-│ Optimized for │ Train + inference │ Training (max perf) │ Train + inference │
-│ HBM per chip │ 16 GB │ 95 GB │ 32 GB │
-│ FLOPS (BF16) │ 197 TFLOPS │ 459 TFLOPS │ 918 TFLOPS │
-│ HBM bandwidth │ 819 GB/s │ 2,765 GB/s │ 1,640 GB/s │
-│ ICI bandwidth │ 1,600 Gbps │ 4,800 Gbps │ 3,584 Gbps │
-│ On-demand/chip/hr │ $1.20 │ $4.20 │ $2.70 │
-│ Spot/chip/hr │ $0.60 │ $2.10 │ $1.35 │
-│ Perf per $ │ 1x (baseline) │ 0.5x │ 2x │
-├────────────────────┼───────────────────┴──────────────────────┴────────────────────────────┤
-│ │ │
-│ DATASET │ │
-│ Text │ 100B tokens │
-│ Audio │ 200k × 20s = ~200M audio tokens + ~13M transcription tokens │
-│ Effective total │ ~100.5B equivalent tokens (audio has ~2-3x encoder overhead) │
-│ Storage │ ~5 GB audio (compressed) + ~400 GB text corpus │
-│ │ │
-├────────────────────┼───────────────────┬──────────────────────┬─────────────┬──────────────┤
-│ 300M multimodal │ v5litepod-4 │ v6e-4 │ v6e-8 │ v6e-16 │
-├────────────────────┼───────────────────┼──────────────────────┼─────────────┼──────────────┤
-│ Chips │ 4 │ 4 │ 8 │ 16 │
-│ Total HBM │ 64 GB │ 128 GB │ 256 GB │ 512 GB │
-│ Est. time │ ~16-21 days │ ~4-5 days │ ~2-3 days │ ~1-1.5 days │
-│ Spot $/hr │ $2.40 │ $5.40 │ $10.80 │ $21.60 │
-│ Est. total cost │ ~$900-1,200 │ ~$550-700 │ ~$550-750 │ ~$550-750 │
-└────────────────────┴───────────────────┴──────────────────────┴─────────────┴──────────────┘
+ Trillium (v6e)
+ ──────────────────────────────────────────
+ HBM per chip 32 GB
+ BF16 FLOPS 918 TFLOPS
+ HBM bandwidth 1,640 GB/s
+ ICI bandwidth 3,584 Gbps
+ On-demand/chip/hr $2.70 (US regions)
+ ──────────────────────────────────────────
+
+ Dataset
+ ──────────────────────────────────────────
+ Text 100B tokens (~400 GB)
+ Audio Emilia subset
+ 200k × 20s clips (~1.1k hrs)
+ ~200M audio tokens
+ ~13M transcription tokens
+ ~27 GB (from ~4.5 TB full dataset)
+ ──────────────────────────────────────────
+
+ 300M multimodal training estimates
+ ──────────────────────────────────────────
+ Total FLOPs 6 × 300M × 100B = 1.8e20
+ MFU (300M model) ~15% → ~138 effective TFLOPS/chip
+ Audio overhead <1% of total (negligible)
+
+ ┌────────────────────┬──────────┬──────────┬──────────┐
+ │ │ v6e-4 │ v6e-8 │ v6e-16 │
+ ├────────────────────┼──────────┼──────────┼──────────┤
+ │ Chips │ 4 │ 8 │ 16 │
+ │ Total HBM │ 128 GB │ 256 GB │ 512 GB │
+ │ Scaling eff. │ 1.0× │ 0.9× │ 0.8× │
+ │ Eff. TFLOPS │ 551 │ 992 │ 1,766 │
+ │ Est. time │ ~96h │ ~50h │ ~29h │
+ │ On-demand $/hr │ $10.80 │ $21.60 │ $43.20 │
+ │ Est. total cost │ ~$1,040 │ ~$1,080 │ ~$1,250 │
+ └────────────────────┴──────────┴──────────┴──────────┘
```
-## Setup For TPU/GCP
+## Setup For TPU/GCP
- Setup gcloud 1: download the `macOS ARM` from [here](https://docs.cloud.google.com/sdk/docs/install-sdk) and uzip.
- Setup gcloud 2: open terminal, cd to ypur downloads and run `./google-cloud-sdk/install.sh`
@@ -159,8 +195,8 @@ needle tpu [command]
┌───────────────────────────────────────────────────────────────────┐
│ │
│ create NAME Create TPU (auto-finds zone) │
- │ --type STR Accelerator type (default: v5litepod-4) │
- │ --version STR TPU OS (default: tpu-ubuntu2204-base) │
+ │ --type STR Accelerator type (default: v6e-4) │
+ │ --version STR TPU OS (auto-detected from --type) │
│ │
│ connect NAME SSH config + first connect (auto-zone) │
│ claude NAME Install Claude Code on instance │
@@ -208,4 +244,4 @@ needle tpu [command]
7. (Optional) Delete instance when no longer needed
needle tpu delete my-experiment
-```
\ No newline at end of file
+```
diff --git a/experiements/datasets.txt b/experiements/datasets.txt
new file mode 100644
index 0000000..e69de29
diff --git a/experiements/pruning.txt b/experiements/pruning.txt
index a3ca264..59828d7 100644
--- a/experiements/pruning.txt
+++ b/experiements/pruning.txt
@@ -1,161 +1,175 @@
================================================================================
-STRUCTURED COMPRESSION OF SMALL ENCODER-DECODER TRANSFORMERS:
-JOINT GROUP-WISE SPARSIFICATION AND LAYER PRUNING
+MULTI-AXIS STRUCTURED COMPRESSION OF SMALL ENCODER-DECODER TRANSFORMERS:
+MATRYOSHKA DIMENSIONS, BLOCK SPARSITY, LAYER PRUNING, AND INT4 QAT
================================================================================
-Target venue: ICML 2026
+Target venue: ICLR 2026
-Thesis: Structured compression (group-aligned block sparsity + layer removal)
-applied in a staged schedule during training yields Pareto-optimal accuracy-
-efficiency tradeoffs for small encoder-decoder models, matching or exceeding
-post-hoc pruning at equivalent FLOPs reduction while preserving INT4
-quantization compatibility.
+Thesis: Four complementary compression axes — Matryoshka dimension pruning,
+group-aligned block sparsity, layer removal, and INT4 quantization-aware
+training — can be jointly applied during training in a staged schedule to
+produce a single checkpoint that is deployable at multiple accuracy-efficiency
+operating points. Training once at d_model=D with MRL loss at dimensions
+{D, D/2, D/4, D/8, ...} yields a family of models from one training run,
+each further compressible via sparsification and layer pruning.
================================================================================
SECTION 1 — EXPERIMENTAL AXES
================================================================================
Independent variables:
- A. Sparsity ratio s ∈ {0.0, 0.10, 0.25, 0.40, 0.50}
- B. Layer prune ratio l ∈ {0.0, 0.125, 0.25, 0.375, 0.50}
- C. Group size g ∈ {16, 32, 64, 128}
- D. Pruning schedule ∈ {staged, simultaneous, reverse-order}
- E. FFN activation ∈ {dReLU, SwiGLU, GeGLU}
- F. Model scale ∈ {BASE-256, LARGE-512}
- G. QAT ∈ {off, int4-group32}
-
-Dependent variables (measured):
+ A. MRL deployment dim d' ∈ {1024, 512, 256, 128, 64}
+ B. Sparsity ratio s ∈ {0.0, 0.10, 0.25, 0.33, 0.50}
+ C. Layer prune ratio l ∈ {0.0, 0.125, 0.25, 0.375, 0.50}
+ D. Group size g ∈ {16, 32, 64, 128}
+ E. Pruning schedule ∈ {staged, simultaneous, reverse-order}
+ F. FFN activation ∈ {dReLU, SwiGLU, GeGLU}
+ G. Model scale ∈ {TOY-128, MAIN-1024}
+ H. QAT ∈ {off, int4-group32}
+
+Dependent variables (measured at EACH MRL dimension):
- Validation perplexity (TinyStories)
+ - Quantized validation perplexity (INT4)
- WikiText-2 perplexity
- LAMBADA accuracy
- HellaSwag accuracy
- ARC-Easy accuracy
+ - Estimated parameter count at deployment dim
- Throughput (tokens/sec on inference)
- Bigram repetition rate (generation diversity)
- - Parameter count (absolute and % of dense baseline)
- - Effective FLOPs (params * density)
- - Weight sparsity (% near-zero, measured at threshold < 1e-6)
+ - Block sparsity (% of fully-zero groups, group_size=g)
+ - Effective FLOPs (est. params × density)
================================================================================
SECTION 2 — MODEL CONFIGURATIONS
================================================================================
-BASE-256 (25.4M params, default):
- d_model=256, num_heads=4, num_kv_heads=2, num_layers=4,
- d_ff=1024, max_seq_len=256, vocab=65536, dtype=bfloat16
+Architecture: MemoryMixer encoder + Transformer decoder, shared embedding,
+GQA (grouped query attention), ZC-RMSNorm, RoPE, SentencePiece BPE (8192).
-LARGE-512 (≈100M params, scaling study):
- d_model=512, num_heads=8, num_kv_heads=4, num_layers=8,
- d_ff=2048, max_seq_len=256, vocab=65536, dtype=bfloat16
+TOY-128 (rapid iteration):
+ mrl_dims=[128, 64], d_model=128, num_heads=4, num_kv_heads=2,
+ num_enc_layers=2, num_dec_layers=2, d_ff=512,
+ max_enc_len=128, max_dec_len=128, vocab=8192, dtype=bfloat16
+ needle train --toy --seed {42,137,256}
-Both trained on TinyStories (500K samples, encoder-decoder split at 30%).
+MAIN-1024 (paper results):
+ mrl_dims=[1024, 512, 256, 128, 64], d_model=1024, num_heads=4,
+ num_kv_heads=2, num_enc_layers=4, num_dec_layers=2, d_ff=4096,
+ max_enc_len=256, max_dec_len=256, vocab=8192, dtype=bfloat16
+ needle train --seed {42,137,256}
+
+Both trained on TinyStories (1M samples, encoder-decoder split at 30%).
Optimizer: Muon (kernels) + AdamW (embeddings/norms), WSD schedule.
+All experiments report metrics at each MRL dimension slice.
================================================================================
SECTION 3 — EXPERIMENT GROUPS
================================================================================
All experiments use 3 seeds (42, 137, 256) for statistical significance.
-Report mean ± std on all metrics.
+Report mean +/- std on all metrics. Every run automatically reports val PPL
+and INT4-quantized val PPL at each MRL dimension slice.
-------------------------------------------------------------------------------
-GROUP 1: Dense Baselines (Table 1)
+GROUP 1: Dense Baselines + MRL Dimension Scaling (Table 1, Figure 1)
-------------------------------------------------------------------------------
-Purpose: Establish upper-bound quality for each model scale.
-
- 1a. BASE-256, 3 epochs, no pruning, no QAT
- needle train --epochs 3 --seed {42,137,256}
+Purpose: Establish quality at each deployment dimension from a single training
+run. This is the MRL baseline — no sparsification or layer pruning.
- 1b. BASE-256, 3 epochs, no pruning, with QAT (int4-group32)
- [QAT is always on in current code — this IS the baseline]
+ 1a. MAIN-1024, 3 epochs, no pruning, MRL dims [1024,512,256,128,64]
+ needle train --sparsity-ratio 0.0 --layer-prune-ratio 0.0 \
+ --seed {42,137,256}
- 1c. LARGE-512, 3 epochs, no pruning
- needle train --epochs 3 --d-model 512 --num-heads 8 --num-layers 8 \
- --max-enc-len 256 --max-dec-len 256 --seed {42,137,256}
+ 1b. TOY-128, 3 epochs, no pruning, MRL dims [128,64]
+ needle train --toy --sparsity-ratio 0.0 --layer-prune-ratio 0.0 \
+ --seed {42,137,256}
Deliverables:
- - Val perplexity, benchmark scores, throughput, generation samples
+ - Table: val PPL and quant PPL at each MRL dim (1024, 512, 256, 128, 64)
+ - Line plot: deployment dim vs val PPL — the MRL scaling curve (Figure 1a)
+ - Line plot: deployment dim vs estimated param count (Figure 1b)
+ - Benchmark scores at each dim
- These are the denominators for all relative comparisons
-------------------------------------------------------------------------------
-GROUP 2: Group-Wise Sparsification Only (Table 2, Figure 1)
+GROUP 2: MRL + Group-Wise Sparsification (Table 2, Figure 2)
-------------------------------------------------------------------------------
-Purpose: Characterise accuracy-sparsity Pareto frontier for block pruning alone.
-Schedule: Epoch 1 dense, epochs 2-3 with sparsity mask locked.
+Purpose: Characterise how MRL dimension scaling interacts with block sparsity.
+At smaller deployment dims, does sparsification hurt more or less?
- For s in {0.10, 0.25, 0.40, 0.50}:
- 2a. BASE-256, 3 epochs, --sparsity-ratio s
- needle train --epochs 3 --sparsity-ratio {s} --seed {42,137,256}
+Schedule: Epoch 1 dense, epochs 2-3 with sparsity mask locked.
- For s in {0.10, 0.25, 0.40, 0.50}:
- 2b. LARGE-512, 3 epochs, --sparsity-ratio s
- needle train --epochs 3 --d-model 512 --num-heads 8 --num-layers 8 \
- --sparsity-ratio {s} --seed {42,137,256}
+ For s in {0.10, 0.25, 0.33, 0.50}:
+ 2a. MAIN-1024, 3 epochs, --sparsity-ratio s
+ needle train --sparsity-ratio {s} --layer-prune-ratio 0.0 \
+ --seed {42,137,256}
Deliverables:
- - Pareto curve: sparsity % vs val perplexity (Figure 1a)
- - Pareto curve: sparsity % vs LAMBADA accuracy (Figure 1b)
- - Table: all metrics at each sparsity level
- - Analysis: at what sparsity does quality degrade >5% relative to baseline?
+ - 2D heatmap: (sparsity, MRL dim) → val PPL (Figure 2a)
+ - Pareto curve per MRL dim: sparsity % vs val PPL (Figure 2b)
+ - Table: all metrics at each (sparsity, dim) pair
+ - Analysis: does sparsification degrade smaller MRL slices more?
+ (Hypothesis: smaller slices are more sensitive because fewer dimensions
+ carry the representation — each zeroed weight matters more)
-------------------------------------------------------------------------------
-GROUP 3: Layer Pruning Only (Table 3, Figure 2)
+GROUP 3: MRL + Layer Pruning (Table 3, Figure 3)
-------------------------------------------------------------------------------
-Purpose: Characterise accuracy-compression frontier for layer removal alone.
-Schedule: Epoch 1 dense, epochs 2-3 with reduced architecture.
+Purpose: Characterise how MRL dimension scaling interacts with layer removal.
- For l in {0.125, 0.25, 0.375, 0.50}:
- 3a. BASE-256 (4+4 layers), 3 epochs, --layer-prune-ratio l
- needle train --epochs 3 --layer-prune-ratio {l} --seed {42,137,256}
- Expected surviving layers: 7, 6, 5, 4 (of 8 total)
+Schedule: Epoch 1 dense, epochs 2-3 with reduced architecture.
For l in {0.125, 0.25, 0.375, 0.50}:
- 3b. LARGE-512 (8+8 layers), 3 epochs, --layer-prune-ratio l
- needle train --epochs 3 --d-model 512 --num-heads 8 --num-layers 8 \
- --layer-prune-ratio {l} --seed {42,137,256}
- Expected surviving layers: 14, 12, 10, 8 (of 16 total)
+ 3a. MAIN-1024 (4 enc + 2 dec layers), 3 epochs, --layer-prune-ratio l
+ needle train --sparsity-ratio 0.0 --layer-prune-ratio {l} \
+ --seed {42,137,256}
+ Expected surviving layers: 5, 4-5, 4, 3 (of 6 total)
Deliverables:
- - Pareto curve: layer ratio vs val perplexity (Figure 2a)
- - Pareto curve: parameter count vs LAMBADA accuracy (Figure 2b)
- - Table: per-run encoder/decoder surviving layer counts
+ - 2D heatmap: (layer ratio, MRL dim) → val PPL (Figure 3a)
+ - Pareto curve: param count vs val PPL at each MRL dim (Figure 3b)
+ - Per-run encoder/decoder surviving layer counts
- Analysis: does the pruner prefer removing encoder or decoder layers?
- (Hypothesis: encoder layers are pruned more aggressively since the
- decoder carries more of the generation burden.)
+ (Hypothesis: encoder MemoryMixer blocks are pruned more aggressively
+ since the decoder carries more of the generation burden)
-------------------------------------------------------------------------------
-GROUP 4: Joint Sparsification + Layer Pruning (Table 4, Figure 3)
- *** CORE CONTRIBUTION ***
+GROUP 4: Full Pipeline — MRL + Sparsification + Layer Pruning (Table 4, Fig 4)
+ *** CORE CONTRIBUTION ***
-------------------------------------------------------------------------------
-Purpose: Show that combining both methods outperforms either alone at matched
-compression ratio. This is the central claim of the paper.
+Purpose: Show that all three compression axes compose — a single checkpoint
+trained with MRL loss, then sparsified and layer-pruned, produces a family
+of models spanning 10-100x parameter range. This is the central claim.
Schedule (staged):
- Epoch 1: dense training
- Epoch 2: apply group-wise sparsification, train with mask
- Epoch 3: apply layer pruning, train with reduced architecture + mask
- Epoch 4: recovery fine-tuning
+ Epoch 1: dense training with MRL loss
+ Epoch 2: apply group-wise sparsification, train with mask + MRL
+ Epoch 3: apply layer pruning, train reduced architecture + mask + MRL
+ (Optional epoch 4: recovery fine-tuning)
For (s, l) in grid:
{(0.10, 0.125), (0.10, 0.25),
(0.25, 0.125), (0.25, 0.25), (0.25, 0.375),
- (0.40, 0.125), (0.40, 0.25),
+ (0.33, 0.125), (0.33, 0.25),
(0.50, 0.25), (0.50, 0.50)}
- 4a. BASE-256, 4 epochs
- needle train --epochs 4 --sparsity-ratio {s} --layer-prune-ratio {l} \
+ 4a. MAIN-1024, 3 epochs
+ needle train --sparsity-ratio {s} --layer-prune-ratio {l} \
--seed {42,137,256}
- 4b. LARGE-512, select best 4 (s,l) pairs from 4a, 4 epochs
- needle train --epochs 4 --d-model 512 --num-heads 8 --num-layers 8 \
- --sparsity-ratio {s} --layer-prune-ratio {l} --seed {42,137,256}
+ 4b. Select best 4 (s,l) pairs from 4a, run with 4 epochs (recovery)
+ needle train --epochs 4 --sparsity-ratio {s} --layer-prune-ratio {l} \
+ --seed {42,137,256}
Deliverables:
- - 2D heatmap: (s, l) → val perplexity (Figure 3a)
- - 2D heatmap: (s, l) → effective parameter count (Figure 3b)
- - Pareto overlay: joint vs sparsity-only vs layer-only at matched
- compression (Figure 3c) — THIS IS THE KEY FIGURE
- - Table: full metrics for all (s, l) pairs
- - Analysis: multiplicative vs additive compression benefit
+ - 3D surface: (s, l, MRL dim) → val PPL (Figure 4a, sliced by dim)
+ - Pareto overlay at each MRL dim: full-pipeline vs sparsity-only vs
+ layer-only at matched effective params (Figure 4b) — KEY FIGURE
+ - Table: full metrics for all (s, l) pairs at each MRL dim
+ - Compression spectrum: for best (s, l), show the range of models:
+ d'=1024 (full) → d'=64 (smallest) with estimated params and PPL
+ - Analysis: multiplicative compression — MRL dimension reduction ×
+ sparsity × layer removal = total compression ratio
-------------------------------------------------------------------------------
GROUP 5: Schedule Ablation (Table 5)
@@ -164,42 +178,42 @@ Purpose: Justify the staged schedule (sparsify first, then prune layers).
Pick the best (s, l) pair from Group 4a. Run 3 schedules:
- 5a. Staged (default): sparsify epoch 2, layer prune epoch 3, recover epoch 4
- needle train --epochs 4 --sparsity-ratio {s} --layer-prune-ratio {l}
+ 5a. Staged (default): sparsify epoch 2, layer prune epoch 3
+ needle train --sparsity-ratio {s} --layer-prune-ratio {l}
- 5b. Reverse: layer prune epoch 2, sparsify epoch 3, recover epoch 4
+ 5b. Reverse: layer prune epoch 2, sparsify epoch 3
[Requires code change to swap epoch triggers]
- 5c. Simultaneous: both at epoch 2, recover epochs 3-4
+ 5c. Simultaneous: both at epoch 2, recover epoch 3
[Requires code change to apply both at same epoch]
Deliverables:
- - Table: val perplexity and benchmark scores for each schedule
- - Analysis: why staged order matters (hypothesis: sparsification is
- lower-risk, lets model adapt before the more destructive layer removal)
+ - Table: val PPL at each MRL dim for each schedule
+ - Analysis: does schedule affect small MRL dims differently from large?
-------------------------------------------------------------------------------
-GROUP 6: Recovery Epoch Scaling (Figure 4)
+GROUP 6: Recovery Epoch Scaling (Figure 5)
-------------------------------------------------------------------------------
Purpose: How many recovery epochs are needed after compression?
Pick the best (s, l) pair from Group 4a. Vary total epochs:
For E in {2, 3, 4, 5, 6}:
- 6a. BASE-256, E epochs, --sparsity-ratio {s} --layer-prune-ratio {l}
- (pruning always at epochs 2 and 3; recovery = E - 3 epochs)
+ 6a. MAIN-1024, E epochs, --sparsity-ratio {s} --layer-prune-ratio {l}
+ (pruning at epochs 2 and 3; recovery = E - 3 epochs)
Deliverables:
- - Line plot: recovery epochs vs val perplexity (Figure 4)
- - Analysis: diminishing returns curve, recommend minimum recovery budget
+ - Line plot per MRL dim: recovery epochs vs val PPL (Figure 5)
+ - Analysis: do smaller MRL dims need more recovery?
-------------------------------------------------------------------------------
GROUP 7: QAT Interaction Study (Table 6)
-------------------------------------------------------------------------------
Purpose: Show that group-wise sparsification is compatible with INT4 QAT
-because both operate on the same group_size=32 granularity.
+because both operate on the same group_size=32 granularity. Measure at
+each MRL dimension — INT4 quantization error at smaller dims may compound.
- 7a. Dense + QAT (baseline, = Group 1b)
+ 7a. Dense + QAT (baseline, = Group 1a)
7b. Sparsified (s=0.25) + QAT
7c. Layer-pruned (l=0.25) + QAT
7d. Joint (s=0.25, l=0.25) + QAT
@@ -208,158 +222,142 @@ because both operate on the same group_size=32 granularity.
to not call _quantize_params.
Deliverables:
- - Table: QAT on vs off for each compression method
- - Analysis: does QAT hurt more when combined with sparsification?
- (Hypothesis: no, because zeroed groups have no quantization error)
- - Effective compression ratio: sparsity + INT4 → bits per parameter
+ - Table: (QAT on/off) × compression method × MRL dim → val PPL
+ - Analysis: does QAT degrade more at smaller MRL dims?
+ (Hypothesis: yes — fewer dimensions means each quantization error
+ has proportionally larger impact on output quality)
+ - Effective compression: MRL dim reduction × sparsity × INT4 → total
+ bits per effective parameter
-------------------------------------------------------------------------------
GROUP 8: Group Size Ablation (Table 7, Figure 6)
-------------------------------------------------------------------------------
-Purpose: Determine optimal granularity for block sparsification. Smaller groups
-give finer-grained pruning decisions (better accuracy retention) but produce
-less structured sparsity (harder to exploit in hardware). Larger groups are
-more hardware-friendly but coarser. The group size also controls QAT
-quantization granularity, so this directly tests the sparsity-quantization
-co-design hypothesis.
+Purpose: Determine optimal block granularity for sparsification.
-Note: requires adding --group-size CLI flag and threading it through to
-_block_prune_params and _fake_quantize_int4.
-
-Fixed configuration: BASE-256, s=0.25, 3 epochs, QAT on.
+Fixed: MAIN-1024, s=0.25, 3 epochs, QAT on.
For g in {16, 32, 64, 128}:
- 8a. Sparsification only (isolate group size effect on pruning)
- needle train --epochs 3 --sparsity-ratio 0.25 --group-size {g} \
- --seed {42,137,256}
+ 8a. Sparsification only
+ needle train --sparsity-ratio 0.25 --layer-prune-ratio 0.0 \
+ --group-size {g} --seed {42,137,256}
8b. Joint sparsification + layer pruning (best l from Group 4a)
- needle train --epochs 4 --sparsity-ratio 0.25 --layer-prune-ratio {l} \
+ needle train --sparsity-ratio 0.25 --layer-prune-ratio {l} \
--group-size {g} --seed {42,137,256}
-Additional comparison at high sparsity to stress-test granularity:
-
+ High sparsity stress test:
For g in {16, 32, 64, 128}:
8c. s=0.50, sparsification only
- needle train --epochs 3 --sparsity-ratio 0.50 --group-size {g} \
- --seed {42,137,256}
+ needle train --sparsity-ratio 0.50 --layer-prune-ratio 0.0 \
+ --group-size {g} --seed {42,137,256}
Deliverables:
- - Line plot: group size vs val perplexity at s=0.25 and s=0.50 (Figure 6a)
- - Line plot: group size vs LAMBADA accuracy (Figure 6b)
- - Table: all metrics at each group size for both sparsity levels
- - Analysis of key tradeoffs:
- (a) g=16: finest granularity, best accuracy, but 2× more scale factors
- to store than g=32 and least structured pattern
- (b) g=32: default, matches common INT4 quantization group size,
- good balance of accuracy and structure
- (c) g=64: coarser, each pruning decision covers 64 weights,
- more aggressive but more hardware-friendly
- (d) g=128: coarsest, each block is a full row for d_model=128,
- approaches per-channel granularity, may degrade significantly
- - Cross-reference with QAT: when group_size is shared between sparsification
- and quantization, zeroed groups save both weight storage AND scale storage
- (no scale needed for an all-zero group). Report effective bits-per-param
- accounting for this: bpp = (1 - sparsity) × (4 + 32/g) where 4 is INT4
- bits and 32/g is amortized scale overhead per weight.
+ - Line plot per MRL dim: group size vs val PPL (Figure 6a)
+ - Table: all metrics at each (group size, MRL dim)
+ - Effective bits-per-param: bpp = (1 - sparsity) * (4 + 32/g)
+ - Analysis: does optimal group size change with MRL deployment dim?
-------------------------------------------------------------------------------
-GROUP 9: Activation Function × Pruning Interaction (Table 8, Figure 8)
+GROUP 9: Activation Function x Pruning Interaction (Table 8, Figure 7)
*** KEY FINDING ***
-------------------------------------------------------------------------------
-Purpose: Test whether activation function choice determines pruning robustness.
-Early observation: dReLU dramatically increases post-prune perplexity compared
-to SwiGLU. This group systematically characterises the interaction.
+Purpose: Test whether activation function choice determines pruning robustness
+across MRL dimension slices.
Activation definitions:
- - dReLU: ReLU(gate) * ReLU(up) — naturally ~75% sparse activations
- - SwiGLU: SiLU(gate) * up — smooth, dense activations
- - GeGLU: GELU(gate) * up — smooth, nearly dense activations
-
-Requires: --activation CLI flag wired to FeedForward module.
+ - dReLU: ReLU(gate) * ReLU(up) — naturally ~75% sparse activations
+ - SwiGLU: SiLU(gate) * up — smooth, dense activations
+ - GeGLU: GELU(gate) * up — smooth, nearly dense activations
9a. Dense baselines (no pruning) — isolate activation quality
For act in {drelu, swiglu, geglu}:
- needle train --epochs 3 --activation {act} --sparsity-ratio 0.0 \
+ needle train --activation {act} --sparsity-ratio 0.0 \
--layer-prune-ratio 0.0 --seed {42,137,256}
9b. Sparsification only — test pruning robustness per activation
For act in {drelu, swiglu, geglu}, s in {0.25, 0.50}:
- needle train --epochs 3 --activation {act} --sparsity-ratio {s} \
- --seed {42,137,256}
+ needle train --activation {act} --sparsity-ratio {s} \
+ --layer-prune-ratio 0.0 --seed {42,137,256}
- 9c. Joint sparsification + layer pruning — full compression
+ 9c. Joint — full compression
For act in {drelu, swiglu, geglu}:
- needle train --epochs 4 --activation {act} --sparsity-ratio 0.25 \
+ needle train --activation {act} --sparsity-ratio 0.25 \
--layer-prune-ratio 0.25 --seed {42,137,256}
- 9d. LARGE-512 validation on best and worst activation from 9b-9c
- needle train --epochs 4 --d-model 512 --num-heads 8 --num-layers 8 \
- --activation {act} --sparsity-ratio 0.25 --layer-prune-ratio 0.25 \
- --seed {42,137,256}
+Deliverables:
+ - Grouped bar chart per MRL dim: activation × compression → val PPL (Fig 7a)
+ - Relative degradation: (pruned_ppl / dense_ppl) per activation per dim
+ (Figure 7b) — KEY CHART: pruning robustness not just raw quality
+ - Analysis: does dReLU's natural activation sparsity help or hurt at
+ smaller MRL dims where redundancy is already low?
+
+-------------------------------------------------------------------------------
+GROUP 10: MRL Dimension-Specific Analysis (Table 9, Figure 8)
+ *** MRL-SPECIFIC CONTRIBUTION ***
+-------------------------------------------------------------------------------
+Purpose: Deep-dive into what the model learns at each MRL dimension slice.
+
+Using best checkpoint from Group 4a:
+
+ 10a. Representation similarity across dims
+ - CKA (centered kernel alignment) between d'=1024 and each smaller d'
+ - Measure: do smaller dims learn a subspace of the full representation?
+
+ 10b. Per-dim generation quality
+ - Generate from each MRL dim slice at inference time
+ - Compare generation diversity, coherence, repetition rate
+
+ 10c. MRL loss weighting ablation
+ - Uniform weighting (current): L = (1/K) * sum(L_k)
+ - Scaled weighting: L = sum(d_k/D * L_k) (larger dims weighted more)
+ - Inverse weighting: L = sum(D/d_k * L_k) (smaller dims weighted more)
Deliverables:
- - Grouped bar chart: activation × {dense, s=0.25, s=0.50} → val ppl (Fig 8a)
- - Table: full metrics for all activation × compression combos
- - Relative degradation plot: (pruned_ppl / dense_ppl) per activation (Fig 8b)
- — THIS IS THE KEY CHART: shows pruning *robustness* not just raw quality
- - Analysis of competing hypotheses:
- (a) dReLU hurts pruning because its ~75% activation sparsity means fewer
- active paths compensate for zeroed weights — the network has less
- redundancy to absorb pruning damage
- (b) dReLU helps pruning because its natural sparsity pattern means weight
- blocks that process mostly-zero activations are truly unimportant
- and can be safely pruned (better signal for the pruning criterion)
- (c) Smooth activations (SwiGLU/GeGLU) spread information across more
- weights, creating redundancy that makes the model more robust to
- weight removal but harder to compress efficiently
- - Recommendation: which activation to use when pruning is planned vs not
-
- Additional metric for this group:
- - Activation sparsity: measure % of zeros in FFN intermediate activations
- for each activation function at each pruning level. Report as:
- act_sparsity = E[fraction of zeros in ReLU(gate) * ReLU(up)]
- This connects weight sparsity to activation sparsity.
+ - CKA heatmap: dim × dim similarity (Figure 8a)
+ - Table: generation quality metrics at each dim
+ - Ablation table: MRL loss weighting strategy × dim → val PPL
-------------------------------------------------------------------------------
-GROUP 10: Inference Efficiency Validation (Table 9, Figure 9)
+GROUP 11: Inference Efficiency Validation (Table 10, Figure 9)
-------------------------------------------------------------------------------
-Purpose: Translate theoretical compression into real speedup.
+Purpose: Translate theoretical compression into real speedup across the
+full spectrum of deployment options.
-For each Pareto-optimal checkpoint from Groups 2-4:
+For each Pareto-optimal checkpoint from Groups 2-4, at each MRL dim:
needle test --checkpoint {path} --throughput-runs 50
-Measure:
+Measure at each deployment dim:
- Tokens/sec (decode throughput)
- Latency per generation (ms)
- Peak memory (via JAX profiler)
Deliverables:
- - Scatter plot: compression ratio vs actual speedup (Figure 5)
- - Table: all Pareto-optimal models with throughput + quality
- - Analysis: does structured sparsity translate to real speedup?
- (Note: dense JAX matmuls don't skip zeros — speedup comes from
- layer removal. Sparsity benefit is in model size / memory only
- until sparse kernel support is added.)
+ - Scatter plot: effective params vs actual throughput (Figure 9a)
+ - Table: all Pareto-optimal (s, l, d') with throughput + quality
+ - The "menu" figure: one chart showing ALL deployment options from a
+ single checkpoint, from largest (d'=1024, no pruning) to smallest
+ (d'=64, s=0.50, l=0.50) — THIS IS THE PAPER'S HERO FIGURE
================================================================================
SECTION 4 — EVALUATION PROTOCOL
================================================================================
-For every checkpoint:
+For every checkpoint, evaluate at EACH MRL dimension slice:
- 1. Validation perplexity (TinyStories val split, full)
+ 1. Validation perplexity (TinyStories val split)
+ - Full-precision val PPL
+ - INT4-quantized val PPL (at each MRL dim)
needle test --checkpoint {path} --max-eval-samples 5000
- 2. Benchmark suite
+ 2. Benchmark suite (at full dim and smallest viable dim)
needle evaluate --checkpoint {path} \
--benchmarks wikitext2 lambada hellaswag arc_easy --max-samples 500
- 3. Generation quality (qualitative + bigram diversity)
+ 3. Generation quality at each MRL dim
needle test --checkpoint {path}
- → Inspect 10 sample generations, report bigram repetition rate
+ Report: bigram repetition rate, avg generation length, sample outputs
- 4. Throughput measurement
+ 4. Throughput at each MRL dim
needle test --checkpoint {path} --throughput-runs 50
================================================================================
@@ -367,50 +365,51 @@ SECTION 5 — EXPECTED TABLES AND FIGURES
================================================================================
Tables:
- T1. Dense baselines (2 scales × {with,without} QAT)
- T2. Group-wise sparsification sweep (2 scales × 4 ratios)
- T3. Layer pruning sweep (2 scales × 4 ratios)
- T4. Joint compression grid (9 pairs × 2 scales)
- T5. Schedule ablation (3 schedules)
- T6. QAT interaction (4 methods × {on,off})
- T7. Group size ablation (4 sizes × 2 sparsity levels + joint)
- T8. Activation × pruning interaction (3 activations × 4 compression levels)
- T9. Inference efficiency (Pareto-optimal checkpoints)
+ T1. Dense baselines + MRL scaling curve (val PPL at each dim)
+ T2. MRL + sparsification (sparsity × dim → val PPL)
+ T3. MRL + layer pruning (layer ratio × dim → val PPL)
+ T4. Full pipeline grid (s × l at each MRL dim)
+ T5. Schedule ablation (3 schedules × MRL dims)
+ T6. QAT interaction (4 methods × {on,off} × MRL dims)
+ T7. Group size ablation (4 sizes × MRL dims)
+ T8. Activation × pruning interaction (3 act × compression × MRL dims)
+ T9. MRL dimension analysis (CKA, generation quality, loss weighting)
+ T10. Inference efficiency (Pareto-optimal checkpoints × MRL dims)
Figures:
- F1. Sparsity-only Pareto: sparsity % vs perplexity + LAMBADA
- F2. Layer-prune-only Pareto: param count vs perplexity + LAMBADA
- F3. Joint compression: (a) heatmap, (b) param heatmap, (c) Pareto overlay
- F4. Recovery epoch scaling curve
- F5. Compression ratio vs actual inference speedup
- F6. Group size: (a) g vs perplexity at s=0.25/0.50, (b) g vs LAMBADA
- F7. Effective bits-per-param vs quality across group sizes
- F8. Activation × pruning: (a) grouped bars, (b) relative degradation ratio
- F9. Activation sparsity vs weight sparsity interaction
+ F1. MRL scaling curve: deployment dim vs PPL + param count
+ F2. MRL + sparsity: (a) heatmap, (b) Pareto per dim
+ F3. MRL + layer pruning: (a) heatmap, (b) Pareto per dim
+ F4. Full pipeline: (a) 3D surface slices, (b) Pareto overlay — KEY FIGURE
+ F5. Recovery epoch scaling per MRL dim
+ F6. Group size: (a) g vs PPL per dim, (b) effective bpp
+ F7. Activation × pruning: (a) grouped bars per dim, (b) degradation ratio
+ F8. MRL representation analysis: (a) CKA heatmap, (b) gen quality vs dim
+ F9. Hero figure: full deployment menu from single checkpoint
================================================================================
SECTION 6 — COMPUTE BUDGET
================================================================================
-Per-run cost (BASE-256, 4 epochs, 500K samples, 4× TPU v5 lite):
- ~10 minutes training + ~2 minutes evaluation
+Per-run cost (MAIN-1024, 3 epochs, 1M samples, 4x TPU v5 lite):
+ ~15 minutes training + ~5 minutes MRL evaluation (5 dims × val + quant)
Estimated total runs:
Group 1: 2 configs × 3 seeds = 6 runs
- Group 2: 4 ratios × 2 scales × 3 seeds = 24 runs
- Group 3: 4 ratios × 2 scales × 3 seeds = 24 runs
- Group 4: 9 pairs × 3 seeds + 4 pairs × 3 seeds (large) = 39 runs
- Group 5: 3 schedules × 3 seeds = 9 runs
- Group 6: 5 epoch counts × 3 seeds = 15 runs
- Group 7: 4 methods × 2 (QAT on/off) × 3 seeds = 24 runs
- Group 8: 4 sizes × 3 configs (s=.25, s=.50, joint) × 3 = 36 runs
- Group 9: 3 act × (3 dense + 6 sparse + 3 joint) × 3 = 36 runs
- + 2 act × 3 seeds (large validation) = 6 runs
- Group 10: evaluation only (reuses checkpoints) = 0 runs
- ─────────────────────────────────────────────────────────────────────
- Total: 219 runs
-
-At ~12 min/run: ~44 TPU-hours (very manageable on v5litepod-4)
+ Group 2: 4 ratios × 3 seeds = 12 runs
+ Group 3: 4 ratios × 3 seeds = 12 runs
+ Group 4: 9 pairs × 3 seeds + 4 pairs × 3 seeds (recov)= 39 runs
+ Group 5: 3 schedules × 3 seeds = 9 runs
+ Group 6: 5 epoch counts × 3 seeds = 15 runs
+ Group 7: 4 methods × 2 (QAT on/off) × 3 seeds = 24 runs
+ Group 8: 4 sizes × 3 configs × 3 seeds = 36 runs
+ Group 9: 3 act × (3 + 6 + 3) × 3 seeds = 36 runs
+ Group 10: 3 weighting strategies × 3 seeds = 9 runs
+ Group 11: evaluation only (reuses checkpoints) = 0 runs
+ ────────────────────────────────────────────────────────────────────
+ Total: 198 runs
+
+At ~20 min/run: ~66 TPU-hours (manageable on v5litepod-4)
================================================================================
SECTION 7 — ABLATION CHECKLIST
@@ -418,41 +417,76 @@ SECTION 7 — ABLATION CHECKLIST
Before submission, verify these are addressed:
- [ ] Statistical significance: 3 seeds, report mean ± std
- [ ] Matched comparisons: compare joint vs individual at SAME compression
- [ ] Scaling behavior: BASE-256 and LARGE-512 show consistent trends
+ [ ] Statistical significance: 3 seeds, report mean +/- std
+ [ ] MRL scaling: smooth degradation curve from full dim to smallest
+ [ ] MRL + compression composability: all three axes combine gracefully
+ [ ] Matched comparisons: compare methods at SAME effective param count
[ ] Schedule sensitivity: staged vs reverse vs simultaneous
[ ] Recovery saturation: diminishing returns on fine-tuning epochs
- [ ] QAT compatibility: sparsity + quantization are not antagonistic
+ [ ] QAT compatibility: sparsity + quantization + MRL are not antagonistic
[ ] Pruning bias: report which layers (enc vs dec) get pruned
- [ ] Generation quality: not just perplexity, show actual text samples
- [ ] Group size sensitivity: optimal g, and does it interact with model scale?
- [ ] Activation × pruning: does dReLU help or hurt compression robustness?
- [ ] Activation sparsity measured: connect weight sparsity to activation sparsity
- [ ] Honest limitations: sparse matmuls don't accelerate on current JAX
+ [ ] Generation quality at each MRL dim, not just perplexity
+ [ ] Group size sensitivity: optimal g, interaction with MRL dim
+ [ ] Activation × pruning × MRL dim: three-way interaction
+ [ ] MRL representation analysis: CKA, subspace structure
+ [ ] Honest limitations: sparse matmuls don't accelerate on current JAX;
+ MRL dimension reduction requires custom kernel for real speedup
+ [ ] Comparison to baselines: training K separate small models vs one MRL
================================================================================
SECTION 8 — EXECUTION ORDER
================================================================================
-Phase 1 (Day 1): Groups 1 + 2 + 3 (54 runs, ~11 hrs)
- Run all baselines and single-method sweeps in parallel across seeds.
- Immediate deliverable: Pareto curves for each method alone.
+Phase 1 (Day 1): Groups 1 + 2 + 3 (30 runs, ~10 hrs)
+ Baselines, sparsity-only, layer-prune-only, all with MRL evaluation.
+ Deliverable: MRL scaling curves and single-axis Pareto frontiers.
-Phase 2 (Day 2): Group 4 (39 runs, ~8 hrs)
- Joint compression grid. This produces the key result.
- Immediate deliverable: heatmaps and overlay Pareto.
+Phase 2 (Day 2): Group 4 (39 runs, ~13 hrs)
+ Full pipeline grid. This produces the key result.
+ Deliverable: multi-axis Pareto and deployment menu.
-Phase 3 (Day 3): Groups 5 + 6 + 7 + 8 (84 runs, ~17 hrs)
- Ablations: schedule, recovery, QAT interaction, group size.
- Immediate deliverable: complete ablation tables.
+Phase 3 (Day 3): Groups 5 + 6 + 7 (48 runs, ~16 hrs)
+ Schedule, recovery, QAT ablations.
+ Deliverable: complete ablation tables.
-Phase 4 (Day 4): Group 9 (42 runs, ~8 hrs)
- Activation × pruning interaction study.
- Immediate deliverable: activation robustness table + degradation plot.
+Phase 4 (Day 4): Groups 8 + 9 (72 runs, ~24 hrs)
+ Group size and activation interaction studies.
+ Deliverable: granularity and activation robustness analysis.
+
+Phase 5 (Day 5): Groups 10 + 11 + writing (9 runs + eval)
+ MRL analysis, inference benchmarks, paper draft.
+ Deliverable: hero figure, all tables populated.
+
+================================================================================
+SECTION 9 — PAPER NARRATIVE
+================================================================================
-Phase 5 (Day 5): Group 10 + writing
- Run inference benchmarks on all Pareto-optimal checkpoints.
- Begin paper draft with all tables and figures populated.
+Framing: "Train once, deploy anywhere."
+
+A single training run with Matryoshka loss produces a checkpoint that can be
+deployed at any dimension slice. Each slice can be further compressed with
+group-wise sparsification and layer pruning. The result is a deployment MENU:
+
+ ┌─────────────────────────────────────────────────────────────────────┐
+ │ Deployment option Est. params Val PPL Quant PPL │
+ │ ───────────────────────────────────────────────────────────────── │
+ │ d'=1024, full ~44M X.XX X.XX │
+ │ d'=512, full ~11M X.XX X.XX │
+ │ d'=512, s=0.33 ~7M X.XX X.XX │
+ │ d'=256, full ~3M X.XX X.XX │
+ │ d'=256, s=0.33 ~2M X.XX X.XX │
+ │ d'=256, s=0.33, l=0.25 ~1.5M X.XX X.XX │
+ │ d'=128, full ~0.8M X.XX X.XX │
+ │ d'=128, s=0.33 ~0.5M X.XX X.XX │
+ │ d'=64, full ~0.2M X.XX X.XX │
+ │ d'=64, s=0.33, l=0.25 ~0.1M X.XX X.XX │
+ └─────────────────────────────────────────────────────────────────────┘
+
+Key claims:
+ 1. MRL dimension pruning is nearly free — minimal degradation at 2x reduction
+ 2. Sparsification and layer pruning compose multiplicatively with MRL
+ 3. A single checkpoint spans 100x+ parameter range
+ 4. INT4 QAT is compatible with all three compression axes
+ 5. MemoryMixer encoder is more compressible than standard transformer encoder
================================================================================
diff --git a/src/cli.py b/src/cli.py
index 98d56bf..ebde0da 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -10,26 +10,34 @@ HELP = """
│ ...the tiny model to rule them all... │
│ │
│ train │
- │ --epochs INT Training epochs (default: 1) │
+ │ --toy Use toy config for quick iteration │
+ │ --epochs INT Training epochs (default: 3) │
│ --batch-size INT Batch size (default: 32) │
│ --lr FLOAT AdamW learning rate (default: 3e-4) │
│ --muon-lr FLOAT Muon learning rate (default: 0.02) │
- │ --d-model INT Model dimension (default: 128) │
+ │ --d-model INT Model dim (default: max of mrl-dims) │
│ --num-heads INT Attention heads (default: 4) │
- │ --num-layers INT Encoder/decoder layers (default: 2) │
- │ --max-enc-len INT Max encoder seq length (default: 128) │
- │ --max-dec-len INT Max decoder seq length (default: 128) │
- │ --max-samples INT Training samples (default: 20000) │
+ │ --num-layers INT Encoder layers (default: 12) │
+ │ --num-dec-layers INT Decoder layers (default: 4) │
+ │ --max-enc-len INT Max encoder seq length (default: 256) │
+ │ --max-dec-len INT Max decoder seq length (default: 256) │
+ │ --max-samples INT Training samples (default: 1000000) │
+ │ --mrl-dims INT [...] MRL dim targets (default: 512 256 ..) │
+ │ --sparsity-ratio FLOAT Block prune ratio (default: 0.33) │
+ │ --layer-prune-ratio FL Layer prune ratio (default: 0.0) │
+ │ --group-size INT Quant/prune group size (default: 32) │
+ │ --activation STR drelu|swiglu|geglu (default: drelu) │
+ │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │
+ │ --eval-every INT Val eval interval (default: 1000) │
+ │ --wandb Enable W&B logging │
+ │ --checkpoint PATH Resume from checkpoint │
│ --checkpoint-dir DIR Checkpoint directory │
- │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │
│ --seed INT Random seed (default: 42) │
│ │
- │ │
- │ sweep │
- │ --sweep-config PATH Sweep YAML config (default: sweep.yaml)│
- │ --project STR W&B project name (default: needle-v1)│
- │ --count INT Number of trials (default: 20) │
- │ (also accepts all train flags as defaults for non-swept params)│
+ │ sweep │
+ │ --sweep-config PATH Sweep YAML config │
+ │ --project STR W&B project name (default: needle-v1) │
+ │ --count INT Number of trials (default: 20) │
│ │
│ run │
│ --checkpoint PATH Path to model checkpoint (required) │
@@ -53,7 +61,7 @@ HELP = """
│ │
│ tpu │
│ create NAME Create TPU (auto-finds zone) │
- │ --type STR Accelerator (default: v5litepod-4) │
+ │ --type STR Accelerator (default: v6e-4) │
│ --version STR TPU OS (auto-detected from --type) │
│ connect NAME SSH config + connect (auto-zone) │
│ claude NAME Install Claude Code on instance │
@@ -67,45 +75,31 @@ HELP = """
"""
TOY_CONFIG = {
- "epochs": 1,
- "batch_size": 32,
- "lr": 3e-4,
- "muon_lr": 0.02,
- "d_model": 128,
"num_heads": 4,
"num_layers": 2,
"num_dec_layers": 2,
"max_enc_len": 128,
"max_dec_len": 128,
"max_samples": 10000,
- "warmup_ratio": 0.05,
- "dtype": "bfloat16",
- "seed": 42,
}
-BASE_CONFIG = {
- "epochs": 1,
- "batch_size": 32,
- "lr": 3e-4,
- "muon_lr": 0.02,
- "d_model": 512,
+MAIN_CONFIG = {
"num_heads": 4,
- "num_layers": 4,
- "num_dec_layers": 2,
+ "num_layers": 12,
+ "num_dec_layers": 4,
"max_enc_len": 256,
"max_dec_len": 256,
- "max_samples": 500000,
- "warmup_ratio": 0.05,
- "dtype": "bfloat16",
- "seed": 42,
+ "max_samples": 1000000,
}
def _apply_train_defaults(args):
- config = TOY_CONFIG if args.toy else BASE_CONFIG
+ config = TOY_CONFIG if args.toy else MAIN_CONFIG
for key, value in config.items():
if getattr(args, key, None) is None:
setattr(args, key, value)
+ if getattr(args, "d_model", None) is None:
+ args.d_model = max(args.mrl_dims)
def main():
@@ -119,10 +113,10 @@ def main():
p = sub.add_parser("train", add_help=False)
p.add_argument("--toy", action="store_true")
p.add_argument("--checkpoint", type=str, default=None)
- p.add_argument("--epochs", type=int, default=None)
- p.add_argument("--batch-size", type=int, default=None)
- p.add_argument("--lr", type=float, default=None)
- p.add_argument("--muon-lr", type=float, default=None)
+ p.add_argument("--epochs", type=int, default=3)
+ p.add_argument("--batch-size", type=int, default=32)
+ p.add_argument("--lr", type=float, default=3e-4)
+ p.add_argument("--muon-lr", type=float, default=0.02)
p.add_argument("--d-model", type=int, default=None)
p.add_argument("--num-heads", type=int, default=None)
p.add_argument("--num-layers", type=int, default=None)
@@ -130,18 +124,20 @@ def main():
p.add_argument("--max-enc-len", type=int, default=None)
p.add_argument("--max-dec-len", type=int, default=None)
p.add_argument("--max-samples", type=int, default=None)
- p.add_argument("--warmup-ratio", type=float, default=None)
+ p.add_argument("--warmup-ratio", type=float, default=0.05)
p.add_argument("--wandb", action="store_true")
- p.add_argument("--dtype", type=str, default=None, choices=["float32", "bfloat16"])
+ p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])
p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
- p.add_argument("--seed", type=int, default=None)
+ p.add_argument("--seed", type=int, default=42)
p.add_argument("--eval-every", type=int, default=1000)
p.add_argument("--max-eval-samples", type=int, default=None)
- p.add_argument("--sparsity-ratio", type=float, default=0.0)
+ p.add_argument("--sparsity-ratio", type=float, default=0.33)
p.add_argument("--layer-prune-ratio", type=float, default=0.0)
p.add_argument("--group-size", type=int, default=32)
p.add_argument("--activation", type=str, default="drelu", choices=["drelu", "swiglu", "geglu"])
p.add_argument("--num-memory-slots", type=int, default=64)
+ p.add_argument("--mrl-dims", type=int, nargs="*", default=[512, 256, 128, 64],
+ help="MRL dimension pruning targets (default: 512 256 128 64)")
p = sub.add_parser("run", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
@@ -165,7 +161,7 @@ def main():
p.add_argument("--project", type=str, default="needle-v1")
p.add_argument("--count", type=int, default=20)
p.add_argument("--epochs", type=int, default=1)
- p.add_argument("--batch-size", type=int, default=32)
+ p.add_argument("--batch-size", type=int, default=16)
p.add_argument("--lr", type=float, default=3e-4)
p.add_argument("--muon-lr", type=float, default=0.02)
p.add_argument("--d-model", type=int, default=128)
@@ -190,7 +186,7 @@ def main():
tp = tpu_sub.add_parser("create", add_help=False)
tp.add_argument("name", type=str)
- tp.add_argument("--type", dest="accel_type", type=str, default="v5litepod-4")
+ tp.add_argument("--type", dest="accel_type", type=str, default="v6e-4")
tp.add_argument("--version", type=str, default=None,
help="Software version (auto-detected from --type if omitted)")
diff --git a/src/data.py b/src/data.py
index a2c7471..7c572b8 100644
--- a/src/data.py
+++ b/src/data.py
@@ -63,7 +63,7 @@ class NeedleTokenizer:
return {"input_ids": all_ids}
-def train_tokenizer(vocab_size=8192, max_samples=100000):
+def train_tokenizer(vocab_size=8192, max_samples=None):
"""Train a SentencePiece BPE tokenizer on TinyStories."""
model_path = TOKENIZER_PREFIX + ".model"
if os.path.exists(model_path):
@@ -72,14 +72,15 @@ def train_tokenizer(vocab_size=8192, max_samples=100000):
os.makedirs(TOKENIZER_DIR, exist_ok=True)
- print(f"Training SentencePiece BPE tokenizer (vocab_size={vocab_size})...")
ds = load_dataset("roneneldan/TinyStories", split="train")
- indices = range(min(max_samples, len(ds)))
- ds_subset = ds.select(indices)
+ if max_samples:
+ ds = ds.select(range(min(max_samples, len(ds))))
+
+ print(f"Training SentencePiece BPE tokenizer (vocab_size={vocab_size}, samples={len(ds):,})...")
corpus_path = os.path.join(TOKENIZER_DIR, "corpus.txt")
with open(corpus_path, "w") as f:
- for example in tqdm(ds_subset, desc="Writing corpus"):
+ for example in tqdm(ds, desc="Writing corpus"):
text = example["text"].strip()
if text:
f.write(text + "\n")
@@ -105,10 +106,10 @@ def train_tokenizer(vocab_size=8192, max_samples=100000):
return model_path
-def get_tokenizer():
+def get_tokenizer(max_samples=None):
model_path = TOKENIZER_PREFIX + ".model"
if not os.path.exists(model_path):
- train_tokenizer()
+ train_tokenizer(max_samples=max_samples)
return NeedleTokenizer(model_path)
@@ -119,8 +120,18 @@ def load_tinystories(split="train", max_samples=None):
return ds
-def _cache_key(n_samples, max_enc_len, max_dec_len, tokenizer_name="needle_spm"):
- key = f"tinystories_{tokenizer_name}_{n_samples}_{max_enc_len}_{max_dec_len}"
+def _tokenizer_hash():
+ """Hash the tokenizer model file to detect retraining."""
+ model_path = TOKENIZER_PREFIX + ".model"
+ if os.path.exists(model_path):
+ with open(model_path, "rb") as f:
+ return hashlib.md5(f.read()).hexdigest()[:8]
+ return "none"
+
+
+def _cache_key(n_samples, max_enc_len, max_dec_len):
+ tok_hash = _tokenizer_hash()
+ key = f"tinystories_{tok_hash}_{n_samples}_{max_enc_len}_{max_dec_len}"
return hashlib.md5(key.encode()).hexdigest()[:12]
diff --git a/src/model.py b/src/model.py
index fe0c995..a8140ba 100644
--- a/src/model.py
+++ b/src/model.py
@@ -325,14 +325,26 @@ class EncoderDecoderTransformer(nn.Module):
)
return logits
- def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None):
+ def forward_with_aux(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, mrl_dims=None):
encoder_out = self.encode(src, src_mask=src_mask)
- logits = self.decode(tgt, encoder_out, self_mask=tgt_mask)
+ x = self.embedding(tgt) * self.embed_scale
+ rope = self._rope(tgt.shape[1])
+ x = self.decoder(x, encoder_out, self_mask=tgt_mask, cross_mask=None, rope=rope)
+ x_f32 = x.astype(jnp.float32)
+ emb = self.embedding.embedding
+ logits = x_f32 @ emb.T
+
+ mrl_logits = []
+ if mrl_dims is not None:
+ for d in mrl_dims:
+ if d < self.config.d_model:
+ mrl_logits.append(x_f32[..., :d] @ emb[:, :d].T)
+
s = encoder_out.astype(jnp.float32)
- gram = jnp.matmul(s, s.transpose(0, 2, 1))
+ gram = jnp.matmul(s, s.transpose(0, 2, 1))
diag_sq = jnp.sum(jnp.diagonal(gram, axis1=1, axis2=2) ** 2)
slot_div = (jnp.sum(gram ** 2) - diag_sq) / s.shape[0]
- return logits, slot_div
+ return logits, slot_div, mrl_logits
def make_causal_mask(seq_len):
diff --git a/src/tpu.py b/src/tpu.py
index c4f286e..717d8ce 100644
--- a/src/tpu.py
+++ b/src/tpu.py
@@ -7,12 +7,6 @@ import sys
PROJECT = "needle-488623"
VERSION_FOR_TYPE = {
- "v2": "tpu-ubuntu2204-base",
- "v3": "tpu-ubuntu2204-base",
- "v4": "tpu-ubuntu2204-base",
- "v5litepod": "v2-alpha-tpuv5-lite",
- "v5e": "v2-alpha-tpuv5-lite",
- "v5p": "v2-alpha-tpuv5",
"v6e": "v2-alpha-tpuv6e",
}
@@ -37,16 +31,15 @@ def _resolve_version(accel_type, explicit_version):
return "tpu-ubuntu2204-base"
+# Trillium (v6e) zones, ordered by on-demand price (cheapest first)
+# us-east1/us-east5: $2.70/chip/hr
+# europe-west4: $2.97/chip/hr
+# asia-northeast1: $3.24/chip/hr
ZONES = [
- "us-central1-a", "us-central1-b", "us-central1-f",
"us-east1-b", "us-east1-c", "us-east1-d",
"us-east5-a", "us-east5-b",
- "us-south1-a", "us-south1-b",
- "us-west1-a", "us-west1-b",
- "us-west4-a",
"europe-west4-a",
"asia-northeast1-b",
- "southamerica-west1-a",
]
TPU_HELP = """
diff --git a/src/train.py b/src/train.py
index ee9a098..9964749 100644
--- a/src/train.py
+++ b/src/train.py
@@ -282,6 +282,7 @@ def _quantize_params(params, group_size=32):
_GROUP_SIZE = 32
+_MRL_DIMS = ()
def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):
@@ -293,23 +294,37 @@ def _train_step(state, ema_params, src, tgt_in, tgt_out, causal_mask):
tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
cross_mask = src_mask
- logits, slot_div = state.apply_fn(
+ logits, slot_div, mrl_logits = state.apply_fn(
{"params": _quantize_params(params, group_size=_GROUP_SIZE)},
src,
tgt_in,
src_mask=src_mask,
tgt_mask=tgt_mask,
cross_mask=cross_mask,
+ mrl_dims=_MRL_DIMS if _MRL_DIMS else None,
method="forward_with_aux",
)
logits_f32 = logits.astype(jnp.float32)
- loss = optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out)
mask = (tgt_out != pad_id).astype(jnp.float32)
- ce_loss = jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)
+ ce_loss = jnp.sum(
+ optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * mask
+ ) / jnp.maximum(jnp.sum(mask), 1.0)
+
+ total_loss = ce_loss
+ num_scales = 1
+ for mrl_log in mrl_logits:
+ mrl_log_f32 = mrl_log.astype(jnp.float32)
+ aux_loss = jnp.sum(
+ optax.softmax_cross_entropy_with_integer_labels(mrl_log_f32, tgt_out) * mask
+ ) / jnp.maximum(jnp.sum(mask), 1.0)
+ total_loss = total_loss + aux_loss
+ num_scales = num_scales + 1
+ total_loss = total_loss / num_scales
+
z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)
div_loss = 1e-4 * slot_div
- return ce_loss + z_loss + div_loss
+ return total_loss + z_loss + div_loss
loss, grads = jax.value_and_grad(loss_fn)(state.params)
grads = jax.lax.pmean(grads, axis_name="batch")
@@ -340,6 +355,50 @@ def _make_val_loss_fn(apply_fn):
return val_loss_batch
+def _make_mrl_val_loss_fn(apply_fn, mrl_dim):
+ """Val loss using truncated decoder hidden states and embedding at dimension mrl_dim."""
+ @jax.jit
+ def val_loss_batch(params, src, tgt_in, tgt_out, causal_mask):
+ pad_id = 0
+ src_mask = make_padding_mask(src, pad_id)
+ tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
+ logits, _, mrl_logits = apply_fn(
+ {"params": params}, src, tgt_in,
+ src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=src_mask,
+ mrl_dims=(mrl_dim,),
+ method="forward_with_aux",
+ )
+ trunc_logits = mrl_logits[0].astype(jnp.float32)
+ loss = optax.softmax_cross_entropy_with_integer_labels(trunc_logits, tgt_out)
+ mask = (tgt_out != pad_id).astype(jnp.float32)
+ return jnp.sum(loss * mask), jnp.sum(mask)
+ return val_loss_batch
+
+
+def _estimate_mrl_params(config, d_prime):
+ """Estimate parameter count if model were deployed at dimension d_prime."""
+ d = d_prime
+ v = config.vocab_size
+ n_enc = config.num_encoder_layers
+ n_dec = config.num_decoder_layers
+ n_h = config.num_heads
+ n_kv = config.num_kv_heads
+ d_ff = d * 4
+ m = config.num_memory_slots
+
+ emb = v * d
+ head_dim = d // n_h
+ kv_dim = n_kv * head_dim
+ attn = d * d + d * kv_dim * 2 + d * d
+ ffn = d * d_ff * 3
+ mixer_token = d * d_ff * 2 + d_ff * m
+ mixer_channel = d * d_ff * 3
+ enc_block = attn + ffn + mixer_token + mixer_channel
+ dec_block = attn * 2 + ffn
+ total = emb + n_enc * enc_block + n_dec * dec_block
+ return int(total)
+
+
def shard_batch(batch, num_devices):
"""Reshape a batch array so leading dim is (num_devices, per_device_batch, ...)."""
return batch.reshape(num_devices, -1, *batch.shape[1:])
@@ -358,7 +417,7 @@ def train(args):
print(f" {num_devices} device(s) for data-parallel training")
print(f"\n[2/4] Loading tokenizer...")
- tokenizer = get_tokenizer()
+ tokenizer = get_tokenizer(max_samples=args.max_samples)
print(f"\n[3/4] Loading TinyStories dataset...")
ds = load_tinystories("train", max_samples=args.max_samples)
@@ -371,9 +430,9 @@ def train(args):
val_enc, val_dec_in, val_dec_tgt = prepare_encoder_decoder_pairs(
val_ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
)
- val_enc = jnp.array(val_enc)
- val_dec_in = jnp.array(val_dec_in)
- val_dec_tgt = jnp.array(val_dec_tgt)
+ val_enc = np.array(val_enc)
+ val_dec_in = np.array(val_dec_in)
+ val_dec_tgt = np.array(val_dec_tgt)
print(f" {len(enc_inputs):,} train / {len(val_enc):,} val pairs")
effective_batch_size = args.batch_size * num_devices
@@ -399,8 +458,13 @@ def train(args):
num_memory_slots=getattr(args, "num_memory_slots", 64),
)
- global _GROUP_SIZE
+ global _GROUP_SIZE, _MRL_DIMS
_GROUP_SIZE = getattr(args, "group_size", 32)
+ mrl_dims_raw = getattr(args, "mrl_dims", None)
+ if mrl_dims_raw:
+ _MRL_DIMS = tuple(d for d in mrl_dims_raw if d < config.d_model)
+ else:
+ _MRL_DIMS = ()
p_train_step = _make_p_train_step()
np.random.seed(args.seed)
@@ -457,9 +521,9 @@ def train(args):
muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps)
tokens_per_batch = effective_batch_size * (args.max_enc_len + args.max_dec_len)
- enc_inputs = jnp.array(enc_inputs)
- dec_inputs = jnp.array(dec_inputs)
- dec_targets = jnp.array(dec_targets)
+ enc_inputs = np.array(enc_inputs)
+ dec_inputs = np.array(dec_inputs)
+ dec_targets = np.array(dec_targets)
last_val_ppl = None
sparsity_ratio = getattr(args, "sparsity_ratio", 0.0)
@@ -555,13 +619,14 @@ def train(args):
dt = time.perf_counter() - t0
eval_every = getattr(args, "eval_every", 100)
if global_step % eval_every == 0 or global_step == total_steps:
- eval_params = jax_utils.unreplicate(ema_params)
+ _eval_params = jax_utils.unreplicate(ema_params)
val_causal = make_causal_mask(args.max_dec_len)
total_loss, total_toks = 0.0, 0.0
for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False):
- vl, vt = val_loss_fn(eval_params, vb[0], vb[1], vb[2], val_causal)
+ vl, vt = val_loss_fn(_eval_params, vb[0], vb[1], vb[2], val_causal)
total_loss += float(vl)
total_toks += float(vt)
+ del _eval_params
last_val_ppl = float(math.exp(total_loss / max(total_toks, 1)))
pbar.set_postfix(loss=f"{loss_val:.4f}", val_ppl=f"{last_val_ppl:.2f}" if last_val_ppl is not None else "?")
@@ -598,24 +663,46 @@ def train(args):
q_total_loss += float(vl)
q_total_toks += float(vt)
quant_val_ppl = float(math.exp(q_total_loss / max(q_total_toks, 1)))
-
- epoch_params = jax.tree.map(np.array, eval_params)
- total_params = sum(x.size for x in jax.tree.leaves(epoch_params))
- near_zero = sum(int(np.sum(np.abs(x) < 1e-6)) for x in jax.tree.leaves(epoch_params))
+ del q_params # free device memory
+
+ # MRL per-dimension evaluation (reuse val_loss_fn's apply_fn)
+ mrl_results = {}
+ if _MRL_DIMS:
+ apply_fn = jax_utils.unreplicate(state).apply_fn
+ for d_prime in _MRL_DIMS:
+ mrl_vl_fn = _make_mrl_val_loss_fn(apply_fn, d_prime)
+ mrl_total_loss, mrl_total_toks = 0.0, 0.0
+ for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False):
+ vl, vt = mrl_vl_fn(eval_params, vb[0], vb[1], vb[2], val_causal)
+ mrl_total_loss += float(vl)
+ mrl_total_toks += float(vt)
+ avg_loss = mrl_total_loss / max(mrl_total_toks, 1)
+ mrl_ppl = float(math.exp(min(avg_loss, 20)))
+ mrl_params = _estimate_mrl_params(config, d_prime)
+ mrl_results[d_prime] = (mrl_ppl, mrl_params)
+ del apply_fn
+
+ # Sparsity + checkpoint — move to CPU, reuse single copy
+ params_np = jax.tree.map(np.array, eval_params)
+ del eval_params # free device memory
+ total_params = sum(x.size for x in jax.tree.leaves(params_np))
+ near_zero = sum(int(np.sum(np.abs(x) < 1e-6)) for x in jax.tree.leaves(params_np))
sparsity = near_zero / total_params * 100
ckpt_name = f"needle_{args.num_layers}_{args.d_model}_{global_step}.pkl"
ckpt_path = os.path.join(args.checkpoint_dir, ckpt_name)
- params_np = jax.tree.map(np.array, jax_utils.unreplicate(ema_params))
with open(ckpt_path, "wb") as f:
pickle.dump({"params": params_np, "config": config.__dict__}, f)
from .test import measure_throughput, benchmark_generation_quality
+ eval_params_jnp = jax.tree.map(jnp.array, params_np)
+ del params_np # free CPU memory
model = EncoderDecoderTransformer(config)
- tp = measure_throughput(model, eval_params, tokenizer, num_runs=5)
+ tp = measure_throughput(model, eval_params_jnp, tokenizer, num_runs=5)
prompts = ["Once upon a time", "The little dog", "She was very happy because"]
- quality = benchmark_generation_quality(model, eval_params, tokenizer, prompts, max_gen_len=64, temperature=0.8)
+ quality = benchmark_generation_quality(model, eval_params_jnp, tokenizer, prompts, max_gen_len=64, temperature=0.8)
+ del eval_params_jnp # free device memory
print(f"\n ─────────────────────────────────────")
print(f" Epoch {epoch + 1}/{args.epochs}")
@@ -626,6 +713,14 @@ def train(args):
print(f" Val ppl {last_val_ppl:>12.2f}")
print(f" Quant val ppl {quant_val_ppl:>12.2f} (INT4 g{_GROUP_SIZE})")
print(f" Sparsity {sparsity:>11.2f}% ({near_zero:,}/{total_params:,})")
+ if mrl_results:
+ print(f" ─────────────────────────────────────")
+ print(f" MRL dimension pruning:")
+ print(f" {'dim':>6} {'val ppl':>10} {'est. params':>12}")
+ print(f" {config.d_model:>6} {last_val_ppl:>10.2f} {total_params:>12,} (full)")
+ for d_prime in sorted(mrl_results.keys(), reverse=True):
+ mrl_ppl, mrl_params = mrl_results[d_prime]
+ print(f" {d_prime:>6} {mrl_ppl:>10.2f} {mrl_params:>12,}")
print(f" ─────────────────────────────────────")
print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")
print(f" Latency {tp['avg_latency_s']:>11.3f}s")
@@ -640,14 +735,18 @@ def train(args):
print(f" ─────────────────────────────────────\n")
if use_wandb:
- wandb.log({
+ log_dict = {
"epoch/avg_loss": epoch_avg_loss,
"epoch/final_loss": final_loss,
"epoch/val_ppl": last_val_ppl,
"epoch/quant_val_ppl": quant_val_ppl,
"epoch/weight_sparsity": sparsity,
"epoch": epoch + 1,
- })
+ }
+ for d_prime, (mrl_ppl, mrl_params) in mrl_results.items():
+ log_dict[f"epoch/mrl_ppl_d{d_prime}"] = mrl_ppl
+ log_dict[f"epoch/mrl_params_d{d_prime}"] = mrl_params
+ wandb.log(log_dict)
if use_wandb:
wandb.finish()