Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Upload wireframe

HenryNdubuakue085770da293
diff --git a/.gitignore b/.gitignore
index b7faf40..2eba63f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -205,3 +205,6 @@ cython_debug/
marimo/_static/
marimo/_lsp/
__marimo__/
+
+checkpoints/
+wandb/
\ No newline at end of file
diff --git a/README.md b/README.md
index 2800bca..78308cc 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,113 @@
-# model
\ No newline at end of file
+```
+ ┌────────────────────────────────────────────────────────┐
+ │ │
+ │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │
+ │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │
+ │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │
+ │ ...the tiny model to rule them all... │
+ └────────────────────────────────────────────────────────┘
+
+ ┌─────────────┐
+ │ Softmax │
+ └──────┬──────┘
+ ┌──────┴──────┐
+ │ Linear (T) │ ← tied weights
+ └──────┬──────┘
+ ┌──────┴──────┐
+ │ LayerNorm │
+ └──────┬──────┘
+ ┌─────────┴─────────┐
+ │ Decoder x 2 │
+ │ ┌───────────────┐ │
+ │ │ Masked Self │ │
+ ┌──────────────┐ │ │ Attn + RoPE │ │
+ │ │ │ ├───────────────┤ │
+ │ Encoder x 2 │ │ │ Cross │ │
+ │ ┌──────────┐ │────────▶ Attention │ │
+ │ │ Self │ │ │ ├───────────────┤ │
+ │ │Attn+RoPE │ │ │ │ Feed-Forward │ │
+ │ ├──────────┤ │ │ └───────────────┘ │
+ │ │ Feed- │ │ └─────────┬─────────┘
+ │ │ Forward │ │ ┌──────┴──────┐
+ │ └──────────┘ │ │ Embedding │ ← shared
+ └──────┬───────┘ └──────┬──────┘
+ ┌──────┴───────┐ │
+ │ Embedding │ ← shared │
+ └──────┬───────┘ │
+ │ │
+ ┌────┴────┐ ┌─────┴─────┐
+ │ Encoder │ │ Decoder │
+ │ Input │ │ Input │
+ └─────────┘ └───────────┘
+
+ ~7.5M params · d=128 · 4 heads · 2+2 layers · RoPE
+```
+
+## Usage
+
+```
+git clone https://github.com/cactus-compute/model.git
+
+source ./setup
+
+needle [command]
+
+ ┌───────────────────────────────────────────────────────────────────┐
+ │ │
+ │ train │
+ │ --epochs INT Training epochs (default: 1) │
+ │ --batch-size INT Batch size (default: 32) │
+ │ --lr FLOAT AdamW learning rate (default: 3e-4) │
+ │ --muon-lr FLOAT Muon learning rate (default: 0.02) │
+ │ --d-model INT Model dimension (default: 128) │
+ │ --num-heads INT Attention heads (default: 4) │
+ │ --num-layers INT Encoder/decoder layers (default: 2) │
+ │ --dropout FLOAT Dropout rate (default: 0.1) │
+ │ --max-enc-len INT Max encoder seq length (default: 128) │
+ │ --max-dec-len INT Max decoder seq length (default: 128) │
+ │ --max-samples INT Training samples (default: 20000) │
+ │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │
+ │ --wandb Enable W&B logging │
+ │ --checkpoint-dir DIR Checkpoint directory │
+ │ --seed INT Random seed (default: 42) │
+ │ │
+ │ sweep │
+ │ --sweep-config PATH Sweep YAML config │
+ │ --project STR W&B project name (default: needle-v1) │
+ │ --count INT Number of trials (default: 20) │
+ │ │
+ │ run │
+ │ --checkpoint PATH Path to model checkpoint (required) │
+ │ --prompts STR [...] One or more prompts to continue │
+ │ --max-len INT Max tokens to generate (default: 128) │
+ │ --temperature FLOAT Sampling temperature (default: 0.8) │
+ │ --seed INT Random seed (default: 0) │
+ │ │
+ │ test │
+ │ --checkpoint PATH Path to model checkpoint (required) │
+ │ --batch-size INT Batch size (default: 32) │
+ │ --max-eval-samples INT Evaluation samples (default: 1000) │
+ │ --max-gen-len INT Max generation length (default: 128) │
+ │ --temperature FLOAT Sampling temperature (default: 0.8) │
+ │ --throughput-runs INT Throughput runs (default: 10) │
+ │ │
+ │ evaluate │
+ │ --checkpoint PATH Path to model checkpoint (required) │
+ │ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │
+ │ --max-samples INT Samples per benchmark (default: 500) │
+ │ │
+ └───────────────────────────────────────────────────────────────────┘
+```
+
+## Project Structure
+
+```
+ src/
+ ├── model.py ······ Transformer architecture
+ ├── data.py ······· TinyStories loading & preprocessing
+ ├── train.py ······ Training loop
+ ├── run.py ········ Story generation from prompts
+ ├── test.py ······· Throughput & quality benchmarks
+ ├── evaluate.py ··· NLP benchmark evaluation
+ └── cli.py ········ CLI entry point
+```
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..6a7c563
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,23 @@
+[project]
+name = "needle"
+version = "0.1.0"
+requires-python = ">=3.10"
+dependencies = [
+ "jax==0.9.0.1",
+ "jaxlib==0.9.0.1",
+ "flax==0.12.4",
+ "optax==0.2.6",
+ "datasets==4.5.0",
+ "transformers==5.2.0",
+ "wandb",
+]
+
+[project.scripts]
+needle = "src.cli:main"
+
+[tool.setuptools.packages.find]
+include = ["src*"]
+
+[build-system]
+requires = ["setuptools>=68.0"]
+build-backend = "setuptools.build_meta"
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..16c0614
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,8 @@
+jax==0.9.0.1
+jaxlib==0.9.0.1
+flax==0.12.4
+optax==0.2.6
+datasets==4.5.0
+transformers==5.2.0
+wandb>=0.19.0
+pyyaml>=6.0
diff --git a/setup b/setup
new file mode 100644
index 0000000..41c7797
--- /dev/null
+++ b/setup
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+VENV_DIR=".venv"
+
+if [ ! -d "$VENV_DIR" ]; then
+ echo "Creating virtual environment..."
+ python3 -m venv "$VENV_DIR"
+else
+ echo "Virtual environment already exists."
+fi
+
+source "$VENV_DIR/bin/activate"
+
+echo "Installing dependencies..."
+pip install --upgrade pip -q
+pip install -e . -q
+
+if [ -z "$WANDB_API_KEY" ]; then
+ echo ""
+ echo " Enter your W&B API key (https://wandb.ai/authorize)"
+ echo " Press Enter to skip."
+ printf " > "
+ read WANDB_API_KEY || true
+ if [ -n "$WANDB_API_KEY" ]; then
+ export WANDB_API_KEY
+ fi
+fi
+
+needle --help
diff --git a/src/__init__.py b/src/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/cli.py b/src/cli.py
new file mode 100644
index 0000000..47e61ac
--- /dev/null
+++ b/src/cli.py
@@ -0,0 +1,146 @@
+import argparse
+import sys
+
+HELP = """
+ ┌───────────────────────────────────────────────────────────────────┐
+ │ │
+ │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │
+ │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │
+ │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │
+ │ ...the tiny model to rule them all... │
+ │ │
+ │ train │
+ │ --epochs INT Training epochs (default: 1) │
+ │ --batch-size INT Batch size (default: 32) │
+ │ --lr FLOAT AdamW learning rate (default: 3e-4) │
+ │ --muon-lr FLOAT Muon learning rate (default: 0.02) │
+ │ --d-model INT Model dimension (default: 128) │
+ │ --num-heads INT Attention heads (default: 4) │
+ │ --num-layers INT Encoder/decoder layers (default: 2) │
+ │ --dropout FLOAT Dropout rate (default: 0.1) │
+ │ --max-enc-len INT Max encoder seq length (default: 128) │
+ │ --max-dec-len INT Max decoder seq length (default: 128) │
+ │ --max-samples INT Training samples (default: 20000) │
+ │ --checkpoint-dir DIR Checkpoint directory │
+ │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │
+ │ --seed INT Random seed (default: 42) │
+ │ │
+ │ │
+ │ sweep │
+ │ --sweep-config PATH Sweep YAML config (default: sweep.yaml)│
+ │ --project STR W&B project name (default: needle-v1)│
+ │ --count INT Number of trials (default: 20) │
+ │ (also accepts all train flags as defaults for non-swept params)│
+ │ │
+ │ run │
+ │ --checkpoint PATH Path to model checkpoint (required) │
+ │ --prompts STR [...] One or more prompts to continue │
+ │ --max-len INT Max tokens to generate (default: 128) │
+ │ --temperature FLOAT Sampling temperature (default: 0.8) │
+ │ --seed INT Random seed (default: 0) │
+ │ │
+ │ test │
+ │ --checkpoint PATH Path to model checkpoint (required) │
+ │ --batch-size INT Batch size (default: 32) │
+ │ --max-eval-samples INT Evaluation samples (default: 1000) │
+ │ --max-gen-len INT Max generation length (default: 128) │
+ │ --temperature FLOAT Sampling temperature (default: 0.8) │
+ │ --throughput-runs INT Throughput runs (default: 10) │
+ │ │
+ │ evaluate │
+ │ --checkpoint PATH Path to model checkpoint (required) │
+ │ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │
+ │ --max-samples INT Samples per benchmark (default: 500) │
+ │ │
+ └───────────────────────────────────────────────────────────────────┘
+"""
+
+
+def main():
+ if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help", "help"):
+ print(HELP)
+ sys.exit(0)
+
+ parser = argparse.ArgumentParser(prog="needle", add_help=False)
+ sub = parser.add_subparsers(dest="command")
+
+ p = sub.add_parser("train", add_help=False)
+ p.add_argument("--epochs", type=int, default=1)
+ p.add_argument("--batch-size", type=int, default=32)
+ p.add_argument("--lr", type=float, default=3e-4)
+ p.add_argument("--muon-lr", type=float, default=0.02)
+ p.add_argument("--d-model", type=int, default=128)
+ p.add_argument("--num-heads", type=int, default=4)
+ p.add_argument("--num-layers", type=int, default=2)
+ p.add_argument("--dropout", type=float, default=0.1)
+ p.add_argument("--max-enc-len", type=int, default=128)
+ p.add_argument("--max-dec-len", type=int, default=128)
+ p.add_argument("--max-samples", type=int, default=20000)
+ p.add_argument("--warmup-ratio", type=float, default=0.05)
+ p.add_argument("--wandb", action="store_true")
+ p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
+ p.add_argument("--seed", type=int, default=42)
+
+ p = sub.add_parser("run", add_help=False)
+ p.add_argument("--checkpoint", type=str, required=True)
+ p.add_argument("--prompts", type=str, nargs="*")
+ p.add_argument("--max-len", type=int, default=128)
+ p.add_argument("--temperature", type=float, default=0.8)
+ p.add_argument("--seed", type=int, default=0)
+
+ p = sub.add_parser("test", add_help=False)
+ p.add_argument("--checkpoint", type=str, required=True)
+ p.add_argument("--batch-size", type=int, default=32)
+ p.add_argument("--max-eval-samples", type=int, default=1000)
+ p.add_argument("--max-enc-len", type=int, default=128)
+ p.add_argument("--max-dec-len", type=int, default=128)
+ p.add_argument("--max-gen-len", type=int, default=128)
+ p.add_argument("--temperature", type=float, default=0.8)
+ p.add_argument("--throughput-runs", type=int, default=10)
+
+ p = sub.add_parser("sweep", add_help=False)
+ p.add_argument("--sweep-config", type=str, default="sweep.yaml")
+ p.add_argument("--project", type=str, default="needle-v1")
+ p.add_argument("--count", type=int, default=20)
+ p.add_argument("--epochs", type=int, default=1)
+ p.add_argument("--batch-size", type=int, default=32)
+ p.add_argument("--lr", type=float, default=3e-4)
+ p.add_argument("--muon-lr", type=float, default=0.02)
+ p.add_argument("--d-model", type=int, default=128)
+ p.add_argument("--num-heads", type=int, default=4)
+ p.add_argument("--num-layers", type=int, default=2)
+ p.add_argument("--dropout", type=float, default=0.1)
+ p.add_argument("--max-enc-len", type=int, default=128)
+ p.add_argument("--max-dec-len", type=int, default=128)
+ p.add_argument("--max-samples", type=int, default=20000)
+ p.add_argument("--warmup-ratio", type=float, default=0.05)
+ p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
+ p.add_argument("--seed", type=int, default=42)
+
+ p = sub.add_parser("evaluate", add_help=False)
+ p.add_argument("--checkpoint", type=str, required=True)
+ p.add_argument("--benchmarks", type=str, nargs="*",
+ choices=["wikitext2", "lambada", "hellaswag", "arc_easy"])
+ p.add_argument("--max-samples", type=int, default=500)
+
+ args = parser.parse_args()
+
+ if not args.command:
+ print(HELP)
+ sys.exit(0)
+
+ if args.command == "train":
+ from .train import train
+ train(args)
+ elif args.command == "sweep":
+ from .train import sweep
+ sweep(args)
+ elif args.command == "run":
+ from .run import main as run_main
+ run_main(args)
+ elif args.command == "test":
+ from .test import main as test_main
+ test_main(args)
+ elif args.command == "evaluate":
+ from .evaluate import main as eval_main
+ eval_main(args)
diff --git a/src/data.py b/src/data.py
new file mode 100644
index 0000000..0adbda5
--- /dev/null
+++ b/src/data.py
@@ -0,0 +1,68 @@
+import os
+os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"
+
+import logging
+logging.getLogger("transformers").setLevel(logging.ERROR)
+
+import numpy as np
+from datasets import load_dataset
+from transformers import AutoTokenizer
+
+
+def get_tokenizer():
+ tokenizer = AutoTokenizer.from_pretrained("gpt2")
+ tokenizer.pad_token = tokenizer.eos_token
+ return tokenizer
+
+
+def load_tinystories(split="train", max_samples=None):
+ ds = load_dataset("roneneldan/TinyStories", split=split)
+ if max_samples:
+ ds = ds.select(range(min(max_samples, len(ds))))
+ return ds
+
+
+def prepare_encoder_decoder_pairs(texts, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):
+ enc_inputs = []
+ dec_inputs = []
+ dec_targets = []
+
+ pad_id = tokenizer.pad_token_id
+ bos_id = tokenizer.eos_token_id
+
+ for text in texts:
+ tokens = tokenizer.encode(text, truncation=True, max_length=max_enc_len + max_dec_len)
+ if len(tokens) < 6:
+ continue
+
+ split_point = max(2, int(len(tokens) * split_ratio))
+
+ enc_tokens = tokens[:split_point][:max_enc_len]
+ dec_tokens = tokens[split_point:][:max_dec_len]
+
+ if len(dec_tokens) < 2:
+ continue
+
+ dec_in = [bos_id] + dec_tokens[:-1]
+ dec_tgt = dec_tokens
+
+ enc_padded = enc_tokens + [pad_id] * (max_enc_len - len(enc_tokens))
+ dec_in_padded = dec_in + [pad_id] * (max_dec_len - len(dec_in))
+ dec_tgt_padded = dec_tgt + [pad_id] * (max_dec_len - len(dec_tgt))
+
+ enc_inputs.append(enc_padded)
+ dec_inputs.append(dec_in_padded)
+ dec_targets.append(dec_tgt_padded)
+
+ return np.array(enc_inputs), np.array(dec_inputs), np.array(dec_targets)
+
+
+def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True):
+ n = len(enc_inputs)
+ indices = np.arange(n)
+ if shuffle:
+ np.random.shuffle(indices)
+
+ for i in range(0, n - batch_size + 1, batch_size):
+ batch_idx = indices[i : i + batch_size]
+ yield enc_inputs[batch_idx], dec_inputs[batch_idx], dec_targets[batch_idx]
diff --git a/src/evaluate.py b/src/evaluate.py
new file mode 100644
index 0000000..816a25a
--- /dev/null
+++ b/src/evaluate.py
@@ -0,0 +1,293 @@
+import argparse
+import math
+import pickle
+
+import jax
+import jax.numpy as jnp
+import numpy as np
+from datasets import load_dataset
+
+from .data import get_tokenizer
+from .model import (
+ EncoderDecoderTransformer,
+ TransformerConfig,
+ make_causal_mask,
+ make_padding_mask,
+)
+
+
+def load_checkpoint(path):
+ with open(path, "rb") as f:
+ data = pickle.load(f)
+ params = jax.tree.map(jnp.array, data["params"])
+ config = TransformerConfig(**data["config"])
+ return params, config
+
+
+def score_sequence(model, params, enc_tokens, dec_tokens, pad_id):
+ """Compute average negative log-likelihood of dec_tokens given enc_tokens."""
+ enc_input = jnp.array([enc_tokens])
+ src_mask = make_padding_mask(enc_input, pad_id)
+
+ encoder_out = model.apply(
+ {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",
+ )
+
+ dec_in = [pad_id] + list(dec_tokens[:-1])
+ dec_input = jnp.array([dec_in])
+ tgt_mask = make_causal_mask(len(dec_in))
+ cross_mask = src_mask
+
+ logits = model.apply(
+ {"params": params}, dec_input, encoder_out,
+ self_mask=tgt_mask, cross_mask=cross_mask, deterministic=True, method="decode",
+ )
+
+ log_probs = jax.nn.log_softmax(logits[0])
+ target_ids = jnp.array(dec_tokens)
+ token_lls = log_probs[jnp.arange(len(dec_tokens)), target_ids]
+ return float(jnp.mean(token_lls))
+
+
+def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256):
+ """Perplexity on WikiText-2 test split."""
+ ds = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="test")
+
+ pad_id = tokenizer.pad_token_id
+ total_nll = 0.0
+ total_tokens = 0
+ evaluated = 0
+
+ for example in ds:
+ text = example["text"].strip()
+ if len(text) < 20:
+ continue
+
+ tokens = tokenizer.encode(text)
+ if len(tokens) < 6:
+ continue
+
+ tokens = tokens[:max_len]
+ split = max(2, len(tokens) // 3)
+ enc_tokens = tokens[:split]
+ dec_tokens = tokens[split:]
+
+ enc_input = jnp.array([enc_tokens])
+ src_mask = make_padding_mask(enc_input, pad_id)
+ encoder_out = model.apply(
+ {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",
+ )
+
+ dec_in = [pad_id] + list(dec_tokens[:-1])
+ dec_input = jnp.array([dec_in])
+ tgt_mask = make_causal_mask(len(dec_in))
+
+ logits = model.apply(
+ {"params": params}, dec_input, encoder_out,
+ self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",
+ )
+
+ log_probs = jax.nn.log_softmax(logits[0])
+ target_ids = jnp.array(dec_tokens)
+ token_lls = log_probs[jnp.arange(len(dec_tokens)), target_ids]
+
+ total_nll += -float(jnp.sum(token_lls))
+ total_tokens += len(dec_tokens)
+ evaluated += 1
+
+ if evaluated >= max_samples:
+ break
+
+ ppl = math.exp(total_nll / max(total_tokens, 1))
+ return {"perplexity": ppl, "samples": evaluated, "tokens": total_tokens}
+
+
+def eval_lambada(model, params, tokenizer, max_samples=500):
+ """Accuracy of predicting the final word on LAMBADA."""
+ ds = load_dataset("EleutherAI/lambada_openai", "default", split="test")
+
+ pad_id = tokenizer.pad_token_id
+ correct = 0
+ total = 0
+
+ for example in ds:
+ text = example["text"].strip()
+ words = text.rsplit(" ", 1)
+ if len(words) < 2:
+ continue
+
+ context, last_word = words
+ context_tokens = tokenizer.encode(context)
+ target_tokens = tokenizer.encode(" " + last_word)
+
+ if not context_tokens or not target_tokens:
+ continue
+
+ enc_input = jnp.array([context_tokens])
+ src_mask = make_padding_mask(enc_input, pad_id)
+ encoder_out = model.apply(
+ {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",
+ )
+
+ dec_in = jnp.array([[pad_id]])
+ tgt_mask = make_causal_mask(1)
+
+ logits = model.apply(
+ {"params": params}, dec_in, encoder_out,
+ self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",
+ )
+
+ predicted = int(jnp.argmax(logits[0, 0]))
+ if predicted == target_tokens[0]:
+ correct += 1
+ total += 1
+
+ if total >= max_samples:
+ break
+
+ acc = correct / max(total, 1)
+ return {"accuracy": acc, "correct": correct, "total": total}
+
+
+def eval_hellaswag(model, params, tokenizer, max_samples=500):
+ """Accuracy on HellaSwag by scoring each candidate ending."""
+ ds = load_dataset("Rowan/hellaswag", split="validation")
+
+ pad_id = tokenizer.pad_token_id
+ correct = 0
+ total = 0
+
+ for example in ds:
+ ctx = example["ctx"].strip()
+ endings = example["endings"]
+ label = int(example["label"])
+
+ enc_tokens = tokenizer.encode(ctx)
+ if not enc_tokens:
+ continue
+
+ # Truncate context to keep it manageable
+ enc_tokens = enc_tokens[:192]
+
+ scores = []
+ for ending in endings:
+ dec_tokens = tokenizer.encode(" " + ending.strip())
+ if not dec_tokens:
+ scores.append(float("-inf"))
+ continue
+ dec_tokens = dec_tokens[:64]
+ score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id)
+ scores.append(score)
+
+ predicted = int(np.argmax(scores))
+ if predicted == label:
+ correct += 1
+ total += 1
+
+ if total >= max_samples:
+ break
+
+ acc = correct / max(total, 1)
+ return {"accuracy": acc, "correct": correct, "total": total}
+
+
+def eval_arc_easy(model, params, tokenizer, max_samples=500):
+ """Accuracy on ARC-Easy by scoring each answer choice."""
+ ds = load_dataset("allenai/ai2_arc", "ARC-Easy", split="test")
+
+ pad_id = tokenizer.pad_token_id
+ correct = 0
+ total = 0
+
+ for example in ds:
+ question = example["question"].strip()
+ choices = example["choices"]
+ answer_key = example["answerKey"]
+
+ labels = choices["label"]
+ texts = choices["text"]
+
+ enc_tokens = tokenizer.encode(question)[:128]
+ if not enc_tokens:
+ continue
+
+ scores = []
+ for choice_text in texts:
+ dec_tokens = tokenizer.encode(" " + choice_text.strip())
+ if not dec_tokens:
+ scores.append(float("-inf"))
+ continue
+ dec_tokens = dec_tokens[:64]
+ score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id)
+ scores.append(score)
+
+ predicted_idx = int(np.argmax(scores))
+ predicted_label = labels[predicted_idx]
+ if predicted_label == answer_key:
+ correct += 1
+ total += 1
+
+ if total >= max_samples:
+ break
+
+ acc = correct / max(total, 1)
+ return {"accuracy": acc, "correct": correct, "total": total}
+
+
+BENCHMARKS = {
+ "wikitext2": eval_wikitext2,
+ "lambada": eval_lambada,
+ "hellaswag": eval_hellaswag,
+ "arc_easy": eval_arc_easy,
+}
+
+
+def main(args):
+ print(f"Loading checkpoint: {args.checkpoint}")
+ params, config = load_checkpoint(args.checkpoint)
+ model = EncoderDecoderTransformer(config)
+ tokenizer = get_tokenizer()
+
+ param_count = sum(x.size for x in jax.tree.leaves(params))
+ print(f"Model parameters: {param_count:,}")
+
+ benchmarks = args.benchmarks or list(BENCHMARKS.keys())
+
+ results = {}
+ for name in benchmarks:
+ if name not in BENCHMARKS:
+ print(f"Unknown benchmark: {name}, skipping")
+ continue
+
+ print(f"\n{'=' * 50}")
+ print(f"Benchmark: {name}")
+ print("=" * 50)
+
+ result = BENCHMARKS[name](model, params, tokenizer, max_samples=args.max_samples)
+ results[name] = result
+
+ for k, v in result.items():
+ if isinstance(v, float):
+ print(f" {k}: {v:.4f}")
+ else:
+ print(f" {k}: {v}")
+
+ print(f"\n{'=' * 50}")
+ print("Summary")
+ print("=" * 50)
+ for name, result in results.items():
+ metric = "perplexity" if "perplexity" in result else "accuracy"
+ print(f" {name:>12s}: {result[metric]:.4f} ({metric})")
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Evaluate on established NLP benchmarks")
+ parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")
+ parser.add_argument("--benchmarks", type=str, nargs="*", choices=list(BENCHMARKS.keys()),
+ help="Benchmarks to run (default: all)")
+ parser.add_argument("--max-samples", type=int, default=500)
+ return parser.parse_args()
+
+
+if __name__ == "__main__":
+ main(parse_args())
diff --git a/src/model.py b/src/model.py
new file mode 100644
index 0000000..e7b0d45
--- /dev/null
+++ b/src/model.py
@@ -0,0 +1,258 @@
+import math
+from dataclasses import dataclass
+
+import jax.numpy as jnp
+import jax.nn.initializers as jinit
+import flax.linen as nn
+
+
+def default_init():
+ return jinit.normal(stddev=0.02)
+
+
+def residual_init(num_layers):
+ return jinit.normal(stddev=0.02 / math.sqrt(2 * num_layers))
+
+
+DTYPE_MAP = {"float32": jnp.float32, "bfloat16": jnp.bfloat16, "float16": jnp.float16}
+
+
+@dataclass
+class TransformerConfig:
+ vocab_size: int = 50257
+ d_model: int = 128
+ num_heads: int = 4
+ num_encoder_layers: int = 2
+ num_decoder_layers: int = 2
+ d_ff: int = 512
+ max_seq_len: int = 128
+ dropout_rate: float = 0.1
+ pad_token_id: int = 50256
+ rope_theta: float = 10000.0
+ dtype: str = "bfloat16"
+
+ @property
+ def jax_dtype(self):
+ return DTYPE_MAP[self.dtype]
+
+ @property
+ def total_layers(self):
+ return self.num_encoder_layers + self.num_decoder_layers
+
+
+def precompute_rope_freqs(head_dim, seq_len, theta=10000.0):
+ freqs = 1.0 / (theta ** (jnp.arange(0, head_dim, 2).astype(jnp.float32) / head_dim))
+ t = jnp.arange(seq_len).astype(jnp.float32)
+ angles = jnp.outer(t, freqs)
+ return jnp.cos(angles), jnp.sin(angles)
+
+
+def apply_rope(x, cos, sin):
+ T = x.shape[2]
+ half = x.shape[-1] // 2
+ cos = cos[:T][None, None, :, :]
+ sin = sin[:T][None, None, :, :]
+ x1 = x[..., :half]
+ x2 = x[..., half:]
+ return jnp.concatenate([x1 * cos - x2 * sin, x2 * cos + x1 * sin], axis=-1)
+
+
+class MultiHeadAttention(nn.Module):
+ num_heads: int
+ d_model: int
+ num_layers: int
+ dropout_rate: float = 0.1
+ dtype: jnp.dtype = jnp.bfloat16
+
+ @nn.compact
+ def __call__(self, q_input, kv_input, mask=None, rope=None, deterministic=True):
+ head_dim = self.d_model // self.num_heads
+ B = q_input.shape[0]
+
+ q = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="q_proj")(q_input)
+ k = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="k_proj")(kv_input)
+ v = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="v_proj")(kv_input)
+
+ q = q.reshape(B, -1, self.num_heads, head_dim).transpose(0, 2, 1, 3)
+ k = k.reshape(B, -1, self.num_heads, head_dim).transpose(0, 2, 1, 3)
+ v = v.reshape(B, -1, self.num_heads, head_dim).transpose(0, 2, 1, 3)
+
+ if rope is not None:
+ cos, sin = rope
+ q = apply_rope(q, cos, sin)
+ k = apply_rope(k, cos, sin)
+
+ scale = jnp.sqrt(jnp.float32(head_dim))
+ attn_weights = jnp.matmul(q, k.transpose(0, 1, 3, 2)) / scale
+
+ if mask is not None:
+ attn_weights = jnp.where(mask, attn_weights, jnp.finfo(attn_weights.dtype).min)
+
+ attn_weights = nn.softmax(attn_weights, axis=-1)
+ attn_weights = nn.Dropout(rate=self.dropout_rate)(attn_weights, deterministic=deterministic)
+
+ out = jnp.matmul(attn_weights, v)
+ out = out.transpose(0, 2, 1, 3).reshape(B, -1, self.d_model)
+ return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="out_proj")(out)
+
+
+class FeedForward(nn.Module):
+ d_model: int
+ d_ff: int
+ num_layers: int
+ dropout_rate: float = 0.1
+ dtype: jnp.dtype = jnp.bfloat16
+
+ @nn.compact
+ def __call__(self, x, deterministic=True):
+ gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="gate_proj")(x)
+ up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="up_proj")(x)
+ x = nn.silu(gate) * up
+ x = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)
+ x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)
+ return x
+
+
+class EncoderBlock(nn.Module):
+ num_heads: int
+ d_model: int
+ d_ff: int
+ num_layers: int
+ dropout_rate: float = 0.1
+ dtype: jnp.dtype = jnp.bfloat16
+
+ @nn.compact
+ def __call__(self, x, mask=None, rope=None, deterministic=True):
+ residual = x
+ x = nn.RMSNorm(dtype=self.dtype)(x)
+ x = MultiHeadAttention(self.num_heads, self.d_model, self.num_layers, self.dropout_rate, self.dtype)(
+ x, x, mask=mask, rope=rope, deterministic=deterministic
+ )
+ x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)
+ x = x + residual
+
+ residual = x
+ x = nn.RMSNorm(dtype=self.dtype)(x)
+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dropout_rate, self.dtype)(x, deterministic=deterministic)
+ x = x + residual
+
+ return x
+
+
+class DecoderBlock(nn.Module):
+ num_heads: int
+ d_model: int
+ d_ff: int
+ num_layers: int
+ dropout_rate: float = 0.1
+ dtype: jnp.dtype = jnp.bfloat16
+
+ @nn.compact
+ def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None, deterministic=True):
+ # Self-attention with RoPE
+ residual = x
+ x = nn.RMSNorm(dtype=self.dtype)(x)
+ x = MultiHeadAttention(self.num_heads, self.d_model, self.num_layers, self.dropout_rate, self.dtype, name="self_attn")(
+ x, x, mask=self_mask, rope=rope, deterministic=deterministic
+ )
+ x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)
+ x = x + residual
+
+ # Cross-attention (no RoPE)
+ residual = x
+ x = nn.RMSNorm(dtype=self.dtype)(x)
+ x = MultiHeadAttention(self.num_heads, self.d_model, self.num_layers, self.dropout_rate, self.dtype, name="cross_attn")(
+ x, encoder_out, mask=cross_mask, deterministic=deterministic
+ )
+ x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)
+ x = x + residual
+
+ residual = x
+ x = nn.RMSNorm(dtype=self.dtype)(x)
+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dropout_rate, self.dtype)(x, deterministic=deterministic)
+ x = x + residual
+
+ return x
+
+
+class Encoder(nn.Module):
+ config: TransformerConfig
+
+ @nn.compact
+ def __call__(self, x, mask=None, rope=None, deterministic=True):
+ cfg = self.config
+ dt = cfg.jax_dtype
+ x = x.astype(dt)
+ x = nn.Dropout(rate=cfg.dropout_rate)(x, deterministic=deterministic)
+
+ for i in range(cfg.num_encoder_layers):
+ x = EncoderBlock(
+ cfg.num_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, cfg.dropout_rate, dt, name=f"block_{i}"
+ )(x, mask=mask, rope=rope, deterministic=deterministic)
+
+ x = nn.RMSNorm(dtype=dt)(x)
+ return x
+
+
+class Decoder(nn.Module):
+ config: TransformerConfig
+
+ @nn.compact
+ def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None, deterministic=True):
+ cfg = self.config
+ dt = cfg.jax_dtype
+ x = x.astype(dt)
+ x = nn.Dropout(rate=cfg.dropout_rate)(x, deterministic=deterministic)
+
+ for i in range(cfg.num_decoder_layers):
+ x = DecoderBlock(
+ cfg.num_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, cfg.dropout_rate, dt, name=f"block_{i}"
+ )(x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope, deterministic=deterministic)
+
+ x = nn.RMSNorm(dtype=dt)(x)
+ return x
+
+
+class EncoderDecoderTransformer(nn.Module):
+ """Encoder-decoder transformer with shared embeddings, tied output, RoPE, and bfloat16."""
+ config: TransformerConfig
+
+ def setup(self):
+ self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))
+ self.encoder = Encoder(self.config)
+ self.decoder = Decoder(self.config)
+
+ def _rope(self, seq_len):
+ head_dim = self.config.d_model // self.config.num_heads
+ return precompute_rope_freqs(head_dim, seq_len, self.config.rope_theta)
+
+ def encode(self, src, src_mask=None, deterministic=True):
+ x = self.embedding(src)
+ rope = self._rope(src.shape[1])
+ return self.encoder(x, mask=src_mask, rope=rope, deterministic=deterministic)
+
+ def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None, deterministic=True):
+ x = self.embedding(tgt)
+ rope = self._rope(tgt.shape[1])
+ x = self.decoder(
+ x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope, deterministic=deterministic
+ )
+ logits = x.astype(jnp.float32) @ self.embedding.embedding.T
+ return logits
+
+ def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, deterministic=True):
+ encoder_out = self.encode(src, src_mask=src_mask, deterministic=deterministic)
+ logits = self.decode(
+ tgt, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask, deterministic=deterministic
+ )
+ return logits
+
+
+def make_causal_mask(seq_len):
+ mask = jnp.tril(jnp.ones((seq_len, seq_len), dtype=jnp.bool_))
+ return mask[None, None, :, :]
+
+
+def make_padding_mask(tokens, pad_token_id):
+ mask = tokens != pad_token_id
+ return mask[:, None, None, :]
diff --git a/src/run.py b/src/run.py
new file mode 100644
index 0000000..b4a7bf2
--- /dev/null
+++ b/src/run.py
@@ -0,0 +1,124 @@
+import argparse
+import pickle
+import sys
+
+import jax
+import jax.numpy as jnp
+
+from .data import get_tokenizer
+from .model import (
+ EncoderDecoderTransformer,
+ TransformerConfig,
+ make_causal_mask,
+ make_padding_mask,
+)
+
+
+def load_checkpoint(path):
+ with open(path, "rb") as f:
+ data = pickle.load(f)
+ params = jax.tree.map(jnp.array, data["params"])
+ config = TransformerConfig(**data["config"])
+ return params, config
+
+
+def generate(model, params, tokenizer, prompt, max_gen_len=128, temperature=0.8, seed=0, stream=True):
+ enc_tokens = tokenizer.encode(prompt)
+ enc_input = jnp.array([enc_tokens])
+
+ pad_id = tokenizer.pad_token_id
+ eos_id = tokenizer.eos_token_id
+
+ src_mask = make_padding_mask(enc_input, pad_id)
+ encoder_out = model.apply(
+ {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode"
+ )
+
+ # Fixed-size buffer so JAX compiles once
+ tgt_mask = make_causal_mask(max_gen_len)
+ dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
+ dec_buffer = dec_buffer.at[0, 0].set(eos_id)
+
+ @jax.jit
+ def decode_step(dec_buffer, encoder_out, src_mask):
+ logits = model.apply(
+ {"params": params},
+ dec_buffer,
+ encoder_out,
+ self_mask=tgt_mask,
+ cross_mask=src_mask,
+ deterministic=True,
+ method="decode",
+ )
+ return logits
+
+ rng = jax.random.PRNGKey(seed)
+ generated_tokens = []
+
+ if stream:
+ sys.stdout.write(f"\n{prompt}")
+ sys.stdout.flush()
+
+ # Compile once on first call
+ logits = decode_step(dec_buffer, encoder_out, src_mask)
+
+ for i in range(max_gen_len - 1):
+ next_logits = logits[0, i] / temperature
+ rng, sample_rng = jax.random.split(rng)
+ next_token = jax.random.categorical(sample_rng, next_logits).item()
+
+ if next_token == eos_id:
+ break
+
+ generated_tokens.append(next_token)
+ dec_buffer = dec_buffer.at[0, i + 1].set(next_token)
+
+ if stream:
+ sys.stdout.write(tokenizer.decode([next_token]))
+ sys.stdout.flush()
+
+ logits = decode_step(dec_buffer, encoder_out, src_mask)
+
+ if stream:
+ sys.stdout.write("\n")
+
+ return tokenizer.decode(generated_tokens)
+
+
+def main(args):
+ print(f"Loading checkpoint: {args.checkpoint}")
+ params, config = load_checkpoint(args.checkpoint)
+
+ model = EncoderDecoderTransformer(config)
+ tokenizer = get_tokenizer()
+
+ param_count = sum(x.size for x in jax.tree.leaves(params))
+ print(f"Model parameters: {param_count:,}")
+
+ prompts = args.prompts or ["Once upon a time", "The little dog", "She was very happy because"]
+
+ for i, prompt in enumerate(prompts):
+ generate(
+ model,
+ params,
+ tokenizer,
+ prompt,
+ max_gen_len=args.max_len,
+ temperature=args.temperature,
+ seed=args.seed + i,
+ stream=True,
+ )
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Generate stories with trained transformer")
+ parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")
+ parser.add_argument("--prompts", type=str, nargs="*", help="Prompts to continue")
+ parser.add_argument("--max-len", type=int, default=128)
+ parser.add_argument("--temperature", type=float, default=0.8)
+ parser.add_argument("--seed", type=int, default=0)
+ return parser.parse_args()
+
+
+if __name__ == "__main__":
+ main(parse_args())
diff --git a/src/test.py b/src/test.py
new file mode 100644
index 0000000..f312057
--- /dev/null
+++ b/src/test.py
@@ -0,0 +1,214 @@
+import argparse
+import math
+import pickle
+import time
+
+import jax
+import jax.numpy as jnp
+import numpy as np
+import optax
+
+from .data import get_batches, get_tokenizer, load_tinystories, prepare_encoder_decoder_pairs
+from .model import (
+ EncoderDecoderTransformer,
+ TransformerConfig,
+ make_causal_mask,
+ make_padding_mask,
+)
+
+
+def load_checkpoint(path):
+ with open(path, "rb") as f:
+ data = pickle.load(f)
+ params = jax.tree.map(jnp.array, data["params"])
+ config = TransformerConfig(**data["config"])
+ return params, config
+
+
+def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batch_size, pad_id):
+ total_loss = 0.0
+ total_tokens = 0
+
+ for src, tgt_in, tgt_out in get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=False):
+ src, tgt_in, tgt_out = jnp.array(src), jnp.array(tgt_in), jnp.array(tgt_out)
+
+ src_mask = make_padding_mask(src, pad_id)
+ tgt_mask = make_causal_mask(tgt_in.shape[1]) & make_padding_mask(tgt_in, pad_id)
+ cross_mask = make_padding_mask(src, pad_id)
+
+ logits = model.apply(
+ {"params": params}, src, tgt_in,
+ src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
+ deterministic=True,
+ )
+
+ loss = optax.softmax_cross_entropy_with_integer_labels(logits, tgt_out)
+ mask = (tgt_out != pad_id).astype(jnp.float32)
+ total_loss += float(jnp.sum(loss * mask))
+ total_tokens += int(jnp.sum(mask))
+
+ avg_nll = total_loss / max(total_tokens, 1)
+ return math.exp(avg_nll)
+
+
+def measure_throughput(model, params, tokenizer, num_runs=10, prompt="Once upon a time", max_gen_len=64):
+ enc_tokens = tokenizer.encode(prompt)
+ enc_input = jnp.array([enc_tokens])
+ pad_id = tokenizer.pad_token_id
+ eos_id = tokenizer.eos_token_id
+
+ src_mask = make_padding_mask(enc_input, pad_id)
+ tgt_mask = make_causal_mask(max_gen_len)
+
+ @jax.jit
+ def decode_step(dec_buffer, encoder_out, src_mask):
+ logits = model.apply(
+ {"params": params}, dec_buffer, encoder_out,
+ self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",
+ )
+ return logits
+
+ # Warmup — compile once
+ encoder_out = model.apply(
+ {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode"
+ )
+ dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
+ dec_buffer = dec_buffer.at[0, 0].set(eos_id)
+ decode_step(dec_buffer, encoder_out, src_mask)
+
+ # Benchmark generation
+ tokens_generated = []
+ latencies = []
+
+ for run in range(num_runs):
+ rng = jax.random.PRNGKey(run)
+ dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)
+ dec_buffer = dec_buffer.at[0, 0].set(eos_id)
+
+ start = time.perf_counter()
+ encoder_out = model.apply(
+ {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode"
+ )
+ logits = decode_step(dec_buffer, encoder_out, src_mask)
+
+ num_tokens = 0
+ for i in range(max_gen_len - 1):
+ rng, sample_rng = jax.random.split(rng)
+ next_token = jax.random.categorical(sample_rng, logits[0, i]).item()
+
+ if next_token == eos_id:
+ break
+
+ num_tokens += 1
+ dec_buffer = dec_buffer.at[0, i + 1].set(next_token)
+ logits = decode_step(dec_buffer, encoder_out, src_mask)
+
+ elapsed = time.perf_counter() - start
+ tokens_generated.append(num_tokens)
+ latencies.append(elapsed)
+
+ avg_tokens = np.mean(tokens_generated)
+ avg_latency = np.mean(latencies)
+ tokens_per_sec = sum(tokens_generated) / sum(latencies)
+
+ return {
+ "avg_tokens_generated": avg_tokens,
+ "avg_latency_s": avg_latency,
+ "tokens_per_second": tokens_per_sec,
+ }
+
+
+def compute_repetition_rate(texts):
+ bigram_rep_rates = []
+ for text in texts:
+ words = text.lower().split()
+ if len(words) < 2:
+ bigram_rep_rates.append(0.0)
+ continue
+ bigrams = [(words[i], words[i + 1]) for i in range(len(words) - 1)]
+ unique = len(set(bigrams))
+ bigram_rep_rates.append(1.0 - unique / len(bigrams))
+ return float(np.mean(bigram_rep_rates))
+
+
+def benchmark_generation_quality(model, params, tokenizer, prompts, max_gen_len=128, temperature=0.8):
+ from .run import generate
+
+ generations = []
+ for i, prompt in enumerate(prompts):
+ text = generate(model, params, tokenizer, prompt, max_gen_len, temperature, seed=i, stream=False)
+ generations.append(text)
+
+ lengths = [len(tokenizer.encode(t)) for t in generations]
+ rep_rate = compute_repetition_rate(generations)
+
+ return {
+ "avg_generation_length": float(np.mean(lengths)),
+ "min_generation_length": int(np.min(lengths)),
+ "max_generation_length": int(np.max(lengths)),
+ "bigram_repetition_rate": rep_rate,
+ "generations": list(zip(prompts, generations)),
+ }
+
+
+def main(args):
+ params, config = load_checkpoint(args.checkpoint)
+ model = EncoderDecoderTransformer(config)
+ tokenizer = get_tokenizer()
+
+ param_count = sum(x.size for x in jax.tree.leaves(params))
+ print(f"\ncheckpoint: {args.checkpoint}")
+ print(f"parameters: {param_count:,}")
+ print(f"config: d={config.d_model}, heads={config.num_heads}, layers={config.num_encoder_layers}/{config.num_decoder_layers}")
+
+ print(f"\nevaluating perplexity ({args.max_eval_samples} samples)...")
+ ds = load_tinystories("validation", max_samples=args.max_eval_samples)
+ texts = [example["text"] for example in ds]
+ enc_inputs, dec_inputs, dec_targets = prepare_encoder_decoder_pairs(
+ texts, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
+ )
+ ppl = compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, args.batch_size, config.pad_token_id)
+ print(f"perplexity: {ppl:.2f}")
+
+ print(f"\nmeasuring throughput ({args.throughput_runs} runs)...")
+ throughput = measure_throughput(model, params, tokenizer, num_runs=args.throughput_runs)
+ print(f"avg tokens: {throughput['avg_tokens_generated']:.1f}")
+ print(f"avg latency: {throughput['avg_latency_s']:.3f}s")
+ print(f"tokens/sec: {throughput['tokens_per_second']:.1f}")
+
+ prompts = [
+ "Once upon a time",
+ "The little dog",
+ "She was very happy because",
+ "One day, a boy named",
+ "The cat and the mouse",
+ ]
+ print(f"\ngenerating samples (t={args.temperature})...")
+ quality = benchmark_generation_quality(
+ model, params, tokenizer, prompts, max_gen_len=args.max_gen_len, temperature=args.temperature
+ )
+ print(f"avg length: {quality['avg_generation_length']:.1f} tokens")
+ print(f"repetition: {quality['bigram_repetition_rate']:.3f}")
+
+ print("\nsamples:")
+ for prompt, gen in quality["generations"]:
+ print(f" [{prompt}]")
+ print(f" {gen[:120]}")
+ print()
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Benchmark transformer on generation tasks")
+ parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")
+ parser.add_argument("--batch-size", type=int, default=32)
+ parser.add_argument("--max-eval-samples", type=int, default=1000)
+ parser.add_argument("--max-enc-len", type=int, default=128)
+ parser.add_argument("--max-dec-len", type=int, default=128)
+ parser.add_argument("--max-gen-len", type=int, default=128)
+ parser.add_argument("--temperature", type=float, default=0.8)
+ parser.add_argument("--throughput-runs", type=int, default=10)
+ return parser.parse_args()
+
+
+if __name__ == "__main__":
+ main(parse_args())
diff --git a/src/train.py b/src/train.py
new file mode 100644
index 0000000..3e3e471
--- /dev/null
+++ b/src/train.py
@@ -0,0 +1,315 @@
+import argparse
+import os
+import pickle
+import time
+from typing import NamedTuple
+
+import jax
+import jax.numpy as jnp
+import numpy as np
+import optax
+from tqdm import tqdm
+from flax.training import train_state
+
+from .data import get_batches, get_tokenizer, load_tinystories, prepare_encoder_decoder_pairs
+from .model import (
+ EncoderDecoderTransformer,
+ TransformerConfig,
+ make_causal_mask,
+ make_padding_mask,
+)
+
+def _newton_schulz(G, steps=5):
+ """Approximate polar decomposition via Newton-Schulz iteration."""
+ a, b, c = 3.4445, -4.7750, 2.0315
+ X = G / (jnp.linalg.norm(G) + 1e-7)
+ transposed = G.shape[0] > G.shape[1]
+ if transposed:
+ X = X.T
+ for _ in range(steps):
+ A = X @ X.T
+ B = b * A + c * A @ A
+ X = a * X + B @ X
+ if transposed:
+ X = X.T
+ return X
+
+
+class MuonState(NamedTuple):
+ mu: optax.Updates
+
+
+def scale_by_muon(momentum=0.95, ns_steps=5):
+ """Muon gradient transform: orthogonalize 2D+ grads, then Nesterov momentum."""
+
+ def init_fn(params):
+ return MuonState(mu=jax.tree.map(jnp.zeros_like, params))
+
+ def update_fn(updates, state, params=None):
+ del params
+
+ def ortho(g):
+ if g.ndim >= 2:
+ return _newton_schulz(g, steps=ns_steps)
+ return g
+
+ ortho_g = jax.tree.map(ortho, updates)
+ new_mu = jax.tree.map(lambda m, g: momentum * m + g, state.mu, ortho_g)
+ new_updates = jax.tree.map(
+ lambda g, m: g + momentum * m, ortho_g, new_mu
+ )
+ return new_updates, MuonState(mu=new_mu)
+
+ return optax.GradientTransformation(init_fn, update_fn)
+
+
+def _param_labels(params):
+ """Label each param: 'muon' for Dense kernels, 'adam' for the rest."""
+
+ def _label(path, _):
+ name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])
+ if name == "kernel":
+ return "muon"
+ return "adam"
+
+ return jax.tree_util.tree_map_with_path(_label, params)
+
+
+def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps):
+ model = EncoderDecoderTransformer(config)
+
+ rng, init_rng, dropout_rng = jax.random.split(rng, 3)
+ dummy_src = jnp.ones((1, 128), dtype=jnp.int32)
+ dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)
+ variables = model.init(
+ {"params": init_rng, "dropout": dropout_rng},
+ dummy_src,
+ dummy_tgt,
+ deterministic=False,
+ )
+
+ adam_schedule = optax.warmup_cosine_decay_schedule(
+ init_value=0.0,
+ peak_value=learning_rate,
+ warmup_steps=warmup_steps,
+ decay_steps=total_steps,
+ end_value=learning_rate * 0.1,
+ )
+ muon_schedule = optax.warmup_cosine_decay_schedule(
+ init_value=0.0,
+ peak_value=muon_lr,
+ warmup_steps=warmup_steps,
+ decay_steps=total_steps,
+ end_value=muon_lr * 0.1,
+ )
+
+ muon_opt = optax.chain(
+ scale_by_muon(momentum=0.95, ns_steps=5),
+ optax.add_decayed_weights(weight_decay=0.01),
+ optax.scale_by_schedule(muon_schedule),
+ optax.scale(-1.0),
+ )
+ adam_opt = optax.chain(
+ optax.adamw(adam_schedule, b2=0.95, weight_decay=0.01),
+ )
+
+ tx = optax.chain(
+ optax.clip_by_global_norm(1.0),
+ optax.multi_transform(
+ {"muon": muon_opt, "adam": adam_opt},
+ _param_labels,
+ ),
+ )
+ return train_state.TrainState.create(
+ apply_fn=model.apply,
+ params=variables["params"],
+ tx=tx,
+ )
+
+
+def train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):
+ pad_id = 50256
+
+ def loss_fn(params):
+ src_mask = make_padding_mask(src, pad_id)
+ tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
+ cross_mask = src_mask
+
+ logits = state.apply_fn(
+ {"params": params},
+ src,
+ tgt_in,
+ src_mask=src_mask,
+ tgt_mask=tgt_mask,
+ cross_mask=cross_mask,
+ deterministic=False,
+ rngs={"dropout": dropout_rng},
+ )
+
+ loss = optax.softmax_cross_entropy_with_integer_labels(logits, tgt_out)
+ mask = (tgt_out != pad_id).astype(jnp.float32)
+ return jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)
+
+ loss, grads = jax.value_and_grad(loss_fn)(state.params)
+ grad_norm = optax.global_norm(grads)
+ state = state.apply_gradients(grads=grads)
+ return state, loss, grad_norm
+
+
+def train(args):
+ use_wandb = getattr(args, "wandb", False)
+ if use_wandb:
+ import wandb
+ if wandb.run is None:
+ wandb.init(project="needle-v1", config=vars(args))
+
+ print("Loading tokenizer...")
+ tokenizer = get_tokenizer()
+
+ print("Loading TinyStories dataset...")
+ ds = load_tinystories("train", max_samples=args.max_samples)
+
+ print("Preparing encoder-decoder pairs...")
+ texts = [example["text"] for example in ds]
+ enc_inputs, dec_inputs, dec_targets = prepare_encoder_decoder_pairs(
+ texts, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
+ )
+ print(f"Prepared {len(enc_inputs)} training pairs")
+
+ config = TransformerConfig(
+ d_model=args.d_model,
+ num_heads=args.num_heads,
+ num_encoder_layers=args.num_layers,
+ num_decoder_layers=args.num_layers,
+ d_ff=args.d_model * 4,
+ max_seq_len=max(args.max_enc_len, args.max_dec_len),
+ dropout_rate=args.dropout,
+ )
+
+ np.random.seed(args.seed)
+ rng = jax.random.PRNGKey(args.seed)
+ rng, init_rng = jax.random.split(rng)
+
+ num_batches = len(enc_inputs) // args.batch_size
+ total_steps = num_batches * args.epochs
+ warmup_steps = max(1, int(total_steps * args.warmup_ratio))
+
+ muon_lr = getattr(args, "muon_lr", 0.02)
+ state = create_train_state(init_rng, config, args.lr, muon_lr, total_steps, warmup_steps)
+
+ param_count = sum(x.size for x in jax.tree.leaves(state.params))
+ print(f"Model parameters: {param_count:,}")
+ print(f"LR schedule: warmup {warmup_steps} steps, cosine decay over {total_steps} steps")
+
+ os.makedirs(args.checkpoint_dir, exist_ok=True)
+ global_step = 0
+ causal_mask = make_causal_mask(args.max_dec_len)
+
+ adam_schedule = optax.warmup_cosine_decay_schedule(
+ init_value=0.0, peak_value=args.lr,
+ warmup_steps=warmup_steps, decay_steps=total_steps, end_value=args.lr * 0.1,
+ )
+ muon_schedule = optax.warmup_cosine_decay_schedule(
+ init_value=0.0, peak_value=muon_lr,
+ warmup_steps=warmup_steps, decay_steps=total_steps, end_value=muon_lr * 0.1,
+ )
+ tokens_per_batch = args.batch_size * (args.max_enc_len + args.max_dec_len)
+
+ for epoch in range(args.epochs):
+ losses = []
+ batches = get_batches(enc_inputs, dec_inputs, dec_targets, args.batch_size)
+ pbar = tqdm(batches, total=num_batches, desc=f"Epoch {epoch + 1}/{args.epochs}")
+
+ for src, tgt_in, tgt_out in pbar:
+ rng, dropout_rng = jax.random.split(rng)
+ t0 = time.perf_counter()
+ state, loss, grad_norm = train_step(
+ state,
+ jnp.array(src),
+ jnp.array(tgt_in),
+ jnp.array(tgt_out),
+ causal_mask,
+ dropout_rng,
+ )
+ loss_val = float(loss)
+ losses.append(loss_val)
+ global_step += 1
+ dt = time.perf_counter() - t0
+ pbar.set_postfix(loss=f"{loss_val:.4f}")
+
+ if use_wandb:
+ wandb.log({
+ "train/loss": loss_val,
+ "train/grad_norm": float(grad_norm),
+ "train/adam_lr": float(adam_schedule(global_step)),
+ "train/muon_lr": float(muon_schedule(global_step)),
+ "train/tokens_per_sec": tokens_per_batch / dt,
+ "train/step": global_step,
+ })
+
+ epoch_avg_loss = sum(losses) / len(losses) if losses else float("nan")
+ final_loss = losses[-1] if losses else float("nan")
+ print(f"Epoch {epoch + 1}/{args.epochs} — avg loss: {epoch_avg_loss:.4f}, final loss: {final_loss:.4f}")
+
+ if use_wandb:
+ wandb.log({
+ "epoch/avg_loss": epoch_avg_loss,
+ "epoch/final_loss": final_loss,
+ "epoch": epoch + 1,
+ })
+
+ ckpt_name = f"needle_{args.num_layers}_{args.d_model}_{global_step}.pkl"
+ ckpt_path = os.path.join(args.checkpoint_dir, ckpt_name)
+ params_np = jax.tree.map(np.array, state.params)
+ with open(ckpt_path, "wb") as f:
+ pickle.dump({"params": params_np, "config": config.__dict__}, f)
+ print(f"Saved checkpoint: {ckpt_path}")
+
+ if use_wandb:
+ wandb.finish()
+ print("Training complete.")
+
+
+def sweep(args):
+ import wandb
+ import yaml
+
+ with open(args.sweep_config) as f:
+ sweep_config = yaml.safe_load(f)
+
+ def sweep_train():
+ wandb.init(project=args.project)
+ for key, value in dict(wandb.config).items():
+ if hasattr(args, key):
+ setattr(args, key, value)
+ args.wandb = True
+ train(args)
+ wandb.finish()
+
+ sweep_id = wandb.sweep(sweep_config, project=args.project)
+ wandb.agent(sweep_id, function=sweep_train, count=args.count)
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Train encoder-decoder transformer on TinyStories")
+ parser.add_argument("--epochs", type=int, default=3)
+ parser.add_argument("--batch-size", type=int, default=32)
+ parser.add_argument("--lr", type=float, default=3e-4)
+ parser.add_argument("--muon-lr", type=float, default=0.02)
+ parser.add_argument("--d-model", type=int, default=128)
+ parser.add_argument("--num-heads", type=int, default=4)
+ parser.add_argument("--num-layers", type=int, default=2)
+ parser.add_argument("--dropout", type=float, default=0.1)
+ parser.add_argument("--max-enc-len", type=int, default=128)
+ parser.add_argument("--max-dec-len", type=int, default=128)
+ parser.add_argument("--max-samples", type=int, default=10000)
+ parser.add_argument("--log-every", type=int, default=50)
+ parser.add_argument("--warmup-ratio", type=float, default=0.05)
+ parser.add_argument("--checkpoint-dir", type=str, default="checkpoints")
+ parser.add_argument("--seed", type=int, default=42)
+ return parser.parse_args()
+
+
+if __name__ == "__main__":
+ train(parse_args())
diff --git a/sweep.yaml b/sweep.yaml
new file mode 100644
index 0000000..6bd0806
--- /dev/null
+++ b/sweep.yaml
@@ -0,0 +1,27 @@
+method: bayes
+metric:
+ name: epoch/avg_loss
+ goal: minimize
+parameters:
+ lr:
+ distribution: log_uniform_values
+ min: 1.0e-5
+ max: 1.0e-2
+ muon_lr:
+ distribution: log_uniform_values
+ min: 5.0e-3
+ max: 0.1
+ warmup_ratio:
+ distribution: uniform
+ min: 0.01
+ max: 0.15
+ dropout:
+ values: [0.0, 0.05, 0.1, 0.15, 0.2]
+ batch_size:
+ values: [16, 32, 64]
+ d_model:
+ values: [64, 128, 256]
+ num_layers:
+ values: [2, 3, 4]
+ num_heads:
+ values: [2, 4, 8]