Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Upload wireframe
HenryNdubuakue085770da293
diff --git a/.gitignore b/.gitignoreindex b7faf40..2eba63f 100644--- a/.gitignore+++ b/.gitignore@@ -205,3 +205,6 @@ cython_debug/marimo/_static/marimo/_lsp/__marimo__/++checkpoints/+wandb/\ No newline at end of filediff --git a/README.md b/README.mdindex 2800bca..78308cc 100644--- a/README.md+++ b/README.md@@ -1 +1,113 @@-# model\ No newline at end of file+```+ ┌────────────────────────────────────────────────────────┐+ │ │+ │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │+ │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │+ │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │+ │ ...the tiny model to rule them all... │+ └────────────────────────────────────────────────────────┘++ ┌─────────────┐+ │ Softmax │+ └──────┬──────┘+ ┌──────┴──────┐+ │ Linear (T) │ ← tied weights+ └──────┬──────┘+ ┌──────┴──────┐+ │ LayerNorm │+ └──────┬──────┘+ ┌─────────┴─────────┐+ │ Decoder x 2 │+ │ ┌───────────────┐ │+ │ │ Masked Self │ │+ ┌──────────────┐ │ │ Attn + RoPE │ │+ │ │ │ ├───────────────┤ │+ │ Encoder x 2 │ │ │ Cross │ │+ │ ┌──────────┐ │────────▶ Attention │ │+ │ │ Self │ │ │ ├───────────────┤ │+ │ │Attn+RoPE │ │ │ │ Feed-Forward │ │+ │ ├──────────┤ │ │ └───────────────┘ │+ │ │ Feed- │ │ └─────────┬─────────┘+ │ │ Forward │ │ ┌──────┴──────┐+ │ └──────────┘ │ │ Embedding │ ← shared+ └──────┬───────┘ └──────┬──────┘+ ┌──────┴───────┐ │+ │ Embedding │ ← shared │+ └──────┬───────┘ │+ │ │+ ┌────┴────┐ ┌─────┴─────┐+ │ Encoder │ │ Decoder │+ │ Input │ │ Input │+ └─────────┘ └───────────┘++ ~7.5M params · d=128 · 4 heads · 2+2 layers · RoPE+```++## Usage++```+git clone https://github.com/cactus-compute/model.git++source ./setup++needle [command]++ ┌───────────────────────────────────────────────────────────────────┐+ │ │+ │ train │+ │ --epochs INT Training epochs (default: 1) │+ │ --batch-size INT Batch size (default: 32) │+ │ --lr FLOAT AdamW learning rate (default: 3e-4) │+ │ --muon-lr FLOAT Muon learning rate (default: 0.02) │+ │ --d-model INT Model dimension (default: 128) │+ │ --num-heads INT Attention heads (default: 4) │+ │ --num-layers INT Encoder/decoder layers (default: 2) │+ │ --dropout FLOAT Dropout rate (default: 0.1) │+ │ --max-enc-len INT Max encoder seq length (default: 128) │+ │ --max-dec-len INT Max decoder seq length (default: 128) │+ │ --max-samples INT Training samples (default: 20000) │+ │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │+ │ --wandb Enable W&B logging │+ │ --checkpoint-dir DIR Checkpoint directory │+ │ --seed INT Random seed (default: 42) │+ │ │+ │ sweep │+ │ --sweep-config PATH Sweep YAML config │+ │ --project STR W&B project name (default: needle-v1) │+ │ --count INT Number of trials (default: 20) │+ │ │+ │ run │+ │ --checkpoint PATH Path to model checkpoint (required) │+ │ --prompts STR [...] One or more prompts to continue │+ │ --max-len INT Max tokens to generate (default: 128) │+ │ --temperature FLOAT Sampling temperature (default: 0.8) │+ │ --seed INT Random seed (default: 0) │+ │ │+ │ test │+ │ --checkpoint PATH Path to model checkpoint (required) │+ │ --batch-size INT Batch size (default: 32) │+ │ --max-eval-samples INT Evaluation samples (default: 1000) │+ │ --max-gen-len INT Max generation length (default: 128) │+ │ --temperature FLOAT Sampling temperature (default: 0.8) │+ │ --throughput-runs INT Throughput runs (default: 10) │+ │ │+ │ evaluate │+ │ --checkpoint PATH Path to model checkpoint (required) │+ │ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │+ │ --max-samples INT Samples per benchmark (default: 500) │+ │ │+ └───────────────────────────────────────────────────────────────────┘+```++## Project Structure++```+ src/+ ├── model.py ······ Transformer architecture+ ├── data.py ······· TinyStories loading & preprocessing+ ├── train.py ······ Training loop+ ├── run.py ········ Story generation from prompts+ ├── test.py ······· Throughput & quality benchmarks+ ├── evaluate.py ··· NLP benchmark evaluation+ └── cli.py ········ CLI entry point+```diff --git a/pyproject.toml b/pyproject.tomlnew file mode 100644index 0000000..6a7c563--- /dev/null+++ b/pyproject.toml@@ -0,0 +1,23 @@+[project]+name = "needle"+version = "0.1.0"+requires-python = ">=3.10"+dependencies = [+ "jax==0.9.0.1",+ "jaxlib==0.9.0.1",+ "flax==0.12.4",+ "optax==0.2.6",+ "datasets==4.5.0",+ "transformers==5.2.0",+ "wandb",+]++[project.scripts]+needle = "src.cli:main"++[tool.setuptools.packages.find]+include = ["src*"]++[build-system]+requires = ["setuptools>=68.0"]+build-backend = "setuptools.build_meta"diff --git a/requirements.txt b/requirements.txtnew file mode 100644index 0000000..16c0614--- /dev/null+++ b/requirements.txt@@ -0,0 +1,8 @@+jax==0.9.0.1+jaxlib==0.9.0.1+flax==0.12.4+optax==0.2.6+datasets==4.5.0+transformers==5.2.0+wandb>=0.19.0+pyyaml>=6.0diff --git a/setup b/setupnew file mode 100644index 0000000..41c7797--- /dev/null+++ b/setup@@ -0,0 +1,29 @@+#!/usr/bin/env bash++VENV_DIR=".venv"++if [ ! -d "$VENV_DIR" ]; then+ echo "Creating virtual environment..."+ python3 -m venv "$VENV_DIR"+else+ echo "Virtual environment already exists."+fi++source "$VENV_DIR/bin/activate"++echo "Installing dependencies..."+pip install --upgrade pip -q+pip install -e . -q++if [ -z "$WANDB_API_KEY" ]; then+ echo ""+ echo " Enter your W&B API key (https://wandb.ai/authorize)"+ echo " Press Enter to skip."+ printf " > "+ read WANDB_API_KEY || true+ if [ -n "$WANDB_API_KEY" ]; then+ export WANDB_API_KEY+ fi+fi++needle --helpdiff --git a/src/__init__.py b/src/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/src/cli.py b/src/cli.pynew file mode 100644index 0000000..47e61ac--- /dev/null+++ b/src/cli.py@@ -0,0 +1,146 @@+import argparse+import sys++HELP = """+ ┌───────────────────────────────────────────────────────────────────┐+ │ │+ │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │+ │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │+ │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │+ │ ...the tiny model to rule them all... │+ │ │+ │ train │+ │ --epochs INT Training epochs (default: 1) │+ │ --batch-size INT Batch size (default: 32) │+ │ --lr FLOAT AdamW learning rate (default: 3e-4) │+ │ --muon-lr FLOAT Muon learning rate (default: 0.02) │+ │ --d-model INT Model dimension (default: 128) │+ │ --num-heads INT Attention heads (default: 4) │+ │ --num-layers INT Encoder/decoder layers (default: 2) │+ │ --dropout FLOAT Dropout rate (default: 0.1) │+ │ --max-enc-len INT Max encoder seq length (default: 128) │+ │ --max-dec-len INT Max decoder seq length (default: 128) │+ │ --max-samples INT Training samples (default: 20000) │+ │ --checkpoint-dir DIR Checkpoint directory │+ │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │+ │ --seed INT Random seed (default: 42) │+ │ │+ │ │+ │ sweep │+ │ --sweep-config PATH Sweep YAML config (default: sweep.yaml)│+ │ --project STR W&B project name (default: needle-v1)│+ │ --count INT Number of trials (default: 20) │+ │ (also accepts all train flags as defaults for non-swept params)│+ │ │+ │ run │+ │ --checkpoint PATH Path to model checkpoint (required) │+ │ --prompts STR [...] One or more prompts to continue │+ │ --max-len INT Max tokens to generate (default: 128) │+ │ --temperature FLOAT Sampling temperature (default: 0.8) │+ │ --seed INT Random seed (default: 0) │+ │ │+ │ test │+ │ --checkpoint PATH Path to model checkpoint (required) │+ │ --batch-size INT Batch size (default: 32) │+ │ --max-eval-samples INT Evaluation samples (default: 1000) │+ │ --max-gen-len INT Max generation length (default: 128) │+ │ --temperature FLOAT Sampling temperature (default: 0.8) │+ │ --throughput-runs INT Throughput runs (default: 10) │+ │ │+ │ evaluate │+ │ --checkpoint PATH Path to model checkpoint (required) │+ │ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │+ │ --max-samples INT Samples per benchmark (default: 500) │+ │ │+ └───────────────────────────────────────────────────────────────────┘+"""+++def main():+ if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help", "help"):+ print(HELP)+ sys.exit(0)++ parser = argparse.ArgumentParser(prog="needle", add_help=False)+ sub = parser.add_subparsers(dest="command")++ p = sub.add_parser("train", add_help=False)+ p.add_argument("--epochs", type=int, default=1)+ p.add_argument("--batch-size", type=int, default=32)+ p.add_argument("--lr", type=float, default=3e-4)+ p.add_argument("--muon-lr", type=float, default=0.02)+ p.add_argument("--d-model", type=int, default=128)+ p.add_argument("--num-heads", type=int, default=4)+ p.add_argument("--num-layers", type=int, default=2)+ p.add_argument("--dropout", type=float, default=0.1)+ p.add_argument("--max-enc-len", type=int, default=128)+ p.add_argument("--max-dec-len", type=int, default=128)+ p.add_argument("--max-samples", type=int, default=20000)+ p.add_argument("--warmup-ratio", type=float, default=0.05)+ p.add_argument("--wandb", action="store_true")+ p.add_argument("--checkpoint-dir", type=str, default="checkpoints")+ p.add_argument("--seed", type=int, default=42)++ p = sub.add_parser("run", add_help=False)+ p.add_argument("--checkpoint", type=str, required=True)+ p.add_argument("--prompts", type=str, nargs="*")+ p.add_argument("--max-len", type=int, default=128)+ p.add_argument("--temperature", type=float, default=0.8)+ p.add_argument("--seed", type=int, default=0)++ p = sub.add_parser("test", add_help=False)+ p.add_argument("--checkpoint", type=str, required=True)+ p.add_argument("--batch-size", type=int, default=32)+ p.add_argument("--max-eval-samples", type=int, default=1000)+ p.add_argument("--max-enc-len", type=int, default=128)+ p.add_argument("--max-dec-len", type=int, default=128)+ p.add_argument("--max-gen-len", type=int, default=128)+ p.add_argument("--temperature", type=float, default=0.8)+ p.add_argument("--throughput-runs", type=int, default=10)++ p = sub.add_parser("sweep", add_help=False)+ p.add_argument("--sweep-config", type=str, default="sweep.yaml")+ p.add_argument("--project", type=str, default="needle-v1")+ p.add_argument("--count", type=int, default=20)+ p.add_argument("--epochs", type=int, default=1)+ p.add_argument("--batch-size", type=int, default=32)+ p.add_argument("--lr", type=float, default=3e-4)+ p.add_argument("--muon-lr", type=float, default=0.02)+ p.add_argument("--d-model", type=int, default=128)+ p.add_argument("--num-heads", type=int, default=4)+ p.add_argument("--num-layers", type=int, default=2)+ p.add_argument("--dropout", type=float, default=0.1)+ p.add_argument("--max-enc-len", type=int, default=128)+ p.add_argument("--max-dec-len", type=int, default=128)+ p.add_argument("--max-samples", type=int, default=20000)+ p.add_argument("--warmup-ratio", type=float, default=0.05)+ p.add_argument("--checkpoint-dir", type=str, default="checkpoints")+ p.add_argument("--seed", type=int, default=42)++ p = sub.add_parser("evaluate", add_help=False)+ p.add_argument("--checkpoint", type=str, required=True)+ p.add_argument("--benchmarks", type=str, nargs="*",+ choices=["wikitext2", "lambada", "hellaswag", "arc_easy"])+ p.add_argument("--max-samples", type=int, default=500)++ args = parser.parse_args()++ if not args.command:+ print(HELP)+ sys.exit(0)++ if args.command == "train":+ from .train import train+ train(args)+ elif args.command == "sweep":+ from .train import sweep+ sweep(args)+ elif args.command == "run":+ from .run import main as run_main+ run_main(args)+ elif args.command == "test":+ from .test import main as test_main+ test_main(args)+ elif args.command == "evaluate":+ from .evaluate import main as eval_main+ eval_main(args)diff --git a/src/data.py b/src/data.pynew file mode 100644index 0000000..0adbda5--- /dev/null+++ b/src/data.py@@ -0,0 +1,68 @@+import os+os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"++import logging+logging.getLogger("transformers").setLevel(logging.ERROR)++import numpy as np+from datasets import load_dataset+from transformers import AutoTokenizer+++def get_tokenizer():+ tokenizer = AutoTokenizer.from_pretrained("gpt2")+ tokenizer.pad_token = tokenizer.eos_token+ return tokenizer+++def load_tinystories(split="train", max_samples=None):+ ds = load_dataset("roneneldan/TinyStories", split=split)+ if max_samples:+ ds = ds.select(range(min(max_samples, len(ds))))+ return ds+++def prepare_encoder_decoder_pairs(texts, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):+ enc_inputs = []+ dec_inputs = []+ dec_targets = []++ pad_id = tokenizer.pad_token_id+ bos_id = tokenizer.eos_token_id++ for text in texts:+ tokens = tokenizer.encode(text, truncation=True, max_length=max_enc_len + max_dec_len)+ if len(tokens) < 6:+ continue++ split_point = max(2, int(len(tokens) * split_ratio))++ enc_tokens = tokens[:split_point][:max_enc_len]+ dec_tokens = tokens[split_point:][:max_dec_len]++ if len(dec_tokens) < 2:+ continue++ dec_in = [bos_id] + dec_tokens[:-1]+ dec_tgt = dec_tokens++ enc_padded = enc_tokens + [pad_id] * (max_enc_len - len(enc_tokens))+ dec_in_padded = dec_in + [pad_id] * (max_dec_len - len(dec_in))+ dec_tgt_padded = dec_tgt + [pad_id] * (max_dec_len - len(dec_tgt))++ enc_inputs.append(enc_padded)+ dec_inputs.append(dec_in_padded)+ dec_targets.append(dec_tgt_padded)++ return np.array(enc_inputs), np.array(dec_inputs), np.array(dec_targets)+++def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True):+ n = len(enc_inputs)+ indices = np.arange(n)+ if shuffle:+ np.random.shuffle(indices)++ for i in range(0, n - batch_size + 1, batch_size):+ batch_idx = indices[i : i + batch_size]+ yield enc_inputs[batch_idx], dec_inputs[batch_idx], dec_targets[batch_idx]diff --git a/src/evaluate.py b/src/evaluate.pynew file mode 100644index 0000000..816a25a--- /dev/null+++ b/src/evaluate.py@@ -0,0 +1,293 @@+import argparse+import math+import pickle++import jax+import jax.numpy as jnp+import numpy as np+from datasets import load_dataset++from .data import get_tokenizer+from .model import (+ EncoderDecoderTransformer,+ TransformerConfig,+ make_causal_mask,+ make_padding_mask,+)+++def load_checkpoint(path):+ with open(path, "rb") as f:+ data = pickle.load(f)+ params = jax.tree.map(jnp.array, data["params"])+ config = TransformerConfig(**data["config"])+ return params, config+++def score_sequence(model, params, enc_tokens, dec_tokens, pad_id):+ """Compute average negative log-likelihood of dec_tokens given enc_tokens."""+ enc_input = jnp.array([enc_tokens])+ src_mask = make_padding_mask(enc_input, pad_id)++ encoder_out = model.apply(+ {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",+ )++ dec_in = [pad_id] + list(dec_tokens[:-1])+ dec_input = jnp.array([dec_in])+ tgt_mask = make_causal_mask(len(dec_in))+ cross_mask = src_mask++ logits = model.apply(+ {"params": params}, dec_input, encoder_out,+ self_mask=tgt_mask, cross_mask=cross_mask, deterministic=True, method="decode",+ )++ log_probs = jax.nn.log_softmax(logits[0])+ target_ids = jnp.array(dec_tokens)+ token_lls = log_probs[jnp.arange(len(dec_tokens)), target_ids]+ return float(jnp.mean(token_lls))+++def eval_wikitext2(model, params, tokenizer, max_samples=500, max_len=256):+ """Perplexity on WikiText-2 test split."""+ ds = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="test")++ pad_id = tokenizer.pad_token_id+ total_nll = 0.0+ total_tokens = 0+ evaluated = 0++ for example in ds:+ text = example["text"].strip()+ if len(text) < 20:+ continue++ tokens = tokenizer.encode(text)+ if len(tokens) < 6:+ continue++ tokens = tokens[:max_len]+ split = max(2, len(tokens) // 3)+ enc_tokens = tokens[:split]+ dec_tokens = tokens[split:]++ enc_input = jnp.array([enc_tokens])+ src_mask = make_padding_mask(enc_input, pad_id)+ encoder_out = model.apply(+ {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",+ )++ dec_in = [pad_id] + list(dec_tokens[:-1])+ dec_input = jnp.array([dec_in])+ tgt_mask = make_causal_mask(len(dec_in))++ logits = model.apply(+ {"params": params}, dec_input, encoder_out,+ self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",+ )++ log_probs = jax.nn.log_softmax(logits[0])+ target_ids = jnp.array(dec_tokens)+ token_lls = log_probs[jnp.arange(len(dec_tokens)), target_ids]++ total_nll += -float(jnp.sum(token_lls))+ total_tokens += len(dec_tokens)+ evaluated += 1++ if evaluated >= max_samples:+ break++ ppl = math.exp(total_nll / max(total_tokens, 1))+ return {"perplexity": ppl, "samples": evaluated, "tokens": total_tokens}+++def eval_lambada(model, params, tokenizer, max_samples=500):+ """Accuracy of predicting the final word on LAMBADA."""+ ds = load_dataset("EleutherAI/lambada_openai", "default", split="test")++ pad_id = tokenizer.pad_token_id+ correct = 0+ total = 0++ for example in ds:+ text = example["text"].strip()+ words = text.rsplit(" ", 1)+ if len(words) < 2:+ continue++ context, last_word = words+ context_tokens = tokenizer.encode(context)+ target_tokens = tokenizer.encode(" " + last_word)++ if not context_tokens or not target_tokens:+ continue++ enc_input = jnp.array([context_tokens])+ src_mask = make_padding_mask(enc_input, pad_id)+ encoder_out = model.apply(+ {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode",+ )++ dec_in = jnp.array([[pad_id]])+ tgt_mask = make_causal_mask(1)++ logits = model.apply(+ {"params": params}, dec_in, encoder_out,+ self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",+ )++ predicted = int(jnp.argmax(logits[0, 0]))+ if predicted == target_tokens[0]:+ correct += 1+ total += 1++ if total >= max_samples:+ break++ acc = correct / max(total, 1)+ return {"accuracy": acc, "correct": correct, "total": total}+++def eval_hellaswag(model, params, tokenizer, max_samples=500):+ """Accuracy on HellaSwag by scoring each candidate ending."""+ ds = load_dataset("Rowan/hellaswag", split="validation")++ pad_id = tokenizer.pad_token_id+ correct = 0+ total = 0++ for example in ds:+ ctx = example["ctx"].strip()+ endings = example["endings"]+ label = int(example["label"])++ enc_tokens = tokenizer.encode(ctx)+ if not enc_tokens:+ continue++ # Truncate context to keep it manageable+ enc_tokens = enc_tokens[:192]++ scores = []+ for ending in endings:+ dec_tokens = tokenizer.encode(" " + ending.strip())+ if not dec_tokens:+ scores.append(float("-inf"))+ continue+ dec_tokens = dec_tokens[:64]+ score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id)+ scores.append(score)++ predicted = int(np.argmax(scores))+ if predicted == label:+ correct += 1+ total += 1++ if total >= max_samples:+ break++ acc = correct / max(total, 1)+ return {"accuracy": acc, "correct": correct, "total": total}+++def eval_arc_easy(model, params, tokenizer, max_samples=500):+ """Accuracy on ARC-Easy by scoring each answer choice."""+ ds = load_dataset("allenai/ai2_arc", "ARC-Easy", split="test")++ pad_id = tokenizer.pad_token_id+ correct = 0+ total = 0++ for example in ds:+ question = example["question"].strip()+ choices = example["choices"]+ answer_key = example["answerKey"]++ labels = choices["label"]+ texts = choices["text"]++ enc_tokens = tokenizer.encode(question)[:128]+ if not enc_tokens:+ continue++ scores = []+ for choice_text in texts:+ dec_tokens = tokenizer.encode(" " + choice_text.strip())+ if not dec_tokens:+ scores.append(float("-inf"))+ continue+ dec_tokens = dec_tokens[:64]+ score = score_sequence(model, params, enc_tokens, dec_tokens, pad_id)+ scores.append(score)++ predicted_idx = int(np.argmax(scores))+ predicted_label = labels[predicted_idx]+ if predicted_label == answer_key:+ correct += 1+ total += 1++ if total >= max_samples:+ break++ acc = correct / max(total, 1)+ return {"accuracy": acc, "correct": correct, "total": total}+++BENCHMARKS = {+ "wikitext2": eval_wikitext2,+ "lambada": eval_lambada,+ "hellaswag": eval_hellaswag,+ "arc_easy": eval_arc_easy,+}+++def main(args):+ print(f"Loading checkpoint: {args.checkpoint}")+ params, config = load_checkpoint(args.checkpoint)+ model = EncoderDecoderTransformer(config)+ tokenizer = get_tokenizer()++ param_count = sum(x.size for x in jax.tree.leaves(params))+ print(f"Model parameters: {param_count:,}")++ benchmarks = args.benchmarks or list(BENCHMARKS.keys())++ results = {}+ for name in benchmarks:+ if name not in BENCHMARKS:+ print(f"Unknown benchmark: {name}, skipping")+ continue++ print(f"\n{'=' * 50}")+ print(f"Benchmark: {name}")+ print("=" * 50)++ result = BENCHMARKS[name](model, params, tokenizer, max_samples=args.max_samples)+ results[name] = result++ for k, v in result.items():+ if isinstance(v, float):+ print(f" {k}: {v:.4f}")+ else:+ print(f" {k}: {v}")++ print(f"\n{'=' * 50}")+ print("Summary")+ print("=" * 50)+ for name, result in results.items():+ metric = "perplexity" if "perplexity" in result else "accuracy"+ print(f" {name:>12s}: {result[metric]:.4f} ({metric})")+++def parse_args():+ parser = argparse.ArgumentParser(description="Evaluate on established NLP benchmarks")+ parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")+ parser.add_argument("--benchmarks", type=str, nargs="*", choices=list(BENCHMARKS.keys()),+ help="Benchmarks to run (default: all)")+ parser.add_argument("--max-samples", type=int, default=500)+ return parser.parse_args()+++if __name__ == "__main__":+ main(parse_args())diff --git a/src/model.py b/src/model.pynew file mode 100644index 0000000..e7b0d45--- /dev/null+++ b/src/model.py@@ -0,0 +1,258 @@+import math+from dataclasses import dataclass++import jax.numpy as jnp+import jax.nn.initializers as jinit+import flax.linen as nn+++def default_init():+ return jinit.normal(stddev=0.02)+++def residual_init(num_layers):+ return jinit.normal(stddev=0.02 / math.sqrt(2 * num_layers))+++DTYPE_MAP = {"float32": jnp.float32, "bfloat16": jnp.bfloat16, "float16": jnp.float16}+++@dataclass+class TransformerConfig:+ vocab_size: int = 50257+ d_model: int = 128+ num_heads: int = 4+ num_encoder_layers: int = 2+ num_decoder_layers: int = 2+ d_ff: int = 512+ max_seq_len: int = 128+ dropout_rate: float = 0.1+ pad_token_id: int = 50256+ rope_theta: float = 10000.0+ dtype: str = "bfloat16"++ @property+ def jax_dtype(self):+ return DTYPE_MAP[self.dtype]++ @property+ def total_layers(self):+ return self.num_encoder_layers + self.num_decoder_layers+++def precompute_rope_freqs(head_dim, seq_len, theta=10000.0):+ freqs = 1.0 / (theta ** (jnp.arange(0, head_dim, 2).astype(jnp.float32) / head_dim))+ t = jnp.arange(seq_len).astype(jnp.float32)+ angles = jnp.outer(t, freqs)+ return jnp.cos(angles), jnp.sin(angles)+++def apply_rope(x, cos, sin):+ T = x.shape[2]+ half = x.shape[-1] // 2+ cos = cos[:T][None, None, :, :]+ sin = sin[:T][None, None, :, :]+ x1 = x[..., :half]+ x2 = x[..., half:]+ return jnp.concatenate([x1 * cos - x2 * sin, x2 * cos + x1 * sin], axis=-1)+++class MultiHeadAttention(nn.Module):+ num_heads: int+ d_model: int+ num_layers: int+ dropout_rate: float = 0.1+ dtype: jnp.dtype = jnp.bfloat16++ @nn.compact+ def __call__(self, q_input, kv_input, mask=None, rope=None, deterministic=True):+ head_dim = self.d_model // self.num_heads+ B = q_input.shape[0]++ q = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="q_proj")(q_input)+ k = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="k_proj")(kv_input)+ v = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="v_proj")(kv_input)++ q = q.reshape(B, -1, self.num_heads, head_dim).transpose(0, 2, 1, 3)+ k = k.reshape(B, -1, self.num_heads, head_dim).transpose(0, 2, 1, 3)+ v = v.reshape(B, -1, self.num_heads, head_dim).transpose(0, 2, 1, 3)++ if rope is not None:+ cos, sin = rope+ q = apply_rope(q, cos, sin)+ k = apply_rope(k, cos, sin)++ scale = jnp.sqrt(jnp.float32(head_dim))+ attn_weights = jnp.matmul(q, k.transpose(0, 1, 3, 2)) / scale++ if mask is not None:+ attn_weights = jnp.where(mask, attn_weights, jnp.finfo(attn_weights.dtype).min)++ attn_weights = nn.softmax(attn_weights, axis=-1)+ attn_weights = nn.Dropout(rate=self.dropout_rate)(attn_weights, deterministic=deterministic)++ out = jnp.matmul(attn_weights, v)+ out = out.transpose(0, 2, 1, 3).reshape(B, -1, self.d_model)+ return nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="out_proj")(out)+++class FeedForward(nn.Module):+ d_model: int+ d_ff: int+ num_layers: int+ dropout_rate: float = 0.1+ dtype: jnp.dtype = jnp.bfloat16++ @nn.compact+ def __call__(self, x, deterministic=True):+ gate = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="gate_proj")(x)+ up = nn.Dense(self.d_ff, dtype=self.dtype, use_bias=False, kernel_init=default_init(), name="up_proj")(x)+ x = nn.silu(gate) * up+ x = nn.Dense(self.d_model, dtype=self.dtype, use_bias=False, kernel_init=residual_init(self.num_layers), name="down_proj")(x)+ x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)+ return x+++class EncoderBlock(nn.Module):+ num_heads: int+ d_model: int+ d_ff: int+ num_layers: int+ dropout_rate: float = 0.1+ dtype: jnp.dtype = jnp.bfloat16++ @nn.compact+ def __call__(self, x, mask=None, rope=None, deterministic=True):+ residual = x+ x = nn.RMSNorm(dtype=self.dtype)(x)+ x = MultiHeadAttention(self.num_heads, self.d_model, self.num_layers, self.dropout_rate, self.dtype)(+ x, x, mask=mask, rope=rope, deterministic=deterministic+ )+ x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)+ x = x + residual++ residual = x+ x = nn.RMSNorm(dtype=self.dtype)(x)+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dropout_rate, self.dtype)(x, deterministic=deterministic)+ x = x + residual++ return x+++class DecoderBlock(nn.Module):+ num_heads: int+ d_model: int+ d_ff: int+ num_layers: int+ dropout_rate: float = 0.1+ dtype: jnp.dtype = jnp.bfloat16++ @nn.compact+ def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None, deterministic=True):+ # Self-attention with RoPE+ residual = x+ x = nn.RMSNorm(dtype=self.dtype)(x)+ x = MultiHeadAttention(self.num_heads, self.d_model, self.num_layers, self.dropout_rate, self.dtype, name="self_attn")(+ x, x, mask=self_mask, rope=rope, deterministic=deterministic+ )+ x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)+ x = x + residual++ # Cross-attention (no RoPE)+ residual = x+ x = nn.RMSNorm(dtype=self.dtype)(x)+ x = MultiHeadAttention(self.num_heads, self.d_model, self.num_layers, self.dropout_rate, self.dtype, name="cross_attn")(+ x, encoder_out, mask=cross_mask, deterministic=deterministic+ )+ x = nn.Dropout(rate=self.dropout_rate)(x, deterministic=deterministic)+ x = x + residual++ residual = x+ x = nn.RMSNorm(dtype=self.dtype)(x)+ x = FeedForward(self.d_model, self.d_ff, self.num_layers, self.dropout_rate, self.dtype)(x, deterministic=deterministic)+ x = x + residual++ return x+++class Encoder(nn.Module):+ config: TransformerConfig++ @nn.compact+ def __call__(self, x, mask=None, rope=None, deterministic=True):+ cfg = self.config+ dt = cfg.jax_dtype+ x = x.astype(dt)+ x = nn.Dropout(rate=cfg.dropout_rate)(x, deterministic=deterministic)++ for i in range(cfg.num_encoder_layers):+ x = EncoderBlock(+ cfg.num_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, cfg.dropout_rate, dt, name=f"block_{i}"+ )(x, mask=mask, rope=rope, deterministic=deterministic)++ x = nn.RMSNorm(dtype=dt)(x)+ return x+++class Decoder(nn.Module):+ config: TransformerConfig++ @nn.compact+ def __call__(self, x, encoder_out, self_mask=None, cross_mask=None, rope=None, deterministic=True):+ cfg = self.config+ dt = cfg.jax_dtype+ x = x.astype(dt)+ x = nn.Dropout(rate=cfg.dropout_rate)(x, deterministic=deterministic)++ for i in range(cfg.num_decoder_layers):+ x = DecoderBlock(+ cfg.num_heads, cfg.d_model, cfg.d_ff, cfg.total_layers, cfg.dropout_rate, dt, name=f"block_{i}"+ )(x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope, deterministic=deterministic)++ x = nn.RMSNorm(dtype=dt)(x)+ return x+++class EncoderDecoderTransformer(nn.Module):+ """Encoder-decoder transformer with shared embeddings, tied output, RoPE, and bfloat16."""+ config: TransformerConfig++ def setup(self):+ self.embedding = nn.Embed(self.config.vocab_size, self.config.d_model, embedding_init=jinit.normal(stddev=0.02))+ self.encoder = Encoder(self.config)+ self.decoder = Decoder(self.config)++ def _rope(self, seq_len):+ head_dim = self.config.d_model // self.config.num_heads+ return precompute_rope_freqs(head_dim, seq_len, self.config.rope_theta)++ def encode(self, src, src_mask=None, deterministic=True):+ x = self.embedding(src)+ rope = self._rope(src.shape[1])+ return self.encoder(x, mask=src_mask, rope=rope, deterministic=deterministic)++ def decode(self, tgt, encoder_out, self_mask=None, cross_mask=None, deterministic=True):+ x = self.embedding(tgt)+ rope = self._rope(tgt.shape[1])+ x = self.decoder(+ x, encoder_out, self_mask=self_mask, cross_mask=cross_mask, rope=rope, deterministic=deterministic+ )+ logits = x.astype(jnp.float32) @ self.embedding.embedding.T+ return logits++ def __call__(self, src, tgt, src_mask=None, tgt_mask=None, cross_mask=None, deterministic=True):+ encoder_out = self.encode(src, src_mask=src_mask, deterministic=deterministic)+ logits = self.decode(+ tgt, encoder_out, self_mask=tgt_mask, cross_mask=cross_mask, deterministic=deterministic+ )+ return logits+++def make_causal_mask(seq_len):+ mask = jnp.tril(jnp.ones((seq_len, seq_len), dtype=jnp.bool_))+ return mask[None, None, :, :]+++def make_padding_mask(tokens, pad_token_id):+ mask = tokens != pad_token_id+ return mask[:, None, None, :]diff --git a/src/run.py b/src/run.pynew file mode 100644index 0000000..b4a7bf2--- /dev/null+++ b/src/run.py@@ -0,0 +1,124 @@+import argparse+import pickle+import sys++import jax+import jax.numpy as jnp++from .data import get_tokenizer+from .model import (+ EncoderDecoderTransformer,+ TransformerConfig,+ make_causal_mask,+ make_padding_mask,+)+++def load_checkpoint(path):+ with open(path, "rb") as f:+ data = pickle.load(f)+ params = jax.tree.map(jnp.array, data["params"])+ config = TransformerConfig(**data["config"])+ return params, config+++def generate(model, params, tokenizer, prompt, max_gen_len=128, temperature=0.8, seed=0, stream=True):+ enc_tokens = tokenizer.encode(prompt)+ enc_input = jnp.array([enc_tokens])++ pad_id = tokenizer.pad_token_id+ eos_id = tokenizer.eos_token_id++ src_mask = make_padding_mask(enc_input, pad_id)+ encoder_out = model.apply(+ {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode"+ )++ # Fixed-size buffer so JAX compiles once+ tgt_mask = make_causal_mask(max_gen_len)+ dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)+ dec_buffer = dec_buffer.at[0, 0].set(eos_id)++ @jax.jit+ def decode_step(dec_buffer, encoder_out, src_mask):+ logits = model.apply(+ {"params": params},+ dec_buffer,+ encoder_out,+ self_mask=tgt_mask,+ cross_mask=src_mask,+ deterministic=True,+ method="decode",+ )+ return logits++ rng = jax.random.PRNGKey(seed)+ generated_tokens = []++ if stream:+ sys.stdout.write(f"\n{prompt}")+ sys.stdout.flush()++ # Compile once on first call+ logits = decode_step(dec_buffer, encoder_out, src_mask)++ for i in range(max_gen_len - 1):+ next_logits = logits[0, i] / temperature+ rng, sample_rng = jax.random.split(rng)+ next_token = jax.random.categorical(sample_rng, next_logits).item()++ if next_token == eos_id:+ break++ generated_tokens.append(next_token)+ dec_buffer = dec_buffer.at[0, i + 1].set(next_token)++ if stream:+ sys.stdout.write(tokenizer.decode([next_token]))+ sys.stdout.flush()++ logits = decode_step(dec_buffer, encoder_out, src_mask)++ if stream:+ sys.stdout.write("\n")++ return tokenizer.decode(generated_tokens)+++def main(args):+ print(f"Loading checkpoint: {args.checkpoint}")+ params, config = load_checkpoint(args.checkpoint)++ model = EncoderDecoderTransformer(config)+ tokenizer = get_tokenizer()++ param_count = sum(x.size for x in jax.tree.leaves(params))+ print(f"Model parameters: {param_count:,}")++ prompts = args.prompts or ["Once upon a time", "The little dog", "She was very happy because"]++ for i, prompt in enumerate(prompts):+ generate(+ model,+ params,+ tokenizer,+ prompt,+ max_gen_len=args.max_len,+ temperature=args.temperature,+ seed=args.seed + i,+ stream=True,+ )+++def parse_args():+ parser = argparse.ArgumentParser(description="Generate stories with trained transformer")+ parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")+ parser.add_argument("--prompts", type=str, nargs="*", help="Prompts to continue")+ parser.add_argument("--max-len", type=int, default=128)+ parser.add_argument("--temperature", type=float, default=0.8)+ parser.add_argument("--seed", type=int, default=0)+ return parser.parse_args()+++if __name__ == "__main__":+ main(parse_args())diff --git a/src/test.py b/src/test.pynew file mode 100644index 0000000..f312057--- /dev/null+++ b/src/test.py@@ -0,0 +1,214 @@+import argparse+import math+import pickle+import time++import jax+import jax.numpy as jnp+import numpy as np+import optax++from .data import get_batches, get_tokenizer, load_tinystories, prepare_encoder_decoder_pairs+from .model import (+ EncoderDecoderTransformer,+ TransformerConfig,+ make_causal_mask,+ make_padding_mask,+)+++def load_checkpoint(path):+ with open(path, "rb") as f:+ data = pickle.load(f)+ params = jax.tree.map(jnp.array, data["params"])+ config = TransformerConfig(**data["config"])+ return params, config+++def compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, batch_size, pad_id):+ total_loss = 0.0+ total_tokens = 0++ for src, tgt_in, tgt_out in get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=False):+ src, tgt_in, tgt_out = jnp.array(src), jnp.array(tgt_in), jnp.array(tgt_out)++ src_mask = make_padding_mask(src, pad_id)+ tgt_mask = make_causal_mask(tgt_in.shape[1]) & make_padding_mask(tgt_in, pad_id)+ cross_mask = make_padding_mask(src, pad_id)++ logits = model.apply(+ {"params": params}, src, tgt_in,+ src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,+ deterministic=True,+ )++ loss = optax.softmax_cross_entropy_with_integer_labels(logits, tgt_out)+ mask = (tgt_out != pad_id).astype(jnp.float32)+ total_loss += float(jnp.sum(loss * mask))+ total_tokens += int(jnp.sum(mask))++ avg_nll = total_loss / max(total_tokens, 1)+ return math.exp(avg_nll)+++def measure_throughput(model, params, tokenizer, num_runs=10, prompt="Once upon a time", max_gen_len=64):+ enc_tokens = tokenizer.encode(prompt)+ enc_input = jnp.array([enc_tokens])+ pad_id = tokenizer.pad_token_id+ eos_id = tokenizer.eos_token_id++ src_mask = make_padding_mask(enc_input, pad_id)+ tgt_mask = make_causal_mask(max_gen_len)++ @jax.jit+ def decode_step(dec_buffer, encoder_out, src_mask):+ logits = model.apply(+ {"params": params}, dec_buffer, encoder_out,+ self_mask=tgt_mask, cross_mask=src_mask, deterministic=True, method="decode",+ )+ return logits++ # Warmup — compile once+ encoder_out = model.apply(+ {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode"+ )+ dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)+ dec_buffer = dec_buffer.at[0, 0].set(eos_id)+ decode_step(dec_buffer, encoder_out, src_mask)++ # Benchmark generation+ tokens_generated = []+ latencies = []++ for run in range(num_runs):+ rng = jax.random.PRNGKey(run)+ dec_buffer = jnp.full((1, max_gen_len), pad_id, dtype=jnp.int32)+ dec_buffer = dec_buffer.at[0, 0].set(eos_id)++ start = time.perf_counter()+ encoder_out = model.apply(+ {"params": params}, enc_input, src_mask=src_mask, deterministic=True, method="encode"+ )+ logits = decode_step(dec_buffer, encoder_out, src_mask)++ num_tokens = 0+ for i in range(max_gen_len - 1):+ rng, sample_rng = jax.random.split(rng)+ next_token = jax.random.categorical(sample_rng, logits[0, i]).item()++ if next_token == eos_id:+ break++ num_tokens += 1+ dec_buffer = dec_buffer.at[0, i + 1].set(next_token)+ logits = decode_step(dec_buffer, encoder_out, src_mask)++ elapsed = time.perf_counter() - start+ tokens_generated.append(num_tokens)+ latencies.append(elapsed)++ avg_tokens = np.mean(tokens_generated)+ avg_latency = np.mean(latencies)+ tokens_per_sec = sum(tokens_generated) / sum(latencies)++ return {+ "avg_tokens_generated": avg_tokens,+ "avg_latency_s": avg_latency,+ "tokens_per_second": tokens_per_sec,+ }+++def compute_repetition_rate(texts):+ bigram_rep_rates = []+ for text in texts:+ words = text.lower().split()+ if len(words) < 2:+ bigram_rep_rates.append(0.0)+ continue+ bigrams = [(words[i], words[i + 1]) for i in range(len(words) - 1)]+ unique = len(set(bigrams))+ bigram_rep_rates.append(1.0 - unique / len(bigrams))+ return float(np.mean(bigram_rep_rates))+++def benchmark_generation_quality(model, params, tokenizer, prompts, max_gen_len=128, temperature=0.8):+ from .run import generate++ generations = []+ for i, prompt in enumerate(prompts):+ text = generate(model, params, tokenizer, prompt, max_gen_len, temperature, seed=i, stream=False)+ generations.append(text)++ lengths = [len(tokenizer.encode(t)) for t in generations]+ rep_rate = compute_repetition_rate(generations)++ return {+ "avg_generation_length": float(np.mean(lengths)),+ "min_generation_length": int(np.min(lengths)),+ "max_generation_length": int(np.max(lengths)),+ "bigram_repetition_rate": rep_rate,+ "generations": list(zip(prompts, generations)),+ }+++def main(args):+ params, config = load_checkpoint(args.checkpoint)+ model = EncoderDecoderTransformer(config)+ tokenizer = get_tokenizer()++ param_count = sum(x.size for x in jax.tree.leaves(params))+ print(f"\ncheckpoint: {args.checkpoint}")+ print(f"parameters: {param_count:,}")+ print(f"config: d={config.d_model}, heads={config.num_heads}, layers={config.num_encoder_layers}/{config.num_decoder_layers}")++ print(f"\nevaluating perplexity ({args.max_eval_samples} samples)...")+ ds = load_tinystories("validation", max_samples=args.max_eval_samples)+ texts = [example["text"] for example in ds]+ enc_inputs, dec_inputs, dec_targets = prepare_encoder_decoder_pairs(+ texts, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len+ )+ ppl = compute_perplexity(model, params, enc_inputs, dec_inputs, dec_targets, args.batch_size, config.pad_token_id)+ print(f"perplexity: {ppl:.2f}")++ print(f"\nmeasuring throughput ({args.throughput_runs} runs)...")+ throughput = measure_throughput(model, params, tokenizer, num_runs=args.throughput_runs)+ print(f"avg tokens: {throughput['avg_tokens_generated']:.1f}")+ print(f"avg latency: {throughput['avg_latency_s']:.3f}s")+ print(f"tokens/sec: {throughput['tokens_per_second']:.1f}")++ prompts = [+ "Once upon a time",+ "The little dog",+ "She was very happy because",+ "One day, a boy named",+ "The cat and the mouse",+ ]+ print(f"\ngenerating samples (t={args.temperature})...")+ quality = benchmark_generation_quality(+ model, params, tokenizer, prompts, max_gen_len=args.max_gen_len, temperature=args.temperature+ )+ print(f"avg length: {quality['avg_generation_length']:.1f} tokens")+ print(f"repetition: {quality['bigram_repetition_rate']:.3f}")++ print("\nsamples:")+ for prompt, gen in quality["generations"]:+ print(f" [{prompt}]")+ print(f" {gen[:120]}")+ print()+++def parse_args():+ parser = argparse.ArgumentParser(description="Benchmark transformer on generation tasks")+ parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")+ parser.add_argument("--batch-size", type=int, default=32)+ parser.add_argument("--max-eval-samples", type=int, default=1000)+ parser.add_argument("--max-enc-len", type=int, default=128)+ parser.add_argument("--max-dec-len", type=int, default=128)+ parser.add_argument("--max-gen-len", type=int, default=128)+ parser.add_argument("--temperature", type=float, default=0.8)+ parser.add_argument("--throughput-runs", type=int, default=10)+ return parser.parse_args()+++if __name__ == "__main__":+ main(parse_args())diff --git a/src/train.py b/src/train.pynew file mode 100644index 0000000..3e3e471--- /dev/null+++ b/src/train.py@@ -0,0 +1,315 @@+import argparse+import os+import pickle+import time+from typing import NamedTuple++import jax+import jax.numpy as jnp+import numpy as np+import optax+from tqdm import tqdm+from flax.training import train_state++from .data import get_batches, get_tokenizer, load_tinystories, prepare_encoder_decoder_pairs+from .model import (+ EncoderDecoderTransformer,+ TransformerConfig,+ make_causal_mask,+ make_padding_mask,+)++def _newton_schulz(G, steps=5):+ """Approximate polar decomposition via Newton-Schulz iteration."""+ a, b, c = 3.4445, -4.7750, 2.0315+ X = G / (jnp.linalg.norm(G) + 1e-7)+ transposed = G.shape[0] > G.shape[1]+ if transposed:+ X = X.T+ for _ in range(steps):+ A = X @ X.T+ B = b * A + c * A @ A+ X = a * X + B @ X+ if transposed:+ X = X.T+ return X+++class MuonState(NamedTuple):+ mu: optax.Updates+++def scale_by_muon(momentum=0.95, ns_steps=5):+ """Muon gradient transform: orthogonalize 2D+ grads, then Nesterov momentum."""++ def init_fn(params):+ return MuonState(mu=jax.tree.map(jnp.zeros_like, params))++ def update_fn(updates, state, params=None):+ del params++ def ortho(g):+ if g.ndim >= 2:+ return _newton_schulz(g, steps=ns_steps)+ return g++ ortho_g = jax.tree.map(ortho, updates)+ new_mu = jax.tree.map(lambda m, g: momentum * m + g, state.mu, ortho_g)+ new_updates = jax.tree.map(+ lambda g, m: g + momentum * m, ortho_g, new_mu+ )+ return new_updates, MuonState(mu=new_mu)++ return optax.GradientTransformation(init_fn, update_fn)+++def _param_labels(params):+ """Label each param: 'muon' for Dense kernels, 'adam' for the rest."""++ def _label(path, _):+ name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])+ if name == "kernel":+ return "muon"+ return "adam"++ return jax.tree_util.tree_map_with_path(_label, params)+++def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps):+ model = EncoderDecoderTransformer(config)++ rng, init_rng, dropout_rng = jax.random.split(rng, 3)+ dummy_src = jnp.ones((1, 128), dtype=jnp.int32)+ dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)+ variables = model.init(+ {"params": init_rng, "dropout": dropout_rng},+ dummy_src,+ dummy_tgt,+ deterministic=False,+ )++ adam_schedule = optax.warmup_cosine_decay_schedule(+ init_value=0.0,+ peak_value=learning_rate,+ warmup_steps=warmup_steps,+ decay_steps=total_steps,+ end_value=learning_rate * 0.1,+ )+ muon_schedule = optax.warmup_cosine_decay_schedule(+ init_value=0.0,+ peak_value=muon_lr,+ warmup_steps=warmup_steps,+ decay_steps=total_steps,+ end_value=muon_lr * 0.1,+ )++ muon_opt = optax.chain(+ scale_by_muon(momentum=0.95, ns_steps=5),+ optax.add_decayed_weights(weight_decay=0.01),+ optax.scale_by_schedule(muon_schedule),+ optax.scale(-1.0),+ )+ adam_opt = optax.chain(+ optax.adamw(adam_schedule, b2=0.95, weight_decay=0.01),+ )++ tx = optax.chain(+ optax.clip_by_global_norm(1.0),+ optax.multi_transform(+ {"muon": muon_opt, "adam": adam_opt},+ _param_labels,+ ),+ )+ return train_state.TrainState.create(+ apply_fn=model.apply,+ params=variables["params"],+ tx=tx,+ )+++def train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):+ pad_id = 50256++ def loss_fn(params):+ src_mask = make_padding_mask(src, pad_id)+ tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)+ cross_mask = src_mask++ logits = state.apply_fn(+ {"params": params},+ src,+ tgt_in,+ src_mask=src_mask,+ tgt_mask=tgt_mask,+ cross_mask=cross_mask,+ deterministic=False,+ rngs={"dropout": dropout_rng},+ )++ loss = optax.softmax_cross_entropy_with_integer_labels(logits, tgt_out)+ mask = (tgt_out != pad_id).astype(jnp.float32)+ return jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)++ loss, grads = jax.value_and_grad(loss_fn)(state.params)+ grad_norm = optax.global_norm(grads)+ state = state.apply_gradients(grads=grads)+ return state, loss, grad_norm+++def train(args):+ use_wandb = getattr(args, "wandb", False)+ if use_wandb:+ import wandb+ if wandb.run is None:+ wandb.init(project="needle-v1", config=vars(args))++ print("Loading tokenizer...")+ tokenizer = get_tokenizer()++ print("Loading TinyStories dataset...")+ ds = load_tinystories("train", max_samples=args.max_samples)++ print("Preparing encoder-decoder pairs...")+ texts = [example["text"] for example in ds]+ enc_inputs, dec_inputs, dec_targets = prepare_encoder_decoder_pairs(+ texts, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len+ )+ print(f"Prepared {len(enc_inputs)} training pairs")++ config = TransformerConfig(+ d_model=args.d_model,+ num_heads=args.num_heads,+ num_encoder_layers=args.num_layers,+ num_decoder_layers=args.num_layers,+ d_ff=args.d_model * 4,+ max_seq_len=max(args.max_enc_len, args.max_dec_len),+ dropout_rate=args.dropout,+ )++ np.random.seed(args.seed)+ rng = jax.random.PRNGKey(args.seed)+ rng, init_rng = jax.random.split(rng)++ num_batches = len(enc_inputs) // args.batch_size+ total_steps = num_batches * args.epochs+ warmup_steps = max(1, int(total_steps * args.warmup_ratio))++ muon_lr = getattr(args, "muon_lr", 0.02)+ state = create_train_state(init_rng, config, args.lr, muon_lr, total_steps, warmup_steps)++ param_count = sum(x.size for x in jax.tree.leaves(state.params))+ print(f"Model parameters: {param_count:,}")+ print(f"LR schedule: warmup {warmup_steps} steps, cosine decay over {total_steps} steps")++ os.makedirs(args.checkpoint_dir, exist_ok=True)+ global_step = 0+ causal_mask = make_causal_mask(args.max_dec_len)++ adam_schedule = optax.warmup_cosine_decay_schedule(+ init_value=0.0, peak_value=args.lr,+ warmup_steps=warmup_steps, decay_steps=total_steps, end_value=args.lr * 0.1,+ )+ muon_schedule = optax.warmup_cosine_decay_schedule(+ init_value=0.0, peak_value=muon_lr,+ warmup_steps=warmup_steps, decay_steps=total_steps, end_value=muon_lr * 0.1,+ )+ tokens_per_batch = args.batch_size * (args.max_enc_len + args.max_dec_len)++ for epoch in range(args.epochs):+ losses = []+ batches = get_batches(enc_inputs, dec_inputs, dec_targets, args.batch_size)+ pbar = tqdm(batches, total=num_batches, desc=f"Epoch {epoch + 1}/{args.epochs}")++ for src, tgt_in, tgt_out in pbar:+ rng, dropout_rng = jax.random.split(rng)+ t0 = time.perf_counter()+ state, loss, grad_norm = train_step(+ state,+ jnp.array(src),+ jnp.array(tgt_in),+ jnp.array(tgt_out),+ causal_mask,+ dropout_rng,+ )+ loss_val = float(loss)+ losses.append(loss_val)+ global_step += 1+ dt = time.perf_counter() - t0+ pbar.set_postfix(loss=f"{loss_val:.4f}")++ if use_wandb:+ wandb.log({+ "train/loss": loss_val,+ "train/grad_norm": float(grad_norm),+ "train/adam_lr": float(adam_schedule(global_step)),+ "train/muon_lr": float(muon_schedule(global_step)),+ "train/tokens_per_sec": tokens_per_batch / dt,+ "train/step": global_step,+ })++ epoch_avg_loss = sum(losses) / len(losses) if losses else float("nan")+ final_loss = losses[-1] if losses else float("nan")+ print(f"Epoch {epoch + 1}/{args.epochs} — avg loss: {epoch_avg_loss:.4f}, final loss: {final_loss:.4f}")++ if use_wandb:+ wandb.log({+ "epoch/avg_loss": epoch_avg_loss,+ "epoch/final_loss": final_loss,+ "epoch": epoch + 1,+ })++ ckpt_name = f"needle_{args.num_layers}_{args.d_model}_{global_step}.pkl"+ ckpt_path = os.path.join(args.checkpoint_dir, ckpt_name)+ params_np = jax.tree.map(np.array, state.params)+ with open(ckpt_path, "wb") as f:+ pickle.dump({"params": params_np, "config": config.__dict__}, f)+ print(f"Saved checkpoint: {ckpt_path}")++ if use_wandb:+ wandb.finish()+ print("Training complete.")+++def sweep(args):+ import wandb+ import yaml++ with open(args.sweep_config) as f:+ sweep_config = yaml.safe_load(f)++ def sweep_train():+ wandb.init(project=args.project)+ for key, value in dict(wandb.config).items():+ if hasattr(args, key):+ setattr(args, key, value)+ args.wandb = True+ train(args)+ wandb.finish()++ sweep_id = wandb.sweep(sweep_config, project=args.project)+ wandb.agent(sweep_id, function=sweep_train, count=args.count)+++def parse_args():+ parser = argparse.ArgumentParser(description="Train encoder-decoder transformer on TinyStories")+ parser.add_argument("--epochs", type=int, default=3)+ parser.add_argument("--batch-size", type=int, default=32)+ parser.add_argument("--lr", type=float, default=3e-4)+ parser.add_argument("--muon-lr", type=float, default=0.02)+ parser.add_argument("--d-model", type=int, default=128)+ parser.add_argument("--num-heads", type=int, default=4)+ parser.add_argument("--num-layers", type=int, default=2)+ parser.add_argument("--dropout", type=float, default=0.1)+ parser.add_argument("--max-enc-len", type=int, default=128)+ parser.add_argument("--max-dec-len", type=int, default=128)+ parser.add_argument("--max-samples", type=int, default=10000)+ parser.add_argument("--log-every", type=int, default=50)+ parser.add_argument("--warmup-ratio", type=float, default=0.05)+ parser.add_argument("--checkpoint-dir", type=str, default="checkpoints")+ parser.add_argument("--seed", type=int, default=42)+ return parser.parse_args()+++if __name__ == "__main__":+ train(parse_args())diff --git a/sweep.yaml b/sweep.yamlnew file mode 100644index 0000000..6bd0806--- /dev/null+++ b/sweep.yaml@@ -0,0 +1,27 @@+method: bayes+metric:+ name: epoch/avg_loss+ goal: minimize+parameters:+ lr:+ distribution: log_uniform_values+ min: 1.0e-5+ max: 1.0e-2+ muon_lr:+ distribution: log_uniform_values+ min: 5.0e-3+ max: 0.1+ warmup_ratio:+ distribution: uniform+ min: 0.01+ max: 0.15+ dropout:+ values: [0.0, 0.05, 0.1, 0.15, 0.2]+ batch_size:+ values: [16, 32, 64]+ d_model:+ values: [64, 128, 256]+ num_layers:+ values: [2, 3, 4]+ num_heads:+ values: [2, 4, 8]