Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Optimise and stabilise train

HenryNdubuaku711811225eeb
diff --git a/.gitignore b/.gitignore
index 7c100f5..9c716c0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -87,3 +87,4 @@ marimo/_lsp/
__marimo__/
checkpoints/
wandb/
+.data_cache/
diff --git a/src/cli.py b/src/cli.py
index a13f53d..1d8d1fb 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -67,6 +67,47 @@ HELP = """
└───────────────────────────────────────────────────────────────────┘
"""
+TOY_CONFIG = {
+ "epochs": 1,
+ "batch_size": 32,
+ "lr": 3e-4,
+ "muon_lr": 0.02,
+ "d_model": 128,
+ "num_heads": 4,
+ "num_layers": 2,
+ "dropout": 0.1,
+ "max_enc_len": 128,
+ "max_dec_len": 128,
+ "max_samples": 10000,
+ "warmup_ratio": 0.05,
+ "dtype": "float32",
+ "seed": 42,
+}
+
+BASE_CONFIG = {
+ "epochs": 3,
+ "batch_size": 32,
+ "lr": 3e-4,
+ "muon_lr": 0.02,
+ "d_model": 256,
+ "num_heads": 4,
+ "num_layers": 4,
+ "dropout": 0.0,
+ "max_enc_len": 256,
+ "max_dec_len": 256,
+ "max_samples": None,
+ "warmup_ratio": 0.05,
+ "dtype": "float32",
+ "seed": 42,
+}
+
+
+def _apply_train_defaults(args):
+ config = TOY_CONFIG if args.toy else BASE_CONFIG
+ for key, value in config.items():
+ if getattr(args, key, None) is None:
+ setattr(args, key, value)
+
def main():
if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help", "help"):
@@ -77,21 +118,23 @@ def main():
sub = parser.add_subparsers(dest="command")
p = sub.add_parser("train", add_help=False)
- p.add_argument("--epochs", type=int, default=1)
- p.add_argument("--batch-size", type=int, default=32)
- p.add_argument("--lr", type=float, default=3e-4)
- p.add_argument("--muon-lr", type=float, default=0.02)
- p.add_argument("--d-model", type=int, default=128)
- p.add_argument("--num-heads", type=int, default=4)
- p.add_argument("--num-layers", type=int, default=2)
- p.add_argument("--dropout", type=float, default=0.1)
- p.add_argument("--max-enc-len", type=int, default=128)
- p.add_argument("--max-dec-len", type=int, default=128)
- p.add_argument("--max-samples", type=int, default=20000)
- p.add_argument("--warmup-ratio", type=float, default=0.05)
+ p.add_argument("--toy", action="store_true")
+ p.add_argument("--epochs", type=int, default=None)
+ p.add_argument("--batch-size", type=int, default=None)
+ p.add_argument("--lr", type=float, default=None)
+ p.add_argument("--muon-lr", type=float, default=None)
+ p.add_argument("--d-model", type=int, default=None)
+ p.add_argument("--num-heads", type=int, default=None)
+ p.add_argument("--num-layers", type=int, default=None)
+ p.add_argument("--dropout", type=float, default=None)
+ p.add_argument("--max-enc-len", type=int, default=None)
+ p.add_argument("--max-dec-len", type=int, default=None)
+ p.add_argument("--max-samples", type=int, default=None)
+ p.add_argument("--warmup-ratio", type=float, default=None)
p.add_argument("--wandb", action="store_true")
+ p.add_argument("--dtype", type=str, default=None, choices=["float32", "bfloat16"])
p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
- p.add_argument("--seed", type=int, default=42)
+ p.add_argument("--seed", type=int, default=None)
p = sub.add_parser("run", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
@@ -126,6 +169,7 @@ def main():
p.add_argument("--max-dec-len", type=int, default=128)
p.add_argument("--max-samples", type=int, default=20000)
p.add_argument("--warmup-ratio", type=float, default=0.05)
+ p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])
p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
p.add_argument("--seed", type=int, default=42)
@@ -173,6 +217,7 @@ def main():
sys.exit(0)
if args.command == "train":
+ _apply_train_defaults(args)
from .train import train
train(args)
elif args.command == "sweep":
diff --git a/src/data.py b/src/data.py
index 0adbda5..59c149b 100644
--- a/src/data.py
+++ b/src/data.py
@@ -1,3 +1,4 @@
+import hashlib
import os
os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"
@@ -6,8 +7,11 @@ logging.getLogger("transformers").setLevel(logging.ERROR)
import numpy as np
from datasets import load_dataset
+from tqdm import tqdm
from transformers import AutoTokenizer
+CACHE_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), ".data_cache")
+
def get_tokenizer():
tokenizer = AutoTokenizer.from_pretrained("gpt2")
@@ -22,47 +26,80 @@ def load_tinystories(split="train", max_samples=None):
return ds
-def prepare_encoder_decoder_pairs(texts, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):
- enc_inputs = []
- dec_inputs = []
- dec_targets = []
+def _cache_key(n_samples, max_enc_len, max_dec_len):
+ key = f"tinystories_{n_samples}_{max_enc_len}_{max_dec_len}"
+ return hashlib.md5(key.encode()).hexdigest()[:12]
+
+
+def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):
+
+ cache_id = _cache_key(len(ds), max_enc_len, max_dec_len)
+ cache_path = os.path.join(CACHE_DIR, cache_id)
+ if os.path.exists(cache_path + "_enc.npy"):
+ print(f"Loading cached data ({cache_id})...")
+ return (
+ np.load(cache_path + "_enc.npy"),
+ np.load(cache_path + "_dec_in.npy"),
+ np.load(cache_path + "_dec_tgt.npy"),
+ )
pad_id = tokenizer.pad_token_id
bos_id = tokenizer.eos_token_id
-
- for text in texts:
- tokens = tokenizer.encode(text, truncation=True, max_length=max_enc_len + max_dec_len)
+ max_total = max_enc_len + max_dec_len
+
+ print("Tokenizing...")
+ texts = list(ds["text"])
+ all_tokens = tokenizer(
+ texts, truncation=True, max_length=max_total,
+ return_attention_mask=False, return_token_type_ids=False,
+ )["input_ids"]
+
+ n = len(all_tokens)
+ enc_inputs = np.full((n, max_enc_len), pad_id, dtype=np.int32)
+ dec_inputs = np.full((n, max_dec_len), pad_id, dtype=np.int32)
+ dec_targets = np.full((n, max_dec_len), pad_id, dtype=np.int32)
+
+ kept = 0
+ for tokens in tqdm(all_tokens, desc="Building pairs"):
if len(tokens) < 6:
continue
split_point = max(2, int(len(tokens) * split_ratio))
-
enc_tokens = tokens[:split_point][:max_enc_len]
dec_tokens = tokens[split_point:][:max_dec_len]
if len(dec_tokens) < 2:
continue
- dec_in = [bos_id] + dec_tokens[:-1]
- dec_tgt = dec_tokens
+ enc_len = len(enc_tokens)
+ dec_len = len(dec_tokens)
+ enc_inputs[kept, :enc_len] = enc_tokens
+ dec_inputs[kept, 0] = bos_id
+ dec_inputs[kept, 1:dec_len] = dec_tokens[:dec_len - 1]
+ dec_targets[kept, :dec_len] = dec_tokens
+ kept += 1
- enc_padded = enc_tokens + [pad_id] * (max_enc_len - len(enc_tokens))
- dec_in_padded = dec_in + [pad_id] * (max_dec_len - len(dec_in))
- dec_tgt_padded = dec_tgt + [pad_id] * (max_dec_len - len(dec_tgt))
+ enc_inputs = enc_inputs[:kept]
+ dec_inputs = dec_inputs[:kept]
+ dec_targets = dec_targets[:kept]
- enc_inputs.append(enc_padded)
- dec_inputs.append(dec_in_padded)
- dec_targets.append(dec_tgt_padded)
+ # Save cache
+ os.makedirs(CACHE_DIR, exist_ok=True)
+ np.save(cache_path + "_enc.npy", enc_inputs)
+ np.save(cache_path + "_dec_in.npy", dec_inputs)
+ np.save(cache_path + "_dec_tgt.npy", dec_targets)
+ print(f"Cached data to {CACHE_DIR}/{cache_id}")
- return np.array(enc_inputs), np.array(dec_inputs), np.array(dec_targets)
+ return enc_inputs, dec_inputs, dec_targets
def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True):
n = len(enc_inputs)
- indices = np.arange(n)
if shuffle:
- np.random.shuffle(indices)
+ perm = np.random.permutation(n)
+ enc_inputs = enc_inputs[perm]
+ dec_inputs = dec_inputs[perm]
+ dec_targets = dec_targets[perm]
for i in range(0, n - batch_size + 1, batch_size):
- batch_idx = indices[i : i + batch_size]
- yield enc_inputs[batch_idx], dec_inputs[batch_idx], dec_targets[batch_idx]
+ yield enc_inputs[i : i + batch_size], dec_inputs[i : i + batch_size], dec_targets[i : i + batch_size]
diff --git a/src/train.py b/src/train.py
index 0623c44..ab2f0cf 100644
--- a/src/train.py
+++ b/src/train.py
@@ -23,6 +23,8 @@ from .model import (
def _newton_schulz(G, steps=5):
"""Approximate polar decomposition via Newton-Schulz iteration."""
a, b, c = 3.4445, -4.7750, 2.0315
+ orig_dtype = G.dtype
+ G = G.astype(jnp.float32)
X = G / (jnp.linalg.norm(G) + 1e-7)
transposed = G.shape[0] > G.shape[1]
if transposed:
@@ -33,7 +35,7 @@ def _newton_schulz(G, steps=5):
X = a * X + B @ X
if transposed:
X = X.T
- return X
+ return X.astype(orig_dtype)
class MuonState(NamedTuple):
@@ -147,7 +149,7 @@ def _train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):
rngs={"dropout": dropout_rng},
)
- loss = optax.softmax_cross_entropy_with_integer_labels(logits, tgt_out)
+ loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)
mask = (tgt_out != pad_id).astype(jnp.float32)
return jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)
@@ -160,7 +162,7 @@ def _train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):
return state, loss, grad_norm
-p_train_step = jax.pmap(_train_step, axis_name="batch")
+p_train_step = jax.pmap(_train_step, axis_name="batch", donate_argnums=(0,))
def shard_batch(batch, num_devices):
@@ -185,9 +187,8 @@ def train(args):
ds = load_tinystories("train", max_samples=args.max_samples)
print("Preparing encoder-decoder pairs...")
- texts = [example["text"] for example in ds]
enc_inputs, dec_inputs, dec_targets = prepare_encoder_decoder_pairs(
- texts, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
+ ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len
)
print(f"Prepared {len(enc_inputs)} training pairs")
@@ -202,6 +203,7 @@ def train(args):
d_ff=args.d_model * 4,
max_seq_len=max(args.max_enc_len, args.max_dec_len),
dropout_rate=args.dropout,
+ dtype=args.dtype,
)
np.random.seed(args.seed)
@@ -212,8 +214,12 @@ def train(args):
total_steps = num_batches * args.epochs
warmup_steps = max(1, int(total_steps * args.warmup_ratio))
- muon_lr = getattr(args, "muon_lr", 0.02)
- state = create_train_state(init_rng, config, args.lr, muon_lr, total_steps, warmup_steps)
+ # Scale LRs for data parallelism: linear for Adam, sqrt for Muon
+ # (Muon's Newton-Schulz orthogonalization is sensitive to large LRs).
+ import math
+ scaled_lr = args.lr * num_devices
+ muon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(num_devices)
+ state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps)
# Replicate state across all devices
state = jax_utils.replicate(state)
@@ -221,6 +227,7 @@ def train(args):
param_count = sum(x.size for x in jax.tree.leaves(jax_utils.unreplicate(state).params))
print(f"Model parameters: {param_count:,}")
print(f"Devices: {num_devices}, per-device batch: {args.batch_size}, effective batch: {effective_batch_size}")
+ print(f"Adam LR: {args.lr} x {num_devices} = {scaled_lr}, Muon LR: {args.muon_lr} x sqrt({num_devices}) = {muon_lr:.4f}")
print(f"LR schedule: warmup {warmup_steps} steps, cosine decay over {total_steps} steps")
os.makedirs(args.checkpoint_dir, exist_ok=True)
@@ -231,8 +238,8 @@ def train(args):
)
adam_schedule = optax.warmup_cosine_decay_schedule(
- init_value=0.0, peak_value=args.lr,
- warmup_steps=warmup_steps, decay_steps=total_steps, end_value=args.lr * 0.1,
+ init_value=0.0, peak_value=scaled_lr,
+ warmup_steps=warmup_steps, decay_steps=total_steps, end_value=scaled_lr * 0.1,
)
muon_schedule = optax.warmup_cosine_decay_schedule(
init_value=0.0, peak_value=muon_lr,
@@ -240,6 +247,10 @@ def train(args):
)
tokens_per_batch = effective_batch_size * (args.max_enc_len + args.max_dec_len)
+ enc_inputs = jnp.array(enc_inputs)
+ dec_inputs = jnp.array(dec_inputs)
+ dec_targets = jnp.array(dec_targets)
+
for epoch in range(args.epochs):
losses = []
batches = get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size)
@@ -254,9 +265,9 @@ def train(args):
# Shard batch across devices: (num_devices, per_device_batch, seq_len)
state, loss, grad_norm = p_train_step(
state,
- shard_batch(jnp.array(src), num_devices),
- shard_batch(jnp.array(tgt_in), num_devices),
- shard_batch(jnp.array(tgt_out), num_devices),
+ shard_batch(src, num_devices),
+ shard_batch(tgt_in, num_devices),
+ shard_batch(tgt_out, num_devices),
causal_mask,
dropout_rngs,
)