Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Optimise and stabilise train
HenryNdubuaku711811225eeb
diff --git a/.gitignore b/.gitignoreindex 7c100f5..9c716c0 100644--- a/.gitignore+++ b/.gitignore@@ -87,3 +87,4 @@ marimo/_lsp/__marimo__/checkpoints/wandb/+.data_cache/diff --git a/src/cli.py b/src/cli.pyindex a13f53d..1d8d1fb 100644--- a/src/cli.py+++ b/src/cli.py@@ -67,6 +67,47 @@ HELP = """└───────────────────────────────────────────────────────────────────┘"""+TOY_CONFIG = {+ "epochs": 1,+ "batch_size": 32,+ "lr": 3e-4,+ "muon_lr": 0.02,+ "d_model": 128,+ "num_heads": 4,+ "num_layers": 2,+ "dropout": 0.1,+ "max_enc_len": 128,+ "max_dec_len": 128,+ "max_samples": 10000,+ "warmup_ratio": 0.05,+ "dtype": "float32",+ "seed": 42,+}++BASE_CONFIG = {+ "epochs": 3,+ "batch_size": 32,+ "lr": 3e-4,+ "muon_lr": 0.02,+ "d_model": 256,+ "num_heads": 4,+ "num_layers": 4,+ "dropout": 0.0,+ "max_enc_len": 256,+ "max_dec_len": 256,+ "max_samples": None,+ "warmup_ratio": 0.05,+ "dtype": "float32",+ "seed": 42,+}+++def _apply_train_defaults(args):+ config = TOY_CONFIG if args.toy else BASE_CONFIG+ for key, value in config.items():+ if getattr(args, key, None) is None:+ setattr(args, key, value)+def main():if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help", "help"):@@ -77,21 +118,23 @@ def main():sub = parser.add_subparsers(dest="command")p = sub.add_parser("train", add_help=False)- p.add_argument("--epochs", type=int, default=1)- p.add_argument("--batch-size", type=int, default=32)- p.add_argument("--lr", type=float, default=3e-4)- p.add_argument("--muon-lr", type=float, default=0.02)- p.add_argument("--d-model", type=int, default=128)- p.add_argument("--num-heads", type=int, default=4)- p.add_argument("--num-layers", type=int, default=2)- p.add_argument("--dropout", type=float, default=0.1)- p.add_argument("--max-enc-len", type=int, default=128)- p.add_argument("--max-dec-len", type=int, default=128)- p.add_argument("--max-samples", type=int, default=20000)- p.add_argument("--warmup-ratio", type=float, default=0.05)+ p.add_argument("--toy", action="store_true")+ p.add_argument("--epochs", type=int, default=None)+ p.add_argument("--batch-size", type=int, default=None)+ p.add_argument("--lr", type=float, default=None)+ p.add_argument("--muon-lr", type=float, default=None)+ p.add_argument("--d-model", type=int, default=None)+ p.add_argument("--num-heads", type=int, default=None)+ p.add_argument("--num-layers", type=int, default=None)+ p.add_argument("--dropout", type=float, default=None)+ p.add_argument("--max-enc-len", type=int, default=None)+ p.add_argument("--max-dec-len", type=int, default=None)+ p.add_argument("--max-samples", type=int, default=None)+ p.add_argument("--warmup-ratio", type=float, default=None)p.add_argument("--wandb", action="store_true")+ p.add_argument("--dtype", type=str, default=None, choices=["float32", "bfloat16"])p.add_argument("--checkpoint-dir", type=str, default="checkpoints")- p.add_argument("--seed", type=int, default=42)+ p.add_argument("--seed", type=int, default=None)p = sub.add_parser("run", add_help=False)p.add_argument("--checkpoint", type=str, required=True)@@ -126,6 +169,7 @@ def main():p.add_argument("--max-dec-len", type=int, default=128)p.add_argument("--max-samples", type=int, default=20000)p.add_argument("--warmup-ratio", type=float, default=0.05)+ p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])p.add_argument("--checkpoint-dir", type=str, default="checkpoints")p.add_argument("--seed", type=int, default=42)@@ -173,6 +217,7 @@ def main():sys.exit(0)if args.command == "train":+ _apply_train_defaults(args)from .train import traintrain(args)elif args.command == "sweep":diff --git a/src/data.py b/src/data.pyindex 0adbda5..59c149b 100644--- a/src/data.py+++ b/src/data.py@@ -1,3 +1,4 @@+import hashlibimport osos.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1"@@ -6,8 +7,11 @@ logging.getLogger("transformers").setLevel(logging.ERROR)import numpy as npfrom datasets import load_dataset+from tqdm import tqdmfrom transformers import AutoTokenizer+CACHE_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), ".data_cache")+def get_tokenizer():tokenizer = AutoTokenizer.from_pretrained("gpt2")@@ -22,47 +26,80 @@ def load_tinystories(split="train", max_samples=None):return ds-def prepare_encoder_decoder_pairs(texts, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):- enc_inputs = []- dec_inputs = []- dec_targets = []+def _cache_key(n_samples, max_enc_len, max_dec_len):+ key = f"tinystories_{n_samples}_{max_enc_len}_{max_dec_len}"+ return hashlib.md5(key.encode()).hexdigest()[:12]+++def prepare_encoder_decoder_pairs(ds, tokenizer, max_enc_len=128, max_dec_len=128, split_ratio=0.3):++ cache_id = _cache_key(len(ds), max_enc_len, max_dec_len)+ cache_path = os.path.join(CACHE_DIR, cache_id)+ if os.path.exists(cache_path + "_enc.npy"):+ print(f"Loading cached data ({cache_id})...")+ return (+ np.load(cache_path + "_enc.npy"),+ np.load(cache_path + "_dec_in.npy"),+ np.load(cache_path + "_dec_tgt.npy"),+ )pad_id = tokenizer.pad_token_idbos_id = tokenizer.eos_token_id-- for text in texts:- tokens = tokenizer.encode(text, truncation=True, max_length=max_enc_len + max_dec_len)+ max_total = max_enc_len + max_dec_len++ print("Tokenizing...")+ texts = list(ds["text"])+ all_tokens = tokenizer(+ texts, truncation=True, max_length=max_total,+ return_attention_mask=False, return_token_type_ids=False,+ )["input_ids"]++ n = len(all_tokens)+ enc_inputs = np.full((n, max_enc_len), pad_id, dtype=np.int32)+ dec_inputs = np.full((n, max_dec_len), pad_id, dtype=np.int32)+ dec_targets = np.full((n, max_dec_len), pad_id, dtype=np.int32)++ kept = 0+ for tokens in tqdm(all_tokens, desc="Building pairs"):if len(tokens) < 6:continuesplit_point = max(2, int(len(tokens) * split_ratio))-enc_tokens = tokens[:split_point][:max_enc_len]dec_tokens = tokens[split_point:][:max_dec_len]if len(dec_tokens) < 2:continue- dec_in = [bos_id] + dec_tokens[:-1]- dec_tgt = dec_tokens+ enc_len = len(enc_tokens)+ dec_len = len(dec_tokens)+ enc_inputs[kept, :enc_len] = enc_tokens+ dec_inputs[kept, 0] = bos_id+ dec_inputs[kept, 1:dec_len] = dec_tokens[:dec_len - 1]+ dec_targets[kept, :dec_len] = dec_tokens+ kept += 1- enc_padded = enc_tokens + [pad_id] * (max_enc_len - len(enc_tokens))- dec_in_padded = dec_in + [pad_id] * (max_dec_len - len(dec_in))- dec_tgt_padded = dec_tgt + [pad_id] * (max_dec_len - len(dec_tgt))+ enc_inputs = enc_inputs[:kept]+ dec_inputs = dec_inputs[:kept]+ dec_targets = dec_targets[:kept]- enc_inputs.append(enc_padded)- dec_inputs.append(dec_in_padded)- dec_targets.append(dec_tgt_padded)+ # Save cache+ os.makedirs(CACHE_DIR, exist_ok=True)+ np.save(cache_path + "_enc.npy", enc_inputs)+ np.save(cache_path + "_dec_in.npy", dec_inputs)+ np.save(cache_path + "_dec_tgt.npy", dec_targets)+ print(f"Cached data to {CACHE_DIR}/{cache_id}")- return np.array(enc_inputs), np.array(dec_inputs), np.array(dec_targets)+ return enc_inputs, dec_inputs, dec_targetsdef get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True):n = len(enc_inputs)- indices = np.arange(n)if shuffle:- np.random.shuffle(indices)+ perm = np.random.permutation(n)+ enc_inputs = enc_inputs[perm]+ dec_inputs = dec_inputs[perm]+ dec_targets = dec_targets[perm]for i in range(0, n - batch_size + 1, batch_size):- batch_idx = indices[i : i + batch_size]- yield enc_inputs[batch_idx], dec_inputs[batch_idx], dec_targets[batch_idx]+ yield enc_inputs[i : i + batch_size], dec_inputs[i : i + batch_size], dec_targets[i : i + batch_size]diff --git a/src/train.py b/src/train.pyindex 0623c44..ab2f0cf 100644--- a/src/train.py+++ b/src/train.py@@ -23,6 +23,8 @@ from .model import (def _newton_schulz(G, steps=5):"""Approximate polar decomposition via Newton-Schulz iteration."""a, b, c = 3.4445, -4.7750, 2.0315+ orig_dtype = G.dtype+ G = G.astype(jnp.float32)X = G / (jnp.linalg.norm(G) + 1e-7)transposed = G.shape[0] > G.shape[1]if transposed:@@ -33,7 +35,7 @@ def _newton_schulz(G, steps=5):X = a * X + B @ Xif transposed:X = X.T- return X+ return X.astype(orig_dtype)class MuonState(NamedTuple):@@ -147,7 +149,7 @@ def _train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):rngs={"dropout": dropout_rng},)- loss = optax.softmax_cross_entropy_with_integer_labels(logits, tgt_out)+ loss = optax.softmax_cross_entropy_with_integer_labels(logits.astype(jnp.float32), tgt_out)mask = (tgt_out != pad_id).astype(jnp.float32)return jnp.sum(loss * mask) / jnp.maximum(jnp.sum(mask), 1.0)@@ -160,7 +162,7 @@ def _train_step(state, src, tgt_in, tgt_out, causal_mask, dropout_rng):return state, loss, grad_norm-p_train_step = jax.pmap(_train_step, axis_name="batch")+p_train_step = jax.pmap(_train_step, axis_name="batch", donate_argnums=(0,))def shard_batch(batch, num_devices):@@ -185,9 +187,8 @@ def train(args):ds = load_tinystories("train", max_samples=args.max_samples)print("Preparing encoder-decoder pairs...")- texts = [example["text"] for example in ds]enc_inputs, dec_inputs, dec_targets = prepare_encoder_decoder_pairs(- texts, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len+ ds, tokenizer, max_enc_len=args.max_enc_len, max_dec_len=args.max_dec_len)print(f"Prepared {len(enc_inputs)} training pairs")@@ -202,6 +203,7 @@ def train(args):d_ff=args.d_model * 4,max_seq_len=max(args.max_enc_len, args.max_dec_len),dropout_rate=args.dropout,+ dtype=args.dtype,)np.random.seed(args.seed)@@ -212,8 +214,12 @@ def train(args):total_steps = num_batches * args.epochswarmup_steps = max(1, int(total_steps * args.warmup_ratio))- muon_lr = getattr(args, "muon_lr", 0.02)- state = create_train_state(init_rng, config, args.lr, muon_lr, total_steps, warmup_steps)+ # Scale LRs for data parallelism: linear for Adam, sqrt for Muon+ # (Muon's Newton-Schulz orthogonalization is sensitive to large LRs).+ import math+ scaled_lr = args.lr * num_devices+ muon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(num_devices)+ state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps)# Replicate state across all devicesstate = jax_utils.replicate(state)@@ -221,6 +227,7 @@ def train(args):param_count = sum(x.size for x in jax.tree.leaves(jax_utils.unreplicate(state).params))print(f"Model parameters: {param_count:,}")print(f"Devices: {num_devices}, per-device batch: {args.batch_size}, effective batch: {effective_batch_size}")+ print(f"Adam LR: {args.lr} x {num_devices} = {scaled_lr}, Muon LR: {args.muon_lr} x sqrt({num_devices}) = {muon_lr:.4f}")print(f"LR schedule: warmup {warmup_steps} steps, cosine decay over {total_steps} steps")os.makedirs(args.checkpoint_dir, exist_ok=True)@@ -231,8 +238,8 @@ def train(args):)adam_schedule = optax.warmup_cosine_decay_schedule(- init_value=0.0, peak_value=args.lr,- warmup_steps=warmup_steps, decay_steps=total_steps, end_value=args.lr * 0.1,+ init_value=0.0, peak_value=scaled_lr,+ warmup_steps=warmup_steps, decay_steps=total_steps, end_value=scaled_lr * 0.1,)muon_schedule = optax.warmup_cosine_decay_schedule(init_value=0.0, peak_value=muon_lr,@@ -240,6 +247,10 @@ def train(args):)tokens_per_batch = effective_batch_size * (args.max_enc_len + args.max_dec_len)+ enc_inputs = jnp.array(enc_inputs)+ dec_inputs = jnp.array(dec_inputs)+ dec_targets = jnp.array(dec_targets)+for epoch in range(args.epochs):losses = []batches = get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size)@@ -254,9 +265,9 @@ def train(args):# Shard batch across devices: (num_devices, per_device_batch, seq_len)state, loss, grad_norm = p_train_step(state,- shard_batch(jnp.array(src), num_devices),- shard_batch(jnp.array(tgt_in), num_devices),- shard_batch(jnp.array(tgt_out), num_devices),+ shard_batch(src, num_devices),+ shard_batch(tgt_in, num_devices),+ shard_batch(tgt_out, num_devices),causal_mask,dropout_rngs,)