Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Matryoshka export
HenryNdubuakufd873c30dd0c
diff --git a/src/cli.py b/src/cli.pyindex b8b2255..160bbe8 100644--- a/src/cli.py+++ b/src/cli.py@@ -4,10 +4,10 @@ import sysHELP = """┌───────────────────────────────────────────────────────────────────┐│ │- │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │- │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │- │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │- │ ...the tiny model to rule them all... │+ │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │+ │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │+ │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │+ │ ...the tiny model to rule them all... ││ ││ train ││ --toy Use toy config for quick iteration │@@ -52,6 +52,11 @@ HELP = """│ --temperature FLOAT Sampling temperature (default: 0.8) ││ --throughput-runs INT Throughput runs (default: 10) ││ │+ │ export │+ │ --checkpoint PATH Path to model checkpoint (required) │+ │ --dim INT Target d_model dimension (required) │+ │ --output PATH Output path (default: auto-named) │+ │ ││ evaluate ││ --checkpoint PATH Path to model checkpoint (required) ││ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │@@ -82,9 +87,11 @@ TOY_CONFIG = {}MAIN_CONFIG = {- "num_heads": 4,- "num_layers": 24,- "num_dec_layers": 4,+ "num_heads": 16,+ "num_kv_heads": 8,+ "num_layers": 28,+ "num_dec_layers": 8,+ "d_ff": 4096,"max_enc_len": 256,"max_dec_len": 256,"max_samples": None,@@ -117,8 +124,10 @@ def main():p.add_argument("--muon-lr", type=float, default=0.02)p.add_argument("--d-model", type=int, default=None)p.add_argument("--num-heads", type=int, default=None)+ p.add_argument("--num-kv-heads", type=int, default=None)p.add_argument("--num-layers", type=int, default=None)p.add_argument("--num-dec-layers", type=int, default=None)+ p.add_argument("--d-ff", type=int, default=None)p.add_argument("--max-enc-len", type=int, default=None)p.add_argument("--max-dec-len", type=int, default=None)p.add_argument("--max-samples", type=int, default=None)@@ -160,6 +169,11 @@ def main():p.add_argument("--temperature", type=float, default=0.8)p.add_argument("--throughput-runs", type=int, default=10)+ p = sub.add_parser("export", add_help=False)+ p.add_argument("--checkpoint", type=str, required=True)+ p.add_argument("--dim", type=int, required=True)+ p.add_argument("--output", type=str, default=None)+p = sub.add_parser("evaluate", add_help=False)p.add_argument("--checkpoint", type=str, required=True)p.add_argument("--benchmarks", type=str, nargs="*",@@ -213,6 +227,9 @@ def main():elif args.command == "test":from .test import main as test_maintest_main(args)+ elif args.command == "export":+ from .export import main as export_main+ export_main(args)elif args.command == "evaluate":from .evaluate import main as eval_maineval_main(args)diff --git a/src/export.py b/src/export.pynew file mode 100644index 0000000..d088c7d--- /dev/null+++ b/src/export.py@@ -0,0 +1,118 @@+"""Export MRL sub-models by slicing a full checkpoint to a target dimension."""++import os+import pickle+from dataclasses import replace+from pathlib import Path++import jax+import jax.numpy as jnp+import numpy as np++from .model import TransformerConfig+++def export_submodel(checkpoint_path, d_prime, output_path):+ """Slice a full MRL checkpoint to a self-contained sub-model at dimension d_prime."""++ # Load checkpoint+ with open(checkpoint_path, "rb") as f:+ data = pickle.load(f)+ params = data["params"]+ config = TransformerConfig(**data["config"])++ head_dim = config.d_model // config.num_heads+ if d_prime % head_dim != 0:+ raise ValueError(+ f"d_prime={d_prime} must be divisible by head_dim={head_dim} "+ f"(d_model={config.d_model} // num_heads={config.num_heads})"+ )++ # Derived dims+ n_h = d_prime // head_dim+ n_kv = min(config.num_kv_heads, n_h)+ d_ff_prime = config.d_ff * d_prime // config.d_model+ kv_dim = config.num_kv_heads * head_dim+ kv_dim_prime = n_kv * head_dim++ d_model = config.d_model+ d_ff = config.d_ff++ def slice_leaf(key_path, leaf):+ arr = np.asarray(leaf)++ if arr.ndim == 1:+ # Norm scales: shape (d_model,)+ return arr[:d_prime]++ if arr.ndim == 3:+ # Memory slots: shape (1, M, d_model)+ return arr[:, :, :d_prime]++ if arr.ndim == 2:+ rows, cols = arr.shape+ # Determine slice for each axis based on original size+ def _target(size):+ if size == d_model:+ return d_prime+ if size == d_ff:+ return d_ff_prime+ if size == kv_dim:+ return kv_dim_prime+ return size # vocab, num_memory_slots, etc.++ r = _target(rows)+ c = _target(cols)+ return arr[:r, :c]++ return arr++ sliced = jax.tree_util.tree_map_with_path(slice_leaf, params)++ # Build new config+ new_config = replace(+ config,+ d_model=d_prime,+ d_ff=d_ff_prime,+ num_heads=n_h,+ num_kv_heads=n_kv,+ )++ # Convert any JAX arrays to numpy for pickling+ sliced_np = jax.tree.map(+ lambda x: np.asarray(x) if isinstance(x, jnp.ndarray) else x, sliced+ )++ os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)+ with open(output_path, "wb") as f:+ pickle.dump({"params": sliced_np, "config": new_config.__dict__}, f)++ # Print summary+ orig_count = sum(x.size for x in jax.tree.leaves(params))+ new_count = sum(x.size for x in jax.tree.leaves(sliced_np))+ orig_bytes = sum(x.nbytes for x in jax.tree.leaves(params))+ new_bytes = sum(x.nbytes for x in jax.tree.leaves(sliced_np))++ print(f"\n Export complete: {output_path}")+ print(f" ─────────────────────────────────────")+ print(f" {'':>20s} {'Original':>12s} {'Exported':>12s}")+ print(f" {'d_model':>20s} {config.d_model:>12d} {d_prime:>12d}")+ print(f" {'d_ff':>20s} {config.d_ff:>12d} {d_ff_prime:>12d}")+ print(f" {'num_heads':>20s} {config.num_heads:>12d} {n_h:>12d}")+ print(f" {'num_kv_heads':>20s} {config.num_kv_heads:>12d} {n_kv:>12d}")+ print(f" {'params':>20s} {orig_count:>12,d} {new_count:>12,d}")+ print(f" {'size (MB)':>20s} {orig_bytes / 1e6:>12.1f} {new_bytes / 1e6:>12.1f}")+ print()+++def main(args):+ checkpoint = args.checkpoint+ d_prime = args.dim+ output = args.output++ if output is None:+ stem = Path(checkpoint).stem+ parent = Path(checkpoint).parent+ output = str(parent / f"{stem}_d{d_prime}.pkl")++ export_submodel(checkpoint, d_prime, output)diff --git a/src/train.py b/src/train.pyindex 7d4112c..668203e 100644--- a/src/train.py+++ b/src/train.py@@ -426,23 +426,31 @@ def _make_mrl_val_loss_fn(apply_fn, mrl_dim):def _estimate_mrl_params(config, d_prime):- """Estimate parameter count if model were deployed at dimension d_prime."""+ """Estimate parameter count of an exported sub-model at dimension d_prime.++ Slicing rules (matching what an actual MRL export would produce):+ - Embedding: [vocab, d_prime] (first d_prime cols)+ - Heads: same head_dim, fewer heads → n_h = d_prime // head_dim+ - KV heads: min(original, n_h)+ - FFN: proportional → d_ff' = d_ff * d_prime // d_model+ - Memory: slots unchanged, but projections shrink with d / d_ff+ """d = d_primev = config.vocab_sizen_enc = config.num_encoder_layersn_dec = config.num_decoder_layers- n_h = config.num_heads- n_kv = config.num_kv_heads- d_ff = d * 4+ head_dim = config.d_model // config.num_heads+ n_h = max(1, d // head_dim)+ n_kv = min(config.num_kv_heads, n_h)+ d_ff = config.d_ff * d // config.d_modelm = config.num_memory_slotsemb = v * d- head_dim = d // n_hkv_dim = n_kv * head_dim- attn = d * d + d * kv_dim * 2 + d * d- ffn = d * d_ff * 3- mixer_token = d * d_ff * 2 + d_ff * m- mixer_channel = d * d_ff * 3+ attn = d * d + d * kv_dim * 2 + d * d+ ffn = d * d_ff * 3+ mixer_token = d * d_ff * 2 + d_ff * m+ mixer_channel = d * d_ff * 3enc_block = attn + ffn + mixer_token + mixer_channeldec_block = attn * 2 + ffntotal = emb + n_enc * enc_block + n_dec * dec_block@@ -499,9 +507,10 @@ def train(args):config = TransformerConfig(d_model=args.d_model,num_heads=args.num_heads,+ num_kv_heads=getattr(args, "num_kv_heads", None) or args.num_heads,num_encoder_layers=args.num_layers,num_decoder_layers=getattr(args, "num_dec_layers", args.num_layers),- d_ff=args.d_model * 4,+ d_ff=getattr(args, "d_ff", None) or args.d_model * 4,max_seq_len=max(args.max_enc_len, args.max_dec_len),dtype=args.dtype,activation=getattr(args, "activation", "drelu"),@@ -783,12 +792,14 @@ def train(args):print(f" Sparsity {sparsity:>11.2f}% ({near_zero:,}/{total_params:,})")if mrl_results:print(f" ─────────────────────────────────────")- print(f" MRL dimension pruning:")- print(f" {'dim':>6} {'val ppl':>10} {'est. params':>12}")- print(f" {config.d_model:>6} {last_val_ppl:>10.2f} {total_params:>12,} (full)")+ print(f" MRL exportable sub-models:")+ print(f" {'dim':>6} {'val ppl':>10} {'params':>12} {'heads':>5}")+ head_dim = config.d_model // config.num_heads+ print(f" {config.d_model:>6} {last_val_ppl:>10.2f} {total_params:>12,} {config.num_heads:>5} (full)")for d_prime in sorted(mrl_results.keys(), reverse=True):mrl_ppl, mrl_params = mrl_results[d_prime]- print(f" {d_prime:>6} {mrl_ppl:>10.2f} {mrl_params:>12,}")+ n_h = max(1, d_prime // head_dim)+ print(f" {d_prime:>6} {mrl_ppl:>10.2f} {mrl_params:>12,} {n_h:>5}")print(f" ─────────────────────────────────────")print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")print(f" Latency {tp['avg_latency_s']:>11.3f}s")