Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Matryoshka export

HenryNdubuakufd873c30dd0c
diff --git a/src/cli.py b/src/cli.py
index b8b2255..160bbe8 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -4,10 +4,10 @@ import sys
HELP = """
┌───────────────────────────────────────────────────────────────────┐
│ │
- │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │
- │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │
- │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │
- │ ...the tiny model to rule them all... │
+ │ ┌─┐┌─┐┌─┐┌┬┐┬ ┬┌─┐ ┌┐┌┌─┐┌─┐┌┬┐┬ ┌─┐ │
+ │ │ ├─┤│ │ │ │└─┐ │││├┤ ├┤ │││ ├┤ │
+ │ └─┘┴ ┴└─┘ ┴ └─┘└─┘ ┘└┘└─┘└─┘─┴┘┴─┘└─┘ │
+ │ ...the tiny model to rule them all... │
│ │
│ train │
│ --toy Use toy config for quick iteration │
@@ -52,6 +52,11 @@ HELP = """
│ --temperature FLOAT Sampling temperature (default: 0.8) │
│ --throughput-runs INT Throughput runs (default: 10) │
│ │
+ │ export │
+ │ --checkpoint PATH Path to model checkpoint (required) │
+ │ --dim INT Target d_model dimension (required) │
+ │ --output PATH Output path (default: auto-named) │
+ │ │
│ evaluate │
│ --checkpoint PATH Path to model checkpoint (required) │
│ --benchmarks [...] wikitext2 lambada hellaswag arc_easy │
@@ -82,9 +87,11 @@ TOY_CONFIG = {
}
MAIN_CONFIG = {
- "num_heads": 4,
- "num_layers": 24,
- "num_dec_layers": 4,
+ "num_heads": 16,
+ "num_kv_heads": 8,
+ "num_layers": 28,
+ "num_dec_layers": 8,
+ "d_ff": 4096,
"max_enc_len": 256,
"max_dec_len": 256,
"max_samples": None,
@@ -117,8 +124,10 @@ def main():
p.add_argument("--muon-lr", type=float, default=0.02)
p.add_argument("--d-model", type=int, default=None)
p.add_argument("--num-heads", type=int, default=None)
+ p.add_argument("--num-kv-heads", type=int, default=None)
p.add_argument("--num-layers", type=int, default=None)
p.add_argument("--num-dec-layers", type=int, default=None)
+ p.add_argument("--d-ff", type=int, default=None)
p.add_argument("--max-enc-len", type=int, default=None)
p.add_argument("--max-dec-len", type=int, default=None)
p.add_argument("--max-samples", type=int, default=None)
@@ -160,6 +169,11 @@ def main():
p.add_argument("--temperature", type=float, default=0.8)
p.add_argument("--throughput-runs", type=int, default=10)
+ p = sub.add_parser("export", add_help=False)
+ p.add_argument("--checkpoint", type=str, required=True)
+ p.add_argument("--dim", type=int, required=True)
+ p.add_argument("--output", type=str, default=None)
+
p = sub.add_parser("evaluate", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
p.add_argument("--benchmarks", type=str, nargs="*",
@@ -213,6 +227,9 @@ def main():
elif args.command == "test":
from .test import main as test_main
test_main(args)
+ elif args.command == "export":
+ from .export import main as export_main
+ export_main(args)
elif args.command == "evaluate":
from .evaluate import main as eval_main
eval_main(args)
diff --git a/src/export.py b/src/export.py
new file mode 100644
index 0000000..d088c7d
--- /dev/null
+++ b/src/export.py
@@ -0,0 +1,118 @@
+"""Export MRL sub-models by slicing a full checkpoint to a target dimension."""
+
+import os
+import pickle
+from dataclasses import replace
+from pathlib import Path
+
+import jax
+import jax.numpy as jnp
+import numpy as np
+
+from .model import TransformerConfig
+
+
+def export_submodel(checkpoint_path, d_prime, output_path):
+ """Slice a full MRL checkpoint to a self-contained sub-model at dimension d_prime."""
+
+ # Load checkpoint
+ with open(checkpoint_path, "rb") as f:
+ data = pickle.load(f)
+ params = data["params"]
+ config = TransformerConfig(**data["config"])
+
+ head_dim = config.d_model // config.num_heads
+ if d_prime % head_dim != 0:
+ raise ValueError(
+ f"d_prime={d_prime} must be divisible by head_dim={head_dim} "
+ f"(d_model={config.d_model} // num_heads={config.num_heads})"
+ )
+
+ # Derived dims
+ n_h = d_prime // head_dim
+ n_kv = min(config.num_kv_heads, n_h)
+ d_ff_prime = config.d_ff * d_prime // config.d_model
+ kv_dim = config.num_kv_heads * head_dim
+ kv_dim_prime = n_kv * head_dim
+
+ d_model = config.d_model
+ d_ff = config.d_ff
+
+ def slice_leaf(key_path, leaf):
+ arr = np.asarray(leaf)
+
+ if arr.ndim == 1:
+ # Norm scales: shape (d_model,)
+ return arr[:d_prime]
+
+ if arr.ndim == 3:
+ # Memory slots: shape (1, M, d_model)
+ return arr[:, :, :d_prime]
+
+ if arr.ndim == 2:
+ rows, cols = arr.shape
+ # Determine slice for each axis based on original size
+ def _target(size):
+ if size == d_model:
+ return d_prime
+ if size == d_ff:
+ return d_ff_prime
+ if size == kv_dim:
+ return kv_dim_prime
+ return size # vocab, num_memory_slots, etc.
+
+ r = _target(rows)
+ c = _target(cols)
+ return arr[:r, :c]
+
+ return arr
+
+ sliced = jax.tree_util.tree_map_with_path(slice_leaf, params)
+
+ # Build new config
+ new_config = replace(
+ config,
+ d_model=d_prime,
+ d_ff=d_ff_prime,
+ num_heads=n_h,
+ num_kv_heads=n_kv,
+ )
+
+ # Convert any JAX arrays to numpy for pickling
+ sliced_np = jax.tree.map(
+ lambda x: np.asarray(x) if isinstance(x, jnp.ndarray) else x, sliced
+ )
+
+ os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
+ with open(output_path, "wb") as f:
+ pickle.dump({"params": sliced_np, "config": new_config.__dict__}, f)
+
+ # Print summary
+ orig_count = sum(x.size for x in jax.tree.leaves(params))
+ new_count = sum(x.size for x in jax.tree.leaves(sliced_np))
+ orig_bytes = sum(x.nbytes for x in jax.tree.leaves(params))
+ new_bytes = sum(x.nbytes for x in jax.tree.leaves(sliced_np))
+
+ print(f"\n Export complete: {output_path}")
+ print(f" ─────────────────────────────────────")
+ print(f" {'':>20s} {'Original':>12s} {'Exported':>12s}")
+ print(f" {'d_model':>20s} {config.d_model:>12d} {d_prime:>12d}")
+ print(f" {'d_ff':>20s} {config.d_ff:>12d} {d_ff_prime:>12d}")
+ print(f" {'num_heads':>20s} {config.num_heads:>12d} {n_h:>12d}")
+ print(f" {'num_kv_heads':>20s} {config.num_kv_heads:>12d} {n_kv:>12d}")
+ print(f" {'params':>20s} {orig_count:>12,d} {new_count:>12,d}")
+ print(f" {'size (MB)':>20s} {orig_bytes / 1e6:>12.1f} {new_bytes / 1e6:>12.1f}")
+ print()
+
+
+def main(args):
+ checkpoint = args.checkpoint
+ d_prime = args.dim
+ output = args.output
+
+ if output is None:
+ stem = Path(checkpoint).stem
+ parent = Path(checkpoint).parent
+ output = str(parent / f"{stem}_d{d_prime}.pkl")
+
+ export_submodel(checkpoint, d_prime, output)
diff --git a/src/train.py b/src/train.py
index 7d4112c..668203e 100644
--- a/src/train.py
+++ b/src/train.py
@@ -426,23 +426,31 @@ def _make_mrl_val_loss_fn(apply_fn, mrl_dim):
def _estimate_mrl_params(config, d_prime):
- """Estimate parameter count if model were deployed at dimension d_prime."""
+ """Estimate parameter count of an exported sub-model at dimension d_prime.
+
+ Slicing rules (matching what an actual MRL export would produce):
+ - Embedding: [vocab, d_prime] (first d_prime cols)
+ - Heads: same head_dim, fewer heads → n_h = d_prime // head_dim
+ - KV heads: min(original, n_h)
+ - FFN: proportional → d_ff' = d_ff * d_prime // d_model
+ - Memory: slots unchanged, but projections shrink with d / d_ff
+ """
d = d_prime
v = config.vocab_size
n_enc = config.num_encoder_layers
n_dec = config.num_decoder_layers
- n_h = config.num_heads
- n_kv = config.num_kv_heads
- d_ff = d * 4
+ head_dim = config.d_model // config.num_heads
+ n_h = max(1, d // head_dim)
+ n_kv = min(config.num_kv_heads, n_h)
+ d_ff = config.d_ff * d // config.d_model
m = config.num_memory_slots
emb = v * d
- head_dim = d // n_h
kv_dim = n_kv * head_dim
- attn = d * d + d * kv_dim * 2 + d * d
- ffn = d * d_ff * 3
- mixer_token = d * d_ff * 2 + d_ff * m
- mixer_channel = d * d_ff * 3
+ attn = d * d + d * kv_dim * 2 + d * d
+ ffn = d * d_ff * 3
+ mixer_token = d * d_ff * 2 + d_ff * m
+ mixer_channel = d * d_ff * 3
enc_block = attn + ffn + mixer_token + mixer_channel
dec_block = attn * 2 + ffn
total = emb + n_enc * enc_block + n_dec * dec_block
@@ -499,9 +507,10 @@ def train(args):
config = TransformerConfig(
d_model=args.d_model,
num_heads=args.num_heads,
+ num_kv_heads=getattr(args, "num_kv_heads", None) or args.num_heads,
num_encoder_layers=args.num_layers,
num_decoder_layers=getattr(args, "num_dec_layers", args.num_layers),
- d_ff=args.d_model * 4,
+ d_ff=getattr(args, "d_ff", None) or args.d_model * 4,
max_seq_len=max(args.max_enc_len, args.max_dec_len),
dtype=args.dtype,
activation=getattr(args, "activation", "drelu"),
@@ -783,12 +792,14 @@ def train(args):
print(f" Sparsity {sparsity:>11.2f}% ({near_zero:,}/{total_params:,})")
if mrl_results:
print(f" ─────────────────────────────────────")
- print(f" MRL dimension pruning:")
- print(f" {'dim':>6} {'val ppl':>10} {'est. params':>12}")
- print(f" {config.d_model:>6} {last_val_ppl:>10.2f} {total_params:>12,} (full)")
+ print(f" MRL exportable sub-models:")
+ print(f" {'dim':>6} {'val ppl':>10} {'params':>12} {'heads':>5}")
+ head_dim = config.d_model // config.num_heads
+ print(f" {config.d_model:>6} {last_val_ppl:>10.2f} {total_params:>12,} {config.num_heads:>5} (full)")
for d_prime in sorted(mrl_results.keys(), reverse=True):
mrl_ppl, mrl_params = mrl_results[d_prime]
- print(f" {d_prime:>6} {mrl_ppl:>10.2f} {mrl_params:>12,}")
+ n_h = max(1, d_prime // head_dim)
+ print(f" {d_prime:>6} {mrl_ppl:>10.2f} {mrl_params:>12,} {n_h:>5}")
print(f" ─────────────────────────────────────")
print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")
print(f" Latency {tp['avg_latency_s']:>11.3f}s")