Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

needle cleanup

HenryNdubuakua20cb7b52e6b
diff --git a/README.md b/README.md
index ca53428..dfb37bc 100644
--- a/README.md
+++ b/README.md
@@ -12,16 +12,16 @@
┌─────────────┐
│ Softmax │ ┌──────────────────────┐
- └──────┬──────┘ │ Forward with MRL │
- ┌─────┬──────┴──────┬─────┐ │ loss at dims │
- │ @E[:d'] for each d' │ │ {512,256,128,64} │
- │ in mrl_dims │ │ + INT4 QAT (g=32) │
+ └──────┬──────┘ │ MRL loss at dims │
+ ┌─────┬──────┴──────┬─────┐ │ {512,256,128,64} │
+ │ @E[:d'] for each d' │ │ + INT4 QAT (g=32) │
+ │ in mrl_dims │ │ + z-loss + slot div │
└─────┬──────┬──────┬─────┘ └──────────┬───────────┘
┌──────┴──────┐ │
│ Linear (T) │ ← tied ┌──────────┴───────────┐
- └──────┬──────┘ │ Muon (2D kernels) │
- ┌──────┴──────┐ │ AdamW (everything │
- │ ZCRMSNorm │ │ else) │
+ └──────┬──────┘ │ Grad clip (norm 1.0) │
+ ┌──────┴──────┐ │ Muon (2D kernels) │
+ │ ZCRMSNorm │ │ AdamW (all else) │
└──────┬──────┘ │ WSD LR schedule │
┌─────────┴─────────┐ └──────────┬───────────┘
│ Decoder x N_dec │ │
@@ -30,22 +30,26 @@
│ │ Attn + RoPE │ │ └──────────┬───────────┘
│ ├───────────────┤ │ │
┌──────────────┐ S │ │ Cross │ │ ┌──────────┴───────────┐
- │ MemoryMixer │─────────▶ Attention │ │ │ Block Prune │
- │ Encoder │ │ ├───────────────┤ │ │ after epoch 1 │
- │ x N_enc │ │ │ Feed-Forward │ │ │ group magnitude │
- │ │ │ │ (dReLU) │ │ │ lock sparsity mask │
- │ ┌──────────┐ │ │ └───────────────┘ │ └──────────┬───────────┘
- │ │Pack: │ │ └─────────┬─────────┘ │
- │ │ S←X Attn │ │ ┌──────┴──────┐ ┌──────────┴───────────┐
- │ │ RoPE keys│ │ │ Embedding │ ← shared │ MRL Checkpoint │
- │ ├──────────┤ │ └──────┬──────┘ │ save per mrl_dim │
- │ │Mix: │ │ │ │ {512,256,128,64} │
- │ │ MLP-Mixer│ │ ┌─────┴─────┐ │ sparse + INT4 │
- │ │ on S │ │ │ Decoder │ └──────────────────────┘
- │ ├──────────┤ │ │ Input │
- │ │Local: │ │ └───────────┘
- │ │ FFN on X │ │
+ │ MemoryMixer │─────────▶ Attention │ │ │ Sparsification │
+ │ Encoder │ │ ├───────────────┤ │ │ cubic schedule │
+ │ x N_enc │ │ │ Gated FFN │ │ │ mask every N steps │
+ │ │ │ └───────────────┘ │ └──────────┬───────────┘
+ │ ┌──────────┐ │ └─────────┬─────────┘ │
+ │ │Pack: │ │ ┌──────┴──────┐ ┌──────────┴───────────┐
+ │ │ S←X Attn │ │ │ Embedding │ ← shared │ Checkpoint │
+ │ │ RoPE keys│ │ └──────┬──────┘ │ full params, can │
+ │ ├──────────┤ │ │ │ export MRL slices │
+ │ │Mix: │ │ ┌─────┴─────┐ └──────────────────────┘
+ │ │ MLP-Mixer│ │ │ Decoder │
+ │ │ on S │ │ │ Input │
+ │ ├──────────┤ │ └───────────┘
+ │ │Local: │ │
+ │ │Gated FFN │ │
+ │ │ on X │ │
│ └──────────┘ │
+ │ │
+ │ Slot Init │ learnable + input-dependent
+ │ DW Conv ↓2 │ stride-2 depthwise-separable
│ S ∈ (M, d) │
└──────┬───────┘
┌──────┴───────┐
@@ -57,9 +61,9 @@
│ Input │
└─────────┘
- d=max(mrl_dims) · 4 heads · 2 KV heads · 64 memory slots
- SentencePiece BPE (8192) · dReLU · ZCRMSNorm · RoPE
- Matryoshka dims · INT4 QAT · Muon + AdamW
+ d=max(mrl_dims) · GQA (4 heads, 2 KV) · QK-norm · 64 slots
+ SentencePiece BPE (8192) · gated dReLU · ZCRMSNorm · RoPE
+ strided DW conv · Matryoshka dims · INT4 QAT · Muon + AdamW
```
## Usage
@@ -74,35 +78,31 @@ needle [command]
┌───────────────────────────────────────────────────────────────────┐
│ │
│ train │
- │ --toy Use toy config for quick iteration │
- │ --epochs INT Training epochs (default: 2) │
+ │ --epochs INT Training epochs (default: 1) │
│ --batch-size INT Batch size (default: 32) │
│ --lr FLOAT AdamW learning rate (default: 3e-4) │
│ --muon-lr FLOAT Muon learning rate (default: 0.02) │
│ --d-model INT Model dim (default: max of mrl-dims) │
│ --num-heads INT Attention heads (default: 4) │
- │ --num-layers INT Encoder layers (default: 12) │
- │ --num-dec-layers INT Decoder layers (default: 4) │
+ │ --num-layers INT Encoder layers (default: 4) │
+ │ --num-dec-layers INT Decoder layers (default: 2) │
│ --max-enc-len INT Max encoder seq length (default: 256) │
│ --max-dec-len INT Max decoder seq length (default: 256) │
- │ --max-samples INT Training samples (default: 1000000) │
- │ --mrl-dims INT [...] MRL dim targets (default: 512 256 128) │
- │ --sparsity-ratio FLOAT Block prune ratio (default: 0.33) │
- │ --layer-prune-ratio FL Layer prune ratio (default: 0.0) │
+ │ --max-samples INT Training samples (default: all) │
+ │ --mrl-dims INT [...] MRL dim targets (default: 1024 512..) │
+ │ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) │
│ --group-size INT Quant/prune group size (default: 32) │
+ │ --prune-interval INT Steps between mask updates (def: 100) │
+ │ --prune-start-frac FL Start pruning at this frac (def: 0.33) │
+ │ --prune-end-frac FL Lock mask at this frac (def: 0.67) │
│ --activation STR drelu|swiglu|geglu (default: drelu) │
- │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │
+ │ --warmup-ratio FLOAT LR warmup ratio (default: 0.05) │
│ --eval-every INT Val eval interval (default: 1000) │
│ --wandb Enable W&B logging │
│ --checkpoint PATH Resume from checkpoint │
│ --checkpoint-dir DIR Checkpoint directory │
│ --seed INT Random seed (default: 42) │
│ │
- │ sweep │
- │ --sweep-config PATH Sweep YAML config │
- │ --project STR W&B project name (default: needle-v1) │
- │ --count INT Number of trials (default: 20) │
- │ │
│ run │
│ --checkpoint PATH Path to model checkpoint (required) │
│ --prompts STR [...] One or more prompts to continue │
diff --git a/src/cli.py b/src/cli.py
index 5c32413..0349cec 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -10,21 +10,19 @@ HELP = """
│ ...the tiny model to rule them all... │
│ │
│ train │
- │ --toy Use toy config for quick iteration │
│ --epochs INT Training epochs (default: 1) │
│ --batch-size INT Batch size (default: 32) │
│ --lr FLOAT AdamW learning rate (default: 3e-4) │
│ --muon-lr FLOAT Muon learning rate (default: 0.02) │
│ --d-model INT Model dim (default: max of mrl-dims) │
│ --num-heads INT Attention heads (default: 4) │
- │ --num-layers INT Encoder layers (default: 12) │
- │ --num-dec-layers INT Decoder layers (default: 4) │
+ │ --num-layers INT Encoder layers (default: 4) │
+ │ --num-dec-layers INT Decoder layers (default: 2) │
│ --max-enc-len INT Max encoder seq length (default: 256) │
│ --max-dec-len INT Max decoder seq length (default: 256) │
- │ --max-samples INT Training samples (default: 1000000) │
- │ --mrl-dims INT [...] MRL dim targets (default: 512 256 ..) │
- │ --sparsity-ratio FLOAT Block prune ratio (default: 0.33) │
- │ --layer-prune-ratio FL Layer prune ratio (default: 0.0) │
+ │ --max-samples INT Training samples (default: all) │
+ │ --mrl-dims INT [...] MRL dim targets (default: 1024 512..) │
+ │ --sparsity-ratio FLOAT Block prune ratio (default: 0.5) │
│ --group-size INT Quant/prune group size (default: 32) │
│ --prune-interval INT Steps between mask updates (def: 100) │
│ --prune-start-frac FL Start pruning at this frac (def: 0.33)│
@@ -72,34 +70,6 @@ HELP = """
└───────────────────────────────────────────────────────────────────┘
"""
-TOY_CONFIG = {
- "num_heads": 4,
- "num_layers": 2,
- "num_dec_layers": 2,
- "max_enc_len": 128,
- "max_dec_len": 128,
- "max_samples": 10000,
-}
-
-MAIN_CONFIG = {
- "num_heads": 4,
- "num_layers": 4,
- "num_dec_layers": 2,
- "max_enc_len": 256,
- "max_dec_len": 256,
- "max_samples": None,
-}
-
-
-def _apply_train_defaults(args):
- config = TOY_CONFIG if args.toy else MAIN_CONFIG
- for key, value in config.items():
- if getattr(args, key, None) is None:
- setattr(args, key, value)
- if getattr(args, "d_model", None) is None:
- args.d_model = max(args.mrl_dims)
-
-
def main():
if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help", "help"):
print(HELP)
@@ -109,18 +79,17 @@ def main():
sub = parser.add_subparsers(dest="command")
p = sub.add_parser("train", add_help=False)
- p.add_argument("--toy", action="store_true")
p.add_argument("--checkpoint", type=str, default=None)
p.add_argument("--epochs", type=int, default=1)
p.add_argument("--batch-size", type=int, default=32)
p.add_argument("--lr", type=float, default=3e-4)
p.add_argument("--muon-lr", type=float, default=0.02)
p.add_argument("--d-model", type=int, default=None)
- p.add_argument("--num-heads", type=int, default=None)
- p.add_argument("--num-layers", type=int, default=None)
- p.add_argument("--num-dec-layers", type=int, default=None)
- p.add_argument("--max-enc-len", type=int, default=None)
- p.add_argument("--max-dec-len", type=int, default=None)
+ p.add_argument("--num-heads", type=int, default=4)
+ p.add_argument("--num-layers", type=int, default=4)
+ p.add_argument("--num-dec-layers", type=int, default=2)
+ p.add_argument("--max-enc-len", type=int, default=256)
+ p.add_argument("--max-dec-len", type=int, default=256)
p.add_argument("--max-samples", type=int, default=None)
p.add_argument("--warmup-ratio", type=float, default=0.05)
p.add_argument("--wandb", action="store_true")
@@ -130,7 +99,6 @@ def main():
p.add_argument("--eval-every", type=int, default=1000)
p.add_argument("--max-eval-samples", type=int, default=None)
p.add_argument("--sparsity-ratio", type=float, default=0.5)
- p.add_argument("--layer-prune-ratio", type=float, default=0.0)
p.add_argument("--group-size", type=int, default=32)
p.add_argument("--prune-interval", type=int, default=100,
help="Steps between mask updates during gradual pruning (default: 100)")
@@ -204,7 +172,8 @@ def main():
sys.exit(0)
if args.command == "train":
- _apply_train_defaults(args)
+ if args.d_model is None:
+ args.d_model = max(args.mrl_dims)
from .train import train
train(args)
elif args.command == "run":
diff --git a/src/train.py b/src/train.py
index ccf9a5a..8dff7e1 100644
--- a/src/train.py
+++ b/src/train.py
@@ -255,72 +255,6 @@ def _make_prune_mask(params, sparsity, group_size):
return pruned, mask
-def _layer_prune(params, config, layer_prune_ratio=0.25):
- """Remove the lowest-scoring encoder/decoder blocks by L1 magnitude.
-
- Returns (new_params, new_config) with surviving blocks renumbered.
- """
- from flax.core import unfreeze
- p = unfreeze(params)
-
- def _score_blocks(module_params, prefix):
- scores = {}
- for key, val in module_params.items():
- if key.startswith("block_"):
- s = sum(float(np.sum(np.abs(np.array(l)))) for l in jax.tree.leaves(val))
- scores[key] = s
- return scores
-
- enc_scores = _score_blocks(p["encoder"], "encoder")
- dec_scores = _score_blocks(p["decoder"], "decoder")
-
- all_scores = list(enc_scores.values()) + list(dec_scores.values())
- if not all_scores:
- return params, config
-
- num_enc = len(enc_scores)
- num_dec = len(dec_scores)
- total_to_remove = max(0, int(len(all_scores) * layer_prune_ratio))
-
- tagged = [(s, "encoder", k) for k, s in enc_scores.items()] + \
- [(s, "decoder", k) for k, s in dec_scores.items()]
- tagged.sort(key=lambda x: x[0])
-
- remove_enc = set()
- remove_dec = set()
- for score, module, key in tagged:
- if len(remove_enc) + len(remove_dec) >= total_to_remove:
- break
- if module == "encoder" and (num_enc - len(remove_enc)) > 1:
- remove_enc.add(key)
- elif module == "decoder" and (num_dec - len(remove_dec)) > 1:
- remove_dec.add(key)
-
- # Rebuild with surviving blocks renumbered
- def _keep_and_renumber(module_params, remove_set):
- surviving = [(k, v) for k, v in sorted(module_params.items()) if k.startswith("block_") and k not in remove_set]
- non_blocks = {k: v for k, v in module_params.items() if not k.startswith("block_")}
- new = dict(non_blocks)
- for i, (_, v) in enumerate(surviving):
- new[f"block_{i}"] = v
- return new
-
- p["encoder"] = _keep_and_renumber(p["encoder"], remove_enc)
- p["decoder"] = _keep_and_renumber(p["decoder"], remove_dec)
-
- new_num_enc = num_enc - len(remove_enc)
- new_num_dec = num_dec - len(remove_dec)
- print(f" Encoder: {num_enc} -> {new_num_enc} blocks (removed {len(remove_enc)})")
- print(f" Decoder: {num_dec} -> {new_num_dec} blocks (removed {len(remove_dec)})")
-
- from dataclasses import replace as dc_replace
- new_config = dc_replace(config,
- num_encoder_layers=new_num_enc,
- num_decoder_layers=new_num_dec)
-
- return p, new_config
-
-
def _quantize_params(params, group_size=32):
"""Fake-quantize all Dense kernels in the param tree."""
def _maybe_quantize(path, leaf):
@@ -586,57 +520,23 @@ def train(args):
last_val_ppl = None
sparsity_ratio = getattr(args, "sparsity_ratio", 0.0)
- layer_prune_ratio = getattr(args, "layer_prune_ratio", 0.0)
prune_mask = None
- gradual_pruning_done = False
+ gradual_sparsify_done = False
prune_interval = getattr(args, "prune_interval", 100)
prune_start_frac = getattr(args, "prune_start_frac", 0.33)
prune_end_frac = getattr(args, "prune_end_frac", 0.67)
weight_prune_epoch = 0 if sparsity_ratio > 0 else -1
- layer_prune_epoch = (1 if sparsity_ratio > 0 else 0) if layer_prune_ratio > 0 else -1
for epoch in range(args.epochs):
- if epoch == weight_prune_epoch and not gradual_pruning_done:
+ if epoch == weight_prune_epoch and not gradual_sparsify_done:
t_start = int(num_batches * prune_start_frac)
t_end = int(num_batches * prune_end_frac)
- print(f"\nGradual magnitude pruning: 0% -> {sparsity_ratio*100:.0f}% over epoch {epoch+1} "
+ print(f"\nGradual magnitude sparsification: 0% -> {sparsity_ratio*100:.0f}% over epoch {epoch+1} "
f"(steps {t_start}-{t_end}/{num_batches}, interval={prune_interval}, group_size={_GROUP_SIZE})")
epoch_step = 0
- if epoch == layer_prune_epoch:
- print(f"\nLayer pruning: removing {layer_prune_ratio*100:.0f}% of layers...")
- current_params = jax_utils.unreplicate(ema_params)
- new_params, config = _layer_prune(current_params, config, layer_prune_ratio=layer_prune_ratio)
-
- rng, reinit_rng = jax.random.split(rng)
- state = create_train_state(reinit_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps)
- matched_params = jax.tree.map(lambda init_leaf, prune_leaf: prune_leaf, state.params, new_params)
- state = state.replace(params=matched_params)
- ema_params = jax.tree.map(jnp.copy, matched_params)
-
- param_count = sum(x.size for x in jax.tree.leaves(matched_params))
- print(f" Parameters: {param_count:,}")
-
- val_loss_fn = _make_val_loss_fn(state.apply_fn)
- val_causal = make_causal_mask(args.max_dec_len)
- total_loss, total_toks = 0.0, 0.0
- for vb in get_batches(val_enc, val_dec_in, val_dec_tgt, args.batch_size, shuffle=False):
- vl, vt = val_loss_fn(matched_params, vb[0], vb[1], vb[2], val_causal)
- total_loss += float(vl)
- total_toks += float(vt)
- layer_prune_ppl = float(math.exp(total_loss / max(total_toks, 1)))
- print(f" Post-layer-prune val ppl: {layer_prune_ppl:.2f} (was {last_val_ppl:.2f})" if last_val_ppl is not None else f" Post-layer-prune val ppl: {layer_prune_ppl:.2f}")
-
- if prune_mask is not None:
- prune_mask = jax.tree.map(lambda w: (jnp.abs(w) > 1e-8).astype(jnp.float32), matched_params)
- prune_mask = jax_utils.replicate(prune_mask)
-
- state = jax_utils.replicate(state)
- ema_params = jax_utils.replicate(ema_params)
- print(f"\nContinuing training with {config.num_encoder_layers}+{config.num_decoder_layers} layers...\n")
-
losses = []
batches = get_batches(enc_inputs, dec_inputs, dec_targets, effective_batch_size)
pbar = tqdm(batches, total=num_batches, desc=f"Epoch {epoch + 1}/{args.epochs}")
@@ -653,8 +553,8 @@ def train(args):
causal_mask,
)
- # --- Gradual magnitude pruning (epoch 1 only) ---
- if epoch == weight_prune_epoch and not gradual_pruning_done:
+ # --- Gradual magnitude sparsification (epoch 1 only) ---
+ if epoch == weight_prune_epoch and not gradual_sparsify_done:
epoch_step += 1
current_sparsity = _cubic_sparsity_schedule(epoch_step, t_start, t_end, sparsity_ratio)
if epoch_step >= t_start and epoch_step % prune_interval == 0 and current_sparsity > 0:
@@ -684,7 +584,7 @@ def train(args):
# --- tqdm postfix with sparsity during epoch 1 ---
postfix = {"loss": f"{loss_val:.4f}", "val_ppl": f"{last_val_ppl:.2f}" if last_val_ppl is not None else "?"}
- if epoch == weight_prune_epoch and not gradual_pruning_done:
+ if epoch == weight_prune_epoch and not gradual_sparsify_done:
postfix["sparsity"] = f"{current_sparsity*100:.1f}%"
pbar.set_postfix(**postfix)
@@ -697,15 +597,15 @@ def train(args):
"train/tokens_per_sec": tokens_per_batch / dt,
"train/step": global_step,
}
- if epoch == weight_prune_epoch and not gradual_pruning_done:
+ if epoch == weight_prune_epoch and not gradual_sparsify_done:
log_dict["train/scheduled_sparsity"] = current_sparsity
if global_step % eval_every == 0 or global_step == total_steps:
log_dict["val/ppl"] = last_val_ppl
wandb.log(log_dict)
- # --- Lock mask after epoch 1 gradual pruning ---
- if epoch == weight_prune_epoch and not gradual_pruning_done:
- gradual_pruning_done = True
+ # --- Lock mask after epoch 1 gradual sparsification ---
+ if epoch == weight_prune_epoch and not gradual_sparsify_done:
+ gradual_sparsify_done = True
if prune_mask is None:
# Edge case: epoch had fewer steps than t_start
ema_unr = jax_utils.unreplicate(ema_params)
@@ -716,7 +616,7 @@ def train(args):
final_pruned = jax.tree.map(np.array, jax_utils.unreplicate(ema_params))
total_p = sum(x.size for x in jax.tree.leaves(final_pruned))
zero_p = sum(int(np.sum(np.abs(x) < 1e-6)) for x in jax.tree.leaves(final_pruned))
- print(f"\n Gradual pruning complete — mask locked.")
+ print(f"\n Gradual sparsification complete — mask locked.")
print(f" Final sparsity: {zero_p/total_p*100:.2f}% ({zero_p:,}/{total_p:,} near-zero)")
del final_pruned