Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
add pretrain
HenryNdubuakud964cdfcf6eb
diff --git a/src/cli.py b/src/cli.pyindex f967292..10cdfc8 100644--- a/src/cli.py+++ b/src/cli.py@@ -1,4 +1,5 @@import argparse+import osimport sysfrom .data import DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_DEC_LEN, DEFAULT_MAX_GEN_LEN@@ -73,6 +74,33 @@ def main():p.add_argument("--calibrate-k", type=float, default=5.0,help="Sigmoid steepness for PPL→confidence mapping (default: 5.0)")+ p = sub.add_parser("pretrain", add_help=False)+ p.add_argument("--name", type=str, default="pretrain",+ help="Experiment name for wandb (default: pretrain)")+ p.add_argument("--epochs", type=int, default=1)+ p.add_argument("--batch-size", type=int, default=128)+ p.add_argument("--lr", type=float, default=3e-4)+ p.add_argument("--muon-lr", type=float, default=0.02)+ p.add_argument("--d-model", type=int, default=512)+ p.add_argument("--num-heads", type=int, default=8)+ p.add_argument("--num-kv-heads", type=int, default=4)+ p.add_argument("--num-layers", type=int, default=12)+ p.add_argument("--num-dec-layers", type=int, default=8)+ p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)+ p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)+ p.add_argument("--warmup-ratio", type=float, default=0.05)+ p.add_argument("--decay-ratio", type=float, default=0.05)+ p.add_argument("--wandb", action="store_true")+ p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])+ p.add_argument("--seed", type=int, default=42)+ p.add_argument("--max-steps", type=int, default=None,+ help="Stop after N steps (default: full epoch)")+ p.add_argument("--save-every", type=int, default=1000,+ help="Save and upload checkpoint every N steps (default: 1000)")+ p.add_argument("--checkpoint-dir", type=str, default="checkpoints")+ p.add_argument("--activation", type=str, default="swiglu", choices=["drelu", "swiglu", "geglu"])+ p.add_argument("--no-feedforward", action=argparse.BooleanOptionalAction, default=True)+p = sub.add_parser("tokenize", add_help=False)p.add_argument("--max-samples", type=int, default=None,help="Limit samples per split (for dev/test)")@@ -179,6 +207,12 @@ def main():tp.add_argument("train_args", nargs=argparse.REMAINDER,help="Extra args passed to needle train")+ tp = tpu_sub.add_parser("pretrain", add_help=False)+ tp.add_argument("name", type=str)+ tp.add_argument("--zone", type=str, default=None)+ tp.add_argument("train_args", nargs=argparse.REMAINDER,+ help="Extra args passed to needle pretrain")+tp = tpu_sub.add_parser("claude", add_help=False)tp.add_argument("name", type=str)tp.add_argument("--zone", type=str, default=None)@@ -206,9 +240,16 @@ def main():if args.command == "tokenize":from .tokenize_data import tokenizetokenize(args)+ elif args.command == "pretrain":+ import jax+ if os.path.exists("/dev/accel0"):+ jax.distributed.initialize()+ from .pretrain import pretrain+ pretrain(args)elif args.command == "train":import jax- jax.distributed.initialize()+ if os.path.exists("/dev/accel0"):+ jax.distributed.initialize()from .train import traintrain(args)elif args.command == "run":diff --git a/src/data.py b/src/data.pyindex 9c0c643..1e0b23d 100644--- a/src/data.py+++ b/src/data.py@@ -16,6 +16,16 @@ from datasets import Audio, Dataset, load_from_diskfrom tqdm import tqdmimport sentencepiece as spm+from .tokenizer import (+ PAD_ID, EOS_ID, BOS_ID, UNK_ID, TOOL_CALL_ID, TOOLS_ID,+ DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_DEC_LEN, DEFAULT_MAX_GEN_LEN,+ to_snake_case, NeedleTokenizer, train_tokenizer, get_tokenizer,+ _download_tokenizer_from_hf, TOKENIZER_DIR, TOKENIZER_PREFIX,+ _HF_TOKENIZER_REPO,+)++import re as _re+HF_DATASET_REPO = "Cactus-Compute/tool-calls"@@ -53,38 +63,11 @@ def download_hf_split(split="train", repo_id=HF_DATASET_REPO):return Dataset(table)-import re as _re--def to_snake_case(name):- """Convert camelCase, PascalCase, or dot.notation name to snake_case."""- # Replace any non-alphanumeric/underscore characters with underscores- s = _re.sub(r'[^a-zA-Z0-9_]+', '_', name)- # Insert underscore before uppercase letters that follow lowercase/digits- s = _re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', s)- # Insert underscore between consecutive uppercase and uppercase+lowercase- s = _re.sub(r'([A-Z]+)([A-Z][a-z])', r'\1_\2', s)- # Collapse multiple underscores and strip edges- s = _re.sub(r'_+', '_', s)- return s.lower().strip('_')-_PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))_DISK_CACHE_DIR = os.path.join(_PROJECT_ROOT, ".data_cache")-TOKENIZER_DIR = os.path.join(_PROJECT_ROOT, "tokenizer")-TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")LOCAL_UNIFIED_DIR = os.path.join(_PROJECT_ROOT, "data", "tool_calls_unified")CACHE_DIR = os.path.join(_PROJECT_ROOT, ".data_cache")-PAD_ID = 0-EOS_ID = 1-BOS_ID = 2-UNK_ID = 3-TOOL_CALL_ID = 4-TOOLS_ID = 5--DEFAULT_MAX_ENC_LEN = 1024-DEFAULT_MAX_DEC_LEN = 512-DEFAULT_MAX_GEN_LEN = 512-_split_dataset_cache = {}@@ -279,55 +262,6 @@ def _set_audio_backend(ds):return ds-class NeedleTokenizer:- """Wrapper around SentencePiece providing the interface the codebase expects."""-- def __init__(self, model_path):- self.sp = spm.SentencePieceProcessor()- self.sp.Load(model_path)-- @property- def pad_token_id(self):- return PAD_ID-- @property- def eos_token_id(self):- return EOS_ID-- @property- def bos_token_id(self):- return BOS_ID-- @property- def tool_call_token_id(self):- return TOOL_CALL_ID-- @property- def tools_token_id(self):- return TOOLS_ID-- @property- def vocab_size(self):- return self.sp.GetPieceSize()-- def encode(self, text):- return self.sp.Encode(text, out_type=int)-- def decode(self, ids):- if isinstance(ids, (list, tuple)) and len(ids) > 0 and isinstance(ids[0], (list, tuple)):- return [self.sp.Decode(seq) for seq in ids]- return self.sp.Decode(list(ids))-- def __call__(self, texts, truncation=True, max_length=None, **kwargs):- all_ids = []- for text in texts:- ids = self.sp.Encode(text, out_type=int)- if truncation and max_length:- ids = ids[:max_length]- all_ids.append(ids)- return {"input_ids": all_ids}--_worker_sp = None_worker_max_len = None@@ -353,57 +287,6 @@ def _compute_classes_chunk(chunk):return results-def train_tokenizer(vocab_size=8192, max_samples=None, force=False):- """Train a SentencePiece BPE tokenizer on tool-calling corpus."""- model_path = TOKENIZER_PREFIX + ".model"- if os.path.exists(model_path) and not force:- print(f"Tokenizer already exists at {model_path}")- return model_path-- os.makedirs(TOKENIZER_DIR, exist_ok=True)-- from datasets import concatenate_datasets- train_ds = _load_split_dataset("train")- val_ds = _load_split_dataset("validation")- ds = concatenate_datasets([train_ds, val_ds])- if max_samples:- ds = ds.select(range(min(max_samples, len(ds))))-- print(f"Training SentencePiece BPE tokenizer (vocab_size={vocab_size}, samples={len(ds):,})...")-- corpus_path = os.path.join(TOKENIZER_DIR, "corpus.txt")- try:- with open(corpus_path, "w") as f:- for example in tqdm(ds, desc="Writing corpus"):- for field in ("query", "tools", "answers"):- text = example[field].strip()- if text:- f.write(text + "\n")-- spm.SentencePieceTrainer.Train(- input=corpus_path,- model_prefix=TOKENIZER_PREFIX,- vocab_size=vocab_size,- model_type="bpe",- pad_id=PAD_ID,- eos_id=EOS_ID,- bos_id=BOS_ID,- unk_id=UNK_ID,- user_defined_symbols=["<tool_call>", "<tools>"],- byte_fallback=True,- normalization_rule_name="identity",- num_threads=min(128, max(1, (os.cpu_count() or 1) * 3 // 4)),- train_extremely_large_corpus=False,- minloglevel=2,- )- finally:- if os.path.exists(corpus_path):- os.remove(corpus_path)- print(f"Tokenizer saved to {model_path}")- return model_path---_HF_TOKENIZER_REPO = "Cactus-Compute/needle-tokenizer"_HF_TOKENIZED_REPO = "Cactus-Compute/tokenized-tool-calls"@@ -428,29 +311,6 @@ def _download_tokenized_from_hf():os.symlink(os.path.join(local, fname), dst)-def _download_tokenizer_from_hf():- """Download tokenizer files from HuggingFace Hub into TOKENIZER_DIR."""- from huggingface_hub import snapshot_download-- local = snapshot_download(- _HF_TOKENIZER_REPO,- repo_type="dataset",- )- os.makedirs(TOKENIZER_DIR, exist_ok=True)- for fname in os.listdir(local):- if fname.startswith("."):- continue- dst = os.path.join(TOKENIZER_DIR, fname)- if not os.path.exists(dst):- os.symlink(os.path.join(local, fname), dst)---def get_tokenizer(max_samples=None):- model_path = TOKENIZER_PREFIX + ".model"- if not os.path.exists(model_path):- print("Downloading pretrained tokenizer from HuggingFace...")- _download_tokenizer_from_hf()- return NeedleTokenizer(model_path)def load_tool_calls(split="train", max_samples=None, return_global_indices=False):diff --git a/src/distributed.py b/src/distributed.pynew file mode 100644index 0000000..5f1f26c--- /dev/null+++ b/src/distributed.py@@ -0,0 +1,58 @@+import os+import threading+import jax+import numpy as np++_HF_CHECKPOINT_REPO = "Cactus-Compute/checkpoints"+++def _replicate(tree):+ """Replicate a pytree across all local devices (multi-host safe)."""+ devices = jax.local_devices()+ return jax.tree.map(+ lambda x: jax.device_put_replicated(x, devices), tree)+++def _unreplicate(tree):+ """Get a single copy from a replicated pytree (multi-host safe).++ Uses addressable_shards instead of x[0] indexing, which fails+ in multi-host pmap because the array spans devices on other hosts.+ Each shard has shape (1, ...) — we index [0] to strip the leading dim.+ """+ def _get_first(x):+ if hasattr(x, 'addressable_shards') and x.addressable_shards:+ shard = x.addressable_shards[0].data+ # Shard has shape (1, ...) from the pmap partition; strip it+ return jax.device_get(shard[0]) if shard.ndim > 0 else jax.device_get(shard)+ return x+ return jax.tree.map(_get_first, tree)+++def shard_batch(batch, num_devices):+ """Reshape a batch array so leading dim is (num_devices, per_device_batch, ...)."""+ return batch.reshape(num_devices, -1, *batch.shape[1:])+++def _upload_checkpoint(ckpt_path):+ """Upload a checkpoint file to HuggingFace Hub in a background thread."""+ import threading++ def _upload():+ try:+ from huggingface_hub import HfApi+ api = HfApi()+ api.create_repo(_HF_CHECKPOINT_REPO, repo_type="model", private=True, exist_ok=True)+ filename = os.path.basename(ckpt_path)+ print(f"[hf] Uploading {filename} to {_HF_CHECKPOINT_REPO} ...")+ api.upload_file(+ path_or_fileobj=ckpt_path,+ path_in_repo=filename,+ repo_id=_HF_CHECKPOINT_REPO,+ repo_type="model",+ )+ print(f"[hf] Checkpoint uploaded: {_HF_CHECKPOINT_REPO}/{filename}")+ except Exception as e:+ print(f"[hf] Warning: checkpoint upload failed: {e}")++ threading.Thread(target=_upload, daemon=True).start()diff --git a/src/optim.py b/src/optim.pynew file mode 100644index 0000000..301402c--- /dev/null+++ b/src/optim.py@@ -0,0 +1,123 @@+import math+from typing import NamedTuple+import jax+import jax.numpy as jnp+import optax+from flax.training import train_state+from .model import EncoderDecoderTransformer+++def _newton_schulz(G, steps=5):+ """Approximate polar decomposition via Newton-Schulz iteration."""+ a, b, c = 3.4445, -4.7750, 2.0315+ orig_dtype = G.dtype+ G = G.astype(jnp.float32)+ X = G / (jnp.linalg.norm(G) + 1e-7)+ transposed = G.shape[0] > G.shape[1]+ if transposed:+ X = X.T+ for _ in range(steps):+ A = X @ X.T+ B = b * A + c * A @ A+ X = a * X + B @ X+ if transposed:+ X = X.T+ return X.astype(orig_dtype)+++class MuonState(NamedTuple):+ mu: optax.Updates+++def scale_by_muon(momentum=0.95, ns_steps=5):+ """Muon gradient transform: orthogonalize 2D+ grads, then Nesterov momentum."""++ def init_fn(params):+ return MuonState(mu=jax.tree.map(jnp.zeros_like, params))++ def update_fn(updates, state, params=None):+ del params++ def ortho(g):+ if g.ndim == 3:+ return jax.vmap(_newton_schulz, in_axes=(0,))(g)+ if g.ndim == 2:+ return _newton_schulz(g, steps=ns_steps)+ return g++ ortho_g = jax.tree.map(ortho, updates)+ new_mu = jax.tree.map(lambda m, g: momentum * m + g, state.mu, ortho_g)+ new_updates = jax.tree.map(+ lambda g, m: g + momentum * m, ortho_g, new_mu+ )+ return new_updates, MuonState(mu=new_mu)++ return optax.GradientTransformation(init_fn, update_fn)+++def _param_labels(params):+ """Label each param: 'muon' for Dense kernels, 'adam' for the rest."""++ def _label(path, leaf):+ name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])+ if name == "kernel" and leaf.ndim in (2, 3):+ return "muon"+ return "adam"++ return jax.tree_util.tree_map_with_path(_label, params)+++def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.15):+ """Warmup-Stable-Decay schedule: linear warmup, hold peak, cosine decay."""+ decay_steps = max(1, int(total_steps * decay_ratio))+ stable_steps = total_steps - warmup_steps - decay_steps+ return optax.join_schedules(+ [+ optax.linear_schedule(0.0, peak_value, warmup_steps),+ optax.constant_schedule(peak_value),+ optax.cosine_decay_schedule(peak_value, decay_steps, alpha=0.05),+ ],+ boundaries=[warmup_steps, warmup_steps + stable_steps],+ )+++def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps, decay_ratio=0.15):+ model = EncoderDecoderTransformer(config)++ rng, init_rng = jax.random.split(rng)+ dummy_src = jnp.ones((1, 128), dtype=jnp.int32)+ dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)+ init_args = [dummy_src, dummy_tgt]+ if config.enable_speech:+ init_args.append(jnp.ones((1, 128, config.n_mels), dtype=jnp.float32))+ variables = model.init(+ {"params": init_rng},+ *init_args,+ method="init_all",+ )++ adam_schedule = _wsd_schedule(learning_rate, total_steps, warmup_steps, decay_ratio)+ muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps, decay_ratio)++ muon_opt = optax.chain(+ scale_by_muon(momentum=0.95, ns_steps=5),+ optax.add_decayed_weights(weight_decay=0.01),+ optax.scale_by_schedule(muon_schedule),+ optax.scale(-1.0),+ )+ adam_opt = optax.chain(+ optax.adamw(adam_schedule, b2=0.95, weight_decay=0.0),+ )++ tx = optax.chain(+ optax.clip_by_global_norm(1.0),+ optax.multi_transform(+ {"muon": muon_opt, "adam": adam_opt},+ _param_labels,+ ),+ )+ return train_state.TrainState.create(+ apply_fn=model.apply,+ params=variables["params"],+ tx=tx,+ )diff --git a/src/pretrain.py b/src/pretrain.pynew file mode 100644index 0000000..cec5a93--- /dev/null+++ b/src/pretrain.py@@ -0,0 +1,352 @@+"""Streaming pretraining on PleIAs/SYNTH dataset.++Streams 80M examples from HuggingFace, tokenizes on-the-fly, and trains+with simple CE loss (uniform weights). Saves needle_base.pkl every N steps.++Usage:+ needle pretrain --wandb+ needle pretrain --wandb --max-steps 10000+ needle tpu train large --name pretrain --wandb+"""++import math+import os+import pickle+import time+import queue+import threading++import jax+import jax.numpy as jnp+import numpy as np+import optax+from tqdm import tqdm++from .tokenizer import get_tokenizer, EOS_ID, TOOLS_ID, PAD_ID+from .model import (+ EncoderDecoderTransformer,+ TransformerConfig,+ make_padding_mask,+ make_causal_mask,+)+from .optim import create_train_state, _wsd_schedule+from .distributed import _replicate, _unreplicate, shard_batch, _upload_checkpoint++_HF_PRETRAIN_REPO = "PleIAs/SYNTH"+++def _stream_batches(tokenizer, batch_size, max_enc_len, max_dec_len, seed=42):+ """Stream batches from PleIAs/SYNTH, tokenizing on-the-fly.++ Encoder: [query_tokens, <tools>, seed_text_tokens] truncated to max_enc_len+ Decoder input: [EOS, answer_tokens] truncated to max_dec_len+ Decoder target: [answer_tokens, EOS] truncated to max_dec_len++ Yields (enc, dec_in, dec_tgt) as numpy int32 arrays of shape (batch_size, seq_len).+ """+ from datasets import load_dataset++ ds = load_dataset(_HF_PRETRAIN_REPO, split="train", streaming=True)+ ds = ds.shuffle(seed=seed, buffer_size=10000)++ tools_sep_id = TOOLS_ID+ eos_id = EOS_ID+ pad_id = PAD_ID++ enc_batch = np.full((batch_size, max_enc_len), pad_id, dtype=np.int32)+ dec_in_batch = np.full((batch_size, max_dec_len), pad_id, dtype=np.int32)+ dec_tgt_batch = np.full((batch_size, max_dec_len), pad_id, dtype=np.int32)+ idx = 0++ for example in ds:+ query = example.get("query") or ""+ seed_text = example.get("query_seed_text") or ""+ answer = example.get("synthetic_answer") or ""++ if not query.strip() or not answer.strip():+ continue++ # Tokenize+ q_toks = tokenizer.encode(query)+ s_toks = tokenizer.encode(seed_text) if seed_text.strip() else []+ a_toks = tokenizer.encode(answer)++ # Build encoder input: [query, <tools>, seed_text]+ max_query = max_enc_len - 2+ if len(q_toks) > max_query:+ q_toks = q_toks[:max_query]+ remaining = max_enc_len - len(q_toks) - 1+ s_toks = s_toks[:remaining]+ enc_tokens = q_toks + [tools_sep_id] + s_toks++ # Build decoder: input=[EOS, answer], target=[answer, EOS]+ max_ans = max_dec_len - 1+ a_toks = a_toks[:max_ans]+ if len(a_toks) == 0:+ continue+ dec_in_tokens = [eos_id] + a_toks+ dec_tgt_tokens = a_toks + [eos_id]++ # Fill batch+ enc_batch[idx, :len(enc_tokens)] = enc_tokens+ dec_in_batch[idx, :len(dec_in_tokens)] = dec_in_tokens+ dec_tgt_batch[idx, :len(dec_tgt_tokens)] = dec_tgt_tokens+ idx += 1++ if idx == batch_size:+ yield enc_batch.copy(), dec_in_batch.copy(), dec_tgt_batch.copy()+ enc_batch[:] = pad_id+ dec_in_batch[:] = pad_id+ dec_tgt_batch[:] = pad_id+ idx = 0+++class _PrefetchStream:+ """Prefetch streaming batches in a background thread."""++ def __init__(self, generator_fn, prefetch=4):+ self._queue = queue.Queue(maxsize=prefetch)+ self._stop = threading.Event()+ self._thread = threading.Thread(target=self._produce, args=(generator_fn,), daemon=True)+ self._thread.start()++ def _produce(self, gen_fn):+ try:+ for batch in gen_fn():+ if self._stop.is_set():+ return+ self._queue.put(batch)+ self._queue.put(None)+ except Exception as e:+ self._queue.put(e)++ def __iter__(self):+ return self++ def __next__(self):+ item = self._queue.get()+ if item is None:+ raise StopIteration+ if isinstance(item, Exception):+ raise item+ return item++ def close(self):+ self._stop.set()+++def _pretrain_step(state, src, tgt_in, tgt_out, rng):+ """Simple CE loss train step for pretraining. No QAT, no contrastive, no pruning."""++ def loss_fn(params):+ src_mask = make_padding_mask(src, PAD_ID)+ tgt_mask = make_causal_mask(tgt_in.shape[1]) & make_padding_mask(tgt_in, PAD_ID)+ cross_mask = src_mask++ logits = state.apply_fn(+ {"params": params}, src, tgt_in,+ src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,+ )+ logits_f32 = logits.astype(jnp.float32)++ padding_mask = (tgt_out != PAD_ID).astype(jnp.float32)+ num_tokens = jnp.maximum(jnp.sum(padding_mask), 1.0)+ ce = jnp.sum(+ optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * padding_mask+ ) / num_tokens+ z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)+ return ce + z_loss++ loss, grads = jax.value_and_grad(loss_fn)(state.params)+ grads = jax.lax.pmean(grads, axis_name="batch")+ loss = jax.lax.pmean(loss, axis_name="batch")+ state = state.apply_gradients(grads=grads)+ return state, loss+++_p_pretrain_step = None+++def _make_p_pretrain_step():+ return jax.pmap(_pretrain_step, axis_name="batch", donate_argnums=(0,))+++def pretrain(args):+ num_devices = jax.local_device_count()+ num_hosts = jax.process_count()+ host_id = jax.process_index()+ total_devices = jax.device_count()+ is_main = host_id == 0++ experiment_name = getattr(args, "name", "pretrain")+ use_wandb = getattr(args, "wandb", False)+ if use_wandb and is_main:+ import wandb+ if wandb.run is None:+ wandb.init(project="needle-v1", name=experiment_name, config=vars(args))++ print(f"\n[1/3] Detecting devices...")+ print(f" {num_devices} local, {total_devices} total across {num_hosts} hosts")++ print(f"\n[2/3] Loading tokenizer...")+ tokenizer = get_tokenizer()++ config = TransformerConfig(+ d_model=args.d_model,+ num_heads=args.num_heads,+ num_kv_heads=getattr(args, "num_kv_heads", None) or args.num_heads,+ num_encoder_layers=args.num_layers,+ num_decoder_layers=getattr(args, "num_dec_layers", args.num_layers),+ d_ff=getattr(args, "d_ff", None) or args.d_model * 4,+ max_seq_len=max(args.max_enc_len, args.max_dec_len),+ dtype=args.dtype,+ activation=getattr(args, "activation", "swiglu"),+ no_feedforward=getattr(args, "no_feedforward", True),+ contrastive_dim=getattr(args, "contrastive_dim", 128),+ enable_speech=False,+ )++ effective_batch_size = args.batch_size * num_devices+ global_batch_size = effective_batch_size * num_hosts++ # Estimate total steps: 80M rows / global_batch_size+ estimated_rows = 80_000_000+ max_steps = getattr(args, "max_steps", None)+ estimated_steps = estimated_rows // global_batch_size+ if max_steps:+ estimated_steps = min(estimated_steps, max_steps)+ total_steps = estimated_steps * args.epochs+ warmup_steps = max(1, int(total_steps * args.warmup_ratio))++ scaled_lr = args.lr * total_devices+ muon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(total_devices)+ decay_ratio = getattr(args, "decay_ratio", 0.05)++ print(f"\n[3/3] Initializing model...")+ rng = jax.random.PRNGKey(args.seed)+ rng, init_rng = jax.random.split(rng)+ state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps, decay_ratio)+ state = _replicate(state)++ param_count = sum(x.size for x in jax.tree.leaves(_unreplicate(state).params))++ global _p_pretrain_step+ _p_pretrain_step = _make_p_pretrain_step()++ save_every = getattr(args, "save_every", 1000)+ checkpoint_dir = getattr(args, "checkpoint_dir", "checkpoints")+ os.makedirs(checkpoint_dir, exist_ok=True)++ if is_main:+ decay_steps = max(1, int(total_steps * decay_ratio))+ stable_steps = total_steps - warmup_steps - decay_steps+ print(f"\n ─────────────────────────────────────")+ print(f" Pretraining on PleIAs/SYNTH")+ print(f" ─────────────────────────────────────")+ print(f" Parameters {param_count:>12,}")+ print(f" d_model {config.d_model:>12}")+ print(f" Heads {config.num_heads:>7} ({config.num_kv_heads} KV)")+ print(f" Layers {config.num_encoder_layers:>7} enc / {config.num_decoder_layers} dec")+ print(f" Dtype {config.dtype:>12}")+ print(f" ─────────────────────────────────────")+ print(f" Hosts {num_hosts:>12}")+ print(f" Devices {num_devices:>5}/host, {total_devices} total")+ print(f" Batch {args.batch_size:>7} x {total_devices} = {global_batch_size}")+ print(f" Adam LR {args.lr:>7} x {total_devices} = {scaled_lr}")+ print(f" Muon LR {getattr(args, 'muon_lr', 0.02):>7.4f} -> {muon_lr:.4f}")+ print(f" Schedule {warmup_steps}w / {stable_steps}s / {decay_steps}d (WSD)")+ print(f" Est. steps {estimated_steps:>12,}")+ print(f" Save every {save_every:>12,}")+ print(f" ─────────────────────────────────────\n")++ global_step = 0++ for epoch in range(args.epochs):+ batch_stream = _PrefetchStream(+ lambda: _stream_batches(tokenizer, global_batch_size,+ args.max_enc_len, args.max_dec_len,+ seed=args.seed + epoch),+ prefetch=8,+ )++ pbar = tqdm(desc=f"Pretrain epoch {epoch + 1}/{args.epochs}",+ total=estimated_steps, disable=not is_main)++ for batch in batch_stream:+ if max_steps and global_step >= max_steps:+ break++ src, tgt_in, tgt_out = batch+ t0 = time.perf_counter()++ # Slice this host's portion+ host_slice = slice(host_id * effective_batch_size,+ (host_id + 1) * effective_batch_size)+ src = src[host_slice]+ tgt_in = tgt_in[host_slice]+ tgt_out = tgt_out[host_slice]++ # Shard for pmap+ src_b = shard_batch(src, num_devices)+ tgt_in_b = shard_batch(tgt_in, num_devices)+ tgt_out_b = shard_batch(tgt_out, num_devices)++ rng, step_rng = jax.random.split(rng)+ step_rngs = jax.random.split(step_rng, num_devices)++ state, loss = _p_pretrain_step(state, src_b, tgt_in_b, tgt_out_b, step_rngs)++ loss_val = float(loss.addressable_shards[0].data[0])+ ppl = math.exp(min(loss_val, 20))+ dt = time.perf_counter() - t0+ global_step += 1++ pbar.update(1)+ pbar.set_postfix(loss=f"{loss_val:.4f}", ppl=f"{ppl:.2f}",+ tok_s=f"{global_batch_size * (args.max_enc_len + args.max_dec_len) / dt:.0f}")++ if use_wandb and is_main:+ import wandb+ wandb.log({+ "pretrain/loss": loss_val,+ "pretrain/ppl": ppl,+ "pretrain/step": global_step,+ "pretrain/tokens_per_sec": global_batch_size * (args.max_enc_len + args.max_dec_len) / dt,+ })++ # Save checkpoint every N steps+ if is_main and global_step % save_every == 0:+ _save_pretrain_checkpoint(state, config, checkpoint_dir, global_step)++ batch_stream.close()+ pbar.close()++ # Final save+ if is_main:+ ckpt_path = _save_pretrain_checkpoint(state, config, checkpoint_dir, global_step)+ print(f"\nPretraining complete. {global_step} steps.")+ print(f"Base checkpoint: {ckpt_path}")++ if use_wandb and is_main:+ import wandb+ wandb.finish()++ # Sync all hosts before exit+ jax.experimental.multihost_utils.sync_global_devices("pretrain_done")+++def _save_pretrain_checkpoint(state, config, checkpoint_dir, global_step):+ """Save and upload needle_base.pkl."""+ params = _unreplicate(state).params+ params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), params)++ ckpt_path = os.path.join(checkpoint_dir, "needle_base.pkl")+ with open(ckpt_path, "wb") as f:+ pickle.dump({"params": params_np, "config": config.__dict__}, f)++ param_count = sum(x.size for x in jax.tree.leaves(params_np))+ size_mb = sum(x.nbytes for x in jax.tree.leaves(params_np)) / 1e6+ print(f"\n [step {global_step}] Saved {ckpt_path} ({param_count:,} params, {size_mb:.1f} MB)")++ _upload_checkpoint(ckpt_path)+ return ckpt_pathdiff --git a/src/quantize.py b/src/quantize.pynew file mode 100644index 0000000..6f86eb8--- /dev/null+++ b/src/quantize.py@@ -0,0 +1,133 @@+import jax+import jax.numpy as jnp+++def _fake_quantize_int4(w, group_size=32):+ """Symmetric group-wise INT4 fake quantization with STE.++ Divides the input dimension (axis 0) into groups of `group_size` elements,+ each with its own scale factor. Falls back to per-channel if in_features < group_size.+ """+ in_feat, out_feat = w.shape+ gs = min(group_size, in_feat)++ pad = (gs - in_feat % gs) % gs+ if pad > 0:+ w_padded = jnp.pad(w, ((0, pad), (0, 0)))+ else:+ w_padded = w++ num_groups = w_padded.shape[0] // gs+ w_grouped = w_padded.reshape(num_groups, gs, out_feat)++ scale = jnp.max(jnp.abs(w_grouped), axis=1, keepdims=True) / 7.0+ scale = jnp.maximum(scale, 1e-8)+ w_q = jnp.clip(jnp.round(w_grouped / scale), -8, 7) * scale++ w_q = w_q.reshape(-1, out_feat)[:in_feat]++ return w + jax.lax.stop_gradient(w_q - w)+++def _fake_quantize_int8(w, group_size=32):+ """Symmetric group-wise INT8 fake quantization with STE.++ Same structure as INT4 but with [-128, 127] range (8-bit signed).+ """+ in_feat, out_feat = w.shape+ gs = min(group_size, in_feat)++ pad = (gs - in_feat % gs) % gs+ if pad > 0:+ w_padded = jnp.pad(w, ((0, pad), (0, 0)))+ else:+ w_padded = w++ num_groups = w_padded.shape[0] // gs+ w_grouped = w_padded.reshape(num_groups, gs, out_feat)++ scale = jnp.max(jnp.abs(w_grouped), axis=1, keepdims=True) / 127.0+ scale = jnp.maximum(scale, 1e-8)+ w_q = jnp.clip(jnp.round(w_grouped / scale), -128, 127) * scale++ w_q = w_q.reshape(-1, out_feat)[:in_feat]++ return w + jax.lax.stop_gradient(w_q - w)+++def _quantize_params(params, group_size=32, precision="int4"):+ """Fake-quantize all Dense kernels in the param tree."""+ qfn = _fake_quantize_int8 if precision == "int8" else _fake_quantize_int4+ def _maybe_quantize(path, leaf):+ name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])+ if name == "kernel" and leaf.ndim == 3:+ return jax.vmap(lambda w: qfn(w, group_size=group_size), in_axes=(0,))(leaf)+ if name == "kernel" and leaf.ndim == 2:+ return qfn(leaf, group_size=group_size)+ return leaf+ return jax.tree_util.tree_map_with_path(_maybe_quantize, params)+++def _cubic_sparsity_schedule(step, t_start, t_end, s_final):+ """Cubic sparsity ramp (Zhu & Gupta 2017). Returns target sparsity at *step*."""+ if step < t_start:+ return 0.0+ if step >= t_end:+ return s_final+ frac = (step - t_start) / (t_end - t_start)+ return s_final * (1.0 - (1.0 - frac) ** 3)+++def _make_prune_mask(params, sparsity, group_size):+ """Compute block-prune mask entirely on-device (no numpy round-trips).++ Returns binary mask tree matching param shapes and dtypes.+ Handles both 2D (in, out) and 3D (num_layers, in, out) kernels from nn.scan.+ """+ all_scores = []+ for _, leaf in jax.tree_util.tree_leaves_with_path(params):+ if leaf.ndim == 2:+ in_feat, out_feat = leaf.shape+ gs = min(group_size, in_feat)+ pad = (gs - in_feat % gs) % gs+ w = jnp.pad(leaf, ((0, pad), (0, 0))) if pad else leaf+ scores = jnp.sum(jnp.abs(w.reshape(-1, gs, out_feat)), axis=1).ravel()+ all_scores.append(scores)+ elif leaf.ndim == 3:+ num_layers, in_feat, out_feat = leaf.shape+ gs = min(group_size, in_feat)+ pad = (gs - in_feat % gs) % gs+ for li in range(num_layers):+ w = jnp.pad(leaf[li], ((0, pad), (0, 0))) if pad else leaf[li]+ scores = jnp.sum(jnp.abs(w.reshape(-1, gs, out_feat)), axis=1).ravel()+ all_scores.append(scores)++ if not all_scores:+ return jax.tree.map(jnp.ones_like, params)++ threshold = jnp.percentile(jnp.concatenate(all_scores), sparsity * 100)++ def _leaf_mask(path, leaf):+ if leaf.ndim == 3:+ num_layers, in_feat, out_feat = leaf.shape+ gs = min(group_size, in_feat)+ pad = (gs - in_feat % gs) % gs+ masks = []+ for li in range(num_layers):+ w = jnp.pad(leaf[li], ((0, pad), (0, 0))) if pad else leaf[li]+ w_grouped = w.reshape(-1, gs, out_feat)+ block_keep = (jnp.sum(jnp.abs(w_grouped), axis=1, keepdims=True) > threshold)+ m = jnp.broadcast_to(block_keep, w_grouped.shape).reshape(-1, out_feat)[:in_feat]+ masks.append(m)+ return jnp.stack(masks, axis=0).astype(leaf.dtype)+ if leaf.ndim != 2:+ return jnp.ones_like(leaf)+ in_feat, out_feat = leaf.shape+ gs = min(group_size, in_feat)+ pad = (gs - in_feat % gs) % gs+ w = jnp.pad(leaf, ((0, pad), (0, 0))) if pad else leaf+ w_grouped = w.reshape(-1, gs, out_feat)+ block_keep = (jnp.sum(jnp.abs(w_grouped), axis=1, keepdims=True) > threshold)+ return jnp.broadcast_to(block_keep, w_grouped.shape).reshape(-1, out_feat)[:in_feat].astype(leaf.dtype)++ return jax.tree_util.tree_map_with_path(_leaf_mask, params)diff --git a/src/tokenizer.py b/src/tokenizer.pynew file mode 100644index 0000000..79991dd--- /dev/null+++ b/src/tokenizer.py@@ -0,0 +1,161 @@+import os+import re as _re++import sentencepiece as spm+from tqdm import tqdm++_PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))+TOKENIZER_DIR = os.path.join(_PROJECT_ROOT, "tokenizer")+TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")++PAD_ID = 0+EOS_ID = 1+BOS_ID = 2+UNK_ID = 3+TOOL_CALL_ID = 4+TOOLS_ID = 5++DEFAULT_MAX_ENC_LEN = 1024+DEFAULT_MAX_DEC_LEN = 512+DEFAULT_MAX_GEN_LEN = 512++_HF_TOKENIZER_REPO = "Cactus-Compute/needle-tokenizer"+++def to_snake_case(name):+ """Convert camelCase, PascalCase, or dot.notation name to snake_case."""+ # Replace any non-alphanumeric/underscore characters with underscores+ s = _re.sub(r'[^a-zA-Z0-9_]+', '_', name)+ # Insert underscore before uppercase letters that follow lowercase/digits+ s = _re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', s)+ # Insert underscore between consecutive uppercase and uppercase+lowercase+ s = _re.sub(r'([A-Z]+)([A-Z][a-z])', r'\1_\2', s)+ # Collapse multiple underscores and strip edges+ s = _re.sub(r'_+', '_', s)+ return s.lower().strip('_')+++class NeedleTokenizer:+ """Wrapper around SentencePiece providing the interface the codebase expects."""++ def __init__(self, model_path):+ self.sp = spm.SentencePieceProcessor()+ self.sp.Load(model_path)++ @property+ def pad_token_id(self):+ return PAD_ID++ @property+ def eos_token_id(self):+ return EOS_ID++ @property+ def bos_token_id(self):+ return BOS_ID++ @property+ def tool_call_token_id(self):+ return TOOL_CALL_ID++ @property+ def tools_token_id(self):+ return TOOLS_ID++ @property+ def vocab_size(self):+ return self.sp.GetPieceSize()++ def encode(self, text):+ return self.sp.Encode(text, out_type=int)++ def decode(self, ids):+ if isinstance(ids, (list, tuple)) and len(ids) > 0 and isinstance(ids[0], (list, tuple)):+ return [self.sp.Decode(seq) for seq in ids]+ return self.sp.Decode(list(ids))++ def __call__(self, texts, truncation=True, max_length=None, **kwargs):+ all_ids = []+ for text in texts:+ ids = self.sp.Encode(text, out_type=int)+ if truncation and max_length:+ ids = ids[:max_length]+ all_ids.append(ids)+ return {"input_ids": all_ids}+++def _download_tokenizer_from_hf():+ """Download tokenizer files from HuggingFace Hub into TOKENIZER_DIR."""+ from huggingface_hub import snapshot_download++ local = snapshot_download(+ _HF_TOKENIZER_REPO,+ repo_type="dataset",+ )+ os.makedirs(TOKENIZER_DIR, exist_ok=True)+ for fname in os.listdir(local):+ if fname.startswith("."):+ continue+ dst = os.path.join(TOKENIZER_DIR, fname)+ if not os.path.exists(dst):+ os.symlink(os.path.join(local, fname), dst)+++def get_tokenizer(max_samples=None):+ model_path = TOKENIZER_PREFIX + ".model"+ if not os.path.exists(model_path):+ print("Downloading pretrained tokenizer from HuggingFace...")+ _download_tokenizer_from_hf()+ return NeedleTokenizer(model_path)+++def train_tokenizer(vocab_size=8192, max_samples=None, force=False):+ """Train a SentencePiece BPE tokenizer on tool-calling corpus."""+ model_path = TOKENIZER_PREFIX + ".model"+ if os.path.exists(model_path) and not force:+ print(f"Tokenizer already exists at {model_path}")+ return model_path++ os.makedirs(TOKENIZER_DIR, exist_ok=True)++ from datasets import concatenate_datasets+ from .data import _load_split_dataset++ train_ds = _load_split_dataset("train")+ val_ds = _load_split_dataset("validation")+ ds = concatenate_datasets([train_ds, val_ds])+ if max_samples:+ ds = ds.select(range(min(max_samples, len(ds))))++ print(f"Training SentencePiece BPE tokenizer (vocab_size={vocab_size}, samples={len(ds):,})...")++ corpus_path = os.path.join(TOKENIZER_DIR, "corpus.txt")+ try:+ with open(corpus_path, "w") as f:+ for example in tqdm(ds, desc="Writing corpus"):+ for field in ("query", "tools", "answers"):+ text = example[field].strip()+ if text:+ f.write(text + "\n")++ spm.SentencePieceTrainer.Train(+ input=corpus_path,+ model_prefix=TOKENIZER_PREFIX,+ vocab_size=vocab_size,+ model_type="bpe",+ pad_id=PAD_ID,+ eos_id=EOS_ID,+ bos_id=BOS_ID,+ unk_id=UNK_ID,+ user_defined_symbols=["<tool_call>", "<tools>"],+ byte_fallback=True,+ normalization_rule_name="identity",+ num_threads=min(128, max(1, (os.cpu_count() or 1) * 3 // 4)),+ train_extremely_large_corpus=False,+ minloglevel=2,+ )+ finally:+ if os.path.exists(corpus_path):+ os.remove(corpus_path)+ print(f"Tokenizer saved to {model_path}")+ return model_pathdiff --git a/src/tpu.py b/src/tpu.pyindex d4dbd6c..5864d57 100644--- a/src/tpu.py+++ b/src/tpu.py@@ -56,6 +56,7 @@ TPU_HELP = """needle tpu setup NAME [--zone ZONE] setup all workers (multi-host)needle tpu sync NAME [--zone ZONE] sync code only (fast, no venv rebuild)needle tpu train NAME [--zone ZONE] launch training on all workers+ needle tpu pretrain NAME [--zone ZONE] launch pretraining on all workersneedle tpu claude NAME [--zone ZONE]needle tpu stop NAME [--zone ZONE]needle tpu start NAME [--zone ZONE]@@ -554,20 +555,20 @@ def _collect_env_exports():return " && ".join(exports)-def tpu_train(args):- """Launch training on all workers of a multi-host TPU."""+def _tpu_run_command(args, needle_cmd="train"):+ """Launch a needle command on all workers of a multi-host TPU."""zone = args.zone or _detect_zone(args.name)num_workers = _get_num_workers(args.name, zone)if num_workers > 1:print(f"[tpu] Killing stale processes on all workers...")_run_all_workers(args.name, zone,- "sudo pkill -9 -f [n]eedle.train || true")+ "sudo pkill -9 -f [n]eedle || true")extra = getattr(args, "train_args", [])if extra and extra[0] == "--":extra = extra[1:]- train_cmd = "cd ~/needle && .venv/bin/needle train"+ train_cmd = f"cd ~/needle && .venv/bin/needle {needle_cmd}"if extra:train_cmd += " " + " ".join(extra)@@ -579,9 +580,8 @@ def tpu_train(args):train_cmd = env_exports + " && " + train_cmdif num_workers <= 1:- print(f"[tpu] Single-host TPU. Running training directly...")- # Use quiet=True to avoid printing tokens, then log sanitized version- print(f"[tpu] $ gcloud compute tpus tpu-vm ssh {args.name} --command 'needle train ...'")+ print(f"[tpu] Single-host TPU. Running directly...")+ print(f"[tpu] $ gcloud ... --command 'needle {needle_cmd} ...'")_run(["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,"--zone", zone, "--project", PROJECT,@@ -590,8 +590,7 @@ def tpu_train(args):)return- print(f"[tpu] Launching training on {num_workers} workers...")- # Run with quiet=True to avoid printing tokens in command log+ print(f"[tpu] Launching 'needle {needle_cmd}' on {num_workers} workers...")result = _run(["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,"--zone", zone, "--project", PROJECT,@@ -600,10 +599,20 @@ def tpu_train(args):check=False, quiet=True,)if result.returncode != 0:- print("[tpu] WARNING: training exited with errors on some workers.",+ print(f"[tpu] WARNING: 'needle {needle_cmd}' exited with errors on some workers.",file=sys.stderr)+def tpu_train(args):+ """Launch 'needle train' on all workers."""+ _tpu_run_command(args, needle_cmd="train")+++def tpu_pretrain(args):+ """Launch 'needle pretrain' on all workers."""+ _tpu_run_command(args, needle_cmd="pretrain")++def tpu_dispatch(args):actions = {"create": tpu_create,@@ -611,6 +620,7 @@ def tpu_dispatch(args):"setup": tpu_setup,"sync": tpu_sync,"train": tpu_train,+ "pretrain": tpu_pretrain,"claude": tpu_claude,"stop": tpu_stop,"start": tpu_start,diff --git a/src/train.py b/src/train.pyindex 3713b90..e9cc7f5 100644--- a/src/train.py+++ b/src/train.py@@ -2,14 +2,12 @@ import mathimport osimport pickleimport time-from typing import NamedTupleimport jaximport jax.numpy as jnpimport numpy as npimport optaxfrom tqdm import tqdm-from flax.training import train_statefrom .data import (get_batches, get_tokenizer,@@ -24,302 +22,11 @@ from .model import (make_causal_packing_mask,make_cross_packing_mask,)--_HF_CHECKPOINT_REPO = "Cactus-Compute/checkpoints"---def _replicate(tree):- """Replicate a pytree across all local devices (multi-host safe)."""- devices = jax.local_devices()- return jax.tree.map(- lambda x: jax.device_put_replicated(x, devices), tree)---def _unreplicate(tree):- """Get a single copy from a replicated pytree (multi-host safe).-- Uses addressable_shards instead of x[0] indexing, which fails- in multi-host pmap because the array spans devices on other hosts.- Each shard has shape (1, ...) — we index [0] to strip the leading dim.- """- def _get_first(x):- if hasattr(x, 'addressable_shards') and x.addressable_shards:- shard = x.addressable_shards[0].data- # Shard has shape (1, ...) from the pmap partition; strip it- return jax.device_get(shard[0]) if shard.ndim > 0 else jax.device_get(shard)- return x- return jax.tree.map(_get_first, tree)---def _upload_checkpoint(ckpt_path):- """Upload a checkpoint file to HuggingFace Hub in a background thread."""- import threading-- def _upload():- try:- from huggingface_hub import HfApi- api = HfApi()- api.create_repo(_HF_CHECKPOINT_REPO, repo_type="model", private=True, exist_ok=True)- filename = os.path.basename(ckpt_path)- print(f"[hf] Uploading {filename} to {_HF_CHECKPOINT_REPO} ...")- api.upload_file(- path_or_fileobj=ckpt_path,- path_in_repo=filename,- repo_id=_HF_CHECKPOINT_REPO,- repo_type="model",- )- print(f"[hf] Checkpoint uploaded: {_HF_CHECKPOINT_REPO}/{filename}")- except Exception as e:- print(f"[hf] Warning: checkpoint upload failed: {e}")-- threading.Thread(target=_upload, daemon=True).start()---def _newton_schulz(G, steps=5):- """Approximate polar decomposition via Newton-Schulz iteration."""- a, b, c = 3.4445, -4.7750, 2.0315- orig_dtype = G.dtype- G = G.astype(jnp.float32)- X = G / (jnp.linalg.norm(G) + 1e-7)- transposed = G.shape[0] > G.shape[1]- if transposed:- X = X.T- for _ in range(steps):- A = X @ X.T- B = b * A + c * A @ A- X = a * X + B @ X- if transposed:- X = X.T- return X.astype(orig_dtype)---class MuonState(NamedTuple):- mu: optax.Updates---def scale_by_muon(momentum=0.95, ns_steps=5):- """Muon gradient transform: orthogonalize 2D+ grads, then Nesterov momentum."""-- def init_fn(params):- return MuonState(mu=jax.tree.map(jnp.zeros_like, params))-- def update_fn(updates, state, params=None):- del params-- def ortho(g):- if g.ndim == 3:- return jax.vmap(_newton_schulz, in_axes=(0,))(g)- if g.ndim == 2:- return _newton_schulz(g, steps=ns_steps)- return g-- ortho_g = jax.tree.map(ortho, updates)- new_mu = jax.tree.map(lambda m, g: momentum * m + g, state.mu, ortho_g)- new_updates = jax.tree.map(- lambda g, m: g + momentum * m, ortho_g, new_mu- )- return new_updates, MuonState(mu=new_mu)-- return optax.GradientTransformation(init_fn, update_fn)---def _param_labels(params):- """Label each param: 'muon' for Dense kernels, 'adam' for the rest."""-- def _label(path, leaf):- name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])- if name == "kernel" and leaf.ndim in (2, 3):- return "muon"- return "adam"-- return jax.tree_util.tree_map_with_path(_label, params)---def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.15):- """Warmup-Stable-Decay schedule: linear warmup, hold peak, cosine decay."""- decay_steps = max(1, int(total_steps * decay_ratio))- stable_steps = total_steps - warmup_steps - decay_steps- return optax.join_schedules(- [- optax.linear_schedule(0.0, peak_value, warmup_steps),- optax.constant_schedule(peak_value),- optax.cosine_decay_schedule(peak_value, decay_steps, alpha=0.05),- ],- boundaries=[warmup_steps, warmup_steps + stable_steps],- )---def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps, decay_ratio=0.15):- model = EncoderDecoderTransformer(config)-- rng, init_rng = jax.random.split(rng)- dummy_src = jnp.ones((1, 128), dtype=jnp.int32)- dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)- init_args = [dummy_src, dummy_tgt]- if config.enable_speech:- init_args.append(jnp.ones((1, 128, config.n_mels), dtype=jnp.float32))- variables = model.init(- {"params": init_rng},- *init_args,- method="init_all",- )-- adam_schedule = _wsd_schedule(learning_rate, total_steps, warmup_steps, decay_ratio)- muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps, decay_ratio)-- muon_opt = optax.chain(- scale_by_muon(momentum=0.95, ns_steps=5),- optax.add_decayed_weights(weight_decay=0.01),- optax.scale_by_schedule(muon_schedule),- optax.scale(-1.0),- )- adam_opt = optax.chain(- optax.adamw(adam_schedule, b2=0.95, weight_decay=0.0),- )-- tx = optax.chain(- optax.clip_by_global_norm(1.0),- optax.multi_transform(- {"muon": muon_opt, "adam": adam_opt},- _param_labels,- ),- )- return train_state.TrainState.create(- apply_fn=model.apply,- params=variables["params"],- tx=tx,- )---def _fake_quantize_int4(w, group_size=32):- """Symmetric group-wise INT4 fake quantization with STE.-- Divides the input dimension (axis 0) into groups of `group_size` elements,- each with its own scale factor. Falls back to per-channel if in_features < group_size.- """- in_feat, out_feat = w.shape- gs = min(group_size, in_feat)-- pad = (gs - in_feat % gs) % gs- if pad > 0:- w_padded = jnp.pad(w, ((0, pad), (0, 0)))- else:- w_padded = w-- num_groups = w_padded.shape[0] // gs- w_grouped = w_padded.reshape(num_groups, gs, out_feat)-- scale = jnp.max(jnp.abs(w_grouped), axis=1, keepdims=True) / 7.0- scale = jnp.maximum(scale, 1e-8)- w_q = jnp.clip(jnp.round(w_grouped / scale), -8, 7) * scale-- w_q = w_q.reshape(-1, out_feat)[:in_feat]-- return w + jax.lax.stop_gradient(w_q - w)---def _fake_quantize_int8(w, group_size=32):- """Symmetric group-wise INT8 fake quantization with STE.-- Same structure as INT4 but with [-128, 127] range (8-bit signed).- """- in_feat, out_feat = w.shape- gs = min(group_size, in_feat)-- pad = (gs - in_feat % gs) % gs- if pad > 0:- w_padded = jnp.pad(w, ((0, pad), (0, 0)))- else:- w_padded = w-- num_groups = w_padded.shape[0] // gs- w_grouped = w_padded.reshape(num_groups, gs, out_feat)-- scale = jnp.max(jnp.abs(w_grouped), axis=1, keepdims=True) / 127.0- scale = jnp.maximum(scale, 1e-8)- w_q = jnp.clip(jnp.round(w_grouped / scale), -128, 127) * scale-- w_q = w_q.reshape(-1, out_feat)[:in_feat]-- return w + jax.lax.stop_gradient(w_q - w)---def _cubic_sparsity_schedule(step, t_start, t_end, s_final):- """Cubic sparsity ramp (Zhu & Gupta 2017). Returns target sparsity at *step*."""- if step < t_start:- return 0.0- if step >= t_end:- return s_final- frac = (step - t_start) / (t_end - t_start)- return s_final * (1.0 - (1.0 - frac) ** 3)---def _make_prune_mask(params, sparsity, group_size):- """Compute block-prune mask entirely on-device (no numpy round-trips).-- Returns binary mask tree matching param shapes and dtypes.- Handles both 2D (in, out) and 3D (num_layers, in, out) kernels from nn.scan.- """- all_scores = []- for _, leaf in jax.tree_util.tree_leaves_with_path(params):- if leaf.ndim == 2:- in_feat, out_feat = leaf.shape- gs = min(group_size, in_feat)- pad = (gs - in_feat % gs) % gs- w = jnp.pad(leaf, ((0, pad), (0, 0))) if pad else leaf- scores = jnp.sum(jnp.abs(w.reshape(-1, gs, out_feat)), axis=1).ravel()- all_scores.append(scores)- elif leaf.ndim == 3:- num_layers, in_feat, out_feat = leaf.shape- gs = min(group_size, in_feat)- pad = (gs - in_feat % gs) % gs- for li in range(num_layers):- w = jnp.pad(leaf[li], ((0, pad), (0, 0))) if pad else leaf[li]- scores = jnp.sum(jnp.abs(w.reshape(-1, gs, out_feat)), axis=1).ravel()- all_scores.append(scores)-- if not all_scores:- return jax.tree.map(jnp.ones_like, params)-- threshold = jnp.percentile(jnp.concatenate(all_scores), sparsity * 100)-- def _leaf_mask(path, leaf):- if leaf.ndim == 3:- num_layers, in_feat, out_feat = leaf.shape- gs = min(group_size, in_feat)- pad = (gs - in_feat % gs) % gs- masks = []- for li in range(num_layers):- w = jnp.pad(leaf[li], ((0, pad), (0, 0))) if pad else leaf[li]- w_grouped = w.reshape(-1, gs, out_feat)- block_keep = (jnp.sum(jnp.abs(w_grouped), axis=1, keepdims=True) > threshold)- m = jnp.broadcast_to(block_keep, w_grouped.shape).reshape(-1, out_feat)[:in_feat]- masks.append(m)- return jnp.stack(masks, axis=0).astype(leaf.dtype)- if leaf.ndim != 2:- return jnp.ones_like(leaf)- in_feat, out_feat = leaf.shape- gs = min(group_size, in_feat)- pad = (gs - in_feat % gs) % gs- w = jnp.pad(leaf, ((0, pad), (0, 0))) if pad else leaf- w_grouped = w.reshape(-1, gs, out_feat)- block_keep = (jnp.sum(jnp.abs(w_grouped), axis=1, keepdims=True) > threshold)- return jnp.broadcast_to(block_keep, w_grouped.shape).reshape(-1, out_feat)[:in_feat].astype(leaf.dtype)-- return jax.tree_util.tree_map_with_path(_leaf_mask, params)---def _quantize_params(params, group_size=32, precision="int4"):- """Fake-quantize all Dense kernels in the param tree."""- qfn = _fake_quantize_int8 if precision == "int8" else _fake_quantize_int4- def _maybe_quantize(path, leaf):- name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])- if name == "kernel" and leaf.ndim == 3:- return jax.vmap(lambda w: qfn(w, group_size=group_size), in_axes=(0,))(leaf)- if name == "kernel" and leaf.ndim == 2:- return qfn(leaf, group_size=group_size)- return leaf- return jax.tree_util.tree_map_with_path(_maybe_quantize, params)+from .optim import create_train_state, _wsd_schedule+from .distributed import _replicate, _unreplicate, shard_batch, _upload_checkpoint+from .quantize import (+ _quantize_params, _cubic_sparsity_schedule, _make_prune_mask,+)_GROUP_SIZE = 32@@ -515,9 +222,6 @@ def _estimate_mat_params(config, matryoshka_factor):return int(total)-def shard_batch(batch, num_devices):- """Reshape a batch array so leading dim is (num_devices, per_device_batch, ...)."""- return batch.reshape(num_devices, -1, *batch.shape[1:])def train(args):