Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

add pretrain

HenryNdubuakud964cdfcf6eb
diff --git a/src/cli.py b/src/cli.py
index f967292..10cdfc8 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -1,4 +1,5 @@
import argparse
+import os
import sys
from .data import DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_DEC_LEN, DEFAULT_MAX_GEN_LEN
@@ -73,6 +74,33 @@ def main():
p.add_argument("--calibrate-k", type=float, default=5.0,
help="Sigmoid steepness for PPL→confidence mapping (default: 5.0)")
+ p = sub.add_parser("pretrain", add_help=False)
+ p.add_argument("--name", type=str, default="pretrain",
+ help="Experiment name for wandb (default: pretrain)")
+ p.add_argument("--epochs", type=int, default=1)
+ p.add_argument("--batch-size", type=int, default=128)
+ p.add_argument("--lr", type=float, default=3e-4)
+ p.add_argument("--muon-lr", type=float, default=0.02)
+ p.add_argument("--d-model", type=int, default=512)
+ p.add_argument("--num-heads", type=int, default=8)
+ p.add_argument("--num-kv-heads", type=int, default=4)
+ p.add_argument("--num-layers", type=int, default=12)
+ p.add_argument("--num-dec-layers", type=int, default=8)
+ p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)
+ p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)
+ p.add_argument("--warmup-ratio", type=float, default=0.05)
+ p.add_argument("--decay-ratio", type=float, default=0.05)
+ p.add_argument("--wandb", action="store_true")
+ p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])
+ p.add_argument("--seed", type=int, default=42)
+ p.add_argument("--max-steps", type=int, default=None,
+ help="Stop after N steps (default: full epoch)")
+ p.add_argument("--save-every", type=int, default=1000,
+ help="Save and upload checkpoint every N steps (default: 1000)")
+ p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
+ p.add_argument("--activation", type=str, default="swiglu", choices=["drelu", "swiglu", "geglu"])
+ p.add_argument("--no-feedforward", action=argparse.BooleanOptionalAction, default=True)
+
p = sub.add_parser("tokenize", add_help=False)
p.add_argument("--max-samples", type=int, default=None,
help="Limit samples per split (for dev/test)")
@@ -179,6 +207,12 @@ def main():
tp.add_argument("train_args", nargs=argparse.REMAINDER,
help="Extra args passed to needle train")
+ tp = tpu_sub.add_parser("pretrain", add_help=False)
+ tp.add_argument("name", type=str)
+ tp.add_argument("--zone", type=str, default=None)
+ tp.add_argument("train_args", nargs=argparse.REMAINDER,
+ help="Extra args passed to needle pretrain")
+
tp = tpu_sub.add_parser("claude", add_help=False)
tp.add_argument("name", type=str)
tp.add_argument("--zone", type=str, default=None)
@@ -206,9 +240,16 @@ def main():
if args.command == "tokenize":
from .tokenize_data import tokenize
tokenize(args)
+ elif args.command == "pretrain":
+ import jax
+ if os.path.exists("/dev/accel0"):
+ jax.distributed.initialize()
+ from .pretrain import pretrain
+ pretrain(args)
elif args.command == "train":
import jax
- jax.distributed.initialize()
+ if os.path.exists("/dev/accel0"):
+ jax.distributed.initialize()
from .train import train
train(args)
elif args.command == "run":
diff --git a/src/data.py b/src/data.py
index 9c0c643..1e0b23d 100644
--- a/src/data.py
+++ b/src/data.py
@@ -16,6 +16,16 @@ from datasets import Audio, Dataset, load_from_disk
from tqdm import tqdm
import sentencepiece as spm
+from .tokenizer import (
+ PAD_ID, EOS_ID, BOS_ID, UNK_ID, TOOL_CALL_ID, TOOLS_ID,
+ DEFAULT_MAX_ENC_LEN, DEFAULT_MAX_DEC_LEN, DEFAULT_MAX_GEN_LEN,
+ to_snake_case, NeedleTokenizer, train_tokenizer, get_tokenizer,
+ _download_tokenizer_from_hf, TOKENIZER_DIR, TOKENIZER_PREFIX,
+ _HF_TOKENIZER_REPO,
+)
+
+import re as _re
+
HF_DATASET_REPO = "Cactus-Compute/tool-calls"
@@ -53,38 +63,11 @@ def download_hf_split(split="train", repo_id=HF_DATASET_REPO):
return Dataset(table)
-import re as _re
-
-def to_snake_case(name):
- """Convert camelCase, PascalCase, or dot.notation name to snake_case."""
- # Replace any non-alphanumeric/underscore characters with underscores
- s = _re.sub(r'[^a-zA-Z0-9_]+', '_', name)
- # Insert underscore before uppercase letters that follow lowercase/digits
- s = _re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', s)
- # Insert underscore between consecutive uppercase and uppercase+lowercase
- s = _re.sub(r'([A-Z]+)([A-Z][a-z])', r'\1_\2', s)
- # Collapse multiple underscores and strip edges
- s = _re.sub(r'_+', '_', s)
- return s.lower().strip('_')
-
_PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))
_DISK_CACHE_DIR = os.path.join(_PROJECT_ROOT, ".data_cache")
-TOKENIZER_DIR = os.path.join(_PROJECT_ROOT, "tokenizer")
-TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")
LOCAL_UNIFIED_DIR = os.path.join(_PROJECT_ROOT, "data", "tool_calls_unified")
CACHE_DIR = os.path.join(_PROJECT_ROOT, ".data_cache")
-PAD_ID = 0
-EOS_ID = 1
-BOS_ID = 2
-UNK_ID = 3
-TOOL_CALL_ID = 4
-TOOLS_ID = 5
-
-DEFAULT_MAX_ENC_LEN = 1024
-DEFAULT_MAX_DEC_LEN = 512
-DEFAULT_MAX_GEN_LEN = 512
-
_split_dataset_cache = {}
@@ -279,55 +262,6 @@ def _set_audio_backend(ds):
return ds
-class NeedleTokenizer:
- """Wrapper around SentencePiece providing the interface the codebase expects."""
-
- def __init__(self, model_path):
- self.sp = spm.SentencePieceProcessor()
- self.sp.Load(model_path)
-
- @property
- def pad_token_id(self):
- return PAD_ID
-
- @property
- def eos_token_id(self):
- return EOS_ID
-
- @property
- def bos_token_id(self):
- return BOS_ID
-
- @property
- def tool_call_token_id(self):
- return TOOL_CALL_ID
-
- @property
- def tools_token_id(self):
- return TOOLS_ID
-
- @property
- def vocab_size(self):
- return self.sp.GetPieceSize()
-
- def encode(self, text):
- return self.sp.Encode(text, out_type=int)
-
- def decode(self, ids):
- if isinstance(ids, (list, tuple)) and len(ids) > 0 and isinstance(ids[0], (list, tuple)):
- return [self.sp.Decode(seq) for seq in ids]
- return self.sp.Decode(list(ids))
-
- def __call__(self, texts, truncation=True, max_length=None, **kwargs):
- all_ids = []
- for text in texts:
- ids = self.sp.Encode(text, out_type=int)
- if truncation and max_length:
- ids = ids[:max_length]
- all_ids.append(ids)
- return {"input_ids": all_ids}
-
-
_worker_sp = None
_worker_max_len = None
@@ -353,57 +287,6 @@ def _compute_classes_chunk(chunk):
return results
-def train_tokenizer(vocab_size=8192, max_samples=None, force=False):
- """Train a SentencePiece BPE tokenizer on tool-calling corpus."""
- model_path = TOKENIZER_PREFIX + ".model"
- if os.path.exists(model_path) and not force:
- print(f"Tokenizer already exists at {model_path}")
- return model_path
-
- os.makedirs(TOKENIZER_DIR, exist_ok=True)
-
- from datasets import concatenate_datasets
- train_ds = _load_split_dataset("train")
- val_ds = _load_split_dataset("validation")
- ds = concatenate_datasets([train_ds, val_ds])
- if max_samples:
- ds = ds.select(range(min(max_samples, len(ds))))
-
- print(f"Training SentencePiece BPE tokenizer (vocab_size={vocab_size}, samples={len(ds):,})...")
-
- corpus_path = os.path.join(TOKENIZER_DIR, "corpus.txt")
- try:
- with open(corpus_path, "w") as f:
- for example in tqdm(ds, desc="Writing corpus"):
- for field in ("query", "tools", "answers"):
- text = example[field].strip()
- if text:
- f.write(text + "\n")
-
- spm.SentencePieceTrainer.Train(
- input=corpus_path,
- model_prefix=TOKENIZER_PREFIX,
- vocab_size=vocab_size,
- model_type="bpe",
- pad_id=PAD_ID,
- eos_id=EOS_ID,
- bos_id=BOS_ID,
- unk_id=UNK_ID,
- user_defined_symbols=["<tool_call>", "<tools>"],
- byte_fallback=True,
- normalization_rule_name="identity",
- num_threads=min(128, max(1, (os.cpu_count() or 1) * 3 // 4)),
- train_extremely_large_corpus=False,
- minloglevel=2,
- )
- finally:
- if os.path.exists(corpus_path):
- os.remove(corpus_path)
- print(f"Tokenizer saved to {model_path}")
- return model_path
-
-
-_HF_TOKENIZER_REPO = "Cactus-Compute/needle-tokenizer"
_HF_TOKENIZED_REPO = "Cactus-Compute/tokenized-tool-calls"
@@ -428,29 +311,6 @@ def _download_tokenized_from_hf():
os.symlink(os.path.join(local, fname), dst)
-def _download_tokenizer_from_hf():
- """Download tokenizer files from HuggingFace Hub into TOKENIZER_DIR."""
- from huggingface_hub import snapshot_download
-
- local = snapshot_download(
- _HF_TOKENIZER_REPO,
- repo_type="dataset",
- )
- os.makedirs(TOKENIZER_DIR, exist_ok=True)
- for fname in os.listdir(local):
- if fname.startswith("."):
- continue
- dst = os.path.join(TOKENIZER_DIR, fname)
- if not os.path.exists(dst):
- os.symlink(os.path.join(local, fname), dst)
-
-
-def get_tokenizer(max_samples=None):
- model_path = TOKENIZER_PREFIX + ".model"
- if not os.path.exists(model_path):
- print("Downloading pretrained tokenizer from HuggingFace...")
- _download_tokenizer_from_hf()
- return NeedleTokenizer(model_path)
def load_tool_calls(split="train", max_samples=None, return_global_indices=False):
diff --git a/src/distributed.py b/src/distributed.py
new file mode 100644
index 0000000..5f1f26c
--- /dev/null
+++ b/src/distributed.py
@@ -0,0 +1,58 @@
+import os
+import threading
+import jax
+import numpy as np
+
+_HF_CHECKPOINT_REPO = "Cactus-Compute/checkpoints"
+
+
+def _replicate(tree):
+ """Replicate a pytree across all local devices (multi-host safe)."""
+ devices = jax.local_devices()
+ return jax.tree.map(
+ lambda x: jax.device_put_replicated(x, devices), tree)
+
+
+def _unreplicate(tree):
+ """Get a single copy from a replicated pytree (multi-host safe).
+
+ Uses addressable_shards instead of x[0] indexing, which fails
+ in multi-host pmap because the array spans devices on other hosts.
+ Each shard has shape (1, ...) — we index [0] to strip the leading dim.
+ """
+ def _get_first(x):
+ if hasattr(x, 'addressable_shards') and x.addressable_shards:
+ shard = x.addressable_shards[0].data
+ # Shard has shape (1, ...) from the pmap partition; strip it
+ return jax.device_get(shard[0]) if shard.ndim > 0 else jax.device_get(shard)
+ return x
+ return jax.tree.map(_get_first, tree)
+
+
+def shard_batch(batch, num_devices):
+ """Reshape a batch array so leading dim is (num_devices, per_device_batch, ...)."""
+ return batch.reshape(num_devices, -1, *batch.shape[1:])
+
+
+def _upload_checkpoint(ckpt_path):
+ """Upload a checkpoint file to HuggingFace Hub in a background thread."""
+ import threading
+
+ def _upload():
+ try:
+ from huggingface_hub import HfApi
+ api = HfApi()
+ api.create_repo(_HF_CHECKPOINT_REPO, repo_type="model", private=True, exist_ok=True)
+ filename = os.path.basename(ckpt_path)
+ print(f"[hf] Uploading {filename} to {_HF_CHECKPOINT_REPO} ...")
+ api.upload_file(
+ path_or_fileobj=ckpt_path,
+ path_in_repo=filename,
+ repo_id=_HF_CHECKPOINT_REPO,
+ repo_type="model",
+ )
+ print(f"[hf] Checkpoint uploaded: {_HF_CHECKPOINT_REPO}/{filename}")
+ except Exception as e:
+ print(f"[hf] Warning: checkpoint upload failed: {e}")
+
+ threading.Thread(target=_upload, daemon=True).start()
diff --git a/src/optim.py b/src/optim.py
new file mode 100644
index 0000000..301402c
--- /dev/null
+++ b/src/optim.py
@@ -0,0 +1,123 @@
+import math
+from typing import NamedTuple
+import jax
+import jax.numpy as jnp
+import optax
+from flax.training import train_state
+from .model import EncoderDecoderTransformer
+
+
+def _newton_schulz(G, steps=5):
+ """Approximate polar decomposition via Newton-Schulz iteration."""
+ a, b, c = 3.4445, -4.7750, 2.0315
+ orig_dtype = G.dtype
+ G = G.astype(jnp.float32)
+ X = G / (jnp.linalg.norm(G) + 1e-7)
+ transposed = G.shape[0] > G.shape[1]
+ if transposed:
+ X = X.T
+ for _ in range(steps):
+ A = X @ X.T
+ B = b * A + c * A @ A
+ X = a * X + B @ X
+ if transposed:
+ X = X.T
+ return X.astype(orig_dtype)
+
+
+class MuonState(NamedTuple):
+ mu: optax.Updates
+
+
+def scale_by_muon(momentum=0.95, ns_steps=5):
+ """Muon gradient transform: orthogonalize 2D+ grads, then Nesterov momentum."""
+
+ def init_fn(params):
+ return MuonState(mu=jax.tree.map(jnp.zeros_like, params))
+
+ def update_fn(updates, state, params=None):
+ del params
+
+ def ortho(g):
+ if g.ndim == 3:
+ return jax.vmap(_newton_schulz, in_axes=(0,))(g)
+ if g.ndim == 2:
+ return _newton_schulz(g, steps=ns_steps)
+ return g
+
+ ortho_g = jax.tree.map(ortho, updates)
+ new_mu = jax.tree.map(lambda m, g: momentum * m + g, state.mu, ortho_g)
+ new_updates = jax.tree.map(
+ lambda g, m: g + momentum * m, ortho_g, new_mu
+ )
+ return new_updates, MuonState(mu=new_mu)
+
+ return optax.GradientTransformation(init_fn, update_fn)
+
+
+def _param_labels(params):
+ """Label each param: 'muon' for Dense kernels, 'adam' for the rest."""
+
+ def _label(path, leaf):
+ name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])
+ if name == "kernel" and leaf.ndim in (2, 3):
+ return "muon"
+ return "adam"
+
+ return jax.tree_util.tree_map_with_path(_label, params)
+
+
+def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.15):
+ """Warmup-Stable-Decay schedule: linear warmup, hold peak, cosine decay."""
+ decay_steps = max(1, int(total_steps * decay_ratio))
+ stable_steps = total_steps - warmup_steps - decay_steps
+ return optax.join_schedules(
+ [
+ optax.linear_schedule(0.0, peak_value, warmup_steps),
+ optax.constant_schedule(peak_value),
+ optax.cosine_decay_schedule(peak_value, decay_steps, alpha=0.05),
+ ],
+ boundaries=[warmup_steps, warmup_steps + stable_steps],
+ )
+
+
+def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps, decay_ratio=0.15):
+ model = EncoderDecoderTransformer(config)
+
+ rng, init_rng = jax.random.split(rng)
+ dummy_src = jnp.ones((1, 128), dtype=jnp.int32)
+ dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)
+ init_args = [dummy_src, dummy_tgt]
+ if config.enable_speech:
+ init_args.append(jnp.ones((1, 128, config.n_mels), dtype=jnp.float32))
+ variables = model.init(
+ {"params": init_rng},
+ *init_args,
+ method="init_all",
+ )
+
+ adam_schedule = _wsd_schedule(learning_rate, total_steps, warmup_steps, decay_ratio)
+ muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps, decay_ratio)
+
+ muon_opt = optax.chain(
+ scale_by_muon(momentum=0.95, ns_steps=5),
+ optax.add_decayed_weights(weight_decay=0.01),
+ optax.scale_by_schedule(muon_schedule),
+ optax.scale(-1.0),
+ )
+ adam_opt = optax.chain(
+ optax.adamw(adam_schedule, b2=0.95, weight_decay=0.0),
+ )
+
+ tx = optax.chain(
+ optax.clip_by_global_norm(1.0),
+ optax.multi_transform(
+ {"muon": muon_opt, "adam": adam_opt},
+ _param_labels,
+ ),
+ )
+ return train_state.TrainState.create(
+ apply_fn=model.apply,
+ params=variables["params"],
+ tx=tx,
+ )
diff --git a/src/pretrain.py b/src/pretrain.py
new file mode 100644
index 0000000..cec5a93
--- /dev/null
+++ b/src/pretrain.py
@@ -0,0 +1,352 @@
+"""Streaming pretraining on PleIAs/SYNTH dataset.
+
+Streams 80M examples from HuggingFace, tokenizes on-the-fly, and trains
+with simple CE loss (uniform weights). Saves needle_base.pkl every N steps.
+
+Usage:
+ needle pretrain --wandb
+ needle pretrain --wandb --max-steps 10000
+ needle tpu train large --name pretrain --wandb
+"""
+
+import math
+import os
+import pickle
+import time
+import queue
+import threading
+
+import jax
+import jax.numpy as jnp
+import numpy as np
+import optax
+from tqdm import tqdm
+
+from .tokenizer import get_tokenizer, EOS_ID, TOOLS_ID, PAD_ID
+from .model import (
+ EncoderDecoderTransformer,
+ TransformerConfig,
+ make_padding_mask,
+ make_causal_mask,
+)
+from .optim import create_train_state, _wsd_schedule
+from .distributed import _replicate, _unreplicate, shard_batch, _upload_checkpoint
+
+_HF_PRETRAIN_REPO = "PleIAs/SYNTH"
+
+
+def _stream_batches(tokenizer, batch_size, max_enc_len, max_dec_len, seed=42):
+ """Stream batches from PleIAs/SYNTH, tokenizing on-the-fly.
+
+ Encoder: [query_tokens, <tools>, seed_text_tokens] truncated to max_enc_len
+ Decoder input: [EOS, answer_tokens] truncated to max_dec_len
+ Decoder target: [answer_tokens, EOS] truncated to max_dec_len
+
+ Yields (enc, dec_in, dec_tgt) as numpy int32 arrays of shape (batch_size, seq_len).
+ """
+ from datasets import load_dataset
+
+ ds = load_dataset(_HF_PRETRAIN_REPO, split="train", streaming=True)
+ ds = ds.shuffle(seed=seed, buffer_size=10000)
+
+ tools_sep_id = TOOLS_ID
+ eos_id = EOS_ID
+ pad_id = PAD_ID
+
+ enc_batch = np.full((batch_size, max_enc_len), pad_id, dtype=np.int32)
+ dec_in_batch = np.full((batch_size, max_dec_len), pad_id, dtype=np.int32)
+ dec_tgt_batch = np.full((batch_size, max_dec_len), pad_id, dtype=np.int32)
+ idx = 0
+
+ for example in ds:
+ query = example.get("query") or ""
+ seed_text = example.get("query_seed_text") or ""
+ answer = example.get("synthetic_answer") or ""
+
+ if not query.strip() or not answer.strip():
+ continue
+
+ # Tokenize
+ q_toks = tokenizer.encode(query)
+ s_toks = tokenizer.encode(seed_text) if seed_text.strip() else []
+ a_toks = tokenizer.encode(answer)
+
+ # Build encoder input: [query, <tools>, seed_text]
+ max_query = max_enc_len - 2
+ if len(q_toks) > max_query:
+ q_toks = q_toks[:max_query]
+ remaining = max_enc_len - len(q_toks) - 1
+ s_toks = s_toks[:remaining]
+ enc_tokens = q_toks + [tools_sep_id] + s_toks
+
+ # Build decoder: input=[EOS, answer], target=[answer, EOS]
+ max_ans = max_dec_len - 1
+ a_toks = a_toks[:max_ans]
+ if len(a_toks) == 0:
+ continue
+ dec_in_tokens = [eos_id] + a_toks
+ dec_tgt_tokens = a_toks + [eos_id]
+
+ # Fill batch
+ enc_batch[idx, :len(enc_tokens)] = enc_tokens
+ dec_in_batch[idx, :len(dec_in_tokens)] = dec_in_tokens
+ dec_tgt_batch[idx, :len(dec_tgt_tokens)] = dec_tgt_tokens
+ idx += 1
+
+ if idx == batch_size:
+ yield enc_batch.copy(), dec_in_batch.copy(), dec_tgt_batch.copy()
+ enc_batch[:] = pad_id
+ dec_in_batch[:] = pad_id
+ dec_tgt_batch[:] = pad_id
+ idx = 0
+
+
+class _PrefetchStream:
+ """Prefetch streaming batches in a background thread."""
+
+ def __init__(self, generator_fn, prefetch=4):
+ self._queue = queue.Queue(maxsize=prefetch)
+ self._stop = threading.Event()
+ self._thread = threading.Thread(target=self._produce, args=(generator_fn,), daemon=True)
+ self._thread.start()
+
+ def _produce(self, gen_fn):
+ try:
+ for batch in gen_fn():
+ if self._stop.is_set():
+ return
+ self._queue.put(batch)
+ self._queue.put(None)
+ except Exception as e:
+ self._queue.put(e)
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ item = self._queue.get()
+ if item is None:
+ raise StopIteration
+ if isinstance(item, Exception):
+ raise item
+ return item
+
+ def close(self):
+ self._stop.set()
+
+
+def _pretrain_step(state, src, tgt_in, tgt_out, rng):
+ """Simple CE loss train step for pretraining. No QAT, no contrastive, no pruning."""
+
+ def loss_fn(params):
+ src_mask = make_padding_mask(src, PAD_ID)
+ tgt_mask = make_causal_mask(tgt_in.shape[1]) & make_padding_mask(tgt_in, PAD_ID)
+ cross_mask = src_mask
+
+ logits = state.apply_fn(
+ {"params": params}, src, tgt_in,
+ src_mask=src_mask, tgt_mask=tgt_mask, cross_mask=cross_mask,
+ )
+ logits_f32 = logits.astype(jnp.float32)
+
+ padding_mask = (tgt_out != PAD_ID).astype(jnp.float32)
+ num_tokens = jnp.maximum(jnp.sum(padding_mask), 1.0)
+ ce = jnp.sum(
+ optax.softmax_cross_entropy_with_integer_labels(logits_f32, tgt_out) * padding_mask
+ ) / num_tokens
+ z_loss = 1e-4 * jnp.mean(jax.nn.logsumexp(logits_f32, axis=-1) ** 2)
+ return ce + z_loss
+
+ loss, grads = jax.value_and_grad(loss_fn)(state.params)
+ grads = jax.lax.pmean(grads, axis_name="batch")
+ loss = jax.lax.pmean(loss, axis_name="batch")
+ state = state.apply_gradients(grads=grads)
+ return state, loss
+
+
+_p_pretrain_step = None
+
+
+def _make_p_pretrain_step():
+ return jax.pmap(_pretrain_step, axis_name="batch", donate_argnums=(0,))
+
+
+def pretrain(args):
+ num_devices = jax.local_device_count()
+ num_hosts = jax.process_count()
+ host_id = jax.process_index()
+ total_devices = jax.device_count()
+ is_main = host_id == 0
+
+ experiment_name = getattr(args, "name", "pretrain")
+ use_wandb = getattr(args, "wandb", False)
+ if use_wandb and is_main:
+ import wandb
+ if wandb.run is None:
+ wandb.init(project="needle-v1", name=experiment_name, config=vars(args))
+
+ print(f"\n[1/3] Detecting devices...")
+ print(f" {num_devices} local, {total_devices} total across {num_hosts} hosts")
+
+ print(f"\n[2/3] Loading tokenizer...")
+ tokenizer = get_tokenizer()
+
+ config = TransformerConfig(
+ d_model=args.d_model,
+ num_heads=args.num_heads,
+ num_kv_heads=getattr(args, "num_kv_heads", None) or args.num_heads,
+ num_encoder_layers=args.num_layers,
+ num_decoder_layers=getattr(args, "num_dec_layers", args.num_layers),
+ d_ff=getattr(args, "d_ff", None) or args.d_model * 4,
+ max_seq_len=max(args.max_enc_len, args.max_dec_len),
+ dtype=args.dtype,
+ activation=getattr(args, "activation", "swiglu"),
+ no_feedforward=getattr(args, "no_feedforward", True),
+ contrastive_dim=getattr(args, "contrastive_dim", 128),
+ enable_speech=False,
+ )
+
+ effective_batch_size = args.batch_size * num_devices
+ global_batch_size = effective_batch_size * num_hosts
+
+ # Estimate total steps: 80M rows / global_batch_size
+ estimated_rows = 80_000_000
+ max_steps = getattr(args, "max_steps", None)
+ estimated_steps = estimated_rows // global_batch_size
+ if max_steps:
+ estimated_steps = min(estimated_steps, max_steps)
+ total_steps = estimated_steps * args.epochs
+ warmup_steps = max(1, int(total_steps * args.warmup_ratio))
+
+ scaled_lr = args.lr * total_devices
+ muon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(total_devices)
+ decay_ratio = getattr(args, "decay_ratio", 0.05)
+
+ print(f"\n[3/3] Initializing model...")
+ rng = jax.random.PRNGKey(args.seed)
+ rng, init_rng = jax.random.split(rng)
+ state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps, decay_ratio)
+ state = _replicate(state)
+
+ param_count = sum(x.size for x in jax.tree.leaves(_unreplicate(state).params))
+
+ global _p_pretrain_step
+ _p_pretrain_step = _make_p_pretrain_step()
+
+ save_every = getattr(args, "save_every", 1000)
+ checkpoint_dir = getattr(args, "checkpoint_dir", "checkpoints")
+ os.makedirs(checkpoint_dir, exist_ok=True)
+
+ if is_main:
+ decay_steps = max(1, int(total_steps * decay_ratio))
+ stable_steps = total_steps - warmup_steps - decay_steps
+ print(f"\n ─────────────────────────────────────")
+ print(f" Pretraining on PleIAs/SYNTH")
+ print(f" ─────────────────────────────────────")
+ print(f" Parameters {param_count:>12,}")
+ print(f" d_model {config.d_model:>12}")
+ print(f" Heads {config.num_heads:>7} ({config.num_kv_heads} KV)")
+ print(f" Layers {config.num_encoder_layers:>7} enc / {config.num_decoder_layers} dec")
+ print(f" Dtype {config.dtype:>12}")
+ print(f" ─────────────────────────────────────")
+ print(f" Hosts {num_hosts:>12}")
+ print(f" Devices {num_devices:>5}/host, {total_devices} total")
+ print(f" Batch {args.batch_size:>7} x {total_devices} = {global_batch_size}")
+ print(f" Adam LR {args.lr:>7} x {total_devices} = {scaled_lr}")
+ print(f" Muon LR {getattr(args, 'muon_lr', 0.02):>7.4f} -> {muon_lr:.4f}")
+ print(f" Schedule {warmup_steps}w / {stable_steps}s / {decay_steps}d (WSD)")
+ print(f" Est. steps {estimated_steps:>12,}")
+ print(f" Save every {save_every:>12,}")
+ print(f" ─────────────────────────────────────\n")
+
+ global_step = 0
+
+ for epoch in range(args.epochs):
+ batch_stream = _PrefetchStream(
+ lambda: _stream_batches(tokenizer, global_batch_size,
+ args.max_enc_len, args.max_dec_len,
+ seed=args.seed + epoch),
+ prefetch=8,
+ )
+
+ pbar = tqdm(desc=f"Pretrain epoch {epoch + 1}/{args.epochs}",
+ total=estimated_steps, disable=not is_main)
+
+ for batch in batch_stream:
+ if max_steps and global_step >= max_steps:
+ break
+
+ src, tgt_in, tgt_out = batch
+ t0 = time.perf_counter()
+
+ # Slice this host's portion
+ host_slice = slice(host_id * effective_batch_size,
+ (host_id + 1) * effective_batch_size)
+ src = src[host_slice]
+ tgt_in = tgt_in[host_slice]
+ tgt_out = tgt_out[host_slice]
+
+ # Shard for pmap
+ src_b = shard_batch(src, num_devices)
+ tgt_in_b = shard_batch(tgt_in, num_devices)
+ tgt_out_b = shard_batch(tgt_out, num_devices)
+
+ rng, step_rng = jax.random.split(rng)
+ step_rngs = jax.random.split(step_rng, num_devices)
+
+ state, loss = _p_pretrain_step(state, src_b, tgt_in_b, tgt_out_b, step_rngs)
+
+ loss_val = float(loss.addressable_shards[0].data[0])
+ ppl = math.exp(min(loss_val, 20))
+ dt = time.perf_counter() - t0
+ global_step += 1
+
+ pbar.update(1)
+ pbar.set_postfix(loss=f"{loss_val:.4f}", ppl=f"{ppl:.2f}",
+ tok_s=f"{global_batch_size * (args.max_enc_len + args.max_dec_len) / dt:.0f}")
+
+ if use_wandb and is_main:
+ import wandb
+ wandb.log({
+ "pretrain/loss": loss_val,
+ "pretrain/ppl": ppl,
+ "pretrain/step": global_step,
+ "pretrain/tokens_per_sec": global_batch_size * (args.max_enc_len + args.max_dec_len) / dt,
+ })
+
+ # Save checkpoint every N steps
+ if is_main and global_step % save_every == 0:
+ _save_pretrain_checkpoint(state, config, checkpoint_dir, global_step)
+
+ batch_stream.close()
+ pbar.close()
+
+ # Final save
+ if is_main:
+ ckpt_path = _save_pretrain_checkpoint(state, config, checkpoint_dir, global_step)
+ print(f"\nPretraining complete. {global_step} steps.")
+ print(f"Base checkpoint: {ckpt_path}")
+
+ if use_wandb and is_main:
+ import wandb
+ wandb.finish()
+
+ # Sync all hosts before exit
+ jax.experimental.multihost_utils.sync_global_devices("pretrain_done")
+
+
+def _save_pretrain_checkpoint(state, config, checkpoint_dir, global_step):
+ """Save and upload needle_base.pkl."""
+ params = _unreplicate(state).params
+ params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), params)
+
+ ckpt_path = os.path.join(checkpoint_dir, "needle_base.pkl")
+ with open(ckpt_path, "wb") as f:
+ pickle.dump({"params": params_np, "config": config.__dict__}, f)
+
+ param_count = sum(x.size for x in jax.tree.leaves(params_np))
+ size_mb = sum(x.nbytes for x in jax.tree.leaves(params_np)) / 1e6
+ print(f"\n [step {global_step}] Saved {ckpt_path} ({param_count:,} params, {size_mb:.1f} MB)")
+
+ _upload_checkpoint(ckpt_path)
+ return ckpt_path
diff --git a/src/quantize.py b/src/quantize.py
new file mode 100644
index 0000000..6f86eb8
--- /dev/null
+++ b/src/quantize.py
@@ -0,0 +1,133 @@
+import jax
+import jax.numpy as jnp
+
+
+def _fake_quantize_int4(w, group_size=32):
+ """Symmetric group-wise INT4 fake quantization with STE.
+
+ Divides the input dimension (axis 0) into groups of `group_size` elements,
+ each with its own scale factor. Falls back to per-channel if in_features < group_size.
+ """
+ in_feat, out_feat = w.shape
+ gs = min(group_size, in_feat)
+
+ pad = (gs - in_feat % gs) % gs
+ if pad > 0:
+ w_padded = jnp.pad(w, ((0, pad), (0, 0)))
+ else:
+ w_padded = w
+
+ num_groups = w_padded.shape[0] // gs
+ w_grouped = w_padded.reshape(num_groups, gs, out_feat)
+
+ scale = jnp.max(jnp.abs(w_grouped), axis=1, keepdims=True) / 7.0
+ scale = jnp.maximum(scale, 1e-8)
+ w_q = jnp.clip(jnp.round(w_grouped / scale), -8, 7) * scale
+
+ w_q = w_q.reshape(-1, out_feat)[:in_feat]
+
+ return w + jax.lax.stop_gradient(w_q - w)
+
+
+def _fake_quantize_int8(w, group_size=32):
+ """Symmetric group-wise INT8 fake quantization with STE.
+
+ Same structure as INT4 but with [-128, 127] range (8-bit signed).
+ """
+ in_feat, out_feat = w.shape
+ gs = min(group_size, in_feat)
+
+ pad = (gs - in_feat % gs) % gs
+ if pad > 0:
+ w_padded = jnp.pad(w, ((0, pad), (0, 0)))
+ else:
+ w_padded = w
+
+ num_groups = w_padded.shape[0] // gs
+ w_grouped = w_padded.reshape(num_groups, gs, out_feat)
+
+ scale = jnp.max(jnp.abs(w_grouped), axis=1, keepdims=True) / 127.0
+ scale = jnp.maximum(scale, 1e-8)
+ w_q = jnp.clip(jnp.round(w_grouped / scale), -128, 127) * scale
+
+ w_q = w_q.reshape(-1, out_feat)[:in_feat]
+
+ return w + jax.lax.stop_gradient(w_q - w)
+
+
+def _quantize_params(params, group_size=32, precision="int4"):
+ """Fake-quantize all Dense kernels in the param tree."""
+ qfn = _fake_quantize_int8 if precision == "int8" else _fake_quantize_int4
+ def _maybe_quantize(path, leaf):
+ name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])
+ if name == "kernel" and leaf.ndim == 3:
+ return jax.vmap(lambda w: qfn(w, group_size=group_size), in_axes=(0,))(leaf)
+ if name == "kernel" and leaf.ndim == 2:
+ return qfn(leaf, group_size=group_size)
+ return leaf
+ return jax.tree_util.tree_map_with_path(_maybe_quantize, params)
+
+
+def _cubic_sparsity_schedule(step, t_start, t_end, s_final):
+ """Cubic sparsity ramp (Zhu & Gupta 2017). Returns target sparsity at *step*."""
+ if step < t_start:
+ return 0.0
+ if step >= t_end:
+ return s_final
+ frac = (step - t_start) / (t_end - t_start)
+ return s_final * (1.0 - (1.0 - frac) ** 3)
+
+
+def _make_prune_mask(params, sparsity, group_size):
+ """Compute block-prune mask entirely on-device (no numpy round-trips).
+
+ Returns binary mask tree matching param shapes and dtypes.
+ Handles both 2D (in, out) and 3D (num_layers, in, out) kernels from nn.scan.
+ """
+ all_scores = []
+ for _, leaf in jax.tree_util.tree_leaves_with_path(params):
+ if leaf.ndim == 2:
+ in_feat, out_feat = leaf.shape
+ gs = min(group_size, in_feat)
+ pad = (gs - in_feat % gs) % gs
+ w = jnp.pad(leaf, ((0, pad), (0, 0))) if pad else leaf
+ scores = jnp.sum(jnp.abs(w.reshape(-1, gs, out_feat)), axis=1).ravel()
+ all_scores.append(scores)
+ elif leaf.ndim == 3:
+ num_layers, in_feat, out_feat = leaf.shape
+ gs = min(group_size, in_feat)
+ pad = (gs - in_feat % gs) % gs
+ for li in range(num_layers):
+ w = jnp.pad(leaf[li], ((0, pad), (0, 0))) if pad else leaf[li]
+ scores = jnp.sum(jnp.abs(w.reshape(-1, gs, out_feat)), axis=1).ravel()
+ all_scores.append(scores)
+
+ if not all_scores:
+ return jax.tree.map(jnp.ones_like, params)
+
+ threshold = jnp.percentile(jnp.concatenate(all_scores), sparsity * 100)
+
+ def _leaf_mask(path, leaf):
+ if leaf.ndim == 3:
+ num_layers, in_feat, out_feat = leaf.shape
+ gs = min(group_size, in_feat)
+ pad = (gs - in_feat % gs) % gs
+ masks = []
+ for li in range(num_layers):
+ w = jnp.pad(leaf[li], ((0, pad), (0, 0))) if pad else leaf[li]
+ w_grouped = w.reshape(-1, gs, out_feat)
+ block_keep = (jnp.sum(jnp.abs(w_grouped), axis=1, keepdims=True) > threshold)
+ m = jnp.broadcast_to(block_keep, w_grouped.shape).reshape(-1, out_feat)[:in_feat]
+ masks.append(m)
+ return jnp.stack(masks, axis=0).astype(leaf.dtype)
+ if leaf.ndim != 2:
+ return jnp.ones_like(leaf)
+ in_feat, out_feat = leaf.shape
+ gs = min(group_size, in_feat)
+ pad = (gs - in_feat % gs) % gs
+ w = jnp.pad(leaf, ((0, pad), (0, 0))) if pad else leaf
+ w_grouped = w.reshape(-1, gs, out_feat)
+ block_keep = (jnp.sum(jnp.abs(w_grouped), axis=1, keepdims=True) > threshold)
+ return jnp.broadcast_to(block_keep, w_grouped.shape).reshape(-1, out_feat)[:in_feat].astype(leaf.dtype)
+
+ return jax.tree_util.tree_map_with_path(_leaf_mask, params)
diff --git a/src/tokenizer.py b/src/tokenizer.py
new file mode 100644
index 0000000..79991dd
--- /dev/null
+++ b/src/tokenizer.py
@@ -0,0 +1,161 @@
+import os
+import re as _re
+
+import sentencepiece as spm
+from tqdm import tqdm
+
+_PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))
+TOKENIZER_DIR = os.path.join(_PROJECT_ROOT, "tokenizer")
+TOKENIZER_PREFIX = os.path.join(TOKENIZER_DIR, "needle")
+
+PAD_ID = 0
+EOS_ID = 1
+BOS_ID = 2
+UNK_ID = 3
+TOOL_CALL_ID = 4
+TOOLS_ID = 5
+
+DEFAULT_MAX_ENC_LEN = 1024
+DEFAULT_MAX_DEC_LEN = 512
+DEFAULT_MAX_GEN_LEN = 512
+
+_HF_TOKENIZER_REPO = "Cactus-Compute/needle-tokenizer"
+
+
+def to_snake_case(name):
+ """Convert camelCase, PascalCase, or dot.notation name to snake_case."""
+ # Replace any non-alphanumeric/underscore characters with underscores
+ s = _re.sub(r'[^a-zA-Z0-9_]+', '_', name)
+ # Insert underscore before uppercase letters that follow lowercase/digits
+ s = _re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', s)
+ # Insert underscore between consecutive uppercase and uppercase+lowercase
+ s = _re.sub(r'([A-Z]+)([A-Z][a-z])', r'\1_\2', s)
+ # Collapse multiple underscores and strip edges
+ s = _re.sub(r'_+', '_', s)
+ return s.lower().strip('_')
+
+
+class NeedleTokenizer:
+ """Wrapper around SentencePiece providing the interface the codebase expects."""
+
+ def __init__(self, model_path):
+ self.sp = spm.SentencePieceProcessor()
+ self.sp.Load(model_path)
+
+ @property
+ def pad_token_id(self):
+ return PAD_ID
+
+ @property
+ def eos_token_id(self):
+ return EOS_ID
+
+ @property
+ def bos_token_id(self):
+ return BOS_ID
+
+ @property
+ def tool_call_token_id(self):
+ return TOOL_CALL_ID
+
+ @property
+ def tools_token_id(self):
+ return TOOLS_ID
+
+ @property
+ def vocab_size(self):
+ return self.sp.GetPieceSize()
+
+ def encode(self, text):
+ return self.sp.Encode(text, out_type=int)
+
+ def decode(self, ids):
+ if isinstance(ids, (list, tuple)) and len(ids) > 0 and isinstance(ids[0], (list, tuple)):
+ return [self.sp.Decode(seq) for seq in ids]
+ return self.sp.Decode(list(ids))
+
+ def __call__(self, texts, truncation=True, max_length=None, **kwargs):
+ all_ids = []
+ for text in texts:
+ ids = self.sp.Encode(text, out_type=int)
+ if truncation and max_length:
+ ids = ids[:max_length]
+ all_ids.append(ids)
+ return {"input_ids": all_ids}
+
+
+def _download_tokenizer_from_hf():
+ """Download tokenizer files from HuggingFace Hub into TOKENIZER_DIR."""
+ from huggingface_hub import snapshot_download
+
+ local = snapshot_download(
+ _HF_TOKENIZER_REPO,
+ repo_type="dataset",
+ )
+ os.makedirs(TOKENIZER_DIR, exist_ok=True)
+ for fname in os.listdir(local):
+ if fname.startswith("."):
+ continue
+ dst = os.path.join(TOKENIZER_DIR, fname)
+ if not os.path.exists(dst):
+ os.symlink(os.path.join(local, fname), dst)
+
+
+def get_tokenizer(max_samples=None):
+ model_path = TOKENIZER_PREFIX + ".model"
+ if not os.path.exists(model_path):
+ print("Downloading pretrained tokenizer from HuggingFace...")
+ _download_tokenizer_from_hf()
+ return NeedleTokenizer(model_path)
+
+
+def train_tokenizer(vocab_size=8192, max_samples=None, force=False):
+ """Train a SentencePiece BPE tokenizer on tool-calling corpus."""
+ model_path = TOKENIZER_PREFIX + ".model"
+ if os.path.exists(model_path) and not force:
+ print(f"Tokenizer already exists at {model_path}")
+ return model_path
+
+ os.makedirs(TOKENIZER_DIR, exist_ok=True)
+
+ from datasets import concatenate_datasets
+ from .data import _load_split_dataset
+
+ train_ds = _load_split_dataset("train")
+ val_ds = _load_split_dataset("validation")
+ ds = concatenate_datasets([train_ds, val_ds])
+ if max_samples:
+ ds = ds.select(range(min(max_samples, len(ds))))
+
+ print(f"Training SentencePiece BPE tokenizer (vocab_size={vocab_size}, samples={len(ds):,})...")
+
+ corpus_path = os.path.join(TOKENIZER_DIR, "corpus.txt")
+ try:
+ with open(corpus_path, "w") as f:
+ for example in tqdm(ds, desc="Writing corpus"):
+ for field in ("query", "tools", "answers"):
+ text = example[field].strip()
+ if text:
+ f.write(text + "\n")
+
+ spm.SentencePieceTrainer.Train(
+ input=corpus_path,
+ model_prefix=TOKENIZER_PREFIX,
+ vocab_size=vocab_size,
+ model_type="bpe",
+ pad_id=PAD_ID,
+ eos_id=EOS_ID,
+ bos_id=BOS_ID,
+ unk_id=UNK_ID,
+ user_defined_symbols=["<tool_call>", "<tools>"],
+ byte_fallback=True,
+ normalization_rule_name="identity",
+ num_threads=min(128, max(1, (os.cpu_count() or 1) * 3 // 4)),
+ train_extremely_large_corpus=False,
+ minloglevel=2,
+ )
+ finally:
+ if os.path.exists(corpus_path):
+ os.remove(corpus_path)
+ print(f"Tokenizer saved to {model_path}")
+ return model_path
diff --git a/src/tpu.py b/src/tpu.py
index d4dbd6c..5864d57 100644
--- a/src/tpu.py
+++ b/src/tpu.py
@@ -56,6 +56,7 @@ TPU_HELP = """
needle tpu setup NAME [--zone ZONE] setup all workers (multi-host)
needle tpu sync NAME [--zone ZONE] sync code only (fast, no venv rebuild)
needle tpu train NAME [--zone ZONE] launch training on all workers
+ needle tpu pretrain NAME [--zone ZONE] launch pretraining on all workers
needle tpu claude NAME [--zone ZONE]
needle tpu stop NAME [--zone ZONE]
needle tpu start NAME [--zone ZONE]
@@ -554,20 +555,20 @@ def _collect_env_exports():
return " && ".join(exports)
-def tpu_train(args):
- """Launch training on all workers of a multi-host TPU."""
+def _tpu_run_command(args, needle_cmd="train"):
+ """Launch a needle command on all workers of a multi-host TPU."""
zone = args.zone or _detect_zone(args.name)
num_workers = _get_num_workers(args.name, zone)
if num_workers > 1:
print(f"[tpu] Killing stale processes on all workers...")
_run_all_workers(args.name, zone,
- "sudo pkill -9 -f [n]eedle.train || true")
+ "sudo pkill -9 -f [n]eedle || true")
extra = getattr(args, "train_args", [])
if extra and extra[0] == "--":
extra = extra[1:]
- train_cmd = "cd ~/needle && .venv/bin/needle train"
+ train_cmd = f"cd ~/needle && .venv/bin/needle {needle_cmd}"
if extra:
train_cmd += " " + " ".join(extra)
@@ -579,9 +580,8 @@ def tpu_train(args):
train_cmd = env_exports + " && " + train_cmd
if num_workers <= 1:
- print(f"[tpu] Single-host TPU. Running training directly...")
- # Use quiet=True to avoid printing tokens, then log sanitized version
- print(f"[tpu] $ gcloud compute tpus tpu-vm ssh {args.name} --command 'needle train ...'")
+ print(f"[tpu] Single-host TPU. Running directly...")
+ print(f"[tpu] $ gcloud ... --command 'needle {needle_cmd} ...'")
_run(
["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,
"--zone", zone, "--project", PROJECT,
@@ -590,8 +590,7 @@ def tpu_train(args):
)
return
- print(f"[tpu] Launching training on {num_workers} workers...")
- # Run with quiet=True to avoid printing tokens in command log
+ print(f"[tpu] Launching 'needle {needle_cmd}' on {num_workers} workers...")
result = _run(
["gcloud", "compute", "tpus", "tpu-vm", "ssh", args.name,
"--zone", zone, "--project", PROJECT,
@@ -600,10 +599,20 @@ def tpu_train(args):
check=False, quiet=True,
)
if result.returncode != 0:
- print("[tpu] WARNING: training exited with errors on some workers.",
+ print(f"[tpu] WARNING: 'needle {needle_cmd}' exited with errors on some workers.",
file=sys.stderr)
+def tpu_train(args):
+ """Launch 'needle train' on all workers."""
+ _tpu_run_command(args, needle_cmd="train")
+
+
+def tpu_pretrain(args):
+ """Launch 'needle pretrain' on all workers."""
+ _tpu_run_command(args, needle_cmd="pretrain")
+
+
def tpu_dispatch(args):
actions = {
"create": tpu_create,
@@ -611,6 +620,7 @@ def tpu_dispatch(args):
"setup": tpu_setup,
"sync": tpu_sync,
"train": tpu_train,
+ "pretrain": tpu_pretrain,
"claude": tpu_claude,
"stop": tpu_stop,
"start": tpu_start,
diff --git a/src/train.py b/src/train.py
index 3713b90..e9cc7f5 100644
--- a/src/train.py
+++ b/src/train.py
@@ -2,14 +2,12 @@ import math
import os
import pickle
import time
-from typing import NamedTuple
import jax
import jax.numpy as jnp
import numpy as np
import optax
from tqdm import tqdm
-from flax.training import train_state
from .data import (
get_batches, get_tokenizer,
@@ -24,302 +22,11 @@ from .model import (
make_causal_packing_mask,
make_cross_packing_mask,
)
-
-_HF_CHECKPOINT_REPO = "Cactus-Compute/checkpoints"
-
-
-def _replicate(tree):
- """Replicate a pytree across all local devices (multi-host safe)."""
- devices = jax.local_devices()
- return jax.tree.map(
- lambda x: jax.device_put_replicated(x, devices), tree)
-
-
-def _unreplicate(tree):
- """Get a single copy from a replicated pytree (multi-host safe).
-
- Uses addressable_shards instead of x[0] indexing, which fails
- in multi-host pmap because the array spans devices on other hosts.
- Each shard has shape (1, ...) — we index [0] to strip the leading dim.
- """
- def _get_first(x):
- if hasattr(x, 'addressable_shards') and x.addressable_shards:
- shard = x.addressable_shards[0].data
- # Shard has shape (1, ...) from the pmap partition; strip it
- return jax.device_get(shard[0]) if shard.ndim > 0 else jax.device_get(shard)
- return x
- return jax.tree.map(_get_first, tree)
-
-
-def _upload_checkpoint(ckpt_path):
- """Upload a checkpoint file to HuggingFace Hub in a background thread."""
- import threading
-
- def _upload():
- try:
- from huggingface_hub import HfApi
- api = HfApi()
- api.create_repo(_HF_CHECKPOINT_REPO, repo_type="model", private=True, exist_ok=True)
- filename = os.path.basename(ckpt_path)
- print(f"[hf] Uploading {filename} to {_HF_CHECKPOINT_REPO} ...")
- api.upload_file(
- path_or_fileobj=ckpt_path,
- path_in_repo=filename,
- repo_id=_HF_CHECKPOINT_REPO,
- repo_type="model",
- )
- print(f"[hf] Checkpoint uploaded: {_HF_CHECKPOINT_REPO}/{filename}")
- except Exception as e:
- print(f"[hf] Warning: checkpoint upload failed: {e}")
-
- threading.Thread(target=_upload, daemon=True).start()
-
-
-def _newton_schulz(G, steps=5):
- """Approximate polar decomposition via Newton-Schulz iteration."""
- a, b, c = 3.4445, -4.7750, 2.0315
- orig_dtype = G.dtype
- G = G.astype(jnp.float32)
- X = G / (jnp.linalg.norm(G) + 1e-7)
- transposed = G.shape[0] > G.shape[1]
- if transposed:
- X = X.T
- for _ in range(steps):
- A = X @ X.T
- B = b * A + c * A @ A
- X = a * X + B @ X
- if transposed:
- X = X.T
- return X.astype(orig_dtype)
-
-
-class MuonState(NamedTuple):
- mu: optax.Updates
-
-
-def scale_by_muon(momentum=0.95, ns_steps=5):
- """Muon gradient transform: orthogonalize 2D+ grads, then Nesterov momentum."""
-
- def init_fn(params):
- return MuonState(mu=jax.tree.map(jnp.zeros_like, params))
-
- def update_fn(updates, state, params=None):
- del params
-
- def ortho(g):
- if g.ndim == 3:
- return jax.vmap(_newton_schulz, in_axes=(0,))(g)
- if g.ndim == 2:
- return _newton_schulz(g, steps=ns_steps)
- return g
-
- ortho_g = jax.tree.map(ortho, updates)
- new_mu = jax.tree.map(lambda m, g: momentum * m + g, state.mu, ortho_g)
- new_updates = jax.tree.map(
- lambda g, m: g + momentum * m, ortho_g, new_mu
- )
- return new_updates, MuonState(mu=new_mu)
-
- return optax.GradientTransformation(init_fn, update_fn)
-
-
-def _param_labels(params):
- """Label each param: 'muon' for Dense kernels, 'adam' for the rest."""
-
- def _label(path, leaf):
- name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])
- if name == "kernel" and leaf.ndim in (2, 3):
- return "muon"
- return "adam"
-
- return jax.tree_util.tree_map_with_path(_label, params)
-
-
-def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.15):
- """Warmup-Stable-Decay schedule: linear warmup, hold peak, cosine decay."""
- decay_steps = max(1, int(total_steps * decay_ratio))
- stable_steps = total_steps - warmup_steps - decay_steps
- return optax.join_schedules(
- [
- optax.linear_schedule(0.0, peak_value, warmup_steps),
- optax.constant_schedule(peak_value),
- optax.cosine_decay_schedule(peak_value, decay_steps, alpha=0.05),
- ],
- boundaries=[warmup_steps, warmup_steps + stable_steps],
- )
-
-
-def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps, decay_ratio=0.15):
- model = EncoderDecoderTransformer(config)
-
- rng, init_rng = jax.random.split(rng)
- dummy_src = jnp.ones((1, 128), dtype=jnp.int32)
- dummy_tgt = jnp.ones((1, 128), dtype=jnp.int32)
- init_args = [dummy_src, dummy_tgt]
- if config.enable_speech:
- init_args.append(jnp.ones((1, 128, config.n_mels), dtype=jnp.float32))
- variables = model.init(
- {"params": init_rng},
- *init_args,
- method="init_all",
- )
-
- adam_schedule = _wsd_schedule(learning_rate, total_steps, warmup_steps, decay_ratio)
- muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps, decay_ratio)
-
- muon_opt = optax.chain(
- scale_by_muon(momentum=0.95, ns_steps=5),
- optax.add_decayed_weights(weight_decay=0.01),
- optax.scale_by_schedule(muon_schedule),
- optax.scale(-1.0),
- )
- adam_opt = optax.chain(
- optax.adamw(adam_schedule, b2=0.95, weight_decay=0.0),
- )
-
- tx = optax.chain(
- optax.clip_by_global_norm(1.0),
- optax.multi_transform(
- {"muon": muon_opt, "adam": adam_opt},
- _param_labels,
- ),
- )
- return train_state.TrainState.create(
- apply_fn=model.apply,
- params=variables["params"],
- tx=tx,
- )
-
-
-def _fake_quantize_int4(w, group_size=32):
- """Symmetric group-wise INT4 fake quantization with STE.
-
- Divides the input dimension (axis 0) into groups of `group_size` elements,
- each with its own scale factor. Falls back to per-channel if in_features < group_size.
- """
- in_feat, out_feat = w.shape
- gs = min(group_size, in_feat)
-
- pad = (gs - in_feat % gs) % gs
- if pad > 0:
- w_padded = jnp.pad(w, ((0, pad), (0, 0)))
- else:
- w_padded = w
-
- num_groups = w_padded.shape[0] // gs
- w_grouped = w_padded.reshape(num_groups, gs, out_feat)
-
- scale = jnp.max(jnp.abs(w_grouped), axis=1, keepdims=True) / 7.0
- scale = jnp.maximum(scale, 1e-8)
- w_q = jnp.clip(jnp.round(w_grouped / scale), -8, 7) * scale
-
- w_q = w_q.reshape(-1, out_feat)[:in_feat]
-
- return w + jax.lax.stop_gradient(w_q - w)
-
-
-def _fake_quantize_int8(w, group_size=32):
- """Symmetric group-wise INT8 fake quantization with STE.
-
- Same structure as INT4 but with [-128, 127] range (8-bit signed).
- """
- in_feat, out_feat = w.shape
- gs = min(group_size, in_feat)
-
- pad = (gs - in_feat % gs) % gs
- if pad > 0:
- w_padded = jnp.pad(w, ((0, pad), (0, 0)))
- else:
- w_padded = w
-
- num_groups = w_padded.shape[0] // gs
- w_grouped = w_padded.reshape(num_groups, gs, out_feat)
-
- scale = jnp.max(jnp.abs(w_grouped), axis=1, keepdims=True) / 127.0
- scale = jnp.maximum(scale, 1e-8)
- w_q = jnp.clip(jnp.round(w_grouped / scale), -128, 127) * scale
-
- w_q = w_q.reshape(-1, out_feat)[:in_feat]
-
- return w + jax.lax.stop_gradient(w_q - w)
-
-
-def _cubic_sparsity_schedule(step, t_start, t_end, s_final):
- """Cubic sparsity ramp (Zhu & Gupta 2017). Returns target sparsity at *step*."""
- if step < t_start:
- return 0.0
- if step >= t_end:
- return s_final
- frac = (step - t_start) / (t_end - t_start)
- return s_final * (1.0 - (1.0 - frac) ** 3)
-
-
-def _make_prune_mask(params, sparsity, group_size):
- """Compute block-prune mask entirely on-device (no numpy round-trips).
-
- Returns binary mask tree matching param shapes and dtypes.
- Handles both 2D (in, out) and 3D (num_layers, in, out) kernels from nn.scan.
- """
- all_scores = []
- for _, leaf in jax.tree_util.tree_leaves_with_path(params):
- if leaf.ndim == 2:
- in_feat, out_feat = leaf.shape
- gs = min(group_size, in_feat)
- pad = (gs - in_feat % gs) % gs
- w = jnp.pad(leaf, ((0, pad), (0, 0))) if pad else leaf
- scores = jnp.sum(jnp.abs(w.reshape(-1, gs, out_feat)), axis=1).ravel()
- all_scores.append(scores)
- elif leaf.ndim == 3:
- num_layers, in_feat, out_feat = leaf.shape
- gs = min(group_size, in_feat)
- pad = (gs - in_feat % gs) % gs
- for li in range(num_layers):
- w = jnp.pad(leaf[li], ((0, pad), (0, 0))) if pad else leaf[li]
- scores = jnp.sum(jnp.abs(w.reshape(-1, gs, out_feat)), axis=1).ravel()
- all_scores.append(scores)
-
- if not all_scores:
- return jax.tree.map(jnp.ones_like, params)
-
- threshold = jnp.percentile(jnp.concatenate(all_scores), sparsity * 100)
-
- def _leaf_mask(path, leaf):
- if leaf.ndim == 3:
- num_layers, in_feat, out_feat = leaf.shape
- gs = min(group_size, in_feat)
- pad = (gs - in_feat % gs) % gs
- masks = []
- for li in range(num_layers):
- w = jnp.pad(leaf[li], ((0, pad), (0, 0))) if pad else leaf[li]
- w_grouped = w.reshape(-1, gs, out_feat)
- block_keep = (jnp.sum(jnp.abs(w_grouped), axis=1, keepdims=True) > threshold)
- m = jnp.broadcast_to(block_keep, w_grouped.shape).reshape(-1, out_feat)[:in_feat]
- masks.append(m)
- return jnp.stack(masks, axis=0).astype(leaf.dtype)
- if leaf.ndim != 2:
- return jnp.ones_like(leaf)
- in_feat, out_feat = leaf.shape
- gs = min(group_size, in_feat)
- pad = (gs - in_feat % gs) % gs
- w = jnp.pad(leaf, ((0, pad), (0, 0))) if pad else leaf
- w_grouped = w.reshape(-1, gs, out_feat)
- block_keep = (jnp.sum(jnp.abs(w_grouped), axis=1, keepdims=True) > threshold)
- return jnp.broadcast_to(block_keep, w_grouped.shape).reshape(-1, out_feat)[:in_feat].astype(leaf.dtype)
-
- return jax.tree_util.tree_map_with_path(_leaf_mask, params)
-
-
-def _quantize_params(params, group_size=32, precision="int4"):
- """Fake-quantize all Dense kernels in the param tree."""
- qfn = _fake_quantize_int8 if precision == "int8" else _fake_quantize_int4
- def _maybe_quantize(path, leaf):
- name = path[-1].key if hasattr(path[-1], "key") else str(path[-1])
- if name == "kernel" and leaf.ndim == 3:
- return jax.vmap(lambda w: qfn(w, group_size=group_size), in_axes=(0,))(leaf)
- if name == "kernel" and leaf.ndim == 2:
- return qfn(leaf, group_size=group_size)
- return leaf
- return jax.tree_util.tree_map_with_path(_maybe_quantize, params)
+from .optim import create_train_state, _wsd_schedule
+from .distributed import _replicate, _unreplicate, shard_batch, _upload_checkpoint
+from .quantize import (
+ _quantize_params, _cubic_sparsity_schedule, _make_prune_mask,
+)
_GROUP_SIZE = 32
@@ -515,9 +222,6 @@ def _estimate_mat_params(config, matryoshka_factor):
return int(total)
-def shard_batch(batch, num_devices):
- """Reshape a batch array so leading dim is (num_devices, per_device_batch, ...)."""
- return batch.reshape(num_devices, -1, *batch.shape[1:])
def train(args):