Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

updates

HenryNdubuakue8a45c3e775c
diff --git a/src/cli.py b/src/cli.py
index f17a7a3..0943516 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -28,6 +28,7 @@ def main():
p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)
p.add_argument("--max-samples", type=int, default=None)
p.add_argument("--warmup-ratio", type=float, default=0.1)
+ p.add_argument("--decay-ratio", type=float, default=0.40)
p.add_argument("--wandb", action="store_true")
p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])
p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
@@ -51,6 +52,10 @@ def main():
help="Dropout rate for residual connections (default: 0.1)")
p.add_argument("--curriculum", action="store_true",
help="Sort batches easy→hard by tool count each epoch")
+ p.add_argument("--contrastive-weight", type=float, default=0.1,
+ help="Weight for CLIP-style contrastive loss (default: 0.1)")
+ p.add_argument("--contrastive-dim", type=int, default=128,
+ help="Dimension of contrastive projection head (default: 128)")
p = sub.add_parser("tokenize", add_help=False)
p.add_argument("--max-samples", type=int, default=None,
@@ -67,6 +72,8 @@ def main():
help="Loss weight for argument key tokens (default: 1.5)")
p.add_argument("--shuffle-tools", action=argparse.BooleanOptionalAction, default=True,
help="Shuffle tool order in encoder input (default: True)")
+ p.add_argument("--max-tool-len", type=int, default=256,
+ help="Max token length for individual tool descriptions (default: 256)")
p = sub.add_parser("run", add_help=False)
p.add_argument("--checkpoint", type=str, required=True)
diff --git a/src/data.py b/src/data.py
index 0e03c45..9c40fd2 100644
--- a/src/data.py
+++ b/src/data.py
@@ -631,9 +631,125 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_
np.save(cache_path + "_tool_count.npy", tool_counts)
print(f"Cached {len(enc_inputs):,} tool-call pairs to {CACHE_DIR}/{cache_id}")
+ max_tool_len = getattr(prepare_tool_call_pairs, '_max_tool_len', 256)
+ _build_contrastive_arrays(ds, enc_texts, tools_texts, cache_path,
+ model_path, max_enc_len, max_tool_len,
+ num_workers, chunk_size, kept_indices)
+
return enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices, tool_counts
+def _build_contrastive_arrays(ds, enc_texts, tools_texts, cache_path,
+ model_path, max_enc_len, max_tool_len,
+ num_workers, chunk_size, kept_indices):
+ """Build and save contrastive arrays: query-only tokens + individual tool tokens."""
+ kept_set = set(kept_indices.tolist()) if kept_indices is not None else set(range(len(enc_texts)))
+ kept_queries = [enc_texts[i] for i in range(len(enc_texts)) if i in kept_set]
+
+ q_chunks = [kept_queries[i:i + chunk_size] for i in range(0, len(kept_queries), chunk_size)]
+ with mp.Pool(num_workers, initializer=_init_worker,
+ initargs=(model_path, max_enc_len)) as pool:
+ q_results = list(tqdm(pool.imap(_tokenize_chunk, q_chunks),
+ total=len(q_chunks), desc="Tokenizing queries (contrastive)"))
+ query_tokens_list = [tok for chunk in q_results for tok in chunk]
+ n_kept = len(query_tokens_list)
+ query_arr = np.full((n_kept, max_enc_len), PAD_ID, dtype=np.int32)
+ for i, toks in enumerate(query_tokens_list):
+ l = min(len(toks), max_enc_len)
+ query_arr[i, :l] = toks[:l]
+ np.save(cache_path + "_query_only.npy", query_arr)
+
+ # Individual tool tokens + mapping arrays
+ tool_texts_individual = []
+ tool_ex_idx = []
+ tool_is_pos = []
+
+ kept_list = sorted(kept_set)
+ kept_to_local = {g: i for i, g in enumerate(kept_list)}
+
+ for global_i in kept_list:
+ local_i = kept_to_local[global_i]
+ tools_str = tools_texts[global_i]
+ ans_str = ds[global_i]["answers"] if global_i < len(ds) else "[]"
+
+ try:
+ tools = _json.loads(tools_str)
+ except (ValueError, TypeError):
+ tools = []
+ if not isinstance(tools, list):
+ tools = []
+
+ try:
+ calls = _json.loads(ans_str)
+ except (ValueError, TypeError):
+ calls = []
+ pos_names = set()
+ if isinstance(calls, list):
+ for c in calls:
+ if isinstance(c, dict) and "name" in c:
+ pos_names.add(c["name"])
+
+ for tool in tools:
+ if not isinstance(tool, dict):
+ continue
+ tool_str = _json.dumps(tool, separators=(",", ":"))
+ tool_texts_individual.append(tool_str)
+ tool_ex_idx.append(local_i)
+ tool_is_pos.append(tool.get("name", "") in pos_names)
+
+ if tool_texts_individual:
+ t_chunks = [tool_texts_individual[i:i + chunk_size]
+ for i in range(0, len(tool_texts_individual), chunk_size)]
+ with mp.Pool(num_workers, initializer=_init_worker,
+ initargs=(model_path, max_tool_len)) as pool:
+ t_results = list(tqdm(pool.imap(_tokenize_chunk, t_chunks),
+ total=len(t_chunks), desc="Tokenizing tools (contrastive)"))
+ tool_tokens_list = [tok for chunk in t_results for tok in chunk]
+
+ n_tools = len(tool_tokens_list)
+ tool_arr = np.full((n_tools, max_tool_len), PAD_ID, dtype=np.int32)
+ for i, toks in enumerate(tool_tokens_list):
+ l = min(len(toks), max_tool_len)
+ tool_arr[i, :l] = toks[:l]
+ else:
+ tool_arr = np.zeros((0, max_tool_len), dtype=np.int32)
+
+ np.save(cache_path + "_tool_individual.npy", tool_arr)
+ np.save(cache_path + "_tool_ex_idx.npy", np.array(tool_ex_idx, dtype=np.int32))
+ np.save(cache_path + "_tool_is_pos.npy", np.array(tool_is_pos, dtype=np.bool_))
+ print(f" Contrastive: {n_kept} queries, {len(tool_ex_idx)} individual tools")
+
+
+def get_contrastive_batches(query_tokens, tool_tokens, tool_ex_idx, tool_is_pos, batch_size):
+ """Yield (query_batch, tool_batch) for CLIP-style contrastive training.
+
+ Each batch: B queries paired with 1 randomly-chosen positive tool each.
+ In-batch negatives provide the contrastive signal.
+ """
+ n_queries = len(query_tokens)
+ pos_map = {}
+ for t_idx in range(len(tool_ex_idx)):
+ if tool_is_pos[t_idx]:
+ pos_map.setdefault(int(tool_ex_idx[t_idx]), []).append(t_idx)
+
+ valid_queries = [i for i in range(n_queries) if i in pos_map]
+ if not valid_queries:
+ return
+
+ indices = np.array(valid_queries)
+ np.random.shuffle(indices)
+
+ for start in range(0, len(indices) - batch_size + 1, batch_size):
+ batch_q_idx = indices[start:start + batch_size]
+ q_batch = np.array(query_tokens[batch_q_idx])
+ t_indices = np.array([
+ pos_map[int(qi)][np.random.randint(len(pos_map[int(qi)]))]
+ for qi in batch_q_idx
+ ])
+ t_batch = np.array(tool_tokens[t_indices])
+ yield q_batch, t_batch
+
+
def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True, loss_mask=None, tool_counts=None):
n = len(enc_inputs)
if tool_counts is not None:
@@ -848,6 +964,12 @@ def load_prepared_data(split, mmap=False):
mmap_mode = "r" if mmap else None
tc_path = cache_path + "_tool_count.npy"
tc = np.load(tc_path, mmap_mode=mmap_mode) if os.path.exists(tc_path) else None
+ def _load_optional(suffix):
+ p = cache_path + suffix
+ if os.path.exists(p):
+ return np.load(p, mmap_mode=mmap_mode)
+ return None
+
return {
"enc_inputs": np.load(cache_path + "_enc.npy", mmap_mode=mmap_mode),
"dec_inputs": np.load(cache_path + "_dec_in.npy", mmap_mode=mmap_mode),
@@ -855,6 +977,10 @@ def load_prepared_data(split, mmap=False):
"loss_mask": np.load(cache_path + "_loss_mask.npy", mmap_mode=mmap_mode),
"kept_indices": np.load(cache_path + "_kept_idx.npy", mmap_mode=mmap_mode),
"tool_counts": tc,
+ "query_only": _load_optional("_query_only.npy"),
+ "tool_individual": _load_optional("_tool_individual.npy"),
+ "tool_ex_idx": _load_optional("_tool_ex_idx.npy"),
+ "tool_is_pos": _load_optional("_tool_is_pos.npy"),
}
diff --git a/src/eval.py b/src/eval.py
index 715adf6..7b30dfd 100644
--- a/src/eval.py
+++ b/src/eval.py
@@ -341,6 +341,90 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=
}
+def benchmark_retrieval(model, params, tokenizer, num_samples=500, max_len=256, ks=(1, 3, 5, 10)):
+ """Benchmark contrastive retrieval: Recall@k and MRR over validation set.
+
+ For each query, ranks all tools from that example by cosine similarity
+ and checks if the positive (called) tools appear in top-k.
+ """
+ import json
+ from .run import encode_for_retrieval
+
+ ds = load_tool_calls("validation", max_samples=num_samples)
+
+ queries = []
+ all_tool_strs = []
+ tool_groups = [] # (start_idx, count, set_of_positive_indices_within_group)
+
+ for ex in ds:
+ query = ex["query"]
+ try:
+ tools = json.loads(ex["tools"])
+ except (ValueError, TypeError):
+ continue
+ if not isinstance(tools, list) or len(tools) == 0:
+ continue
+ try:
+ calls = json.loads(ex["answers"])
+ except (ValueError, TypeError):
+ calls = []
+ pos_names = set()
+ if isinstance(calls, list):
+ for c in calls:
+ if isinstance(c, dict) and "name" in c:
+ pos_names.add(c["name"])
+ if not pos_names:
+ continue
+
+ start = len(all_tool_strs)
+ pos_indices = set()
+ for j, tool in enumerate(tools):
+ if not isinstance(tool, dict):
+ continue
+ all_tool_strs.append(json.dumps(tool, separators=(",", ":")))
+ if tool.get("name", "") in pos_names:
+ pos_indices.add(len(all_tool_strs) - 1 - start)
+
+ if pos_indices:
+ queries.append(query)
+ tool_groups.append((start, len(all_tool_strs) - start, pos_indices))
+
+ if not queries:
+ return {"recall@k": {}, "mrr": 0.0, "num_queries": 0}
+
+ # Encode all queries and tools
+ q_embs = encode_for_retrieval(model, params, tokenizer, queries, max_len=max_len)
+ t_embs = encode_for_retrieval(model, params, tokenizer, all_tool_strs, max_len=max_len)
+
+ max_k = max(ks)
+ recall = {k: 0 for k in ks}
+ mrr_sum = 0.0
+
+ for i, (start, count, pos_set) in enumerate(tool_groups):
+ group_embs = t_embs[start:start + count] # (count, dim)
+ scores = q_embs[i:i+1] @ group_embs.T # (1, count)
+ ranked = np.argsort(-scores[0])
+
+ # MRR: reciprocal rank of first positive
+ for rank, idx in enumerate(ranked):
+ if idx in pos_set:
+ mrr_sum += 1.0 / (rank + 1)
+ break
+
+ # Recall@k
+ for k in ks:
+ top_k_set = set(ranked[:k].tolist())
+ if top_k_set & pos_set:
+ recall[k] += 1
+
+ n = len(queries)
+ return {
+ "recall@k": {k: recall[k] / n for k in ks},
+ "mrr": mrr_sum / n,
+ "num_queries": n,
+ }
+
+
def main(args):
params, config = load_checkpoint(args.checkpoint)
model = EncoderDecoderTransformer(config)
diff --git a/src/model.py b/src/model.py
index 593cddb..14b95ec 100644
--- a/src/model.py
+++ b/src/model.py
@@ -47,6 +47,7 @@ class TransformerConfig:
num_memory_slots: int = 64
n_mels: int = 80
dropout_rate: float = 0.1
+ contrastive_dim: int = 128
@property
def jax_dtype(self):
@@ -309,6 +310,11 @@ class EncoderDecoderTransformer(nn.Module):
self.spec_augment = SpecAugment()
self.encoder = Encoder(self.config)
self.decoder = Decoder(self.config)
+ self.contrastive_proj = nn.Dense(
+ self.config.contrastive_dim, dtype=self.config.jax_dtype,
+ use_bias=False, kernel_init=default_init(), name="contrastive_proj",
+ )
+ self.log_temp = self.param("log_temp", jinit.zeros, ())
def _rope(self, seq_len):
head_dim = self.config.d_model // self.config.num_heads
@@ -337,6 +343,32 @@ class EncoderDecoderTransformer(nn.Module):
logits = x.astype(jnp.float32) @ self.embedding.embedding.T
return logits
+ def _mean_pool(self, encoder_out, enc_mask):
+ """Mean-pool encoder output over non-padded positions. Returns (B, d_model)."""
+ if enc_mask is not None:
+ # enc_mask shape: (B, 1, 1, T) -> (B, T)
+ mask_2d = enc_mask[:, 0, 0, :]
+ else:
+ mask_2d = jnp.ones(encoder_out.shape[:2], dtype=encoder_out.dtype)
+ mask_3d = mask_2d[:, :, None].astype(encoder_out.dtype) # (B, T, 1)
+ summed = jnp.sum(encoder_out * mask_3d, axis=1) # (B, d_model)
+ counts = jnp.maximum(jnp.sum(mask_2d, axis=1, keepdims=True), 1.0) # (B, 1)
+ return summed / counts
+
+ def encode_contrastive(self, tokens, deterministic=True):
+ """Encode tokens and project to L2-normalized contrastive space. Returns (B, contrastive_dim)."""
+ src_mask = make_padding_mask(tokens, self.config.pad_token_id)
+ encoder_out, enc_mask = self.encode_text(tokens, src_mask=src_mask, deterministic=deterministic)
+ pooled = self._mean_pool(encoder_out, enc_mask)
+ projected = self.contrastive_proj(pooled)
+ return projected / jnp.maximum(jnp.linalg.norm(projected, axis=-1, keepdims=True), 1e-8)
+
+ def forward_contrastive(self, query_tokens, tool_tokens, deterministic=True):
+ """Encode query and tool tokens, return (q_emb, t_emb, log_temp)."""
+ q_emb = self.encode_contrastive(query_tokens, deterministic=deterministic)
+ t_emb = self.encode_contrastive(tool_tokens, deterministic=deterministic)
+ return q_emb, t_emb, self.log_temp
+
def __call__(self, src, tgt, src_mask=None, tgt_mask=None):
encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)
logits = self.decode(tgt, encoder_out, self_mask=tgt_mask, cross_mask=enc_mask)
@@ -418,6 +450,7 @@ class EncoderDecoderTransformer(nn.Module):
speech_out, speech_enc_mask = self.encode_speech(mel, src_mask=mel_mask, deterministic=True)
_ = self._run_decoder(text_out, tgt, tgt_mask=tgt_mask, cross_mask=text_enc_mask)
_ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask, cross_mask=speech_enc_mask)
+ _ = self.encode_contrastive(src)
return jnp.zeros(())
diff --git a/src/run.py b/src/run.py
index 20dc1ae..dbdd8f3 100644
--- a/src/run.py
+++ b/src/run.py
@@ -387,6 +387,45 @@ def main(args):
)
+def encode_for_retrieval(model, params, tokenizer, texts, max_len=256, batch_size=64):
+ """Encode texts into contrastive embeddings for retrieval. Returns (N, contrastive_dim) numpy array."""
+ pad_id = tokenizer.pad_token_id
+ all_embs = []
+
+ for start in range(0, len(texts), batch_size):
+ batch_texts = texts[start:start + batch_size]
+ token_lists = [tokenizer.encode(t)[:max_len] for t in batch_texts]
+ max_t = max(len(toks) for toks in token_lists)
+ tokens = np.full((len(batch_texts), max_t), pad_id, dtype=np.int32)
+ for i, toks in enumerate(token_lists):
+ tokens[i, :len(toks)] = toks
+ embs = model.apply(
+ {"params": params}, jnp.array(tokens),
+ deterministic=True, method="encode_contrastive",
+ )
+ all_embs.append(np.array(embs))
+
+ return np.concatenate(all_embs, axis=0)
+
+
+def retrieve_tools(model, params, tokenizer, query, tool_descriptions, top_k=5, max_len=256):
+ """Retrieve top-k tools by cosine similarity to query.
+
+ Args:
+ query: query string
+ tool_descriptions: list of tool description strings (one per tool)
+ top_k: number of results to return
+
+ Returns:
+ list of (index, score) tuples sorted by descending similarity
+ """
+ q_emb = encode_for_retrieval(model, params, tokenizer, [query], max_len=max_len)
+ t_emb = encode_for_retrieval(model, params, tokenizer, tool_descriptions, max_len=max_len)
+ scores = (q_emb @ t_emb.T)[0] # (N_tools,)
+ top_indices = np.argsort(-scores)[:top_k]
+ return [(int(idx), float(scores[idx])) for idx in top_indices]
+
+
def parse_args():
parser = argparse.ArgumentParser(description="Generate tool calls with trained transformer")
parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")
diff --git a/src/tokenize_data.py b/src/tokenize_data.py
index 9735a6c..f6214ec 100644
--- a/src/tokenize_data.py
+++ b/src/tokenize_data.py
@@ -77,6 +77,8 @@ def tokenize(args):
w_value = getattr(args, "w_value", 2.0)
w_key = getattr(args, "w_key", 1.5)
shuffle_tools = getattr(args, "shuffle_tools", True)
+ max_tool_len = getattr(args, "max_tool_len", 256)
+ prepare_tool_call_pairs._max_tool_len = max_tool_len
_, _, _, _, kept_indices, _ = prepare_tool_call_pairs(
ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len,
w_name=w_name, w_value=w_value, w_key=w_key, shuffle_tools=shuffle_tools,
diff --git a/src/train.py b/src/train.py
index 06ba2b5..9eefc6f 100644
--- a/src/train.py
+++ b/src/train.py
@@ -16,6 +16,7 @@ from .data import (
get_batches, get_tokenizer,
load_prepared_data, load_tool_calls,
PrefetchIterator, count_batches,
+ get_contrastive_batches,
)
from .model import (
EncoderDecoderTransformer,
@@ -82,21 +83,21 @@ def _param_labels(params):
return jax.tree_util.tree_map_with_path(_label, params)
-def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.15):
- """Warmup-Stable-Decay schedule: linear warmup, hold peak, linear decay."""
+def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.40):
+ """Warmup-Stable-Decay schedule: linear warmup, hold peak, cosine decay."""
decay_steps = max(1, int(total_steps * decay_ratio))
stable_steps = total_steps - warmup_steps - decay_steps
return optax.join_schedules(
[
optax.linear_schedule(0.0, peak_value, warmup_steps),
optax.constant_schedule(peak_value),
- optax.linear_schedule(peak_value, peak_value * 0.1, decay_steps),
+ optax.cosine_decay_schedule(peak_value, decay_steps, alpha=0.05),
],
boundaries=[warmup_steps, warmup_steps + stable_steps],
)
-def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps):
+def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps, decay_ratio=0.40):
model = EncoderDecoderTransformer(config)
rng, init_rng = jax.random.split(rng)
@@ -111,8 +112,8 @@ def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_
method="init_all",
)
- adam_schedule = _wsd_schedule(learning_rate, total_steps, warmup_steps)
- muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps)
+ adam_schedule = _wsd_schedule(learning_rate, total_steps, warmup_steps, decay_ratio)
+ muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps, decay_ratio)
muon_opt = optax.chain(
scale_by_muon(momentum=0.95, ns_steps=5),
@@ -222,16 +223,29 @@ def _quantize_params(params, group_size=32):
_GROUP_SIZE = 32
_MAT_FACTORS = ()
-_MAT_FF_WIDTHS = ()
-_D_FF = 2048
+_MAT_FF_WIDTHS = ()
+_D_FF = 2048
+_CONTRASTIVE_WEIGHT = 0.1
+
+
+def _clip_contrastive_loss(q_emb, t_emb, log_temp):
+ """CLIP-style symmetric contrastive loss with learnable temperature."""
+ temp = jnp.exp(jnp.clip(log_temp, -jnp.log(100.0), jnp.log(100.0)))
+ logits = q_emb @ t_emb.T / temp # (B, B)
+ B = logits.shape[0]
+ labels = jnp.arange(B)
+ loss_q = optax.softmax_cross_entropy_with_integer_labels(logits, labels)
+ loss_t = optax.softmax_cross_entropy_with_integer_labels(logits.T, labels)
+ return (jnp.mean(loss_q) + jnp.mean(loss_t)) / 2.0
def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask):
pad_id = 0
+ q_params = _quantize_params(params, group_size=_GROUP_SIZE)
src_mask = make_padding_mask(src, pad_id)
tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)
logits, slot_div = state.apply_fn(
- {"params": _quantize_params(params, group_size=_GROUP_SIZE)},
+ {"params": q_params},
src, tgt_in, src_mask=src_mask, tgt_mask=tgt_mask,
ffn_mask=ffn_mask,
deterministic=False,
@@ -247,6 +261,19 @@ def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rn
return ce_loss + z_loss
+def _contrastive_loss_fn(state, params, query_tokens, tool_tokens, rng):
+ """Compute CLIP contrastive loss on query/tool pairs."""
+ q_params = _quantize_params(params, group_size=_GROUP_SIZE)
+ q_emb, t_emb, log_temp = state.apply_fn(
+ {"params": q_params},
+ query_tokens, tool_tokens,
+ deterministic=False,
+ method="forward_contrastive",
+ rngs={"dropout": rng},
+ )
+ return _clip_contrastive_loss(q_emb, t_emb, log_temp)
+
+
def _make_ffn_mask(batch_size, d_ff, mat_ff_widths):
"""Build (batch, d_ff) prefix mask: batch split equally across widths.
@@ -295,6 +322,20 @@ def _train_step_text_masked(state, ema_params, src, tgt_in, tgt_out, causal_mask
return state, ema_params, loss, grad_norm
+def _train_step_contrastive(state, ema_params, query_tokens, tool_tokens, rng):
+ """Separate contrastive training step using CLIP loss."""
+ ema_decay = 0.999
+ cl_loss, grads = jax.value_and_grad(
+ lambda p: _contrastive_loss_fn(state, p, query_tokens, tool_tokens, rng)
+ )(state.params)
+ grads = jax.lax.pmean(grads, axis_name="batch")
+ cl_loss = jax.lax.pmean(cl_loss, axis_name="batch")
+ grads = jax.tree.map(lambda g: g * _CONTRASTIVE_WEIGHT, grads)
+ state = state.apply_gradients(grads=grads)
+ ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, state.params)
+ return state, ema_params, cl_loss
+
+
def _make_p_train_step():
@@ -305,6 +346,10 @@ def _make_p_train_step_masked():
return jax.pmap(_train_step_text_masked, axis_name="batch", donate_argnums=(0, 1))
+def _make_p_train_step_contrastive():
+ return jax.pmap(_train_step_contrastive, axis_name="batch", donate_argnums=(0, 1))
+
+
def _make_val_loss_fn(apply_fn):
@jax.jit
def val_loss_batch(params, src, tgt_in, tgt_out, causal_mask, loss_mask):
@@ -397,6 +442,15 @@ def train(args):
val_loss_mask = val_data["loss_mask"]
print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs (memory-mapped)")
+ # Contrastive data (optional — graceful if missing)
+ cl_query_tokens = train_data.get("query_only")
+ cl_tool_tokens = train_data.get("tool_individual")
+ cl_tool_ex_idx = train_data.get("tool_ex_idx")
+ cl_tool_is_pos = train_data.get("tool_is_pos")
+ has_contrastive = all(x is not None for x in [cl_query_tokens, cl_tool_tokens, cl_tool_ex_idx, cl_tool_is_pos])
+ if has_contrastive:
+ print(f" Contrastive: {len(cl_query_tokens):,} queries, {len(cl_tool_tokens):,} tools")
+
effective_batch_size = args.batch_size * num_devices
resume_checkpoint = getattr(args, "checkpoint", None)
@@ -419,10 +473,12 @@ def train(args):
dtype=args.dtype,
activation=getattr(args, "activation", "drelu"),
num_memory_slots=getattr(args, "num_memory_slots", 64),
+ contrastive_dim=getattr(args, "contrastive_dim", 128),
)
- global _GROUP_SIZE, _MAT_FACTORS, _MAT_FF_WIDTHS, _D_FF
+ global _GROUP_SIZE, _MAT_FACTORS, _MAT_FF_WIDTHS, _D_FF, _CONTRASTIVE_WEIGHT
_GROUP_SIZE = getattr(args, "group_size", 32)
+ _CONTRASTIVE_WEIGHT = getattr(args, "contrastive_weight", 0.1)
_D_FF = config.d_ff
mat_factors_raw = getattr(args, "mat_factors", None)
if mat_factors_raw:
@@ -434,6 +490,7 @@ def train(args):
n_widths = 1 + len(_MAT_FF_WIDTHS) if _MAT_FF_WIDTHS else 1
p_train_step = _make_p_train_step()
p_train_step_masked = _make_p_train_step_masked()
+ p_train_step_contrastive = _make_p_train_step_contrastive()
np.random.seed(args.seed)
rng = jax.random.PRNGKey(args.seed)
@@ -447,7 +504,8 @@ def train(args):
scaled_lr = args.lr * num_devices
muon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(num_devices)
- state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps)
+ decay_ratio = getattr(args, "decay_ratio", 0.40)
+ state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps, decay_ratio)
val_loss_fn = _make_val_loss_fn(state.apply_fn)
if resume_checkpoint:
@@ -459,8 +517,10 @@ def train(args):
ema_params = jax_utils.replicate(ema_params)
param_count = sum(x.size for x in jax.tree.leaves(jax_utils.unreplicate(state).params))
- decay_steps = max(1, int(total_steps * 0.15))
+ decay_steps = max(1, int(total_steps * decay_ratio))
stable_steps = total_steps - warmup_steps - decay_steps
+ best_call_f1 = 0.0
+ best_ckpt_path = None
print(f"\n ─────────────────────────────────────")
print(f" Parameters {param_count:>12,}")
@@ -529,6 +589,16 @@ def train(args):
prefetch=4,
)
+ # Contrastive batch iterator (cycles through contrastive data alongside text)
+ cl_batch_iter = None
+ if has_contrastive and _CONTRASTIVE_WEIGHT > 0:
+ cl_batch_iter = PrefetchIterator(
+ lambda: get_contrastive_batches(
+ cl_query_tokens, cl_tool_tokens, cl_tool_ex_idx, cl_tool_is_pos,
+ unique_batch_size),
+ prefetch=4,
+ )
+
pbar = tqdm(range(text_batches_per_epoch), desc=f"Epoch {epoch + 1}/{args.epochs}")
for step_i in pbar:
@@ -536,6 +606,17 @@ def train(args):
src, tgt_in, tgt_out, lm = next(text_batch_iter)
+ # Get contrastive batch (may be exhausted before text batches)
+ cl_q_b = None
+ cl_t_b = None
+ if cl_batch_iter is not None:
+ try:
+ cl_q, cl_t = next(cl_batch_iter)
+ cl_q_b = shard_batch(cl_q, num_devices)
+ cl_t_b = shard_batch(cl_t, num_devices)
+ except StopIteration:
+ cl_batch_iter = None
+
src_b = shard_batch(src, num_devices)
tgt_in_b = shard_batch(tgt_in, num_devices)
tgt_out_b = shard_batch(tgt_out, num_devices)
@@ -553,6 +634,14 @@ def train(args):
state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, text_rngs, lm_b,
)
+ # Separate contrastive step (if data available)
+ if cl_q_b is not None and cl_t_b is not None:
+ rng, cl_rng = jax.random.split(rng)
+ cl_rngs = jax.random.split(cl_rng, num_devices)
+ state, ema_params, cl_loss = p_train_step_contrastive(
+ state, ema_params, cl_q_b, cl_t_b, cl_rngs,
+ )
+
text_loss_val = float(loss[0])
text_losses.append(text_loss_val)
step_grad_norm = float(grad_norm[0])
@@ -609,6 +698,8 @@ def train(args):
wandb.log(log_dict)
text_batch_iter.close()
+ if cl_batch_iter is not None:
+ cl_batch_iter.close()
if epoch == weight_prune_epoch and not gradual_sparsify_done:
gradual_sparsify_done = True
@@ -840,6 +931,25 @@ def train(args):
tc_metrics["param_miss"] = tc_missing_params / max(tc_total_ref_params, 1)
tc_metrics["value_acc"] = tc_correct_values / max(tc_matched_params, 1)
+ # Save best checkpoint based on call_f1
+ if tc_metrics and tc_metrics["call_f1"] > best_call_f1:
+ best_call_f1 = tc_metrics["call_f1"]
+ best_ckpt_path = os.path.join(args.checkpoint_dir, f"needle_{args.num_layers}_{args.d_model}_best.pkl")
+ params_best = jax.tree.map(np.array, jax_utils.unreplicate(ema_params))
+ with open(best_ckpt_path, "wb") as f:
+ pickle.dump({"params": params_best, "config": config.__dict__}, f)
+ del params_best
+ print(f" ** New best call_f1={best_call_f1:.1%} → {best_ckpt_path}")
+
+ # Contrastive retrieval eval
+ retrieval_metrics = None
+ if has_contrastive and _CONTRASTIVE_WEIGHT > 0:
+ from .eval import benchmark_retrieval
+ retrieval_metrics = benchmark_retrieval(
+ eval_model, eval_params, tokenizer,
+ num_samples=min(500, getattr(args, "max_eval_samples", 500)),
+ )
+
del eval_params
print(f"\n ─────────────────────────────────────")
@@ -867,6 +977,12 @@ def train(args):
print(f" Args acc {tc_metrics['args_acc']:>10.1%}")
print(f" Call F1 {tc_metrics['call_f1']:>10.1%}")
print(f" Exact match {tc_metrics['exact_match']:>10.1%}")
+ if retrieval_metrics and retrieval_metrics["num_queries"] > 0:
+ rm = retrieval_metrics
+ print(f" ─── Retrieval ({rm['num_queries']} queries) ─────")
+ for k, v in sorted(rm["recall@k"].items()):
+ print(f" Recall@{k:<3} {v:>10.1%}")
+ print(f" MRR {rm['mrr']:>10.3f}")
print(f" ─────────────────────────────────────")
print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")
print(f" Latency {tp['avg_latency_s']:>11.3f}s")
@@ -906,10 +1022,16 @@ def train(args):
log_dict["epoch/tc_value_acc"] = tc_metrics["value_acc"]
log_dict["epoch/tc_args_acc"] = tc_metrics["args_acc"]
log_dict["epoch/tc_call_f1"] = tc_metrics["call_f1"]
+ if retrieval_metrics and retrieval_metrics["num_queries"] > 0:
+ for k, v in retrieval_metrics["recall@k"].items():
+ log_dict[f"epoch/retrieval_recall@{k}"] = v
+ log_dict["epoch/retrieval_mrr"] = retrieval_metrics["mrr"]
wandb.log(log_dict)
if use_wandb:
wandb.finish()
+ if best_ckpt_path:
+ print(f"\nBest checkpoint (call_f1={best_call_f1:.1%}): {best_ckpt_path}")
print("\nTraining complete.")