Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
updates
HenryNdubuakue8a45c3e775c
diff --git a/src/cli.py b/src/cli.pyindex f17a7a3..0943516 100644--- a/src/cli.py+++ b/src/cli.py@@ -28,6 +28,7 @@ def main():p.add_argument("--max-dec-len", type=int, default=DEFAULT_MAX_DEC_LEN)p.add_argument("--max-samples", type=int, default=None)p.add_argument("--warmup-ratio", type=float, default=0.1)+ p.add_argument("--decay-ratio", type=float, default=0.40)p.add_argument("--wandb", action="store_true")p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])p.add_argument("--checkpoint-dir", type=str, default="checkpoints")@@ -51,6 +52,10 @@ def main():help="Dropout rate for residual connections (default: 0.1)")p.add_argument("--curriculum", action="store_true",help="Sort batches easy→hard by tool count each epoch")+ p.add_argument("--contrastive-weight", type=float, default=0.1,+ help="Weight for CLIP-style contrastive loss (default: 0.1)")+ p.add_argument("--contrastive-dim", type=int, default=128,+ help="Dimension of contrastive projection head (default: 128)")p = sub.add_parser("tokenize", add_help=False)p.add_argument("--max-samples", type=int, default=None,@@ -67,6 +72,8 @@ def main():help="Loss weight for argument key tokens (default: 1.5)")p.add_argument("--shuffle-tools", action=argparse.BooleanOptionalAction, default=True,help="Shuffle tool order in encoder input (default: True)")+ p.add_argument("--max-tool-len", type=int, default=256,+ help="Max token length for individual tool descriptions (default: 256)")p = sub.add_parser("run", add_help=False)p.add_argument("--checkpoint", type=str, required=True)diff --git a/src/data.py b/src/data.pyindex 0e03c45..9c40fd2 100644--- a/src/data.py+++ b/src/data.py@@ -631,9 +631,125 @@ def prepare_tool_call_pairs(ds, tokenizer, max_enc_len=DEFAULT_MAX_ENC_LEN, max_np.save(cache_path + "_tool_count.npy", tool_counts)print(f"Cached {len(enc_inputs):,} tool-call pairs to {CACHE_DIR}/{cache_id}")+ max_tool_len = getattr(prepare_tool_call_pairs, '_max_tool_len', 256)+ _build_contrastive_arrays(ds, enc_texts, tools_texts, cache_path,+ model_path, max_enc_len, max_tool_len,+ num_workers, chunk_size, kept_indices)+return enc_inputs, dec_inputs, dec_targets, loss_mask, kept_indices, tool_counts+def _build_contrastive_arrays(ds, enc_texts, tools_texts, cache_path,+ model_path, max_enc_len, max_tool_len,+ num_workers, chunk_size, kept_indices):+ """Build and save contrastive arrays: query-only tokens + individual tool tokens."""+ kept_set = set(kept_indices.tolist()) if kept_indices is not None else set(range(len(enc_texts)))+ kept_queries = [enc_texts[i] for i in range(len(enc_texts)) if i in kept_set]++ q_chunks = [kept_queries[i:i + chunk_size] for i in range(0, len(kept_queries), chunk_size)]+ with mp.Pool(num_workers, initializer=_init_worker,+ initargs=(model_path, max_enc_len)) as pool:+ q_results = list(tqdm(pool.imap(_tokenize_chunk, q_chunks),+ total=len(q_chunks), desc="Tokenizing queries (contrastive)"))+ query_tokens_list = [tok for chunk in q_results for tok in chunk]+ n_kept = len(query_tokens_list)+ query_arr = np.full((n_kept, max_enc_len), PAD_ID, dtype=np.int32)+ for i, toks in enumerate(query_tokens_list):+ l = min(len(toks), max_enc_len)+ query_arr[i, :l] = toks[:l]+ np.save(cache_path + "_query_only.npy", query_arr)++ # Individual tool tokens + mapping arrays+ tool_texts_individual = []+ tool_ex_idx = []+ tool_is_pos = []++ kept_list = sorted(kept_set)+ kept_to_local = {g: i for i, g in enumerate(kept_list)}++ for global_i in kept_list:+ local_i = kept_to_local[global_i]+ tools_str = tools_texts[global_i]+ ans_str = ds[global_i]["answers"] if global_i < len(ds) else "[]"++ try:+ tools = _json.loads(tools_str)+ except (ValueError, TypeError):+ tools = []+ if not isinstance(tools, list):+ tools = []++ try:+ calls = _json.loads(ans_str)+ except (ValueError, TypeError):+ calls = []+ pos_names = set()+ if isinstance(calls, list):+ for c in calls:+ if isinstance(c, dict) and "name" in c:+ pos_names.add(c["name"])++ for tool in tools:+ if not isinstance(tool, dict):+ continue+ tool_str = _json.dumps(tool, separators=(",", ":"))+ tool_texts_individual.append(tool_str)+ tool_ex_idx.append(local_i)+ tool_is_pos.append(tool.get("name", "") in pos_names)++ if tool_texts_individual:+ t_chunks = [tool_texts_individual[i:i + chunk_size]+ for i in range(0, len(tool_texts_individual), chunk_size)]+ with mp.Pool(num_workers, initializer=_init_worker,+ initargs=(model_path, max_tool_len)) as pool:+ t_results = list(tqdm(pool.imap(_tokenize_chunk, t_chunks),+ total=len(t_chunks), desc="Tokenizing tools (contrastive)"))+ tool_tokens_list = [tok for chunk in t_results for tok in chunk]++ n_tools = len(tool_tokens_list)+ tool_arr = np.full((n_tools, max_tool_len), PAD_ID, dtype=np.int32)+ for i, toks in enumerate(tool_tokens_list):+ l = min(len(toks), max_tool_len)+ tool_arr[i, :l] = toks[:l]+ else:+ tool_arr = np.zeros((0, max_tool_len), dtype=np.int32)++ np.save(cache_path + "_tool_individual.npy", tool_arr)+ np.save(cache_path + "_tool_ex_idx.npy", np.array(tool_ex_idx, dtype=np.int32))+ np.save(cache_path + "_tool_is_pos.npy", np.array(tool_is_pos, dtype=np.bool_))+ print(f" Contrastive: {n_kept} queries, {len(tool_ex_idx)} individual tools")+++def get_contrastive_batches(query_tokens, tool_tokens, tool_ex_idx, tool_is_pos, batch_size):+ """Yield (query_batch, tool_batch) for CLIP-style contrastive training.++ Each batch: B queries paired with 1 randomly-chosen positive tool each.+ In-batch negatives provide the contrastive signal.+ """+ n_queries = len(query_tokens)+ pos_map = {}+ for t_idx in range(len(tool_ex_idx)):+ if tool_is_pos[t_idx]:+ pos_map.setdefault(int(tool_ex_idx[t_idx]), []).append(t_idx)++ valid_queries = [i for i in range(n_queries) if i in pos_map]+ if not valid_queries:+ return++ indices = np.array(valid_queries)+ np.random.shuffle(indices)++ for start in range(0, len(indices) - batch_size + 1, batch_size):+ batch_q_idx = indices[start:start + batch_size]+ q_batch = np.array(query_tokens[batch_q_idx])+ t_indices = np.array([+ pos_map[int(qi)][np.random.randint(len(pos_map[int(qi)]))]+ for qi in batch_q_idx+ ])+ t_batch = np.array(tool_tokens[t_indices])+ yield q_batch, t_batch++def get_batches(enc_inputs, dec_inputs, dec_targets, batch_size, shuffle=True, loss_mask=None, tool_counts=None):n = len(enc_inputs)if tool_counts is not None:@@ -848,6 +964,12 @@ def load_prepared_data(split, mmap=False):mmap_mode = "r" if mmap else Nonetc_path = cache_path + "_tool_count.npy"tc = np.load(tc_path, mmap_mode=mmap_mode) if os.path.exists(tc_path) else None+ def _load_optional(suffix):+ p = cache_path + suffix+ if os.path.exists(p):+ return np.load(p, mmap_mode=mmap_mode)+ return None+return {"enc_inputs": np.load(cache_path + "_enc.npy", mmap_mode=mmap_mode),"dec_inputs": np.load(cache_path + "_dec_in.npy", mmap_mode=mmap_mode),@@ -855,6 +977,10 @@ def load_prepared_data(split, mmap=False):"loss_mask": np.load(cache_path + "_loss_mask.npy", mmap_mode=mmap_mode),"kept_indices": np.load(cache_path + "_kept_idx.npy", mmap_mode=mmap_mode),"tool_counts": tc,+ "query_only": _load_optional("_query_only.npy"),+ "tool_individual": _load_optional("_tool_individual.npy"),+ "tool_ex_idx": _load_optional("_tool_ex_idx.npy"),+ "tool_is_pos": _load_optional("_tool_is_pos.npy"),}diff --git a/src/eval.py b/src/eval.pyindex 715adf6..7b30dfd 100644--- a/src/eval.py+++ b/src/eval.py@@ -341,6 +341,90 @@ def benchmark_tool_calls(model, params, tokenizer, num_samples=200, max_gen_len=}+def benchmark_retrieval(model, params, tokenizer, num_samples=500, max_len=256, ks=(1, 3, 5, 10)):+ """Benchmark contrastive retrieval: Recall@k and MRR over validation set.++ For each query, ranks all tools from that example by cosine similarity+ and checks if the positive (called) tools appear in top-k.+ """+ import json+ from .run import encode_for_retrieval++ ds = load_tool_calls("validation", max_samples=num_samples)++ queries = []+ all_tool_strs = []+ tool_groups = [] # (start_idx, count, set_of_positive_indices_within_group)++ for ex in ds:+ query = ex["query"]+ try:+ tools = json.loads(ex["tools"])+ except (ValueError, TypeError):+ continue+ if not isinstance(tools, list) or len(tools) == 0:+ continue+ try:+ calls = json.loads(ex["answers"])+ except (ValueError, TypeError):+ calls = []+ pos_names = set()+ if isinstance(calls, list):+ for c in calls:+ if isinstance(c, dict) and "name" in c:+ pos_names.add(c["name"])+ if not pos_names:+ continue++ start = len(all_tool_strs)+ pos_indices = set()+ for j, tool in enumerate(tools):+ if not isinstance(tool, dict):+ continue+ all_tool_strs.append(json.dumps(tool, separators=(",", ":")))+ if tool.get("name", "") in pos_names:+ pos_indices.add(len(all_tool_strs) - 1 - start)++ if pos_indices:+ queries.append(query)+ tool_groups.append((start, len(all_tool_strs) - start, pos_indices))++ if not queries:+ return {"recall@k": {}, "mrr": 0.0, "num_queries": 0}++ # Encode all queries and tools+ q_embs = encode_for_retrieval(model, params, tokenizer, queries, max_len=max_len)+ t_embs = encode_for_retrieval(model, params, tokenizer, all_tool_strs, max_len=max_len)++ max_k = max(ks)+ recall = {k: 0 for k in ks}+ mrr_sum = 0.0++ for i, (start, count, pos_set) in enumerate(tool_groups):+ group_embs = t_embs[start:start + count] # (count, dim)+ scores = q_embs[i:i+1] @ group_embs.T # (1, count)+ ranked = np.argsort(-scores[0])++ # MRR: reciprocal rank of first positive+ for rank, idx in enumerate(ranked):+ if idx in pos_set:+ mrr_sum += 1.0 / (rank + 1)+ break++ # Recall@k+ for k in ks:+ top_k_set = set(ranked[:k].tolist())+ if top_k_set & pos_set:+ recall[k] += 1++ n = len(queries)+ return {+ "recall@k": {k: recall[k] / n for k in ks},+ "mrr": mrr_sum / n,+ "num_queries": n,+ }++def main(args):params, config = load_checkpoint(args.checkpoint)model = EncoderDecoderTransformer(config)diff --git a/src/model.py b/src/model.pyindex 593cddb..14b95ec 100644--- a/src/model.py+++ b/src/model.py@@ -47,6 +47,7 @@ class TransformerConfig:num_memory_slots: int = 64n_mels: int = 80dropout_rate: float = 0.1+ contrastive_dim: int = 128@propertydef jax_dtype(self):@@ -309,6 +310,11 @@ class EncoderDecoderTransformer(nn.Module):self.spec_augment = SpecAugment()self.encoder = Encoder(self.config)self.decoder = Decoder(self.config)+ self.contrastive_proj = nn.Dense(+ self.config.contrastive_dim, dtype=self.config.jax_dtype,+ use_bias=False, kernel_init=default_init(), name="contrastive_proj",+ )+ self.log_temp = self.param("log_temp", jinit.zeros, ())def _rope(self, seq_len):head_dim = self.config.d_model // self.config.num_heads@@ -337,6 +343,32 @@ class EncoderDecoderTransformer(nn.Module):logits = x.astype(jnp.float32) @ self.embedding.embedding.Treturn logits+ def _mean_pool(self, encoder_out, enc_mask):+ """Mean-pool encoder output over non-padded positions. Returns (B, d_model)."""+ if enc_mask is not None:+ # enc_mask shape: (B, 1, 1, T) -> (B, T)+ mask_2d = enc_mask[:, 0, 0, :]+ else:+ mask_2d = jnp.ones(encoder_out.shape[:2], dtype=encoder_out.dtype)+ mask_3d = mask_2d[:, :, None].astype(encoder_out.dtype) # (B, T, 1)+ summed = jnp.sum(encoder_out * mask_3d, axis=1) # (B, d_model)+ counts = jnp.maximum(jnp.sum(mask_2d, axis=1, keepdims=True), 1.0) # (B, 1)+ return summed / counts++ def encode_contrastive(self, tokens, deterministic=True):+ """Encode tokens and project to L2-normalized contrastive space. Returns (B, contrastive_dim)."""+ src_mask = make_padding_mask(tokens, self.config.pad_token_id)+ encoder_out, enc_mask = self.encode_text(tokens, src_mask=src_mask, deterministic=deterministic)+ pooled = self._mean_pool(encoder_out, enc_mask)+ projected = self.contrastive_proj(pooled)+ return projected / jnp.maximum(jnp.linalg.norm(projected, axis=-1, keepdims=True), 1e-8)++ def forward_contrastive(self, query_tokens, tool_tokens, deterministic=True):+ """Encode query and tool tokens, return (q_emb, t_emb, log_temp)."""+ q_emb = self.encode_contrastive(query_tokens, deterministic=deterministic)+ t_emb = self.encode_contrastive(tool_tokens, deterministic=deterministic)+ return q_emb, t_emb, self.log_temp+def __call__(self, src, tgt, src_mask=None, tgt_mask=None):encoder_out, enc_mask = self.encode_text(src, src_mask=src_mask)logits = self.decode(tgt, encoder_out, self_mask=tgt_mask, cross_mask=enc_mask)@@ -418,6 +450,7 @@ class EncoderDecoderTransformer(nn.Module):speech_out, speech_enc_mask = self.encode_speech(mel, src_mask=mel_mask, deterministic=True)_ = self._run_decoder(text_out, tgt, tgt_mask=tgt_mask, cross_mask=text_enc_mask)_ = self._run_decoder(speech_out, tgt, tgt_mask=tgt_mask, cross_mask=speech_enc_mask)+ _ = self.encode_contrastive(src)return jnp.zeros(())diff --git a/src/run.py b/src/run.pyindex 20dc1ae..dbdd8f3 100644--- a/src/run.py+++ b/src/run.py@@ -387,6 +387,45 @@ def main(args):)+def encode_for_retrieval(model, params, tokenizer, texts, max_len=256, batch_size=64):+ """Encode texts into contrastive embeddings for retrieval. Returns (N, contrastive_dim) numpy array."""+ pad_id = tokenizer.pad_token_id+ all_embs = []++ for start in range(0, len(texts), batch_size):+ batch_texts = texts[start:start + batch_size]+ token_lists = [tokenizer.encode(t)[:max_len] for t in batch_texts]+ max_t = max(len(toks) for toks in token_lists)+ tokens = np.full((len(batch_texts), max_t), pad_id, dtype=np.int32)+ for i, toks in enumerate(token_lists):+ tokens[i, :len(toks)] = toks+ embs = model.apply(+ {"params": params}, jnp.array(tokens),+ deterministic=True, method="encode_contrastive",+ )+ all_embs.append(np.array(embs))++ return np.concatenate(all_embs, axis=0)+++def retrieve_tools(model, params, tokenizer, query, tool_descriptions, top_k=5, max_len=256):+ """Retrieve top-k tools by cosine similarity to query.++ Args:+ query: query string+ tool_descriptions: list of tool description strings (one per tool)+ top_k: number of results to return++ Returns:+ list of (index, score) tuples sorted by descending similarity+ """+ q_emb = encode_for_retrieval(model, params, tokenizer, [query], max_len=max_len)+ t_emb = encode_for_retrieval(model, params, tokenizer, tool_descriptions, max_len=max_len)+ scores = (q_emb @ t_emb.T)[0] # (N_tools,)+ top_indices = np.argsort(-scores)[:top_k]+ return [(int(idx), float(scores[idx])) for idx in top_indices]++def parse_args():parser = argparse.ArgumentParser(description="Generate tool calls with trained transformer")parser.add_argument("--checkpoint", type=str, default="checkpoints/checkpoint_epoch3.pkl")diff --git a/src/tokenize_data.py b/src/tokenize_data.pyindex 9735a6c..f6214ec 100644--- a/src/tokenize_data.py+++ b/src/tokenize_data.py@@ -77,6 +77,8 @@ def tokenize(args):w_value = getattr(args, "w_value", 2.0)w_key = getattr(args, "w_key", 1.5)shuffle_tools = getattr(args, "shuffle_tools", True)+ max_tool_len = getattr(args, "max_tool_len", 256)+ prepare_tool_call_pairs._max_tool_len = max_tool_len_, _, _, _, kept_indices, _ = prepare_tool_call_pairs(ds, tokenizer, max_enc_len=max_enc_len, max_dec_len=max_dec_len,w_name=w_name, w_value=w_value, w_key=w_key, shuffle_tools=shuffle_tools,diff --git a/src/train.py b/src/train.pyindex 06ba2b5..9eefc6f 100644--- a/src/train.py+++ b/src/train.py@@ -16,6 +16,7 @@ from .data import (get_batches, get_tokenizer,load_prepared_data, load_tool_calls,PrefetchIterator, count_batches,+ get_contrastive_batches,)from .model import (EncoderDecoderTransformer,@@ -82,21 +83,21 @@ def _param_labels(params):return jax.tree_util.tree_map_with_path(_label, params)-def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.15):- """Warmup-Stable-Decay schedule: linear warmup, hold peak, linear decay."""+def _wsd_schedule(peak_value, total_steps, warmup_steps, decay_ratio=0.40):+ """Warmup-Stable-Decay schedule: linear warmup, hold peak, cosine decay."""decay_steps = max(1, int(total_steps * decay_ratio))stable_steps = total_steps - warmup_steps - decay_stepsreturn optax.join_schedules([optax.linear_schedule(0.0, peak_value, warmup_steps),optax.constant_schedule(peak_value),- optax.linear_schedule(peak_value, peak_value * 0.1, decay_steps),+ optax.cosine_decay_schedule(peak_value, decay_steps, alpha=0.05),],boundaries=[warmup_steps, warmup_steps + stable_steps],)-def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps):+def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_steps, decay_ratio=0.40):model = EncoderDecoderTransformer(config)rng, init_rng = jax.random.split(rng)@@ -111,8 +112,8 @@ def create_train_state(rng, config, learning_rate, muon_lr, total_steps, warmup_method="init_all",)- adam_schedule = _wsd_schedule(learning_rate, total_steps, warmup_steps)- muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps)+ adam_schedule = _wsd_schedule(learning_rate, total_steps, warmup_steps, decay_ratio)+ muon_schedule = _wsd_schedule(muon_lr, total_steps, warmup_steps, decay_ratio)muon_opt = optax.chain(scale_by_muon(momentum=0.95, ns_steps=5),@@ -222,16 +223,29 @@ def _quantize_params(params, group_size=32):_GROUP_SIZE = 32_MAT_FACTORS = ()-_MAT_FF_WIDTHS = ()-_D_FF = 2048+_MAT_FF_WIDTHS = ()+_D_FF = 2048+_CONTRASTIVE_WEIGHT = 0.1+++def _clip_contrastive_loss(q_emb, t_emb, log_temp):+ """CLIP-style symmetric contrastive loss with learnable temperature."""+ temp = jnp.exp(jnp.clip(log_temp, -jnp.log(100.0), jnp.log(100.0)))+ logits = q_emb @ t_emb.T / temp # (B, B)+ B = logits.shape[0]+ labels = jnp.arange(B)+ loss_q = optax.softmax_cross_entropy_with_integer_labels(logits, labels)+ loss_t = optax.softmax_cross_entropy_with_integer_labels(logits.T, labels)+ return (jnp.mean(loss_q) + jnp.mean(loss_t)) / 2.0def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rng, loss_mask):pad_id = 0+ q_params = _quantize_params(params, group_size=_GROUP_SIZE)src_mask = make_padding_mask(src, pad_id)tgt_mask = causal_mask & make_padding_mask(tgt_in, pad_id)logits, slot_div = state.apply_fn(- {"params": _quantize_params(params, group_size=_GROUP_SIZE)},+ {"params": q_params},src, tgt_in, src_mask=src_mask, tgt_mask=tgt_mask,ffn_mask=ffn_mask,deterministic=False,@@ -247,6 +261,19 @@ def _text_loss_fn(state, params, src, tgt_in, tgt_out, causal_mask, ffn_mask, rnreturn ce_loss + z_loss+def _contrastive_loss_fn(state, params, query_tokens, tool_tokens, rng):+ """Compute CLIP contrastive loss on query/tool pairs."""+ q_params = _quantize_params(params, group_size=_GROUP_SIZE)+ q_emb, t_emb, log_temp = state.apply_fn(+ {"params": q_params},+ query_tokens, tool_tokens,+ deterministic=False,+ method="forward_contrastive",+ rngs={"dropout": rng},+ )+ return _clip_contrastive_loss(q_emb, t_emb, log_temp)++def _make_ffn_mask(batch_size, d_ff, mat_ff_widths):"""Build (batch, d_ff) prefix mask: batch split equally across widths.@@ -295,6 +322,20 @@ def _train_step_text_masked(state, ema_params, src, tgt_in, tgt_out, causal_maskreturn state, ema_params, loss, grad_norm+def _train_step_contrastive(state, ema_params, query_tokens, tool_tokens, rng):+ """Separate contrastive training step using CLIP loss."""+ ema_decay = 0.999+ cl_loss, grads = jax.value_and_grad(+ lambda p: _contrastive_loss_fn(state, p, query_tokens, tool_tokens, rng)+ )(state.params)+ grads = jax.lax.pmean(grads, axis_name="batch")+ cl_loss = jax.lax.pmean(cl_loss, axis_name="batch")+ grads = jax.tree.map(lambda g: g * _CONTRASTIVE_WEIGHT, grads)+ state = state.apply_gradients(grads=grads)+ ema_params = jax.tree.map(lambda e, p: ema_decay * e + (1 - ema_decay) * p, ema_params, state.params)+ return state, ema_params, cl_loss++def _make_p_train_step():@@ -305,6 +346,10 @@ def _make_p_train_step_masked():return jax.pmap(_train_step_text_masked, axis_name="batch", donate_argnums=(0, 1))+def _make_p_train_step_contrastive():+ return jax.pmap(_train_step_contrastive, axis_name="batch", donate_argnums=(0, 1))++def _make_val_loss_fn(apply_fn):@jax.jitdef val_loss_batch(params, src, tgt_in, tgt_out, causal_mask, loss_mask):@@ -397,6 +442,15 @@ def train(args):val_loss_mask = val_data["loss_mask"]print(f" {len(enc_inputs):,} train / {len(val_enc):,} val tool-call pairs (memory-mapped)")+ # Contrastive data (optional — graceful if missing)+ cl_query_tokens = train_data.get("query_only")+ cl_tool_tokens = train_data.get("tool_individual")+ cl_tool_ex_idx = train_data.get("tool_ex_idx")+ cl_tool_is_pos = train_data.get("tool_is_pos")+ has_contrastive = all(x is not None for x in [cl_query_tokens, cl_tool_tokens, cl_tool_ex_idx, cl_tool_is_pos])+ if has_contrastive:+ print(f" Contrastive: {len(cl_query_tokens):,} queries, {len(cl_tool_tokens):,} tools")+effective_batch_size = args.batch_size * num_devicesresume_checkpoint = getattr(args, "checkpoint", None)@@ -419,10 +473,12 @@ def train(args):dtype=args.dtype,activation=getattr(args, "activation", "drelu"),num_memory_slots=getattr(args, "num_memory_slots", 64),+ contrastive_dim=getattr(args, "contrastive_dim", 128),)- global _GROUP_SIZE, _MAT_FACTORS, _MAT_FF_WIDTHS, _D_FF+ global _GROUP_SIZE, _MAT_FACTORS, _MAT_FF_WIDTHS, _D_FF, _CONTRASTIVE_WEIGHT_GROUP_SIZE = getattr(args, "group_size", 32)+ _CONTRASTIVE_WEIGHT = getattr(args, "contrastive_weight", 0.1)_D_FF = config.d_ffmat_factors_raw = getattr(args, "mat_factors", None)if mat_factors_raw:@@ -434,6 +490,7 @@ def train(args):n_widths = 1 + len(_MAT_FF_WIDTHS) if _MAT_FF_WIDTHS else 1p_train_step = _make_p_train_step()p_train_step_masked = _make_p_train_step_masked()+ p_train_step_contrastive = _make_p_train_step_contrastive()np.random.seed(args.seed)rng = jax.random.PRNGKey(args.seed)@@ -447,7 +504,8 @@ def train(args):scaled_lr = args.lr * num_devicesmuon_lr = getattr(args, "muon_lr", 0.02) * math.sqrt(num_devices)- state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps)+ decay_ratio = getattr(args, "decay_ratio", 0.40)+ state = create_train_state(init_rng, config, scaled_lr, muon_lr, total_steps, warmup_steps, decay_ratio)val_loss_fn = _make_val_loss_fn(state.apply_fn)if resume_checkpoint:@@ -459,8 +517,10 @@ def train(args):ema_params = jax_utils.replicate(ema_params)param_count = sum(x.size for x in jax.tree.leaves(jax_utils.unreplicate(state).params))- decay_steps = max(1, int(total_steps * 0.15))+ decay_steps = max(1, int(total_steps * decay_ratio))stable_steps = total_steps - warmup_steps - decay_steps+ best_call_f1 = 0.0+ best_ckpt_path = Noneprint(f"\n ─────────────────────────────────────")print(f" Parameters {param_count:>12,}")@@ -529,6 +589,16 @@ def train(args):prefetch=4,)+ # Contrastive batch iterator (cycles through contrastive data alongside text)+ cl_batch_iter = None+ if has_contrastive and _CONTRASTIVE_WEIGHT > 0:+ cl_batch_iter = PrefetchIterator(+ lambda: get_contrastive_batches(+ cl_query_tokens, cl_tool_tokens, cl_tool_ex_idx, cl_tool_is_pos,+ unique_batch_size),+ prefetch=4,+ )+pbar = tqdm(range(text_batches_per_epoch), desc=f"Epoch {epoch + 1}/{args.epochs}")for step_i in pbar:@@ -536,6 +606,17 @@ def train(args):src, tgt_in, tgt_out, lm = next(text_batch_iter)+ # Get contrastive batch (may be exhausted before text batches)+ cl_q_b = None+ cl_t_b = None+ if cl_batch_iter is not None:+ try:+ cl_q, cl_t = next(cl_batch_iter)+ cl_q_b = shard_batch(cl_q, num_devices)+ cl_t_b = shard_batch(cl_t, num_devices)+ except StopIteration:+ cl_batch_iter = None+src_b = shard_batch(src, num_devices)tgt_in_b = shard_batch(tgt_in, num_devices)tgt_out_b = shard_batch(tgt_out, num_devices)@@ -553,6 +634,14 @@ def train(args):state, ema_params, src_b, tgt_in_b, tgt_out_b, causal_mask, text_ffn_mask, text_rngs, lm_b,)+ # Separate contrastive step (if data available)+ if cl_q_b is not None and cl_t_b is not None:+ rng, cl_rng = jax.random.split(rng)+ cl_rngs = jax.random.split(cl_rng, num_devices)+ state, ema_params, cl_loss = p_train_step_contrastive(+ state, ema_params, cl_q_b, cl_t_b, cl_rngs,+ )+text_loss_val = float(loss[0])text_losses.append(text_loss_val)step_grad_norm = float(grad_norm[0])@@ -609,6 +698,8 @@ def train(args):wandb.log(log_dict)text_batch_iter.close()+ if cl_batch_iter is not None:+ cl_batch_iter.close()if epoch == weight_prune_epoch and not gradual_sparsify_done:gradual_sparsify_done = True@@ -840,6 +931,25 @@ def train(args):tc_metrics["param_miss"] = tc_missing_params / max(tc_total_ref_params, 1)tc_metrics["value_acc"] = tc_correct_values / max(tc_matched_params, 1)+ # Save best checkpoint based on call_f1+ if tc_metrics and tc_metrics["call_f1"] > best_call_f1:+ best_call_f1 = tc_metrics["call_f1"]+ best_ckpt_path = os.path.join(args.checkpoint_dir, f"needle_{args.num_layers}_{args.d_model}_best.pkl")+ params_best = jax.tree.map(np.array, jax_utils.unreplicate(ema_params))+ with open(best_ckpt_path, "wb") as f:+ pickle.dump({"params": params_best, "config": config.__dict__}, f)+ del params_best+ print(f" ** New best call_f1={best_call_f1:.1%} → {best_ckpt_path}")++ # Contrastive retrieval eval+ retrieval_metrics = None+ if has_contrastive and _CONTRASTIVE_WEIGHT > 0:+ from .eval import benchmark_retrieval+ retrieval_metrics = benchmark_retrieval(+ eval_model, eval_params, tokenizer,+ num_samples=min(500, getattr(args, "max_eval_samples", 500)),+ )+del eval_paramsprint(f"\n ─────────────────────────────────────")@@ -867,6 +977,12 @@ def train(args):print(f" Args acc {tc_metrics['args_acc']:>10.1%}")print(f" Call F1 {tc_metrics['call_f1']:>10.1%}")print(f" Exact match {tc_metrics['exact_match']:>10.1%}")+ if retrieval_metrics and retrieval_metrics["num_queries"] > 0:+ rm = retrieval_metrics+ print(f" ─── Retrieval ({rm['num_queries']} queries) ─────")+ for k, v in sorted(rm["recall@k"].items()):+ print(f" Recall@{k:<3} {v:>10.1%}")+ print(f" MRR {rm['mrr']:>10.3f}")print(f" ─────────────────────────────────────")print(f" Throughput {tp['tokens_per_second']:>10.1f} tok/s")print(f" Latency {tp['avg_latency_s']:>11.3f}s")@@ -906,10 +1022,16 @@ def train(args):log_dict["epoch/tc_value_acc"] = tc_metrics["value_acc"]log_dict["epoch/tc_args_acc"] = tc_metrics["args_acc"]log_dict["epoch/tc_call_f1"] = tc_metrics["call_f1"]+ if retrieval_metrics and retrieval_metrics["num_queries"] > 0:+ for k, v in retrieval_metrics["recall@k"].items():+ log_dict[f"epoch/retrieval_recall@{k}"] = v+ log_dict["epoch/retrieval_mrr"] = retrieval_metrics["mrr"]wandb.log(log_dict)if use_wandb:wandb.finish()+ if best_ckpt_path:+ print(f"\nBest checkpoint (call_f1={best_call_f1:.1%}): {best_ckpt_path}")print("\nTraining complete.")