Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

GRPO

HenryNdubuakue767e9a7d911
diff --git a/src/cli.py b/src/cli.py
index 12c7186..3be4001 100644
--- a/src/cli.py
+++ b/src/cli.py
@@ -205,6 +205,31 @@ def main():
p.add_argument("--activation", type=str, default="swiglu", choices=["drelu", "swiglu", "geglu"])
p.add_argument("--no-feedforward", action=argparse.BooleanOptionalAction, default=True)
+ p = sub.add_parser("grpo", add_help=False)
+ p.add_argument("--checkpoint", type=str, default="checkpoints/needle.pkl",
+ help="SFT checkpoint to init both policy and reference (default: checkpoints/needle.pkl)")
+ p.add_argument("--name", type=str, default="grpo")
+ p.add_argument("--batch-size", type=int, default=16,
+ help="Prompts per step (B)")
+ p.add_argument("--k", type=int, default=4,
+ help="Samples per prompt (K)")
+ p.add_argument("--lr", type=float, default=1e-6)
+ p.add_argument("--beta", type=float, default=0.04,
+ help="KL penalty weight against frozen reference")
+ p.add_argument("--max-steps", type=int, default=5000)
+ p.add_argument("--max-enc-len", type=int, default=DEFAULT_MAX_ENC_LEN)
+ p.add_argument("--max-gen-len", type=int, default=DEFAULT_MAX_GEN_LEN)
+ p.add_argument("--eval-every", type=int, default=100,
+ help="Run quick val PPL every N steps (default: 100)")
+ p.add_argument("--eval-batch-size", type=int, default=32,
+ help="Batch size for val PPL eval")
+ p.add_argument("--tool-call-samples", type=int, default=100,
+ help="Samples for stratified tool-call benchmark at save time (split evenly single/multi)")
+ p.add_argument("--checkpoint-dir", type=str, default="checkpoints")
+ p.add_argument("--wandb", action="store_true")
+ p.add_argument("--dtype", type=str, default="bfloat16", choices=["float32", "bfloat16"])
+ p.add_argument("--seed", type=int, default=42)
+
p = sub.add_parser("tokenize", add_help=False)
p.add_argument("--max-samples", type=int, default=None,
help="Limit samples per split (for dev/test)")
@@ -356,6 +381,12 @@ def main():
jax.distributed.initialize()
from .train import train
train(args)
+ elif args.command == "grpo":
+ import jax
+ if os.path.exists("/dev/accel0"):
+ jax.distributed.initialize()
+ from .grpo import grpo
+ grpo(args)
elif args.command == "run":
from .run import main as run_main
run_main(args)
diff --git a/src/grpo.py b/src/grpo.py
new file mode 100644
index 0000000..e2fd953
--- /dev/null
+++ b/src/grpo.py
@@ -0,0 +1,609 @@
+"""GRPO fine-tuning from an SFT checkpoint.
+
+For each prompt we sample K completions from the current policy, score them
+with a programmatic reward (exact call-set match against the reference answer),
+normalize rewards within the group to get advantages, and take a policy
+gradient step with a KL penalty against a frozen reference (the SFT ckpt).
+
+No KV cache, no constrained decoding, no reward model.
+
+Usage:
+ needle grpo --checkpoint checkpoints/needle.pkl --wandb
+"""
+
+import json
+import math
+import os
+import pickle
+import time
+
+import jax
+import jax.numpy as jnp
+import numpy as np
+import optax
+from flax.training import train_state
+from tqdm import tqdm
+
+from .data import (
+ load_tool_calls, load_prepared_data, get_batches, to_snake_case,
+)
+from .distributed import (
+ _replicate, _unreplicate, _upload_checkpoint,
+)
+
+_HF_SFT_REPO = "Cactus-Compute/needle"
+
+
+def _download_sft_checkpoint(path_or_name):
+ """Return a local path to the SFT checkpoint, downloading from HF if needed.
+
+ The SFT checkpoint lives in Cactus-Compute/needle, not the generic
+ checkpoints repo that _download_checkpoint targets.
+ """
+ if os.path.exists(path_or_name):
+ return path_or_name
+ from huggingface_hub import hf_hub_download
+ local_dir = os.path.dirname(path_or_name) or "checkpoints"
+ os.makedirs(local_dir, exist_ok=True)
+ print(f" Downloading {os.path.basename(path_or_name)} "
+ f"from {_HF_SFT_REPO}...", flush=True)
+ local_path = hf_hub_download(
+ repo_id=_HF_SFT_REPO,
+ filename=os.path.basename(path_or_name),
+ repo_type="model",
+ local_dir=local_dir,
+ )
+ print(f" Downloaded to {local_path}", flush=True)
+ return local_path
+from .model import (
+ EncoderDecoderTransformer, TransformerConfig,
+ make_causal_mask, make_padding_mask,
+ make_packing_mask, make_causal_packing_mask, make_cross_packing_mask,
+)
+from .optim import _wsd_schedule
+from .run import _build_encoder_input
+from .tokenizer import EOS_ID, PAD_ID, get_tokenizer
+
+
+def _call_key_set(calls):
+ """Canonicalize a list of tool calls: snake_case names, sorted."""
+ keys = []
+ for c in calls:
+ if not isinstance(c, dict):
+ continue
+ name = to_snake_case(c.get("name", ""))
+ args = c.get("arguments", {}) or {}
+ keys.append(json.dumps({"name": name, "arguments": args}, sort_keys=True))
+ return sorted(keys)
+
+
+def _score(pred_text, ref_answers):
+ """Binary reward: 1.0 if pred call-set == ref call-set (order-invariant)."""
+ if pred_text.startswith("<tool_call>"):
+ pred_text = pred_text[len("<tool_call>"):]
+ try:
+ pred_calls = json.loads(pred_text)
+ if isinstance(pred_calls, dict):
+ pred_calls = [pred_calls]
+ elif not isinstance(pred_calls, list):
+ return 0.0
+ except (json.JSONDecodeError, TypeError):
+ return 0.0
+ try:
+ ref_calls = json.loads(ref_answers)
+ if not isinstance(ref_calls, list):
+ ref_calls = []
+ except (json.JSONDecodeError, TypeError):
+ return 0.0
+ return 1.0 if _call_key_set(pred_calls) == _call_key_set(ref_calls) else 0.0
+
+
+def _build_prompts(tokenizer, batch_examples, max_enc_len):
+ """Tokenize and pad encoder inputs for a batch of prompts."""
+ enc_lists = [
+ _build_encoder_input(tokenizer, ex["query"], ex["tools"], max_enc_len)
+ for ex in batch_examples
+ ]
+ B = len(enc_lists)
+ enc = np.full((B, max_enc_len), PAD_ID, dtype=np.int32)
+ for i, toks in enumerate(enc_lists):
+ enc[i, :len(toks)] = toks
+ return enc
+
+
+def _make_fns(model, max_gen_len):
+ """Build pmapped encode / sample-step / log-prob functions."""
+
+ @jax.pmap
+ def p_encode(params, enc_input):
+ src_mask = make_padding_mask(enc_input, PAD_ID)
+ return model.apply({"params": params}, enc_input,
+ src_mask=src_mask, method="encode")
+
+ @jax.pmap
+ def p_sample_step(params, dec_buffer, encoder_out, cross_mask,
+ rng, step_idx, finished):
+ sm = make_causal_mask(max_gen_len)
+ logits = model.apply(
+ {"params": params}, dec_buffer, encoder_out,
+ self_mask=sm, cross_mask=cross_mask, method="decode",
+ )
+ next_logits = jax.lax.dynamic_index_in_dim(
+ logits, step_idx, axis=1, keepdims=False,
+ )
+ next_tokens = jax.random.categorical(rng, next_logits, axis=-1)
+ next_tokens = jnp.where(finished, PAD_ID, next_tokens)
+ new_finished = finished | (next_tokens == EOS_ID)
+ return next_tokens, new_finished
+
+ @jax.pmap
+ def p_log_probs(params, enc_input, dec_in, dec_tgt):
+ src_mask = make_padding_mask(enc_input, PAD_ID)
+ tm = make_causal_mask(dec_in.shape[-1]) & make_padding_mask(dec_in, PAD_ID)
+ logits = model.apply({"params": params}, enc_input, dec_in,
+ src_mask=src_mask, tgt_mask=tm)
+ lp = jax.nn.log_softmax(logits.astype(jnp.float32), axis=-1)
+ return jnp.take_along_axis(lp, dec_tgt[..., None], axis=-1).squeeze(-1)
+
+ return p_encode, p_sample_step, p_log_probs
+
+
+def _make_grpo_step(model, beta):
+ """Build pmapped GRPO gradient step."""
+
+ def loss_fn(params, enc_input, dec_in, dec_tgt, advantages,
+ response_mask, ref_logp):
+ src_mask = make_padding_mask(enc_input, PAD_ID)
+ tm = make_causal_mask(dec_in.shape[-1]) & make_padding_mask(dec_in, PAD_ID)
+ logits = model.apply({"params": params}, enc_input, dec_in,
+ src_mask=src_mask, tgt_mask=tm)
+ lp = jax.nn.log_softmax(logits.astype(jnp.float32), axis=-1)
+ pol_logp = jnp.take_along_axis(lp, dec_tgt[..., None], axis=-1).squeeze(-1)
+
+ denom = jnp.maximum(response_mask.sum(), 1.0)
+ pg = -(advantages[:, None] * pol_logp * response_mask).sum() / denom
+
+ # Schulman k3 KL estimator: unbiased, always non-negative.
+ log_ratio = ref_logp - pol_logp
+ kl_tok = (jnp.exp(log_ratio) - log_ratio - 1.0) * response_mask
+ kl = kl_tok.sum() / denom
+
+ return pg + beta * kl, (pg, kl)
+
+ def step(state, enc_input, dec_in, dec_tgt, advantages,
+ response_mask, ref_logp):
+ (loss, (pg, kl)), grads = jax.value_and_grad(loss_fn, has_aux=True)(
+ state.params, enc_input, dec_in, dec_tgt, advantages,
+ response_mask, ref_logp,
+ )
+ grads = jax.lax.pmean(grads, axis_name="batch")
+ loss = jax.lax.pmean(loss, axis_name="batch")
+ pg = jax.lax.pmean(pg, axis_name="batch")
+ kl = jax.lax.pmean(kl, axis_name="batch")
+ state = state.apply_gradients(grads=grads)
+ return state, loss, pg, kl
+
+ return jax.pmap(step, axis_name="batch", donate_argnums=(0,))
+
+
+def _sample_completions(p_encode, p_sample_step, r_params, enc_sharded,
+ num_devices, max_gen_len, rng):
+ """Sample one completion per input row, in parallel across devices.
+
+ enc_sharded: (num_devices, per_device, max_enc_len) already on-device.
+ Returns numpy array (num_devices, per_device, max_gen_len): each row is
+ [EOS, sampled..., EOS, PAD...] or [EOS, sampled...] if EOS wasn't emitted.
+ """
+ encoder_out, cross_mask = p_encode(r_params, enc_sharded)
+ per_device = enc_sharded.shape[1]
+
+ dec_buffer = np.full((num_devices, per_device, max_gen_len), PAD_ID, dtype=np.int32)
+ dec_buffer[:, :, 0] = EOS_ID
+ dec_buffer = jnp.asarray(dec_buffer)
+ finished = jnp.zeros((num_devices, per_device), dtype=jnp.bool_)
+
+ for step in range(max_gen_len - 1):
+ rng, sub = jax.random.split(rng)
+ sub_per = jax.random.split(sub, num_devices)
+ step_idx = jnp.full((num_devices,), step, dtype=jnp.int32)
+ next_tokens, finished = p_sample_step(
+ r_params, dec_buffer, encoder_out, cross_mask,
+ sub_per, step_idx, finished,
+ )
+ dec_buffer = dec_buffer.at[:, :, step + 1].set(next_tokens)
+
+ return np.asarray(dec_buffer), rng
+
+
+def _make_val_loss_fn(model):
+ """Teacher-forced packed val loss (matches train.py)."""
+
+ @jax.jit
+ def fn(params, src, dec_in, dec_tgt, enc_seg, dec_seg):
+ src_mask = make_packing_mask(enc_seg)
+ tgt_mask = make_causal_packing_mask(dec_seg)
+ cross_mask = make_cross_packing_mask(enc_seg, dec_seg)
+ logits = model.apply({"params": params}, src, dec_in,
+ src_mask=src_mask, tgt_mask=tgt_mask,
+ cross_mask=cross_mask)
+ loss = optax.softmax_cross_entropy_with_integer_labels(
+ logits.astype(jnp.float32), dec_tgt)
+ pmask = (dec_seg > 0).astype(jnp.float32)
+ return jnp.sum(loss * pmask), jnp.sum(pmask)
+
+ return fn
+
+
+def _compute_val_ppl(val_loss_fn, params, val_data, batch_size):
+ total_loss = 0.0
+ total_toks = 0.0
+ for vb in get_batches(
+ val_data["packed_enc"], val_data["packed_dec_in"], val_data["packed_dec_tgt"],
+ batch_size, shuffle=False,
+ loss_mask=val_data["packed_loss"],
+ enc_seg_ids=val_data["packed_enc_seg"],
+ dec_seg_ids=val_data["packed_dec_seg"],
+ ):
+ src, di, dt, _lm, es, ds = vb
+ vl, vt = val_loss_fn(
+ params,
+ jnp.asarray(src, dtype=jnp.int32),
+ jnp.asarray(di, dtype=jnp.int32),
+ jnp.asarray(dt, dtype=jnp.int32),
+ jnp.asarray(es, dtype=jnp.int32),
+ jnp.asarray(ds, dtype=jnp.int32),
+ )
+ total_loss += float(vl)
+ total_toks += float(vt)
+ return math.exp(min(total_loss / max(total_toks, 1), 20))
+
+
+def _stratified_val_slice(val_ds, per_bucket):
+ """Pick up to per_bucket single-call + per_bucket multi-call examples."""
+ single, multi = [], []
+ for ex in val_ds:
+ try:
+ calls = json.loads(ex["answers"])
+ except (json.JSONDecodeError, TypeError):
+ continue
+ if not isinstance(calls, list) or not calls:
+ continue
+ (single if len(calls) == 1 else multi).append(ex)
+ if len(single) >= per_bucket and len(multi) >= per_bucket:
+ break
+ return single[:per_bucket], multi[:per_bucket]
+
+
+def _run_tool_call_eval(model, params, tokenizer, val_ds, per_bucket,
+ max_enc_len, max_gen_len):
+ """Stratified single/multi-call benchmark with display samples.
+
+ Mirrors train.py's end-of-epoch tool-call eval. Uses greedy decode WITHOUT
+ constrained decoding — this reveals whether the underlying model is
+ actually learning to produce correct tool names, rather than leaning on
+ the grammar. `constrained=True` is the right choice at deployment; for
+ training telemetry we want to see raw model behavior.
+ """
+ from .eval import benchmark_tool_calls
+
+ single, multi = _stratified_val_slice(val_ds, per_bucket)
+ results = {}
+ for label, examples in (("single", single), ("multi", multi)):
+ if not examples:
+ continue
+ tc = benchmark_tool_calls(
+ model, params, tokenizer, num_samples=len(examples),
+ max_gen_len=max_gen_len, max_enc_len=max_enc_len,
+ constrained=False, ds=examples,
+ )
+ results[label] = {
+ "name_f1": tc["name_f1"],
+ "call_f1": tc["call_f1"],
+ "exact_match": tc["exact_match"],
+ "args_acc": tc["args_acc"],
+ "value_acc": tc["value_acc"],
+ "param_haluc": tc["param_haluc"],
+ "param_miss": tc["param_miss"],
+ "json_parse": tc["json_parse_rate"],
+ "samples": tc["samples"],
+ "n": len(examples),
+ }
+ return results
+
+
+def _print_eval_block(step, val_ppl, tc_results, pbar):
+ """Mirror the end-of-epoch print layout from train.py."""
+ pbar.write(f"\n ─────── step {step} eval ───────")
+ if val_ppl is not None:
+ pbar.write(f" val_ppl {val_ppl:>10.3f}")
+ for label, m in tc_results.items():
+ pbar.write(f" ─── {label} ({m['n']} samples, unconstrained) ───")
+ pbar.write(f" JSON parse {m['json_parse']:>10.1%}")
+ pbar.write(f" Name F1 {m['name_f1']:>10.1%}")
+ pbar.write(f" Call F1 {m['call_f1']:>10.1%}")
+ pbar.write(f" Exact match {m['exact_match']:>10.1%}")
+ pbar.write(f" Args acc {m['args_acc']:>10.1%}")
+ pbar.write(f" Value acc {m['value_acc']:>10.1%}")
+ pbar.write(f" Param haluc {m['param_haluc']:>10.1%}")
+ pbar.write(f" Param miss {m['param_miss']:>10.1%}")
+ # Show up to 3 display samples from the "single" pool
+ single = tc_results.get("single") or tc_results.get("multi")
+ if single and single.get("samples"):
+ pbar.write(f" ─── samples ───")
+ for q, ref, pred in single["samples"][:3]:
+ pbar.write(f" Q: {q}")
+ pbar.write(f" R: {ref}")
+ pbar.write(f" P: {pred}")
+ pbar.write("")
+
+
+def _decode_sample(tokenizer, token_row):
+ """Decode one sampled row to text, stopping at the first EOS after pos 0."""
+ sampled = token_row[1:]
+ eos_positions = np.where(sampled == EOS_ID)[0]
+ end = int(eos_positions[0]) if len(eos_positions) > 0 else len(sampled)
+ ids = [int(t) for t in sampled[:end] if t != PAD_ID]
+ return tokenizer.decode(ids)
+
+
+def grpo(args):
+ num_devices = jax.local_device_count()
+ num_hosts = jax.process_count()
+ host_id = jax.process_index()
+ total_devices = jax.device_count()
+ is_main = host_id == 0
+
+ use_wandb = getattr(args, "wandb", False) and is_main
+ if use_wandb:
+ import wandb
+ if wandb.run is None:
+ wandb.init(project="needle-v1", name=args.name, config=vars(args))
+
+ print(f"\n[1/4] Devices: {num_devices} local, {total_devices} total "
+ f"across {num_hosts} host(s)", flush=True)
+
+ print(f"\n[2/4] Loading checkpoint {args.checkpoint} ...", flush=True)
+ ckpt_path = _download_sft_checkpoint(args.checkpoint)
+ with open(ckpt_path, "rb") as f:
+ data = pickle.load(f)
+ config_dict = {k: v for k, v in data["config"].items()
+ if k in TransformerConfig.__dataclass_fields__}
+ config = TransformerConfig(**config_dict)
+ model = EncoderDecoderTransformer(config)
+
+ dt = jnp.bfloat16 if args.dtype == "bfloat16" else jnp.float32
+ params = jax.tree.map(lambda x: jnp.array(x, dtype=dt), data["params"])
+ ref_params = jax.tree.map(lambda x: jnp.array(x, dtype=dt), data["params"])
+
+ param_count = sum(x.size for x in jax.tree.leaves(params))
+ print(f" params: {param_count:,} "
+ f"(d={config.d_model}, heads={config.num_heads}, "
+ f"layers={config.num_encoder_layers}/{config.num_decoder_layers})")
+
+ print(f"\n[3/4] Building optimizer + pmap ...", flush=True)
+ total_steps = args.max_steps
+ warmup = max(1, int(total_steps * 0.05))
+ scaled_lr = args.lr * total_devices
+ schedule = _wsd_schedule(scaled_lr, total_steps, warmup, decay_ratio=0.1)
+ tx = optax.chain(
+ optax.clip_by_global_norm(1.0),
+ optax.adamw(schedule, b2=0.95, weight_decay=0.0),
+ )
+ state = train_state.TrainState.create(apply_fn=model.apply, params=params, tx=tx)
+ del params
+
+ r_state = _replicate(state)
+ r_ref_params = _replicate(ref_params)
+ del state, ref_params
+
+ p_encode, p_sample_step, p_log_probs = _make_fns(model, args.max_gen_len)
+ p_grpo_step = _make_grpo_step(model, args.beta)
+
+ print(f"\n[4/4] Loading tokenizer + data ...", flush=True)
+ tokenizer = get_tokenizer()
+ train_ds = load_tool_calls("train")
+ n_examples = len(train_ds)
+ print(f" train: {n_examples:,} examples")
+
+ val_data = None
+ val_ds = None
+ val_loss_fn = None
+ if is_main:
+ try:
+ val_data = load_prepared_data("val", mmap=True)
+ print(f" val (packed): {len(val_data['packed_enc']):,} bins")
+ except FileNotFoundError:
+ print(" WARN: no packed val data — skipping val PPL "
+ "(run `needle tokenize` to generate)")
+ try:
+ val_ds = load_tool_calls("val")
+ print(f" val (tool-call): {len(val_ds):,} examples")
+ except Exception as e:
+ print(f" WARN: could not load tool-call val set: {e}")
+ val_loss_fn = _make_val_loss_fn(model)
+
+ B = args.batch_size
+ K = args.k
+ BK = B * K
+ if BK % num_devices != 0:
+ raise ValueError(f"B*K ({BK}) must be divisible by num_devices ({num_devices})")
+ per_device = BK // num_devices
+
+ if is_main:
+ print(f"\n ─────────────────────────────────────")
+ print(f" GRPO")
+ print(f" ─────────────────────────────────────")
+ print(f" B (prompts/step) {B}")
+ print(f" K (samples/prompt) {K}")
+ print(f" B*K {BK} ({per_device}/device)")
+ print(f" LR {args.lr} -> {scaled_lr}")
+ print(f" beta (KL) {args.beta}")
+ print(f" max_steps {args.max_steps}")
+ print(f" max_enc_len {args.max_enc_len}")
+ print(f" max_gen_len {args.max_gen_len}")
+ print(f" ─────────────────────────────────────\n")
+
+ rng = jax.random.PRNGKey(args.seed)
+ perm = np.random.RandomState(args.seed).permutation(n_examples)
+ cursor = 0
+
+ best_call_f1 = -1.0
+
+ pbar = tqdm(total=args.max_steps, disable=not is_main, desc="GRPO")
+
+ for step in range(args.max_steps):
+ if cursor + B > len(perm):
+ perm = np.random.RandomState(args.seed + step).permutation(n_examples)
+ cursor = 0
+ batch_idx = perm[cursor:cursor + B]
+ cursor += B
+ batch = [train_ds[int(i)] for i in batch_idx]
+
+ # Build encoder inputs, replicate K times per prompt, shard across devices.
+ enc_np = _build_prompts(tokenizer, batch, args.max_enc_len)
+ enc_rep = np.repeat(enc_np, K, axis=0)
+ enc_sharded = jnp.asarray(
+ enc_rep.reshape(num_devices, per_device, args.max_enc_len)
+ )
+
+ # Sample K completions per prompt under the current policy.
+ t0 = time.perf_counter()
+ dec_buffer_np, rng = _sample_completions(
+ p_encode, p_sample_step, r_state.params, enc_sharded,
+ num_devices, args.max_gen_len, rng,
+ )
+ sample_time = time.perf_counter() - t0
+ dec_buffer_flat = dec_buffer_np.reshape(BK, args.max_gen_len)
+
+ # Score each completion.
+ rewards = np.zeros(BK, dtype=np.float32)
+ for i in range(BK):
+ prompt_idx = i // K
+ text = _decode_sample(tokenizer, dec_buffer_flat[i])
+ rewards[i] = _score(text, batch[prompt_idx]["answers"])
+
+ # Group-relative advantages: normalize within each prompt's K samples.
+ rewards_grp = rewards.reshape(B, K)
+ mean = rewards_grp.mean(axis=1, keepdims=True)
+ std = rewards_grp.std(axis=1, keepdims=True)
+ advantages = ((rewards_grp - mean) / (std + 1e-4)).reshape(BK).astype(np.float32)
+
+ # Teacher-forced shapes: dec_in = buffer[:-1], dec_tgt = buffer[1:].
+ dec_in_np = dec_buffer_flat[:, :-1].astype(np.int32)
+ dec_tgt_np = dec_buffer_flat[:, 1:].astype(np.int32)
+ response_mask_np = (dec_tgt_np != PAD_ID).astype(np.float32)
+ T_resp = dec_in_np.shape[1]
+
+ dec_in_sh = jnp.asarray(dec_in_np.reshape(num_devices, per_device, T_resp))
+ dec_tgt_sh = jnp.asarray(dec_tgt_np.reshape(num_devices, per_device, T_resp))
+ resp_sh = jnp.asarray(response_mask_np.reshape(num_devices, per_device, T_resp))
+ adv_sh = jnp.asarray(advantages.reshape(num_devices, per_device))
+
+ ref_logp_sh = p_log_probs(r_ref_params, enc_sharded, dec_in_sh, dec_tgt_sh)
+
+ r_state, loss, pg, kl = p_grpo_step(
+ r_state, enc_sharded, dec_in_sh, dec_tgt_sh,
+ adv_sh, resp_sh, ref_logp_sh,
+ )
+
+ loss_val = float(loss.addressable_shards[0].data[0])
+ pg_val = float(pg.addressable_shards[0].data[0])
+ kl_val = float(kl.addressable_shards[0].data[0])
+ mean_r = float(rewards.mean())
+ std_r = float(rewards.std())
+
+ pbar.update(1)
+ pbar.set_postfix(
+ loss=f"{loss_val:+.4f}", pg=f"{pg_val:+.4f}",
+ kl=f"{kl_val:.4f}", R=f"{mean_r:.3f}", sT=f"{sample_time:.1f}s",
+ )
+
+ if use_wandb:
+ import wandb
+ wandb.log({
+ "grpo/loss": loss_val,
+ "grpo/pg": pg_val,
+ "grpo/kl": kl_val,
+ "grpo/reward_mean": mean_r,
+ "grpo/reward_std": std_r,
+ "grpo/sample_time_s": sample_time,
+ "grpo/step": step,
+ })
+
+ # Full eval — val PPL + stratified tool-call benchmark + samples.
+ # Mirrors the end-of-epoch eval in train.py, run every eval_every steps.
+ if is_main and (step + 1) % args.eval_every == 0:
+ eval_params = _unreplicate(r_state).params
+
+ val_ppl = None
+ if val_data is not None:
+ val_ppl = _compute_val_ppl(
+ val_loss_fn, eval_params, val_data, args.eval_batch_size,
+ )
+
+ tc_results = {}
+ if val_ds is not None:
+ tc_results = _run_tool_call_eval(
+ model, eval_params, tokenizer, val_ds,
+ per_bucket=args.tool_call_samples // 2,
+ max_enc_len=args.max_enc_len,
+ max_gen_len=args.max_gen_len,
+ )
+
+ _print_eval_block(step + 1, val_ppl, tc_results, pbar)
+
+ # Best-checkpoint tracking: match train.py, use single-call F1.
+ cur_f1 = tc_results.get("single", {}).get("call_f1", -1.0)
+ is_new_best = cur_f1 > best_call_f1
+ if is_new_best and cur_f1 >= 0:
+ best_call_f1 = cur_f1
+ _save_grpo_checkpoint(
+ r_state, config, args.checkpoint_dir, step + 1,
+ filename="needle_grpo_best.pkl",
+ )
+ pbar.write(f" ** New best single call_f1={best_call_f1:.1%}")
+
+ if use_wandb:
+ import wandb
+ log = {"grpo/step": step + 1, "val/best_call_f1": best_call_f1}
+ if val_ppl is not None:
+ log["val/text_ppl"] = val_ppl
+ for label, metrics in tc_results.items():
+ for k, v in metrics.items():
+ if k in ("n", "samples"):
+ continue
+ log[f"val/{label}_{k}"] = v
+ wandb.log(log)
+
+ del eval_params
+
+ pbar.close()
+
+ if is_main:
+ best_path = os.path.join(args.checkpoint_dir, "needle_grpo_best.pkl")
+ if best_call_f1 >= 0:
+ print(f"\nGRPO complete. Best single call_f1={best_call_f1:.1%} -> {best_path}")
+ else:
+ print(f"\nGRPO complete. No eval ran — no best checkpoint saved.")
+
+ if use_wandb:
+ import wandb
+ wandb.finish()
+
+ if num_hosts > 1:
+ jax.experimental.multihost_utils.sync_global_devices("grpo_done")
+
+
+def _save_grpo_checkpoint(r_state, config, checkpoint_dir, step,
+ filename="needle_grpo.pkl"):
+ os.makedirs(checkpoint_dir, exist_ok=True)
+ params = _unreplicate(r_state).params
+ params_np = jax.tree.map(lambda x: np.array(x).astype(np.float16), params)
+ ckpt_path = os.path.join(checkpoint_dir, filename)
+ with open(ckpt_path, "wb") as f:
+ pickle.dump({"params": params_np, "config": config.__dict__,
+ "grpo_step": step}, f)
+ size_mb = sum(x.nbytes for x in jax.tree.leaves(params_np)) / 1e6
+ print(f"\n [step {step}] Saved {ckpt_path} ({size_mb:.1f} MB)", flush=True)
+ _upload_checkpoint(ckpt_path)
+ return ckpt_path