Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(train): print the optimizer-update budget before a run
Bruce-Yiiae6d4698cf08
diff --git a/docs/finetune.md b/docs/finetune.mdindex dbf4c06b..f63e4097 100644--- a/docs/finetune.md+++ b/docs/finetune.md@@ -75,8 +75,9 @@ laya-train --data tickets.csv --text-column body --label-column department --dry````--dry-run` builds the training items with the tokenizer and config of `--base`, without loading-its weights or training, and prints how many rows were read, how many items they produced, and how many questions-were skipped and why:+its weights or training, and prints how many rows were read, how many items they produced, how many+questions were skipped and why, and the optimizer-update budget the configured defaults will spend+(see [Sizing a run in optimizer updates](#sizing-a-run-in-optimizer-updates)):| reason | the question was skipped because ||---|---|@@ -85,6 +86,31 @@ were skipped and why:| `options_collapsed` | the head budget left two options with the same tokens (#538) || `options_beyond_max_len` | the question's options do not fit in `max_len` |+### Sizing a run in optimizer updates++`--epochs`, `--micro-batch` and `--grad-accum` decide how much training a run gets, but the number+that matters is the optimizer updates it performs. An epoch is split into+`ceil(items / micro_batch)` micro-batches; one update runs per `grad_accum` micro-batches, plus one+for the final partial window of each epoch, which steps once even when it holds fewer:++```+updates = epochs * ceil(ceil(train_items / micro_batch) / grad_accum)+```++`--dry-run` and the start of a run both print this budget, from the post-calibration training+count, with the effective batch:++```+optimizer budget: 1100 train item(s), effective batch up to 64 (micro_batch 8 x grad_accum 8), 72 optimizer updates over 4 epoch(s)+```++The defaults (4 epochs, effective batch 64) are the notebook's; on typed-decisions' 6,000+decisions they work out to about 376 updates, fewer once the calibration slice is held out. That+number is a reference for comparing runs of different sizes, not a cutoff: the same defaults on+1,100 items give about 72 updates, and [#963](https://github.com/NandhaKishorM/laya/issues/963)+measured a head that collapsed there while 8 epochs still failed on the task itself. A small dataset+therefore needs more epochs to reach a similar update count.+### What a run writes`--out` gets the layout `laya.load` reads (`model.safetensors`, `encoder/`, `tokenizer/`,diff --git a/laya/train.py b/laya/train.pyindex facf1cfb..343ba294 100644--- a/laya/train.py+++ b/laya/train.py@@ -604,6 +604,34 @@ def save_checkpoint(model, tok, cfg: Dict[str, Any], path: str) -> None:# ------------------------------------------------------------------------------------ train+def optimizer_updates(n_items: int, micro_batch: int, grad_accum: int, epochs: int) -> int:+ """Optimizer steps for `n_items` training items.++ `train_model` calls this, so the count a pre-run diagnostic prints is the number of+ `optimizer.step()` calls the run performs: one per full accumulation window+ (`grad_accum` micro-batches), plus one for the final partial window of every epoch, which+ still steps once even when it holds fewer micro-batches.+ """+ if n_items < 1:+ raise ValueError("n_items must be a positive integer, got %r" % (n_items,))+ steps_per_epoch = math.ceil(n_items / micro_batch)+ updates_per_epoch = math.ceil(steps_per_epoch / grad_accum)+ return updates_per_epoch * epochs+++def update_budget_message(n_train: int, config: TrainConfig) -> str:+ """The pre-run optimizer-update budget line, shared by `finetune` startup and `--dry-run`.++ Says "effective batch up to" because the last window of an epoch is often partial, so not+ every update sees a full `micro_batch x grad_accum` batch.+ """+ updates = optimizer_updates(n_train, config.micro_batch, config.grad_accum, config.epochs)+ return ("optimizer budget: %d train item(s), effective batch up to %d "+ "(micro_batch %d x grad_accum %d), %d optimizer updates over %d epoch(s)"+ % (n_train, config.micro_batch * config.grad_accum,+ config.micro_batch, config.grad_accum, updates, config.epochs))++def _forward(model, batch, device, amp: bool, detach_encoder: bool):args = (batch["input_ids"].to(device), batch["attention_mask"].to(device),batch["marker_pos"].to(device), batch["marker_mask"].to(device), batch["qtype"].to(device))@@ -647,8 +675,10 @@ def train_model(model, tok, items: Sequence[Dict[str, Any]], config: TrainConfigif n.startswith("encoder.") and p.requires_grad], "lr": config.encoder_lr})optimizer = torch.optim.AdamW(groups, weight_decay=config.weight_decay)steps_per_epoch = math.ceil(len(items) / config.micro_batch)- updates = max(1, math.ceil(steps_per_epoch / config.grad_accum) * config.epochs)- scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=updates, eta_min=config.min_lr)+ scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(+ optimizer,+ T_max=optimizer_updates(len(items), config.micro_batch, config.grad_accum, config.epochs),+ eta_min=config.min_lr)scaler = torch.amp.GradScaler("cuda") if amp and device.type == "cuda" else Nonetorch.manual_seed(config.seed)@@ -1079,6 +1109,7 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainCprint("train items %d, calibration items %d, eval items %d%s, skipped %r, device %s"% (len(train_items), len(calib_items), len(eval_items),(" (" + str(eval_source) + ")") if eval_source else "", skipped, dev), flush=True)+ print(update_budget_message(len(train_items), config), flush=True)# Evaluate base checkpoint on eval items prior to trainingbefore_eval = None@@ -1264,10 +1295,18 @@ def dry_run(data: str, model_dir: str, config: Optional[TrainConfig] = None) ->question_id=config.question_id, instructions=config.instructions)items, skipped = items_from_rows(tok, rows, max_len, head_max_len,label_smoothing=config.label_smoothing)+ # The training budget is set by the post-calibration item count, so split the same way+ # finetune does rather than report on the raw valid-item count.+ train_items, calib_items = split_calibration(items, config.calib_max, config.calib_frac, config.calib_seed)summary = {"data": data,"rows_read": len(rows),"valid_items": len(items),+ "train_items": len(train_items),+ "calibration_items": len(calib_items),+ "effective_batch": config.micro_batch * config.grad_accum,+ "optimizer_updates": optimizer_updates(len(train_items), config.micro_batch,+ config.grad_accum, config.epochs),"skipped": skipped,"max_len": max_len,"head_max_len": head_max_len,@@ -1285,4 +1324,5 @@ def dry_run(data: str, model_dir: str, config: Optional[TrainConfig] = None) ->print("dry-run: %d rows read, %d valid items, skipped: %r"% (len(rows), len(items), skipped), flush=True)+ print("dry-run: " + update_budget_message(len(train_items), config), flush=True)return summarydiff --git a/tests/test_train.py b/tests/test_train.pyindex 85dfc5c3..6f7933ac 100644--- a/tests/test_train.py+++ b/tests/test_train.py@@ -561,6 +561,19 @@ class EndToEndTests(unittest.TestCase):self.assertIn(agent.predict("app", {"department": DEPARTMENT})["answers"]["department"]["choice"],DEPARTMENT["criteria"])+ def test_startup_prints_the_optimizer_budget(self):+ import contextlib+ import io++ from laya.train import optimizer_updates++ buf = io.StringIO()+ with contextlib.redirect_stdout(buf):+ report = self.run_finetune("out_budget", epochs=2)+ expected = optimizer_updates(report["train_items"], 8, 1, 2)+ self.assertIn("optimizer budget", buf.getvalue())+ self.assertIn("%d optimizer updates" % expected, buf.getvalue())+def test_rlcd_objective_still_trains(self):report = self.run_finetune("out_rlcd", loss="rlcd")self.assertLess(report["epoch_loss"][-1], report["epoch_loss"][0])@@ -1091,5 +1104,92 @@ class PriorCollapseTests(unittest.TestCase):self.assertFalse(any("collapsed to the class prior" in m for m in peaked_msgs), peaked_msgs)+class OptimizerBudgetTests(unittest.TestCase):+ """#963 proposal 3: the optimizer-update budget is derived once and surfaced before a run."""++ def test_updates_count_every_accumulation_window_including_the_final_partial_one(self):+ from laya.train import optimizer_updates++ # steps_per_epoch = ceil(N / micro_batch); one update per full window plus the final+ # partial window of every epoch. typed-decisions and the #963 set are the reference points.+ self.assertEqual(optimizer_updates(6000, 8, 8, 4), 376)+ self.assertEqual(optimizer_updates(1100, 8, 8, 4), 72)+ self.assertEqual(optimizer_updates(5, 2, 2, 2), 4) # 3 micro-steps -> 2 updates/epoch+ self.assertEqual(optimizer_updates(63, 8, 8, 4), 4) # fewer micro-steps than a window+ self.assertEqual(optimizer_updates(8, 8, 8, 1), 1)+ self.assertEqual(optimizer_updates(9, 8, 8, 1), 1) # 2 micro-steps -> 1 update+ with self.assertRaises(ValueError):+ optimizer_updates(0, 8, 8, 4)++ def test_train_model_takes_exactly_optimizer_updates_steps(self):+ from laya.train import optimizer_updates++ class ScalarModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.encoder = torch.nn.Identity()+ self.value = torch.nn.Parameter(torch.tensor(0.0))++ steps = []++ class CountingSGD(torch.optim.SGD):+ def step(self, *args, **kwargs):+ steps.append(1)+ return super().step(*args, **kwargs)++ tok = type("Tokenizer", (), {"pad_token_id": 0})()+ item = {"q": {"t": "choice"}, "k": 2}+ batch = {+ "marker_mask": torch.ones((1, 1), dtype=torch.bool),+ "target": torch.ones((1, 1)),+ }+ for n_items, micro_batch, grad_accum, epochs in [(5, 2, 2, 2), (3, 1, 2, 1),+ (7, 3, 2, 3), (8, 8, 8, 1)]:+ with self.subTest(n_items=n_items, micro_batch=micro_batch, grad_accum=grad_accum):+ model = ScalarModel()+ steps.clear()+ config = TrainConfig(+ epochs=epochs, micro_batch=micro_batch, grad_accum=grad_accum,+ head_lr=1.0, min_lr=1.0, weight_decay=0.0, grad_clip=100.0,+ loss="soft-ce", freeze_encoder=True, amp=False, log_every=0,+ )+ with patch("laya.train.encode_item", return_value={}), \+ patch("laya.train.collate_items", return_value=batch), \+ patch("laya.train._forward", side_effect=lambda current, *_args: current.value), \+ patch("laya.train.soft_ce_loss", side_effect=lambda logits, *_args: logits), \+ patch("laya.train.torch.optim.AdamW",+ side_effect=lambda groups, weight_decay: CountingSGD(+ groups, weight_decay=weight_decay)):+ train_model(model, tok, [item] * n_items, config, torch.device("cpu"), 1, 1)++ self.assertEqual(len(steps), optimizer_updates(n_items, micro_batch, grad_accum, epochs))++ def test_dry_run_reports_the_post_calibration_budget(self):+ import contextlib+ import io++ from laya.train import dry_run, optimizer_updates++ with tempfile.TemporaryDirectory() as d:+ root = Path(d)+ make_checkpoint(root / "base", make_tokenizer())+ data = root / "train.jsonl"+ data.write_text("\n".join(json.dumps(r) for r in rows(2)), encoding="utf-8")+ config = TrainConfig(epochs=3, micro_batch=4, grad_accum=2, calib_frac=0.25, log_every=0)++ buf = io.StringIO()+ with contextlib.redirect_stdout(buf):+ summary = dry_run(str(data), str(root / "base"), config)++ # The budget must come from the post-calibration split, not the raw valid-item count.+ self.assertGreater(summary["calibration_items"], 0)+ self.assertEqual(summary["train_items"] + summary["calibration_items"], summary["valid_items"])+ self.assertLess(summary["train_items"], summary["valid_items"])+ self.assertEqual(summary["optimizer_updates"],+ optimizer_updates(summary["train_items"], 4, 2, 3))+ self.assertIn("optimizer budget", buf.getvalue())+ self.assertIn("%d optimizer updates" % summary["optimizer_updates"], buf.getvalue())++if __name__ == "__main__":unittest.main()