Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(train): print the optimizer-update budget before a run

Bruce-Yiiae6d4698cf08
diff --git a/docs/finetune.md b/docs/finetune.md
index dbf4c06b..f63e4097 100644
--- a/docs/finetune.md
+++ b/docs/finetune.md
@@ -75,8 +75,9 @@ laya-train --data tickets.csv --text-column body --label-column department --dry
```
`--dry-run` builds the training items with the tokenizer and config of `--base`, without loading
-its weights or training, and prints how many rows were read, how many items they produced, and how many questions
-were skipped and why:
+its weights or training, and prints how many rows were read, how many items they produced, how many
+questions were skipped and why, and the optimizer-update budget the configured defaults will spend
+(see [Sizing a run in optimizer updates](#sizing-a-run-in-optimizer-updates)):
| reason | the question was skipped because |
|---|---|
@@ -85,6 +86,31 @@ were skipped and why:
| `options_collapsed` | the head budget left two options with the same tokens (#538) |
| `options_beyond_max_len` | the question's options do not fit in `max_len` |
+### Sizing a run in optimizer updates
+
+`--epochs`, `--micro-batch` and `--grad-accum` decide how much training a run gets, but the number
+that matters is the optimizer updates it performs. An epoch is split into
+`ceil(items / micro_batch)` micro-batches; one update runs per `grad_accum` micro-batches, plus one
+for the final partial window of each epoch, which steps once even when it holds fewer:
+
+```
+updates = epochs * ceil(ceil(train_items / micro_batch) / grad_accum)
+```
+
+`--dry-run` and the start of a run both print this budget, from the post-calibration training
+count, with the effective batch:
+
+```
+optimizer budget: 1100 train item(s), effective batch up to 64 (micro_batch 8 x grad_accum 8), 72 optimizer updates over 4 epoch(s)
+```
+
+The defaults (4 epochs, effective batch 64) are the notebook's; on typed-decisions' 6,000
+decisions they work out to about 376 updates, fewer once the calibration slice is held out. That
+number is a reference for comparing runs of different sizes, not a cutoff: the same defaults on
+1,100 items give about 72 updates, and [#963](https://github.com/NandhaKishorM/laya/issues/963)
+measured a head that collapsed there while 8 epochs still failed on the task itself. A small dataset
+therefore needs more epochs to reach a similar update count.
+
### What a run writes
`--out` gets the layout `laya.load` reads (`model.safetensors`, `encoder/`, `tokenizer/`,
diff --git a/laya/train.py b/laya/train.py
index facf1cfb..343ba294 100644
--- a/laya/train.py
+++ b/laya/train.py
@@ -604,6 +604,34 @@ def save_checkpoint(model, tok, cfg: Dict[str, Any], path: str) -> None:
# ------------------------------------------------------------------------------------ train
+def optimizer_updates(n_items: int, micro_batch: int, grad_accum: int, epochs: int) -> int:
+ """Optimizer steps for `n_items` training items.
+
+ `train_model` calls this, so the count a pre-run diagnostic prints is the number of
+ `optimizer.step()` calls the run performs: one per full accumulation window
+ (`grad_accum` micro-batches), plus one for the final partial window of every epoch, which
+ still steps once even when it holds fewer micro-batches.
+ """
+ if n_items < 1:
+ raise ValueError("n_items must be a positive integer, got %r" % (n_items,))
+ steps_per_epoch = math.ceil(n_items / micro_batch)
+ updates_per_epoch = math.ceil(steps_per_epoch / grad_accum)
+ return updates_per_epoch * epochs
+
+
+def update_budget_message(n_train: int, config: TrainConfig) -> str:
+ """The pre-run optimizer-update budget line, shared by `finetune` startup and `--dry-run`.
+
+ Says "effective batch up to" because the last window of an epoch is often partial, so not
+ every update sees a full `micro_batch x grad_accum` batch.
+ """
+ updates = optimizer_updates(n_train, config.micro_batch, config.grad_accum, config.epochs)
+ return ("optimizer budget: %d train item(s), effective batch up to %d "
+ "(micro_batch %d x grad_accum %d), %d optimizer updates over %d epoch(s)"
+ % (n_train, config.micro_batch * config.grad_accum,
+ config.micro_batch, config.grad_accum, updates, config.epochs))
+
+
def _forward(model, batch, device, amp: bool, detach_encoder: bool):
args = (batch["input_ids"].to(device), batch["attention_mask"].to(device),
batch["marker_pos"].to(device), batch["marker_mask"].to(device), batch["qtype"].to(device))
@@ -647,8 +675,10 @@ def train_model(model, tok, items: Sequence[Dict[str, Any]], config: TrainConfig
if n.startswith("encoder.") and p.requires_grad], "lr": config.encoder_lr})
optimizer = torch.optim.AdamW(groups, weight_decay=config.weight_decay)
steps_per_epoch = math.ceil(len(items) / config.micro_batch)
- updates = max(1, math.ceil(steps_per_epoch / config.grad_accum) * config.epochs)
- scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=updates, eta_min=config.min_lr)
+ scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
+ optimizer,
+ T_max=optimizer_updates(len(items), config.micro_batch, config.grad_accum, config.epochs),
+ eta_min=config.min_lr)
scaler = torch.amp.GradScaler("cuda") if amp and device.type == "cuda" else None
torch.manual_seed(config.seed)
@@ -1079,6 +1109,7 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainC
print("train items %d, calibration items %d, eval items %d%s, skipped %r, device %s"
% (len(train_items), len(calib_items), len(eval_items),
(" (" + str(eval_source) + ")") if eval_source else "", skipped, dev), flush=True)
+ print(update_budget_message(len(train_items), config), flush=True)
# Evaluate base checkpoint on eval items prior to training
before_eval = None
@@ -1264,10 +1295,18 @@ def dry_run(data: str, model_dir: str, config: Optional[TrainConfig] = None) ->
question_id=config.question_id, instructions=config.instructions)
items, skipped = items_from_rows(tok, rows, max_len, head_max_len,
label_smoothing=config.label_smoothing)
+ # The training budget is set by the post-calibration item count, so split the same way
+ # finetune does rather than report on the raw valid-item count.
+ train_items, calib_items = split_calibration(items, config.calib_max, config.calib_frac, config.calib_seed)
summary = {
"data": data,
"rows_read": len(rows),
"valid_items": len(items),
+ "train_items": len(train_items),
+ "calibration_items": len(calib_items),
+ "effective_batch": config.micro_batch * config.grad_accum,
+ "optimizer_updates": optimizer_updates(len(train_items), config.micro_batch,
+ config.grad_accum, config.epochs),
"skipped": skipped,
"max_len": max_len,
"head_max_len": head_max_len,
@@ -1285,4 +1324,5 @@ def dry_run(data: str, model_dir: str, config: Optional[TrainConfig] = None) ->
print("dry-run: %d rows read, %d valid items, skipped: %r"
% (len(rows), len(items), skipped), flush=True)
+ print("dry-run: " + update_budget_message(len(train_items), config), flush=True)
return summary
diff --git a/tests/test_train.py b/tests/test_train.py
index 85dfc5c3..6f7933ac 100644
--- a/tests/test_train.py
+++ b/tests/test_train.py
@@ -561,6 +561,19 @@ class EndToEndTests(unittest.TestCase):
self.assertIn(agent.predict("app", {"department": DEPARTMENT})["answers"]["department"]["choice"],
DEPARTMENT["criteria"])
+ def test_startup_prints_the_optimizer_budget(self):
+ import contextlib
+ import io
+
+ from laya.train import optimizer_updates
+
+ buf = io.StringIO()
+ with contextlib.redirect_stdout(buf):
+ report = self.run_finetune("out_budget", epochs=2)
+ expected = optimizer_updates(report["train_items"], 8, 1, 2)
+ self.assertIn("optimizer budget", buf.getvalue())
+ self.assertIn("%d optimizer updates" % expected, buf.getvalue())
+
def test_rlcd_objective_still_trains(self):
report = self.run_finetune("out_rlcd", loss="rlcd")
self.assertLess(report["epoch_loss"][-1], report["epoch_loss"][0])
@@ -1091,5 +1104,92 @@ class PriorCollapseTests(unittest.TestCase):
self.assertFalse(any("collapsed to the class prior" in m for m in peaked_msgs), peaked_msgs)
+class OptimizerBudgetTests(unittest.TestCase):
+ """#963 proposal 3: the optimizer-update budget is derived once and surfaced before a run."""
+
+ def test_updates_count_every_accumulation_window_including_the_final_partial_one(self):
+ from laya.train import optimizer_updates
+
+ # steps_per_epoch = ceil(N / micro_batch); one update per full window plus the final
+ # partial window of every epoch. typed-decisions and the #963 set are the reference points.
+ self.assertEqual(optimizer_updates(6000, 8, 8, 4), 376)
+ self.assertEqual(optimizer_updates(1100, 8, 8, 4), 72)
+ self.assertEqual(optimizer_updates(5, 2, 2, 2), 4) # 3 micro-steps -> 2 updates/epoch
+ self.assertEqual(optimizer_updates(63, 8, 8, 4), 4) # fewer micro-steps than a window
+ self.assertEqual(optimizer_updates(8, 8, 8, 1), 1)
+ self.assertEqual(optimizer_updates(9, 8, 8, 1), 1) # 2 micro-steps -> 1 update
+ with self.assertRaises(ValueError):
+ optimizer_updates(0, 8, 8, 4)
+
+ def test_train_model_takes_exactly_optimizer_updates_steps(self):
+ from laya.train import optimizer_updates
+
+ class ScalarModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.encoder = torch.nn.Identity()
+ self.value = torch.nn.Parameter(torch.tensor(0.0))
+
+ steps = []
+
+ class CountingSGD(torch.optim.SGD):
+ def step(self, *args, **kwargs):
+ steps.append(1)
+ return super().step(*args, **kwargs)
+
+ tok = type("Tokenizer", (), {"pad_token_id": 0})()
+ item = {"q": {"t": "choice"}, "k": 2}
+ batch = {
+ "marker_mask": torch.ones((1, 1), dtype=torch.bool),
+ "target": torch.ones((1, 1)),
+ }
+ for n_items, micro_batch, grad_accum, epochs in [(5, 2, 2, 2), (3, 1, 2, 1),
+ (7, 3, 2, 3), (8, 8, 8, 1)]:
+ with self.subTest(n_items=n_items, micro_batch=micro_batch, grad_accum=grad_accum):
+ model = ScalarModel()
+ steps.clear()
+ config = TrainConfig(
+ epochs=epochs, micro_batch=micro_batch, grad_accum=grad_accum,
+ head_lr=1.0, min_lr=1.0, weight_decay=0.0, grad_clip=100.0,
+ loss="soft-ce", freeze_encoder=True, amp=False, log_every=0,
+ )
+ with patch("laya.train.encode_item", return_value={}), \
+ patch("laya.train.collate_items", return_value=batch), \
+ patch("laya.train._forward", side_effect=lambda current, *_args: current.value), \
+ patch("laya.train.soft_ce_loss", side_effect=lambda logits, *_args: logits), \
+ patch("laya.train.torch.optim.AdamW",
+ side_effect=lambda groups, weight_decay: CountingSGD(
+ groups, weight_decay=weight_decay)):
+ train_model(model, tok, [item] * n_items, config, torch.device("cpu"), 1, 1)
+
+ self.assertEqual(len(steps), optimizer_updates(n_items, micro_batch, grad_accum, epochs))
+
+ def test_dry_run_reports_the_post_calibration_budget(self):
+ import contextlib
+ import io
+
+ from laya.train import dry_run, optimizer_updates
+
+ with tempfile.TemporaryDirectory() as d:
+ root = Path(d)
+ make_checkpoint(root / "base", make_tokenizer())
+ data = root / "train.jsonl"
+ data.write_text("\n".join(json.dumps(r) for r in rows(2)), encoding="utf-8")
+ config = TrainConfig(epochs=3, micro_batch=4, grad_accum=2, calib_frac=0.25, log_every=0)
+
+ buf = io.StringIO()
+ with contextlib.redirect_stdout(buf):
+ summary = dry_run(str(data), str(root / "base"), config)
+
+ # The budget must come from the post-calibration split, not the raw valid-item count.
+ self.assertGreater(summary["calibration_items"], 0)
+ self.assertEqual(summary["train_items"] + summary["calibration_items"], summary["valid_items"])
+ self.assertLess(summary["train_items"], summary["valid_items"])
+ self.assertEqual(summary["optimizer_updates"],
+ optimizer_updates(summary["train_items"], 4, 2, 3))
+ self.assertIn("optimizer budget", buf.getvalue())
+ self.assertIn("%d optimizer updates" % summary["optimizer_updates"], buf.getvalue())
+
+
if __name__ == "__main__":
unittest.main()