Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
docs(finetune): table the three laya-train flags no page documented
Aashish58919f14ca1a
diff --git a/docs/finetune.md b/docs/finetune.mdindex a36babe8..b4c648fa 100644--- a/docs/finetune.md+++ b/docs/finetune.md@@ -167,6 +167,9 @@ warning the run prints before gating on `min_confidence`.| `--epochs`, `--micro-batch`, `--grad-accum` | 4, 8, 8 | the notebook's 4 epochs and effective batch of 64 || `--encoder-lr`, `--head-lr` | 2.5e-5, 1e-4 | learning rates || `--max-len`, `--head-max-len` | the base checkpoint's | token budgets; the notebook uses 1024 and 256 |+| `--eval`, `--eval-data` | none | a second dataset, answered before and after the fine-tune. Without it the "after" numbers come from the calibration slice; the run's report says which of the two it is (`eval_mode`, `is_held_out`, `note`), and rows whose state and question also occur in the training or calibration inputs count as overlap and stop the metrics being held out (#967) |+| `--target-error` | `0.10` | the error the run fits the checkpoint's per-bucket abstention thresholds to keep among the answers it accepts |+| `--min-abstain-n` | `10` | a bucket with fewer items than this is left out of that map, and `resolve_min_confidence` gates that option count at 0.0 -- nothing abstains -- unless the caller sets `min_confidence` itself || `--device`, `--seed` | `auto`, 0 | |Mixed precision is on for CUDA. On a GPU where fp16 runs slower than fp32 (the GTX 16xx cards, fordiff --git a/tests/test_finetune_entrypoints.py b/tests/test_finetune_entrypoints.pyindex f2cea726..cfab445b 100644--- a/tests/test_finetune_entrypoints.py+++ b/tests/test_finetune_entrypoints.py@@ -1,6 +1,7 @@import argparseimport importlib.utilimport json+import reimport sysfrom pathlib import Pathfrom unittest import mock@@ -151,3 +152,95 @@ def test_no_duplicated_fitter_or_training_loop_remains():assert "fit_temperature(" not in mps_textassert "def preprocess" not in single_textassert "while n_batches % args.grad_accum" not in mps_text+++# ------------------------------------------------- `laya-train`'s options table, against the parser++# `docs/finetune.md`'s Options table is where a reader learns what `laya-train` accepts and what each+# flag does to the run; `--help` gives one line per flag and no consequence. Three flags were in+# neither the table nor any other page under `docs/` or the README -- `--eval`, `--target-error` and+# `--min-abstain-n` -- although the last two decide the per-bucket abstention map a fine-tuned+# checkpoint ships with, which is the number `Router` and `laya-serve` later gate answers on.+#+# Not a completeness gate over the parser, on purpose: #982 is adding six flags to `build_parser` as+# this lands, and a check demanding a row for every flag would go red on that PR instead of on the+# gap it exists to catch. What is checked is derived from the parser and applied to whatever the page+# claims, in both directions: a flag the table names has to exist, and a default the table prints has+# to be the parser's. The three flags this table gained stay named by name.++TABLE_ROW = re.compile(r"^\|\s*((?:`--[a-z0-9-]+`)(?:\s*,\s*`--[a-z0-9-]+`)*)\s*\|\s*([^|]*?)\s*\|")+FLAG_IN_CELL = re.compile(r"`(--[a-z0-9-]+)`")+FINETUNE_PAGE = ROOT / "docs" / "finetune.md"+++def _parser_defaults():+ """`{option string: default}` for every flag `laya-train` accepts, aliases included."""+ from laya.train_cli import build_parser++ out = {}+ for action in build_parser()._actions:+ for option in action.option_strings:+ out[option] = action.default+ return out+++def _option_table():+ """The Options table's rows as `(flags, default cell)`, read out of `docs/finetune.md`."""+ lines = FINETUNE_PAGE.read_text(encoding="utf-8").splitlines()+ start = next((i for i, line in enumerate(lines) if line.strip() == "### Options"), None)+ assert start is not None, "%s no longer has an `### Options` section" % FINETUNE_PAGE+ rows = []+ for line in lines[start:start + 40]:+ if line.startswith("###") and rows:+ break+ match = TABLE_ROW.match(line)+ if match:+ rows.append((FLAG_IN_CELL.findall(match.group(1)), match.group(2)))+ assert rows, "`### Options` in %s holds no table rows this sweep can read" % FINETUNE_PAGE+ return rows+++def test_every_flag_in_the_options_table_exists():+ """The table cannot promise a flag `laya-train` does not take."""+ defaults = _parser_defaults()+ rows = _option_table()+ unknown = sorted({flag for flags, _cell in rows for flag in flags if flag not in defaults})+ assert not unknown, (+ "docs/finetune.md's Options table names %s, which `laya-train` does not accept; the flags it "+ "takes are %s" % (unknown, sorted(defaults)))+++def test_the_options_table_prints_the_parser_defaults():+ """Where the table states numbers for a row of flags, they are the parser's, in that order."""+ defaults = _parser_defaults()+ checked = 0+ stale = []+ for flags, cell in _option_table():+ parts = [part.strip().strip("`") for part in cell.split(",")]+ if len(parts) != len(flags):+ continue # a shared row stating one thing for several flags, or prose+ try:+ stated = [float(part) for part in parts]+ except ValueError:+ continue # `rlcd`, `off`, `auto`: not a number this check can compare+ for flag, value in zip(flags, stated):+ got = defaults.get(flag)+ if isinstance(got, bool) or not isinstance(got, (int, float)):+ continue # unknown flags are the other check's business+ checked += 1+ if float(got) != value:+ stale.append("%s: table says %r, `build_parser` defaults to %r" % (flag, value, got))+ assert not stale, "docs/finetune.md prints a default the parser does not have: %s" % stale+ assert checked >= 6, (+ "fixture: this check compared %d defaults, and the table's numeric rows have to keep it "+ "meaningful -- the rows it reads are no longer the ones it was written against" % checked)+++def test_the_abstention_and_eval_flags_are_documented():+ """The three flags that decide what a fine-tuned checkpoint ships with stay in the table."""+ rows = _option_table()+ named = {flag for flags, _cell in rows for flag in flags}+ for flag in ("--eval", "--target-error", "--min-abstain-n"):+ assert flag in named, (+ "%s is what decides a fine-tuned checkpoint's held-out numbers and its abstention map, "+ "and it is no longer in docs/finetune.md's Options table" % flag)