Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(train): handle fractional scores, CSV BOM, atomic questions.json, and dry-run (#887)
Swatantra Yadav70cd9a74ca94
diff --git a/laya/train.py b/laya/train.pyindex 0216df0f..c29cf9c0 100644--- a/laya/train.py+++ b/laya/train.py@@ -204,23 +204,41 @@ def target_from_expected(q: Dict[str, Any], expected_val: Any,raise ValueError("expected answer %r cannot be parsed as a noul boolean" % (expected_val,))else: # scorekeys = list(crit.keys()) if isinstance(crit, dict) else [str(i) for i in range(len(crit))]- try:- if not isinstance(expected_val, bool):- num = int(expected_val)- if 0 <= num < len(keys):- idx = num- except (ValueError, TypeError):- pass- if idx is None:+ K = len(keys)+ target = None+ if isinstance(expected_val, str) and expected_val in keys:+ idx = keys.index(expected_val)+ target = [1.0 if i == idx else 0.0 for i in range(K)]+ else:+ try:+ if not isinstance(expected_val, bool):+ val = float(expected_val)+ if 0.0 <= val <= float(K - 1):+ low = int(math.floor(val))+ if low == K - 1 or val == float(low):+ target = [1.0 if i == low else 0.0 for i in range(K)]+ else:+ frac = val - float(low)+ target = [0.0] * K+ target[low] = 1.0 - frac+ target[low + 1] = frac+ else:+ raise ValueError("expected score %r out of bounds [0, %d]" % (expected_val, K - 1))+ except (ValueError, TypeError) as exc:+ if isinstance(exc, ValueError) and "out of bounds" in str(exc):+ raise+ pass+ if target is None:str_val = str(expected_val)for i, key in enumerate(keys):if str(key) == str_val:- idx = i+ target = [1.0 if j == i else 0.0 for j in range(K)]break- if idx is None:+ if target is None:raise ValueError("expected answer %r not found in score levels %r" % (expected_val, keys))- target = [1.0 if i == idx else 0.0 for i in range(k)]+ if target is None:+ target = [1.0 if i == idx else 0.0 for i in range(k)]if label_smoothing > 0.0:smooth_val = label_smoothing / ktarget = [(1.0 - label_smoothing) * v + smooth_val for v in target]@@ -258,14 +276,19 @@ def items_from_rows(tok, rows: Iterable[Dict[str, Any]], max_len: int,"""Items from `{state, questions, gold}` or `{state, questions, expected}` rows; returns `(items, skipped)`.`skipped` counts, by reason, questions that were labelled but could not become an item:- `invalid_question` and `invalid_target` (a `ValueError` from `to_internal`, `target_from_gold` or- `target_from_expected`), plus the two reasons `make_item` gives (`options_collapsed` and `target_mismatch`).+ `empty_text`, `empty_label`, `invalid_question` and `invalid_target` (a `ValueError` from+ `to_internal`, `target_from_gold` or `target_from_expected`), plus the two reasons `make_item`+ gives (`options_collapsed` and `target_mismatch`).Questions with neither `gold` nor `expected` entry are not counted -- the row simply does not label them."""items, skipped = [], {}for row in rows:- state_ids = encode_state(tok, row["state"], max_len)- for qid, question in row["questions"].items():+ state = row.get("state")+ if state is None or (isinstance(state, str) and not state.strip()):+ skipped["empty_text"] = skipped.get("empty_text", 0) + 1+ continue+ state_ids = encode_state(tok, state, max_len)+ for qid, question in row.get("questions", {}).items():gold_q = row.get("gold", {}).get(qid) if isinstance(row.get("gold"), dict) else Noneexpected_q = row.get("expected", {}).get(qid) if isinstance(row.get("expected"), dict) else Noneif gold_q is None and expected_q is None:@@ -280,7 +303,10 @@ def items_from_rows(tok, rows: Iterable[Dict[str, Any]], max_len: int,if gold_q is not None:target = target_from_gold(q, gold_q)else:- target = target_from_expected(q, expected_q, label_smoothing=label_smoothing)+ if isinstance(expected_q, str) and not expected_q.strip():+ reason = "empty_label"+ else:+ target = target_from_expected(q, expected_q, label_smoothing=label_smoothing)except (ValueError, TypeError, KeyError):reason = "invalid_target"if reason is None:@@ -298,12 +324,13 @@ def rows_from_csv(path: str, text_column: str = "text", label_column: str = "lab"""Read a CSV file with one labelled column per row into `{state, questions, expected}` rows.Discovers distinct labels in `label_column` (preserving order of appearance) and- builds a choice question with those labels as criteria.+ builds a choice question with those labels as a list of criteria to render each option once.+ Uses `utf-8-sig` encoding so UTF-8 BOM headers from Excel are handled transparently."""import csvrows_raw = []- with open(path, encoding="utf-8") as f:+ with open(path, encoding="utf-8-sig") as f:reader = csv.DictReader(f)if reader.fieldnames is None:raise ValueError("%s is an empty CSV file" % (path,))@@ -331,12 +358,12 @@ def rows_from_csv(path: str, text_column: str = "text", label_column: str = "labraise ValueError("CSV file %s must contain at least 2 distinct labels in %r, found %r"% (path, label_column, labels))- instr = instructions or ("Classify the input into one of: %s" % (", ".join(labels)))+ instr = instructions or "Classify the input into the correct category."questions = {question_id: {"type": "choice","instructions": instr,- "criteria": {lbl: lbl for lbl in labels},+ "criteria": labels,}}@@ -344,12 +371,11 @@ def rows_from_csv(path: str, text_column: str = "text", label_column: str = "labfor r in rows_raw:text = r.get(text_column, "")lbl = r.get(label_column, "").strip()- if text and lbl:- output_rows.append({- "state": text,- "questions": questions,- "expected": {question_id: lbl},- })+ output_rows.append({+ "state": text,+ "questions": questions,+ "expected": {question_id: lbl},+ })return output_rows@@ -683,6 +709,26 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainCout_cfg["temperature_by_options"] = fitted["temperature_by_options"]out_cfg["training"] = dict(out_cfg.get("training") or {}, laya_train=asdict(config))save_checkpoint(model, tok, out_cfg, output_dir)++ # Save the questions schema alongside the checkpoint for inference reuse+ sample_questions = {}+ for r in rows:+ qs = r.get("questions")+ if isinstance(qs, dict):+ for qid, qdef in qs.items():+ if qid not in sample_questions:+ sample_questions[qid] = qdef+ elif sample_questions[qid] != qdef:+ import warnings+ warnings.warn("Conflicting schema detected for question %r across rows; keeping first seen definition." % (qid,))+ if sample_questions:+ for d in (output_dir, os.path.join(output_dir, "checkpoint_latest")):+ os.makedirs(d, exist_ok=True)+ tmp_path = os.path.join(d, "questions.json.tmp")+ with open(tmp_path, "w", encoding="utf-8") as f:+ json.dump(sample_questions, f, indent=2)+ os.replace(tmp_path, os.path.join(d, "questions.json"))+return {"train_items": len(train_items),"calibration_items": len(calib_items),@@ -693,3 +739,35 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainC"n_by_bucket": fitted["n_by_bucket"],"output_dir": output_dir,}+++def dry_run(data: str, model_dir: str, config: Optional[TrainConfig] = None) -> Dict[str, Any]:+ """Inspect dataset and build items without loading model weights; returns dataset stats."""+ from transformers import AutoTokenizer+ from .agent import _fix_tokenizer_config++ config = config or TrainConfig()+ config.validate()+ resolved_dir = resolve_checkpoint_dir(model_dir)+ _fix_tokenizer_config(resolved_dir)+ tok = AutoTokenizer.from_pretrained(os.path.join(resolved_dir, "tokenizer"))+ with open(os.path.join(resolved_dir, "rl_agent_config.json"), encoding="utf-8") as f:+ cfg = json.load(f)+ max_len = config.max_len or cfg.get("max_len", 512)+ head_max_len = config.head_max_len or cfg.get("head_max_len", 192)++ rows = read_data(data, text_column=config.text_column, label_column=config.label_column,+ question_id=config.question_id, instructions=config.instructions)+ items, skipped = items_from_rows(tok, rows, max_len, head_max_len,+ label_smoothing=config.label_smoothing)+ summary = {+ "data": data,+ "rows_read": len(rows),+ "valid_items": len(items),+ "skipped": skipped,+ "max_len": max_len,+ "head_max_len": head_max_len,+ }+ print("dry-run: %d rows read, %d valid items, skipped: %r"+ % (len(rows), len(items), skipped), flush=True)+ return summarydiff --git a/laya/train_cli.py b/laya/train_cli.pyindex aeea279f..c7bdcde5 100644--- a/laya/train_cli.py+++ b/laya/train_cli.py@@ -30,7 +30,8 @@ def build_parser() -> argparse.ArgumentParser:"--out","--output-dir",dest="output_dir",- required=True,+ required=False,+ default=None,help="Output directory where the fine-tuned checkpoint will be saved.",)parser.add_argument(@@ -140,6 +141,11 @@ def build_parser() -> argparse.ArgumentParser:default=None,help="Override maximum question head token budget.",)+ parser.add_argument(+ "--dry-run",+ action="store_true",+ help="Inspect dataset, build items, and print stats without loading model weights or training.",+ )return parser@@ -147,11 +153,13 @@ def main(argv: Optional[Sequence[str]] = None) -> int:parser = build_parser()try:args = parser.parse_args(argv)+ if not args.dry_run and not args.output_dir:+ parser.error("--out is required unless --dry-run is set")except SystemExit as exc:return exc.codetry:- from .train import TrainConfig, finetune+ from .train import TrainConfig, dry_run, finetuneexcept ImportError as err:if "torch" in str(err).lower():print("laya-train: error: PyTorch is required for training. Please install torch.",@@ -181,6 +189,13 @@ def main(argv: Optional[Sequence[str]] = None) -> int:try:config.validate()+ if args.dry_run:+ dry_run(+ data=args.data,+ model_dir=args.model_dir,+ config=config,+ )+ return 0summary = finetune(data=args.data,model_dir=args.model_dir,diff --git a/tests/test_train.py b/tests/test_train.pyindex 4e7732d2..9a60ae9a 100644--- a/tests/test_train.py+++ b/tests/test_train.py@@ -168,6 +168,14 @@ class DataTests(unittest.TestCase):self.assertEqual(target(LEVEL, 1), [0.0, 1.0])self.assertEqual(target(LEVEL, "1"), [0.0, 1.0])+ # Fractional score answers (e.g. from laya-evals ScoreMAE / ScoreWithin)+ three_level = {"type": "score", "instructions": "which level ?", "criteria": ["low", "mid", "high"]}+ self.assertEqual(target(three_level, 1.5), [0.0, 0.5, 0.5])+ res_frac2 = target(three_level, 0.2)+ self.assertAlmostEqual(res_frac2[0], 0.8)+ self.assertAlmostEqual(res_frac2[1], 0.2)+ self.assertAlmostEqual(res_frac2[2], 0.0)+def test_target_from_expected_label_smoothing(self):res = target_from_expected(to_internal("q", DEPARTMENT), "billing", label_smoothing=0.1)self.assertAlmostEqual(res[0], 0.9 + 0.1 / 3)@@ -190,6 +198,8 @@ class DataTests(unittest.TestCase):target_from_expected(to_internal("q", URGENT), bad_num)with self.assertRaises(ValueError):target_from_expected(to_internal("q", LEVEL), 99)+ with self.assertRaises(ValueError):+ target_from_expected(to_internal("q", LEVEL), -0.5)for bad_smooth in (-0.1, 1.0, 1.5, True):with self.assertRaises(ValueError):target_from_expected(to_internal("q", DEPARTMENT), "billing", label_smoothing=bad_smooth)@@ -214,7 +224,14 @@ class DataTests(unittest.TestCase):self.assertEqual(data[0]["expected"]["label"], "billing")q = data[0]["questions"]["label"]self.assertEqual(q["type"], "choice")- self.assertEqual(list(q["criteria"].keys()), ["billing", "technical"])+ self.assertEqual(q["criteria"], ["billing", "technical"])++ # Test Excel UTF-8 BOM encoding+ bom_csv = Path(tmpdir) / "bom.csv"+ bom_csv.write_bytes("\ufeffbody,department\nrefund invoice,billing\ncrash,technical\n".encode("utf-8"))+ bom_data = read_data(str(bom_csv), text_column="body", label_column="department")+ self.assertEqual(len(bom_data), 2)+ self.assertEqual(bom_data[0]["state"], "refund invoice")def test_rows_from_csv_validation(self):with tempfile.TemporaryDirectory() as tmpdir:@@ -239,6 +256,17 @@ class DataTests(unittest.TestCase):self.assertEqual(items[0]["target"], [1.0, 0.0, 0.0])self.assertEqual(items[1]["target"], [0.0, 1.0, 0.0])+ def test_items_from_rows_skip_tracking(self):+ exp_rows_with_empty = [+ {"state": "", "questions": {"department": DEPARTMENT}, "expected": {"department": "billing"}},+ {"state": "valid text", "questions": {"department": DEPARTMENT}, "expected": {"department": ""}},+ {"state": "valid text", "questions": {"department": DEPARTMENT}, "expected": {"department": "billing"}},+ ]+ items, skipped = items_from_rows(self.tok, exp_rows_with_empty, 64, 40)+ self.assertEqual(len(items), 1)+ self.assertEqual(skipped.get("empty_text"), 1)+ self.assertEqual(skipped.get("empty_label"), 1)+def test_questions_are_rendered_exactly_as_inference_renders_them(self):# The sequence a training item produces must be the one `Agent` builds for the same# question, or the model is trained on option texts it never sees.@@ -504,6 +532,15 @@ class EndToEndTests(unittest.TestCase):self.assertTrue(args.shuffle_options)self.assertTrue(args.freeze_encoder)+ # --out is required when --dry-run is not set+ code_missing_out = train_cli_main(["--data", "train.csv"])+ self.assertEqual(code_missing_out, 2)++ # --dry-run without --out succeeds parsing+ dry_args = parser.parse_args(["--data", "train.csv", "--dry-run"])+ self.assertIsNone(dry_args.output_dir)+ self.assertTrue(dry_args.dry_run)+def test_laya_train_cli_runs_on_csv(self):csv_path = self.root / "tickets.csv"lines = ["text,department"]@@ -530,11 +567,26 @@ class EndToEndTests(unittest.TestCase):self.assertEqual(code, 0)self.assertTrue((out_dir / "model.safetensors").exists())self.assertTrue((out_dir / "rl_agent_config.json").exists())+ self.assertTrue((out_dir / "questions.json").exists())+ saved_q = json.loads((out_dir / "questions.json").read_text(encoding="utf-8"))+ self.assertIn("label", saved_q)+ self.assertEqual(saved_q["label"]["criteria"], ["billing", "technical"])+agent = load(str(out_dir), device="cpu")ans = agent.predict("please refund invoice", {"label": {"type": "choice", "instructions": "Classify","criteria": ["billing", "technical"]}})self.assertIn(ans["answers"]["label"]["choice"], ["billing", "technical"])+ # Test --dry-run+ dry_code = train_cli_main([+ "--data", str(csv_path),+ "--base", str(self.root / "base"),+ "--text-column", "text",+ "--label-column", "department",+ "--dry-run",+ ])+ self.assertEqual(dry_code, 0)+if __name__ == "__main__":unittest.main()