Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(train): handle fractional scores, CSV BOM, atomic questions.json, and dry-run (#887)

Swatantra Yadav70cd9a74ca94
diff --git a/laya/train.py b/laya/train.py
index 0216df0f..c29cf9c0 100644
--- a/laya/train.py
+++ b/laya/train.py
@@ -204,23 +204,41 @@ def target_from_expected(q: Dict[str, Any], expected_val: Any,
raise ValueError("expected answer %r cannot be parsed as a noul boolean" % (expected_val,))
else: # score
keys = list(crit.keys()) if isinstance(crit, dict) else [str(i) for i in range(len(crit))]
- try:
- if not isinstance(expected_val, bool):
- num = int(expected_val)
- if 0 <= num < len(keys):
- idx = num
- except (ValueError, TypeError):
- pass
- if idx is None:
+ K = len(keys)
+ target = None
+ if isinstance(expected_val, str) and expected_val in keys:
+ idx = keys.index(expected_val)
+ target = [1.0 if i == idx else 0.0 for i in range(K)]
+ else:
+ try:
+ if not isinstance(expected_val, bool):
+ val = float(expected_val)
+ if 0.0 <= val <= float(K - 1):
+ low = int(math.floor(val))
+ if low == K - 1 or val == float(low):
+ target = [1.0 if i == low else 0.0 for i in range(K)]
+ else:
+ frac = val - float(low)
+ target = [0.0] * K
+ target[low] = 1.0 - frac
+ target[low + 1] = frac
+ else:
+ raise ValueError("expected score %r out of bounds [0, %d]" % (expected_val, K - 1))
+ except (ValueError, TypeError) as exc:
+ if isinstance(exc, ValueError) and "out of bounds" in str(exc):
+ raise
+ pass
+ if target is None:
str_val = str(expected_val)
for i, key in enumerate(keys):
if str(key) == str_val:
- idx = i
+ target = [1.0 if j == i else 0.0 for j in range(K)]
break
- if idx is None:
+ if target is None:
raise ValueError("expected answer %r not found in score levels %r" % (expected_val, keys))
- target = [1.0 if i == idx else 0.0 for i in range(k)]
+ if target is None:
+ target = [1.0 if i == idx else 0.0 for i in range(k)]
if label_smoothing > 0.0:
smooth_val = label_smoothing / k
target = [(1.0 - label_smoothing) * v + smooth_val for v in target]
@@ -258,14 +276,19 @@ def items_from_rows(tok, rows: Iterable[Dict[str, Any]], max_len: int,
"""Items from `{state, questions, gold}` or `{state, questions, expected}` rows; returns `(items, skipped)`.
`skipped` counts, by reason, questions that were labelled but could not become an item:
- `invalid_question` and `invalid_target` (a `ValueError` from `to_internal`, `target_from_gold` or
- `target_from_expected`), plus the two reasons `make_item` gives (`options_collapsed` and `target_mismatch`).
+ `empty_text`, `empty_label`, `invalid_question` and `invalid_target` (a `ValueError` from
+ `to_internal`, `target_from_gold` or `target_from_expected`), plus the two reasons `make_item`
+ gives (`options_collapsed` and `target_mismatch`).
Questions with neither `gold` nor `expected` entry are not counted -- the row simply does not label them.
"""
items, skipped = [], {}
for row in rows:
- state_ids = encode_state(tok, row["state"], max_len)
- for qid, question in row["questions"].items():
+ state = row.get("state")
+ if state is None or (isinstance(state, str) and not state.strip()):
+ skipped["empty_text"] = skipped.get("empty_text", 0) + 1
+ continue
+ state_ids = encode_state(tok, state, max_len)
+ for qid, question in row.get("questions", {}).items():
gold_q = row.get("gold", {}).get(qid) if isinstance(row.get("gold"), dict) else None
expected_q = row.get("expected", {}).get(qid) if isinstance(row.get("expected"), dict) else None
if gold_q is None and expected_q is None:
@@ -280,7 +303,10 @@ def items_from_rows(tok, rows: Iterable[Dict[str, Any]], max_len: int,
if gold_q is not None:
target = target_from_gold(q, gold_q)
else:
- target = target_from_expected(q, expected_q, label_smoothing=label_smoothing)
+ if isinstance(expected_q, str) and not expected_q.strip():
+ reason = "empty_label"
+ else:
+ target = target_from_expected(q, expected_q, label_smoothing=label_smoothing)
except (ValueError, TypeError, KeyError):
reason = "invalid_target"
if reason is None:
@@ -298,12 +324,13 @@ def rows_from_csv(path: str, text_column: str = "text", label_column: str = "lab
"""Read a CSV file with one labelled column per row into `{state, questions, expected}` rows.
Discovers distinct labels in `label_column` (preserving order of appearance) and
- builds a choice question with those labels as criteria.
+ builds a choice question with those labels as a list of criteria to render each option once.
+ Uses `utf-8-sig` encoding so UTF-8 BOM headers from Excel are handled transparently.
"""
import csv
rows_raw = []
- with open(path, encoding="utf-8") as f:
+ with open(path, encoding="utf-8-sig") as f:
reader = csv.DictReader(f)
if reader.fieldnames is None:
raise ValueError("%s is an empty CSV file" % (path,))
@@ -331,12 +358,12 @@ def rows_from_csv(path: str, text_column: str = "text", label_column: str = "lab
raise ValueError("CSV file %s must contain at least 2 distinct labels in %r, found %r"
% (path, label_column, labels))
- instr = instructions or ("Classify the input into one of: %s" % (", ".join(labels)))
+ instr = instructions or "Classify the input into the correct category."
questions = {
question_id: {
"type": "choice",
"instructions": instr,
- "criteria": {lbl: lbl for lbl in labels},
+ "criteria": labels,
}
}
@@ -344,12 +371,11 @@ def rows_from_csv(path: str, text_column: str = "text", label_column: str = "lab
for r in rows_raw:
text = r.get(text_column, "")
lbl = r.get(label_column, "").strip()
- if text and lbl:
- output_rows.append({
- "state": text,
- "questions": questions,
- "expected": {question_id: lbl},
- })
+ output_rows.append({
+ "state": text,
+ "questions": questions,
+ "expected": {question_id: lbl},
+ })
return output_rows
@@ -683,6 +709,26 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainC
out_cfg["temperature_by_options"] = fitted["temperature_by_options"]
out_cfg["training"] = dict(out_cfg.get("training") or {}, laya_train=asdict(config))
save_checkpoint(model, tok, out_cfg, output_dir)
+
+ # Save the questions schema alongside the checkpoint for inference reuse
+ sample_questions = {}
+ for r in rows:
+ qs = r.get("questions")
+ if isinstance(qs, dict):
+ for qid, qdef in qs.items():
+ if qid not in sample_questions:
+ sample_questions[qid] = qdef
+ elif sample_questions[qid] != qdef:
+ import warnings
+ warnings.warn("Conflicting schema detected for question %r across rows; keeping first seen definition." % (qid,))
+ if sample_questions:
+ for d in (output_dir, os.path.join(output_dir, "checkpoint_latest")):
+ os.makedirs(d, exist_ok=True)
+ tmp_path = os.path.join(d, "questions.json.tmp")
+ with open(tmp_path, "w", encoding="utf-8") as f:
+ json.dump(sample_questions, f, indent=2)
+ os.replace(tmp_path, os.path.join(d, "questions.json"))
+
return {
"train_items": len(train_items),
"calibration_items": len(calib_items),
@@ -693,3 +739,35 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainC
"n_by_bucket": fitted["n_by_bucket"],
"output_dir": output_dir,
}
+
+
+def dry_run(data: str, model_dir: str, config: Optional[TrainConfig] = None) -> Dict[str, Any]:
+ """Inspect dataset and build items without loading model weights; returns dataset stats."""
+ from transformers import AutoTokenizer
+ from .agent import _fix_tokenizer_config
+
+ config = config or TrainConfig()
+ config.validate()
+ resolved_dir = resolve_checkpoint_dir(model_dir)
+ _fix_tokenizer_config(resolved_dir)
+ tok = AutoTokenizer.from_pretrained(os.path.join(resolved_dir, "tokenizer"))
+ with open(os.path.join(resolved_dir, "rl_agent_config.json"), encoding="utf-8") as f:
+ cfg = json.load(f)
+ max_len = config.max_len or cfg.get("max_len", 512)
+ head_max_len = config.head_max_len or cfg.get("head_max_len", 192)
+
+ rows = read_data(data, text_column=config.text_column, label_column=config.label_column,
+ question_id=config.question_id, instructions=config.instructions)
+ items, skipped = items_from_rows(tok, rows, max_len, head_max_len,
+ label_smoothing=config.label_smoothing)
+ summary = {
+ "data": data,
+ "rows_read": len(rows),
+ "valid_items": len(items),
+ "skipped": skipped,
+ "max_len": max_len,
+ "head_max_len": head_max_len,
+ }
+ print("dry-run: %d rows read, %d valid items, skipped: %r"
+ % (len(rows), len(items), skipped), flush=True)
+ return summary
diff --git a/laya/train_cli.py b/laya/train_cli.py
index aeea279f..c7bdcde5 100644
--- a/laya/train_cli.py
+++ b/laya/train_cli.py
@@ -30,7 +30,8 @@ def build_parser() -> argparse.ArgumentParser:
"--out",
"--output-dir",
dest="output_dir",
- required=True,
+ required=False,
+ default=None,
help="Output directory where the fine-tuned checkpoint will be saved.",
)
parser.add_argument(
@@ -140,6 +141,11 @@ def build_parser() -> argparse.ArgumentParser:
default=None,
help="Override maximum question head token budget.",
)
+ parser.add_argument(
+ "--dry-run",
+ action="store_true",
+ help="Inspect dataset, build items, and print stats without loading model weights or training.",
+ )
return parser
@@ -147,11 +153,13 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
parser = build_parser()
try:
args = parser.parse_args(argv)
+ if not args.dry_run and not args.output_dir:
+ parser.error("--out is required unless --dry-run is set")
except SystemExit as exc:
return exc.code
try:
- from .train import TrainConfig, finetune
+ from .train import TrainConfig, dry_run, finetune
except ImportError as err:
if "torch" in str(err).lower():
print("laya-train: error: PyTorch is required for training. Please install torch.",
@@ -181,6 +189,13 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
try:
config.validate()
+ if args.dry_run:
+ dry_run(
+ data=args.data,
+ model_dir=args.model_dir,
+ config=config,
+ )
+ return 0
summary = finetune(
data=args.data,
model_dir=args.model_dir,
diff --git a/tests/test_train.py b/tests/test_train.py
index 4e7732d2..9a60ae9a 100644
--- a/tests/test_train.py
+++ b/tests/test_train.py
@@ -168,6 +168,14 @@ class DataTests(unittest.TestCase):
self.assertEqual(target(LEVEL, 1), [0.0, 1.0])
self.assertEqual(target(LEVEL, "1"), [0.0, 1.0])
+ # Fractional score answers (e.g. from laya-evals ScoreMAE / ScoreWithin)
+ three_level = {"type": "score", "instructions": "which level ?", "criteria": ["low", "mid", "high"]}
+ self.assertEqual(target(three_level, 1.5), [0.0, 0.5, 0.5])
+ res_frac2 = target(three_level, 0.2)
+ self.assertAlmostEqual(res_frac2[0], 0.8)
+ self.assertAlmostEqual(res_frac2[1], 0.2)
+ self.assertAlmostEqual(res_frac2[2], 0.0)
+
def test_target_from_expected_label_smoothing(self):
res = target_from_expected(to_internal("q", DEPARTMENT), "billing", label_smoothing=0.1)
self.assertAlmostEqual(res[0], 0.9 + 0.1 / 3)
@@ -190,6 +198,8 @@ class DataTests(unittest.TestCase):
target_from_expected(to_internal("q", URGENT), bad_num)
with self.assertRaises(ValueError):
target_from_expected(to_internal("q", LEVEL), 99)
+ with self.assertRaises(ValueError):
+ target_from_expected(to_internal("q", LEVEL), -0.5)
for bad_smooth in (-0.1, 1.0, 1.5, True):
with self.assertRaises(ValueError):
target_from_expected(to_internal("q", DEPARTMENT), "billing", label_smoothing=bad_smooth)
@@ -214,7 +224,14 @@ class DataTests(unittest.TestCase):
self.assertEqual(data[0]["expected"]["label"], "billing")
q = data[0]["questions"]["label"]
self.assertEqual(q["type"], "choice")
- self.assertEqual(list(q["criteria"].keys()), ["billing", "technical"])
+ self.assertEqual(q["criteria"], ["billing", "technical"])
+
+ # Test Excel UTF-8 BOM encoding
+ bom_csv = Path(tmpdir) / "bom.csv"
+ bom_csv.write_bytes("\ufeffbody,department\nrefund invoice,billing\ncrash,technical\n".encode("utf-8"))
+ bom_data = read_data(str(bom_csv), text_column="body", label_column="department")
+ self.assertEqual(len(bom_data), 2)
+ self.assertEqual(bom_data[0]["state"], "refund invoice")
def test_rows_from_csv_validation(self):
with tempfile.TemporaryDirectory() as tmpdir:
@@ -239,6 +256,17 @@ class DataTests(unittest.TestCase):
self.assertEqual(items[0]["target"], [1.0, 0.0, 0.0])
self.assertEqual(items[1]["target"], [0.0, 1.0, 0.0])
+ def test_items_from_rows_skip_tracking(self):
+ exp_rows_with_empty = [
+ {"state": "", "questions": {"department": DEPARTMENT}, "expected": {"department": "billing"}},
+ {"state": "valid text", "questions": {"department": DEPARTMENT}, "expected": {"department": ""}},
+ {"state": "valid text", "questions": {"department": DEPARTMENT}, "expected": {"department": "billing"}},
+ ]
+ items, skipped = items_from_rows(self.tok, exp_rows_with_empty, 64, 40)
+ self.assertEqual(len(items), 1)
+ self.assertEqual(skipped.get("empty_text"), 1)
+ self.assertEqual(skipped.get("empty_label"), 1)
+
def test_questions_are_rendered_exactly_as_inference_renders_them(self):
# The sequence a training item produces must be the one `Agent` builds for the same
# question, or the model is trained on option texts it never sees.
@@ -504,6 +532,15 @@ class EndToEndTests(unittest.TestCase):
self.assertTrue(args.shuffle_options)
self.assertTrue(args.freeze_encoder)
+ # --out is required when --dry-run is not set
+ code_missing_out = train_cli_main(["--data", "train.csv"])
+ self.assertEqual(code_missing_out, 2)
+
+ # --dry-run without --out succeeds parsing
+ dry_args = parser.parse_args(["--data", "train.csv", "--dry-run"])
+ self.assertIsNone(dry_args.output_dir)
+ self.assertTrue(dry_args.dry_run)
+
def test_laya_train_cli_runs_on_csv(self):
csv_path = self.root / "tickets.csv"
lines = ["text,department"]
@@ -530,11 +567,26 @@ class EndToEndTests(unittest.TestCase):
self.assertEqual(code, 0)
self.assertTrue((out_dir / "model.safetensors").exists())
self.assertTrue((out_dir / "rl_agent_config.json").exists())
+ self.assertTrue((out_dir / "questions.json").exists())
+ saved_q = json.loads((out_dir / "questions.json").read_text(encoding="utf-8"))
+ self.assertIn("label", saved_q)
+ self.assertEqual(saved_q["label"]["criteria"], ["billing", "technical"])
+
agent = load(str(out_dir), device="cpu")
ans = agent.predict("please refund invoice", {"label": {"type": "choice", "instructions": "Classify",
"criteria": ["billing", "technical"]}})
self.assertIn(ans["answers"]["label"]["choice"], ["billing", "technical"])
+ # Test --dry-run
+ dry_code = train_cli_main([
+ "--data", str(csv_path),
+ "--base", str(self.root / "base"),
+ "--text-column", "text",
+ "--label-column", "department",
+ "--dry-run",
+ ])
+ self.assertEqual(dry_code, 0)
+
if __name__ == "__main__":
unittest.main()