Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(train): add laya-train CLI and CSV/expected dataset loaders (#887)

Swatantra Yadavfb74cb976dc8
diff --git a/laya/cli.py b/laya/cli.py
index a2192071..89857b4f 100644
--- a/laya/cli.py
+++ b/laya/cli.py
@@ -383,6 +383,9 @@ def main(argv=None):
if argv and argv[0] == "eval": # `laya eval ...` mirrors the `laya-evals` script
from .evals_cli import main as eval_main
return eval_main(argv[1:])
+ if argv and argv[0] == "train": # `laya train ...` mirrors the `laya-train` script
+ from .train_cli import main as train_main
+ return train_main(argv[1:])
args = build_parser().parse_args(argv)
# Redirected output is encoded with the locale's codec by default (cp1252 on Windows), which
# cannot hold a non-Latin request or answer, so `laya --batch FILE > out.txt` failed with a
diff --git a/laya/train.py b/laya/train.py
index 5e861f85..0216df0f 100644
--- a/laya/train.py
+++ b/laya/train.py
@@ -88,6 +88,11 @@ class TrainConfig:
amp: Optional[bool] = None
gradient_checkpointing: Optional[bool] = None
log_every: int = 100
+ label_smoothing: float = 0.0
+ text_column: str = "text"
+ label_column: str = "label"
+ question_id: str = "label"
+ instructions: Optional[str] = None
def validate(self) -> None:
if self.loss not in LOSSES:
@@ -101,6 +106,8 @@ class TrainConfig:
raise ValueError("%s must be a positive integer, got %r" % (name, value))
if not 0.0 <= self.calib_frac < 1.0:
raise ValueError("calib_frac must be in [0, 1), got %r" % (self.calib_frac,))
+ if isinstance(self.label_smoothing, bool) or not (0.0 <= self.label_smoothing < 1.0):
+ raise ValueError("label_smoothing must be in [0, 1), got %r" % (self.label_smoothing,))
# ------------------------------------------------------------------------------------- data
@@ -148,6 +155,78 @@ def target_from_gold(q: Dict[str, Any], gold_q: Dict[str, Any]) -> List[float]:
return [1.0 / len(target)] * len(target)
+def target_from_expected(q: Dict[str, Any], expected_val: Any,
+ label_smoothing: float = 0.0) -> List[float]:
+ """The normalised target distribution from a single expected answer, in option order.
+
+ `expected_val` is:
+ - a choice label (str or value matching a criterion) for choice questions
+ - a boolean for noul questions
+ - an integer level index or level string for score questions
+
+ When `label_smoothing > 0`, smooths probability mass uniformly:
+ target[i] = (1 - eps) * one_hot[i] + eps / K.
+ """
+ if isinstance(label_smoothing, bool) or not (0.0 <= label_smoothing < 1.0):
+ raise ValueError("label_smoothing must be in [0, 1), got %r" % (label_smoothing,))
+
+ t, crit = q["t"], q["crit"]
+ k = len(render_options(q))
+ idx = None
+
+ if t == "choice":
+ keys = list(crit.keys())
+ if expected_val in crit:
+ idx = keys.index(expected_val)
+ else:
+ str_expected = str(expected_val)
+ for i, key in enumerate(keys):
+ if str(key) == str_expected:
+ idx = i
+ break
+ if idx is None:
+ raise ValueError("expected answer %r not found in choice options %r" % (expected_val, keys))
+ elif t == "noul":
+ if isinstance(expected_val, bool):
+ idx = 1 if expected_val else 0
+ elif isinstance(expected_val, (int, float)):
+ if expected_val in (0, 1) or expected_val in (0.0, 1.0):
+ idx = int(expected_val)
+ else:
+ raise ValueError("numeric noul expected answer must be 0 or 1, got %r" % (expected_val,))
+ elif isinstance(expected_val, str):
+ lower = expected_val.strip().lower()
+ if lower in ("true", "1", "yes"):
+ idx = 1
+ elif lower in ("false", "0", "no"):
+ idx = 0
+ if idx is None:
+ raise ValueError("expected answer %r cannot be parsed as a noul boolean" % (expected_val,))
+ else: # score
+ keys = list(crit.keys()) if isinstance(crit, dict) else [str(i) for i in range(len(crit))]
+ try:
+ if not isinstance(expected_val, bool):
+ num = int(expected_val)
+ if 0 <= num < len(keys):
+ idx = num
+ except (ValueError, TypeError):
+ pass
+ if idx is None:
+ str_val = str(expected_val)
+ for i, key in enumerate(keys):
+ if str(key) == str_val:
+ idx = i
+ break
+ if idx is None:
+ raise ValueError("expected answer %r not found in score levels %r" % (expected_val, keys))
+
+ target = [1.0 if i == idx else 0.0 for i in range(k)]
+ if label_smoothing > 0.0:
+ smooth_val = label_smoothing / k
+ target = [(1.0 - label_smoothing) * v + smooth_val for v in target]
+ return target
+
+
def encode_state(tok, state: Any, max_len: int) -> List[int]:
"""State token ids, capped at `max_len`: no sequence can hold more of the state than that."""
text = serialize_state(state).replace(tok.mask_token, " ")
@@ -174,29 +253,34 @@ def make_item(tok, q: Dict[str, Any], target: Sequence[float], state_ids: List[i
def items_from_rows(tok, rows: Iterable[Dict[str, Any]], max_len: int,
- head_max_len: int) -> Tuple[List[Dict[str, Any]], Dict[str, int]]:
- """Items from `{state, questions, gold}` rows; returns `(items, skipped)`.
+ head_max_len: int,
+ label_smoothing: float = 0.0) -> Tuple[List[Dict[str, Any]], Dict[str, int]]:
+ """Items from `{state, questions, gold}` or `{state, questions, expected}` rows; returns `(items, skipped)`.
`skipped` counts, by reason, questions that were labelled but could not become an item:
- `invalid_question` and `invalid_target` (a `ValueError` from `to_internal` or
- `target_from_gold`), plus the two reasons `make_item` gives. Questions with no gold entry
- are not counted -- the row simply does not label them.
+ `invalid_question` and `invalid_target` (a `ValueError` from `to_internal`, `target_from_gold` or
+ `target_from_expected`), plus the two reasons `make_item` gives (`options_collapsed` and `target_mismatch`).
+ Questions with neither `gold` nor `expected` entry are not counted -- the row simply does not label them.
"""
items, skipped = [], {}
for row in rows:
state_ids = encode_state(tok, row["state"], max_len)
for qid, question in row["questions"].items():
- gold_q = row["gold"].get(qid)
- if gold_q is None:
+ gold_q = row.get("gold", {}).get(qid) if isinstance(row.get("gold"), dict) else None
+ expected_q = row.get("expected", {}).get(qid) if isinstance(row.get("expected"), dict) else None
+ if gold_q is None and expected_q is None:
continue
reason = None
try:
q = to_internal(qid, question)
- except ValueError:
+ except (ValueError, TypeError, KeyError):
reason = "invalid_question"
if reason is None:
try:
- target = target_from_gold(q, gold_q)
+ if gold_q is not None:
+ target = target_from_gold(q, gold_q)
+ else:
+ target = target_from_expected(q, expected_q, label_smoothing=label_smoothing)
except (ValueError, TypeError, KeyError):
reason = "invalid_target"
if reason is None:
@@ -208,6 +292,77 @@ def items_from_rows(tok, rows: Iterable[Dict[str, Any]], max_len: int,
return items, skipped
+def rows_from_csv(path: str, text_column: str = "text", label_column: str = "label",
+ question_id: str = "label",
+ instructions: Optional[str] = None) -> List[Dict[str, Any]]:
+ """Read a CSV file with one labelled column per row into `{state, questions, expected}` rows.
+
+ Discovers distinct labels in `label_column` (preserving order of appearance) and
+ builds a choice question with those labels as criteria.
+ """
+ import csv
+
+ rows_raw = []
+ with open(path, encoding="utf-8") as f:
+ reader = csv.DictReader(f)
+ if reader.fieldnames is None:
+ raise ValueError("%s is an empty CSV file" % (path,))
+ if text_column not in reader.fieldnames:
+ raise ValueError("CSV file %s missing text column %r (available: %r)"
+ % (path, text_column, reader.fieldnames))
+ if label_column not in reader.fieldnames:
+ raise ValueError("CSV file %s missing label column %r (available: %r)"
+ % (path, label_column, reader.fieldnames))
+ for r in reader:
+ rows_raw.append(r)
+
+ if not rows_raw:
+ raise ValueError("%s has no rows" % (path,))
+
+ labels = []
+ seen = set()
+ for r in rows_raw:
+ val = r.get(label_column, "").strip()
+ if val and val not in seen:
+ seen.add(val)
+ labels.append(val)
+
+ if len(labels) < 2:
+ raise ValueError("CSV file %s must contain at least 2 distinct labels in %r, found %r"
+ % (path, label_column, labels))
+
+ instr = instructions or ("Classify the input into one of: %s" % (", ".join(labels)))
+ questions = {
+ question_id: {
+ "type": "choice",
+ "instructions": instr,
+ "criteria": {lbl: lbl for lbl in labels},
+ }
+ }
+
+ output_rows = []
+ for r in rows_raw:
+ text = r.get(text_column, "")
+ lbl = r.get(label_column, "").strip()
+ if text and lbl:
+ output_rows.append({
+ "state": text,
+ "questions": questions,
+ "expected": {question_id: lbl},
+ })
+ return output_rows
+
+
+def read_data(path: str, text_column: str = "text", label_column: str = "label",
+ question_id: str = "label",
+ instructions: Optional[str] = None) -> List[Dict[str, Any]]:
+ """Read training/eval rows from either a JSONL file or a CSV file."""
+ if path.lower().endswith(".csv"):
+ return rows_from_csv(path, text_column=text_column, label_column=label_column,
+ question_id=question_id, instructions=instructions)
+ return read_jsonl(path)
+
+
def read_jsonl(path: str) -> List[Dict[str, Any]]:
rows = []
with open(path, encoding="utf-8") as f:
@@ -309,13 +464,47 @@ def resolve_device(device: Optional[str] = "auto") -> torch.device:
return torch.device("cpu")
+def resolve_checkpoint_dir(model_id_or_path: str, token: Optional[str] = None) -> str:
+ """Resolve a local path, model alias ('multilingual', 'english') or Hub repo ID to a local directory."""
+ if os.path.isdir(model_id_or_path) and os.path.exists(os.path.join(model_id_or_path, "rl_agent_config.json")):
+ return model_id_or_path
+
+ if model_id_or_path.startswith(("/", "./", "../")) or os.path.isabs(model_id_or_path):
+ if not os.path.isdir(model_id_or_path):
+ raise FileNotFoundError("Local model path not found: %r" % (model_id_or_path,))
+ return model_id_or_path
+
+ from .router import resolve_model_spec
+ spec = resolve_model_spec(model_id_or_path)
+ subfolder = None
+ if spec is not None:
+ model_id_or_path, subfolder = spec
+
+ from huggingface_hub import snapshot_download
+ prefix = f"{subfolder}/" if subfolder else ""
+ kw = {
+ "token": token or os.environ.get("HF_TOKEN") or None,
+ "allow_patterns": [prefix + name for name in (
+ "rl_agent_config.json", "model.safetensors", "tokenizer/*", "encoder/*",
+ )],
+ }
+ model_dir = snapshot_download(model_id_or_path, **kw)
+ if subfolder:
+ model_dir = os.path.join(model_dir, subfolder)
+ if not os.path.isdir(model_dir) or not os.path.exists(os.path.join(model_dir, "rl_agent_config.json")):
+ raise FileNotFoundError("Checkpoint directory not found for %r (resolved to %r)"
+ % (model_id_or_path, model_dir))
+ return model_dir
+
+
def load_checkpoint(model_dir: str):
- """`(model, tokenizer, cfg)` from a local checkpoint directory, model on CPU in fp32."""
+ """`(model, tokenizer, cfg)` from a local checkpoint directory or Hub ID, model on CPU in fp32."""
from safetensors.torch import load_file
from transformers import AutoTokenizer
from .agent import _fix_tokenizer_config
+ model_dir = resolve_checkpoint_dir(model_dir)
_fix_tokenizer_config(model_dir)
with open(os.path.join(model_dir, "rl_agent_config.json"), encoding="utf-8") as f:
cfg = json.load(f)
@@ -456,7 +645,8 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainC
device: Optional[str] = "auto") -> Dict[str, Any]:
"""Preprocess, train, calibrate and save; returns a summary of the run.
- `data` is a JSONL file of `{state, questions, gold}` rows (the schema in `docs/finetune.md`).
+ `data` is a JSONL file of `{state, questions, gold}` or `{state, questions, expected}` rows,
+ or a CSV file with text and label columns.
`model_dir` is a local checkpoint directory -- the layout `laya.load` reads. The result in
`output_dir` loads with `laya.load(output_dir)`.
"""
@@ -467,7 +657,10 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainC
max_len = config.max_len or cfg.get("max_len", 512)
head_max_len = config.head_max_len or cfg.get("head_max_len", 192)
- items, skipped = items_from_rows(tok, read_jsonl(data), max_len, head_max_len)
+ rows = read_data(data, text_column=config.text_column, label_column=config.label_column,
+ question_id=config.question_id, instructions=config.instructions)
+ items, skipped = items_from_rows(tok, rows, max_len, head_max_len,
+ label_smoothing=config.label_smoothing)
if not items:
raise ValueError("%s produced no training items (skipped: %r)" % (data, skipped))
train_items, calib_items = split_calibration(items, config.calib_max, config.calib_frac, config.calib_seed)
diff --git a/laya/train_cli.py b/laya/train_cli.py
new file mode 100644
index 00000000..aeea279f
--- /dev/null
+++ b/laya/train_cli.py
@@ -0,0 +1,202 @@
+"""`laya-train`: fine-tune a Laya checkpoint on labelled data (CSV or JSONL).
+
+Usage:
+ laya-train --data tickets.csv --text-column body --label-column department \\
+ --base convaiinnovations/laya --out ./my-checkpoint
+
+ laya-train --data train.jsonl --base multilingual --out ./my-checkpoint
+
+`laya train ...` dispatches here from the main CLI, so both spellings work.
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from typing import Optional, Sequence
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ prog="laya-train",
+ description="Fine-tune a Laya checkpoint on labelled decisions (JSONL or CSV).",
+ )
+ # Required data and output options
+ parser.add_argument(
+ "--data",
+ required=True,
+ help="Path to training data file (.jsonl or .csv).",
+ )
+ parser.add_argument(
+ "--out",
+ "--output-dir",
+ dest="output_dir",
+ required=True,
+ help="Output directory where the fine-tuned checkpoint will be saved.",
+ )
+ parser.add_argument(
+ "--base",
+ "--model",
+ dest="model_dir",
+ default="convaiinnovations/laya",
+ help="Base checkpoint path or model ID (default: 'convaiinnovations/laya').",
+ )
+
+ # CSV mapping options
+ parser.add_argument(
+ "--text-column",
+ default="text",
+ help="CSV column containing the input text/state (default: 'text').",
+ )
+ parser.add_argument(
+ "--label-column",
+ default="label",
+ help="CSV column containing the target label (default: 'label').",
+ )
+ parser.add_argument(
+ "--question-id",
+ default="label",
+ help="Question identifier when synthesizing a question from CSV (default: 'label').",
+ )
+ parser.add_argument(
+ "--instructions",
+ default=None,
+ help="Instructions string for synthesized choice question.",
+ )
+
+ # Training hyperparameters
+ parser.add_argument(
+ "--epochs",
+ type=int,
+ default=4,
+ help="Number of training epochs (default: 4).",
+ )
+ parser.add_argument(
+ "--micro-batch",
+ type=int,
+ default=8,
+ help="Micro-batch size per forward pass (default: 8).",
+ )
+ parser.add_argument(
+ "--grad-accum",
+ type=int,
+ default=8,
+ help="Gradient accumulation steps (default: 8).",
+ )
+ parser.add_argument(
+ "--encoder-lr",
+ type=float,
+ default=2.5e-5,
+ help="Learning rate for encoder (default: 2.5e-5).",
+ )
+ parser.add_argument(
+ "--head-lr",
+ type=float,
+ default=1e-4,
+ help="Learning rate for classification heads (default: 1e-4).",
+ )
+ parser.add_argument(
+ "--loss",
+ choices=["soft-ce", "rlcd"],
+ default="rlcd",
+ help="Training objective: 'rlcd' or 'soft-ce' (default: 'rlcd').",
+ )
+ parser.add_argument(
+ "--label-smoothing",
+ type=float,
+ default=0.0,
+ help="Label smoothing epsilon for hard labels in [0, 1) (default: 0.0).",
+ )
+ parser.add_argument(
+ "--shuffle-options",
+ action="store_true",
+ help="Enable random option-order augmentation during training for choice questions.",
+ )
+ parser.add_argument(
+ "--freeze-encoder",
+ action="store_true",
+ help="Freeze encoder weights, training classification heads only.",
+ )
+ parser.add_argument(
+ "--device",
+ default="auto",
+ help="Compute device ('auto', 'cpu', 'cuda', 'mps'; default: 'auto').",
+ )
+ parser.add_argument(
+ "--seed",
+ type=int,
+ default=0,
+ help="Random seed for reproducibility (default: 0).",
+ )
+ parser.add_argument(
+ "--max-len",
+ type=int,
+ default=None,
+ help="Override maximum state token budget.",
+ )
+ parser.add_argument(
+ "--head-max-len",
+ dest="head_max_len",
+ type=int,
+ default=None,
+ help="Override maximum question head token budget.",
+ )
+ return parser
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+ parser = build_parser()
+ try:
+ args = parser.parse_args(argv)
+ except SystemExit as exc:
+ return exc.code
+
+ try:
+ from .train import TrainConfig, finetune
+ except ImportError as err:
+ if "torch" in str(err).lower():
+ print("laya-train: error: PyTorch is required for training. Please install torch.",
+ file=sys.stderr)
+ return 1
+ raise
+
+ shuffle_opts = ("choice",) if args.shuffle_options else ()
+ config = TrainConfig(
+ epochs=args.epochs,
+ micro_batch=args.micro_batch,
+ grad_accum=args.grad_accum,
+ encoder_lr=args.encoder_lr,
+ head_lr=args.head_lr,
+ loss=args.loss,
+ label_smoothing=args.label_smoothing,
+ shuffle_options=shuffle_opts,
+ freeze_encoder=args.freeze_encoder,
+ seed=args.seed,
+ max_len=args.max_len,
+ head_max_len=args.head_max_len,
+ text_column=args.text_column,
+ label_column=args.label_column,
+ question_id=args.question_id,
+ instructions=args.instructions,
+ )
+
+ try:
+ config.validate()
+ summary = finetune(
+ data=args.data,
+ model_dir=args.model_dir,
+ output_dir=args.output_dir,
+ config=config,
+ device=args.device,
+ )
+ print("Training complete! Model saved to %s" % summary["output_dir"])
+ return 0
+ except (ValueError, FileNotFoundError, TypeError) as err:
+ print("laya-train: error: %s" % (err,), file=sys.stderr)
+ return 1
+ except Exception as err:
+ print("laya-train: unexpected error: %s" % (err,), file=sys.stderr)
+ return 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/pyproject.toml b/pyproject.toml
index 420661e2..5ef73b19 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -81,6 +81,7 @@ laya = "laya.cli:main"
laya-serve = "laya.serve:main"
laya-evals = "laya.evals_cli:main"
laya-mcp-server = "laya.mcp.server:main"
+laya-train = "laya.train_cli:main"
[project.urls]
Homepage = "https://huggingface.co/convaiinnovations/laya"
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 6589d0db..0abd6059 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -703,6 +703,10 @@ code, out, err, stub = run_cli(["--predict", "--min-confidence", "0.5", "--json"
check("threshold: --json still carries the raw key",
'"low_confidence": true' in out, out)
+# --------------------------------------------------------------------- train subcommand
+code, out, err, stub = run_cli(["train", "--help"])
+check("train subcommand: dispatches to train_cli and shows help", code == 0 and "laya-train" in out, out)
+
# Drift pin, the other direction: every control `predict` takes that the parser can name must
# arrive in the call, and the ones it cannot name must be the accepted three. A control added to
# `predict` lands in the second check and has to be placed -- flagged and forwarded, or written
diff --git a/tests/test_train.py b/tests/test_train.py
index f7ca5558..4e7732d2 100644
--- a/tests/test_train.py
+++ b/tests/test_train.py
@@ -46,13 +46,17 @@ from laya.train import ( # noqa: E402
finetune,
items_from_rows,
make_item,
+ read_data,
+ rows_from_csv,
rlcd_loss,
soft_ce_loss,
split_calibration,
+ target_from_expected,
target_from_gold,
to_internal,
train_model,
)
+from laya.train_cli import build_parser as build_train_parser, main as train_cli_main # noqa: E402
WORDS = ["refund", "invoice", "charged", "twice", "crash", "error", "app", "screen", "please",
"billing", "technical", "other", "is", "it", "urgent", "low", "high", "level", "question",
@@ -136,6 +140,105 @@ class DataTests(unittest.TestCase):
with self.assertRaises(ValueError):
target(DEPARTMENT, bad)
+ def test_target_from_expected_choice(self):
+ def target(question, exp, smoothing=0.0):
+ return target_from_expected(to_internal("q", question), exp, label_smoothing=smoothing)
+
+ self.assertEqual(target(DEPARTMENT, "billing"), [1.0, 0.0, 0.0])
+ self.assertEqual(target(DEPARTMENT, "technical"), [0.0, 1.0, 0.0])
+ self.assertEqual(target(DEPARTMENT, "other"), [0.0, 0.0, 1.0])
+ # Non-string labels matching keys
+ self.assertEqual(target({"type": "choice", "instructions": "which ?", "criteria": {1: "a", 2: "b"}}, 1),
+ [1.0, 0.0])
+
+ def test_target_from_expected_noul(self):
+ def target(question, exp, smoothing=0.0):
+ return target_from_expected(to_internal("q", question), exp, label_smoothing=smoothing)
+
+ self.assertEqual(target(URGENT, True), [0.0, 1.0])
+ self.assertEqual(target(URGENT, False), [1.0, 0.0])
+ self.assertEqual(target(URGENT, "true"), [0.0, 1.0])
+ self.assertEqual(target(URGENT, "false"), [1.0, 0.0])
+
+ def test_target_from_expected_score(self):
+ def target(question, exp, smoothing=0.0):
+ return target_from_expected(to_internal("q", question), exp, label_smoothing=smoothing)
+
+ self.assertEqual(target(LEVEL, 0), [1.0, 0.0])
+ self.assertEqual(target(LEVEL, 1), [0.0, 1.0])
+ self.assertEqual(target(LEVEL, "1"), [0.0, 1.0])
+
+ def test_target_from_expected_label_smoothing(self):
+ res = target_from_expected(to_internal("q", DEPARTMENT), "billing", label_smoothing=0.1)
+ self.assertAlmostEqual(res[0], 0.9 + 0.1 / 3)
+ self.assertAlmostEqual(res[1], 0.1 / 3)
+ self.assertAlmostEqual(res[2], 0.1 / 3)
+ self.assertAlmostEqual(sum(res), 1.0)
+
+ res_noul = target_from_expected(to_internal("q", URGENT), True, label_smoothing=0.2)
+ self.assertAlmostEqual(res_noul[0], 0.1)
+ self.assertAlmostEqual(res_noul[1], 0.9)
+ self.assertAlmostEqual(sum(res_noul), 1.0)
+
+ def test_target_from_expected_rejects_invalid(self):
+ with self.assertRaises(ValueError):
+ target_from_expected(to_internal("q", DEPARTMENT), "unknown_choice")
+ with self.assertRaises(ValueError):
+ target_from_expected(to_internal("q", URGENT), "not_a_bool")
+ for bad_num in (-3, 7, 0.5, -0.1, 2):
+ with self.assertRaises(ValueError):
+ target_from_expected(to_internal("q", URGENT), bad_num)
+ with self.assertRaises(ValueError):
+ target_from_expected(to_internal("q", LEVEL), 99)
+ for bad_smooth in (-0.1, 1.0, 1.5, True):
+ with self.assertRaises(ValueError):
+ target_from_expected(to_internal("q", DEPARTMENT), "billing", label_smoothing=bad_smooth)
+
+ def test_resolve_checkpoint_dir(self):
+ from laya.train import resolve_checkpoint_dir
+ with tempfile.TemporaryDirectory() as tmpdir:
+ p = Path(tmpdir)
+ (p / "rl_agent_config.json").write_text("{}", encoding="utf-8")
+ self.assertEqual(resolve_checkpoint_dir(str(p)), str(p))
+ with self.assertRaises(FileNotFoundError):
+ resolve_checkpoint_dir(str(p / "nonexistent"))
+
+ def test_rows_from_csv_and_read_data(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ csv_path = Path(tmpdir) / "tickets.csv"
+ csv_path.write_text("body,department\nrefund my invoice,billing\napp crash on startup,technical\n",
+ encoding="utf-8")
+ data = read_data(str(csv_path), text_column="body", label_column="department")
+ self.assertEqual(len(data), 2)
+ self.assertEqual(data[0]["state"], "refund my invoice")
+ self.assertEqual(data[0]["expected"]["label"], "billing")
+ q = data[0]["questions"]["label"]
+ self.assertEqual(q["type"], "choice")
+ self.assertEqual(list(q["criteria"].keys()), ["billing", "technical"])
+
+ def test_rows_from_csv_validation(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ p = Path(tmpdir) / "bad.csv"
+ p.write_text("text,dept\nhi,billing\n", encoding="utf-8")
+ with self.assertRaises(ValueError):
+ rows_from_csv(str(p), text_column="body", label_column="dept")
+ p.write_text("text,dept\nhi,billing\nhello,billing\n", encoding="utf-8")
+ with self.assertRaises(ValueError):
+ rows_from_csv(str(p), text_column="text", label_column="dept")
+
+ def test_items_from_rows_supports_expected(self):
+ exp_rows = [
+ {"state": "please refund invoice", "questions": {"department": DEPARTMENT},
+ "expected": {"department": "billing"}},
+ {"state": "app crash", "questions": {"department": DEPARTMENT},
+ "expected": {"department": "technical"}},
+ ]
+ items, skipped = items_from_rows(self.tok, exp_rows, 64, 40)
+ self.assertEqual(len(items), 2)
+ self.assertEqual(skipped, {})
+ self.assertEqual(items[0]["target"], [1.0, 0.0, 0.0])
+ self.assertEqual(items[1]["target"], [0.0, 1.0, 0.0])
+
def test_questions_are_rendered_exactly_as_inference_renders_them(self):
# The sequence a training item produces must be the one `Agent` builds for the same
# question, or the model is trained on option texts it never sees.
@@ -391,6 +494,47 @@ class EndToEndTests(unittest.TestCase):
with self.assertRaises(ValueError):
train_model(None, self.tok, [], TrainConfig(), torch.device("cpu"), 64, 40)
+ def test_laya_train_cli_parser_and_validation(self):
+ parser = build_train_parser()
+ args = parser.parse_args(["--data", "train.csv", "--out", "./out", "--label-smoothing", "0.1",
+ "--shuffle-options", "--freeze-encoder"])
+ self.assertEqual(args.data, "train.csv")
+ self.assertEqual(args.output_dir, "./out")
+ self.assertEqual(args.label_smoothing, 0.1)
+ self.assertTrue(args.shuffle_options)
+ self.assertTrue(args.freeze_encoder)
+
+ def test_laya_train_cli_runs_on_csv(self):
+ csv_path = self.root / "tickets.csv"
+ lines = ["text,department"]
+ for s in BILLING_STATES:
+ lines.append(f"{s},billing")
+ for s in TECH_STATES:
+ lines.append(f"{s},technical")
+ csv_path.write_text("\n".join(lines), encoding="utf-8")
+
+ out_dir = self.root / "out_cli"
+ code = train_cli_main([
+ "--data", str(csv_path),
+ "--base", str(self.root / "base"),
+ "--out", str(out_dir),
+ "--text-column", "text",
+ "--label-column", "department",
+ "--epochs", "1",
+ "--micro-batch", "4",
+ "--grad-accum", "1",
+ "--loss", "soft-ce",
+ "--label-smoothing", "0.05",
+ "--device", "cpu",
+ ])
+ self.assertEqual(code, 0)
+ self.assertTrue((out_dir / "model.safetensors").exists())
+ self.assertTrue((out_dir / "rl_agent_config.json").exists())
+ agent = load(str(out_dir), device="cpu")
+ ans = agent.predict("please refund invoice", {"label": {"type": "choice", "instructions": "Classify",
+ "criteria": ["billing", "technical"]}})
+ self.assertIn(ans["answers"]["label"]["choice"], ["billing", "technical"])
+
if __name__ == "__main__":
unittest.main()