Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(evals): a dataset or report saved with a BOM is read, not refused
Aashish7e89ed3fc38d
diff --git a/laya/evals.py b/laya/evals.pyindex 40717279..73408688 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -255,7 +255,11 @@ class Dataset:# *is* a `ValueError`, so the wider class would re-wrap the row errors named just below.examples: List[Example] = []try:- with open(path, "r", encoding="utf-8") as handle:+ # `utf-8-sig`, not `utf-8`: a dataset exported from Excel or saved by Notepad starts+ # with a BOM, and a plain `utf-8` read leaves it as U+FEFF on line 1, so `json` refuses a+ # file that is valid JSON -- and says so with its own "decode using utf-8-sig" hint.+ # `laya/cli.py`, `laya/train.py` and `laya/evidence.py` open a caller's file this way.+ with open(path, "r", encoding="utf-8-sig") as handle:for lineno, line in enumerate(handle, 1):line = line.strip()if not line or line.startswith("#"):diff --git a/laya/evals_cli.py b/laya/evals_cli.pyindex 56c3b74e..f8d95a02 100644--- a/laya/evals_cli.py+++ b/laya/evals_cli.py@@ -290,7 +290,12 @@ def _load_report(path: str) -> Dict[str, Any]:# distinction the exit codes exist to draw. `laya-evals agreement` has always read its three# runs this way; this makes the shared report loader satisfy the same contract.try:- with open(path, "r", encoding="utf-8") as handle:+ # `utf-8-sig`, not `utf-8`: a report saved through a Windows editor or a spreadsheet export+ # starts with a BOM, and a plain `utf-8` read leaves it as U+FEFF, so `json.load` refuses a+ # document that is valid JSON -- with its own "decode using utf-8-sig" hint. `laya/cli.py`,+ # `laya/train.py` and the dataset reader in `laya/evals.py` open a caller's file this way+ # for the same reason.+ with open(path, "r", encoding="utf-8-sig") as handle:document = json.load(handle)except (OSError, ValueError, RecursionError) as exc:raise EvalError("cannot read report %r: %s" % (path, exc)) from exc@@ -592,7 +597,10 @@ def _cmd_agreement(args) -> int:runs = []for path in (args.reference, args.repeat, args.student):try:- with open(path, encoding="utf-8") as handle:+ # `utf-8-sig` for the same reason as `_load_report` above: a run re-saved through an+ # editor or a spreadsheet export starts with a BOM, which a plain `utf-8` read leaves as+ # U+FEFF and `json` refuses -- naming an agreement run as unreadable when it is valid.+ with open(path, encoding="utf-8-sig") as handle:runs.append(json.load(handle, object_pairs_hook=unique_object))except (OSError, ValueError, RecursionError) as exc:raise EvalError("cannot read agreement run %r: %s" % (path, exc)) from excdiff --git a/tests/test_evals.py b/tests/test_evals.pyindex f7edda1c..04baeb0f 100644--- a/tests/test_evals.py+++ b/tests/test_evals.py@@ -765,6 +765,29 @@ def test_dataset_rejects_bad_rows(row, fragment):assert fragment in str(exc.value)+def test_dataset_from_jsonl_reads_a_file_that_starts_with_a_bom(tmp_path):+ """A dataset saved by Notepad or exported from Excel is the same file with three bytes in front.++ A `utf-8` read leaves that BOM as U+FEFF on line 1, so `json.loads` refused the whole document+ with `"...:1 is not valid JSON: Unexpected UTF-8 BOM (decode using utf-8-sig)"` -- a refusal+ that names the fix this reader could have applied itself. `laya/cli.py` (--questions, --batch+ and stdin), `laya/train.py` and `laya/evidence.py` open a caller's file with `utf-8-sig` for+ exactly this reason, and the comment in `laya/cli.py` claims "the other file readers in this+ repo" do too; this was the dataset door that did not.+ """+ rows = [+ json.dumps({"state": "s1", "questions": Q, "expected": {"intent": "a"}, "language": "en"}),+ "# a comment line",+ json.dumps({"state": "s2", "questions": Q, "expected": {"intent": "b"}, "tags": ["t"]}),+ ]+ path = tmp_path / "bom.jsonl"+ path.write_text("" + "\n".join(rows), encoding="utf-8")+ dataset = Dataset.from_jsonl(str(path))+ assert len(dataset) == 2+ assert dataset.examples[0].language == "en"+ assert dataset.examples[1].tags == ("t",)++def test_dataset_rejects_malformed_json_and_empty(tmp_path):bad = tmp_path / "bad.jsonl"bad.write_text("{not json\n", encoding="utf-8")@@ -2955,6 +2978,30 @@ def test_cli_report_reads_refuse_an_unreadable_report_with_code_2(monkeypatch, tmonkeypatch.setattr("builtins.open", real_open)+def test_load_report_reads_a_file_that_starts_with_a_bom(tmp_path):+ """The report door is the same door as the dataset one, and had the same three-byte problem.++ `_load_report` opened `utf-8`, so a report re-saved through a Windows editor arrived with a BOM,+ `json.load` refused it, and the tool answered `cannot read report ... Unexpected UTF-8 BOM+ (decode using utf-8-sig)` with exit 2 -- a usage error whose own text names the decoding this+ reader could have asked for. `laya/cli.py` and `laya/evidence.py` open a caller's file with+ `utf-8-sig` for exactly this reason; the refusal paths above are unchanged, since a BOM is not a+ file that cannot be read.+ """+ from laya import evals_cli++ document = {"overall": {"choice_accuracy": 0.5}, "slices": {}, "cases": []}+ path = tmp_path / "bom-report.json"+ path.write_text("" + json.dumps(document), encoding="utf-8")+ assert evals_cli._load_report(str(path)) == document+ # Still refused, in the same words, for a document that is not JSON with or without a BOM.+ broken = tmp_path / "broken.json"+ broken.write_text("{nope", encoding="utf-8")+ with pytest.raises(EvalError) as exc:+ evals_cli._load_report(str(broken))+ assert "cannot read report" in str(exc.value)++def test_cli_dataset_reads_refuse_an_unreadable_dataset_with_code_2(monkeypatch, tmp_path, capsys):"""`validate` and `run` both call a dataset that cannot be read a usage error, not a crash.diff --git a/tests/test_evals_agreement.py b/tests/test_evals_agreement.pyindex ac6da4a7..bb35fd4f 100644--- a/tests/test_evals_agreement.py+++ b/tests/test_evals_agreement.py@@ -513,6 +513,25 @@ def test_cli_saved_run_replay_matches_public_api(tmp_path):assert json.loads(output.read_text(encoding="utf-8")) == expected+def test_cli_reads_bom_prefixed_saved_runs(tmp_path):+ """A run re-saved through an editor or a spreadsheet export is the same run with three bytes in front.++ The three agreement reads opened `utf-8`, so a UTF-8 BOM stayed as U+FEFF and `json.load` refused+ a document that is valid JSON, answering `cannot read agreement run ... Unexpected UTF-8 BOM+ (decode using utf-8-sig)` -- a refusal naming the decoding this CLI could have asked for itself.+ `laya/cli.py`, `laya/train.py` and `laya/evidence.py` open a caller's file with `utf-8-sig` for+ exactly this reason, and the byte-for-byte equality below is the point: the BOM changes nothing+ about the numbers, so it must change nothing about the verdict.+ """+ paths, runs = saved_runs(tmp_path)+ for path in paths:+ path.write_text("" + path.read_text(encoding="utf-8"), encoding="utf-8")+ expected = compare(runs, bootstrap_samples=100, seed=12)+ result = cli(*paths, "--bootstrap-samples", "100", "--seed", "12")+ assert result.returncode == 0, result.stderr+ assert json.loads(result.stdout) == expected++@pytest.mark.parametrize("invalid", ["duplicate_key", "nan", "malformed", "nested", "missing", "misaligned"])def test_cli_refuses_bad_inputs_without_a_report(tmp_path, invalid):paths, runs = saved_runs(tmp_path)