Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): a dataset or report saved with a BOM is read, not refused

Aashish7e89ed3fc38d
diff --git a/laya/evals.py b/laya/evals.py
index 40717279..73408688 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -255,7 +255,11 @@ class Dataset:
# *is* a `ValueError`, so the wider class would re-wrap the row errors named just below.
examples: List[Example] = []
try:
- with open(path, "r", encoding="utf-8") as handle:
+ # `utf-8-sig`, not `utf-8`: a dataset exported from Excel or saved by Notepad starts
+ # with a BOM, and a plain `utf-8` read leaves it as U+FEFF on line 1, so `json` refuses a
+ # file that is valid JSON -- and says so with its own "decode using utf-8-sig" hint.
+ # `laya/cli.py`, `laya/train.py` and `laya/evidence.py` open a caller's file this way.
+ with open(path, "r", encoding="utf-8-sig") as handle:
for lineno, line in enumerate(handle, 1):
line = line.strip()
if not line or line.startswith("#"):
diff --git a/laya/evals_cli.py b/laya/evals_cli.py
index 56c3b74e..f8d95a02 100644
--- a/laya/evals_cli.py
+++ b/laya/evals_cli.py
@@ -290,7 +290,12 @@ def _load_report(path: str) -> Dict[str, Any]:
# distinction the exit codes exist to draw. `laya-evals agreement` has always read its three
# runs this way; this makes the shared report loader satisfy the same contract.
try:
- with open(path, "r", encoding="utf-8") as handle:
+ # `utf-8-sig`, not `utf-8`: a report saved through a Windows editor or a spreadsheet export
+ # starts with a BOM, and a plain `utf-8` read leaves it as U+FEFF, so `json.load` refuses a
+ # document that is valid JSON -- with its own "decode using utf-8-sig" hint. `laya/cli.py`,
+ # `laya/train.py` and the dataset reader in `laya/evals.py` open a caller's file this way
+ # for the same reason.
+ with open(path, "r", encoding="utf-8-sig") as handle:
document = json.load(handle)
except (OSError, ValueError, RecursionError) as exc:
raise EvalError("cannot read report %r: %s" % (path, exc)) from exc
@@ -592,7 +597,10 @@ def _cmd_agreement(args) -> int:
runs = []
for path in (args.reference, args.repeat, args.student):
try:
- with open(path, encoding="utf-8") as handle:
+ # `utf-8-sig` for the same reason as `_load_report` above: a run re-saved through an
+ # editor or a spreadsheet export starts with a BOM, which a plain `utf-8` read leaves as
+ # U+FEFF and `json` refuses -- naming an agreement run as unreadable when it is valid.
+ with open(path, encoding="utf-8-sig") as handle:
runs.append(json.load(handle, object_pairs_hook=unique_object))
except (OSError, ValueError, RecursionError) as exc:
raise EvalError("cannot read agreement run %r: %s" % (path, exc)) from exc
diff --git a/tests/test_evals.py b/tests/test_evals.py
index f7edda1c..04baeb0f 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -765,6 +765,29 @@ def test_dataset_rejects_bad_rows(row, fragment):
assert fragment in str(exc.value)
+def test_dataset_from_jsonl_reads_a_file_that_starts_with_a_bom(tmp_path):
+ """A dataset saved by Notepad or exported from Excel is the same file with three bytes in front.
+
+ A `utf-8` read leaves that BOM as U+FEFF on line 1, so `json.loads` refused the whole document
+ with `"...:1 is not valid JSON: Unexpected UTF-8 BOM (decode using utf-8-sig)"` -- a refusal
+ that names the fix this reader could have applied itself. `laya/cli.py` (--questions, --batch
+ and stdin), `laya/train.py` and `laya/evidence.py` open a caller's file with `utf-8-sig` for
+ exactly this reason, and the comment in `laya/cli.py` claims "the other file readers in this
+ repo" do too; this was the dataset door that did not.
+ """
+ rows = [
+ json.dumps({"state": "s1", "questions": Q, "expected": {"intent": "a"}, "language": "en"}),
+ "# a comment line",
+ json.dumps({"state": "s2", "questions": Q, "expected": {"intent": "b"}, "tags": ["t"]}),
+ ]
+ path = tmp_path / "bom.jsonl"
+ path.write_text("" + "\n".join(rows), encoding="utf-8")
+ dataset = Dataset.from_jsonl(str(path))
+ assert len(dataset) == 2
+ assert dataset.examples[0].language == "en"
+ assert dataset.examples[1].tags == ("t",)
+
+
def test_dataset_rejects_malformed_json_and_empty(tmp_path):
bad = tmp_path / "bad.jsonl"
bad.write_text("{not json\n", encoding="utf-8")
@@ -2955,6 +2978,30 @@ def test_cli_report_reads_refuse_an_unreadable_report_with_code_2(monkeypatch, t
monkeypatch.setattr("builtins.open", real_open)
+def test_load_report_reads_a_file_that_starts_with_a_bom(tmp_path):
+ """The report door is the same door as the dataset one, and had the same three-byte problem.
+
+ `_load_report` opened `utf-8`, so a report re-saved through a Windows editor arrived with a BOM,
+ `json.load` refused it, and the tool answered `cannot read report ... Unexpected UTF-8 BOM
+ (decode using utf-8-sig)` with exit 2 -- a usage error whose own text names the decoding this
+ reader could have asked for. `laya/cli.py` and `laya/evidence.py` open a caller's file with
+ `utf-8-sig` for exactly this reason; the refusal paths above are unchanged, since a BOM is not a
+ file that cannot be read.
+ """
+ from laya import evals_cli
+
+ document = {"overall": {"choice_accuracy": 0.5}, "slices": {}, "cases": []}
+ path = tmp_path / "bom-report.json"
+ path.write_text("" + json.dumps(document), encoding="utf-8")
+ assert evals_cli._load_report(str(path)) == document
+ # Still refused, in the same words, for a document that is not JSON with or without a BOM.
+ broken = tmp_path / "broken.json"
+ broken.write_text("{nope", encoding="utf-8")
+ with pytest.raises(EvalError) as exc:
+ evals_cli._load_report(str(broken))
+ assert "cannot read report" in str(exc.value)
+
+
def test_cli_dataset_reads_refuse_an_unreadable_dataset_with_code_2(monkeypatch, tmp_path, capsys):
"""`validate` and `run` both call a dataset that cannot be read a usage error, not a crash.
diff --git a/tests/test_evals_agreement.py b/tests/test_evals_agreement.py
index ac6da4a7..bb35fd4f 100644
--- a/tests/test_evals_agreement.py
+++ b/tests/test_evals_agreement.py
@@ -513,6 +513,25 @@ def test_cli_saved_run_replay_matches_public_api(tmp_path):
assert json.loads(output.read_text(encoding="utf-8")) == expected
+def test_cli_reads_bom_prefixed_saved_runs(tmp_path):
+ """A run re-saved through an editor or a spreadsheet export is the same run with three bytes in front.
+
+ The three agreement reads opened `utf-8`, so a UTF-8 BOM stayed as U+FEFF and `json.load` refused
+ a document that is valid JSON, answering `cannot read agreement run ... Unexpected UTF-8 BOM
+ (decode using utf-8-sig)` -- a refusal naming the decoding this CLI could have asked for itself.
+ `laya/cli.py`, `laya/train.py` and `laya/evidence.py` open a caller's file with `utf-8-sig` for
+ exactly this reason, and the byte-for-byte equality below is the point: the BOM changes nothing
+ about the numbers, so it must change nothing about the verdict.
+ """
+ paths, runs = saved_runs(tmp_path)
+ for path in paths:
+ path.write_text("" + path.read_text(encoding="utf-8"), encoding="utf-8")
+ expected = compare(runs, bootstrap_samples=100, seed=12)
+ result = cli(*paths, "--bootstrap-samples", "100", "--seed", "12")
+ assert result.returncode == 0, result.stderr
+ assert json.loads(result.stdout) == expected
+
+
@pytest.mark.parametrize("invalid", ["duplicate_key", "nan", "malformed", "nested", "missing", "misaligned"])
def test_cli_refuses_bad_inputs_without_a_report(tmp_path, invalid):
paths, runs = saved_runs(tmp_path)