Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(cli): read the --questions and --batch files as utf-8-sig, so a BOM is not data

PerryLink9493bcb6e639
diff --git a/laya/cli.py b/laya/cli.py
index d5794be0..9a67e97b 100644
--- a/laya/cli.py
+++ b/laya/cli.py
@@ -160,7 +160,12 @@ def load_questions(path):
file declares it. Default ``"request"``, matching bare ``--predict``.
"""
try:
- with open(path, encoding="utf-8") as handle:
+ # `utf-8-sig`, not `utf-8`: a questions file written by Notepad or Excel starts with a
+ # UTF-8 BOM, and a plain `utf-8` read leaves it as U+FEFF, so `json.load` refuses a file
+ # that is valid JSON -- with its own "decode using utf-8-sig" hint. The other file readers
+ # in this repo open the same way for the same reason (docker/entrypoint.py,
+ # laya/train.py, scripts/http_inference_smoke.py).
+ with open(path, encoding="utf-8-sig") as handle:
raw = json.load(handle)
except FileNotFoundError:
raise ValueError("no such --questions file: %s" % path)
@@ -321,10 +326,15 @@ def read_batch_lines(source):
# A pipe is decoded with the locale's codec by default (cp1252 on Windows), not the utf-8 a
# FILE is read with, so piped non-Latin requests arrived as mojibake.
if hasattr(sys.stdin, "reconfigure"):
- sys.stdin.reconfigure(encoding="utf-8")
+ # `utf-8-sig` for the same reason as FILE below: a producer that wrote the stream with
+ # a Windows editor prefixes U+FEFF, which would otherwise become the first request's
+ # first character. It reads plain UTF-8 unchanged when there is no BOM.
+ sys.stdin.reconfigure(encoding="utf-8-sig")
lines = sys.stdin.read().splitlines()
else:
- with open(source, encoding="utf-8") as handle:
+ # `utf-8-sig`: a batch file saved by Notepad or Excel starts with a UTF-8 BOM, which plain
+ # `utf-8` would keep as U+FEFF at the front of the first request.
+ with open(source, encoding="utf-8-sig") as handle:
lines = handle.read().splitlines()
return [line.strip() for line in lines if line.strip()]
diff --git a/tests/test_cli.py b/tests/test_cli.py
index a5454140..2f88c23c 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -406,6 +406,29 @@ check("batch empty file: exit 2, no calls",
code == 2 and "no requests" in err and stub.predict_batch_calls == [],
"code %r err %r" % (code, err))
+# Same BOM, one reader over: a batch file saved by a Windows editor used to send U+FEFF as the
+# first character of the first request, which the model then read as part of the text.
+bom_batch = os.path.join(tmp, "bom-requests.txt")
+with open(bom_batch, "wb") as handle:
+ handle.write(b"\xef\xbb\xbffirst ticket\nsecond ticket\n")
+code, out, err, stub = run_batch_cli(["--batch", bom_batch, "--predict"])
+requests = stub.predict_batch_calls[0][0] if stub.predict_batch_calls else []
+check("batch: a UTF-8 BOM does not become part of the first request",
+ code == 0 and [r["state"]["request"] for r in requests] == ["first ticket", "second ticket"],
+ "code %r states %r err %r" % (code, [r["state"] for r in requests], err))
+
+_bom_stdin = io.TextIOWrapper(io.BytesIO(b"\xef\xbb\xbffirst ticket\nsecond ticket\n"), encoding="utf-8")
+_original_stdin = sys.stdin
+sys.stdin = _bom_stdin
+try:
+ code, out, err, stub = run_batch_cli(["--batch", "-", "--predict"])
+finally:
+ sys.stdin = _original_stdin
+requests = stub.predict_batch_calls[0][0] if stub.predict_batch_calls else []
+check("batch -: a UTF-8 BOM on a piped stream is not part of the first request",
+ code == 0 and [r["state"]["request"] for r in requests] == ["first ticket", "second ticket"],
+ "code %r states %r err %r" % (code, [r["state"] for r in requests], err))
+
_original_stdin = sys.stdin
sys.stdin = io.StringIO("piped one\npiped two\n")
try:
@@ -710,6 +733,14 @@ check("questions: too-deep JSON is exit 2, not a download-error message",
code == 2 and "nested deeper" in err and "checkpoint" not in err,
"code %r err %r" % (code, err))
+# A UTF-8 BOM is what Notepad and Excel write. The other file readers in this repo open with
+# `utf-8-sig` for exactly that reason (docker/entrypoint.py, laya/train.py), but this one read
+# plain utf-8, so the BOM became U+FEFF and a valid questions file was refused as "not valid JSON".
+bom_questions = write_raw("bom-questions.json", b"\xef\xbb\xbf" + _json.dumps(INTENTS).encode("utf-8"))
+code, out, err, stub = run_cli(["hello", "--questions", bom_questions], router=QuestionRecorder())
+check("questions: a UTF-8 BOM file is read, not refused as invalid JSON",
+ code == 0 and stub.questions == INTENTS, "code %r err %r" % (code, err))
+
code, out, err, stub = run_cli(["hi", "--questions", INTENTS_PATH, "--preset", "triage"],
router=QuestionRecorder())
check("questions: two sources refuse each other",