Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(cli): read the --questions and --batch files as utf-8-sig, so a BOM is not data
PerryLink9493bcb6e639
diff --git a/laya/cli.py b/laya/cli.pyindex d5794be0..9a67e97b 100644--- a/laya/cli.py+++ b/laya/cli.py@@ -160,7 +160,12 @@ def load_questions(path):file declares it. Default ``"request"``, matching bare ``--predict``."""try:- with open(path, encoding="utf-8") as handle:+ # `utf-8-sig`, not `utf-8`: a questions file written by Notepad or Excel starts with a+ # UTF-8 BOM, and a plain `utf-8` read leaves it as U+FEFF, so `json.load` refuses a file+ # that is valid JSON -- with its own "decode using utf-8-sig" hint. The other file readers+ # in this repo open the same way for the same reason (docker/entrypoint.py,+ # laya/train.py, scripts/http_inference_smoke.py).+ with open(path, encoding="utf-8-sig") as handle:raw = json.load(handle)except FileNotFoundError:raise ValueError("no such --questions file: %s" % path)@@ -321,10 +326,15 @@ def read_batch_lines(source):# A pipe is decoded with the locale's codec by default (cp1252 on Windows), not the utf-8 a# FILE is read with, so piped non-Latin requests arrived as mojibake.if hasattr(sys.stdin, "reconfigure"):- sys.stdin.reconfigure(encoding="utf-8")+ # `utf-8-sig` for the same reason as FILE below: a producer that wrote the stream with+ # a Windows editor prefixes U+FEFF, which would otherwise become the first request's+ # first character. It reads plain UTF-8 unchanged when there is no BOM.+ sys.stdin.reconfigure(encoding="utf-8-sig")lines = sys.stdin.read().splitlines()else:- with open(source, encoding="utf-8") as handle:+ # `utf-8-sig`: a batch file saved by Notepad or Excel starts with a UTF-8 BOM, which plain+ # `utf-8` would keep as U+FEFF at the front of the first request.+ with open(source, encoding="utf-8-sig") as handle:lines = handle.read().splitlines()return [line.strip() for line in lines if line.strip()]diff --git a/tests/test_cli.py b/tests/test_cli.pyindex a5454140..2f88c23c 100644--- a/tests/test_cli.py+++ b/tests/test_cli.py@@ -406,6 +406,29 @@ check("batch empty file: exit 2, no calls",code == 2 and "no requests" in err and stub.predict_batch_calls == [],"code %r err %r" % (code, err))+# Same BOM, one reader over: a batch file saved by a Windows editor used to send U+FEFF as the+# first character of the first request, which the model then read as part of the text.+bom_batch = os.path.join(tmp, "bom-requests.txt")+with open(bom_batch, "wb") as handle:+ handle.write(b"\xef\xbb\xbffirst ticket\nsecond ticket\n")+code, out, err, stub = run_batch_cli(["--batch", bom_batch, "--predict"])+requests = stub.predict_batch_calls[0][0] if stub.predict_batch_calls else []+check("batch: a UTF-8 BOM does not become part of the first request",+ code == 0 and [r["state"]["request"] for r in requests] == ["first ticket", "second ticket"],+ "code %r states %r err %r" % (code, [r["state"] for r in requests], err))++_bom_stdin = io.TextIOWrapper(io.BytesIO(b"\xef\xbb\xbffirst ticket\nsecond ticket\n"), encoding="utf-8")+_original_stdin = sys.stdin+sys.stdin = _bom_stdin+try:+ code, out, err, stub = run_batch_cli(["--batch", "-", "--predict"])+finally:+ sys.stdin = _original_stdin+requests = stub.predict_batch_calls[0][0] if stub.predict_batch_calls else []+check("batch -: a UTF-8 BOM on a piped stream is not part of the first request",+ code == 0 and [r["state"]["request"] for r in requests] == ["first ticket", "second ticket"],+ "code %r states %r err %r" % (code, [r["state"] for r in requests], err))+_original_stdin = sys.stdinsys.stdin = io.StringIO("piped one\npiped two\n")try:@@ -710,6 +733,14 @@ check("questions: too-deep JSON is exit 2, not a download-error message",code == 2 and "nested deeper" in err and "checkpoint" not in err,"code %r err %r" % (code, err))+# A UTF-8 BOM is what Notepad and Excel write. The other file readers in this repo open with+# `utf-8-sig` for exactly that reason (docker/entrypoint.py, laya/train.py), but this one read+# plain utf-8, so the BOM became U+FEFF and a valid questions file was refused as "not valid JSON".+bom_questions = write_raw("bom-questions.json", b"\xef\xbb\xbf" + _json.dumps(INTENTS).encode("utf-8"))+code, out, err, stub = run_cli(["hello", "--questions", bom_questions], router=QuestionRecorder())+check("questions: a UTF-8 BOM file is read, not refused as invalid JSON",+ code == 0 and stub.questions == INTENTS, "code %r err %r" % (code, err))+code, out, err, stub = run_cli(["hi", "--questions", INTENTS_PATH, "--preset", "triage"],router=QuestionRecorder())check("questions: two sources refuse each other",