Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(cli): read --batch - from stdin as utf-8

plox-sumitdd19e5a16bb4
diff --git a/laya/cli.py b/laya/cli.py
index 6b5a7955..a33a6cbb 100644
--- a/laya/cli.py
+++ b/laya/cli.py
@@ -243,6 +243,10 @@ def run(text, args, router=None):
def read_batch_lines(source):
"""One request per non-blank line of FILE, or of stdin for '-'."""
if source == "-":
+ # A pipe is decoded with the locale's codec by default (cp1252 on Windows), not the utf-8 a
+ # FILE is read with, so piped non-Latin requests arrived as mojibake.
+ if hasattr(sys.stdin, "reconfigure"):
+ sys.stdin.reconfigure(encoding="utf-8")
lines = sys.stdin.read().splitlines()
else:
with open(source, encoding="utf-8") as handle:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index b895e3c7..9a3403aa 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -325,6 +325,18 @@ check("batch -: reads stdin", code == 0
and len(stub.predict_batch_calls[0][0]) == 2
and len([l for l in out.splitlines() if l.strip()]) == 2, "code %r err %r" % (code, err))
+# A pipe arrives in the locale's codec unless told otherwise, and FILE is read as utf-8, so `-`
+# has to be too or a non-Latin request reaches the router as mojibake.
+_hindi = "मुझसे दो बार"
+sys.stdin = io.TextIOWrapper(io.BytesIO(_hindi.encode("utf-8")), encoding="latin-1")
+try:
+ code, out, err, stub = run_batch_cli(["--batch", "-", "--predict", "--json"])
+finally:
+ sys.stdin = _original_stdin
+_states = [list(r["state"].values()) for r in stub.predict_batch_calls[0][0]]
+check("batch -: stdin is decoded as utf-8, like FILE", code == 0 and _states == [[_hindi]],
+ "code %r states %r" % (code, _states))
+
# ------------------------------------------------------------- --sort-by-length (#294 knob)
#
# `Agent.predict_batch` and `Router.predict_batch` have grouped similarly sized states into one