Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(cli): write redirected output as utf-8

plox-sumit2f27840f0d2d
diff --git a/laya/cli.py b/laya/cli.py
index 94616f48..a2192071 100644
--- a/laya/cli.py
+++ b/laya/cli.py
@@ -384,6 +384,11 @@ def main(argv=None):
from .evals_cli import main as eval_main
return eval_main(argv[1:])
args = build_parser().parse_args(argv)
+ # Redirected output is encoded with the locale's codec by default (cp1252 on Windows), which
+ # cannot hold a non-Latin request or answer, so `laya --batch FILE > out.txt` failed with a
+ # codec error instead of writing it. utf-8 is what --batch reads back.
+ if hasattr(sys.stdout, "reconfigure"):
+ sys.stdout.reconfigure(encoding="utf-8")
text = " ".join(args.text).strip()
# Routing answers nothing, so there is no confidence to gate: `route` does not take the
# threshold at all. Accepting the flag there would let `laya --min-confidence 0.9 "..."` exit 0
diff --git a/tests/test_cli.py b/tests/test_cli.py
index ae15a909..6589d0db 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -337,6 +337,25 @@ _states = [list(r["state"].values()) for r in stub.predict_batch_calls[0][0]]
check("batch -: stdin is decoded as utf-8, like FILE", code == 0 and _states == [[_hindi]],
"code %r states %r" % (code, _states))
+# The same holds on the way out: redirected output is encoded with the locale's codec, which
+# cannot hold the request each batch line echoes, so the run failed instead of writing it.
+_hindi_path = os.path.join(tmp, "hindi.txt")
+with open(_hindi_path, "w", encoding="utf-8") as handle:
+ handle.write(_hindi + "\n")
+_raw = io.BytesIO()
+_redirected = io.TextIOWrapper(_raw, encoding="latin-1")
+_original_make_router = cli.make_router
+cli.make_router = lambda args: BatchRouter()
+try:
+ with redirect_stdout(_redirected), redirect_stderr(io.StringIO()) as err:
+ code = cli.main(["--batch", _hindi_path])
+ _redirected.flush()
+finally:
+ cli.make_router = _original_make_router
+check("batch: redirected stdout is written as utf-8", code == 0
+ and _hindi in _raw.getvalue().decode("utf-8", "replace"),
+ "code %r err %r out %r" % (code, err.getvalue(), _raw.getvalue()))
+
# ------------------------------------------------------------- --sort-by-length (#294 knob)
#
# `Agent.predict_batch` and `Router.predict_batch` have grouped similarly sized states into one