Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(rans_verify): a stdout that cannot encode U+2014 loses the verdict

kevin932753748e1e1e84
diff --git a/c/tests/test_rans_repack.py b/c/tests/test_rans_repack.py
index 2eebbce3..979f9ce6 100644
--- a/c/tests/test_rans_repack.py
+++ b/c/tests/test_rans_repack.py
@@ -445,6 +445,28 @@ class TestRansRepack(unittest.TestCase):
self.assertEqual(r.returncode, 1, msg=f"{label}: {r.stdout}")
self.assertIn("REFUSE", r.stdout, msg=f"{label}: no REFUSE line")
+ def test_verifier_survives_a_non_utf8_stdout(self):
+ """On a Korean or Japanese Windows (cp949, cp932) a redirected stdout
+ is encoded in a code page with no U+2014, and the verifier's own
+ `[note] ... no repack-manifest.json — ...` line raised
+ UnicodeEncodeError. Its E_INTERNAL fallback printed the same text and
+ raised again, so a valid shard without a manifest came back as exit 1,
+ a traceback and no verdict. PYTHONIOENCODING stands in for that code
+ page, so this runs the same on every platform, and the output is read
+ back in that code page, the way such a caller would read it."""
+ shard = self.repack(self.root / "codepage")[0]
+ (shard.parent / "repack-manifest.json").unlink()
+ env = dict(os.environ, PYTHONIOENCODING="cp949")
+ r = subprocess.run([sys.executable, str(TOOLS / "rans_verify.py"),
+ str(shard)], capture_output=True, env=env)
+ out = r.stdout.decode("cp949")
+ err = r.stderr.decode("cp949", "backslashreplace")
+ self.assertNotIn("Traceback", err, msg=err)
+ self.assertEqual(r.returncode, 0, msg=out + err)
+ self.assertIn("no repack-manifest.json", out)
+ self.assertIn("whole-artifact digest check skipped", out)
+ self.assertIn("RESULT: TRUST", out)
+
def test_partial_output_refusal(self):
"""An outdir already holding repack output is refused (crash
evidence: an interrupted set must not be papered over); --force
diff --git a/c/tools/rans_verify.py b/c/tools/rans_verify.py
index 81cc5fda..641675b5 100644
--- a/c/tools/rans_verify.py
+++ b/c/tools/rans_verify.py
@@ -88,6 +88,17 @@ import json
import os
import sys
+# The notes and refusal details print U+2014, which a stdout in cp949 or cp932
+# (a redirected stdout on Korean or Japanese Windows) cannot encode: the print
+# raised, the E_INTERNAL fallback printed the same text and raised again, and
+# the verdict was lost. Escape what the encoding cannot hold, as CPython already
+# does on stderr; the encoding itself is left alone, so whoever reads this
+# output in that code page still can. A UTF-8 stdout is unaffected.
+try:
+ sys.stdout.reconfigure(errors="backslashreplace")
+except AttributeError:
+ pass
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import rans_format as rf # noqa: E402