Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
test(portability): gate that no repo read in tests/ leaves the codec to the runner
Aashish506d7225bfcd
diff --git a/tests/test_portability.py b/tests/test_portability.pyindex 7a19a921..01140bad 100644--- a/tests/test_portability.py+++ b/tests/test_portability.py@@ -1,7 +1,7 @@"""Portability regressions: checkpoints saved by transformers 5, and non-CUDA devices.-Two failures found while getting Laya to run on a macOS/Intel box, both silent or fatal depending-on the machine:+Three failures found while getting Laya to run on a machine that is not the one it was written+on, all silent or fatal depending on the platform:1. transformers 5 records ModernBERT's RoPE bases as`rope_parameters = {"full_attention": {...}, "sliding_attention": {...}}`. transformers 4.x@@ -14,7 +14,17 @@ on the machine:autocast backend for, even when disabled: 'User specified an unsupported autocast device_typemps'. Laya only ever enables autocast on CUDA, but it still entered the context on every call,so `predict()` died outright on the MPS device torch selects on Apple/AMD machines.++3. `open(path)` and `Path.read_text()` with no `encoding=` take `locale.getpreferredencoding()`,+ which is cp1252 on the `tests (windows)` runner. The pages the suites police -- README.md, the+ docs, the examples -- are UTF-8 and carry characters cp1252 has no mapping for, so a test that+ reads one of them dies at the read with `UnicodeDecodeError: 'charmap' codec can't decode byte+ 0x81` instead of failing one assertion. It has cost the runner twice: once at a metadata gate+ that read README.md, and once before that at the batch-shape gate, whose fix is recorded in+ tests/test_mcp.py's comment at its README read. Section 4 below gates the convention."""+import ast+import globimport osimport sysfrom contextlib import nullcontext@@ -296,6 +306,102 @@ with mock.patch.object(torch.Tensor, "to", _to_that_ignores_mps):FAIL.append("autocast/streak was not survived: %s: %s" % (type(e).__name__, e))+# ------------------------------------------------ 4. a repo read must pin its encoding, not take the+# locale's. See the module docstring: on `tests (windows)` the locale codec is cp1252, so reading a+# UTF-8 page raises at the read and the whole suite exits 1 rather than failing one assertion.+TEXT_SUFFIXES = (".md", ".py", ".nix", ".txt", ".yml", ".yaml", ".json", ".toml")+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))++# The one suite this gate does not police, and why leaving it is not leaving a live crash: its repo+# read targets `laya/tl_kernels.py`, which cp1252 decodes today, and seven open pull requests append+# to this file from hunks that start at lines 946-970 -- inside the context of the read at line 960,+# so pinning it here would collide with all seven to fix a read that cannot raise. It leaves the set+# when they land, and the check below fails the moment the exemption stops being earned.+LOCALE_READ_EXEMPT = {"tests/test_hooks_api.py"}+++def _repo_reads(src):+ """The reads of a repo file `src` carries, as `(lineno, path expression, encoding pinned)`.++ Writes are out of scope: this is about decoding, which only a read does. A read is in scope when+ its path expression names a file that exists in the repo, or builds from the module's own repo+ root (`ROOT`, `parents[`, `dirname(`, `__file__`). The second arm is deliberately blunt --+ `open(os.path.join(ROOT, rel))` is how both crashes were written, and `rel` cannot be resolved+ without running the module, so a read of that shape is asked to pin its encoding rather than+ proven dangerous.+ """+ out = []+ for node in ast.walk(ast.parse(src)):+ if not isinstance(node, ast.Call):+ continue+ fn = node.func+ name = fn.attr if isinstance(fn, ast.Attribute) else (fn.id if isinstance(fn, ast.Name) else "")+ if name == "open" and node.args:+ target = node.args[0]+ elif name == "read_text" and isinstance(fn, ast.Attribute):+ # `Path.read_text()` carries its path on the object, not in `args`.+ target = fn.value+ else:+ continue+ modes = [ast.unparse(k.value) for k in node.keywords if k.arg == "mode"]+ if name == "open" and len(node.args) > 1:+ modes.append(ast.unparse(node.args[1]))+ if any("w" in m or "a" in m or "b" in m for m in modes):+ continue+ arg = ast.unparse(target)+ literals = [n.value for n in ast.walk(target)+ if isinstance(n, ast.Constant) and isinstance(n.value, str)]+ names_repo_file = any(l.endswith(TEXT_SUFFIXES) and os.path.exists(os.path.join(REPO, l))+ for l in literals)+ from_root = any(tok in arg for tok in ("ROOT", "parents[", "dirname(", "__file__"))+ if names_repo_file or from_root:+ out.append((node.lineno, arg, any(k.arg == "encoding" for k in node.keywords)))+ return out+++reads_by_suite = {}+for _path in sorted(glob.glob(os.path.join(REPO, "tests", "*.py"))):+ _rel = "tests/" + os.path.basename(_path)+ if _rel in LOCALE_READ_EXEMPT:+ continue+ with open(_path, encoding="utf-8") as _fh:+ reads_by_suite[_rel] = _repo_reads(_fh.read())++unpinned = ["%s:%d %s" % (_rel, _lineno, _arg)+ for _rel, _reads in sorted(reads_by_suite.items())+ for _lineno, _arg, _pinned in _reads if not _pinned]+check("encoding/no repo read in tests/ leaves the codec to the locale", unpinned, [])++# The rule must not pass by finding nothing: it has to see the reads that already pin the encoding.+pinned_seen = sum(1 for _reads in reads_by_suite.values() for _, _, _p in _reads if _p)+check_true("encoding/the scan sees the reads that already pin it", pinned_seen >= 10,+ "only %d pinned repo reads found, so the scan is not reaching the suites" % pinned_seen)+for _rel in sorted(LOCALE_READ_EXEMPT):+ with open(os.path.join(REPO, _rel), encoding="utf-8") as _fh:+ _exempt = _repo_reads(_fh.read())+ check_true("encoding/%s still earns its exemption" % _rel,+ any(not _p for _, _, _p in _exempt),+ "every repo read there pins its encoding now, so drop it from the exemption set")++# And the premise has to hold: pages the suites read really are UTF-8 that cp1252 cannot decode.+# Without one such page the rule is a style preference, not a crash guard.+_repo_text = []+for _pattern in ("README.md", os.path.join("docs", "**", "*.md"),+ os.path.join("examples", "*.py"), os.path.join("laya", "*.py")):+ _repo_text.extend(glob.glob(os.path.join(REPO, _pattern), recursive=True))+_undecodable = []+for _page in _repo_text:+ try:+ with open(_page, encoding="cp1252") as _fh:+ _fh.read()+ except UnicodeDecodeError:+ _undecodable.append(os.path.relpath(_page, REPO).replace(os.sep, "/"))+check_true("encoding/some repo page is UTF-8 the Windows locale cannot decode", bool(_undecodable),+ "every repo page decodes as cp1252, so the crash this gate guards has nothing to crash on")+print("encoding: %d repo pages cp1252 cannot decode, %d repo reads in tests/ all pinned"+ % (len(_undecodable), pinned_seen))++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:print(" FAIL " + f)