Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
"""Model-free regressions: python laya-rust/tools/test_ci_tools.py."""
import contextlib
import io
import types
import unittest
from unittest.mock import mock_open, patch
from google.protobuf.message import DecodeError
import check_test_skips as guard
import regen_golden as regen
from repr_cases import build_repr_probe
def summary(passed=10, failed=0, ignored=0, filtered=0):
outcome = "FAILED" if failed else "ok"
return ("test result: %s. %d passed; %d failed; %d ignored; 0 measured; %d filtered out;\n"
% (outcome, passed, failed, ignored, filtered))
class SkipGuardTests(unittest.TestCase):
def check(self, text, minimum=1):
with contextlib.redirect_stdout(io.StringIO()):
return guard.check(text, minimum)
def test_passed_run_including_empty_sample_and_doc_binaries(self):
self.assertEqual(0, self.check(summary() + summary(0)))
def test_early_returns_counted_as_passes_are_rejected(self):
for reason in [
"skip: no ONNX artifacts for 'english'",
"skip: no split-layout ONNX artifacts for 'multilingual'",
"skip: no golden data for 'typed-decisions'",
"test parity ... skip: unexpected reason",
]:
with self.subTest(reason=reason):
self.assertEqual(1, self.check(reason + "\n" + summary()))
def test_ignored_and_failed_tests_are_rejected(self):
for text in [summary(ignored=1), summary(failed=1), summary(filtered=1)]:
with self.subTest(text=text):
self.assertEqual(1, self.check(text))
def test_empty_wrong_filter_and_build_errors_fail_closed(self):
for text in ["", summary(0), summary(1), summary() + "error: compiler failed\n"]:
with self.subTest(text=text):
self.assertEqual(1, self.check(text, minimum=2))
def test_unknown_or_incomplete_runs_fail_closed(self):
for text in [
summary() + "test result: Unknown. 1 passed;\n",
" Running tests/one.rs (target/one)\n" + summary()
+ " Running tests/two.rs (target/two)\n",
]:
with self.subTest(text=text):
self.assertEqual(1, self.check(text))
def test_shard_floor_is_per_shard_and_selected_checkpoint_skips_fail(self):
# Each parity shard passes its own --min-passed. A shard that loaded no checkpoint scores
# only the model-free tests, which a whole-lane floor of 200 would have let through if the
# other shards had made up the count; its own floor does not.
model_free_only = summary(passed=60)
self.assertEqual(0, self.check(model_free_only, minimum=60))
self.assertEqual(1, self.check(model_free_only, minimum=61))
self.assertEqual(1, self.check(
"skip: no ONNX artifacts for 'english'; set LAYA_ONNX_ROOT\n" + summary(passed=500),
minimum=61))
def test_missing_output_fails(self):
with patch("sys.argv", ["guard", "missing.log"]), patch.object(
guard.Path, "read_text", side_effect=FileNotFoundError("missing")
), contextlib.redirect_stdout(io.StringIO()):
self.assertEqual(1, guard.main())
class ExportCacheTests(unittest.TestCase):
def validate(self, stamp="expected", sidecar_size=100, length=100, broken_graph=False):
import onnx
tensor = onnx.TensorProto(data_location=onnx.TensorProto.EXTERNAL)
for key, value in [("location", "encoder.onnx.data"), ("length", str(length))]:
entry = tensor.external_data.add()
entry.key, entry.value = key, value
model = types.SimpleNamespace(graph=types.SimpleNamespace(initializer=[tensor]))
def size(path):
if path.endswith(".data"):
if sidecar_size is None:
raise FileNotFoundError(path)
return sidecar_size
return 100
with patch("builtins.open", mock_open(read_data='{"inputs_sha256": "%s"}' % stamp)):
with patch.object(regen.os.path, "getsize", side_effect=size):
with patch.object(onnx, "load", return_value=model,
side_effect=DecodeError("broken") if broken_graph else None):
return regen.is_valid("artifacts", ["encoder.onnx"], "expected")
def test_complete_cache_is_reused(self):
self.assertTrue(self.validate())
def test_stale_stamp_is_rejected(self):
self.assertFalse(self.validate(stamp="old"))
def test_missing_empty_or_truncated_sidecar_is_rejected(self):
for size in [None, 0, 99]:
with self.subTest(size=size):
self.assertFalse(self.validate(sidecar_size=size))
def test_malformed_graph_is_rejected(self):
self.assertFalse(self.validate(broken_graph=True))
def test_cached_exports_still_regenerate_goldens(self):
with patch.object(regen, "is_valid", return_value=True), patch.object(
regen, "inputs_hash", return_value="stamp"
), patch.object(regen, "run") as run, patch.object(regen, "snapshot") as snapshot:
regen.regen_checkpoint("english", "artifacts", "revision", False)
snapshot.assert_not_called()
run.assert_called_once()
self.assertIn("dump_golden.py", run.call_args.args[0][1])
self.assertIn("--checkpoint", run.call_args.args[0])
def test_routing_regeneration_is_unconditional(self):
with patch("sys.argv", ["regen", "--checkpoint", "routing"]), patch.object(
regen, "run"
) as run, patch.object(regen.shutil, "rmtree"):
self.assertEqual(0, regen.main())
calls = [c.args[0] for c in run.call_args_list]
self.assertEqual(2, len(calls))
self.assertIn("dump_routing_golden.py", calls[0][1])
self.assertIn("--force", calls[0])
self.assertIn("dump_differential_golden.py", calls[1][1])
def test_repr_regeneration_needs_no_models(self):
with patch("sys.argv", ["regen", "--checkpoint", "repr"]), patch.object(
regen, "run"
) as run, patch.object(regen, "regen_checkpoint") as model, patch.object(regen.shutil, "rmtree"):
self.assertEqual(0, regen.main())
model.assert_not_called()
run.assert_called_once()
self.assertIn("repr_cases.py", run.call_args.args[0][1])
class FloatProbeTests(unittest.TestCase):
def test_sweep_is_reproducible_and_preserves_bit_patterns(self):
import math
import struct
probe = build_repr_probe()
self.assertEqual(probe, build_repr_probe())
self.assertEqual(2261, len(probe))
self.assertEqual(len(probe), len({entry["bits"] for entry in probe}))
for entry in probe:
value = struct.unpack(">d", bytes.fromhex(entry["bits"]))[0]
self.assertTrue(math.isfinite(value))
self.assertEqual(repr(value), entry["repr"])
self.assertEqual(
"1658206780088562.2",
next(entry["repr"] for entry in probe if entry["bits"] == "43179085685d83c9"),
)
class SelfCheckTests(unittest.TestCase):
def check(self, actual_act, reference_act, actual_logits=None):
import numpy as np
import torch
import dump_golden as dump
arrays = {f"single_noul/{n}": np.zeros((1, 2), dtype=np.int64)
for n in ["input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype"]}
arrays["single_noul/ref_logits"] = np.array([[0.1, 0.2]], dtype=np.float32)
arrays["single_noul/ref_act"] = np.array([reference_act], dtype=np.float32)
class Fixture:
files = list(arrays)
def __getitem__(self, key):
return arrays[key]
logits = arrays["single_noul/ref_logits"] if actual_logits is None else np.array(
[actual_logits], dtype=np.float32
)
agent = types.SimpleNamespace(model=lambda *args: (
torch.from_numpy(logits), torch.tensor([actual_act], dtype=torch.float32)
))
with patch.object(dump.os.path, "exists", return_value=True), patch.object(
dump.np, "load", return_value=Fixture()
), contextlib.redirect_stdout(io.StringIO()):
return dump.self_check(agent, "artifacts")
def test_saturated_act_logits_may_drift_without_changing_signal(self):
report = self.check([4807.878, -3937.442], [4807.720, -3937.599])
self.assertGreater(report["single_noul"]["act_maxdiff"], 0.15)
self.assertEqual(0.0, report["single_noul"]["act_probs_maxdiff"])
def test_real_probability_drift_is_rejected_even_below_old_logit_tolerance(self):
with self.assertRaisesRegex(SystemExit, "act_probs.*refusing"):
self.check([0.005, 0.0], [0.0, 0.0])
def test_saturated_action_flip_is_rejected(self):
with self.assertRaises(SystemExit):
self.check([-3937.599, 4807.720], [4807.720, -3937.599])
def test_small_unsaturated_probability_drift_is_accepted(self):
report = self.check([0.0002, 0.0], [0.0, 0.0])
self.assertLess(report["single_noul"]["act_probs_maxdiff"], 1e-4)
def test_option_logit_drift_remains_rejected(self):
with self.assertRaises(SystemExit):
self.check([0.0, 0.0], [0.0, 0.0], [0.102, 0.2])
def test_nonfinite_or_wrong_shape_outputs_fail_closed(self):
for actual, reference in [
([float("nan"), 0.0], [0.0, 0.0]),
([float("inf"), 0.0], [0.0, 0.0]),
([0.0, 0.0], [float("nan"), 0.0]),
([0.0], [0.0, 0.0]),
]:
with self.subTest(actual=actual, reference=reference), self.assertRaises(SystemExit):
self.check(actual, reference)
if __name__ == "__main__":
unittest.main()