Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

test(evals): pin the subclass acceptance and the scalar exactness it keeps

Aashish912a4f874c4d
diff --git a/tests/test_evals_agreement.py b/tests/test_evals_agreement.py
index edb5e173..3f7e95d0 100644
--- a/tests/test_evals_agreement.py
+++ b/tests/test_evals_agreement.py
@@ -1,10 +1,12 @@
"""Model-free checks for paired agreement and case-level uncertainty."""
from copy import deepcopy
+import collections
import json
from pathlib import Path
import subprocess
import sys
+import numpy as np
import pytest
from laya import confidence, evals
@@ -230,6 +232,91 @@ def test_malformed_runs_are_rejected(change):
compare(runs)
+def _deep(value, dict_factory, list_factory):
+ """Rebuild every mapping as `dict_factory(...)` and every list as `list_factory(...)`.
+
+ Scalars are left alone: these arms are about the container spellings a caller reaches for
+ when it wants a JSON object whose key order it controls, not about scalar subclasses.
+ """
+ if isinstance(value, dict):
+ return dict_factory({k: _deep(v, dict_factory, list_factory) for k, v in value.items()})
+ if isinstance(value, list):
+ return list_factory([_deep(v, dict_factory, list_factory) for v in value])
+ return value
+
+
+class SubDict(dict):
+ pass
+
+
+class SubList(list):
+ pass
+
+
[email protected]("flavour", ["ordered", "pairs_hook", "defaultdict", "counter",
+ "subdict", "sublist"])
+def test_container_subclasses_encode_as_the_mapping_they_wrap(flavour):
+ """`_json` matched containers by exact type, so it refused payloads `json.dumps` encodes
+ byte-for-byte as the plain mapping. `object_pairs_hook=OrderedDict` is how a caller reads a
+ saved run *and* keeps the key order this module fingerprints, and `Counter`/`defaultdict` are
+ what a caller builds criteria and settings from; all of them reached
+ "unsupported value of type OrderedDict" while every `isinstance(..., dict)` guard in `_run`,
+ `_provenance` and `_labels` had already accepted the same object."""
+ expected = compare(documents())
+ runs = documents()
+ if flavour == "ordered":
+ runs = [_deep(run, collections.OrderedDict, list) for run in runs]
+ elif flavour == "pairs_hook":
+ runs = [json.loads(json.dumps(run), object_pairs_hook=collections.OrderedDict)
+ for run in runs]
+ elif flavour == "defaultdict":
+ runs = [_deep(run, lambda d: collections.defaultdict(dict, d), list) for run in runs]
+ elif flavour == "counter":
+ for run in runs:
+ for case in run["cases"]:
+ for question in case["questions"].values():
+ question["criteria"] = collections.Counter(question["criteria"])
+ elif flavour == "subdict":
+ runs = [_deep(run, SubDict, list) for run in runs]
+ else:
+ runs = [_deep(run, dict, SubList) for run in runs]
+ report = compare(runs)
+ assert report == expected
+ assert report["config"]["input_fingerprints"] == expected["config"]["input_fingerprints"]
+
+
[email protected]("value,typed", [(("x", "y"), "tuple"), ({"x", "y"}, "set"),
+ (np.float64(0.5), "float64"), (np.int64(7), "int64"),
+ (np.str_("teacher"), "str_")])
+def test_scalars_stay_matched_by_exact_type(value, typed):
+ """The other half of the rule: a tuple is an array to `json.dumps` and a set is not
+ encodable, so exactness there protects the encoding; a scalar subclass would silently
+ substitute a type the writer did not declare (`np.int64` and `np.bool_` do not encode at
+ all). Containers are the only match relaxed."""
+ runs = documents()
+ runs[0]["provenance"]["settings"]["temperature"] = value
+ with pytest.raises(evals.EvalError,
+ match="must be strict JSON: unsupported value of type %s" % typed):
+ compare(runs)
+
+
+def test_a_non_string_key_inside_a_subclass_is_still_refused():
+ runs = documents()
+ runs[0]["provenance"]["settings"] = collections.OrderedDict([(1, 0.5)])
+ with pytest.raises(evals.EvalError, match="must be strict JSON: object keys must be strings"):
+ compare(runs)
+
+
+def test_ordered_spellings_do_not_blind_the_order_check():
+ # `_align` compares the *encoding*, so a criteria mapping spelled as an OrderedDict in both
+ # runs must still fail when the two runs disagree on option order.
+ runs = [_deep(run, collections.OrderedDict, list) for run in documents()]
+ question = runs[1]["cases"][0]["questions"]["q0"]
+ question["criteria"] = collections.OrderedDict(reversed(list(question["criteria"].items())))
+ with pytest.raises(evals.EvalError, match="including option order"):
+ compare(runs)
+
+
_DEFAULT_RESULT = object()