Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

test(evals): pin which field the calibration metrics read, at the server's own projector

Aashishbdf3b89bd6ed
diff --git a/tests/test_evals.py b/tests/test_evals.py
index 59cbaea2..31cc09ef 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -86,6 +86,127 @@ def test_calibration_uses_answer_confidence():
assert report.overall["mean_confidence"] == pytest.approx(0.9)
+# ---------------------------------------------------------------- which confidence a metric reads
+# `_answer_confidence` prefers `answer_confidence`, and an answer that carries none gets the entropy
+# `confidence` -- a different quantity on a different scale (#394), which the docstring used to call
+# "the calibrated confidence Laya reports ... on every answer type" while docs/evals.md told the
+# operator the same. The shape that really arrives without the field is the strict Jev wire contract:
+# `LAYA_JEV_STRICT` drops it (`laya/serve.py::_project_jev_strict`, documented in docs/http-api.md).
+# So the stripped answers below are made by the server's own projector rather than hand-written -- a
+# hand copy could drift from what the server serves and this gate would still pass. `laya.serve`
+# imports with neither fastapi nor torch at module scope, which is why it is affordable here.
+
+_CONTRACT_PROBS = {"choice": [0.95, 0.03, 0.02], "score": [0.05, 0.90, 0.05], "noul": [0.13, 0.87]}
+_CONTRACT_KEYS = {"choice": ["billing", "sales", "other"], "score": ["low", "mid", "high"],
+ "noul": ["no", "yes"]}
+_CONTRACT_LABEL = {"choice": "billing", "score": "mid", "noul": True}
+
+
+def _agent_answer(kind):
+ """The answer dict `laya/agent.py` publishes, built the way the decoder builds it: entropy
+ `confidence` for `choice` and `score`, `max(p_true, 1 - p_true)` for `noul`, and
+ `answer_confidence` = max(p) -- see laya/agent.py:1373, :1392, :1400."""
+ p = _CONTRACT_PROBS[kind]
+ argmax = max(range(len(p)), key=lambda i: p[i])
+ from laya.common import confidence_from_probs
+ conf = (round(max(p[1], 1.0 - p[1]), 4) if kind == "noul"
+ else round(confidence_from_probs(np.asarray(p), len(p)), 4))
+ answer = {"confidence": conf, "answer_confidence": round(float(p[argmax]), 4),
+ "type": kind}
+ if kind == "choice":
+ return dict(answer, choice=_CONTRACT_KEYS[kind][argmax],
+ probabilities=dict(zip(_CONTRACT_KEYS[kind], p)))
+ if kind == "score":
+ return dict(answer, score=round(sum(i * v for i, v in enumerate(p)), 4),
+ legend={str(i): k for i, k in enumerate(_CONTRACT_KEYS[kind])},
+ probabilities={str(i): v for i, v in enumerate(p)})
+ return dict(answer, noul=p[1])
+
+
+def _contract_shapes():
+ """`(full, strict)` -- the payload the agents publish, and the same payload put through the
+ projector that serves the strict Jev contract."""
+ from laya.serve import _project_jev_strict
+ full = {kind: _agent_answer(kind) for kind in _CONTRACT_PROBS}
+ strict = _project_jev_strict({"model": "stub", "answers": dict(full),
+ "usage": {"prompt_tokens": 1, "completion_tokens": 1}})["answers"]
+ return full, strict
+
+
+def _contract_report(shapes):
+ questions = {"q": {"type": "choice", "instructions": "?",
+ "criteria": dict(zip(_CONTRACT_KEYS["choice"], ["c"] * 3))}}
+ by_state = {"state-%s" % kind: {"q": shapes[kind]} for kind in shapes}
+ dataset = Dataset([Example("state-%s" % kind, questions, {"q": _CONTRACT_LABEL[kind]})
+ for kind in shapes])
+ return evaluate(StubRunner(by_state), dataset)
+
+
+def test_serve_contract_projection_is_the_shape_without_the_field():
+ full, strict = _contract_shapes()
+ for kind in ("choice", "score", "noul"):
+ assert "answer_confidence" in full[kind]
+ assert "answer_confidence" not in strict[kind], (
+ "the projector is what makes the fallback reachable; if it stopped dropping the field, "
+ "the prose this test guards would need re-reading, not these numbers")
+ assert "confidence" in strict["choice"] and "confidence" in strict["score"]
+ assert "confidence" not in strict["noul"], "docs/http-api.md: noul keeps only `noul`"
+
+
+def test_a_strict_answer_is_calibrated_against_the_entropy_number():
+ full, strict = _contract_shapes()
+ for kind in ("choice", "score"):
+ assert evals._answer_confidence(full[kind]) == pytest.approx(full[kind]["answer_confidence"])
+ assert evals._answer_confidence(strict[kind]) == pytest.approx(full[kind]["confidence"]), \
+ "with the field gone the column silently becomes the entropy number"
+ # A strict `noul` carries neither field, so it is the only fallback that lands on the right
+ # quantity: max(p_true, 1 - p_true) is what the decoder reports as its answer_confidence.
+ assert evals._answer_confidence(strict["noul"]) == pytest.approx(
+ full["noul"]["answer_confidence"])
+
+
+def test_the_two_shapes_do_not_score_the_same_report():
+ full, strict = _contract_shapes()
+ over_full, over_strict = _contract_report(full), _contract_report(strict)
+ assert over_full.overall["mean_confidence"] == pytest.approx(0.9066666666666666)
+ assert over_strict.overall["mean_confidence"] == pytest.approx(0.7665666666666667)
+ # ece sees the two rows whose answer type yields a known label (choice, noul); the score row has
+ # no correctness rule, so only mean_confidence moves for it.
+ assert over_full.overall["ece"] == pytest.approx(0.09)
+ assert over_strict.overall["ece"] == pytest.approx(0.17065)
+ assert over_strict.overall["ece"] - over_full.overall["ece"] == pytest.approx(0.08065), \
+ "the same answers, the same labels, a different field: 0.09 -> 0.17065"
+
+
+def test_the_docs_name_the_fallback_and_refuse_to_call_the_shipped_number_calibrated():
+ import inspect
+ import pathlib
+
+ doc = inspect.getsource(evals._answer_confidence)
+ flat = " ".join(doc.split())
+ assert "falls through" in flat and "normalized entropy" in flat
+ assert "LAYA_JEV_STRICT" in flat, "the shape that lacks the field has to be named"
+ assert "over-confident" in flat, "the shipped level must not be called calibrated"
+ assert "is not *calibrated* as shipped" in flat
+ assert "The calibrated confidence" not in doc, "main's opening claim, verbatim"
+
+ page = (pathlib.Path(__file__).resolve().parent.parent
+ / "docs" / "evals.md").read_text(encoding="utf-8")
+ prose = " ".join(page.split())
+ assert "the calibrated probability Laya reports on every answer type" not in prose
+ assert "`laya.evals._answer_confidence`" in prose, "the page names the function it describes"
+ assert "then\n`max(p, 1 - p)`" in page or "then `max(p, 1 - p)`" in prose
+ assert "LAYA_JEV_STRICT" in prose and "over-confident" in prose
+ assert "## Which confidence a metric reads" in page, "the fallback needs a home, not a footnote"
+
+ # And the claim the evals page leans on has to still be the one the HTTP page makes.
+ api = (pathlib.Path(__file__).resolve().parent.parent
+ / "docs" / "http-api.md").read_text(encoding="utf-8")
+ row = [line for line in api.splitlines() if "`LAYA_JEV_STRICT`" in line]
+ assert row and "no per-answer `action` / `answer_confidence`" in row[0]
+
+
+
def test_compare_ignores_latency_by_default():
report = EvalReport(overall={"choice_accuracy": 0.8, "latency_p50_ms": 12.0})
baseline = {"overall": {"choice_accuracy": 0.8, "latency_p50_ms": 5.0}}