Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(evals): attribute shortlist retrieval and decision errors
xp76d69bbec8a6
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 39843b20..893909a2 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -247,6 +247,9 @@ jobs:- name: Eval harness testsrun: python -m pytest tests/test_evals.py -q+ - name: Shortlist attribution tests+ run: python -m pytest tests/test_evals_shortlist.py -q+- name: Eval API guardrun: python tests/test_evals_api.pydiff --git a/docs/evals.md b/docs/evals.mdindex 0e11aefc..4f55a56a 100644--- a/docs/evals.md+++ b/docs/evals.md@@ -29,6 +29,50 @@ Exit codes: `0` on success, `1` when a threshold or a baseline tolerance fails,usage error. `run` prints the overall metrics and any requested slices to stdout, and writesthe full report and a Markdown summary when `--json` / `--markdown` are given.+## Attributing shortlist errors++For a labelled high-cardinality choice set, `laya.evals_shortlist.evaluate_shortlist`+uses the existing `predict_shortlist` path and the regular evaluation harness. It+answers two separate questions: did retrieval keep the gold label, and did Laya+choose it when it was present? This is an opt-in Python API for choice labels;+ordinary `laya-evals run` reports are unchanged.++```python+import laya+from laya.evals import Dataset+from laya.evals_shortlist import evaluate_shortlist+from laya.shortlist import embed_fn_from_agent++agent = laya.load()+dataset_path = "intents.jsonl"+dataset = Dataset.from_jsonl(dataset_path)+report = evaluate_shortlist(+ agent, dataset, embed_fn_from_agent(agent), k=20,+ checkpoint_id="my-checkpoint@revision", embedder_id="my-encoder@revision",+ dataset_path=dataset_path,+)+print(report.overall)+print(report.cases[0]["shortlist_status"])+```++Use the same embedding function and checkpoint as the deployment being measured.+The two identifiers are supplied by the caller and should name immutable revisions;+the report cannot infer the weights behind an arbitrary callable. `dataset_path`+records the file's SHA256 alongside the existing question fingerprint. Each case+keeps the actual shortlist labels and one of `correct`, `retrieval_miss`, or+`decision_miss`. `shortlist_recall_at_k` is the fraction of gold labels retained.+`shortlist_accuracy_on_recalled` is correct decisions divided by retained cases;+it is omitted when none were retained. The existing `choice_accuracy` remains+end-to-end accuracy over all cases, including retrieval misses. The shortlist+metrics appear in the same language, model, question and tag slices. Request+latency includes embedding and the decision call; the report does not isolate+stage timings. With `k >= n`, the original question passes through and retrieval+recall is 1 without calling the embedder.++This does not reproduce the BANKING77 results in [issue #102](https://github.com/NandhaKishorM/laya/issues/102):+those numbers depend on its dataset, checkpoint and bi-encoder. This API makes the+same kind of diagnosis repeatable on a caller's own labelled set.+## Evaluating an ONNX export`run --onnx PATH` scores an exported ONNX model through `ONNXAgent` instead of the torchdiff --git a/laya/evals.py b/laya/evals.pyindex 8b780f62..73bd217d 100644--- a/laya/evals.py+++ b/laya/evals.py@@ -654,7 +654,7 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evaluaraise EvalError("runner returned no answer for question %r" % qid)errors.append({"index": index + offset, "question": qid, "error": "missing answer"})continue- cases.append({+ case = {"qid": qid,"language": example.language,"model": example.model or _answered_model(result),@@ -665,7 +665,13 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua"correct": _correct(answer, expected),"scores": {evaluator.name: evaluator.score(answer, expected)for evaluator in evaluators},- })+ }+ # Shortlist metadata describes the retrieval stage, not the answer. Keep it+ # beside the case so opt-in evaluations can attribute the source of an error.+ shortlist = (result or {}).get("shortlist")+ if isinstance(shortlist, dict) and qid in shortlist:+ case["shortlist"] = shortlist[qid]+ cases.append(case)index += len(chunk)report = EvalReport(config=dict(config or {}), cases=cases)diff --git a/laya/evals_shortlist.py b/laya/evals_shortlist.pynew file mode 100644index 00000000..2c092d11--- /dev/null+++ b/laya/evals_shortlist.py@@ -0,0 +1,151 @@+"""Opt-in attribution of shortlist retrieval and decision errors.++The runner calls the existing ``predict_shortlist`` once per row. The regular evaluation+loop still owns answer scoring, timing, cases and slices; this module only adds metrics+that require the shortlist metadata returned alongside an answer.+"""+from __future__ import annotations++from typing import Any, Callable, Dict, Optional, Sequence++from .evals import Dataset, EvalError, EvalReport, _group_cases, evaluate, file_fingerprint+from .shortlist import DEFAULT_SHORTLIST_K, predict_shortlist++SHORTLIST_REPORT_SCHEMA = "laya-shortlist-evals-report/1"+++def _labels(question: Dict[str, Any], where: str) -> list[str]:+ criteria = question.get("criteria")+ if isinstance(criteria, dict):+ labels = list(criteria)+ elif isinstance(criteria, list):+ labels = criteria+ else:+ raise EvalError("%s: choice criteria must be a dict or list" % where)+ if not labels or any(not isinstance(label, str) for label in labels):+ raise EvalError("%s: choice criteria must contain string labels" % where)+ if len(set(labels)) != len(labels):+ raise EvalError("%s: choice criteria contain duplicate labels" % where)+ return labels+++def _validate_dataset(dataset: Dataset) -> None:+ if not isinstance(dataset, Dataset) or not dataset.examples:+ raise EvalError("shortlist evaluation needs a non-empty Dataset")+ for index, example in enumerate(dataset.examples):+ if not example.expected:+ raise EvalError("row %d: expected must contain a choice label" % index)+ for qid, expected in example.expected.items():+ where = "row %d question %r" % (index, qid)+ question = example.questions.get(qid)+ if not isinstance(question, dict) or question.get("type") != "choice":+ raise EvalError("%s: shortlist evaluation requires a choice question" % where)+ labels = _labels(question, where)+ if not isinstance(expected, str) or expected not in labels:+ raise EvalError("%s: expected label %r is not in criteria" % (where, expected))+++class _ShortlistRunner:+ def __init__(self, agent: Any, embed_fn: Callable[[Sequence[str]], Any], k: int):+ self.agent, self.embed_fn, self.k = agent, embed_fn, k++ def predict(self, state: Any, questions: Dict[str, Any], model: Optional[str] = None) -> Dict[str, Any]:+ kwargs = {"model": model} if model is not None else {}+ result = predict_shortlist(self.agent, state, questions, self.embed_fn, self.k, **kwargs)+ meta = result.get("shortlist")+ if not isinstance(meta, dict):+ raise EvalError("predict_shortlist returned no shortlist metadata")+ answers = result.get("answers")+ for qid, question in questions.items():+ if not isinstance(question, dict) or question.get("type") != "choice":+ continue+ where = "question %r" % qid+ original = _labels(question, where)+ item = meta.get(qid)+ if not isinstance(item, dict):+ raise EvalError("%s: missing shortlist metadata" % where)+ kept = item.get("labels")+ if (not isinstance(kept, list) or len(kept) != min(self.k, len(original))+ or any(not isinstance(label, str) or label not in original for label in kept)+ or len(set(kept)) != len(kept)+ or item.get("n") != len(original) or item.get("k") != self.k+ or item.get("passthrough") is not (self.k >= len(original))):+ raise EvalError("%s: inconsistent shortlist metadata" % where)+ scores = item.get("scores")+ if ((self.k >= len(original) and scores is not None)+ or (self.k < len(original) and+ (not isinstance(scores, list) or len(scores) != len(kept)))):+ raise EvalError("%s: inconsistent shortlist scores" % where)+ answer = answers.get(qid) if isinstance(answers, dict) else None+ if isinstance(answer, dict) and (answer.get("type") != "choice" or answer.get("choice") not in kept):+ raise EvalError("%s: choice answer is outside the shortlist" % where)+ return result+++def _metrics(cases: Sequence[Dict[str, Any]]) -> Dict[str, float]:+ if not cases:+ return {}+ recalled = sum(case["shortlist_status"] != "retrieval_miss" for case in cases)+ correct = sum(case["shortlist_status"] == "correct" for case in cases)+ metrics = {"shortlist_recall_at_k": recalled / len(cases)}+ # There is no decision-stage denominator when retrieval recalls nothing.+ if recalled:+ metrics["shortlist_accuracy_on_recalled"] = correct / recalled+ return metrics+++def evaluate_shortlist(+ agent: Any,+ dataset: Dataset,+ embed_fn: Callable[[Sequence[str]], Any],+ *,+ k: int = DEFAULT_SHORTLIST_K,+ checkpoint_id: str,+ embedder_id: str,+ dataset_path: Optional[str] = None,+) -> EvalReport:+ """Evaluate labelled choice questions with retrieval/decision error attribution.++ ``checkpoint_id`` and ``embedder_id`` are caller-supplied experiment identifiers,+ ideally immutable revisions. ``dataset_path`` adds the SHA256 of the JSONL bytes when+ the caller loaded ``dataset`` from that file. The same ``embed_fn`` used at inference+ is used here. Timing includes both embedding and the decision call.++ ``shortlist_recall_at_k`` counts a gold label kept by retrieval. Conditional+ ``shortlist_accuracy_on_recalled`` is absent when recall is zero. The existing+ ``choice_accuracy`` remains the end-to-end accuracy over every scored case.+ """+ if isinstance(k, bool) or not isinstance(k, int) or k < 1:+ raise EvalError("k must be a positive integer, got %r" % k)+ if not callable(embed_fn):+ raise EvalError("embed_fn must be callable")+ for name, value in (("checkpoint_id", checkpoint_id), ("embedder_id", embedder_id)):+ if not isinstance(value, str) or not value.strip():+ raise EvalError("%s must be a non-empty string" % name)+ _validate_dataset(dataset)+ config: Dict[str, Any] = {"shortlist": {"k": k, "checkpoint_id": checkpoint_id,+ "embedder_id": embedder_id}}+ if dataset_path is not None:+ config["dataset"] = dataset_path+ config["dataset_sha256"] = file_fingerprint(dataset_path)+ report = evaluate(_ShortlistRunner(agent, embed_fn, k), dataset, config=config)+ for case in report.cases:+ item = case.get("shortlist")+ if not isinstance(item, dict) or not isinstance(item.get("labels"), list):+ raise EvalError("question %r: missing shortlist metadata" % case["qid"])+ if case["expected"] not in item["labels"]:+ case["shortlist_status"] = "retrieval_miss"+ elif case["correct"]:+ case["shortlist_status"] = "correct"+ else:+ case["shortlist_status"] = "decision_miss"+ report.overall.update(_metrics(report.cases))+ for dimension in ("language", "model", "qid", "tag"):+ for value, group in _group_cases(report.cases, dimension).items():+ report.slices[dimension][value].update(_metrics(group))+ report.config["schema"] = SHORTLIST_REPORT_SCHEMA+ report.config["shortlist"]["counts"] = {+ status: sum(case["shortlist_status"] == status for case in report.cases)+ for status in ("correct", "retrieval_miss", "decision_miss")+ }+ return reportdiff --git a/tests/test_evals_shortlist.py b/tests/test_evals_shortlist.pynew file mode 100644index 00000000..c2077aa0--- /dev/null+++ b/tests/test_evals_shortlist.py@@ -0,0 +1,137 @@+"""Weight-free checks for two-stage shortlist evaluation."""+import json++import pytest++from laya.evals import Dataset, EvalError, Example, evaluate+from laya.evals_shortlist import SHORTLIST_REPORT_SCHEMA, evaluate_shortlist+++QUESTION = {"intent": {"type": "choice", "instructions": "Which?",+ "criteria": ["a", "b", "c"]}}+VECTORS = {+ "Which?\none": [1.0, 0.0],+ "Which?\ntwo": [1.0, 0.0],+ "Which?\nthree": [-1.0, 0.0],+ "a": [1.0, 0.0],+ "b": [0.0, 1.0],+ "c": [-1.0, 0.0],+}+++class TableEmbed:+ def __init__(self):+ self.calls = []++ def __call__(self, texts):+ self.calls.append(list(texts))+ return [VECTORS[text] for text in texts]+++class ChoiceAgent:+ def __init__(self, chosen):+ self.chosen = chosen+ self.received = []++ def system_one(self, state, questions):+ labels = list(questions["intent"]["criteria"])+ self.received.append(labels)+ return {"model": "stub", "answers": {"intent": {+ "type": "choice", "choice": self.chosen[state],+ "answer_confidence": 0.7,+ }}}+++def sample(state, gold, tag):+ return Example(state, QUESTION, {"intent": gold}, tags=(tag,))+++def run(agent, dataset, embed, k=2):+ return evaluate_shortlist(agent, dataset, embed, k=k,+ checkpoint_id="checkpoint@123", embedder_id="embedder@456")+++def test_stage_attribution_and_slices():+ agent = ChoiceAgent({"one": "a", "two": "a", "three": "c"})+ dataset = Dataset([sample("one", "a", "ok"), sample("two", "b", "bad"),+ sample("three", "a", "bad")])+ report = run(agent, dataset, TableEmbed())+ assert [case["shortlist_status"] for case in report.cases] == [+ "correct", "decision_miss", "retrieval_miss"]+ assert report.overall["shortlist_recall_at_k"] == pytest.approx(2 / 3)+ assert report.overall["shortlist_accuracy_on_recalled"] == pytest.approx(1 / 2)+ assert report.overall["choice_accuracy"] == pytest.approx(1 / 3)+ assert report.slices["tag"]["bad"]["shortlist_recall_at_k"] == 0.5+ assert report.slices["tag"]["bad"]["shortlist_accuracy_on_recalled"] == 0.0+ assert report.config["shortlist"]["counts"] == {+ "correct": 1, "retrieval_miss": 1, "decision_miss": 1}+ assert report.config["schema"] == SHORTLIST_REPORT_SCHEMA+ assert report.config["shortlist"]["k"] == 2+ assert json.loads(json.dumps(report.to_json()))["cases"][2]["shortlist_status"] == "retrieval_miss"+ assert agent.received == [["a", "b"], ["a", "b"], ["c", "b"]]+++def test_zero_recall_has_no_conditional_accuracy():+ report = run(ChoiceAgent({"three": "c"}), Dataset([sample("three", "a", "bad")]),+ TableEmbed(), k=1)+ assert report.overall["shortlist_recall_at_k"] == 0.0+ assert "shortlist_accuracy_on_recalled" not in report.overall+ assert "shortlist_accuracy_on_recalled" not in report.slices["tag"]["bad"]+ assert report.overall["choice_accuracy"] == 0.0+++def test_passthrough_skips_embedding():+ def no_embedding(_texts):+ raise AssertionError("k >= n should not embed")++ report = run(ChoiceAgent({"one": "a"}), Dataset([sample("one", "a", "ok")]),+ no_embedding, k=3)+ assert report.overall["shortlist_recall_at_k"] == 1.0+ assert report.cases[0]["shortlist"]["passthrough"] is True+ assert report.cases[0]["shortlist"]["scores"] is None+++def test_invalid_gold_and_out_of_shortlist_answer_fail():+ with pytest.raises(EvalError, match="not in criteria"):+ run(ChoiceAgent({}), Dataset([sample("one", "missing", "bad")]), TableEmbed())+ with pytest.raises(EvalError, match="outside the shortlist"):+ run(ChoiceAgent({"one": "c"}), Dataset([sample("one", "a", "bad")]), TableEmbed())+ with pytest.raises(EvalError, match="positive integer"):+ run(ChoiceAgent({}), Dataset([sample("one", "a", "ok")]), TableEmbed(), k=True)+++def test_malformed_shortlist_metadata_fails(monkeypatch):+ def malformed(*_args, **_kwargs):+ return {"answers": {"intent": {"type": "choice", "choice": "a"}},+ "shortlist": {"intent": {"labels": ["a", "a"], "k": 2, "n": 3,+ "passthrough": False, "scores": [1.0, 1.0]}}}++ monkeypatch.setattr("laya.evals_shortlist.predict_shortlist", malformed)+ with pytest.raises(EvalError, match="inconsistent shortlist metadata"):+ run(ChoiceAgent({"one": "a"}), Dataset([sample("one", "a", "ok")]), TableEmbed())+++def test_dataset_bytes_are_recorded_when_path_is_supplied(tmp_path):+ path = tmp_path / "intents.jsonl"+ path.write_text(json.dumps({"state": "one", "questions": QUESTION,+ "expected": {"intent": "a"}}) + "\n", encoding="utf-8")+ dataset = Dataset.from_jsonl(str(path))+ report = evaluate_shortlist(ChoiceAgent({"one": "a"}), dataset, TableEmbed(), k=2,+ checkpoint_id="checkpoint@123", embedder_id="embedder@456",+ dataset_path=str(path))+ from laya.evals import file_fingerprint++ assert report.config["dataset_sha256"] == file_fingerprint(str(path))+ assert report.config["questions_sha256"]+++def test_plain_evaluate_keeps_its_schema_and_metrics():+ class Runner:+ def predict(self, state, questions, model=None):+ return {"answers": {"intent": {"type": "choice", "choice": "a"}}}++ report = evaluate(Runner(), Dataset([sample("one", "a", "ok")]))+ assert report.config["schema"] == "laya-evals-report/1"+ assert report.overall["choice_accuracy"] == 1.0+ assert "shortlist_recall_at_k" not in report.overall+ assert "shortlist" not in report.cases[0]diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.pyindex ccd59572..8da74ba3 100644--- a/tests/test_hooks_api.py+++ b/tests/test_hooks_api.py@@ -839,6 +839,15 @@ check("serve/BODY_REFUSALS names nothing but hooks",[key for key in _http_refusals if "hook" not in key and "predict" not in key], [])check("serve forwards and refuses disjoint sets", sorted(set(_http_controls) & set(_http_refusals)), [])+# The opt-in shortlist evaluator is a public Python entry point. Pin its required+# provenance arguments without adding an eager import to the package root.+from laya.evals_shortlist import evaluate_shortlist # noqa: E402++check_param("evaluate_shortlist", evaluate_shortlist, "k", 20)+check_param("evaluate_shortlist", evaluate_shortlist, "dataset_path", None)+for param in ("checkpoint_id", "embedder_id"):+ check_param("evaluate_shortlist", evaluate_shortlist, param, inspect.Parameter.empty)+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL: