Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(evals): attribute shortlist retrieval and decision errors

xp76d69bbec8a6
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 39843b20..893909a2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -247,6 +247,9 @@ jobs:
- name: Eval harness tests
run: python -m pytest tests/test_evals.py -q
+ - name: Shortlist attribution tests
+ run: python -m pytest tests/test_evals_shortlist.py -q
+
- name: Eval API guard
run: python tests/test_evals_api.py
diff --git a/docs/evals.md b/docs/evals.md
index 0e11aefc..4f55a56a 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -29,6 +29,50 @@ Exit codes: `0` on success, `1` when a threshold or a baseline tolerance fails,
usage error. `run` prints the overall metrics and any requested slices to stdout, and writes
the full report and a Markdown summary when `--json` / `--markdown` are given.
+## Attributing shortlist errors
+
+For a labelled high-cardinality choice set, `laya.evals_shortlist.evaluate_shortlist`
+uses the existing `predict_shortlist` path and the regular evaluation harness. It
+answers two separate questions: did retrieval keep the gold label, and did Laya
+choose it when it was present? This is an opt-in Python API for choice labels;
+ordinary `laya-evals run` reports are unchanged.
+
+```python
+import laya
+from laya.evals import Dataset
+from laya.evals_shortlist import evaluate_shortlist
+from laya.shortlist import embed_fn_from_agent
+
+agent = laya.load()
+dataset_path = "intents.jsonl"
+dataset = Dataset.from_jsonl(dataset_path)
+report = evaluate_shortlist(
+ agent, dataset, embed_fn_from_agent(agent), k=20,
+ checkpoint_id="my-checkpoint@revision", embedder_id="my-encoder@revision",
+ dataset_path=dataset_path,
+)
+print(report.overall)
+print(report.cases[0]["shortlist_status"])
+```
+
+Use the same embedding function and checkpoint as the deployment being measured.
+The two identifiers are supplied by the caller and should name immutable revisions;
+the report cannot infer the weights behind an arbitrary callable. `dataset_path`
+records the file's SHA256 alongside the existing question fingerprint. Each case
+keeps the actual shortlist labels and one of `correct`, `retrieval_miss`, or
+`decision_miss`. `shortlist_recall_at_k` is the fraction of gold labels retained.
+`shortlist_accuracy_on_recalled` is correct decisions divided by retained cases;
+it is omitted when none were retained. The existing `choice_accuracy` remains
+end-to-end accuracy over all cases, including retrieval misses. The shortlist
+metrics appear in the same language, model, question and tag slices. Request
+latency includes embedding and the decision call; the report does not isolate
+stage timings. With `k >= n`, the original question passes through and retrieval
+recall is 1 without calling the embedder.
+
+This does not reproduce the BANKING77 results in [issue #102](https://github.com/NandhaKishorM/laya/issues/102):
+those numbers depend on its dataset, checkpoint and bi-encoder. This API makes the
+same kind of diagnosis repeatable on a caller's own labelled set.
+
## Evaluating an ONNX export
`run --onnx PATH` scores an exported ONNX model through `ONNXAgent` instead of the torch
diff --git a/laya/evals.py b/laya/evals.py
index 8b780f62..73bd217d 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -654,7 +654,7 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
raise EvalError("runner returned no answer for question %r" % qid)
errors.append({"index": index + offset, "question": qid, "error": "missing answer"})
continue
- cases.append({
+ case = {
"qid": qid,
"language": example.language,
"model": example.model or _answered_model(result),
@@ -665,7 +665,13 @@ def evaluate(runner: Any, dataset: Dataset, evaluators: Optional[Sequence[Evalua
"correct": _correct(answer, expected),
"scores": {evaluator.name: evaluator.score(answer, expected)
for evaluator in evaluators},
- })
+ }
+ # Shortlist metadata describes the retrieval stage, not the answer. Keep it
+ # beside the case so opt-in evaluations can attribute the source of an error.
+ shortlist = (result or {}).get("shortlist")
+ if isinstance(shortlist, dict) and qid in shortlist:
+ case["shortlist"] = shortlist[qid]
+ cases.append(case)
index += len(chunk)
report = EvalReport(config=dict(config or {}), cases=cases)
diff --git a/laya/evals_shortlist.py b/laya/evals_shortlist.py
new file mode 100644
index 00000000..2c092d11
--- /dev/null
+++ b/laya/evals_shortlist.py
@@ -0,0 +1,151 @@
+"""Opt-in attribution of shortlist retrieval and decision errors.
+
+The runner calls the existing ``predict_shortlist`` once per row. The regular evaluation
+loop still owns answer scoring, timing, cases and slices; this module only adds metrics
+that require the shortlist metadata returned alongside an answer.
+"""
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, Optional, Sequence
+
+from .evals import Dataset, EvalError, EvalReport, _group_cases, evaluate, file_fingerprint
+from .shortlist import DEFAULT_SHORTLIST_K, predict_shortlist
+
+SHORTLIST_REPORT_SCHEMA = "laya-shortlist-evals-report/1"
+
+
+def _labels(question: Dict[str, Any], where: str) -> list[str]:
+ criteria = question.get("criteria")
+ if isinstance(criteria, dict):
+ labels = list(criteria)
+ elif isinstance(criteria, list):
+ labels = criteria
+ else:
+ raise EvalError("%s: choice criteria must be a dict or list" % where)
+ if not labels or any(not isinstance(label, str) for label in labels):
+ raise EvalError("%s: choice criteria must contain string labels" % where)
+ if len(set(labels)) != len(labels):
+ raise EvalError("%s: choice criteria contain duplicate labels" % where)
+ return labels
+
+
+def _validate_dataset(dataset: Dataset) -> None:
+ if not isinstance(dataset, Dataset) or not dataset.examples:
+ raise EvalError("shortlist evaluation needs a non-empty Dataset")
+ for index, example in enumerate(dataset.examples):
+ if not example.expected:
+ raise EvalError("row %d: expected must contain a choice label" % index)
+ for qid, expected in example.expected.items():
+ where = "row %d question %r" % (index, qid)
+ question = example.questions.get(qid)
+ if not isinstance(question, dict) or question.get("type") != "choice":
+ raise EvalError("%s: shortlist evaluation requires a choice question" % where)
+ labels = _labels(question, where)
+ if not isinstance(expected, str) or expected not in labels:
+ raise EvalError("%s: expected label %r is not in criteria" % (where, expected))
+
+
+class _ShortlistRunner:
+ def __init__(self, agent: Any, embed_fn: Callable[[Sequence[str]], Any], k: int):
+ self.agent, self.embed_fn, self.k = agent, embed_fn, k
+
+ def predict(self, state: Any, questions: Dict[str, Any], model: Optional[str] = None) -> Dict[str, Any]:
+ kwargs = {"model": model} if model is not None else {}
+ result = predict_shortlist(self.agent, state, questions, self.embed_fn, self.k, **kwargs)
+ meta = result.get("shortlist")
+ if not isinstance(meta, dict):
+ raise EvalError("predict_shortlist returned no shortlist metadata")
+ answers = result.get("answers")
+ for qid, question in questions.items():
+ if not isinstance(question, dict) or question.get("type") != "choice":
+ continue
+ where = "question %r" % qid
+ original = _labels(question, where)
+ item = meta.get(qid)
+ if not isinstance(item, dict):
+ raise EvalError("%s: missing shortlist metadata" % where)
+ kept = item.get("labels")
+ if (not isinstance(kept, list) or len(kept) != min(self.k, len(original))
+ or any(not isinstance(label, str) or label not in original for label in kept)
+ or len(set(kept)) != len(kept)
+ or item.get("n") != len(original) or item.get("k") != self.k
+ or item.get("passthrough") is not (self.k >= len(original))):
+ raise EvalError("%s: inconsistent shortlist metadata" % where)
+ scores = item.get("scores")
+ if ((self.k >= len(original) and scores is not None)
+ or (self.k < len(original) and
+ (not isinstance(scores, list) or len(scores) != len(kept)))):
+ raise EvalError("%s: inconsistent shortlist scores" % where)
+ answer = answers.get(qid) if isinstance(answers, dict) else None
+ if isinstance(answer, dict) and (answer.get("type") != "choice" or answer.get("choice") not in kept):
+ raise EvalError("%s: choice answer is outside the shortlist" % where)
+ return result
+
+
+def _metrics(cases: Sequence[Dict[str, Any]]) -> Dict[str, float]:
+ if not cases:
+ return {}
+ recalled = sum(case["shortlist_status"] != "retrieval_miss" for case in cases)
+ correct = sum(case["shortlist_status"] == "correct" for case in cases)
+ metrics = {"shortlist_recall_at_k": recalled / len(cases)}
+ # There is no decision-stage denominator when retrieval recalls nothing.
+ if recalled:
+ metrics["shortlist_accuracy_on_recalled"] = correct / recalled
+ return metrics
+
+
+def evaluate_shortlist(
+ agent: Any,
+ dataset: Dataset,
+ embed_fn: Callable[[Sequence[str]], Any],
+ *,
+ k: int = DEFAULT_SHORTLIST_K,
+ checkpoint_id: str,
+ embedder_id: str,
+ dataset_path: Optional[str] = None,
+) -> EvalReport:
+ """Evaluate labelled choice questions with retrieval/decision error attribution.
+
+ ``checkpoint_id`` and ``embedder_id`` are caller-supplied experiment identifiers,
+ ideally immutable revisions. ``dataset_path`` adds the SHA256 of the JSONL bytes when
+ the caller loaded ``dataset`` from that file. The same ``embed_fn`` used at inference
+ is used here. Timing includes both embedding and the decision call.
+
+ ``shortlist_recall_at_k`` counts a gold label kept by retrieval. Conditional
+ ``shortlist_accuracy_on_recalled`` is absent when recall is zero. The existing
+ ``choice_accuracy`` remains the end-to-end accuracy over every scored case.
+ """
+ if isinstance(k, bool) or not isinstance(k, int) or k < 1:
+ raise EvalError("k must be a positive integer, got %r" % k)
+ if not callable(embed_fn):
+ raise EvalError("embed_fn must be callable")
+ for name, value in (("checkpoint_id", checkpoint_id), ("embedder_id", embedder_id)):
+ if not isinstance(value, str) or not value.strip():
+ raise EvalError("%s must be a non-empty string" % name)
+ _validate_dataset(dataset)
+ config: Dict[str, Any] = {"shortlist": {"k": k, "checkpoint_id": checkpoint_id,
+ "embedder_id": embedder_id}}
+ if dataset_path is not None:
+ config["dataset"] = dataset_path
+ config["dataset_sha256"] = file_fingerprint(dataset_path)
+ report = evaluate(_ShortlistRunner(agent, embed_fn, k), dataset, config=config)
+ for case in report.cases:
+ item = case.get("shortlist")
+ if not isinstance(item, dict) or not isinstance(item.get("labels"), list):
+ raise EvalError("question %r: missing shortlist metadata" % case["qid"])
+ if case["expected"] not in item["labels"]:
+ case["shortlist_status"] = "retrieval_miss"
+ elif case["correct"]:
+ case["shortlist_status"] = "correct"
+ else:
+ case["shortlist_status"] = "decision_miss"
+ report.overall.update(_metrics(report.cases))
+ for dimension in ("language", "model", "qid", "tag"):
+ for value, group in _group_cases(report.cases, dimension).items():
+ report.slices[dimension][value].update(_metrics(group))
+ report.config["schema"] = SHORTLIST_REPORT_SCHEMA
+ report.config["shortlist"]["counts"] = {
+ status: sum(case["shortlist_status"] == status for case in report.cases)
+ for status in ("correct", "retrieval_miss", "decision_miss")
+ }
+ return report
diff --git a/tests/test_evals_shortlist.py b/tests/test_evals_shortlist.py
new file mode 100644
index 00000000..c2077aa0
--- /dev/null
+++ b/tests/test_evals_shortlist.py
@@ -0,0 +1,137 @@
+"""Weight-free checks for two-stage shortlist evaluation."""
+import json
+
+import pytest
+
+from laya.evals import Dataset, EvalError, Example, evaluate
+from laya.evals_shortlist import SHORTLIST_REPORT_SCHEMA, evaluate_shortlist
+
+
+QUESTION = {"intent": {"type": "choice", "instructions": "Which?",
+ "criteria": ["a", "b", "c"]}}
+VECTORS = {
+ "Which?\none": [1.0, 0.0],
+ "Which?\ntwo": [1.0, 0.0],
+ "Which?\nthree": [-1.0, 0.0],
+ "a": [1.0, 0.0],
+ "b": [0.0, 1.0],
+ "c": [-1.0, 0.0],
+}
+
+
+class TableEmbed:
+ def __init__(self):
+ self.calls = []
+
+ def __call__(self, texts):
+ self.calls.append(list(texts))
+ return [VECTORS[text] for text in texts]
+
+
+class ChoiceAgent:
+ def __init__(self, chosen):
+ self.chosen = chosen
+ self.received = []
+
+ def system_one(self, state, questions):
+ labels = list(questions["intent"]["criteria"])
+ self.received.append(labels)
+ return {"model": "stub", "answers": {"intent": {
+ "type": "choice", "choice": self.chosen[state],
+ "answer_confidence": 0.7,
+ }}}
+
+
+def sample(state, gold, tag):
+ return Example(state, QUESTION, {"intent": gold}, tags=(tag,))
+
+
+def run(agent, dataset, embed, k=2):
+ return evaluate_shortlist(agent, dataset, embed, k=k,
+ checkpoint_id="checkpoint@123", embedder_id="embedder@456")
+
+
+def test_stage_attribution_and_slices():
+ agent = ChoiceAgent({"one": "a", "two": "a", "three": "c"})
+ dataset = Dataset([sample("one", "a", "ok"), sample("two", "b", "bad"),
+ sample("three", "a", "bad")])
+ report = run(agent, dataset, TableEmbed())
+ assert [case["shortlist_status"] for case in report.cases] == [
+ "correct", "decision_miss", "retrieval_miss"]
+ assert report.overall["shortlist_recall_at_k"] == pytest.approx(2 / 3)
+ assert report.overall["shortlist_accuracy_on_recalled"] == pytest.approx(1 / 2)
+ assert report.overall["choice_accuracy"] == pytest.approx(1 / 3)
+ assert report.slices["tag"]["bad"]["shortlist_recall_at_k"] == 0.5
+ assert report.slices["tag"]["bad"]["shortlist_accuracy_on_recalled"] == 0.0
+ assert report.config["shortlist"]["counts"] == {
+ "correct": 1, "retrieval_miss": 1, "decision_miss": 1}
+ assert report.config["schema"] == SHORTLIST_REPORT_SCHEMA
+ assert report.config["shortlist"]["k"] == 2
+ assert json.loads(json.dumps(report.to_json()))["cases"][2]["shortlist_status"] == "retrieval_miss"
+ assert agent.received == [["a", "b"], ["a", "b"], ["c", "b"]]
+
+
+def test_zero_recall_has_no_conditional_accuracy():
+ report = run(ChoiceAgent({"three": "c"}), Dataset([sample("three", "a", "bad")]),
+ TableEmbed(), k=1)
+ assert report.overall["shortlist_recall_at_k"] == 0.0
+ assert "shortlist_accuracy_on_recalled" not in report.overall
+ assert "shortlist_accuracy_on_recalled" not in report.slices["tag"]["bad"]
+ assert report.overall["choice_accuracy"] == 0.0
+
+
+def test_passthrough_skips_embedding():
+ def no_embedding(_texts):
+ raise AssertionError("k >= n should not embed")
+
+ report = run(ChoiceAgent({"one": "a"}), Dataset([sample("one", "a", "ok")]),
+ no_embedding, k=3)
+ assert report.overall["shortlist_recall_at_k"] == 1.0
+ assert report.cases[0]["shortlist"]["passthrough"] is True
+ assert report.cases[0]["shortlist"]["scores"] is None
+
+
+def test_invalid_gold_and_out_of_shortlist_answer_fail():
+ with pytest.raises(EvalError, match="not in criteria"):
+ run(ChoiceAgent({}), Dataset([sample("one", "missing", "bad")]), TableEmbed())
+ with pytest.raises(EvalError, match="outside the shortlist"):
+ run(ChoiceAgent({"one": "c"}), Dataset([sample("one", "a", "bad")]), TableEmbed())
+ with pytest.raises(EvalError, match="positive integer"):
+ run(ChoiceAgent({}), Dataset([sample("one", "a", "ok")]), TableEmbed(), k=True)
+
+
+def test_malformed_shortlist_metadata_fails(monkeypatch):
+ def malformed(*_args, **_kwargs):
+ return {"answers": {"intent": {"type": "choice", "choice": "a"}},
+ "shortlist": {"intent": {"labels": ["a", "a"], "k": 2, "n": 3,
+ "passthrough": False, "scores": [1.0, 1.0]}}}
+
+ monkeypatch.setattr("laya.evals_shortlist.predict_shortlist", malformed)
+ with pytest.raises(EvalError, match="inconsistent shortlist metadata"):
+ run(ChoiceAgent({"one": "a"}), Dataset([sample("one", "a", "ok")]), TableEmbed())
+
+
+def test_dataset_bytes_are_recorded_when_path_is_supplied(tmp_path):
+ path = tmp_path / "intents.jsonl"
+ path.write_text(json.dumps({"state": "one", "questions": QUESTION,
+ "expected": {"intent": "a"}}) + "\n", encoding="utf-8")
+ dataset = Dataset.from_jsonl(str(path))
+ report = evaluate_shortlist(ChoiceAgent({"one": "a"}), dataset, TableEmbed(), k=2,
+ checkpoint_id="checkpoint@123", embedder_id="embedder@456",
+ dataset_path=str(path))
+ from laya.evals import file_fingerprint
+
+ assert report.config["dataset_sha256"] == file_fingerprint(str(path))
+ assert report.config["questions_sha256"]
+
+
+def test_plain_evaluate_keeps_its_schema_and_metrics():
+ class Runner:
+ def predict(self, state, questions, model=None):
+ return {"answers": {"intent": {"type": "choice", "choice": "a"}}}
+
+ report = evaluate(Runner(), Dataset([sample("one", "a", "ok")]))
+ assert report.config["schema"] == "laya-evals-report/1"
+ assert report.overall["choice_accuracy"] == 1.0
+ assert "shortlist_recall_at_k" not in report.overall
+ assert "shortlist" not in report.cases[0]
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index ccd59572..8da74ba3 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -839,6 +839,15 @@ check("serve/BODY_REFUSALS names nothing but hooks",
[key for key in _http_refusals if "hook" not in key and "predict" not in key], [])
check("serve forwards and refuses disjoint sets", sorted(set(_http_controls) & set(_http_refusals)), [])
+# The opt-in shortlist evaluator is a public Python entry point. Pin its required
+# provenance arguments without adding an eager import to the package root.
+from laya.evals_shortlist import evaluate_shortlist # noqa: E402
+
+check_param("evaluate_shortlist", evaluate_shortlist, "k", 20)
+check_param("evaluate_shortlist", evaluate_shortlist, "dataset_path", None)
+for param in ("checkpoint_id", "embedder_id"):
+ check_param("evaluate_shortlist", evaluate_shortlist, param, inspect.Parameter.empty)
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL: