OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat(eval): score the rank of the answer chunk (#2478) (#2572)
Nathan Bakare9beff91e4c14
diff --git a/scripts/eval/retrieval/README.md b/scripts/eval/retrieval/README.mdindex 15a8aebf0..20bd4b40c 100644--- a/scripts/eval/retrieval/README.md+++ b/scripts/eval/retrieval/README.md@@ -276,6 +276,29 @@ All three run from 0 to 1, and higher is better. Runs are compared on recall@5.If either run did not evaluate k=5, the comparison uses a cutoff both runs haveand says so. Two runs with no cutoff in common cannot be compared.+### Passage scores++A file can rank first while the chunk that answers the question sits much+lower. For every question with an `expected_text`, the harness also records+the rank of the first returned chunk that comes from an expected file and+contains that text. It counts every chunk this run uploaded, not just the first+chunk of each file.++| Metric | Meaning |+|---|---|+| passage hit@k | 1 when the answer chunk is in the top k chunks, otherwise 0 |+| Passage MRR | 1 divided by the rank of the answer chunk, averaged over questions; 0 when no chunk holds the text |++Chunks are matched on their text, not on chunk IDs, so passage scores can still+be compared when the chunk size changes. Questions without an `expected_text`+have no passage scores and are left out of these averages. The summary says how+many questions the passage scores cover.++Passage scores are reported for information. Questions are still classified as+regressed or improved on the file scores above. A baseline written before+passage scores existed still compares; the comparison notes that only file+scores were compared.+## Results filesEach run writes four files to the output directory: `results.json` and@@ -295,7 +318,9 @@ Each run writes four files to the output directory: `results.json` andEach record in `queries` holds the question, its labels, the ranked files withthe score of their first chunk, the rank of each expected file(`expected_ranks`, `null` when a file was not returned), whether the expected-text was found, and the metrics for that question.+text was found, the rank of the answer chunk (`passage_rank`, only for questions+with an `expected_text`, `null` when no chunk held it), and the metrics for that+question.## Add or change a questiondiff --git a/scripts/eval/retrieval/metrics.py b/scripts/eval/retrieval/metrics.pyindex e6b6cd23d..55103c1cb 100644--- a/scripts/eval/retrieval/metrics.py+++ b/scripts/eval/retrieval/metrics.py@@ -3,7 +3,8 @@``/v1/search`` returns chunks, but questions are labeled by file because chunkboundaries move whenever chunking changes. This module turns a ranked chunklist into a ranked list of unique files and scores it with recall@k,-reciprocal rank and nDCG@k.+reciprocal rank and nDCG@k. For questions with an ``expected_text`` it also+scores the rank of the chunk that holds the answer.Standard library only, so it can be unit-tested without the SDK or a stack."""@@ -139,11 +140,55 @@ def score_query(return {**recall, "reciprocal_rank": reciprocal_rank, **ndcg}+def passage_rank(+ chunks: Iterable[Chunk],+ run_prefix: str,+ corpus_names: Collection[str],+ expected_files: Collection[str],+ expected_text: str,+) -> int | None:+ """Rank of the first chunk that holds the answer, or None if no chunk does.++ The answer chunk is a chunk of an expected file that contains+ ``expected_text`` (case- and whitespace-insensitive). Ranks count only the+ chunks of files this run uploaded, so stray documents never shift them.+ Matching on text rather than chunk IDs keeps the rank meaningful when the+ chunk size changes.+ """+ needle = _normalize(expected_text)+ expected = set(expected_files)+ position = 0+ for chunk in chunks:+ name = canonical_name(chunk.filename, run_prefix, corpus_names)+ if name is None:+ continue+ position += 1+ if name in expected and needle in _normalize(chunk.text):+ return position+ return None+++def score_passage(rank: int | None, k_values: Sequence[int] = DEFAULT_K_VALUES) -> dict[str, float]:+ """Score one question on the rank of its answer chunk.++ - ``passage_hit@k``: 1 if the answer chunk is in the top k chunks, else 0.+ - ``passage_rr``: 1 / rank of the answer chunk, 0 if none was returned.+ """+ hits = {f"passage_hit@{k}": 1.0 if rank is not None and rank <= k else 0.0 for k in k_values}+ return {**hits, "passage_rr": 1.0 / rank if rank is not None else 0.0}+++_MEAN_NAMES = {"reciprocal_rank": "mrr", "passage_rr": "passage_mrr"}++def _mean_scores(rows: Sequence[Mapping[str, float]]) -> dict[str, float]:+ # Passage scores exist only for questions with an expected_text, so each+ # metric is averaged over the questions that have it.+ keys = list(dict.fromkeys(key for row in rows for key in row))means: dict[str, float] = {}- for key in rows[0]:- name = "mrr" if key == "reciprocal_rank" else key- means[name] = sum(row[key] for row in rows) / len(rows)+ for key in keys:+ values = [row[key] for row in rows if key in row]+ means[_MEAN_NAMES.get(key, key)] = sum(values) / len(values)return means@@ -167,7 +212,13 @@ def aggregate(by_tag.setdefault(tag, []).append(scores)return {"overall": _mean_scores(per_query),- "by_tag": {- tag: {"count": len(rows), **_mean_scores(rows)} for tag, rows in sorted(by_tag.items())- },+ "by_tag": {tag: _tag_scores(rows) for tag, rows in sorted(by_tag.items())},}+++def _tag_scores(rows: Sequence[Mapping[str, float]]) -> dict[str, float]:+ scores: dict[str, float] = {"count": len(rows)}+ passage_count = sum(1 for row in rows if "passage_rr" in row)+ if passage_count:+ scores["passage_count"] = passage_count+ return {**scores, **_mean_scores(rows)}diff --git a/scripts/eval/retrieval/report.py b/scripts/eval/retrieval/report.pyindex 952a703c9..371d0f6e5 100644--- a/scripts/eval/retrieval/report.py+++ b/scripts/eval/retrieval/report.py@@ -205,6 +205,7 @@ def diff_results("baseline": _snippet_missing(base),"current": _snippet_missing(cur),},+ "passage": {"baseline": _passage(base), "current": _passage(cur)},}(improved if direction > 0 else regressed).append(entry)@@ -229,6 +230,14 @@ def diff_results(f"The baseline has no recall@{current['run']['primary_k']}, "f"so questions are compared on recall@{k}.")+ # Notes are informational. Unlike warnings, they do not make the file scores+ # incomparable, so the recall gate and baseline promotion ignore them.+ notes: list[str] = []+ if "passage_mrr" in cur_overall and "passage_mrr" not in base_overall:+ notes.append(+ "The baseline has no passage scores, so only file scores are compared. "+ "Rerun the baseline to compare passages."+ )if added or removed:warnings.append("The runs do not cover the same questions, so the aggregate deltas "@@ -240,6 +249,7 @@ def diff_results("current_run_id": current["run"].get("run_id"),"compare_metrics": metrics,"warnings": warnings,+ "notes": notes,"aggregate": aggregate,"regressed": regressed,"improved": improved,@@ -314,7 +324,17 @@ def _cell(text: object) -> str:def _label(metric: str) -> str:if metric in ("mrr", "reciprocal_rank"):return "MRR" if metric == "mrr" else "Reciprocal rank"- return metric.replace("ndcg", "nDCG")+ if metric in ("passage_mrr", "passage_rr"):+ return "Passage MRR" if metric == "passage_mrr" else "Passage reciprocal rank"+ return metric.replace("ndcg", "nDCG").replace("passage_hit", "passage hit")+++def _passage(query: Mapping[str, Any]) -> str:+ """Where the answer chunk ranked, for a question with an expected_text."""+ if "passage_rank" not in query:+ return "n/a"+ rank = query["passage_rank"]+ return "not returned" if rank is None else f"#{rank}"def _ranks(ranks: Mapping[str, int | None], snippet_missing: Sequence[str] = ()) -> str:@@ -330,8 +350,10 @@ def _ranks(ranks: Mapping[str, int | None], snippet_missing: Sequence[str] = ())def _changed_table(entries: Sequence[Mapping[str, Any]], metrics: Sequence[str]) -> list[str]:lines = [- "| ID | Query | " + " | ".join(_label(m) for m in metrics) + " | Expected file ranks |",- "|---|---|" + "---|" * len(metrics) + "---|",+ "| ID | Query | "+ + " | ".join(_label(m) for m in metrics)+ + " | Expected file ranks | Answer chunk rank |",+ "|---|---|" + "---|" * len(metrics) + "---|---|",]for entry in entries:cells = []@@ -344,10 +366,12 @@ def _changed_table(entries: Sequence[Mapping[str, Any]], metrics: Sequence[str])missing = entry.get("snippet_missing", {})before = _ranks(ranks["baseline"], missing.get("baseline", ()))after = _ranks(ranks["current"], missing.get("current", ()))+ passage = entry.get("passage", {})lines.append(f"| {_cell(entry['id'])} | {_cell(entry['query'])} | "+ " | ".join(cells)- + f" | {_cell(before)} → {_cell(after)} |"+ + f" | {_cell(before)} → {_cell(after)} "+ + f"| {passage.get('baseline', 'n/a')} → {passage.get('current', 'n/a')} |")return lines@@ -367,7 +391,9 @@ def render_diff(diff: Mapping[str, Any]) -> str:]for warning in diff["warnings"]:lines.append(f"> **Warning:** {warning}")- if diff["warnings"]:+ for note in diff.get("notes", ()):+ lines.append(f"> **Note:** {note}")+ if diff["warnings"] or diff.get("notes"):lines.append("")lines += [f"### Regressed ({len(regressed)})", ""]@@ -434,8 +460,17 @@ def render_summary(results: Mapping[str, Any], diff: Mapping[str, Any] | None ="|---|---|",]lines += [f"| {_label(name)} | {value:.3f} |" for name, value in overall.items()]+ if "passage_mrr" in overall:+ lines += [+ "",+ f"Passage scores cover the {run.get('passage_questions', 'n/a')} of "+ f"{run['query_count']} questions that have an expected_text. They rank the "+ "first chunk holding that text among all chunks returned.",+ ]if by_tag:+ if f"passage_hit@{k}" in overall:+ headline += [f"passage_hit@{k}", "passage_mrr"]lines += ["","## By tag",@@ -446,7 +481,8 @@ def render_summary(results: Mapping[str, Any], diff: Mapping[str, Any] | None =for tag, scores in by_tag.items():lines.append(f"| {_cell(tag)} | {scores['count']} | "- + " | ".join(f"{scores[m]:.3f}" for m in headline)+ # A tag whose questions all lack an expected_text has no passage scores.+ + " | ".join(f"{scores[m]:.3f}" if m in scores else "n/a" for m in headline)+ " |")@@ -454,15 +490,17 @@ def render_summary(results: Mapping[str, Any], diff: Mapping[str, Any] | None =lines += ["", f"## Questions below full recall@{k} ({len(missed)})", ""]if missed:lines += [- f"| ID | Query | recall@{k} | Expected file ranks | Snippet found | Top files returned |",- "|---|---|---|---|---|---|",+ f"| ID | Query | recall@{k} | Expected file ranks | Snippet found "+ "| Answer chunk rank | Top files returned |",+ "|---|---|---|---|---|---|---|",]for q in missed:snippet = {True: "yes", False: "no", None: "n/a"}[q.get("snippet_found")]top = ", ".join(item["filename"] for item in q["ranked_files"][:3]) or "none"lines.append(f"| {_cell(q['id'])} | {_cell(q['query'])} | {q['metrics'][f'recall@{k}']:.3f} "- f"| {_cell(_ranks(q['expected_ranks'], _snippet_missing(q)))} | {snippet} | {_cell(top)} |"+ f"| {_cell(_ranks(q['expected_ranks'], _snippet_missing(q)))} | {snippet} "+ f"| {_passage(q)} | {_cell(top)} |")else:lines.append(f"Every question reached full recall@{k}.")diff --git a/scripts/eval/retrieval/run_eval.py b/scripts/eval/retrieval/run_eval.pyindex 145084de5..5d5186fb4 100644--- a/scripts/eval/retrieval/run_eval.py+++ b/scripts/eval/retrieval/run_eval.py@@ -275,11 +275,18 @@ async def _run_queries()chunks = [metrics.Chunk(r.filename, r.text, r.score) for r in response.results]ranked = metrics.rank_files(chunks, run_prefix, corpus_names, query.expected_text)+ scores = metrics.score_query(ranked, query.expected_files, args.k_values)snippet_found = None+ passage: dict[str, Any] = {}if query.expected_text is not None:snippet_found = any(item.snippet_found for item in ranked if item.filename in query.expected_files)+ rank = metrics.passage_rank(+ chunks, run_prefix, corpus_names, query.expected_files, query.expected_text+ )+ passage = {"passage_rank": rank}+ scores.update(metrics.score_passage(rank, args.k_values))records.append({"id": query.id,@@ -300,7 +307,8 @@ async def _run_queries(],"expected_ranks": metrics.expected_ranks(ranked, query.expected_files),"snippet_found": snippet_found,- "metrics": metrics.score_query(ranked, query.expected_files, args.k_values),+ **passage,+ "metrics": scores,})return records@@ -389,6 +397,7 @@ async def run(args: argparse.Namespace) -> int:"corpus_files": len(corpus_names),"corpus_hash": dataset.hash_corpus(args.corpus, corpus_names),"query_count": len(queries),+ "passage_questions": sum(1 for query in queries if query.expected_text is not None),"queries_hash": dataset.hash_file(args.queries),"k_values": args.k_values,"primary_k": metrics.primary_k(args.k_values),diff --git a/tests/unit/test_retrieval_eval.py b/tests/unit/test_retrieval_eval.pyindex 776e03188..40886c090 100644--- a/tests/unit/test_retrieval_eval.py+++ b/tests/unit/test_retrieval_eval.py@@ -167,6 +167,63 @@ def test_expected_file_without_the_snippet_is_not_relevant() -> None:assert scores["reciprocal_rank"] == 0.0+def test_passage_rank_finds_the_answer_chunk_below_the_files_first_chunk() -> None:+ chunks = [+ chunk("a.md", "Introduction to keys."),+ chunk("b.md", "Signing secrets also rotate."),+ chunk("a.md", "Keep the OVERLAP\n window short."),+ ]++ rank = metrics.passage_rank(chunks, PREFIX, CORPUS, ["a.md"], "overlap window")++ # File recall gives a.md full marks at rank 1; the answer itself is chunk 3.+ assert rank == 3+ assert metrics.score_passage(rank, [1, 3, 5]) == {+ "passage_hit@1": 0.0,+ "passage_hit@3": 1.0,+ "passage_hit@5": 1.0,+ "passage_rr": pytest.approx(1 / 3),+ }+++def test_passage_rank_ignores_the_text_in_files_that_are_not_expected() -> None:+ chunks = [chunk("b.md", "overlap window"), chunk("a.md", "Nothing here.")]++ rank = metrics.passage_rank(chunks, PREFIX, CORPUS, ["a.md"], "overlap window")++ assert rank is None+ assert metrics.score_passage(rank, [5]) == {"passage_hit@5": 0.0, "passage_rr": 0.0}+++def test_passage_rank_does_not_count_chunks_from_other_documents() -> None:+ chunks = [+ chunk("a.md", "overlap window", prefix="eval-other123-"),+ chunk("stray.md", "overlap window"),+ chunk("a.md", "overlap window"),+ ]++ assert metrics.passage_rank(chunks, PREFIX, CORPUS, ["a.md"], "overlap window") == 1+++def test_aggregate_averages_passage_scores_over_questions_that_have_them() -> None:+ per_query = [+ {"recall@5": 1.0, "passage_hit@5": 1.0, "passage_rr": 0.5},+ {"recall@5": 0.0, "passage_hit@5": 0.0, "passage_rr": 0.0},+ {"recall@5": 1.0},+ ]+ tags = [["semantic"], ["semantic"], ["semantic"]]++ result: Any = metrics.aggregate(per_query, tags)++ assert result["overall"] == {+ "recall@5": pytest.approx(2 / 3),+ "passage_hit@5": 0.5,+ "passage_mrr": 0.25,+ }+ assert result["by_tag"]["semantic"]["count"] == 3+ assert result["by_tag"]["semantic"]["passage_count"] == 2++def test_primary_k_prefers_five_then_largest() -> None:assert metrics.primary_k([1, 3, 5, 10]) == 5assert metrics.primary_k([1, 3]) == 3@@ -499,6 +556,53 @@ def test_diff_explains_a_drop_caused_by_a_missing_snippet() -> None:assert "a.md: #1 → a.md: #1 (snippet not found)" in report.render_diff(diff)+def with_passage(query: dict[str, Any], rank: int | None) -> dict[str, Any]:+ query["expected_text"] = "overlap window"+ query["passage_rank"] = rank+ query["metrics"].update(metrics.score_passage(rank, [5]))+ return query+++def test_summary_and_diff_show_the_answer_chunk_rank() -> None:+ baseline = make_results([with_passage(make_query("q", 1.0, 1.0, 1.0, 1), 2)])+ current = make_results(+ [with_passage(make_query("q", 0.0, 0.0, 0.0, None), None)], passage_questions=1+ )++ summary = report.render_summary(current, report.diff_results(baseline, current))++ assert "| Passage MRR | 0.000 |" in summary+ assert "| passage hit@5 | 0.000 |" in summary+ assert "cover the 1 of 1 questions that have an expected_text" in summary+ assert "| semantic | 1 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 |" in summary+ # The questions-below-recall table and the regressed table both show it.+ assert "| n/a | not returned |" in summary+ assert "| #2 → not returned |" in summary+++def test_diff_with_an_old_baseline_compares_file_scores_and_says_why() -> None:+ baseline = make_results([make_query("q", 1.0, 1.0, 1.0, 1)])+ current = make_results([with_passage(make_query("q", 1.0, 1.0, 1.0, 1), 3)])++ diff = report.diff_results(baseline, current)++ assert diff["unchanged"] == 1+ assert "passage_mrr" not in diff["aggregate"]+ assert diff["warnings"] == []+ assert any("baseline has no passage scores" in note for note in diff["notes"])+ assert "> **Note:** The baseline has no passage scores" in report.render_diff(diff)+++def test_an_old_baseline_still_passes_the_gate_and_allows_promotion() -> None:+ old = _results_with_recall(0.5)+ same = make_results([with_passage(make_query("q", 0.5, 0.5, 0.5, 1), 1)])+ better = make_results([with_passage(make_query("q", 1.0, 1.0, 1.0, 1), 1)])++ assert report.recall_gate_failure(report.diff_results(old, same), 0.02) is None+ promoted, reason = report.promotion_decision(old, better)+ assert promoted, reason++def test_render_diff_says_so_when_nothing_changed() -> None:results = make_results([make_query("q", 1.0, 1.0, 1.0, 1)])