Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat(eval): score the rank of the answer chunk (#2478) (#2572)

Nathan Bakare9beff91e4c14
diff --git a/scripts/eval/retrieval/README.md b/scripts/eval/retrieval/README.md
index 15a8aebf0..20bd4b40c 100644
--- a/scripts/eval/retrieval/README.md
+++ b/scripts/eval/retrieval/README.md
@@ -276,6 +276,29 @@ All three run from 0 to 1, and higher is better. Runs are compared on recall@5.
If either run did not evaluate k=5, the comparison uses a cutoff both runs have
and says so. Two runs with no cutoff in common cannot be compared.
+### Passage scores
+
+A file can rank first while the chunk that answers the question sits much
+lower. For every question with an `expected_text`, the harness also records
+the rank of the first returned chunk that comes from an expected file and
+contains that text. It counts every chunk this run uploaded, not just the first
+chunk of each file.
+
+| Metric | Meaning |
+|---|---|
+| passage hit@k | 1 when the answer chunk is in the top k chunks, otherwise 0 |
+| Passage MRR | 1 divided by the rank of the answer chunk, averaged over questions; 0 when no chunk holds the text |
+
+Chunks are matched on their text, not on chunk IDs, so passage scores can still
+be compared when the chunk size changes. Questions without an `expected_text`
+have no passage scores and are left out of these averages. The summary says how
+many questions the passage scores cover.
+
+Passage scores are reported for information. Questions are still classified as
+regressed or improved on the file scores above. A baseline written before
+passage scores existed still compares; the comparison notes that only file
+scores were compared.
+
## Results files
Each run writes four files to the output directory: `results.json` and
@@ -295,7 +318,9 @@ Each run writes four files to the output directory: `results.json` and
Each record in `queries` holds the question, its labels, the ranked files with
the score of their first chunk, the rank of each expected file
(`expected_ranks`, `null` when a file was not returned), whether the expected
-text was found, and the metrics for that question.
+text was found, the rank of the answer chunk (`passage_rank`, only for questions
+with an `expected_text`, `null` when no chunk held it), and the metrics for that
+question.
## Add or change a question
diff --git a/scripts/eval/retrieval/metrics.py b/scripts/eval/retrieval/metrics.py
index e6b6cd23d..55103c1cb 100644
--- a/scripts/eval/retrieval/metrics.py
+++ b/scripts/eval/retrieval/metrics.py
@@ -3,7 +3,8 @@
``/v1/search`` returns chunks, but questions are labeled by file because chunk
boundaries move whenever chunking changes. This module turns a ranked chunk
list into a ranked list of unique files and scores it with recall@k,
-reciprocal rank and nDCG@k.
+reciprocal rank and nDCG@k. For questions with an ``expected_text`` it also
+scores the rank of the chunk that holds the answer.
Standard library only, so it can be unit-tested without the SDK or a stack.
"""
@@ -139,11 +140,55 @@ def score_query(
return {**recall, "reciprocal_rank": reciprocal_rank, **ndcg}
+def passage_rank(
+ chunks: Iterable[Chunk],
+ run_prefix: str,
+ corpus_names: Collection[str],
+ expected_files: Collection[str],
+ expected_text: str,
+) -> int | None:
+ """Rank of the first chunk that holds the answer, or None if no chunk does.
+
+ The answer chunk is a chunk of an expected file that contains
+ ``expected_text`` (case- and whitespace-insensitive). Ranks count only the
+ chunks of files this run uploaded, so stray documents never shift them.
+ Matching on text rather than chunk IDs keeps the rank meaningful when the
+ chunk size changes.
+ """
+ needle = _normalize(expected_text)
+ expected = set(expected_files)
+ position = 0
+ for chunk in chunks:
+ name = canonical_name(chunk.filename, run_prefix, corpus_names)
+ if name is None:
+ continue
+ position += 1
+ if name in expected and needle in _normalize(chunk.text):
+ return position
+ return None
+
+
+def score_passage(rank: int | None, k_values: Sequence[int] = DEFAULT_K_VALUES) -> dict[str, float]:
+ """Score one question on the rank of its answer chunk.
+
+ - ``passage_hit@k``: 1 if the answer chunk is in the top k chunks, else 0.
+ - ``passage_rr``: 1 / rank of the answer chunk, 0 if none was returned.
+ """
+ hits = {f"passage_hit@{k}": 1.0 if rank is not None and rank <= k else 0.0 for k in k_values}
+ return {**hits, "passage_rr": 1.0 / rank if rank is not None else 0.0}
+
+
+_MEAN_NAMES = {"reciprocal_rank": "mrr", "passage_rr": "passage_mrr"}
+
+
def _mean_scores(rows: Sequence[Mapping[str, float]]) -> dict[str, float]:
+ # Passage scores exist only for questions with an expected_text, so each
+ # metric is averaged over the questions that have it.
+ keys = list(dict.fromkeys(key for row in rows for key in row))
means: dict[str, float] = {}
- for key in rows[0]:
- name = "mrr" if key == "reciprocal_rank" else key
- means[name] = sum(row[key] for row in rows) / len(rows)
+ for key in keys:
+ values = [row[key] for row in rows if key in row]
+ means[_MEAN_NAMES.get(key, key)] = sum(values) / len(values)
return means
@@ -167,7 +212,13 @@ def aggregate(
by_tag.setdefault(tag, []).append(scores)
return {
"overall": _mean_scores(per_query),
- "by_tag": {
- tag: {"count": len(rows), **_mean_scores(rows)} for tag, rows in sorted(by_tag.items())
- },
+ "by_tag": {tag: _tag_scores(rows) for tag, rows in sorted(by_tag.items())},
}
+
+
+def _tag_scores(rows: Sequence[Mapping[str, float]]) -> dict[str, float]:
+ scores: dict[str, float] = {"count": len(rows)}
+ passage_count = sum(1 for row in rows if "passage_rr" in row)
+ if passage_count:
+ scores["passage_count"] = passage_count
+ return {**scores, **_mean_scores(rows)}
diff --git a/scripts/eval/retrieval/report.py b/scripts/eval/retrieval/report.py
index 952a703c9..371d0f6e5 100644
--- a/scripts/eval/retrieval/report.py
+++ b/scripts/eval/retrieval/report.py
@@ -205,6 +205,7 @@ def diff_results(
"baseline": _snippet_missing(base),
"current": _snippet_missing(cur),
},
+ "passage": {"baseline": _passage(base), "current": _passage(cur)},
}
(improved if direction > 0 else regressed).append(entry)
@@ -229,6 +230,14 @@ def diff_results(
f"The baseline has no recall@{current['run']['primary_k']}, "
f"so questions are compared on recall@{k}."
)
+ # Notes are informational. Unlike warnings, they do not make the file scores
+ # incomparable, so the recall gate and baseline promotion ignore them.
+ notes: list[str] = []
+ if "passage_mrr" in cur_overall and "passage_mrr" not in base_overall:
+ notes.append(
+ "The baseline has no passage scores, so only file scores are compared. "
+ "Rerun the baseline to compare passages."
+ )
if added or removed:
warnings.append(
"The runs do not cover the same questions, so the aggregate deltas "
@@ -240,6 +249,7 @@ def diff_results(
"current_run_id": current["run"].get("run_id"),
"compare_metrics": metrics,
"warnings": warnings,
+ "notes": notes,
"aggregate": aggregate,
"regressed": regressed,
"improved": improved,
@@ -314,7 +324,17 @@ def _cell(text: object) -> str:
def _label(metric: str) -> str:
if metric in ("mrr", "reciprocal_rank"):
return "MRR" if metric == "mrr" else "Reciprocal rank"
- return metric.replace("ndcg", "nDCG")
+ if metric in ("passage_mrr", "passage_rr"):
+ return "Passage MRR" if metric == "passage_mrr" else "Passage reciprocal rank"
+ return metric.replace("ndcg", "nDCG").replace("passage_hit", "passage hit")
+
+
+def _passage(query: Mapping[str, Any]) -> str:
+ """Where the answer chunk ranked, for a question with an expected_text."""
+ if "passage_rank" not in query:
+ return "n/a"
+ rank = query["passage_rank"]
+ return "not returned" if rank is None else f"#{rank}"
def _ranks(ranks: Mapping[str, int | None], snippet_missing: Sequence[str] = ()) -> str:
@@ -330,8 +350,10 @@ def _ranks(ranks: Mapping[str, int | None], snippet_missing: Sequence[str] = ())
def _changed_table(entries: Sequence[Mapping[str, Any]], metrics: Sequence[str]) -> list[str]:
lines = [
- "| ID | Query | " + " | ".join(_label(m) for m in metrics) + " | Expected file ranks |",
- "|---|---|" + "---|" * len(metrics) + "---|",
+ "| ID | Query | "
+ + " | ".join(_label(m) for m in metrics)
+ + " | Expected file ranks | Answer chunk rank |",
+ "|---|---|" + "---|" * len(metrics) + "---|---|",
]
for entry in entries:
cells = []
@@ -344,10 +366,12 @@ def _changed_table(entries: Sequence[Mapping[str, Any]], metrics: Sequence[str])
missing = entry.get("snippet_missing", {})
before = _ranks(ranks["baseline"], missing.get("baseline", ()))
after = _ranks(ranks["current"], missing.get("current", ()))
+ passage = entry.get("passage", {})
lines.append(
f"| {_cell(entry['id'])} | {_cell(entry['query'])} | "
+ " | ".join(cells)
- + f" | {_cell(before)} → {_cell(after)} |"
+ + f" | {_cell(before)} → {_cell(after)} "
+ + f"| {passage.get('baseline', 'n/a')} → {passage.get('current', 'n/a')} |"
)
return lines
@@ -367,7 +391,9 @@ def render_diff(diff: Mapping[str, Any]) -> str:
]
for warning in diff["warnings"]:
lines.append(f"> **Warning:** {warning}")
- if diff["warnings"]:
+ for note in diff.get("notes", ()):
+ lines.append(f"> **Note:** {note}")
+ if diff["warnings"] or diff.get("notes"):
lines.append("")
lines += [f"### Regressed ({len(regressed)})", ""]
@@ -434,8 +460,17 @@ def render_summary(results: Mapping[str, Any], diff: Mapping[str, Any] | None =
"|---|---|",
]
lines += [f"| {_label(name)} | {value:.3f} |" for name, value in overall.items()]
+ if "passage_mrr" in overall:
+ lines += [
+ "",
+ f"Passage scores cover the {run.get('passage_questions', 'n/a')} of "
+ f"{run['query_count']} questions that have an expected_text. They rank the "
+ "first chunk holding that text among all chunks returned.",
+ ]
if by_tag:
+ if f"passage_hit@{k}" in overall:
+ headline += [f"passage_hit@{k}", "passage_mrr"]
lines += [
"",
"## By tag",
@@ -446,7 +481,8 @@ def render_summary(results: Mapping[str, Any], diff: Mapping[str, Any] | None =
for tag, scores in by_tag.items():
lines.append(
f"| {_cell(tag)} | {scores['count']} | "
- + " | ".join(f"{scores[m]:.3f}" for m in headline)
+ # A tag whose questions all lack an expected_text has no passage scores.
+ + " | ".join(f"{scores[m]:.3f}" if m in scores else "n/a" for m in headline)
+ " |"
)
@@ -454,15 +490,17 @@ def render_summary(results: Mapping[str, Any], diff: Mapping[str, Any] | None =
lines += ["", f"## Questions below full recall@{k} ({len(missed)})", ""]
if missed:
lines += [
- f"| ID | Query | recall@{k} | Expected file ranks | Snippet found | Top files returned |",
- "|---|---|---|---|---|---|",
+ f"| ID | Query | recall@{k} | Expected file ranks | Snippet found "
+ "| Answer chunk rank | Top files returned |",
+ "|---|---|---|---|---|---|---|",
]
for q in missed:
snippet = {True: "yes", False: "no", None: "n/a"}[q.get("snippet_found")]
top = ", ".join(item["filename"] for item in q["ranked_files"][:3]) or "none"
lines.append(
f"| {_cell(q['id'])} | {_cell(q['query'])} | {q['metrics'][f'recall@{k}']:.3f} "
- f"| {_cell(_ranks(q['expected_ranks'], _snippet_missing(q)))} | {snippet} | {_cell(top)} |"
+ f"| {_cell(_ranks(q['expected_ranks'], _snippet_missing(q)))} | {snippet} "
+ f"| {_passage(q)} | {_cell(top)} |"
)
else:
lines.append(f"Every question reached full recall@{k}.")
diff --git a/scripts/eval/retrieval/run_eval.py b/scripts/eval/retrieval/run_eval.py
index 145084de5..5d5186fb4 100644
--- a/scripts/eval/retrieval/run_eval.py
+++ b/scripts/eval/retrieval/run_eval.py
@@ -275,11 +275,18 @@ async def _run_queries(
)
chunks = [metrics.Chunk(r.filename, r.text, r.score) for r in response.results]
ranked = metrics.rank_files(chunks, run_prefix, corpus_names, query.expected_text)
+ scores = metrics.score_query(ranked, query.expected_files, args.k_values)
snippet_found = None
+ passage: dict[str, Any] = {}
if query.expected_text is not None:
snippet_found = any(
item.snippet_found for item in ranked if item.filename in query.expected_files
)
+ rank = metrics.passage_rank(
+ chunks, run_prefix, corpus_names, query.expected_files, query.expected_text
+ )
+ passage = {"passage_rank": rank}
+ scores.update(metrics.score_passage(rank, args.k_values))
records.append(
{
"id": query.id,
@@ -300,7 +307,8 @@ async def _run_queries(
],
"expected_ranks": metrics.expected_ranks(ranked, query.expected_files),
"snippet_found": snippet_found,
- "metrics": metrics.score_query(ranked, query.expected_files, args.k_values),
+ **passage,
+ "metrics": scores,
}
)
return records
@@ -389,6 +397,7 @@ async def run(args: argparse.Namespace) -> int:
"corpus_files": len(corpus_names),
"corpus_hash": dataset.hash_corpus(args.corpus, corpus_names),
"query_count": len(queries),
+ "passage_questions": sum(1 for query in queries if query.expected_text is not None),
"queries_hash": dataset.hash_file(args.queries),
"k_values": args.k_values,
"primary_k": metrics.primary_k(args.k_values),
diff --git a/tests/unit/test_retrieval_eval.py b/tests/unit/test_retrieval_eval.py
index 776e03188..40886c090 100644
--- a/tests/unit/test_retrieval_eval.py
+++ b/tests/unit/test_retrieval_eval.py
@@ -167,6 +167,63 @@ def test_expected_file_without_the_snippet_is_not_relevant() -> None:
assert scores["reciprocal_rank"] == 0.0
+def test_passage_rank_finds_the_answer_chunk_below_the_files_first_chunk() -> None:
+ chunks = [
+ chunk("a.md", "Introduction to keys."),
+ chunk("b.md", "Signing secrets also rotate."),
+ chunk("a.md", "Keep the OVERLAP\n window short."),
+ ]
+
+ rank = metrics.passage_rank(chunks, PREFIX, CORPUS, ["a.md"], "overlap window")
+
+ # File recall gives a.md full marks at rank 1; the answer itself is chunk 3.
+ assert rank == 3
+ assert metrics.score_passage(rank, [1, 3, 5]) == {
+ "passage_hit@1": 0.0,
+ "passage_hit@3": 1.0,
+ "passage_hit@5": 1.0,
+ "passage_rr": pytest.approx(1 / 3),
+ }
+
+
+def test_passage_rank_ignores_the_text_in_files_that_are_not_expected() -> None:
+ chunks = [chunk("b.md", "overlap window"), chunk("a.md", "Nothing here.")]
+
+ rank = metrics.passage_rank(chunks, PREFIX, CORPUS, ["a.md"], "overlap window")
+
+ assert rank is None
+ assert metrics.score_passage(rank, [5]) == {"passage_hit@5": 0.0, "passage_rr": 0.0}
+
+
+def test_passage_rank_does_not_count_chunks_from_other_documents() -> None:
+ chunks = [
+ chunk("a.md", "overlap window", prefix="eval-other123-"),
+ chunk("stray.md", "overlap window"),
+ chunk("a.md", "overlap window"),
+ ]
+
+ assert metrics.passage_rank(chunks, PREFIX, CORPUS, ["a.md"], "overlap window") == 1
+
+
+def test_aggregate_averages_passage_scores_over_questions_that_have_them() -> None:
+ per_query = [
+ {"recall@5": 1.0, "passage_hit@5": 1.0, "passage_rr": 0.5},
+ {"recall@5": 0.0, "passage_hit@5": 0.0, "passage_rr": 0.0},
+ {"recall@5": 1.0},
+ ]
+ tags = [["semantic"], ["semantic"], ["semantic"]]
+
+ result: Any = metrics.aggregate(per_query, tags)
+
+ assert result["overall"] == {
+ "recall@5": pytest.approx(2 / 3),
+ "passage_hit@5": 0.5,
+ "passage_mrr": 0.25,
+ }
+ assert result["by_tag"]["semantic"]["count"] == 3
+ assert result["by_tag"]["semantic"]["passage_count"] == 2
+
+
def test_primary_k_prefers_five_then_largest() -> None:
assert metrics.primary_k([1, 3, 5, 10]) == 5
assert metrics.primary_k([1, 3]) == 3
@@ -499,6 +556,53 @@ def test_diff_explains_a_drop_caused_by_a_missing_snippet() -> None:
assert "a.md: #1 → a.md: #1 (snippet not found)" in report.render_diff(diff)
+def with_passage(query: dict[str, Any], rank: int | None) -> dict[str, Any]:
+ query["expected_text"] = "overlap window"
+ query["passage_rank"] = rank
+ query["metrics"].update(metrics.score_passage(rank, [5]))
+ return query
+
+
+def test_summary_and_diff_show_the_answer_chunk_rank() -> None:
+ baseline = make_results([with_passage(make_query("q", 1.0, 1.0, 1.0, 1), 2)])
+ current = make_results(
+ [with_passage(make_query("q", 0.0, 0.0, 0.0, None), None)], passage_questions=1
+ )
+
+ summary = report.render_summary(current, report.diff_results(baseline, current))
+
+ assert "| Passage MRR | 0.000 |" in summary
+ assert "| passage hit@5 | 0.000 |" in summary
+ assert "cover the 1 of 1 questions that have an expected_text" in summary
+ assert "| semantic | 1 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 |" in summary
+ # The questions-below-recall table and the regressed table both show it.
+ assert "| n/a | not returned |" in summary
+ assert "| #2 → not returned |" in summary
+
+
+def test_diff_with_an_old_baseline_compares_file_scores_and_says_why() -> None:
+ baseline = make_results([make_query("q", 1.0, 1.0, 1.0, 1)])
+ current = make_results([with_passage(make_query("q", 1.0, 1.0, 1.0, 1), 3)])
+
+ diff = report.diff_results(baseline, current)
+
+ assert diff["unchanged"] == 1
+ assert "passage_mrr" not in diff["aggregate"]
+ assert diff["warnings"] == []
+ assert any("baseline has no passage scores" in note for note in diff["notes"])
+ assert "> **Note:** The baseline has no passage scores" in report.render_diff(diff)
+
+
+def test_an_old_baseline_still_passes_the_gate_and_allows_promotion() -> None:
+ old = _results_with_recall(0.5)
+ same = make_results([with_passage(make_query("q", 0.5, 0.5, 0.5, 1), 1)])
+ better = make_results([with_passage(make_query("q", 1.0, 1.0, 1.0, 1), 1)])
+
+ assert report.recall_gate_failure(report.diff_results(old, same), 0.02) is None
+ promoted, reason = report.promotion_decision(old, better)
+ assert promoted, reason
+
+
def test_render_diff_says_so_when_nothing_changed() -> None:
results = make_results([make_query("q", 1.0, 1.0, 1.0, 1)])