Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

test(rag): cover eval-diff, rewrite provenance and the kb eval-diff CLI

Kungeb67f53b491d
diff --git a/tests/cli/test_kb_eval_cli.py b/tests/cli/test_kb_eval_cli.py
index 87bcf56fc..4cd882f0f 100644
--- a/tests/cli/test_kb_eval_cli.py
+++ b/tests/cli/test_kb_eval_cli.py
@@ -260,3 +260,110 @@ def test_parse_rewrite_flag_is_case_insensitive() -> None:
from deeptutor_cli.kb import _parse_rewrite_flag
assert _parse_rewrite_flag(" Standalone ") is True
+
+
+def _metric_block(k: int, **overrides: float) -> dict[str, float]:
+ """A complete aggregate metric block for ``k``, zero-filled unless overridden."""
+ keys = (f"recall@{k}", f"precision@{k}", f"ndcg@{k}", "mrr", f"hit@{k}", f"map@{k}")
+ return {key: overrides.get(key, 0.0) for key in keys}
+
+
+def _write_report(path: Path, *, k: int = 5, metrics: dict[str, float] | None = None) -> Path:
+ """Write a minimal ``kb eval --save`` report for ``kb eval-diff`` to load."""
+ path.write_text(
+ json.dumps(
+ {
+ "dataset": "set",
+ "kb": "kb1",
+ "provider": "llamaindex",
+ "k": k,
+ "mode": "",
+ "duration_seconds": 0.1,
+ "match_policy": {"min_ratio": 0.5, "min_tokens": 3},
+ "metrics": _metric_block(k) if metrics is None else metrics,
+ "queries": [],
+ }
+ ),
+ encoding="utf-8",
+ )
+ return path
+
+
+def test_eval_diff_reports_metric_movement(tmp_path: Path) -> None:
+ baseline = _write_report(tmp_path / "base.json", metrics=_metric_block(5, **{"recall@5": 0.4}))
+ candidate = _write_report(tmp_path / "cand.json", metrics=_metric_block(5, **{"recall@5": 0.8}))
+
+ result = runner.invoke(app, ["kb", "eval-diff", str(baseline), str(candidate)])
+
+ assert result.exit_code == 0, result.output
+ assert "Baseline: base" in result.output
+ assert "Candidate: cand" in result.output
+ assert "recall@5" in result.output
+ assert "k=5" in result.output
+ assert "improved" in result.output
+
+
+def test_eval_diff_json_output_is_machine_readable(tmp_path: Path) -> None:
+ baseline = _write_report(tmp_path / "base.json", metrics=_metric_block(5, **{"recall@5": 0.4}))
+ candidate = _write_report(tmp_path / "cand.json", metrics=_metric_block(5, **{"recall@5": 0.8}))
+
+ result = runner.invoke(
+ app, ["kb", "eval-diff", str(baseline), str(candidate), "--format", "json"]
+ )
+
+ assert result.exit_code == 0, result.output
+ payload = json.loads(result.stdout)
+ assert payload["baseline"] == "base"
+ assert payload["candidate"] == "cand"
+ assert payload["metrics"]["recall@5"]["direction"] == "improved"
+ assert payload["metrics"]["recall@5"]["delta"] == pytest.approx(0.4)
+
+
+def test_eval_diff_exits_two_on_a_regression(tmp_path: Path) -> None:
+ baseline = _write_report(tmp_path / "base.json", metrics=_metric_block(5, **{"recall@5": 0.8}))
+ candidate = _write_report(tmp_path / "cand.json", metrics=_metric_block(5, **{"recall@5": 0.4}))
+
+ result = runner.invoke(app, ["kb", "eval-diff", str(baseline), str(candidate)])
+
+ assert result.exit_code == 2, result.output
+
+
+def test_eval_diff_rejects_a_missing_report(tmp_path: Path) -> None:
+ candidate = _write_report(tmp_path / "cand.json")
+
+ result = runner.invoke(app, ["kb", "eval-diff", str(tmp_path / "absent.json"), str(candidate)])
+
+ assert result.exit_code == 1
+ assert "Report not found" in result.output
+
+
+def test_eval_diff_rejects_unparseable_json(tmp_path: Path) -> None:
+ broken = tmp_path / "broken.json"
+ broken.write_text("{not json", encoding="utf-8")
+ candidate = _write_report(tmp_path / "cand.json")
+
+ result = runner.invoke(app, ["kb", "eval-diff", str(broken), str(candidate)])
+
+ assert result.exit_code == 1
+ assert "Could not read report" in result.output
+
+
+def test_eval_diff_rejects_a_non_object_payload(tmp_path: Path) -> None:
+ array = tmp_path / "array.json"
+ array.write_text("[]", encoding="utf-8")
+ candidate = _write_report(tmp_path / "cand.json")
+
+ result = runner.invoke(app, ["kb", "eval-diff", str(array), str(candidate)])
+
+ assert result.exit_code == 1
+ assert "is not a JSON object" in result.output
+
+
+def test_eval_diff_rejects_reports_at_different_k(tmp_path: Path) -> None:
+ baseline = _write_report(tmp_path / "base.json", k=5)
+ candidate = _write_report(tmp_path / "cand.json", k=3)
+
+ result = runner.invoke(app, ["kb", "eval-diff", str(baseline), str(candidate)])
+
+ assert result.exit_code == 1
+ assert "different k" in result.output
diff --git a/tests/services/rag/eval/test_eval_diff.py b/tests/services/rag/eval/test_eval_diff.py
new file mode 100644
index 000000000..e7fd538cd
--- /dev/null
+++ b/tests/services/rag/eval/test_eval_diff.py
@@ -0,0 +1,206 @@
+"""Tests for diffing two retrieval-evaluation reports (the ``kb eval-diff`` core)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from deeptutor.services.rag.eval import EvalReport, MatchPolicy, QueryEvaluation, evaluate_hits
+from deeptutor.services.rag.eval.diff import diff_reports
+
+K = 5
+KEYS = (f"recall@{K}", f"precision@{K}", f"ndcg@{K}", "mrr", f"hit@{K}", f"map@{K}")
+
+
+def _metrics(**overrides: float) -> dict[str, float]:
+ """A complete metric block for the cutoff, zero-filled unless overridden."""
+ return {key: 0.0 for key in KEYS} | overrides
+
+
+def _case(case_id: str, *, error: str = "", **overrides: float) -> dict[str, Any]:
+ """A serialized per-case row in the shape ``kb eval --save`` writes."""
+ return {
+ "case_id": case_id,
+ "query": f"query {case_id}",
+ "error": error,
+ **_metrics(**overrides),
+ }
+
+
+def _report(
+ *,
+ k: int = K,
+ metrics: dict[str, float] | None = None,
+ queries: list[dict[str, Any]] | None = None,
+) -> dict[str, Any]:
+ """A serialized report mapping in the shape ``kb eval --save`` writes."""
+ return {
+ "dataset": "set",
+ "kb": "kb1",
+ "provider": "llamaindex",
+ "k": k,
+ "mode": "",
+ "duration_seconds": 0.1,
+ "match_policy": {"min_ratio": 0.5, "min_tokens": 3},
+ "metrics": _metrics() if metrics is None else metrics,
+ "queries": queries or [],
+ }
+
+
+def _report_object(*rankings: tuple[frozenset[int], ...]) -> EvalReport:
+ """A live report object; each ranking is one case's per-rank matched-gold sets."""
+ evaluations = tuple(
+ QueryEvaluation(
+ case_id=f"q{index}",
+ query=f"query q{index}",
+ k=K,
+ gold_count=1,
+ hits=ranking,
+ metrics=evaluate_hits(f"q{index}", ranking, 1, K),
+ provider="fake",
+ )
+ for index, ranking in enumerate(rankings)
+ )
+ return EvalReport(
+ dataset_name="set",
+ kb_name="kb1",
+ provider="fake",
+ k=K,
+ mode=None,
+ policy=MatchPolicy(),
+ evaluations=evaluations,
+ duration_seconds=0.1,
+ )
+
+
+def test_metric_deltas_are_signed_and_directioned() -> None:
+ baseline = _report(metrics=_metrics(**{"recall@5": 0.5, "mrr": 1.0, "hit@5": 1.0}))
+ candidate = _report(metrics=_metrics(**{"recall@5": 1.0, "mrr": 1.0, "hit@5": 0.5}))
+
+ diff = diff_reports(baseline, candidate)
+
+ assert diff.k == K
+ assert diff.metrics["recall@5"].delta == pytest.approx(0.5)
+ assert diff.metrics["recall@5"].improved
+ assert diff.metrics["hit@5"].delta == pytest.approx(-0.5)
+ assert diff.metrics["hit@5"].regressed
+ flat = diff.metrics["mrr"].to_dict()
+ assert flat["direction"] == "flat"
+ assert flat["delta"] == 0.0
+ assert diff.has_regression
+
+
+def test_a_delta_below_the_epsilon_is_not_a_direction() -> None:
+ baseline = _report(metrics=_metrics(mrr=0.3))
+ candidate = _report(metrics=_metrics(mrr=0.3 + 1e-12))
+
+ delta = diff_reports(baseline, candidate).metrics["mrr"]
+
+ assert not delta.improved
+ assert not delta.regressed
+ assert delta.to_dict()["direction"] == "flat"
+
+
+def test_object_and_serialized_forms_diff_the_same() -> None:
+ late_hit = (frozenset(), frozenset({0}))
+ early_hit = (frozenset({0}), frozenset())
+ baseline = _report_object(late_hit)
+ candidate = _report_object(early_hit)
+
+ as_objects = diff_reports(baseline, candidate)
+ as_dicts = diff_reports(baseline.to_dict(), candidate.to_dict())
+
+ # ``QueryMetrics.to_dict`` rounds to 4 decimals, so the serialized form is
+ # compared with a tolerance; in practice both sides of a diff are saved reports.
+ payload = as_dicts.to_dict()
+ assert as_objects.to_dict()["metrics"].keys() == payload["metrics"].keys()
+ assert as_objects.metrics["mrr"].delta == pytest.approx(0.5)
+ for key, delta in as_objects.metrics.items():
+ assert delta.delta == pytest.approx(payload["metrics"][key]["delta"], abs=1e-3)
+ assert set(as_objects.cases) == set(payload["cases"]) == {"q0"}
+ assert as_objects.cases["q0"].comparable
+ for key, delta in as_objects.cases["q0"].metrics.items():
+ assert delta.delta == pytest.approx(
+ payload["cases"]["q0"]["metrics"][key]["delta"], abs=1e-3
+ )
+
+
+def test_reports_at_different_k_are_rejected() -> None:
+ with pytest.raises(ValueError, match="different k"):
+ diff_reports(_report(k=5), _report(k=3))
+
+
+def test_cases_unique_to_one_run_are_added_and_removed() -> None:
+ baseline = _report(queries=[_case("q1", **{"recall@5": 1.0}), _case("q2", **{"recall@5": 1.0})])
+ candidate = _report(
+ queries=[_case("q2", **{"recall@5": 1.0}), _case("q3", **{"recall@5": 1.0})]
+ )
+
+ diff = diff_reports(baseline, candidate)
+
+ assert diff.added_cases == ("q3",)
+ assert diff.removed_cases == ("q1",)
+ assert set(diff.cases) == {"q2"}
+ assert diff.cases["q2"].comparable
+
+
+def test_a_case_failed_by_either_run_is_not_comparable() -> None:
+ baseline = _report(queries=[_case("q1", **{"recall@5": 1.0})])
+ candidate = _report(queries=[_case("q1", error="index missing", **{"recall@5": 0.0})])
+
+ case = diff_reports(baseline, candidate).cases["q1"]
+
+ assert case.baseline_failed is False
+ assert case.candidate_failed is True
+ assert not case.comparable
+ assert case.metrics == {}
+ assert case.worst_key is None
+
+
+def test_case_movement_classifies_improvements_and_regressions() -> None:
+ baseline = _report(
+ queries=[_case("up", **{"recall@5": 0.5}), _case("down", **{"recall@5": 1.0})]
+ )
+ candidate = _report(
+ queries=[_case("up", **{"recall@5": 1.0}), _case("down", **{"recall@5": 0.0})]
+ )
+
+ diff = diff_reports(baseline, candidate)
+
+ assert diff.cases["up"] in diff.improved_cases
+ assert diff.cases["down"] in diff.regressed_cases
+ assert diff.cases["down"].worst_key == "recall@5"
+ assert diff.has_regression
+
+ flat_candidate = _report(
+ queries=[_case("up", **{"recall@5": 1.0}), _case("down", **{"recall@5": 1.0})]
+ )
+ assert not diff_reports(baseline, flat_candidate).has_regression
+
+
+def test_to_dict_and_summary_lines_render_for_the_cli() -> None:
+ baseline = _report(
+ metrics=_metrics(**{"recall@5": 0.4}), queries=[_case("q1", **{"recall@5": 0.4})]
+ )
+ candidate = _report(
+ metrics=_metrics(**{"recall@5": 0.8}),
+ queries=[_case("q1", **{"recall@5": 0.8}), _case("q2", **{"recall@5": 1.0})],
+ )
+
+ diff = diff_reports(baseline, candidate, baseline_label="before", candidate_label="after")
+ payload = diff.to_dict()
+
+ assert payload["baseline"] == "before"
+ assert payload["candidate"] == "after"
+ assert payload["k"] == K
+ assert payload["metrics"]["recall@5"]["direction"] == "improved"
+ assert payload["metrics"]["recall@5"]["delta"] == pytest.approx(0.4)
+ assert payload["added_cases"] == ["q2"]
+ assert set(payload["cases"]) == {"q1"}
+
+ joined = "\n".join(diff.summary_lines())
+ assert "Baseline: before" in joined
+ assert "Candidate: after" in joined
+ assert f"k={K}" in joined
+ assert "1 comparable, 1 improved, 0 regressed" in joined
diff --git a/tests/services/rag/eval/test_eval_runner.py b/tests/services/rag/eval/test_eval_runner.py
index 192512ed1..cae645c63 100644
--- a/tests/services/rag/eval/test_eval_runner.py
+++ b/tests/services/rag/eval/test_eval_runner.py
@@ -254,3 +254,67 @@ async def test_a_custom_match_policy_flows_through_the_run() -> None:
assert report.policy.min_tokens == 3
assert report.evaluations[0].metrics.recall == 1.0
+
+
+async def test_explicit_rewrite_flag_reaches_search_but_the_default_does_not() -> None:
+ search = _Recorder(_sources(GOLD))
+
+ await evaluate_dataset(_dataset("q"), kb_name="kb", k=5, search_fn=search)
+ await evaluate_dataset(_dataset("q"), kb_name="kb", k=5, search_fn=search, rewrite=True)
+ await evaluate_dataset(_dataset("q"), kb_name="kb", k=5, search_fn=search, rewrite=False)
+
+ assert "rewrite" not in search.calls[0]
+ assert search.calls[1]["rewrite"] is True
+ assert search.calls[2]["rewrite"] is False
+
+
+async def test_query_rewrite_provenance_is_carried_into_the_report() -> None:
+ provenance = {
+ "strategy": "standalone",
+ "applied": True,
+ "rewritten": "alpha beta",
+ "latency_ms": 12.5,
+ }
+ search = _Recorder({**_sources(GOLD), "query_rewrite": provenance})
+
+ report = await evaluate_dataset(_dataset("q"), kb_name="kb", k=5, search_fn=search)
+
+ assert report.evaluations[0].query_rewrite == provenance
+ assert report.to_dict()["queries"][0]["query_rewrite"] == provenance
+
+
+async def test_non_mapping_provenance_is_dropped_rather_than_crashing() -> None:
+ search = _Recorder({**_sources(GOLD), "query_rewrite": "standalone"})
+
+ report = await evaluate_dataset(_dataset("q"), kb_name="kb", k=5, search_fn=search)
+
+ assert report.evaluations[0].query_rewrite is None
+
+
+def test_default_search_forwards_only_an_explicit_rewrite_flag(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ captured: list[dict[str, Any]] = []
+
+ async def fake_search(**kwargs: Any) -> dict[str, Any]:
+ captured.append(kwargs)
+ return {}
+
+ module = types.ModuleType("deeptutor.tools.rag_tool")
+ module.rag_search = fake_search # type: ignore[attr-defined]
+ monkeypatch.setitem(sys.modules, "deeptutor.tools.rag_tool", module)
+
+ asyncio.run(
+ runner_module._default_search(
+ "q", kb_name="kb", kb_base_dir=None, mode=None, top_k=2, rewrite=False
+ )
+ )
+ asyncio.run(
+ runner_module._default_search("q", kb_name="kb", kb_base_dir=None, mode=None, top_k=2)
+ )
+
+ assert captured[0]["rewrite"] is False
+ assert "rewrite" not in captured[1]