DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
test(rag): cover eval-diff, rewrite provenance and the kb eval-diff CLI
Kungeb67f53b491d
diff --git a/tests/cli/test_kb_eval_cli.py b/tests/cli/test_kb_eval_cli.pyindex 87bcf56fc..4cd882f0f 100644--- a/tests/cli/test_kb_eval_cli.py+++ b/tests/cli/test_kb_eval_cli.py@@ -260,3 +260,110 @@ def test_parse_rewrite_flag_is_case_insensitive() -> None:from deeptutor_cli.kb import _parse_rewrite_flagassert _parse_rewrite_flag(" Standalone ") is True+++def _metric_block(k: int, **overrides: float) -> dict[str, float]:+ """A complete aggregate metric block for ``k``, zero-filled unless overridden."""+ keys = (f"recall@{k}", f"precision@{k}", f"ndcg@{k}", "mrr", f"hit@{k}", f"map@{k}")+ return {key: overrides.get(key, 0.0) for key in keys}+++def _write_report(path: Path, *, k: int = 5, metrics: dict[str, float] | None = None) -> Path:+ """Write a minimal ``kb eval --save`` report for ``kb eval-diff`` to load."""+ path.write_text(+ json.dumps(+ {+ "dataset": "set",+ "kb": "kb1",+ "provider": "llamaindex",+ "k": k,+ "mode": "",+ "duration_seconds": 0.1,+ "match_policy": {"min_ratio": 0.5, "min_tokens": 3},+ "metrics": _metric_block(k) if metrics is None else metrics,+ "queries": [],+ }+ ),+ encoding="utf-8",+ )+ return path+++def test_eval_diff_reports_metric_movement(tmp_path: Path) -> None:+ baseline = _write_report(tmp_path / "base.json", metrics=_metric_block(5, **{"recall@5": 0.4}))+ candidate = _write_report(tmp_path / "cand.json", metrics=_metric_block(5, **{"recall@5": 0.8}))++ result = runner.invoke(app, ["kb", "eval-diff", str(baseline), str(candidate)])++ assert result.exit_code == 0, result.output+ assert "Baseline: base" in result.output+ assert "Candidate: cand" in result.output+ assert "recall@5" in result.output+ assert "k=5" in result.output+ assert "improved" in result.output+++def test_eval_diff_json_output_is_machine_readable(tmp_path: Path) -> None:+ baseline = _write_report(tmp_path / "base.json", metrics=_metric_block(5, **{"recall@5": 0.4}))+ candidate = _write_report(tmp_path / "cand.json", metrics=_metric_block(5, **{"recall@5": 0.8}))++ result = runner.invoke(+ app, ["kb", "eval-diff", str(baseline), str(candidate), "--format", "json"]+ )++ assert result.exit_code == 0, result.output+ payload = json.loads(result.stdout)+ assert payload["baseline"] == "base"+ assert payload["candidate"] == "cand"+ assert payload["metrics"]["recall@5"]["direction"] == "improved"+ assert payload["metrics"]["recall@5"]["delta"] == pytest.approx(0.4)+++def test_eval_diff_exits_two_on_a_regression(tmp_path: Path) -> None:+ baseline = _write_report(tmp_path / "base.json", metrics=_metric_block(5, **{"recall@5": 0.8}))+ candidate = _write_report(tmp_path / "cand.json", metrics=_metric_block(5, **{"recall@5": 0.4}))++ result = runner.invoke(app, ["kb", "eval-diff", str(baseline), str(candidate)])++ assert result.exit_code == 2, result.output+++def test_eval_diff_rejects_a_missing_report(tmp_path: Path) -> None:+ candidate = _write_report(tmp_path / "cand.json")++ result = runner.invoke(app, ["kb", "eval-diff", str(tmp_path / "absent.json"), str(candidate)])++ assert result.exit_code == 1+ assert "Report not found" in result.output+++def test_eval_diff_rejects_unparseable_json(tmp_path: Path) -> None:+ broken = tmp_path / "broken.json"+ broken.write_text("{not json", encoding="utf-8")+ candidate = _write_report(tmp_path / "cand.json")++ result = runner.invoke(app, ["kb", "eval-diff", str(broken), str(candidate)])++ assert result.exit_code == 1+ assert "Could not read report" in result.output+++def test_eval_diff_rejects_a_non_object_payload(tmp_path: Path) -> None:+ array = tmp_path / "array.json"+ array.write_text("[]", encoding="utf-8")+ candidate = _write_report(tmp_path / "cand.json")++ result = runner.invoke(app, ["kb", "eval-diff", str(array), str(candidate)])++ assert result.exit_code == 1+ assert "is not a JSON object" in result.output+++def test_eval_diff_rejects_reports_at_different_k(tmp_path: Path) -> None:+ baseline = _write_report(tmp_path / "base.json", k=5)+ candidate = _write_report(tmp_path / "cand.json", k=3)++ result = runner.invoke(app, ["kb", "eval-diff", str(baseline), str(candidate)])++ assert result.exit_code == 1+ assert "different k" in result.outputdiff --git a/tests/services/rag/eval/test_eval_diff.py b/tests/services/rag/eval/test_eval_diff.pynew file mode 100644index 000000000..e7fd538cd--- /dev/null+++ b/tests/services/rag/eval/test_eval_diff.py@@ -0,0 +1,206 @@+"""Tests for diffing two retrieval-evaluation reports (the ``kb eval-diff`` core)."""++from __future__ import annotations++from typing import Any++import pytest++from deeptutor.services.rag.eval import EvalReport, MatchPolicy, QueryEvaluation, evaluate_hits+from deeptutor.services.rag.eval.diff import diff_reports++K = 5+KEYS = (f"recall@{K}", f"precision@{K}", f"ndcg@{K}", "mrr", f"hit@{K}", f"map@{K}")+++def _metrics(**overrides: float) -> dict[str, float]:+ """A complete metric block for the cutoff, zero-filled unless overridden."""+ return {key: 0.0 for key in KEYS} | overrides+++def _case(case_id: str, *, error: str = "", **overrides: float) -> dict[str, Any]:+ """A serialized per-case row in the shape ``kb eval --save`` writes."""+ return {+ "case_id": case_id,+ "query": f"query {case_id}",+ "error": error,+ **_metrics(**overrides),+ }+++def _report(+ *,+ k: int = K,+ metrics: dict[str, float] | None = None,+ queries: list[dict[str, Any]] | None = None,+) -> dict[str, Any]:+ """A serialized report mapping in the shape ``kb eval --save`` writes."""+ return {+ "dataset": "set",+ "kb": "kb1",+ "provider": "llamaindex",+ "k": k,+ "mode": "",+ "duration_seconds": 0.1,+ "match_policy": {"min_ratio": 0.5, "min_tokens": 3},+ "metrics": _metrics() if metrics is None else metrics,+ "queries": queries or [],+ }+++def _report_object(*rankings: tuple[frozenset[int], ...]) -> EvalReport:+ """A live report object; each ranking is one case's per-rank matched-gold sets."""+ evaluations = tuple(+ QueryEvaluation(+ case_id=f"q{index}",+ query=f"query q{index}",+ k=K,+ gold_count=1,+ hits=ranking,+ metrics=evaluate_hits(f"q{index}", ranking, 1, K),+ provider="fake",+ )+ for index, ranking in enumerate(rankings)+ )+ return EvalReport(+ dataset_name="set",+ kb_name="kb1",+ provider="fake",+ k=K,+ mode=None,+ policy=MatchPolicy(),+ evaluations=evaluations,+ duration_seconds=0.1,+ )+++def test_metric_deltas_are_signed_and_directioned() -> None:+ baseline = _report(metrics=_metrics(**{"recall@5": 0.5, "mrr": 1.0, "hit@5": 1.0}))+ candidate = _report(metrics=_metrics(**{"recall@5": 1.0, "mrr": 1.0, "hit@5": 0.5}))++ diff = diff_reports(baseline, candidate)++ assert diff.k == K+ assert diff.metrics["recall@5"].delta == pytest.approx(0.5)+ assert diff.metrics["recall@5"].improved+ assert diff.metrics["hit@5"].delta == pytest.approx(-0.5)+ assert diff.metrics["hit@5"].regressed+ flat = diff.metrics["mrr"].to_dict()+ assert flat["direction"] == "flat"+ assert flat["delta"] == 0.0+ assert diff.has_regression+++def test_a_delta_below_the_epsilon_is_not_a_direction() -> None:+ baseline = _report(metrics=_metrics(mrr=0.3))+ candidate = _report(metrics=_metrics(mrr=0.3 + 1e-12))++ delta = diff_reports(baseline, candidate).metrics["mrr"]++ assert not delta.improved+ assert not delta.regressed+ assert delta.to_dict()["direction"] == "flat"+++def test_object_and_serialized_forms_diff_the_same() -> None:+ late_hit = (frozenset(), frozenset({0}))+ early_hit = (frozenset({0}), frozenset())+ baseline = _report_object(late_hit)+ candidate = _report_object(early_hit)++ as_objects = diff_reports(baseline, candidate)+ as_dicts = diff_reports(baseline.to_dict(), candidate.to_dict())++ # ``QueryMetrics.to_dict`` rounds to 4 decimals, so the serialized form is+ # compared with a tolerance; in practice both sides of a diff are saved reports.+ payload = as_dicts.to_dict()+ assert as_objects.to_dict()["metrics"].keys() == payload["metrics"].keys()+ assert as_objects.metrics["mrr"].delta == pytest.approx(0.5)+ for key, delta in as_objects.metrics.items():+ assert delta.delta == pytest.approx(payload["metrics"][key]["delta"], abs=1e-3)+ assert set(as_objects.cases) == set(payload["cases"]) == {"q0"}+ assert as_objects.cases["q0"].comparable+ for key, delta in as_objects.cases["q0"].metrics.items():+ assert delta.delta == pytest.approx(+ payload["cases"]["q0"]["metrics"][key]["delta"], abs=1e-3+ )+++def test_reports_at_different_k_are_rejected() -> None:+ with pytest.raises(ValueError, match="different k"):+ diff_reports(_report(k=5), _report(k=3))+++def test_cases_unique_to_one_run_are_added_and_removed() -> None:+ baseline = _report(queries=[_case("q1", **{"recall@5": 1.0}), _case("q2", **{"recall@5": 1.0})])+ candidate = _report(+ queries=[_case("q2", **{"recall@5": 1.0}), _case("q3", **{"recall@5": 1.0})]+ )++ diff = diff_reports(baseline, candidate)++ assert diff.added_cases == ("q3",)+ assert diff.removed_cases == ("q1",)+ assert set(diff.cases) == {"q2"}+ assert diff.cases["q2"].comparable+++def test_a_case_failed_by_either_run_is_not_comparable() -> None:+ baseline = _report(queries=[_case("q1", **{"recall@5": 1.0})])+ candidate = _report(queries=[_case("q1", error="index missing", **{"recall@5": 0.0})])++ case = diff_reports(baseline, candidate).cases["q1"]++ assert case.baseline_failed is False+ assert case.candidate_failed is True+ assert not case.comparable+ assert case.metrics == {}+ assert case.worst_key is None+++def test_case_movement_classifies_improvements_and_regressions() -> None:+ baseline = _report(+ queries=[_case("up", **{"recall@5": 0.5}), _case("down", **{"recall@5": 1.0})]+ )+ candidate = _report(+ queries=[_case("up", **{"recall@5": 1.0}), _case("down", **{"recall@5": 0.0})]+ )++ diff = diff_reports(baseline, candidate)++ assert diff.cases["up"] in diff.improved_cases+ assert diff.cases["down"] in diff.regressed_cases+ assert diff.cases["down"].worst_key == "recall@5"+ assert diff.has_regression++ flat_candidate = _report(+ queries=[_case("up", **{"recall@5": 1.0}), _case("down", **{"recall@5": 1.0})]+ )+ assert not diff_reports(baseline, flat_candidate).has_regression+++def test_to_dict_and_summary_lines_render_for_the_cli() -> None:+ baseline = _report(+ metrics=_metrics(**{"recall@5": 0.4}), queries=[_case("q1", **{"recall@5": 0.4})]+ )+ candidate = _report(+ metrics=_metrics(**{"recall@5": 0.8}),+ queries=[_case("q1", **{"recall@5": 0.8}), _case("q2", **{"recall@5": 1.0})],+ )++ diff = diff_reports(baseline, candidate, baseline_label="before", candidate_label="after")+ payload = diff.to_dict()++ assert payload["baseline"] == "before"+ assert payload["candidate"] == "after"+ assert payload["k"] == K+ assert payload["metrics"]["recall@5"]["direction"] == "improved"+ assert payload["metrics"]["recall@5"]["delta"] == pytest.approx(0.4)+ assert payload["added_cases"] == ["q2"]+ assert set(payload["cases"]) == {"q1"}++ joined = "\n".join(diff.summary_lines())+ assert "Baseline: before" in joined+ assert "Candidate: after" in joined+ assert f"k={K}" in joined+ assert "1 comparable, 1 improved, 0 regressed" in joineddiff --git a/tests/services/rag/eval/test_eval_runner.py b/tests/services/rag/eval/test_eval_runner.pyindex 192512ed1..cae645c63 100644--- a/tests/services/rag/eval/test_eval_runner.py+++ b/tests/services/rag/eval/test_eval_runner.py@@ -254,3 +254,67 @@ async def test_a_custom_match_policy_flows_through_the_run() -> None:assert report.policy.min_tokens == 3assert report.evaluations[0].metrics.recall == 1.0+++async def test_explicit_rewrite_flag_reaches_search_but_the_default_does_not() -> None:+ search = _Recorder(_sources(GOLD))++ await evaluate_dataset(_dataset("q"), kb_name="kb", k=5, search_fn=search)+ await evaluate_dataset(_dataset("q"), kb_name="kb", k=5, search_fn=search, rewrite=True)+ await evaluate_dataset(_dataset("q"), kb_name="kb", k=5, search_fn=search, rewrite=False)++ assert "rewrite" not in search.calls[0]+ assert search.calls[1]["rewrite"] is True+ assert search.calls[2]["rewrite"] is False+++async def test_query_rewrite_provenance_is_carried_into_the_report() -> None:+ provenance = {+ "strategy": "standalone",+ "applied": True,+ "rewritten": "alpha beta",+ "latency_ms": 12.5,+ }+ search = _Recorder({**_sources(GOLD), "query_rewrite": provenance})++ report = await evaluate_dataset(_dataset("q"), kb_name="kb", k=5, search_fn=search)++ assert report.evaluations[0].query_rewrite == provenance+ assert report.to_dict()["queries"][0]["query_rewrite"] == provenance+++async def test_non_mapping_provenance_is_dropped_rather_than_crashing() -> None:+ search = _Recorder({**_sources(GOLD), "query_rewrite": "standalone"})++ report = await evaluate_dataset(_dataset("q"), kb_name="kb", k=5, search_fn=search)++ assert report.evaluations[0].query_rewrite is None+++def test_default_search_forwards_only_an_explicit_rewrite_flag(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ captured: list[dict[str, Any]] = []++ async def fake_search(**kwargs: Any) -> dict[str, Any]:+ captured.append(kwargs)+ return {}++ module = types.ModuleType("deeptutor.tools.rag_tool")+ module.rag_search = fake_search # type: ignore[attr-defined]+ monkeypatch.setitem(sys.modules, "deeptutor.tools.rag_tool", module)++ asyncio.run(+ runner_module._default_search(+ "q", kb_name="kb", kb_base_dir=None, mode=None, top_k=2, rewrite=False+ )+ )+ asyncio.run(+ runner_module._default_search("q", kb_name="kb", kb_base_dir=None, mode=None, top_k=2)+ )++ assert captured[0]["rewrite"] is False+ assert "rewrite" not in captured[1]