Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

feat(eval): add report diffing and record query-rewrite provenance

Kunge9ed03bf5031
diff --git a/deeptutor/services/rag/eval/__init__.py b/deeptutor/services/rag/eval/__init__.py
index 785743977..9c332f842 100644
--- a/deeptutor/services/rag/eval/__init__.py
+++ b/deeptutor/services/rag/eval/__init__.py
@@ -21,6 +21,7 @@ Example:
"""
from .dataset import EvalCase, EvalDataset, EvalDatasetError, load_dataset
+from .diff import CaseDelta, EvalDiff, MetricDelta, diff_reports
from .matching import (
DEFAULT_MIN_RATIO,
DEFAULT_MIN_TOKENS,
@@ -54,14 +55,17 @@ from .runner import (
)
__all__ = [
+ "CaseDelta",
"DEFAULT_MIN_RATIO",
"DEFAULT_MIN_TOKENS",
"EvalCase",
"EvalDataset",
"EvalDatasetError",
+ "EvalDiff",
"EvalReport",
"MatchPolicy",
"MatchResult",
+ "MetricDelta",
"ProgressFn",
"QueryEvaluation",
"QueryMetrics",
@@ -71,6 +75,7 @@ __all__ = [
"SOURCE_MODE_CONTEXT",
"aggregate_query_metrics",
"average_precision",
+ "diff_reports",
"evaluate_dataset",
"evaluate_hits",
"format_metric",
diff --git a/deeptutor/services/rag/eval/diff.py b/deeptutor/services/rag/eval/diff.py
new file mode 100644
index 000000000..1e6b07cc7
--- /dev/null
+++ b/deeptutor/services/rag/eval/diff.py
@@ -0,0 +1,312 @@
+"""Compare two retrieval-evaluation reports to quantify a settings change.
+
+A single ``kb eval`` run reports absolute metrics; it cannot tell you whether a
+change to the retrieval stack (an embedding model, a reranker, chunk size, or the
+query-rewrite layer) actually helped. This module answers that by diffing two
+stored reports: a *baseline* recorded before the change and a *candidate* recorded
+after. It aligns the two by query id and reports the per-metric delta plus the
+per-case movement, so a reordered or truncated set still compares like-for-like.
+
+Nothing here calls a model: every number is recomputed from the two reports, so
+the diff is reproducible and cannot drift.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from .metrics import metric_keys
+from .report import format_metric
+
+#: A delta above this magnitude counts as a move (symmetric for regressions).
+_IMPROVE_EPSILON = 1e-9
+
+
+@dataclass(frozen=True)
+class _Normalized:
+ """A report reduced to just what a diff needs: ``k``, metric keys, metrics, cases.
+
+ Both a live :class:`~deeptutor.services.rag.eval.report.EvalReport` object and a
+ serialized ``to_dict`` (e.g. loaded from a ``--save`` JSON file) normalize to the
+ same shape, so a diff works without reconstructing report objects.
+ """
+
+ k: int
+ keys: tuple[str, ...]
+ metrics: dict[str, float]
+ cases: dict[str, dict[str, Any]]
+
+
+def _normalize(report: Any) -> _Normalized:
+ """Reduce a report object or its ``to_dict`` to the fields a diff compares.
+
+ Args:
+ report: A report object or its serialized ``to_dict`` mapping.
+
+ Returns:
+ A :class:`_Normalized` holding the macro metrics and per-case scores.
+
+ Raises:
+ TypeError: If ``report`` is neither a report object nor a mapping.
+ """
+ if isinstance(report, dict):
+ k = int(report.get("k") or 0)
+ aggregate: dict[str, Any] = report.get("metrics") or {}
+ raw_cases: Sequence[Any] = report.get("queries") or []
+ object_form = False
+ else:
+ k = int(getattr(report, "k", 0) or 0)
+ aggregate = dict(report.aggregate())
+ raw_cases = list(getattr(report, "evaluations", ()))
+ object_form = True
+
+ keys = metric_keys(k) if k >= 1 else ()
+ metrics = {key: float(aggregate.get(key, 0.0) or 0.0) for key in keys}
+
+ cases: dict[str, dict[str, Any]] = {}
+ for case in raw_cases:
+ if object_form:
+ case_id = str(case.case_id)
+ query = str(case.query)
+ failed = case.failed
+ values = case.metrics.values()
+ else:
+ case_id = str(case.get("case_id") or case.get("query_id") or case.get("query"))
+ query = str(case.get("query") or case_id)
+ failed = bool(case.get("error"))
+ values = case
+ metrics_by_key = {key: float(values.get(key, 0.0) or 0.0) for key in keys}
+ cases[case_id] = {"query": query, "failed": failed, "metrics": metrics_by_key}
+
+ return _Normalized(k=k, keys=keys, metrics=metrics, cases=cases)
+
+
+@dataclass(frozen=True)
+class MetricDelta:
+ """One metric's baseline value, candidate value, and signed change."""
+
+ key: str
+ baseline: float
+ candidate: float
+
+ @property
+ def delta(self) -> float:
+ """``candidate - baseline``; positive means the change improved this metric."""
+ return self.candidate - self.baseline
+
+ @property
+ def improved(self) -> bool:
+ """Whether the metric went up by more than a rounding epsilon."""
+ return self.delta > _IMPROVE_EPSILON
+
+ @property
+ def regressed(self) -> bool:
+ """Whether the metric went down by more than a rounding epsilon."""
+ return self.delta < -_IMPROVE_EPSILON
+
+ def to_dict(self) -> dict[str, Any]:
+ """A JSON-serializable row for this metric's delta."""
+ return {
+ "metric": self.key,
+ "baseline": round(self.baseline, 6),
+ "candidate": round(self.candidate, 6),
+ "delta": round(self.delta, 6),
+ "direction": "improved" if self.improved else "regressed" if self.regressed else "flat",
+ }
+
+
+@dataclass(frozen=True)
+class CaseDelta:
+ """One query's metric deltas, comparing only where both runs scored it."""
+
+ query: str
+ baseline_failed: bool
+ candidate_failed: bool
+ metrics: dict[str, MetricDelta] = field(default_factory=dict)
+
+ @property
+ def comparable(self) -> bool:
+ """Whether both runs produced a score for this query."""
+ return not (self.baseline_failed or self.candidate_failed)
+
+ @property
+ def worst_key(self) -> str | None:
+ """The metric with the most negative delta, if the case has metrics."""
+ if not self.metrics:
+ return None
+ return min(self.metrics, key=lambda key: self.metrics[key].delta)
+
+ def to_dict(self) -> dict[str, Any]:
+ """A JSON-serializable row for this case's per-metric deltas."""
+ return {
+ "query": self.query,
+ "comparable": self.comparable,
+ "baseline_failed": self.baseline_failed,
+ "candidate_failed": self.candidate_failed,
+ "metrics": {key: delta.to_dict() for key, delta in self.metrics.items()},
+ }
+
+
+@dataclass(frozen=True)
+class EvalDiff:
+ """The per-metric and per-case movement between a baseline and a candidate run.
+
+ Macro deltas cover the scored cases of each run. Per-case deltas are only
+ computed where *both* runs scored the query; a case one run failed (a transient
+ engine error) is reported as added/removed rather than pretending a zero-vs-score
+ delta is meaningful.
+ """
+
+ baseline_label: str
+ candidate_label: str
+ k: int
+ metrics: dict[str, MetricDelta] = field(default_factory=dict)
+ cases: dict[str, CaseDelta] = field(default_factory=dict)
+ added_cases: tuple[str, ...] = ()
+ removed_cases: tuple[str, ...] = ()
+
+ @property
+ def comparable_cases(self) -> tuple[CaseDelta, ...]:
+ """Cases both runs scored, the ones whose per-case movement is meaningful."""
+ return tuple(case for case in self.cases.values() if case.comparable)
+
+ @property
+ def improved_cases(self) -> tuple[CaseDelta, ...]:
+ """Comparable cases where some metric rose and none fell."""
+ return tuple(
+ case
+ for case in self.comparable_cases
+ if any(delta.improved for delta in case.metrics.values())
+ and not any(delta.regressed for delta in case.metrics.values())
+ )
+
+ @property
+ def regressed_cases(self) -> tuple[CaseDelta, ...]:
+ """Comparable cases where any metric fell."""
+ return tuple(
+ case
+ for case in self.comparable_cases
+ if any(delta.regressed for delta in case.metrics.values())
+ )
+
+ @property
+ def has_regression(self) -> bool:
+ """Whether any macro metric or comparable case regressed."""
+ if any(delta.regressed for delta in self.metrics.values()):
+ return True
+ return bool(self.regressed_cases)
+
+ def to_dict(self) -> dict[str, Any]:
+ """A JSON-serializable form of the whole diff."""
+ return {
+ "baseline": self.baseline_label,
+ "candidate": self.candidate_label,
+ "k": self.k,
+ "metrics": {key: delta.to_dict() for key, delta in self.metrics.items()},
+ "cases": {case_id: case.to_dict() for case_id, case in self.cases.items()},
+ "added_cases": list(self.added_cases),
+ "removed_cases": list(self.removed_cases),
+ }
+
+ def summary_lines(self) -> list[str]:
+ """Plain-text lines suitable for a CLI: macro table then case movement."""
+ lines = [
+ f"Baseline: {self.baseline_label}",
+ f"Candidate: {self.candidate_label}",
+ f"k={self.k}",
+ "",
+ f"{'metric':<12} {'baseline':>10} {'candidate':>10} {'delta':>10}",
+ ]
+ for key, delta in self.metrics.items():
+ lines.append(
+ f"{key:<12} "
+ f"{format_metric(delta.baseline):>10} "
+ f"{format_metric(delta.candidate):>10} "
+ f"{format_metric(delta.delta):>10} {delta.to_dict()['direction']}"
+ )
+ lines.append("")
+ lines.append(
+ f"cases: {len(self.comparable_cases)} comparable, "
+ f"{len(self.improved_cases)} improved, {len(self.regressed_cases)} regressed"
+ )
+ if self.added_cases:
+ lines.append(f"added (candidate only): {', '.join(self.added_cases)}")
+ if self.removed_cases:
+ lines.append(f"removed (baseline only): {', '.join(self.removed_cases)}")
+ return lines
+
+
+def diff_reports(
+ baseline: Any,
+ candidate: Any,
+ *,
+ baseline_label: str = "baseline",
+ candidate_label: str = "candidate",
+) -> EvalDiff:
+ """Diff two evaluation reports (objects or ``to_dict`` mappings).
+
+ Args:
+ baseline: The pre-change report (object or serialized mapping).
+ candidate: The post-change report (object or serialized mapping).
+ baseline_label: Human label for the baseline in the summary.
+ candidate_label: Human label for the candidate in the summary.
+
+ Returns:
+ An :class:`EvalDiff` with per-metric and per-case movement. Cases are
+ aligned by query id; a case scored by only one run is reported as
+ added/removed rather than as a spurious delta.
+
+ Raises:
+ ValueError: If the two reports were evaluated at different ``k`` and so
+ cannot be compared like-for-like.
+ """
+ base = _normalize(baseline)
+ cand = _normalize(candidate)
+ if base.k != cand.k:
+ raise ValueError(
+ f"reports use different k (baseline k={base.k}, candidate k={cand.k}); "
+ "re-evaluate both at the same k to compare"
+ )
+
+ metrics = {
+ key: MetricDelta(
+ key=key, baseline=base.metrics.get(key, 0.0), candidate=cand.metrics.get(key, 0.0)
+ )
+ for key in base.keys
+ }
+
+ base_ids = set(base.cases)
+ cand_ids = set(cand.cases)
+ shared = sorted(base_ids & cand_ids)
+
+ cases: dict[str, CaseDelta] = {}
+ for case_id in shared:
+ base_case = base.cases[case_id]
+ cand_case = cand.cases[case_id]
+ comparable = not (base_case["failed"] or cand_case["failed"])
+ per_metric = {
+ key: MetricDelta(
+ key=key,
+ baseline=base_case["metrics"].get(key, 0.0),
+ candidate=cand_case["metrics"].get(key, 0.0),
+ )
+ for key in base.keys
+ }
+ cases[case_id] = CaseDelta(
+ query=cand_case["query"] or base_case["query"],
+ baseline_failed=base_case["failed"],
+ candidate_failed=cand_case["failed"],
+ metrics=per_metric if comparable else {},
+ )
+
+ return EvalDiff(
+ baseline_label=baseline_label,
+ candidate_label=candidate_label,
+ k=base.k,
+ metrics=metrics,
+ cases=cases,
+ added_cases=tuple(sorted(cand_ids - base_ids)),
+ removed_cases=tuple(sorted(base_ids - cand_ids)),
+ )
diff --git a/deeptutor/services/rag/eval/runner.py b/deeptutor/services/rag/eval/runner.py
index 2cde2da54..a88edd89e 100644
--- a/deeptutor/services/rag/eval/runner.py
+++ b/deeptutor/services/rag/eval/runner.py
@@ -54,6 +54,7 @@ class QueryEvaluation:
provider: str = ""
error: str = ""
sources: tuple[dict[str, Any], ...] = ()
+ query_rewrite: dict[str, Any] | None = None
@property
def failed(self) -> bool:
@@ -69,6 +70,7 @@ class QueryEvaluation:
"provider": self.provider,
"source_mode": self.source_mode,
"matched_gold": [sorted(indices) for indices in self.hits],
+ "query_rewrite": self.query_rewrite,
**self.metrics.to_dict(),
}
@@ -113,6 +115,7 @@ async def _default_search(
kb_base_dir: str | None,
mode: str | None,
top_k: int,
+ rewrite: bool | None = None,
) -> dict[str, Any]:
"""Retrieve through the production path (``rag_search`` → the bound pipeline)."""
from deeptutor.tools.rag_tool import rag_search
@@ -120,6 +123,8 @@ async def _default_search(
kwargs: dict[str, Any] = {"top_k": top_k}
if mode:
kwargs["mode"] = mode
+ if rewrite is not None:
+ kwargs["rewrite"] = rewrite
return await rag_search(query=query, kb_name=kb_name, kb_base_dir=kb_base_dir, **kwargs)
@@ -137,6 +142,9 @@ class RetrievalEvaluator:
search_fn: Search callable override, for tests and embedders.
provider: Provider label for the report; a search result's own
``provider`` still wins per case.
+ rewrite: Force the query-rewrite layer on (``True``), off (``False``),
+ or defer to the runtime ``query_rewrite`` setting (``None``, the
+ default). Passed through to :func:`deeptutor.tools.rag_tool.rag_search`.
Raises:
ValueError: ``kb_name`` is empty or ``k`` is below 1.
@@ -153,6 +161,7 @@ class RetrievalEvaluator:
policy: MatchPolicy | None = None,
search_fn: SearchFn | None = None,
provider: str = "",
+ rewrite: bool | None = None,
) -> None:
self.kb_name = (kb_name or "").strip()
if not self.kb_name:
@@ -166,6 +175,7 @@ class RetrievalEvaluator:
self.policy = policy or MatchPolicy()
self.search_fn: SearchFn = search_fn or _default_search
self.provider = provider
+ self.rewrite = rewrite
async def evaluate(
self,
@@ -204,13 +214,16 @@ class RetrievalEvaluator:
"""Search one case and score the citations it came back with."""
matcher = RelevanceMatcher(case.gold, self.policy)
try:
- result = await self.search_fn(
- query=case.query,
- kb_name=self.kb_name,
- kb_base_dir=self.kb_base_dir,
- mode=self.mode,
- top_k=self.k,
- )
+ search_kwargs: dict[str, Any] = {
+ "query": case.query,
+ "kb_name": self.kb_name,
+ "kb_base_dir": self.kb_base_dir,
+ "mode": self.mode,
+ "top_k": self.k,
+ }
+ if self.rewrite is not None:
+ search_kwargs["rewrite"] = self.rewrite
+ result = await self.search_fn(**search_kwargs)
except Exception as exc:
return self._failure(case, matcher, f"{type(exc).__name__}: {exc}")
@@ -226,6 +239,7 @@ class RetrievalEvaluator:
hits = tuple(
frozenset(matcher.match(source_text(source)).indices) for source in sources[: self.k]
)
+ provenance = result.get("query_rewrite")
return QueryEvaluation(
case_id=case.id,
query=case.query,
@@ -236,6 +250,7 @@ class RetrievalEvaluator:
source_mode=source_mode,
provider=str(result.get("provider") or self.provider),
sources=tuple(sources),
+ query_rewrite=dict(provenance) if isinstance(provenance, Mapping) else None,
)
def _failure(self, case: EvalCase, matcher: RelevanceMatcher, message: str) -> QueryEvaluation:
@@ -263,6 +278,7 @@ async def evaluate_dataset(
policy: MatchPolicy | None = None,
search_fn: SearchFn | None = None,
provider: str = "",
+ rewrite: bool | None = None,
progress: ProgressFn | None = None,
) -> EvalReport:
"""Score one dataset against one knowledge base (see :class:`RetrievalEvaluator`)."""
@@ -275,6 +291,7 @@ async def evaluate_dataset(
policy=policy,
search_fn=search_fn,
provider=provider,
+ rewrite=rewrite,
)
return await evaluator.evaluate(dataset, progress=progress)