DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
feat(eval): add report diffing and record query-rewrite provenance
Kunge9ed03bf5031
diff --git a/deeptutor/services/rag/eval/__init__.py b/deeptutor/services/rag/eval/__init__.pyindex 785743977..9c332f842 100644--- a/deeptutor/services/rag/eval/__init__.py+++ b/deeptutor/services/rag/eval/__init__.py@@ -21,6 +21,7 @@ Example:"""from .dataset import EvalCase, EvalDataset, EvalDatasetError, load_dataset+from .diff import CaseDelta, EvalDiff, MetricDelta, diff_reportsfrom .matching import (DEFAULT_MIN_RATIO,DEFAULT_MIN_TOKENS,@@ -54,14 +55,17 @@ from .runner import ()__all__ = [+ "CaseDelta","DEFAULT_MIN_RATIO","DEFAULT_MIN_TOKENS","EvalCase","EvalDataset","EvalDatasetError",+ "EvalDiff","EvalReport","MatchPolicy","MatchResult",+ "MetricDelta","ProgressFn","QueryEvaluation","QueryMetrics",@@ -71,6 +75,7 @@ __all__ = ["SOURCE_MODE_CONTEXT","aggregate_query_metrics","average_precision",+ "diff_reports","evaluate_dataset","evaluate_hits","format_metric",diff --git a/deeptutor/services/rag/eval/diff.py b/deeptutor/services/rag/eval/diff.pynew file mode 100644index 000000000..1e6b07cc7--- /dev/null+++ b/deeptutor/services/rag/eval/diff.py@@ -0,0 +1,312 @@+"""Compare two retrieval-evaluation reports to quantify a settings change.++A single ``kb eval`` run reports absolute metrics; it cannot tell you whether a+change to the retrieval stack (an embedding model, a reranker, chunk size, or the+query-rewrite layer) actually helped. This module answers that by diffing two+stored reports: a *baseline* recorded before the change and a *candidate* recorded+after. It aligns the two by query id and reports the per-metric delta plus the+per-case movement, so a reordered or truncated set still compares like-for-like.++Nothing here calls a model: every number is recomputed from the two reports, so+the diff is reproducible and cannot drift.+"""++from __future__ import annotations++from collections.abc import Sequence+from dataclasses import dataclass, field+from typing import Any++from .metrics import metric_keys+from .report import format_metric++#: A delta above this magnitude counts as a move (symmetric for regressions).+_IMPROVE_EPSILON = 1e-9+++@dataclass(frozen=True)+class _Normalized:+ """A report reduced to just what a diff needs: ``k``, metric keys, metrics, cases.++ Both a live :class:`~deeptutor.services.rag.eval.report.EvalReport` object and a+ serialized ``to_dict`` (e.g. loaded from a ``--save`` JSON file) normalize to the+ same shape, so a diff works without reconstructing report objects.+ """++ k: int+ keys: tuple[str, ...]+ metrics: dict[str, float]+ cases: dict[str, dict[str, Any]]+++def _normalize(report: Any) -> _Normalized:+ """Reduce a report object or its ``to_dict`` to the fields a diff compares.++ Args:+ report: A report object or its serialized ``to_dict`` mapping.++ Returns:+ A :class:`_Normalized` holding the macro metrics and per-case scores.++ Raises:+ TypeError: If ``report`` is neither a report object nor a mapping.+ """+ if isinstance(report, dict):+ k = int(report.get("k") or 0)+ aggregate: dict[str, Any] = report.get("metrics") or {}+ raw_cases: Sequence[Any] = report.get("queries") or []+ object_form = False+ else:+ k = int(getattr(report, "k", 0) or 0)+ aggregate = dict(report.aggregate())+ raw_cases = list(getattr(report, "evaluations", ()))+ object_form = True++ keys = metric_keys(k) if k >= 1 else ()+ metrics = {key: float(aggregate.get(key, 0.0) or 0.0) for key in keys}++ cases: dict[str, dict[str, Any]] = {}+ for case in raw_cases:+ if object_form:+ case_id = str(case.case_id)+ query = str(case.query)+ failed = case.failed+ values = case.metrics.values()+ else:+ case_id = str(case.get("case_id") or case.get("query_id") or case.get("query"))+ query = str(case.get("query") or case_id)+ failed = bool(case.get("error"))+ values = case+ metrics_by_key = {key: float(values.get(key, 0.0) or 0.0) for key in keys}+ cases[case_id] = {"query": query, "failed": failed, "metrics": metrics_by_key}++ return _Normalized(k=k, keys=keys, metrics=metrics, cases=cases)+++@dataclass(frozen=True)+class MetricDelta:+ """One metric's baseline value, candidate value, and signed change."""++ key: str+ baseline: float+ candidate: float++ @property+ def delta(self) -> float:+ """``candidate - baseline``; positive means the change improved this metric."""+ return self.candidate - self.baseline++ @property+ def improved(self) -> bool:+ """Whether the metric went up by more than a rounding epsilon."""+ return self.delta > _IMPROVE_EPSILON++ @property+ def regressed(self) -> bool:+ """Whether the metric went down by more than a rounding epsilon."""+ return self.delta < -_IMPROVE_EPSILON++ def to_dict(self) -> dict[str, Any]:+ """A JSON-serializable row for this metric's delta."""+ return {+ "metric": self.key,+ "baseline": round(self.baseline, 6),+ "candidate": round(self.candidate, 6),+ "delta": round(self.delta, 6),+ "direction": "improved" if self.improved else "regressed" if self.regressed else "flat",+ }+++@dataclass(frozen=True)+class CaseDelta:+ """One query's metric deltas, comparing only where both runs scored it."""++ query: str+ baseline_failed: bool+ candidate_failed: bool+ metrics: dict[str, MetricDelta] = field(default_factory=dict)++ @property+ def comparable(self) -> bool:+ """Whether both runs produced a score for this query."""+ return not (self.baseline_failed or self.candidate_failed)++ @property+ def worst_key(self) -> str | None:+ """The metric with the most negative delta, if the case has metrics."""+ if not self.metrics:+ return None+ return min(self.metrics, key=lambda key: self.metrics[key].delta)++ def to_dict(self) -> dict[str, Any]:+ """A JSON-serializable row for this case's per-metric deltas."""+ return {+ "query": self.query,+ "comparable": self.comparable,+ "baseline_failed": self.baseline_failed,+ "candidate_failed": self.candidate_failed,+ "metrics": {key: delta.to_dict() for key, delta in self.metrics.items()},+ }+++@dataclass(frozen=True)+class EvalDiff:+ """The per-metric and per-case movement between a baseline and a candidate run.++ Macro deltas cover the scored cases of each run. Per-case deltas are only+ computed where *both* runs scored the query; a case one run failed (a transient+ engine error) is reported as added/removed rather than pretending a zero-vs-score+ delta is meaningful.+ """++ baseline_label: str+ candidate_label: str+ k: int+ metrics: dict[str, MetricDelta] = field(default_factory=dict)+ cases: dict[str, CaseDelta] = field(default_factory=dict)+ added_cases: tuple[str, ...] = ()+ removed_cases: tuple[str, ...] = ()++ @property+ def comparable_cases(self) -> tuple[CaseDelta, ...]:+ """Cases both runs scored, the ones whose per-case movement is meaningful."""+ return tuple(case for case in self.cases.values() if case.comparable)++ @property+ def improved_cases(self) -> tuple[CaseDelta, ...]:+ """Comparable cases where some metric rose and none fell."""+ return tuple(+ case+ for case in self.comparable_cases+ if any(delta.improved for delta in case.metrics.values())+ and not any(delta.regressed for delta in case.metrics.values())+ )++ @property+ def regressed_cases(self) -> tuple[CaseDelta, ...]:+ """Comparable cases where any metric fell."""+ return tuple(+ case+ for case in self.comparable_cases+ if any(delta.regressed for delta in case.metrics.values())+ )++ @property+ def has_regression(self) -> bool:+ """Whether any macro metric or comparable case regressed."""+ if any(delta.regressed for delta in self.metrics.values()):+ return True+ return bool(self.regressed_cases)++ def to_dict(self) -> dict[str, Any]:+ """A JSON-serializable form of the whole diff."""+ return {+ "baseline": self.baseline_label,+ "candidate": self.candidate_label,+ "k": self.k,+ "metrics": {key: delta.to_dict() for key, delta in self.metrics.items()},+ "cases": {case_id: case.to_dict() for case_id, case in self.cases.items()},+ "added_cases": list(self.added_cases),+ "removed_cases": list(self.removed_cases),+ }++ def summary_lines(self) -> list[str]:+ """Plain-text lines suitable for a CLI: macro table then case movement."""+ lines = [+ f"Baseline: {self.baseline_label}",+ f"Candidate: {self.candidate_label}",+ f"k={self.k}",+ "",+ f"{'metric':<12} {'baseline':>10} {'candidate':>10} {'delta':>10}",+ ]+ for key, delta in self.metrics.items():+ lines.append(+ f"{key:<12} "+ f"{format_metric(delta.baseline):>10} "+ f"{format_metric(delta.candidate):>10} "+ f"{format_metric(delta.delta):>10} {delta.to_dict()['direction']}"+ )+ lines.append("")+ lines.append(+ f"cases: {len(self.comparable_cases)} comparable, "+ f"{len(self.improved_cases)} improved, {len(self.regressed_cases)} regressed"+ )+ if self.added_cases:+ lines.append(f"added (candidate only): {', '.join(self.added_cases)}")+ if self.removed_cases:+ lines.append(f"removed (baseline only): {', '.join(self.removed_cases)}")+ return lines+++def diff_reports(+ baseline: Any,+ candidate: Any,+ *,+ baseline_label: str = "baseline",+ candidate_label: str = "candidate",+) -> EvalDiff:+ """Diff two evaluation reports (objects or ``to_dict`` mappings).++ Args:+ baseline: The pre-change report (object or serialized mapping).+ candidate: The post-change report (object or serialized mapping).+ baseline_label: Human label for the baseline in the summary.+ candidate_label: Human label for the candidate in the summary.++ Returns:+ An :class:`EvalDiff` with per-metric and per-case movement. Cases are+ aligned by query id; a case scored by only one run is reported as+ added/removed rather than as a spurious delta.++ Raises:+ ValueError: If the two reports were evaluated at different ``k`` and so+ cannot be compared like-for-like.+ """+ base = _normalize(baseline)+ cand = _normalize(candidate)+ if base.k != cand.k:+ raise ValueError(+ f"reports use different k (baseline k={base.k}, candidate k={cand.k}); "+ "re-evaluate both at the same k to compare"+ )++ metrics = {+ key: MetricDelta(+ key=key, baseline=base.metrics.get(key, 0.0), candidate=cand.metrics.get(key, 0.0)+ )+ for key in base.keys+ }++ base_ids = set(base.cases)+ cand_ids = set(cand.cases)+ shared = sorted(base_ids & cand_ids)++ cases: dict[str, CaseDelta] = {}+ for case_id in shared:+ base_case = base.cases[case_id]+ cand_case = cand.cases[case_id]+ comparable = not (base_case["failed"] or cand_case["failed"])+ per_metric = {+ key: MetricDelta(+ key=key,+ baseline=base_case["metrics"].get(key, 0.0),+ candidate=cand_case["metrics"].get(key, 0.0),+ )+ for key in base.keys+ }+ cases[case_id] = CaseDelta(+ query=cand_case["query"] or base_case["query"],+ baseline_failed=base_case["failed"],+ candidate_failed=cand_case["failed"],+ metrics=per_metric if comparable else {},+ )++ return EvalDiff(+ baseline_label=baseline_label,+ candidate_label=candidate_label,+ k=base.k,+ metrics=metrics,+ cases=cases,+ added_cases=tuple(sorted(cand_ids - base_ids)),+ removed_cases=tuple(sorted(base_ids - cand_ids)),+ )diff --git a/deeptutor/services/rag/eval/runner.py b/deeptutor/services/rag/eval/runner.pyindex 2cde2da54..a88edd89e 100644--- a/deeptutor/services/rag/eval/runner.py+++ b/deeptutor/services/rag/eval/runner.py@@ -54,6 +54,7 @@ class QueryEvaluation:provider: str = ""error: str = ""sources: tuple[dict[str, Any], ...] = ()+ query_rewrite: dict[str, Any] | None = None@propertydef failed(self) -> bool:@@ -69,6 +70,7 @@ class QueryEvaluation:"provider": self.provider,"source_mode": self.source_mode,"matched_gold": [sorted(indices) for indices in self.hits],+ "query_rewrite": self.query_rewrite,**self.metrics.to_dict(),}@@ -113,6 +115,7 @@ async def _default_search(kb_base_dir: str | None,mode: str | None,top_k: int,+ rewrite: bool | None = None,) -> dict[str, Any]:"""Retrieve through the production path (``rag_search`` → the bound pipeline)."""from deeptutor.tools.rag_tool import rag_search@@ -120,6 +123,8 @@ async def _default_search(kwargs: dict[str, Any] = {"top_k": top_k}if mode:kwargs["mode"] = mode+ if rewrite is not None:+ kwargs["rewrite"] = rewritereturn await rag_search(query=query, kb_name=kb_name, kb_base_dir=kb_base_dir, **kwargs)@@ -137,6 +142,9 @@ class RetrievalEvaluator:search_fn: Search callable override, for tests and embedders.provider: Provider label for the report; a search result's own``provider`` still wins per case.+ rewrite: Force the query-rewrite layer on (``True``), off (``False``),+ or defer to the runtime ``query_rewrite`` setting (``None``, the+ default). Passed through to :func:`deeptutor.tools.rag_tool.rag_search`.Raises:ValueError: ``kb_name`` is empty or ``k`` is below 1.@@ -153,6 +161,7 @@ class RetrievalEvaluator:policy: MatchPolicy | None = None,search_fn: SearchFn | None = None,provider: str = "",+ rewrite: bool | None = None,) -> None:self.kb_name = (kb_name or "").strip()if not self.kb_name:@@ -166,6 +175,7 @@ class RetrievalEvaluator:self.policy = policy or MatchPolicy()self.search_fn: SearchFn = search_fn or _default_searchself.provider = provider+ self.rewrite = rewriteasync def evaluate(self,@@ -204,13 +214,16 @@ class RetrievalEvaluator:"""Search one case and score the citations it came back with."""matcher = RelevanceMatcher(case.gold, self.policy)try:- result = await self.search_fn(- query=case.query,- kb_name=self.kb_name,- kb_base_dir=self.kb_base_dir,- mode=self.mode,- top_k=self.k,- )+ search_kwargs: dict[str, Any] = {+ "query": case.query,+ "kb_name": self.kb_name,+ "kb_base_dir": self.kb_base_dir,+ "mode": self.mode,+ "top_k": self.k,+ }+ if self.rewrite is not None:+ search_kwargs["rewrite"] = self.rewrite+ result = await self.search_fn(**search_kwargs)except Exception as exc:return self._failure(case, matcher, f"{type(exc).__name__}: {exc}")@@ -226,6 +239,7 @@ class RetrievalEvaluator:hits = tuple(frozenset(matcher.match(source_text(source)).indices) for source in sources[: self.k])+ provenance = result.get("query_rewrite")return QueryEvaluation(case_id=case.id,query=case.query,@@ -236,6 +250,7 @@ class RetrievalEvaluator:source_mode=source_mode,provider=str(result.get("provider") or self.provider),sources=tuple(sources),+ query_rewrite=dict(provenance) if isinstance(provenance, Mapping) else None,)def _failure(self, case: EvalCase, matcher: RelevanceMatcher, message: str) -> QueryEvaluation:@@ -263,6 +278,7 @@ async def evaluate_dataset(policy: MatchPolicy | None = None,search_fn: SearchFn | None = None,provider: str = "",+ rewrite: bool | None = None,progress: ProgressFn | None = None,) -> EvalReport:"""Score one dataset against one knowledge base (see :class:`RetrievalEvaluator`)."""@@ -275,6 +291,7 @@ async def evaluate_dataset(policy=policy,search_fn=search_fn,provider=provider,+ rewrite=rewrite,)return await evaluator.evaluate(dataset, progress=progress)