DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(rag-eval): discount duplicate gold and keep JSON output parseable
Bingxi Zhao (Frank)dcb157886d05
diff --git a/README.md b/README.mdindex 94aefec5c..c76a9e2fb 100644--- a/README.md+++ b/README.md@@ -803,7 +803,7 @@ The built-in LightRAG engine is installed with `pip install 'deeptutor[rag-lightNative LightRAG queries and incremental indexing require the embedding configuration recorded by the published index, including the model, dimension, and endpoint identity. If it changes, restore the original configuration or rebuild with the current embedding; indexes without a recorded embedding identity require a rebuild. The knowledge-base detail and index-version views show recovery guidance, while files remain available for viewing and download.-**Scoring a knowledge base.** With several engines available, "which one retrieves best here?" is an empirical question — so `deeptutor kb eval` answers it. Write a QA set (one JSON object per line: a `query` plus the `gold` passages an ideal retrieval should return), then score the KB with `deeptutor kb eval <name> --dataset qa.jsonl --top-k 5 [--mode hybrid] [--save baseline.json]`. Each case is retrieved through the same path a chat turn uses, its ranked citations are matched against the gold passages (verbatim, truncated, or paraphrased — thresholds are tunable with `--min-ratio`), and the run reports **Recall@k / Precision@k / nDCG@k / MRR / MAP / Hit@k**. Metrics are pure functions over the ranking, so no model judges the output: scores are reproducible and diffable, and a saved baseline turns the set into a regression gate for an embedding switch, a reranker change, or a chunk-size experiment. Cases whose search failed (a missing index, bad credentials) carry no score and are reported separately rather than dragging the means down. PageIndex uses reasoning as retrieval and returns no ranked chunks, so it is excluded.+**Scoring a knowledge base.** With several engines available, "which one retrieves best here?" is an empirical question — so `deeptutor kb eval` answers it. Write a QA set (one JSON object per line: a `query` plus the `gold` passages an ideal retrieval should return), then score the KB with `deeptutor kb eval <name> --dataset qa.jsonl --top-k 5 [--mode hybrid] [--save baseline.json]`. Each case is retrieved through the same path a chat turn uses, its ranked citations are matched against the gold passages using exact text or lexical overlap (thresholds are tunable with `--min-ratio`; semantic equivalence is not assessed), and the run reports **Recall@k / Precision@k / nDCG@k / MRR / MAP / Hit@k**. Metrics are pure functions over the ranking, so no model judges the output: scores are reproducible and diffable, and a saved baseline turns the set into a regression gate for an embedding switch, a reranker change, or a chunk-size experiment. Cases whose search failed (a missing index, bad credentials) carry no score and are reported separately rather than dragging the means down. PageIndex uses reasoning as retrieval and returns no ranked chunks, so it is excluded.```jsonl{"query": "Why do transformers scale attention?", "gold": ["Scaling keeps the dot products from growing with the dimension."]}diff --git a/deeptutor/services/rag/eval/metrics.py b/deeptutor/services/rag/eval/metrics.pyindex 69da0f4aa..21579c1c0 100644--- a/deeptutor/services/rag/eval/metrics.py+++ b/deeptutor/services/rag/eval/metrics.py@@ -112,12 +112,18 @@ def ndcg_at_k(hits: Hits, gold_count: int, k: int) -> float:The ideal ranking holds the case's gold passages (capped at ``k``) ahead ofeverything else, so a case whose gold is all retrieved but buried below- irrelevant chunks scores below 1.+ irrelevant chunks scores below 1. Each rank contributes at most one gain,+ and only when it introduces a gold passage not covered by an earlier rank.+ Repeating the same passage cannot substitute for missing gold passages."""cutoff = _require_k(k)if gold_count <= 0:return 0.0- gains = [1.0 if matched else 0.0 for matched in hits[:cutoff]]+ seen: set[int] = set()+ gains: list[float] = []+ for matched in hits[:cutoff]:+ gains.append(1.0 if set(matched) - seen else 0.0)+ seen.update(matched)ideal_dcg = _dcg([1.0] * min(gold_count, cutoff))if ideal_dcg == 0:return 0.0diff --git a/deeptutor_cli/kb.py b/deeptutor_cli/kb.pyindex 939203e89..189c80718 100644--- a/deeptutor_cli/kb.py+++ b/deeptutor_cli/kb.py@@ -113,9 +113,9 @@ def _save_eval_report(save: str, payload: str) -> None:target.parent.mkdir(parents=True, exist_ok=True)target.write_text(payload, encoding="utf-8")except OSError as exc:- console.print(f"[red]Could not write report: {exc}[/]")+ typer.echo(f"Could not write report: {exc}", err=True)raise typer.Exit(code=1) from exc- console.print(f"[green]Report written to {target}[/]")+ typer.echo(f"Report written to {target}", err=True)def _render_eval_report(report: EvalReport, name: str, provider: str) -> None:diff --git a/tests/cli/test_kb_eval_cli.py b/tests/cli/test_kb_eval_cli.pyindex 5430090db..7b52d10e2 100644--- a/tests/cli/test_kb_eval_cli.py+++ b/tests/cli/test_kb_eval_cli.py@@ -186,3 +186,24 @@ def test_eval_rejects_an_impossible_match_ratio(eval_env: SimpleNamespace) -> Noassert result.exit_code == 1assert "min_ratio must be in (0, 1]" in result.output+++def test_eval_json_with_save_keeps_stdout_machine_readable(eval_env, tmp_path):+ target = tmp_path / "saved.json"+ result = runner.invoke(+ app,+ [+ "kb",+ "eval",+ "kb1",+ "--dataset",+ str(eval_env.set_path),+ "--format",+ "json",+ "--save",+ str(target),+ ],+ )+ assert result.exit_code == 0, result.output+ assert json.loads(result.stdout) == json.loads(target.read_text())+ assert "Report written to" in result.stderrdiff --git a/tests/services/rag/eval/test_eval_metrics.py b/tests/services/rag/eval/test_eval_metrics.pyindex cc577e061..b22f34c10 100644--- a/tests/services/rag/eval/test_eval_metrics.py+++ b/tests/services/rag/eval/test_eval_metrics.py@@ -153,3 +153,22 @@ def test_aggregate_of_nothing_is_zeroed_not_an_error() -> None:assert summary["queries"] == 0assert summary["recall@5"] == 0.0assert summary["ndcg@5"] == 0.0++[email protected]("count", [2, 3, 8])+def test_duplicate_gold_cannot_replace_missing_passages(count):+ duplicated = evaluate_hits("duplicate", [{0}] * count, 2, count)+ single = evaluate_hits("single", [{0}], 2, count)+ assert duplicated.recall == 0.5+ assert duplicated.ndcg == pytest.approx(single.ndcg)+ assert duplicated.ndcg == pytest.approx(0.6131, abs=1e-4)+ assert evaluate_hits("complete", [{0}, {1}], 2, count).ndcg == 1.0+++def test_only_new_gold_adds_binary_gain_at_each_rank():+ import math++ assert ndcg_at_k([{0}, {0}, {0, 1}], 2, 3) == pytest.approx(+ (1 + 1 / math.log2(4)) / (1 + 1 / math.log2(3))+ )+ assert ndcg_at_k([{0, 1}], 2, 2) == pytest.approx(1 / (1 + 1 / math.log2(3)))