Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(rag): report failed retrievals in evaluations

dajiaohuang591473725eb0
diff --git a/deeptutor/services/rag/eval/runner.py b/deeptutor/services/rag/eval/runner.py
index 2cde2da54..3fb6cc22d 100644
--- a/deeptutor/services/rag/eval/runner.py
+++ b/deeptutor/services/rag/eval/runner.py
@@ -103,6 +103,8 @@ def _error_message(result: Mapping[str, Any]) -> str:
if result.get("error_type"):
message = result.get("answer") or result.get("content") or result["error_type"]
return str(message)
+ if result.get("error"):
+ return str(result["error"])
return ""
diff --git a/tests/services/rag/eval/test_search_failure_envelope.py b/tests/services/rag/eval/test_search_failure_envelope.py
new file mode 100644
index 000000000..85d6454e7
--- /dev/null
+++ b/tests/services/rag/eval/test_search_failure_envelope.py
@@ -0,0 +1,45 @@
+"""Keep production retrieval failures out of quality averages."""
+
+from pathlib import Path
+
+import pytest
+
+from deeptutor.services.rag.eval import EvalDataset, evaluate_dataset
+from deeptutor.services.rag.pipelines.llamaindex import pipeline as pipeline_module
+
+
+async def test_native_index_read_failure_is_excluded_from_quality_average(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ index_dir = tmp_path / "index"
+ index_dir.mkdir()
+ (index_dir / "docstore.json").write_text("{}", encoding="utf-8")
+ pipeline = pipeline_module.LlamaIndexPipeline(
+ kb_base_dir=str(tmp_path), signature_provider=lambda: None
+ )
+ monkeypatch.setattr(pipeline, "_configure_settings", lambda: None)
+ monkeypatch.setattr(pipeline_module, "resolve_storage_dir_for_read", lambda *_: index_dir)
+
+ def retrieve(*_, **__):
+ return (index_dir / "missing-index.json").read_text(encoding="utf-8")
+
+ monkeypatch.setattr(pipeline_module.storage, "retrieve_nodes", retrieve)
+
+ async def search(query: str, **kwargs):
+ if query == "working":
+ return {"sources": [{"content": "alpha beta gamma delta"}]}
+ return await pipeline.search(query, kwargs["kb_name"], top_k=kwargs["top_k"])
+
+ dataset = EvalDataset.from_cases(
+ [
+ {"query": "broken", "gold": "alpha beta gamma delta"},
+ {"query": "working", "gold": "alpha beta gamma delta"},
+ ]
+ )
+ report = await evaluate_dataset(dataset, kb_name="example", search_fn=search)
+ assert report.evaluations[0].failed
+ assert "missing-index.json" in report.evaluations[0].error
+ assert report.aggregate()["queries_failed"] == 1
+ assert report.aggregate()["queries_scored"] == 1
+ assert report.aggregate()["recall@5"] == 1.0