DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(rag): report failed retrievals in evaluations
dajiaohuang591473725eb0
diff --git a/deeptutor/services/rag/eval/runner.py b/deeptutor/services/rag/eval/runner.pyindex 2cde2da54..3fb6cc22d 100644--- a/deeptutor/services/rag/eval/runner.py+++ b/deeptutor/services/rag/eval/runner.py@@ -103,6 +103,8 @@ def _error_message(result: Mapping[str, Any]) -> str:if result.get("error_type"):message = result.get("answer") or result.get("content") or result["error_type"]return str(message)+ if result.get("error"):+ return str(result["error"])return ""diff --git a/tests/services/rag/eval/test_search_failure_envelope.py b/tests/services/rag/eval/test_search_failure_envelope.pynew file mode 100644index 000000000..85d6454e7--- /dev/null+++ b/tests/services/rag/eval/test_search_failure_envelope.py@@ -0,0 +1,45 @@+"""Keep production retrieval failures out of quality averages."""++from pathlib import Path++import pytest++from deeptutor.services.rag.eval import EvalDataset, evaluate_dataset+from deeptutor.services.rag.pipelines.llamaindex import pipeline as pipeline_module+++async def test_native_index_read_failure_is_excluded_from_quality_average(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ index_dir = tmp_path / "index"+ index_dir.mkdir()+ (index_dir / "docstore.json").write_text("{}", encoding="utf-8")+ pipeline = pipeline_module.LlamaIndexPipeline(+ kb_base_dir=str(tmp_path), signature_provider=lambda: None+ )+ monkeypatch.setattr(pipeline, "_configure_settings", lambda: None)+ monkeypatch.setattr(pipeline_module, "resolve_storage_dir_for_read", lambda *_: index_dir)++ def retrieve(*_, **__):+ return (index_dir / "missing-index.json").read_text(encoding="utf-8")++ monkeypatch.setattr(pipeline_module.storage, "retrieve_nodes", retrieve)++ async def search(query: str, **kwargs):+ if query == "working":+ return {"sources": [{"content": "alpha beta gamma delta"}]}+ return await pipeline.search(query, kwargs["kb_name"], top_k=kwargs["top_k"])++ dataset = EvalDataset.from_cases(+ [+ {"query": "broken", "gold": "alpha beta gamma delta"},+ {"query": "working", "gold": "alpha beta gamma delta"},+ ]+ )+ report = await evaluate_dataset(dataset, kb_name="example", search_fn=search)+ assert report.evaluations[0].failed+ assert "missing-index.json" in report.evaluations[0].error+ assert report.aggregate()["queries_failed"] == 1+ assert report.aggregate()["queries_scored"] == 1+ assert report.aggregate()["recall@5"] == 1.0