Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(rag): avoid false IMA citations on empty and failed search (#1500)

Bingxi Zhao (Frank)69058b48bba4
diff --git a/deeptutor/tools/builtin/__init__.py b/deeptutor/tools/builtin/__init__.py
index 85bc536e3..f3e203ccd 100644
--- a/deeptutor/tools/builtin/__init__.py
+++ b/deeptutor/tools/builtin/__init__.py
@@ -73,10 +73,17 @@ def _rag_sources(result: dict[str, Any], *, query: str, kb_name: str) -> list[di
LightRAG-server pipelines). Forward those so a grounded claim is traceable
to the chunk / entity / report behind it; without this the tool reported
only an echo of its own query (issue #694). ``type``/``kb_name`` are kept on
- every entry so consumers that key on them still work, and an engine that
- surfaces no provenance still yields the echo rather than nothing.
+ every entry so consumers that key on them still work. Only a successful
+ non-empty answer may use the query echo as a fallback; an empty or failed
+ search has no source to cite (issue #1500).
"""
retrieved = [item for item in (result.get("sources") or []) if isinstance(item, dict)]
+ if not retrieved and (
+ result.get("error_type")
+ or result.get("needs_reindex")
+ or not (result.get("answer") or result.get("content"))
+ ):
+ return []
if not retrieved:
return [{"type": "rag", "query": query, "kb_name": kb_name}]
return [{"type": "rag", "kb_name": kb_name, **item} for item in retrieved]
@@ -124,7 +131,12 @@ class RAGTool(_PromptHintsMixin, BaseTool):
**extra_kwargs,
)
content = result.get("answer") or result.get("content", "")
- if not content and not result.get("error_type") and not result.get("sources"):
+ failed = bool(result.get("error_type") or result.get("needs_reindex"))
+ if not content and result.get("error_type"):
+ content = f"Knowledge base '{kb_name}' search failed ({result['error_type']})."
+ elif not content and result.get("needs_reindex"):
+ content = f"Knowledge base '{kb_name}' needs reindexing before it can be searched."
+ elif not content and not result.get("sources"):
content = (
f"No matching content was found in knowledge base '{kb_name}'. "
"The search completed successfully."
@@ -133,6 +145,7 @@ class RAGTool(_PromptHintsMixin, BaseTool):
content=content,
sources=_rag_sources(result, query=query, kb_name=kb_name),
metadata=result,
+ success=not failed,
)
diff --git a/tests/core/test_builtin_tools.py b/tests/core/test_builtin_tools.py
index 849e66bd5..ee33105f0 100644
--- a/tests/core/test_builtin_tools.py
+++ b/tests/core/test_builtin_tools.py
@@ -349,6 +349,8 @@ async def test_rag_tool_reports_successful_empty_retrieval(
"No matching content was found in knowledge base 'demo-kb'. "
"The search completed successfully."
)
+ assert result.sources == []
+ assert result.success is True
assert "error_type" not in result.metadata
@@ -369,10 +371,52 @@ async def test_rag_tool_does_not_disguise_errors_as_empty_results(
result = await RAGTool().execute(query="what is a tensor", kb_name="demo-kb")
- assert result.content == ""
+ assert result.content == "Knowledge base 'demo-kb' search failed (retrieval_error)."
+ assert result.sources == []
+ assert result.success is False
assert result.metadata["error_type"] == "retrieval_error"
+async def test_rag_tool_preserves_ima_failure_message_without_claiming_a_source(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A failed remote search is not a completed retrieval or a citable match."""
+
+ async def fake_rag_search(**_kwargs: Any) -> dict[str, Any]:
+ return {
+ "answer": "Could not reach Tencent IMA. Try again shortly.",
+ "content": "",
+ "sources": [],
+ "provider": "ima",
+ "error_type": "retrieval_error",
+ }
+
+ _install_module(monkeypatch, "deeptutor.tools.rag_tool", rag_search=fake_rag_search)
+
+ result = await RAGTool().execute(query="multiplication", kb_name="demo-kb")
+
+ assert result.content == "Could not reach Tencent IMA. Try again shortly."
+ assert result.sources == []
+ assert result.success is False
+
+
+async def test_rag_tool_does_not_report_reindex_required_as_no_matches(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fake_rag_search(**_kwargs: Any) -> dict[str, Any]:
+ return {"answer": "", "content": "", "sources": [], "needs_reindex": True}
+
+ _install_module(monkeypatch, "deeptutor.tools.rag_tool", rag_search=fake_rag_search)
+
+ result = await RAGTool().execute(query="multiplication", kb_name="demo-kb")
+
+ assert result.content == "Knowledge base 'demo-kb' needs reindexing before it can be searched."
+ assert result.sources == []
+ assert result.success is False
+
+
@pytest.mark.asyncio
async def test_rag_tool_rejects_empty_query(monkeypatch: pytest.MonkeyPatch) -> None:
called = False