Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(ima): distinguish retrieval failures and expose incomplete source evidence (#1500)

Bingxi Zhao (Frank)bac2c39875c8
diff --git a/deeptutor/services/rag/pipelines/ima/client.py b/deeptutor/services/rag/pipelines/ima/client.py
index 96ce8918a..187e50c77 100644
--- a/deeptutor/services/rag/pipelines/ima/client.py
+++ b/deeptutor/services/rag/pipelines/ima/client.py
@@ -103,6 +103,7 @@ class ImaClient:
folders = []
cursor = ""
is_end = False
+ unverified = 0
for _ in range(_MAX_SEARCH_PAGES):
page = parse_knowledge_page(
await self._wire.post(
@@ -116,6 +117,7 @@ class ImaClient:
)
documents.extend(page.documents)
folders.extend(page.folders)
+ unverified += page.unverified_documents
cursor = page.next_cursor
is_end = page.is_end
if len(documents) >= limit or page.is_end or not cursor:
@@ -125,6 +127,7 @@ class ImaClient:
folders=tuple(folders),
next_cursor=cursor,
is_end=is_end,
+ unverified_documents=unverified,
)
async def get_knowledge_list(
diff --git a/deeptutor/services/rag/pipelines/ima/models.py b/deeptutor/services/rag/pipelines/ima/models.py
index 62d7d6fa1..ed23f6457 100644
--- a/deeptutor/services/rag/pipelines/ima/models.py
+++ b/deeptutor/services/rag/pipelines/ima/models.py
@@ -59,6 +59,7 @@ class ImaKnowledgePage:
next_cursor: str = ""
is_end: bool = False
+ unverified_documents: int = 0
@dataclass(frozen=True, slots=True)
@@ -129,6 +130,7 @@ def parse_knowledge_page(data: Mapping[str, Any]) -> ImaKnowledgePage:
folders: list[ImaFolder] = []
seen_documents: set[str] = set()
seen_folders: set[str] = set()
+ unverified = 0
for raw in _entries(data, _DOCUMENT_KEYS):
# A folder can appear inside the document array (search matches folder
@@ -142,6 +144,8 @@ def parse_knowledge_page(data: Mapping[str, Any]) -> ImaKnowledgePage:
folders.append(folder)
continue
document = _document_from(raw)
+ if document is None:
+ unverified += 1
if document is not None and document.media_id not in seen_documents:
seen_documents.add(document.media_id)
documents.append(document)
@@ -158,6 +162,7 @@ def parse_knowledge_page(data: Mapping[str, Any]) -> ImaKnowledgePage:
path=_breadcrumb(data.get("current_path")),
next_cursor=str(data.get("next_cursor") or ""),
is_end=bool(data.get("is_end")),
+ unverified_documents=unverified,
)
diff --git a/deeptutor/services/rag/pipelines/ima/pipeline.py b/deeptutor/services/rag/pipelines/ima/pipeline.py
index 5a803a6dd..fa6b49e98 100644
--- a/deeptutor/services/rag/pipelines/ima/pipeline.py
+++ b/deeptutor/services/rag/pipelines/ima/pipeline.py
@@ -21,12 +21,15 @@ import asyncio
import logging
from typing import Any, Dict, List, Optional
+import httpx
+
from deeptutor.runtime.home import get_runtime_data_root
from deeptutor.services.rag.provider_binding import load_kb_config_entry
from . import media as media_ops
from . import sources as source_policy
from .config import ImaNotConfiguredError, resolve_kb_config
+from .envelope import ImaAuthError, ImaRateLimitError
logger = logging.getLogger(__name__)
@@ -78,14 +81,33 @@ class ImaPipeline:
client = self._client(config)
page = await client.search_knowledge(query, limit=self._top_k(kwargs))
except Exception as exc:
- self.logger.error("IMA search failed for '%s': %s", kb_name, exc)
+ self.logger.error("IMA search failed for '%s' (%s)", kb_name, type(exc).__name__)
return self._error_result(query, exc, error_type="retrieval_error")
matched = source_policy.documents_to_sources(page.documents)
- await self._hydrate(client, matched)
+ hydration_failures = await self._hydrate(client, matched)
# A title match without a snippet or readable media is useful for
# hydration, but it is not evidence that the rag tool may cite (#1500).
sources = [source for source in matched if str(source.get("content") or "").strip()]
+ diagnostics = {
+ "knowledge_base": kb_name,
+ "matched_documents": len(matched),
+ "readable_documents": len(sources),
+ "unreadable_documents": len(matched) - len(sources),
+ "unverified_documents": page.unverified_documents + len(page.documents) - len(matched),
+ "hydration_failures": hydration_failures,
+ }
+ if diagnostics["unverified_documents"] and not matched:
+ return {
+ "query": query,
+ "answer": "Tencent IMA returned matches without usable source identifiers. No source passage could be verified.",
+ "content": "",
+ "sources": [],
+ "provider": PROVIDER,
+ "error_type": "unverified_sources",
+ "retrieval_status": "unverified_sources",
+ "diagnostics": diagnostics,
+ }
if matched and not sources:
return {
"query": query,
@@ -97,17 +119,33 @@ class ImaPipeline:
"sources": [],
"provider": PROVIDER,
"error_type": "content_unavailable",
+ "retrieval_status": "content_unavailable",
+ "diagnostics": diagnostics,
}
content = source_policy.render_context(sources)
+ status = "ready"
+ if not matched:
+ status = "no_matches"
+ content = f"Tencent IMA returned no matching passages for this query in '{kb_name}'."
+ elif (
+ len(sources) < len(matched) or hydration_failures or diagnostics["unverified_documents"]
+ ):
+ status = "partial"
+ content += (
+ "\n\nSome IMA matches could not be read in full. Use only the returned "
+ "passages as evidence; do not invent missing source text or locations."
+ )
return {
"query": query,
"answer": content,
"content": content,
"sources": sources,
"provider": PROVIDER,
+ "retrieval_status": status,
+ "diagnostics": diagnostics,
}
- async def _hydrate(self, client, sources: list[dict[str, Any]]) -> None:
+ async def _hydrate(self, client, sources: list[dict[str, Any]]) -> list[dict[str, str]]:
"""Replace thin or missing snippets with real source text, concurrently.
Each fetch is independent, so they run together — a search that needs
@@ -117,11 +155,12 @@ class ImaPipeline:
"""
targets = source_policy.hydration_targets(sources)
if not targets:
- return
+ return []
results = await asyncio.gather(
*(self._fetch_text(client, sources[index]) for index in targets),
return_exceptions=True,
)
+ failures = []
for index, result in zip(targets, results):
if isinstance(result, BaseException):
# HTTP errors may embed a signed COS URL; log only the error
@@ -131,8 +170,16 @@ class ImaPipeline:
sources[index]["chunk_id"],
type(result).__name__,
)
+ failures.append(
+ {"source_id": sources[index]["chunk_id"], "error_type": type(result).__name__}
+ )
elif result:
sources[index]["content"] = result
+ else:
+ failures.append(
+ {"source_id": sources[index]["chunk_id"], "error_type": "content_unavailable"}
+ )
+ return failures
@staticmethod
async def _fetch_text(client, source: dict[str, Any]) -> str:
@@ -144,13 +191,29 @@ class ImaPipeline:
)
def _error_result(self, query: str, exc: Exception, *, error_type: str) -> Dict[str, Any]:
+ # #1500: preserve observable failure classes without exposing signed
+ # media URLs or credentials carried by transport exceptions.
+ message = str(exc)
+ if isinstance(exc, ImaAuthError):
+ error_type = "authentication_error"
+ message = "Tencent IMA rejected the credentials or library access. Check this account's IMA connection and permissions."
+ elif isinstance(exc, ImaRateLimitError):
+ error_type = "rate_limited"
+ message = "Tencent IMA rate limit reached. Wait before retrying this search."
+ elif isinstance(exc, httpx.TimeoutException):
+ error_type = "timeout"
+ message = "Tencent IMA did not respond in time. Retry the search."
+ elif isinstance(exc, httpx.TransportError):
+ error_type = "network_error"
+ message = "Tencent IMA could not be reached. Check the connection and retry."
return {
"query": query,
- "answer": str(exc),
+ "answer": message,
"content": "",
"sources": [],
"provider": PROVIDER,
"error_type": error_type,
+ "retrieval_status": "failed",
}
# ----- indexing (not applicable — owned by IMA) ------------------------
diff --git a/deeptutor/services/rag/pipelines/ima/transport.py b/deeptutor/services/rag/pipelines/ima/transport.py
index 52632fcfc..abff7949f 100644
--- a/deeptutor/services/rag/pipelines/ima/transport.py
+++ b/deeptutor/services/rag/pipelines/ima/transport.py
@@ -25,7 +25,7 @@ from typing import Any, Optional
import httpx
from .config import ImaConfig
-from .envelope import ImaRateLimitError, unwrap
+from .envelope import ImaAPIError, ImaAuthError, ImaRateLimitError, unwrap
API_BASE_URL = "https://ima.qq.com"
WIKI_PREFIX = "/openapi/wiki/v1"
@@ -91,10 +91,18 @@ class ImaTransport:
@staticmethod
def _unwrap(response: httpx.Response) -> dict[str, Any]:
+ if response.status_code in {401, 403}:
+ raise ImaAuthError("IMA rejected the credentials or knowledge-base permissions.")
# A transport-level 429 never reaches the envelope, so it is mapped here
# to the same error the envelope's rate-limit codes raise.
if response.status_code == 429:
raise ImaRateLimitError("IMA rate limit reached. Try again shortly.")
+ if not response.is_success:
+ # #1500: a failed gateway response must not become a successful
+ # empty retrieval merely because its JSON contains code=0.
+ raise ImaAPIError(
+ f"IMA returned HTTP {response.status_code}. Retry or check the connection."
+ )
try:
payload = response.json()
except Exception:
diff --git a/tests/services/rag/test_ima_pipeline.py b/tests/services/rag/test_ima_pipeline.py
index 46150fe73..9c2834258 100644
--- a/tests/services/rag/test_ima_pipeline.py
+++ b/tests/services/rag/test_ima_pipeline.py
@@ -648,6 +648,16 @@ class _SearchStub:
class TestPipelineSearch:
+ @pytest.mark.parametrize(
+ ("status", "error"), [(401, ImaAuthError), (403, ImaAuthError), (503, ImaAPIError)]
+ )
+ def test_http_failure_cannot_be_accepted_as_an_empty_success(self, status, error):
+ def handler(_request):
+ return httpx.Response(status, json={"code": 0, "data": {"info_list": []}})
+
+ with pytest.raises(error):
+ asyncio.run(_client(handler).search_knowledge("multiplication", limit=5))
+
def test_search_shapes_snippets_into_context_and_sources(self, tmp_path) -> None:
base = _kb_config(
tmp_path,
@@ -863,7 +873,10 @@ class TestPipelineSearch:
stub = _SearchStub([{"highlight_content": "orphan"}])
pipeline = ImaPipeline(kb_base_dir=base, client_factory=lambda _c: stub)
- assert asyncio.run(pipeline.search("q", "IMA"))["sources"] == []
+ result = asyncio.run(pipeline.search("q", "IMA"))
+ assert result["sources"] == []
+ assert result["retrieval_status"] == "unverified_sources"
+ assert result["diagnostics"]["unverified_documents"] == 1
def test_top_k_is_clamped(self, tmp_path) -> None:
base = _kb_config(
@@ -911,6 +924,59 @@ class TestPipelineSearch:
assert result["content"] == ""
assert result["sources"] == []
+ @pytest.mark.parametrize(
+ ("error", "kind"),
+ [
+ (ImaAuthError("rejected"), "authentication_error"),
+ (ImaRateLimitError("too many calls"), "rate_limited"),
+ (httpx.ReadTimeout("https://media.example/?token=private"), "timeout"),
+ (httpx.ConnectError("https://media.example/?token=private"), "network_error"),
+ ],
+ )
+ def test_failure_states_are_actionable_without_exposing_transport_urls(
+ self, tmp_path, error, kind
+ ):
+ base = _kb_config(
+ tmp_path, {"client_id": "cid", "api_key": "key", "knowledge_base_id": "kb-1"}
+ )
+ pipeline = ImaPipeline(kb_base_dir=base, client_factory=lambda _: _SearchStub(error=error))
+ result = asyncio.run(pipeline.search("multiplication", "IMA"))
+ assert result["error_type"] == kind
+ assert result["retrieval_status"] == "failed"
+ assert result["sources"] == []
+ assert "private" not in json.dumps(result)
+ assert result["answer"]
+
+ def test_partial_retrieval_keeps_real_passages_and_reports_unreadable_matches(self, tmp_path):
+ base = _kb_config(
+ tmp_path, {"client_id": "cid", "api_key": "key", "knowledge_base_id": "kb-1"}
+ )
+ stub = _SearchStub(
+ [
+ {
+ "media_id": "readable",
+ "title": "Math",
+ "highlight_content": _thick("Three groups of four make twelve."),
+ },
+ {"media_id": "missing", "title": "Unavailable chapter"},
+ ],
+ media_error=ImaAuthError("forbidden"),
+ )
+ result = asyncio.run(
+ ImaPipeline(kb_base_dir=base, client_factory=lambda _: stub).search(
+ "multiplication", "IMA"
+ )
+ )
+ assert result["retrieval_status"] == "partial"
+ assert [source["chunk_id"] for source in result["sources"]] == ["readable"]
+ assert "Three groups of four" in result["content"]
+ assert "do not invent" in result["answer"]
+ assert result["diagnostics"]["matched_documents"] == 2
+ assert result["diagnostics"]["readable_documents"] == 1
+ assert result["diagnostics"]["hydration_failures"] == [
+ {"source_id": "missing", "error_type": "ImaAuthError"}
+ ]
+
def test_transport_failure_reports_retrieval_error(self, tmp_path) -> None:
base = _kb_config(
tmp_path,