DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(ima): distinguish retrieval failures and expose incomplete source evidence (#1500)
Bingxi Zhao (Frank)bac2c39875c8
diff --git a/deeptutor/services/rag/pipelines/ima/client.py b/deeptutor/services/rag/pipelines/ima/client.pyindex 96ce8918a..187e50c77 100644--- a/deeptutor/services/rag/pipelines/ima/client.py+++ b/deeptutor/services/rag/pipelines/ima/client.py@@ -103,6 +103,7 @@ class ImaClient:folders = []cursor = ""is_end = False+ unverified = 0for _ in range(_MAX_SEARCH_PAGES):page = parse_knowledge_page(await self._wire.post(@@ -116,6 +117,7 @@ class ImaClient:)documents.extend(page.documents)folders.extend(page.folders)+ unverified += page.unverified_documentscursor = page.next_cursoris_end = page.is_endif len(documents) >= limit or page.is_end or not cursor:@@ -125,6 +127,7 @@ class ImaClient:folders=tuple(folders),next_cursor=cursor,is_end=is_end,+ unverified_documents=unverified,)async def get_knowledge_list(diff --git a/deeptutor/services/rag/pipelines/ima/models.py b/deeptutor/services/rag/pipelines/ima/models.pyindex 62d7d6fa1..ed23f6457 100644--- a/deeptutor/services/rag/pipelines/ima/models.py+++ b/deeptutor/services/rag/pipelines/ima/models.py@@ -59,6 +59,7 @@ class ImaKnowledgePage:next_cursor: str = ""is_end: bool = False+ unverified_documents: int = 0@dataclass(frozen=True, slots=True)@@ -129,6 +130,7 @@ def parse_knowledge_page(data: Mapping[str, Any]) -> ImaKnowledgePage:folders: list[ImaFolder] = []seen_documents: set[str] = set()seen_folders: set[str] = set()+ unverified = 0for raw in _entries(data, _DOCUMENT_KEYS):# A folder can appear inside the document array (search matches folder@@ -142,6 +144,8 @@ def parse_knowledge_page(data: Mapping[str, Any]) -> ImaKnowledgePage:folders.append(folder)continuedocument = _document_from(raw)+ if document is None:+ unverified += 1if document is not None and document.media_id not in seen_documents:seen_documents.add(document.media_id)documents.append(document)@@ -158,6 +162,7 @@ def parse_knowledge_page(data: Mapping[str, Any]) -> ImaKnowledgePage:path=_breadcrumb(data.get("current_path")),next_cursor=str(data.get("next_cursor") or ""),is_end=bool(data.get("is_end")),+ unverified_documents=unverified,)diff --git a/deeptutor/services/rag/pipelines/ima/pipeline.py b/deeptutor/services/rag/pipelines/ima/pipeline.pyindex 5a803a6dd..fa6b49e98 100644--- a/deeptutor/services/rag/pipelines/ima/pipeline.py+++ b/deeptutor/services/rag/pipelines/ima/pipeline.py@@ -21,12 +21,15 @@ import asyncioimport loggingfrom typing import Any, Dict, List, Optional+import httpx+from deeptutor.runtime.home import get_runtime_data_rootfrom deeptutor.services.rag.provider_binding import load_kb_config_entryfrom . import media as media_opsfrom . import sources as source_policyfrom .config import ImaNotConfiguredError, resolve_kb_config+from .envelope import ImaAuthError, ImaRateLimitErrorlogger = logging.getLogger(__name__)@@ -78,14 +81,33 @@ class ImaPipeline:client = self._client(config)page = await client.search_knowledge(query, limit=self._top_k(kwargs))except Exception as exc:- self.logger.error("IMA search failed for '%s': %s", kb_name, exc)+ self.logger.error("IMA search failed for '%s' (%s)", kb_name, type(exc).__name__)return self._error_result(query, exc, error_type="retrieval_error")matched = source_policy.documents_to_sources(page.documents)- await self._hydrate(client, matched)+ hydration_failures = await self._hydrate(client, matched)# A title match without a snippet or readable media is useful for# hydration, but it is not evidence that the rag tool may cite (#1500).sources = [source for source in matched if str(source.get("content") or "").strip()]+ diagnostics = {+ "knowledge_base": kb_name,+ "matched_documents": len(matched),+ "readable_documents": len(sources),+ "unreadable_documents": len(matched) - len(sources),+ "unverified_documents": page.unverified_documents + len(page.documents) - len(matched),+ "hydration_failures": hydration_failures,+ }+ if diagnostics["unverified_documents"] and not matched:+ return {+ "query": query,+ "answer": "Tencent IMA returned matches without usable source identifiers. No source passage could be verified.",+ "content": "",+ "sources": [],+ "provider": PROVIDER,+ "error_type": "unverified_sources",+ "retrieval_status": "unverified_sources",+ "diagnostics": diagnostics,+ }if matched and not sources:return {"query": query,@@ -97,17 +119,33 @@ class ImaPipeline:"sources": [],"provider": PROVIDER,"error_type": "content_unavailable",+ "retrieval_status": "content_unavailable",+ "diagnostics": diagnostics,}content = source_policy.render_context(sources)+ status = "ready"+ if not matched:+ status = "no_matches"+ content = f"Tencent IMA returned no matching passages for this query in '{kb_name}'."+ elif (+ len(sources) < len(matched) or hydration_failures or diagnostics["unverified_documents"]+ ):+ status = "partial"+ content += (+ "\n\nSome IMA matches could not be read in full. Use only the returned "+ "passages as evidence; do not invent missing source text or locations."+ )return {"query": query,"answer": content,"content": content,"sources": sources,"provider": PROVIDER,+ "retrieval_status": status,+ "diagnostics": diagnostics,}- async def _hydrate(self, client, sources: list[dict[str, Any]]) -> None:+ async def _hydrate(self, client, sources: list[dict[str, Any]]) -> list[dict[str, str]]:"""Replace thin or missing snippets with real source text, concurrently.Each fetch is independent, so they run together — a search that needs@@ -117,11 +155,12 @@ class ImaPipeline:"""targets = source_policy.hydration_targets(sources)if not targets:- return+ return []results = await asyncio.gather(*(self._fetch_text(client, sources[index]) for index in targets),return_exceptions=True,)+ failures = []for index, result in zip(targets, results):if isinstance(result, BaseException):# HTTP errors may embed a signed COS URL; log only the error@@ -131,8 +170,16 @@ class ImaPipeline:sources[index]["chunk_id"],type(result).__name__,)+ failures.append(+ {"source_id": sources[index]["chunk_id"], "error_type": type(result).__name__}+ )elif result:sources[index]["content"] = result+ else:+ failures.append(+ {"source_id": sources[index]["chunk_id"], "error_type": "content_unavailable"}+ )+ return failures@staticmethodasync def _fetch_text(client, source: dict[str, Any]) -> str:@@ -144,13 +191,29 @@ class ImaPipeline:)def _error_result(self, query: str, exc: Exception, *, error_type: str) -> Dict[str, Any]:+ # #1500: preserve observable failure classes without exposing signed+ # media URLs or credentials carried by transport exceptions.+ message = str(exc)+ if isinstance(exc, ImaAuthError):+ error_type = "authentication_error"+ message = "Tencent IMA rejected the credentials or library access. Check this account's IMA connection and permissions."+ elif isinstance(exc, ImaRateLimitError):+ error_type = "rate_limited"+ message = "Tencent IMA rate limit reached. Wait before retrying this search."+ elif isinstance(exc, httpx.TimeoutException):+ error_type = "timeout"+ message = "Tencent IMA did not respond in time. Retry the search."+ elif isinstance(exc, httpx.TransportError):+ error_type = "network_error"+ message = "Tencent IMA could not be reached. Check the connection and retry."return {"query": query,- "answer": str(exc),+ "answer": message,"content": "","sources": [],"provider": PROVIDER,"error_type": error_type,+ "retrieval_status": "failed",}# ----- indexing (not applicable — owned by IMA) ------------------------diff --git a/deeptutor/services/rag/pipelines/ima/transport.py b/deeptutor/services/rag/pipelines/ima/transport.pyindex 52632fcfc..abff7949f 100644--- a/deeptutor/services/rag/pipelines/ima/transport.py+++ b/deeptutor/services/rag/pipelines/ima/transport.py@@ -25,7 +25,7 @@ from typing import Any, Optionalimport httpxfrom .config import ImaConfig-from .envelope import ImaRateLimitError, unwrap+from .envelope import ImaAPIError, ImaAuthError, ImaRateLimitError, unwrapAPI_BASE_URL = "https://ima.qq.com"WIKI_PREFIX = "/openapi/wiki/v1"@@ -91,10 +91,18 @@ class ImaTransport:@staticmethoddef _unwrap(response: httpx.Response) -> dict[str, Any]:+ if response.status_code in {401, 403}:+ raise ImaAuthError("IMA rejected the credentials or knowledge-base permissions.")# A transport-level 429 never reaches the envelope, so it is mapped here# to the same error the envelope's rate-limit codes raise.if response.status_code == 429:raise ImaRateLimitError("IMA rate limit reached. Try again shortly.")+ if not response.is_success:+ # #1500: a failed gateway response must not become a successful+ # empty retrieval merely because its JSON contains code=0.+ raise ImaAPIError(+ f"IMA returned HTTP {response.status_code}. Retry or check the connection."+ )try:payload = response.json()except Exception:diff --git a/tests/services/rag/test_ima_pipeline.py b/tests/services/rag/test_ima_pipeline.pyindex 46150fe73..9c2834258 100644--- a/tests/services/rag/test_ima_pipeline.py+++ b/tests/services/rag/test_ima_pipeline.py@@ -648,6 +648,16 @@ class _SearchStub:class TestPipelineSearch:+ @pytest.mark.parametrize(+ ("status", "error"), [(401, ImaAuthError), (403, ImaAuthError), (503, ImaAPIError)]+ )+ def test_http_failure_cannot_be_accepted_as_an_empty_success(self, status, error):+ def handler(_request):+ return httpx.Response(status, json={"code": 0, "data": {"info_list": []}})++ with pytest.raises(error):+ asyncio.run(_client(handler).search_knowledge("multiplication", limit=5))+def test_search_shapes_snippets_into_context_and_sources(self, tmp_path) -> None:base = _kb_config(tmp_path,@@ -863,7 +873,10 @@ class TestPipelineSearch:stub = _SearchStub([{"highlight_content": "orphan"}])pipeline = ImaPipeline(kb_base_dir=base, client_factory=lambda _c: stub)- assert asyncio.run(pipeline.search("q", "IMA"))["sources"] == []+ result = asyncio.run(pipeline.search("q", "IMA"))+ assert result["sources"] == []+ assert result["retrieval_status"] == "unverified_sources"+ assert result["diagnostics"]["unverified_documents"] == 1def test_top_k_is_clamped(self, tmp_path) -> None:base = _kb_config(@@ -911,6 +924,59 @@ class TestPipelineSearch:assert result["content"] == ""assert result["sources"] == []+ @pytest.mark.parametrize(+ ("error", "kind"),+ [+ (ImaAuthError("rejected"), "authentication_error"),+ (ImaRateLimitError("too many calls"), "rate_limited"),+ (httpx.ReadTimeout("https://media.example/?token=private"), "timeout"),+ (httpx.ConnectError("https://media.example/?token=private"), "network_error"),+ ],+ )+ def test_failure_states_are_actionable_without_exposing_transport_urls(+ self, tmp_path, error, kind+ ):+ base = _kb_config(+ tmp_path, {"client_id": "cid", "api_key": "key", "knowledge_base_id": "kb-1"}+ )+ pipeline = ImaPipeline(kb_base_dir=base, client_factory=lambda _: _SearchStub(error=error))+ result = asyncio.run(pipeline.search("multiplication", "IMA"))+ assert result["error_type"] == kind+ assert result["retrieval_status"] == "failed"+ assert result["sources"] == []+ assert "private" not in json.dumps(result)+ assert result["answer"]++ def test_partial_retrieval_keeps_real_passages_and_reports_unreadable_matches(self, tmp_path):+ base = _kb_config(+ tmp_path, {"client_id": "cid", "api_key": "key", "knowledge_base_id": "kb-1"}+ )+ stub = _SearchStub(+ [+ {+ "media_id": "readable",+ "title": "Math",+ "highlight_content": _thick("Three groups of four make twelve."),+ },+ {"media_id": "missing", "title": "Unavailable chapter"},+ ],+ media_error=ImaAuthError("forbidden"),+ )+ result = asyncio.run(+ ImaPipeline(kb_base_dir=base, client_factory=lambda _: stub).search(+ "multiplication", "IMA"+ )+ )+ assert result["retrieval_status"] == "partial"+ assert [source["chunk_id"] for source in result["sources"]] == ["readable"]+ assert "Three groups of four" in result["content"]+ assert "do not invent" in result["answer"]+ assert result["diagnostics"]["matched_documents"] == 2+ assert result["diagnostics"]["readable_documents"] == 1+ assert result["diagnostics"]["hydration_failures"] == [+ {"source_id": "missing", "error_type": "ImaAuthError"}+ ]+def test_transport_failure_reports_retrieval_error(self, tmp_path) -> None:base = _kb_config(tmp_path,