RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Refine agentic search (#18798)
Yingfeng862ee95df256
diff --git a/api/db/services/dialog_service.py b/api/db/services/dialog_service.pyindex 01e3663d04..189bad8924 100644--- a/api/db/services/dialog_service.py+++ b/api/db/services/dialog_service.py@@ -24,7 +24,6 @@ from rag.advanced_rag.agentic_rag import RAGToolslogger = logging.getLogger(__name__)from datetime import datetime, timezone-from functools import partialfrom timeit import default_timer as timerfrom langfuse import Langfuse, propagate_attributesfrom peewee import fn@@ -45,7 +44,6 @@ from api.db.joint_services.tenant_model_service import get_tenant_default_model_from common.time_utils import current_timestamp, datetime_formatfrom common.text_utils import normalize_arabic_digitsfrom rag.advanced_rag.knowlege_compile.mind_map_extractor import MindMapExtractor-from rag.advanced_rag import DeepResearcherfrom rag.app.tag import label_questionfrom rag.nlp.search import index_namefrom rag.prompts.generator import chunks_format, citation_prompt, cross_languages, full_question, kb_prompt, keyword_extraction, message_fit_in, PROMPT_JINJA_ENV, ASK_SUMMARY@@ -746,80 +744,40 @@ async def async_chat(dialog, messages, stream=True, **kwargs):logging.debug("Proceeding with retrieval")tenant_ids = list(set([kb.tenant_id for kb in kbs]))knowledges = []- # replaced by extension of reasoning: 0, 1, 2- if False: # prompt_config.get("reasoning", False) or kwargs.get("reasoning"):- reasoner = DeepResearcher(- chat_mdl,- prompt_config,- partial(- retriever.retrieval,- embd_mdl=embd_mdl,- tenant_ids=tenant_ids,- kb_ids=dialog.kb_ids,- page=1,- page_size=dialog.top_n,- similarity_threshold=0.2,- vector_similarity_weight=0.3,- doc_ids=scoped_doc_ids,- rerank_candidates_count=rerank_candidates_count,- ),- internet_enabled=use_web_search,+ if embd_mdl:+ kbinfos = await retriever.retrieval(+ " ".join(questions),+ embd_mdl,+ tenant_ids,+ dialog.kb_ids,+ 1,+ dialog.top_n,+ dialog.similarity_threshold,+ dialog.vector_similarity_weight,+ doc_ids=scoped_doc_ids,+ knn_top_k=dialog.top_k,+ aggs=True,+ rerank_mdl=rerank_mdl,+ rank_feature=label_question(" ".join(questions), kbs),+ rerank_candidates_count=rerank_candidates_count,)- queue = asyncio.Queue()-- async def callback(msg: str):- nonlocal queue- await queue.put(msg + "<br/>")-- await callback("<START_DEEP_RESEARCH>")- task = asyncio.create_task(reasoner.research(kbinfos, questions[-1], questions[-1], callback=callback))- while True:- msg = await queue.get()- if msg.find("<START_DEEP_RESEARCH>") == 0:- yield {"answer": "<retrieving>", "reference": {}, "audio_binary": None, "final": False}- elif msg.find("<END_DEEP_RESEARCH>") == 0:- yield {"answer": "</retrieving>", "reference": {}, "audio_binary": None, "final": False}- break- else:- yield {"answer": msg, "reference": {}, "audio_binary": None, "final": False}-- await task-- else:- if embd_mdl:- kbinfos = await retriever.retrieval(- " ".join(questions),- embd_mdl,- tenant_ids,- dialog.kb_ids,- 1,- dialog.top_n,- dialog.similarity_threshold,- dialog.vector_similarity_weight,- doc_ids=scoped_doc_ids,- knn_top_k=dialog.top_k,- aggs=True,- rerank_mdl=rerank_mdl,- rank_feature=label_question(" ".join(questions), kbs),- rerank_candidates_count=rerank_candidates_count,- )- if prompt_config.get("toc_enhance"):- cks = await retriever.retrieval_by_toc(" ".join(questions), kbinfos["chunks"], tenant_ids, chat_mdl, dialog.top_n)- if cks:- kbinfos["chunks"] = cks- kbinfos["chunks"] = retriever.retrieval_by_children(kbinfos["chunks"], tenant_ids)- if use_web_search:- web_search = create_web_search_provider(prompt_config)- web_res = web_search.retrieve_chunks(" ".join(questions))- kbinfos["chunks"].extend(web_res["chunks"])- kbinfos["doc_aggs"].extend(web_res["doc_aggs"])- if prompt_config.get("use_kg"):- default_chat_model = get_tenant_default_model_by_type(dialog.tenant_id, LLMType.CHAT)- ck = await settings.kg_retriever.retrieval(- " ".join(questions), tenant_ids, dialog.kb_ids, embd_mdl, LLMBundle(dialog.tenant_id, default_chat_model, trace_context=trace_context, langfuse_session_id=session_id)- )- if ck["content_with_weight"]:- kbinfos["chunks"].insert(0, ck)+ if prompt_config.get("toc_enhance"):+ cks = await retriever.retrieval_by_toc(" ".join(questions), kbinfos["chunks"], tenant_ids, chat_mdl, dialog.top_n)+ if cks:+ kbinfos["chunks"] = cks+ kbinfos["chunks"] = retriever.retrieval_by_children(kbinfos["chunks"], tenant_ids)+ if use_web_search:+ web_search = create_web_search_provider(prompt_config)+ web_res = web_search.retrieve_chunks(" ".join(questions))+ kbinfos["chunks"].extend(web_res["chunks"])+ kbinfos["doc_aggs"].extend(web_res["doc_aggs"])+ if prompt_config.get("use_kg"):+ default_chat_model = get_tenant_default_model_by_type(dialog.tenant_id, LLMType.CHAT)+ ck = await settings.kg_retriever.retrieval(+ " ".join(questions), tenant_ids, dialog.kb_ids, embd_mdl, LLMBundle(dialog.tenant_id, default_chat_model, trace_context=trace_context, langfuse_session_id=session_id)+ )+ if ck["content_with_weight"]:+ kbinfos["chunks"].insert(0, ck)if include_reference_metadata:logging.debug(diff --git a/internal/agent/harness/grounded_llm.go b/internal/agent/harness/grounded_llm.goindex 430ce73e93..941ffb9d23 100644--- a/internal/agent/harness/grounded_llm.go+++ b/internal/agent/harness/grounded_llm.go@@ -27,12 +27,11 @@ import ("ragflow/internal/agent/chat")-// LLM groundedness review (draft review), mirroring Python grounded_llm.py +-// rag/prompts/grounded_select.md. It reviews each claim's draft report against-// the cited evidence semantically, catching over-claims/relation errors that the-// lexical code-level cross-check cannot see.+// LLM groundedness review (draft review). It reviews each claim's draft report+// against the cited evidence semantically, catching over-claims/relation errors+// that the lexical code-level cross-check cannot see.-// groundedSelectPrompt mirrors rag/prompts/grounded_select.md.+// groundedSelectPrompt is the groundedness reviewer system prompt (self-contained).const groundedSelectPrompt = `You are an answer-groundedness reviewer. For each claim's report (the draft answer), determine whether every assertion is supported by the provided evidence.A claim's report is GROUNDED only if each of its assertions can be inferred from the evidence. The assertion does NOT need to use the same words as the evidence (semantic paraphrase is fine), but it must NOT:diff --git a/rag/advanced_rag/__init__.py b/rag/advanced_rag/__init__.pyindex d2b3ddc294..cf3a6b3366 100644--- a/rag/advanced_rag/__init__.py+++ b/rag/advanced_rag/__init__.py@@ -14,13 +14,11 @@# limitations under the License.#-from .tree_structured_query_decomposition_retrieval import TreeStructuredQueryDecompositionRetrieval as DeepResearcherfrom .harness.config import THINKING_MODES, get_modefrom .harness.types import RouteDecision, ExecutionStrategy, ClaimTarget, WorkflowPlan, SufficiencyVerdict__all__ = [- "DeepResearcher","THINKING_MODES","get_mode","RouteDecision",diff --git a/rag/advanced_rag/agentic_rag.py b/rag/advanced_rag/agentic_rag.pyindex 3af595fe18..576b78df81 100644--- a/rag/advanced_rag/agentic_rag.py+++ b/rag/advanced_rag/agentic_rag.py@@ -46,7 +46,9 @@ from common import settingsfrom common.misc_utils import thread_pool_execfrom common.token_utils import num_tokens_from_stringfrom rag.advanced_rag.agentic_rag_graph import _split_think_stream+from rag.advanced_rag.harness.keywords import extract_weighted_keywordsfrom rag.advanced_rag.harness.stats import CountingChatModel, LLMUsageStats, in_phase, using_stats+from rag.advanced_rag.harness.tools.search import _compact_keywordsfrom rag.app.tag import label_questionfrom rag.llm.tool_decorator import toolfrom rag.prompts.generator import (@@ -150,6 +152,29 @@ _RAG_CACHE_STOPWORDS = frozenset()+def _resolve_effective_question(question: str, original_user_question: str) -> str:+ """Prefer the user's ORIGINAL, complete question over the outer model's+ rewritten `question` argument, but only when the two are clearly the SAME+ user turn (>=2 shared significant keywords). The outer smart agent's rewrite+ frequently drops the FINAL target of a multi-hop question (Q317 lost the+ purchaser's death date, Q305 lost the "shortest abbreviation" attribute) by+ compressing to the first hop; using the lossy rewrite means the inner graph+ answers a question whose answer-attribute was deleted, and no planner /+ sufficiency fix can recover it. Keyword-overlap guards against a genuine+ multi-turn re-ask for a DIFFERENT question, which must keep its own query.+ """+ if not question or not original_user_question:+ return question or ""+ _oq = (original_user_question or "").strip()+ if not _oq:+ return question+ _qk = set(_question_keywords(question)[0])+ _ok = set(_question_keywords(_oq)[0])+ if _qk and _ok and len(_qk & _ok) >= 2:+ return _oq+ return question++def _question_keywords(question: str) -> tuple[set[str], set[str]]:"""(significant words, numeric tokens) of a question.@@ -208,9 +233,20 @@ class RAGTools:do_refer: bool | None = True,thinking_mode: str = "medium",text_attachments_content: str = "",+ original_user_question: str = "",system_prompt: str = "",):self.tenant_ids = tenant_ids+ # The user's ORIGINAL, complete question as received from the chat layer+ # (before the outer smart agent may have rewritten/compressed it). The+ # outer LLM's `rag(question=...)` argument is model-generated and+ # frequently drops the FINAL target of a multi-hop question (Q317 lost+ # the purchaser's death, Q305 lost the "shortest abbreviation" attribute)+ # because it compresses the question to the first hop. When this original+ # is available and this is the same user turn, the `rag` tool must use it+ # instead of the model-rewritten argument, so the inner graph never+ # answers a question whose answer-attribute was silently deleted.+ self.original_user_question = original_user_question# P0 instrumentation: count LLM calls / token usage per harness phase.# The wrapper proxies every ``async_chat*`` entry point (and ``clone``)# of the bundle, keeping the rest of the harness untouched.@@ -265,6 +301,15 @@ class RAGTools:# threshold (≥0.85 n-gram Jaccard) so near-identical phrasing is caught# without confusing genuinely different questions.self._rag_cache: dict[str, tuple[str, set[str]]] = {}+ # Sufficiency verdict of the most recent agentic-graph run, used by the outer+ # loop (via `rag`) to decide whether to re-run research and how to rephrase+ # the next question. Set in run_agentic_rag.+ self._rag_verdict: dict | None = None+ # Outer-loop guardrail: count of consecutive rag calls that ended+ # UNANSWERABLE / INSUFFICIENT (evidence conflicts or missing). Once this+ # exceeds a threshold, `rag` tells the outer LLM to stop re-running, so we+ # keep the outer loop's question-rewrite value while bounding useless retries.+ self._consecutive_unanswerable = 0# Per-request retrieval cache keyed by the effective query + scope, so# the same question is never retrieved twice within one turn (e.g.@@ -330,6 +375,16 @@ class RAGTools:"After the `rag` tool returns, do not call `rag` again for the same ""user question. Use the returned cited answer as the final answer ""unless the user explicitly asks a new question.\n"+ "CRITICAL — preserve the full multi-hop structure when phrasing the "+ "`rag` question. A question that compares two or more DISTINCT "+ 'targets or needs an arithmetic result across them ("how much taller '+ 'is X than Y", "how many days after A\'s death did B die", "which of '+ 'these was discovered last") MUST keep every target and relation in '+ "the question you pass to `rag`. Never rewrite a comparison into a "+ "single-entity question — dropping the second entity (e.g. the "+ 'purchaser in "how many days after his death did the man who '+ 'purchased it in 1933 die") makes the pipeline answer only the first '+ "part. Pass the complete comparison.\n"f"{summarize_line}""Do not invent facts and do not fabricate document IDs.")@@ -370,21 +425,31 @@ class RAGTools:lines.append(f"{prefix}: {content}")transcript = "\n".join(lines)+ user_msgs = [m for m in messages if (isinstance(m, str) or m.get("role", "user") == "user")]+ multi_turn = len(user_msgs) > 1+ if not multi_turn and last_user:+ _LOG.info("[Formalize] Single-turn self-contained question — kept verbatim (no rewrite): %s", last_user.strip()[:120])+ try:+ keywords = await self.extract_keywords(last_user)+ except Exception as exc:+ _LOG.info("[Formalize] extract_keywords failed for single-turn question: %s", exc)+ keywords = ""+ return last_user.strip(), keywords+system = ("You are given a conversation. Do BOTH of the following and return JSON only:\n""1. Rewrite the LAST user message into a single, self-contained question that can be "- "understood without seeing the prior conversation — resolve pronouns, ellipses and "- "follow-up shortcuts using earlier turns. Preserve the original language of the last "- "user message. If it is already a complete standalone question, keep it unchanged.\n"- "2. Extract keywords ONLY from the wording of the STANDALONE QUESTION itself — the "- "salient content words and phrases that literally appear in it (key nouns, named "- "entities, domain terms). Do NOT answer the question, and do NOT include any term that "- "would be part of the answer or is not present in the question. Then, for each extracted "- "term, you MAY add 1-2 close synonyms or alternative phrasings OF THAT SAME TERM. Output "- "them all together as one comma-separated list, in the SAME language as the question.\n"- ' Example — question "In which year did Apple acquire Beats?": keywords = '- '"Apple, Apple Inc., acquire, acquisition, Beats" (terms from the question + synonyms; '- "the year is the ANSWER, so it must NOT appear).\n\n"+ "understood without the prior conversation — resolve pronouns, ellipses and follow-up "+ "shortcuts using the earlier turns. In most cases, it should be EXACTLY THE SAME as the "+ "last user query — only rewrite when there is something to resolve (a pronoun/ellipsis "+ "pointing back at an earlier turn). Preserve the original language.\n"+ "2. Extract keywords for a keyword search: the salient content words and phrases that "+ "literally appear in the (standalone) question — key nouns, named entities, domain "+ "terms — PLUS 2-3 close synonyms/abbreviations/aliases/alternative spellings of each, "+ "in the SAME language as the question. Maximize recall. Do NOT include terms that would "+ "be part of the answer.\n"+ ' Example — "In which year did Apple acquire Beats?" -> keywords = "Apple, Apple '+ 'Inc., AAPL, acquire, acquisition, acquired, Beats, Beats Electronics".\n\n'"Output ONLY JSON, no prose, no code fences: "'{"question": "<standalone question>", "keywords": "<term1, term2, synonym1, ...>"}')@@ -408,11 +473,16 @@ class RAGTools:# Fall back to the raw last user message rather than an empty question.question = (last_user or "").strip()+ # Multi-turn: keywords come from the SAME single LLM call (the JSON above+ # outputs both question and keywords) — no second LLM round-trip.keywords = data.get("keywords") or ""if isinstance(keywords, list):keywords = ", ".join(str(k).strip() for k in keywords if str(k).strip())keywords = str(keywords).strip()- return question, keywords+ # Same source-level compacting as extract_keywords: the multi-turn+ # formalize prompt asks for synonyms/aliases per term, which models over-+ # answer with a long redundant run. Cap + dedupe so it stays a hint.+ return question, _compact_keywords(keywords)async def pick_documents(self, question: str) -> list[str] | None:"""Narrow the search to a document subset for ``question``.@@ -442,8 +512,8 @@ class RAGTools:return []logic = filters.get("logic", "and")try:- doc_ids = await thread_pool_exec(- DocMetadataService.filter_doc_ids_by_meta_pushdown,+ # peewee MySQL lookup — call directly to reuse the pool's connection.+ doc_ids = DocMetadataService.filter_doc_ids_by_meta_pushdown(self.kb_ids,conditions,logic,@@ -454,7 +524,8 @@ class RAGTools:return doc_ids or []async def _select_by_titles(self, question: str, max_docs: int = 512) -> list[str]:- docs = await thread_pool_exec(self._collect_doc_titles, max_docs)+ # peewee MySQL lookup — call directly to reuse the pool's connection.+ docs = self._collect_doc_titles(max_docs)if not docs:return []@@ -484,32 +555,27 @@ class RAGTools:known = {doc_id for doc_id, _ in docs}return [doc_id for doc_id in ids if isinstance(doc_id, str) and doc_id in known]+ async def _extract_keywords_weighted(self, question: str) -> tuple[str, str]:+ """Extract FOUR weighted keyword aspects for ``question``.++ Returns ``(query, keywords)``: ``query`` is the entity-weighted search+ string (entity terms repeated x3 so BM25 weights them up, other aspects+ once), ``keywords`` is the plain deduped union used to narrow retrieved+ chunks. Thin wrapper over ``harness.keywords.extract_weighted_keywords``.+ """+ return await extract_weighted_keywords(self.chat_mdl, question)+async def extract_keywords(self, question: str) -> str:- """Produce a compact keyword string (terms + a few close synonyms).+ """Produce a compact keyword string (the deduped union of the four+ weighted aspects, plus close synonyms).Replaces the keywords the outer LLM used to hand to the retrievaltool. Falls back to the question itself when extraction fails."""if not question:return ""- system = (- "Extract the search terms for a knowledge-base query from the "- "question below. Output 3-8 of the most important content terms, "- "plus 1-2 close synonyms or alternative phrasings for any ambiguous "- "term. Single words or short noun phrases, space-separated, in the "- "SAME language as the question. Output ONLY the terms — no labels, "- "no punctuation lists, no explanation."- )- try:- _, msg = message_fit_in(form_message(system, question), self.chat_mdl.max_length)- ans = await self.chat_mdl.async_chat(msg[0]["content"], msg[1:], {"temperature": 0.2})- if isinstance(ans, tuple):- ans = ans[0]- ans = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL).strip()- except Exception:- logging.exception("extract_keywords failed")- ans = ""- return ans or question+ _weighted, union = await self._extract_keywords_weighted(question)+ return _compact_keywords(union)async def retrieve(self,@@ -537,7 +603,8 @@ class RAGTools:return {"chunks": [], "doc_aggs": []}if doc_scope:candidates = [d for d in doc_scope if isinstance(d, str)]- known = await thread_pool_exec(self._filter_known_doc_ids, candidates)+ # peewee MySQL lookup — call directly to reuse the pool's connection.+ known = self._filter_known_doc_ids(candidates)valid = [d for d in candidates if d in known]if valid:doc_scope = valid@@ -666,7 +733,8 @@ class RAGTools:return {"chunks": [], "doc_aggs": []}if self.doc_scope is not None and doc_id not in self.doc_scope:return {"chunks": [], "doc_aggs": []}- resolved = await thread_pool_exec(self._resolve_doc_tenant, doc_id)+ # peewee MySQL lookup — call directly to reuse the pool's connection.+ resolved = self._resolve_doc_tenant(doc_id)if resolved is None:logging.warning(f"fetch_full_document: doc_id {doc_id!r} not in any bound KB — refusing to fetch")return {"chunks": [], "doc_aggs": []}@@ -733,18 +801,43 @@ class RAGTools:# the cache (their content is appended to the question message below).if question and not self.text_attachments_content:qk = _question_keywords(question)- if self._rag_cache:+ # If the last research round was not sufficient, do not reuse a cached+ # (similarly-worded) answer — the outer loop asked again because it+ # needs more evidence, so re-run the graph instead of returning the+ # same incomplete answer (Google "iteration" phase).+ last_status = ""+ v = getattr(self, "_rag_verdict", None)+ if isinstance(v, dict):+ last_status = str(v.get("status") or "")+ _cache_ok = not last_status or last_status == "SUFFICIENT"+ if _cache_ok and self._rag_cache:for cached_q, (cached_answer, cached_gram) in list(self._rag_cache.items()):if cached_gram and _cache_similar(qk, cached_gram):shared = len(qk[0] & cached_gram[0])_LOG.info("[Agentic RAG] Cache hit — reused prior answer for near-identical question %r (%d shared words); skipped research.", question, shared)return cached_answer- messages = [{"role": "user", "content": question}] if question else []+ # Prefer the user's ORIGINAL, complete question over the outer model's+ # rewritten `question` argument (same-turn only). See+ # `_resolve_effective_question` for the rationale — the outer rewrite+ # drops the FINAL target of multi-hop questions (Q317 purchaser death,+ # Q305 "shortest abbreviation").+ effective_q = _resolve_effective_question(question or "", self.original_user_question)+ if effective_q and effective_q != question:+ _LOG.info(+ "[Agentic RAG] Using original user question over outer rewrite (original=%r → rewrite=%r)",+ self.original_user_question[:80],+ (question or "")[:80],+ )+ messages = [{"role": "user", "content": effective_q}] if effective_q else []if self.text_attachments_content and messages:messages[-1]["content"] += self.text_attachments_content++ # Run the inner research executor (the static LangGraph pipeline).+ research = run_agentic_rag(self, messages)+final = ""- async for kind, delta in _split_think_stream(run_agentic_rag(self, messages)):+ async for kind, delta in _split_think_stream(research):if kind == "answer":final += deltaif self.answer_sink is not None:@@ -754,6 +847,46 @@ class RAGTools:# Cache the freshly produced answer for later near-identical questions.if question and final and not self.text_attachments_content:self._rag_cache[question] = (final, _question_keywords(question))++ # Expose the sufficiency verdict to the outer LLM so it can decide whether+ # to re-run `rag` from the reported gaps (Google "missing pieces" feedback)+ # instead of guessing from the answer text. This is appended to the tool+ # result — it does NOT change the final answer (the graph's formalize_answer+ # composes that independently from kbinfos).+ #+ # Outer-loop guardrail: bound useless re-runs. If consecutive rag calls keep+ # ending insufficient (UNANSWERABLE / INSUFFICIENT / CONFLICTING), tell the+ # outer LLM to STOP re-running and answer from the existing evidence — the+ # corpus likely lacks the data, so re-querying different angles won't help+ # and only inflates latency / risks timeout.+ verdict = getattr(self, "_rag_verdict", None)+ insufficient_statuses = {"UNANSWERABLE", "INSUFFICIENT", "CONFLICTING"}+ if isinstance(verdict, dict):+ status = verdict.get("status")+ missing = verdict.get("missing_claims") or []+ feedback = verdict.get("feedback") or ""+ hard = verdict.get("hard_violations") or []+ conf = verdict.get("agent_confidence")+ if status and status != "SUFFICIENT":+ status_hint = {+ "USEFUL_BUT_INCOMPLETE": "evidence is partially sufficient (gaps remain)",+ "INSUFFICIENT": "evidence is not yet sufficient",+ "CONFLICTING": "evidence contains conflicts",+ }.get(status, f"sufficiency status: {status}")+ missing_txt = ("; missing: " + "; ".join(missing[:3])) if missing else ""+ hard_txt = ("; hard gaps: " + ", ".join(str(x) for x in hard[:3])) if hard else ""+ conf_txt = f"; agent confidence: {conf:.2f}" if isinstance(conf, (int, float)) else ""+ fb_txt = f"; feedback: {feedback[:200]}" if feedback else ""+ # Guardrail counter.+ if status in insufficient_statuses:+ self._consecutive_unanswerable += 1+ else:+ self._consecutive_unanswerable = 0+ _GUA = 2 # consecutive-insufficient threshold+ if self._consecutive_unanswerable >= _GUA:+ final = f"{final}\n\n[Research status] {status_hint}{missing_txt}{hard_txt}{conf_txt}{fb_txt}. STOP calling rag again: {self._consecutive_unanswerable} consecutive research rounds returned insufficient evidence. The sources likely lack the required data. Give your best answer from the evidence already gathered; do not re-run rag."+ else:+ final = f"{final}\n\n[Research status] {status_hint}{missing_txt}{hard_txt}{conf_txt}{fb_txt}. If these gaps are material, call rag again with a question focused on them."call_stats.log()return final@@ -792,7 +925,8 @@ class RAGTools:if not self.kb_ids:self._metas_cache = {}return self._metas_cache- self._metas_cache = await thread_pool_exec(DocMetadataService.get_flatted_meta_by_kbs, self.kb_ids)+ # peewee MySQL lookup — call directly to reuse the pool's connection.+ self._metas_cache = DocMetadataService.get_flatted_meta_by_kbs(self.kb_ids)return self._metas_cache or {}def _collect_doc_titles(self, max_docs: int = 512) -> list[tuple[str, str]] | None:diff --git a/rag/advanced_rag/agentic_rag_graph.py b/rag/advanced_rag/agentic_rag_graph.pyindex d00d529a66..1fb084c373 100644--- a/rag/advanced_rag/agentic_rag_graph.py+++ b/rag/advanced_rag/agentic_rag_graph.py@@ -14,19 +14,38 @@# limitations under the License.#-"""LangGraph agentic-search graph — 4 nodes.--Architecture:-- formalize_question → route → planner → orchestrator_loop → formalize_answer--The ``orchestrator_loop`` node internally dispatches to one of three execution-strategies based on the thinking mode:-- low: direct_search — single hybrid search, no decomposition- medium: decompose_and_search — decompose → parallel search → sufficiency- high: agentic_research — two-level loop (orchestrator + research agent)- ultra: deep_research — same as high + dynamic claim expansion + replan+"""LangGraph agentic-search graph — Agentic RAG alignment.++Agentic RAG sequence this graph implements:++ Question+ │+ ▼+ Planner Agent (Phase 1 — fan-out expansion)+ │ query fanouts+ ▼+ RAG Agent / Search Fanout (Phase 2 — ALL fanouts researched+ │ AT ONCE: one programmatic+ ▼ pre-fetch + one native tool-calling+ Search results + drafts research pass whose first instruction+ │ carries the full fan-out checklist)+ ▼+ Sufficient Context Agent (Phase 3 — reviews ORIGINAL question ++ │ suff / insuff retrieved snippets + intermediate draft,+ ▼ emits missing-pieces Reason/Feedback)+ Synthesis Gap analisis + feedback+ │ (Phase 4 — Query Rewriter targets the+ ▼ gap, pre-fetches new evidence into+ RAG Agent kbinfos AND hands the same requests to+ (loop until sufficient the RAG Agent for a fresh research pass)+ or budget exhausted)+ │ (Phase 5 — clean synthesis of the final+ ▼ answer from the approved draft + snippets)+ Synthesis Agent++All five phases live in ONE compiled graph for medium / high / ultra+thinking modes (``build_agentic_graph``). ``low`` keeps the lightweight+direct-search graph (``build_low_graph``)."""from __future__ import annotations@@ -35,38 +54,30 @@ import asyncioimport jsonimport loggingimport re-from typing import Any, TypedDict+import time+from typing import Annotated, Any, TypedDictfrom langgraph.graph import END, START, StateGraph+from langgraph.graph.message import add_messagesfrom rag.advanced_rag.harness.stats import in_phasefrom rag.prompts.generator import form_message, kb_prompt, message_fit_in_LOG = logging.getLogger(__name__)-_ANSWER_TARGET_SYSTEM = """You are an answer-target verifier for a multi-hop RAG system.-Resolve what entity, value, or fact the user's question is asking for.-Do not choose bridge entities that merely explain a clue. In inverse-relation-questions, a later clue may identify the target's partner, relative, employer,-team, or source; keep tracing until the requested answer role is satisfied.-Return JSON only."""--_ANSWER_TARGET_USER = """Question:-{question}--Research summary:-{pre_summary}--Evidence:-{evidence}--Return JSON:-{{- "target_role": "the role the final answer must satisfy",- "must_satisfy": ["short evidence-backed condition the answer must meet"],- "bridge_entities": ["entities that are useful clues but should not be the answer unless they also satisfy target_role"],- "reason": "one short explanation of the answer shape"-}}"""+# ── Global research budget & per-call timeouts (source-level switches) ──+#+# The benchmark client cuts a request at 300s (read timeout). These bounds keep+# one question's WHOLE pipeline comfortably under that line: when the budget+# runs out the routing guards steer the graph straight to synthesis with+# whatever evidence is on hand instead of starting another research round.+_TOTAL_BUDGET_S = 180.0 # whole-graph wall-clock ceiling per question+_MIN_ROUND_HEADROOM_S = 50.0 # need at least this much left to start a new round+_PASS_TIMEOUT_S = 120.0 # slot research pass wall-clock+_PREFETCH_TIMEOUT_S = 90.0 # programmatic fan-out fetch+_DRAFT_TIMEOUT_S = 60.0 # fallback draft synthesis+_SCA_TIMEOUT_S = 60.0 # sufficient-context review call+_REWRITE_TIMEOUT_S = 45.0 # gap → query rewrite calldef _snip(value: Any, limit: int = 240) -> str:@@ -80,96 +91,126 @@ def _snip(value: Any, limit: int = 240) -> str:return s-def _extract_json(text: str) -> dict:- text = re.sub(r"^.*</think>", "", text or "", flags=re.DOTALL).strip()- text = re.sub(r"```(?:json)?\s*|\s*```", "", text).strip()- try:- import json_repair+def _safe_list(value, what: str) -> list:+ """Coerce a possibly-poisoned graph-state field into a plain list.- return json_repair.loads(text)- except Exception:- try:- return json.loads(text)- except Exception:- return {}+ A 'coroutine' object has been observed reaching node payloads through+ graph-state fields under production concurrency. Instead of crashing the+ whole question three nodes later with an opaque iterable error, log WHO+ created it — a coroutine's ``repr`` contains its function name — and+ degrade to empty.+ """+ if value is None:+ return []+ r = repr(value)[:200]+ if isinstance(value, (list, tuple)):+ return list(value)+ if asyncio.iscoroutine(value):+ _LOG.error("[StateGuard] %s is a raw COROUTINE — creator: %s", what, r)+ else:+ _LOG.warning("[StateGuard] %s has unexpected type %s (%s); treating as empty", what, type(value).__name__, r)+ return []+++def _is_poisoned(value) -> bool:+ """True when a state value should be discarded before it poisons LLM prompts."""+ return asyncio.iscoroutine(value)+++def _select_sca_view(chunks: list, focus_terms: list[str], cap: int | None = None) -> tuple[list, str]:+ """Rank the stored pool down to the SCA review view (storage ≠ review).++ Score = retrieval relevance (similarity) + surface-term coverage against+ the question + current gap queries + freshness bonus for chunks admitted+ in later rounds. Returns ``(view, identity)`` where ``identity`` is a+ stable hash of the selected chunk ids — the caller uses it to detect an+ unproductive round (same view twice despite new storage ⇒ nothing new to+ look at).+ """+ from rag.advanced_rag.harness.tools.search import _chunk_id+ capped = cap or _SCA_VIEW_CAP+ terms = [t.lower() for t in dict.fromkeys(focus_terms or []) if len(t) >= 3]-def _compact_text(text: str, limit: int) -> str:- text = text or ""- if len(text) <= limit:- return text- return text[:limit] + f"\n...(+{len(text) - limit} chars)"+ def _score(i: int, c: dict) -> float:+ text = " ".join(str(c.get(k, "")) for k in ("content", "content_with_weight", "title", "question_toks")).lower()+ cov = sum(1 for t in terms if t in text)+ cov_ratio = cov / len(terms) if terms else 0.5+ rel = float(c.get("similarity") or c.get("score") or 0.0)+ fresh = min(i / 20.0, 0.2) # late arrivals (gap-pursuit evidence) get seen+ return rel * 0.45 + min(cov_ratio, 1.0) * 0.45 + fresh+ ranked = sorted(list(enumerate(chunks)), key=lambda ic: _score(*ic), reverse=True)+ view = [c for _, c in ranked[:capped]]+ ident = "|".join(sorted((_chunk_id(c) or "") for c in view))+ return view, str(hash(ident))-def _string_items(value) -> list[str]:- if isinstance(value, str):- return [value.strip()] if value.strip() else []- if not isinstance(value, list):- return []- return [str(v).strip() for v in value if str(v).strip()]+def _view_terms(state: AgenticState) -> list[str]:+ """Terms describing what the SCA should look FOR this round."""+ from rag.advanced_rag.harness.tools.search import _query_to_terms++ terms = _query_to_terms(state.get("question") or "")+ for q in _safe_list(state.get("current_queries"), "state.current_queries"):+ if isinstance(q, str):+ terms.extend(_query_to_terms(q))+ return list(dict.fromkeys(terms))-def _format_answer_target_contract(contract: dict) -> str:- target_role = str(contract.get("target_role") or "").strip()- reason = str(contract.get("reason") or "").strip()- lines = []- if target_role:- lines.append(f"Final answer must satisfy: {target_role}")- must = _string_items(contract.get("must_satisfy"))- if must:- lines.append("Must meet these conditions:")- lines.extend(f"- {item}" for item in must)- bridges = _string_items(contract.get("bridge_entities"))- if bridges:- lines.append("Bridge entities to verify but not answer with:")- lines.extend(f"- {item}" for item in bridges)- if reason:- lines.append(f"Reasoning guard: {reason}")- lines.append("Do not answer with an intermediate clue entity unless it also satisfies the final answer role.")- return "\n".join(lines)+def _remaining_s(state: dict) -> float:+ """Seconds left in the question's global research budget (huge if unset)."""+ dl = state.get("deadline") or 0.0+ return dl - time.monotonic() if dl else float("inf")-async def _answer_target_contract(tools, question: str, kbinfos: dict, evidence: str) -> str:- """Build a compact guardrail that tells final synthesis what to answer."""- fallback = "Final answer must directly satisfy the user's top-level who/what request. Use bridge entities only as clues, and verify any proposed answer against the evidence."- pre_summary = kbinfos.get("pre_summary") or ""- if not question or not pre_summary:- return fallback++async def _bounded(coro, timeout_s: float, what: str):+ """Await ``coro`` under a wall-clock bound; log & surface None on expiry."""try:- user = _ANSWER_TARGET_USER.format(- question=question,- pre_summary=_compact_text(pre_summary, 2400),- evidence=_compact_text(evidence, 6000),- )- msg = await tools._fit_messages(_ANSWER_TARGET_SYSTEM, user)- ans = await tools.chat_mdl.async_chat(msg[0]["content"], msg[1:], {"temperature": 0.0})- if isinstance(ans, tuple):- ans = ans[0]- parsed = _extract_json(ans)- formatted = _format_answer_target_contract(parsed) if parsed else ""- return formatted or fallback- except Exception:- _LOG.exception("[Composing the answer] Failed to build answer-target contract")- return fallback+ async with asyncio.timeout(timeout_s):+ return await coro+ except TimeoutError:+ _LOG.warning("[Budget] %s exceeded %.0fs — moving on without it", what, timeout_s)+ return Noneclass AgenticState(TypedDict, total=False):- messages: list+ # ── conversation input ──+ messages: list # raw conversation; read once by formalize_questionquestion: strkeywords: str # search keywords + close synonyms for the formalized question- seed_chunks: list # preliminary hybrid_search chunks used to ground the plan- route: dict # RouteDecision serialized- plan: dict # WorkflowPlan serialized- claims: list # ClaimTarget[] serialized- kbinfos: dict # accumulated chunks & doc_aggs- verdict: dict # SufficiencyVerdict serialized++ # ── evolving research state ──+ plan: dict # {"fanouts": [...] } planner output (Phase 1)+ current_queries: list # active research targets (fanouts, then rewritten gap queries)++ # outer draft/SCA/rewrite stages can validate and close unresolved slots.+ slot_table: dict # {"slots": [{id,type,question_clues,discovered_clues,candidate,candidate_strength}], "answer_variable": int}+ slot_draft: str # slot-rendered fact draft fed to the SCA (structured view)+ collected_answer: str # non-terminal <answer> candidate surfaced for SCA validation+ unresolved_slots: list # [{id,type,question_clues,discovered_clues}] still unfilled after the tree round+ # Append-only accumulation of Phase-4 feedback instructions handed to the+ # RAG Agent between passes. Annotated with ``add_messages`` so each+ # query_rewrite invocation may return JUST its new instruction(s); the seed+ # instruction itself is deterministic (question + fan-outs) and therefore+ # rebuilt per pass rather than round-tripped through the reducer.+ research_feedback: Annotated[list, add_messages]+ kbinfos: dict # accumulated chunks & doc_aggs (+ "pre_summary" after the draft node)+ draft: str # intermediate fact-preserving draft (Phase 3 reviewee)+ rag_answer: str # latest RAG-agent research reportpartial_answer: boolabstain: boolempty_result: bool- final_answer: str- loop: int+ verdict: dict # SufficiencyVerdict serialized+ sca: dict # raw SCA payload (its gaps feed the Query Rewriter)++ # ── budgets & counters ──max_loops: int- feedback: str # replanning feedback+ deadline: float # time.monotonic() stamp when research budget expires+ search_rounds: int # completed SCA→query_rewrite iterations+ sca_view_id: str # identity hash of the last SCA review view (unproductive-round detector)+ attempted: list # [{"q": query, "r": round, "new": n_new}] — every issued search + outcome (rewriter context)+ fills_found: bool # research-tree mode: at least one slot got a confident fill+ no_progress: bool # last iteration produced neither gaps nor queries# ── Think tag helpers ──@@ -246,232 +287,1119 @@ async def _split_think_stream(stream):yield ("think" if in_think else "answer"), re.sub(r"</?think>", "", buf)-# ── Graph construction ──+# ── Agentic RAG helpers ──++def _sca_gaps_to_rewrite(sca: dict) -> list[tuple[str, str]]:+ """Extract the Query-Rewriter gaps from a Sufficient Context Agent verdict.-def _merge_result_into_kbinfos(tools, result: dict) -> None:- """Merge a search result's chunks/doc_aggs into ``tools.kbinfos``, deduped.+ Preference order:+ 1. Unsatisfied ``sub_queries`` (the precise "what is missing / where to+ search next" signal from Q-CARE) — ``(missing_fact, search_hint)``.+ 2. Per-claim ``missing_information`` items — ``(what, search_hint)``.- Mirrors the orchestrators' merge so seed evidence and orchestrator evidence- share one deduplicated pool.+ Returns ``[(what, search_hint), ...]`` (deduped, non-empty). Empty when the+ verdict carries no concrete gap (caller then accepts the draft)."""- if not result or not result.get("chunks"):- return- kb = tools.kbinfos- seen = {c.get("chunk_id") or c.get("id") or id(c) for c in kb.get("chunks", [])}- for c in result.get("chunks", []):- k = c.get("chunk_id") or c.get("id") or id(c)- if k in seen:+ gaps: list[tuple[str, str]] = []+ seen: set[str] = set()++ def _add(what: str, hint: str) -> None:+ what = str(what or "").strip()+ hint = str(hint or "").strip()+ if not what and not hint:+ return+ key = f"{what}|{hint}"+ if key in seen:+ return+ seen.add(key)+ gaps.append((what or hint, hint or what))++ for sq in sca.get("sub_queries") or []:+ if not isinstance(sq, dict):continue- seen.add(k)- kb.setdefault("chunks", []).append(c)- dseen = {d.get("doc_id") for d in kb.get("doc_aggs", [])}- for d in result.get("doc_aggs", []):- if d.get("doc_id") in dseen:+ if sq.get("satisfied"):continue- dseen.add(d.get("doc_id"))- kb.setdefault("doc_aggs", []).append(d)+ _add(sq.get("missing_fact") or sq.get("sub_query") or "", sq.get("search_hint") or "")+ if gaps:+ return gaps[:8]+ for g in (sca.get("claims") or {}).values():+ for mi in g.get("missing_information") or []:+ if isinstance(mi, dict):+ _add(mi.get("what") or "", mi.get("search_hint") or "")+ elif mi:+ _add(str(mi), "")+ return gaps[:8]+++_FANOUT_PROMPT = (+ "Break the user's question into 2 to 5 independent, directly searchable sub-questions "+ "(fan-outs). Each must be self-contained enough to retrieve relevant passages from a "+ "document corpus on its own. For multi-hop questions, produce ONLY the first-hop "+ "sub-questions needed to start (the anchor facts); do not invent downstream hops that "+ "depend on answers you do not have yet.\n"+ 'Respond with a JSON object: {"fanouts": ["...", "..."]}. No prose, JSON only.'+)+++def _extract_json_object(text: str):+ """Return the first parseable JSON object found in ``text``.++ The fan-out model sometimes emits prose before/after the object or even a second+ JSON object; a greedy brace-match then captures multiple objects and ``json.loads``+ fails with "Extra data". We brace-match to isolate the FIRST complete object and+ only return it if it parses; otherwise try the next opening brace.+ Returns ``None`` when no object parses.+ """+ i = 0+ n = len(text)+ while i < n:+ start = text.find("{", i)+ if start < 0:+ return None+ depth = 0+ for j in range(start, n):+ ch = text[j]+ if ch == "{":+ depth += 1+ elif ch == "}":+ depth -= 1+ if depth == 0:+ candidate = text[start : j + 1]+ try:+ return json.loads(candidate)+ except Exception:+ break # not a valid object; try the next "{"+ i = start + 1+ return None+++async def _expand_fanouts(tools, question: str, answer_conf: dict) -> list[str]:+ """Phase-1 fan-out expansion: break the question into searchable sub-questions.++ Uses ONE chat call (no tools). Returns 2-5 first-hop fan-outs; falls back to the+ raw question alone on any failure — a fan-out failure never blocks the pipeline.+ """+ try:+ from rag.advanced_rag.harness.tools.search import _base_chat_mdl+ except Exception:+ _LOG.warning("[rag_agent] could not import _base_chat_mdl for fan-out expansion", exc_info=True)+ return [question] if question else []+ try:+ mdl = _base_chat_mdl(tools)+ except Exception:+ _LOG.warning("[rag_agent] could not resolve base chat model for fan-out expansion", exc_info=True)+ return [question] if question else []+ if mdl is None or not question:+ return [question] if question else []+ try:+ ans, _ = await mdl.async_chat(+ _FANOUT_PROMPT,+ [{"role": "user", "content": f"Question: {question}"}],+ dict(answer_conf or {}),+ )+ text = str(ans or "")+ data = _extract_json_object(text)+ if data is not None:+ fanouts = [str(f).strip() for f in (data.get("fanouts") or []) if str(f).strip()]+ else:+ # Fallback: split on line items from a loose answer.+ fanouts = [ln.strip("-•0123456789. ").strip() for ln in text.splitlines() if ln.strip()]+ fanouts = list(dict.fromkeys(fanouts))[:5]+ _LOG.info("[Planner] fan-out expansion: %d sub-question(s): %s", len(fanouts), fanouts)+ return fanouts or ([question] if question else [])+ except Exception:+ _LOG.warning("[Planner] fan-out expansion failed; falling back to raw question", exc_info=True)+ return [question] if question else []+++# Storage ceiling of the snippet pool across ALL rounds. Storage and REVIEW+# are decoupled: the SCA only reads a ranked view (``_SCA_VIEW_CAP``), so the+# pool may accumulate freely while prompts stay bounded. The former single+# 30-chunk ceiling starved every enrichment channel dead (observed: Drill+# admitted 0 for entire runs because prefetch filled the pool first).+_MAX_SNIPPET_POOL = 60+_DRILL_RESERVE = 12 # slots kept free after the FIRST prefetch so the research+# executor can top up evidence+_SCA_VIEW_CAP = 24 # chunks shown to the Sufficient Context Agent per review++async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: int | None = None) -> int:+ """Programmatic fan-out retrieval: fetch all queries and fill ``kbinfos``.-def build_agentic_graph(tools, token_queue: asyncio.Queue, gen_conf: dict | None = None):- """Compile the 4-node agentic-search graph."""+ Phase 2 ("The RAG Agent searches ... all the query fanouts at once")+ for the SNIPPET pool the Sufficient Context Agent reviews (Phase 3 reads+ actual retrieved text). Two collector channels run per query:++ * Channel A (BM25 + ``narrow_by_terms``): exact-name recall, rewritten to a+ short match window.+ * Channel B (hybrid vector+BM25, **narrow bypass**): semantic hits whose+ text does NOT share surface words with the query. The 2026-08-27 hybrid+ experiment proved these blocks are valuable but are ALL filtered out by the keyword narrow step while+ crowding candidates out of top_n — so they now bypass it entirely.++ Called after planning and again after every Query-Rewriter pass. Returns+ how many NEW chunks were added; failures are swallowed per-query.+ """+ try:+ from rag.advanced_rag.harness.grep_sed_narrow import narrow_by_terms+ from rag.advanced_rag.harness.memory import _STOPWORDS as _FANOUT_STOPWORDS+ from rag.advanced_rag.harness.tools.search import (+ _chunk_id,+ _get_kb_ids,+ _query_to_terms,+ bm25_search,+ hybrid_search,+ )+ except Exception:+ _LOG.warning("[rag_agent] could not import fan-out search helpers", exc_info=True)+ return 0++ kbinfos = getattr(tools, "kbinfos", None)+ if kbinfos is None:+ kbinfos = {"chunks": [], "doc_aggs": []}+ tools.kbinfos = kbinfos+ seen = {_chunk_id(c) for c in kbinfos.get("chunks", [])}+ kb_ids = _get_kb_ids(tools) or None++ # Input-shape guard: this node runs several graph rounds in, fed from+ # planner output AND rewriter output. A malformed upstream value must never+ # crash the whole question (observed: a 'coroutine' object is not iterable+ # here when a stale byte-code path slipped a coroutine-shaped argument in).+ if isinstance(fanouts, str):+ fanouts = [fanouts]+ elif fanouts is None:+ return 0+ elif isinstance(fanouts, (list, tuple, set)):+ try:+ fanouts = [f for f in fanouts if isinstance(f, str)]+ except Exception:+ return 0+ else:+ _LOG.warning("[Prefetch] unexpected queries payload of type %s; skipping fan-out search", type(fanouts).__name__)+ return 0++ async def _search_one(fq: str) -> tuple[list, list]:+ # Each coroutine ONLY retrieves and returns chunk lists; it does NOT+ # touch ``kbinfos``. All mutation happens in the main coroutine below,+ # so concurrent fan-out searches cannot race on the shared list.+ # Channel A: exact-surface collector.+ try:+ res = await bm25_search(tools, fq, kb_ids=kb_ids, top_n=60)+ candidates = res.get("chunks", []) or []+ except Exception:+ _LOG.warning("[rag_agent] BM25 search failed for %r", fq, exc_info=True)+ candidates = []++ kept_a: list = []+ terms = _query_to_terms(fq)+ if candidates:+ try:+ # Feed the matcher CONCISE entity terms instead of the full sentence:+ # a long question buries the discriminating entities under stopwords,+ # so generic docs outrank the specific one. Numbers are kept.+ keyed = [t for t in terms if len(t) >= 3 and t.lower() not in _FANOUT_STOPWORDS and not t.isdigit() or (len(t) >= 4 and t.isdigit())]+ narrowed = narrow_by_terms(+ candidates,+ keyed or terms,+ fallback_terms=None,+ context={"before": 0, "after": 1},+ keywords=fq,+ max_out_chars_per_chunk=1200,+ max_out_total_chars=16000,+ )+ kept_a = (narrowed.get("kept", []) or [])[: max(1, top_n)]+ except Exception:+ _LOG.warning("[rag_agent] narrowing failed for %r; using raw BM25 head", fq, exc_info=True)+ kept_a = candidates[: max(1, top_n)]++ # Channel B: semantic collector, narrow BYPASS. Keep only hits that+ # Channel A did not already surface (dedup happens again at merge).+ kept_b: list = []+ seen_ids_a = {_chunk_id(c) for c in kept_a}+ try:+ hres = await hybrid_search(tools, fq, kb_ids=kb_ids, top_n=30)+ for c in hres.get("chunks", []) or []:+ if _chunk_id(c) in seen_ids_a:+ continue+ kept_b.append(c)+ if len(kept_b) >= 4: # modest semantic quota per query+ break+ except Exception:+ _LOG.warning("[rag_agent] hybrid channel failed for %r; skipping", fq, exc_info=True)+ return kept_a, kept_b++ # Search all fan-outs at once (parallel) — then merge into the shared pool.+ results = await asyncio.gather(*(_search_one(fq) for fq in fanouts), return_exceptions=True)++ # Cap the TOTAL evidence pool so the SCA never has to read unbounded chunks.+ # This is a GLOBAL ceiling across all rounds (planning prefetch + every+ # Query-Rewriter top-up) — a per-call cap would let the pool grow with each+ # insufficient round and bloat every subsequent SCA prompt (observed: 113+ # chunks after round 3). Once the pool is full, later calls add nothing and+ # the caller's zero-new-chunks early-exit can kick in.+ max_total = capacity or _MAX_SNIPPET_POOL+ room = max(0, max_total - len(seen))+ added = 0++ def _admit(batch) -> bool:+ nonlocal added+ if isinstance(batch, Exception) or not batch:+ return False+ stop = False+ for c in batch:+ if added >= room:+ stop = True+ break+ k = _chunk_id(c)+ if k and k in seen:+ continue+ if k:+ seen.add(k)+ kbinfos.setdefault("chunks", []).append(c)+ added += 1+ return stop++ # Channel A first (exact matches earn their slots), then semantic extras.+ for pair in results:+ if isinstance(pair, Exception):+ continue+ kept_a, kept_b = pair if isinstance(pair, tuple) else ([], [])+ if _admit(kept_a):+ break+ for pair in results:+ if isinstance(pair, Exception):+ continue+ _, kept_b = pair if isinstance(pair, tuple) else ([], [])+ if _admit(kept_b):+ break++ if room == 0:+ _LOG.info("[Prefetch] snippet pool FULL (%d chunks); nothing new admitted", max_total)+ return added+ _LOG.info("[Prefetch] fan-out channels added %d new chunk(s) (total %d)", added, len(kbinfos.get("chunks", [])))+ return added+++# Reused by build_low_graph — lightweight direct search node.+async def _direct_search_node_impl(state: AgenticState, tools) -> dict:+ from rag.advanced_rag.harness.orchestrator.direct import direct_search++ return await direct_search(state, tools)+++# ── Shared final-answer composition (Phase 5 Synthesis) ──+++async def _compose_answer_from_evidence(state: AgenticState, tools, token_queue: asyncio.Queue, answer_conf: dict) -> dict:+ """Compose the final grounded answer from the gathered evidence."""+ kbinfos = state.get("kbinfos") or {"chunks": [], "doc_aggs": []}+ question = state.get("question") or ""+ partial = state.get("partial_answer", False)+ abstain = state.get("abstain", False)+ empty_result = state.get("empty_result", False)++ _note = " — partial answer, some gaps remain" if partial else (" — not enough evidence to answer" if abstain else "")+ _LOG.info('[Composing the answer] Writing the final answer to "%s" from %d gathered passage(s)%s.', _snip(question), len(kbinfos["chunks"]), _note)++ tools.kbinfos = kbinfos++ no_evidence = abstain or empty_result or not kbinfos["chunks"]+ if no_evidence and getattr(tools, "empty_response", ""):+ _LOG.info("[Composing the answer] No supporting evidence was found; returning the configured empty response without calling the answer model.")+ token_queue.put_nowait(tools.empty_response)+ return {"final_answer": tools.empty_response}++ # Primary evidence is the fact-preserving ``pre_summary`` (the intermediate+ # draft reviewed by the SCA — it carries the exact numbers/entities). A small+ # set of top-similarity chunks stays as citation reference / fallback detail.+ pre_summary = kbinfos.get("pre_summary")+ all_chunks = kbinfos.get("chunks") or []+ ranked = sorted(+ (c for c in all_chunks),+ key=lambda c: float(c.get("similarity", 0.0) or c.get("score", 0.0) or 0.0),+ reverse=True,+ )+ from rag.advanced_rag.agentic_rag import _EVIDENCE_BUDGET_TOKENS++ _CITE_CHUNK_CAP = 6+ cite_chunks = ranked[:_CITE_CHUNK_CAP] or all_chunks+ evidence_kbinfos = dict(kbinfos, chunks=cite_chunks)+ evidence_blocks = kb_prompt(evidence_kbinfos, min(tools.chat_mdl.max_length, _EVIDENCE_BUDGET_TOKENS))+ evidence = "\n".join(evidence_blocks) if isinstance(evidence_blocks, list) else str(evidence_blocks)++ parts = [f"Question:\n{question}\n"]++ # Static answer-target guardrail (no extra LLM call). Keeps the essential+ # instruction: satisfy the top-level request, treat bridge entities as clues.+ parts.append(+ "Answer Target Contract:\n"+ "Final answer must directly satisfy the user's top-level who/what request. "+ "Use bridge entities only as clues, and verify any proposed answer against "+ "the evidence. In EXTREME-SELECTION questions (shortest/longest/smallest/"+ "largest/most/least/最), compare the alternatives in the evidence and name "+ "the EXTREME one rather than the most common or first-listed.\n"+ )++ if no_evidence:+ if pre_summary:+ parts.append(+ "The retrieved passages are limited. Answer as completely as possible "+ "from the Research Summary below, using the known facts; where a "+ "specific number/entity is missing, say what is known and avoid "+ "flatly refusing to answer.\n"+ )+ else:+ parts.append("No supporting evidence was retrieved. State clearly that the available sources are insufficient, and do not answer from general knowledge.\n")++ if pre_summary:+ parts.append(f"Research Summary (primary evidence):\n{pre_summary}\n")++ if partial:+ from rag.advanced_rag.harness.prompts.report_prompt import PARTIAL_ANSWER_PREAMBLE++ parts.append(f"{PARTIAL_ANSWER_PREAMBLE}\n")++ from rag.advanced_rag.harness.prompts.report_prompt import FINAL_ANSWER_SYSTEM+ from rag.prompts.generator import citation_prompt as cp++ rules = cp(tools.user_defined_prompts).strip()+ system = FINAL_ANSWER_SYSTEM.format(cite_rules=rules)+ # Honor the dialog-level system prompt (UI-configured) the same way the+ # reasoning-disabled path does — merge from upstream/main.+ if getattr(tools, "system_prompt", "").strip():+ system = f"{tools.system_prompt.strip()}\n\n{system}"++ parts.append(f"Evidence:\n{evidence}")+ user_content = "\n".join(parts)++ _LOG.info(+ "[Formalize][pre_summary] question=%r pre_summary_len=%d evidence_len=%d\npre_summary=%r",+ question[:160],+ len(pre_summary or ""),+ len(evidence or ""),+ (pre_summary or "")[:3000],+ )++ _, msg = message_fit_in(form_message(system, user_content), min(tools.chat_mdl.max_length, _EVIDENCE_BUDGET_TOKENS))+ try:+ async for tok in tools.chat_mdl.async_chat_streamly_delta(msg[0]["content"], msg[1:], answer_conf):+ token_queue.put_nowait(tok)+ except Exception:+ _LOG.exception("formalize_answer: stream failed")+ token_queue.put_nowait("I'm sorry, I encountered an error while composing the answer.")++ return {"final_answer": ""}+++# ── Graph construction ──+++def build_low_graph(tools, token_queue: asyncio.Queue, gen_conf: dict | None = None):+ """Compile the lightweight low-mode graph: formalize → direct_search → answer."""answer_conf = dict(gen_conf) if gen_conf else {"temperature": 0.3}- # ── Node: formalize_question ──@in_phase("formalize")async def formalize_question(state: AgenticState) -> dict:msgs = state.get("messages") or []- _LOG.info("[Formalizing the question] Reading the conversation (%d message(s)) to work out the standalone question...", len(msgs))+ _LOG.info("[Formalizing the question] Reading the conversation (%d message(s))...", len(msgs))q, kw = await tools.formalize(msgs)q = (q or "").strip()kw = (kw or "").strip()- _LOG.info('[Formalizing the question] Understood the question as: "%s" — searching with keywords: %s', _snip(q), _snip(kw))return {"question": q,"keywords": kw,"kbinfos": {"chunks": [], "doc_aggs": []},- "loop": 0,"partial_answer": False,"abstain": False,}- # ── Node: route ──- @in_phase("route")- async def route(state: AgenticState) -> dict:- from rag.advanced_rag.harness.route import route_node+ @in_phase("orchestrator")+ async def direct_search_node(state: AgenticState) -> dict:+ return await _direct_search_node_impl(state, tools)++ @in_phase("finalize")+ async def formalize_answer(state: AgenticState) -> dict:+ return await _compose_answer_from_evidence(state, tools, token_queue, answer_conf)++ g = StateGraph(AgenticState)+ g.add_node("formalize_question", formalize_question)+ g.add_node("direct_search", direct_search_node)+ g.add_node("formalize_answer", formalize_answer)++ g.add_edge(START, "formalize_question")+ g.add_edge("formalize_question", "direct_search")+ g.add_edge("direct_search", "formalize_answer")+ g.add_edge("formalize_answer", END)- return await route_node(state, tools)+ return g.compile()- # ── Node: pre_search ──- async def pre_search(state: AgenticState) -> dict:- """Preliminary hybrid_search to ground the planner's decomposition.- Only runs for decomposition modes (direct/low mode retrieves in- orchestrator_loop anyway, so we skip the duplicate search). The result- is narrowed by keywords inside ``hybrid_search`` and merged into the- shared citation pool so it also enriches the final answer.- """- route = state.get("route")- if not route or not getattr(route, "requires_decomposition", False):- _LOG.info("[Preliminary search] Skipping the first look — this question goes straight to a single search.")- return {"seed_chunks": []}+def build_agentic_graph(+ tools,+ token_queue: asyncio.Queue,+ gen_conf: dict | None = None,+ enable_sca: bool = False,+ use_fanout: bool = False,+):+ """Compile the slot-table-driven agentic-search graph (medium / high / ultra).- from rag.advanced_rag.harness.tools.search import hybrid_search+ * ``use_fanout`` adds Phase 1 (planner) + programmatic pre-fetch.+ * ``enable_sca`` wraps the research in the Phase 3-4 SCA↔Query-Rewriter+ iteration loop (bounded by ``_SCA_MAX_ROUNDS``).+ * The RAG-Agent research executor is slot-table-driven: the planner builds a+ slot table, and rag_agent runs one action_session per unresolved slot.- q = state.get("question", "")- kw = state.get("keywords", "")- _LOG.info('[Preliminary search] Taking a first look in the knowledge base for: "%s" (keywords: %s)', _snip(q), _snip(kw))- try:- result = await hybrid_search(tools, query=q, keywords=kw)- except Exception:- _LOG.exception("[Preliminary search] hybrid_search failed")- return {"seed_chunks": []}+ Structure::++ formalize_question → [planner → prefetch] → rag_agent → draft → sca+ ├─ sufficient ──────────────────────────────→ formalize_answer+ └─ insufficient → query_rewrite ────────────→ rag_agent (next round)+ """+ answer_conf = dict(gen_conf) if gen_conf else {"temperature": 0.3}+ # SCA iteration budget (Phase 4 loops). ultra is intentionally deeper:+ # more research rounds before giving up, at the cost of latency — this is+ # one of the two differentiators from high (the other is the ultra-only+ # graph_explore relational tool). high/medium stay at 3.+ _mode_label = str(getattr(tools, "thinking_mode", "") or "").lower()+ sca_max_rounds = 5 if _mode_label == "ultra" else 3- chunks = result.get("chunks", []) or []- _merge_result_into_kbinfos(tools, result)- _LOG.info("[Preliminary search] First look found %d passage(s); %d gathered so far.", len(chunks), len(tools.kbinfos.get("chunks", [])))- return {"seed_chunks": chunks}+ # ── Node: formalize_question ──+ @in_phase("formalize")+ async def formalize_question(state: AgenticState) -> dict:+ msgs = state.get("messages") or []+ _LOG.info("[Formalizing the question] Reading the conversation (%d message(s))...", len(msgs))+ q, kw = await tools.formalize(msgs)+ q = (q or "").strip()+ kw = (kw or "").strip()+ return {+ "question": q,+ "keywords": kw,+ "kbinfos": {"chunks": [], "doc_aggs": []},+ "partial_answer": False,+ "abstain": False,+ "empty_result": True,+ "current_queries": [],+ "research_feedback": [],+ "rag_answer": "",+ "draft": "",+ "search_rounds": 0,+ "sca_view_id": "",+ "no_progress": False,+ "deadline": time.monotonic() + _TOTAL_BUDGET_S,+ }- # ── Node: planner ──+ # ── Node: planner (Phase 1) ──@in_phase("planner")async def planner(state: AgenticState) -> dict:- from rag.advanced_rag.harness.planner import planner_node-- return await planner_node(state, tools)+ question = state.get("question") or ""+ _LOG.info("[Planner] Decomposing the question into first-hop fan-outs...")+ fanouts = await _expand_fanouts(tools, question, answer_conf)+ out = {"plan": {"fanouts": fanouts}, "current_queries": fanouts}+ # Build the slot table right after fan-out decomposition so the research+ # pass is slot-directed (each slot = one unknown to resolve, with+ # question_clues driving its action_session).+ root, first_queries = await _build_slot_table(+ tools,+ question,+ fanouts,+ answer_conf,+ lambda: _remaining_s(state) - 15.0,+ )+ out["slot_table"] = _slot_table_serialize(root)+ if first_queries:+ out["current_queries"] = [str(q) for q in first_queries]+ return out- # ── Node: orchestrator_loop ──+ # ── Node: prefetch (programmatic Phase 2 snippet pool for the SCA) ──@in_phase("orchestrator")- async def orchestrator_loop(state: AgenticState) -> dict:- from rag.advanced_rag.harness.orchestrator import orchestrator_loop as _run+ async def prefetch(state: AgenticState) -> dict:+ queries = state.get("current_queries") or ([state.get("question")] if state.get("question") else [])+ if not queries:+ return {}+ # Sync the shared pool from graph state before appending.+ tools.kbinfos = dict(getattr(tools, "kbinfos", None) or state.get("kbinfos") or {"chunks": [], "doc_aggs": []})+ t = min(_PREFETCH_TIMEOUT_S, max(10.0, _remaining_s(state) - _MIN_ROUND_HEADROOM_S))+ # First-round prefetch leaves drill slots free (see _DRILL_RESERVE).+ added = await _bounded(+ _fanout_search(tools, queries, capacity=_MAX_SNIPPET_POOL - _DRILL_RESERVE),+ t,+ "programmatic prefetch",+ )+ # Book the planning fanouts into the rewriter's research ledger.+ prev = [e for e in _safe_list(state.get("attempted"), "state.attempted") if isinstance(e, dict)]+ for q in queries:+ prev.append({"q": q, "r": 0, "new": int(added or 0)})+ return {"kbinfos": tools.kbinfos, "attempted": prev}++ # ── Node: rag_agent (Phases 2 & 4 — the retrieval researcher) ──+ # Runs slot-table-driven research: the planner built a slot table; here each+ # unresolved slot gets its own graph-edge action_session (retrieve /+ # search_chunks / list_chunks) that targets that slot. Slot patches fold+ # back into the table; a <answer> is surfaced as a candidate for the outer+ # SCA to validate instead of ending the question with a guess.+ @in_phase("dynamic")+ async def rag_agent(state: AgenticState) -> dict:+ question = state.get("question") or ""+ time_left = _remaining_s(state)+ if time_left < _MIN_ROUND_HEADROOM_S:+ _LOG.info("[RAGAgent] only %.0fs left of the research budget; skipping further passes.", time_left)+ return {}+ t = max(20.0, min(_PASS_TIMEOUT_S, time_left - 25.0))+ slot_result = await _bounded(+ _run_slot_research_pass(tools, question, state, answer_conf, deadline_left=t),+ t,+ "slot research pass",+ )+ if not slot_result:+ return {}+ _LOG.info(+ "[RAGAgent] pass %d — %d slot(s) filled, unresolved=%d",+ int(state.get("search_rounds", 0)) + 1,+ sum(1 for s in (slot_result.get("slot_table") or {}).get("slots", []) if s.get("candidate")),+ len(slot_result.get("unresolved_slots") or []),+ )+ return slot_result++ # ── Node: draft (intermediate draft — Phase 3 reviewee) ──+ @in_phase("draft")+ async def draft(state: AgenticState) -> dict:+ report = (state.get("rag_answer") or "").strip()+ kbinfos = dict(getattr(tools, "kbinfos", None) or state.get("kbinfos") or {"chunks": [], "doc_aggs": []})+ draft_text = report+ if not draft_text:+ # Budget exhaustion without a report: synthesize one from snippets.+ t = min(_DRAFT_TIMEOUT_S, max(15.0, _remaining_s(state) - 10.0))+ composed = await _bounded(+ _compose_fallback_draft(tools, state, answer_conf),+ t,+ "fallback draft synthesis",+ )+ draft_text = composed or ""+ if draft_text:+ kbinfos["pre_summary"] = draft_text+ tools.kbinfos = kbinfos+ _LOG.info("[Draft] intermediate draft %d chars (evidence=%d chunks)", len(draft_text or ""), len(kbinfos.get("chunks", [])))+ return {"draft": draft_text, "kbinfos": kbinfos}++ # ── Node: sca (Phase 3 — quality-control inspector) ──+ @in_phase("sca")+ async def sca(state: AgenticState) -> dict:+ from rag.advanced_rag.harness.orchestrator.sufficient_context import sufficient_context_agent++ question = state.get("question") or ""+ kbinfos = getattr(tools, "kbinfos", None)+ if not isinstance(kbinfos, dict):+ _LOG.error("[SCA] tools.kbinfos is %s (%r)", type(kbinfos).__name__, repr(kbinfos)[:120])+ kbinfos = {}+ chunks = [c for c in _safe_list(kbinfos.get("chunks"), "kbinfos.chunks") if isinstance(c, dict)]+ draft_text = (state.get("draft") or "").strip()+ if not isinstance(draft_text, str):+ _LOG.error("[SCA] state.draft is %s (%r)", type(draft_text).__name__, repr(draft_text)[:120])+ draft_text = ""++ # Storage ≠ review: rank the whole pool down to a gap-focused view.+ view, view_id = _select_sca_view(chunks, _view_terms(state))+ prev_id = str(state.get("sca_view_id") or "")+ if prev_id and view_id == prev_id:+ # Same evidence review twice despite new storage — further rounds+ # cannot change the verdict; stop instead of looping.+ _LOG.info("[SCA] review view UNCHANGED since last round (%d stored / %d viewed); closing out.", len(chunks), len(view))+ return {"verdict": {"status": "INSUFFICIENT"}, "sca": {}, "no_progress": True}++ # The SCA renders claim evidence from ``tools.kbinfos`` by INDEX, so swap+ # in a view-only copy for the duration of the review — the indexed ids+ # then point at exactly the selected chunks. Restore afterwards so other+ # nodes (drill/cite) keep seeing the full pool.+ orig_kbinfos = kbinfos+ tools.kbinfos = dict(kbinfos, chunks=view)+ try:+ if not draft_text and not view:+ _LOG.info("[SCA] nothing retrieved nor drafted; marking INSUFFICIENT to trigger a targeted re-search.")+ return {"verdict": {"status": "INSUFFICIENT"}, "sca": {}, "sca_view_id": view_id}+ sca_payload = await _bounded(+ sufficient_context_agent(+ tools,+ question,+ claims=[("c0", draft_text or "(no draft)", list(range(len(view))))],+ ),+ min(_SCA_TIMEOUT_S, max(15.0, _remaining_s(state) - 10.0)),+ "sufficient-context review",+ )+ finally:+ tools.kbinfos = orig_kbinfos+ if not sca_payload:+ _LOG.info("[SCA] unavailable; accepting current draft.")+ return {"verdict": {"status": "SUFFICIENT"}, "sca": {}, "sca_view_id": view_id}+ status = "SUFFICIENT" if sca_payload.get("is_sufficient") else "INSUFFICIENT"+ _LOG.info("[SCA] verdict=%s (confidence=%s; view=%d/%d)", status, sca_payload.get("confidence"), len(view), len(chunks))+ return {"verdict": {"status": status}, "sca": sca_payload, "sca_view_id": view_id}++ # ── Node: query_rewrite (Phase 4 — targeted gap pursuit) ──+ @in_phase("rewrite")+ async def query_rewrite(state: AgenticState) -> dict:+ from rag.advanced_rag.harness.orchestrator.query_rewriter import rewrite_gap_to_query- return await _run(state, tools)+ question = state.get("question") or ""+ sca_payload = state.get("sca")+ if not isinstance(sca_payload, dict):+ sca_payload = {}+ if _is_poisoned(state.get("sca")):+ _LOG.error("[QueryRewriter] state.sca arrived as %r — upstream node returned an un-awaited call!", repr(state.get("sca"))[:160])+ gaps = [g for g in _sca_gaps_to_rewrite(sca_payload) if isinstance(g, tuple)]+ if not gaps:+ _LOG.info("[QueryRewriter] SCA insufficient but no concrete gap; accepting the draft.")+ return {"no_progress": True}++ # Information-augmented rewriting (the Google-style lever): instead of+ # rule-based dedupe, give the rewriter FULL VISIBILITY — what was tried+ # (with outcomes), what the evidence pool currently holds — so it aims at+ # uncovered angles itself.+ ledger = [e for e in _safe_list(state.get("attempted"), "state.attempted") if isinstance(e, dict)]+ history_lines = []+ for e in ledger:+ q = str(e.get("q", ""))[:120]+ r = e.get("r", "?")+ new = e.get("new")+ outcome = "no new passages" if new == 0 else f"{new} new passage(s)"+ history_lines.append(f"- {q} (round {r}: {outcome})")+ kbinfos_now = getattr(tools, "kbinfos", None)+ pool_head_lines = []+ for c in ((kbinfos_now or {}).get("chunks") or [])[:12]:+ first_line = str(c.get("content") or c.get("content_with_weight") or "").split("\n")[0].strip()+ if first_line:+ pool_head_lines.append(f"- {first_line[:140]}")+ research_context_parts = []+ if history_lines:+ research_context_parts.append("Previously searched queries and their outcomes:\n" + "\n".join(history_lines))+ if pool_head_lines:+ research_context_parts.append("Evidence currently at hand (first lines of top stored snippets):\n" + "\n".join(pool_head_lines))+ research_context = "\n\n".join(research_context_parts)++ rewritten = await _bounded(+ rewrite_gap_to_query(tools, question, gaps, research_context=research_context),+ min(_REWRITE_TIMEOUT_S, max(10.0, _remaining_s(state) - 10.0)),+ "gap → query rewrite",+ )+ if rewritten is not None and not isinstance(rewritten, (list, tuple)):+ _LOG.error("[QueryRewriter] rewrite_gap_to_query returned non-list type %s (%r)", type(rewritten).__name__, repr(rewritten)[:160])+ rewritten = []+ queries = [str(q.get("query", "")).strip() for q in (rewritten or []) if isinstance(q, dict) and str(q.get("query", "")).strip()]+ # Fold the still-unresolved slots into the rewrite queries so the next+ # slot research pass targets exactly those unknowns (their question_clues+ # become the action_session directions). This makes an insufficient+ # verdict drive slot completion, not a blind re-search.+ for us in state.get("unresolved_slots") or []:+ if isinstance(us, dict):+ for qc in (us.get("question_clues") or [])[:2]:+ qc = str(qc).strip()+ if qc:+ queries.append(qc)+ queries = list(dict.fromkeys(q for q in queries if q))+ if not queries:+ _LOG.info("[QueryRewriter] no actionable query produced; accepting the draft.")+ return {"no_progress": True}++ # Dual-track pursuit of the gap, like the reference implementation:+ # (a) programmatically pre-fetch new snippets into the SCA pool;+ tools.kbinfos = dict(getattr(tools, "kbinfos", None) or {"chunks": [], "doc_aggs": []})+ added = await _fanout_search(tools, queries, top_n=6)+ # Retrieval saturation early-exit: if a rewrite round produced ZERO new+ # snippets (pool full / queries exhausted), further full research passes+ # just burn latency without new signal — accept the current draft.+ if added == 0 and int(state.get("search_rounds", 0)) >= 1:+ _LOG.info("[QueryRewriter] retrieval saturated (0 new chunks after another insufficient round); stopping iteration.")+ return {"no_progress": True, "current_queries": queries}+ # (b) the next slot research pass picks these up via the persisted+ # slot_table + unresolved_slots — no separate feedback channel needed.+ _LOG.info("[QueryRewriter] insufficient round %d → %d targeted query(s): %s", int(state.get("search_rounds", 0)) + 1, len(queries), queries)+ ledger = [e for e in _safe_list(state.get("attempted"), "state.attempted") if isinstance(e, dict)]+ for q in queries:+ ledger.append({"q": q, "r": int(state.get("search_rounds", 0)) + 1, "new": int(added or 0)})+ return {+ "no_progress": False,+ "current_queries": queries,+ "search_rounds": int(state.get("search_rounds", 0)) + 1,+ "attempted": ledger,+ "kbinfos": tools.kbinfos,+ }- # ── Node: formalize_answer ──+ # ── Node: formalize_answer (Phase 5 Synthesis) ──@in_phase("finalize")async def formalize_answer(state: AgenticState) -> dict:- kbinfos = state.get("kbinfos") or {"chunks": [], "doc_aggs": []}- question = state.get("question") or ""- partial = state.get("partial_answer", False)- abstain = state.get("abstain", False)- empty_result = state.get("empty_result", False)+ kbinfos = state.get("kbinfos") or (getattr(tools, "kbinfos", None) or {"chunks": [], "doc_aggs": []})+ if kbinfos.get("chunks"):+ tools.kbinfos = kbinfos+ state = dict(state)+ if bool(state.get("no_progress")) or str((state.get("verdict") or {}).get("status")) == "INSUFFICIENT":+ # All research attempts exhausted without a satisfying context —+ # surface the residual findings honestly instead of refusing.+ state["partial_answer"] = True+ return await _compose_answer_from_evidence(state, tools, token_queue, answer_conf)++ # ── Routing ──+ def _route_sca(state: AgenticState) -> str:+ if state.get("no_progress"):+ return "formalize_answer"+ if not enable_sca:+ # medium: single research pass — the SCA verdict is informational only.+ return "formalize_answer"+ if (+ str((state.get("verdict") or {}).get("status")) == "INSUFFICIENT"+ and int(state.get("search_rounds", 0)) < sca_max_rounds+ # Budget guard: starting another rewrite+research round only when+ # enough headroom remains for the whole round; otherwise close out.+ and _remaining_s(state) > _MIN_ROUND_HEADROOM_S+ ):+ return "query_rewrite"+ return "formalize_answer"++ def _route_rewrite(state: AgenticState) -> str:+ if state.get("no_progress"):+ return "formalize_answer"+ if int(state.get("search_rounds", 0)) >= sca_max_rounds:+ return "formalize_answer"+ if _remaining_s(state) <= _MIN_ROUND_HEADROOM_S:+ _LOG.info("[Routing] research budget nearly exhausted (%.0fs left); closing out with current evidence.", _remaining_s(state))+ return "formalize_answer"+ return "rag_agent"- _note = " — partial answer, some gaps remain" if partial else (" — not enough evidence to answer" if abstain else "")- _LOG.info('[Composing the answer] Writing the final answer to "%s" from %d gathered passage(s)%s.', _snip(question), len(kbinfos["chunks"]), _note)+ g = StateGraph(AgenticState)+ g.add_node("formalize_question", formalize_question)+ if use_fanout:+ g.add_node("planner", planner)+ g.add_node("prefetch", prefetch)+ g.add_node("rag_agent", rag_agent)+ g.add_node("draft", draft)+ g.add_node("sca", sca)+ g.add_node("query_rewrite", query_rewrite)+ g.add_node("formalize_answer", formalize_answer)- tools.kbinfos = kbinfos+ # Slot mode owns its first-hop queries (slot-table first_queries) — the+ # planning prefetch would duplicate them at full BM25+hybrid cost for ~20s+ use_prefetch = use_fanout- no_evidence = abstain or empty_result or not kbinfos["chunks"]- if no_evidence and tools.empty_response:- _LOG.info("[Composing the answer] No supporting evidence was found; returning the configured empty response without calling the answer model.")- token_queue.put_nowait(tools.empty_response)- return {"final_answer": tools.empty_response}-- # Build evidence — narrow the gathered chunks to keyword-bearing- # snippets BEFORE feeding the answer model, instead of dumping every- # full chunk into the prompt. The retriever already narrows per-query,- # but multi-search accumulation and the "keep-all-on-no-match" fallback- # can still leave many full passages here (one run produced a 46-passage- # / 70K-token call). Re-narrowing on the final question keeps each- # passage a snippet and bounds the prompt, while preserving originals if- # nothing matches (so we never answer with empty evidence).- #- # IMPORTANT: narrowing runs on a COPY of the chunk list — we never mutate- # ``kbinfos["chunks"]`` because that pool is also the main-agent citation- # reference; in-place narrowing there would shrink what the agent can cite- # and make it re-ask (more sub-questions). Also, a chunk whose original- # carried a number but whose narrowed snippet lost every digit is kept- # whole (a numeric answer sentence often lacks the keyword itself).- kw = state.get("keywords") or ""- from rag.advanced_rag.harness.orchestrator.sufficiency_llm import _narrow_snippet_safe-- kw_list = [k for k in re.split(r"[,\s]+", kw or "") if k]- evidence_chunks = []- for c in kbinfos.get("chunks") or []:- if not kw_list:- evidence_chunks.append(c)- continue- raw = c.get("content_with_weight") or c.get("text") or ""- # General informative-sentence guard (same policy as `_evidence_md`):- # keep the narrowed snippet, but fall back to the whole chunk when- # narrowing would drop every fact-bearing sentence (numbers / proper- # nouns / quotes) — the answer may be far from any keyword. Bounds- # the prompt while never losing critical evidence.- narrowed = _narrow_snippet_safe(raw, kw_list)- if narrowed:- evidence_chunks.append({**c, "content_with_weight": narrowed})- else:- evidence_chunks.append(c)- evidence_kbinfos = dict(kbinfos, chunks=evidence_chunks)- # Bounded evidence budget: the raw model context (``max_length``) is far- # too large — with a big pool it lets evidence fill the whole window- # (observed 141K tokens in one call). Use a fixed, modest budget so the- # answer model sees only a compact evidence set.- from rag.advanced_rag.agentic_rag import _EVIDENCE_BUDGET_TOKENS-- evidence_blocks = kb_prompt(evidence_kbinfos, min(tools.chat_mdl.max_length, _EVIDENCE_BUDGET_TOKENS))- evidence = "\n".join(evidence_blocks) if isinstance(evidence_blocks, list) else str(evidence_blocks)- parts = [f"Question:\n{question}\n"]-- answer_target = await _answer_target_contract(tools, question, kbinfos, evidence)- if answer_target:- parts.append(f"Answer Target Contract:\n{answer_target}\n")-- if no_evidence:- parts.append("No supporting evidence was retrieved. State clearly that the available sources are insufficient, and do not answer from general knowledge.\n")+ g.add_edge(START, "formalize_question")+ g.add_edge("formalize_question", "planner" if use_fanout else ("prefetch" if use_prefetch else "rag_agent"))+ if use_fanout:+ g.add_edge("planner", "prefetch" if use_prefetch else "rag_agent")+ if use_prefetch:+ g.add_edge("prefetch", "rag_agent")+ g.add_edge("rag_agent", "draft")+ g.add_edge("draft", "sca")+ g.add_conditional_edges("sca", _route_sca, {"query_rewrite": "query_rewrite", "formalize_answer": "formalize_answer"})+ g.add_conditional_edges("query_rewrite", _route_rewrite, {"rag_agent": "rag_agent", "formalize_answer": "formalize_answer"})+ g.add_edge("formalize_answer", END)- # Include pre_summary from agent results if available- pre_summary = kbinfos.get("pre_summary")- if pre_summary:- parts.append(f"Research Summary:\n{pre_summary}\n")+ return g.compile()- if partial:- from rag.advanced_rag.harness.prompts.report_prompt import PARTIAL_ANSWER_PREAMBLE- parts.append(f"{PARTIAL_ANSWER_PREAMBLE}\n")+def _render_slot_draft(slot_table: dict, collected_answer: str | None = None) -> str:+ """Render a slot table into a fact-preserving draft for the SCA.- from rag.advanced_rag.harness.prompts.report_prompt import FINAL_ANSWER_SYSTEM- from rag.prompts.generator import citation_prompt as cp+ Each resolved slot becomes a ``<type>: <candidate> [strength]`` line with its+ discovered-clue tail; unresolved slots are listed explicitly so the SCA can+ call them out as gaps. When the tree surfaced a ``collected_answer`` it leads+ the draft (it is the strongest candidate); the slot view stays as structured+ context below it.+ """+ slots = slot_table.get("slots") if isinstance(slot_table, dict) else None+ lines = []+ if collected_answer:+ lines.append(f"Candidate answer: {collected_answer}")+ lines.append("")+ if not slots:+ return "\n".join(lines) if lines else ""+ for s in slots:+ if not isinstance(s, dict):+ continue+ vid = s.get("id")+ vtype = s.get("type") or "entity"+ cand = s.get("candidate")+ if cand:+ cs = s.get("candidate_strength")+ strength = f"{float(cs):.2f}" if isinstance(cs, (int, float)) else "?"+ clues = s.get("discovered_clues") or []+ tail = "; ".join(str(c)[:80] for c in clues[-2:])+ lines.append(f"- slot {vid} [{vtype}]: {cand} (strength={strength})" + (f" — {tail}" if tail else ""))+ else:+ qc = s.get("question_clues") or []+ lines.append(f"- slot {vid} [{vtype}]: NOT RESOLVED ({'; '.join(str(c)[:80] for c in qc[:2])})")+ return "\n".join(lines)- rules = cp(tools.user_defined_prompts).strip()- system = FINAL_ANSWER_SYSTEM.format(cite_rules=rules)- # Honor the dialog-level system prompt in the reasoning path the same- # way async_chat does when reasoning is disabled.- if tools.system_prompt.strip():- system = f"{tools.system_prompt.strip()}\n\n{system}"- parts.append(f"Evidence:\n{evidence}")- user_content = "\n".join(parts)+def _slot_table_serialize(state) -> dict:+ """Serialize a tree ``State`` (slot table) into a JSON-friendly dict for+ graph-state transport. ``None``/empty yields a minimal empty table."""+ if state is None:+ return {"slots": [], "answer_variable": None}+ slots = []+ for v in getattr(state, "state", []):+ slots.append(+ {+ "id": v.id,+ "type": v.type,+ "question_clues": list(v.question_clues),+ "discovered_clues": list(v.discovered_clues),+ "candidate": v.candidate,+ "candidate_strength": v.candidate_strength,+ }+ )+ return {"slots": slots, "answer_variable": getattr(state, "answer_variable", None)}- # Same bounded budget for the final message fit — never fill the whole- # model context with evidence.- _, msg = message_fit_in(form_message(system, user_content), min(tools.chat_mdl.max_length, _EVIDENCE_BUDGET_TOKENS))- try:- async for tok in tools.chat_mdl.async_chat_streamly_delta(msg[0]["content"], msg[1:], answer_conf):- token_queue.put_nowait(tok)- except Exception:- _LOG.exception("formalize_answer: stream failed")- token_queue.put_nowait("I'm sorry, I encountered an error while composing the answer.")- return {"final_answer": ""}+def _slot_table_deserialize(slot_table: dict):+ """Rebuild a tree ``State`` from a serialized slot-table dict (方案 B).+ Returns ``None`` when no usable table is present."""+ from rag.advanced_rag.harness.action_session import State, Variable- # ── Build graph ──- g = StateGraph(AgenticState)- g.add_node("formalize_question", formalize_question)- g.add_node("route", route)- g.add_node("pre_search", pre_search)- g.add_node("planner", planner)- g.add_node("orchestrator_loop", orchestrator_loop)- g.add_node("formalize_answer", formalize_answer)+ slots = (slot_table or {}).get("slots") if isinstance(slot_table, dict) else None+ if not slots:+ return None+ vars_ = []+ for s in slots:+ if not isinstance(s, dict):+ continue+ vars_.append(+ Variable(+ id=int(s.get("id", len(vars_))),+ type=str(s.get("type") or "entity"),+ question_clues=[str(c) for c in (s.get("question_clues") or [])],+ discovered_clues=[str(c) for c in (s.get("discovered_clues") or [])],+ candidate=s.get("candidate"),+ candidate_strength=s.get("candidate_strength"),+ )+ )+ if not vars_:+ return None+ return State(state=vars_, depth=0, answer_variable=(slot_table or {}).get("answer_variable"))- g.add_edge(START, "formalize_question")- g.add_edge("formalize_question", "route")- g.add_edge("route", "pre_search")- g.add_edge("pre_search", "planner")- g.add_edge("planner", "orchestrator_loop")- g.add_edge("orchestrator_loop", "formalize_answer")- g.add_edge("formalize_answer", END)- return g.compile()+async def _build_slot_table(tools, question: str, fanouts: list, answer_conf: dict, deadline_left_fn=None) -> tuple:+ """Build a slot table for the research executor.++ Reuses the tree's ``initialize_state`` (LLM decomposition into typed+ unknowns with question_clues). Planner fanouts are passed as ``fanout_hint``+ so the table's first-hop slots align with the planner's sub-questions.+ Returns ``(State, first_queries)``.+ """+ from rag.advanced_rag.harness.action_session import initialize_state++ try:+ root, first_queries = await initialize_state(+ tools,+ question,+ fanouts or [],+ deadline_left_fn() if deadline_left_fn else None,+ )+ except Exception:+ _LOG.warning("[SlotTable] initialize_state failed; building from fanouts", exc_info=True)+ root = None+ first_queries = fanouts or [question]+ if root is None or not root.state:+ # fallback: one answer slot per fanout (or the raw question)+ from rag.advanced_rag.harness.action_session import State, Variable++ queries = fanouts or [question]+ root = State(+ state=[Variable(id=i, type="aspect", question_clues=[str(q)[:160]]) for i, q in enumerate(queries[:4])],+ depth=0,+ answer_variable=0,+ )+ first_queries = first_queries or queries[:3]+ _LOG.info("[SlotTable] built %d slot(s): %s", len(root.state), root.brief())+ return root, (first_queries or [question])+++async def _run_slot_research_pass(tools, question: str, state: AgenticState, answer_conf: dict, deadline_left: float) -> dict:+ """Drive ONE research round with slot-aware action+ sessions.+ """+ from rag.advanced_rag.harness.action_session import run_action_session++ slot_table = _slot_table_deserialize(state.get("slot_table"))+ if slot_table is None:+ # No planner ran (medium single-pass, or planner failed): build the slot+ # table from the raw question so the slot research still executes. The+ # empty-fanout fallback in _build_slot_table yields one answer slot.+ root, _ = await _build_slot_table(+ tools,+ state.get("question") or question,+ [],+ answer_conf,+ lambda: max(15.0, deadline_left - 10.0),+ )+ slot_table = root+ question = state.get("question") or question+ unresolved = slot_table.unresolved()+ if not unresolved:+ _LOG.info("[SlotResearch] all slots filled; no session to run.")+ return {}++ # Bound sessions for the unresolved slots (parallel, capped concurrency).+ sem = asyncio.Semaphore(2)+ base = getattr(tools, "kbinfos", None) or state.get("kbinfos") or {"chunks": [], "doc_aggs": []}+ tools.kbinfos = dict(base)++ async def _one(v):+ direction = v.question_clues[0] if v.question_clues else question+ async with sem:+ return await run_action_session(+ tools=tools,+ direction=direction,+ parent_state=slot_table,+ deadline_left=max(20.0, deadline_left - 10.0),+ base_summary="",+ )++ results = await asyncio.gather(+ *[_one(v) for v in unresolved[:3]],+ return_exceptions=True,+ )++ collected = state.get("collected_answer")+ ledger = [e for e in _safe_list(state.get("attempted"), "state.attempted") if isinstance(e, dict)]+ # Fold each session's new-state branches back into the slot table.+ for r in results:+ if isinstance(r, Exception):+ _LOG.warning("[SlotResearch] action_session raised: %s", r)+ continue+ if r.found_answer and not collected:+ collected = r.found_answer+ for ns in r.new_states or []:+ # merge the branch's candidate values into the shared table+ merged = _merge_slot_patch(slot_table, ns)+ if merged is not None:+ slot_table = merged++ ledger.append({"q": (r.found_answer or str(getattr(r, "messages", [])))[:80] if hasattr(r, "found_answer") else "", "new": 1})++ # Serialize the updated table and expose unresolved slots for the rewriter.+ out_table = _slot_table_serialize(slot_table)+ unresolved_slots = [+ {+ "id": v.id,+ "type": v.type,+ "question_clues": list(v.question_clues),+ "discovered_clues": list(v.discovered_clues[-4:]),+ }+ for v in slot_table.unresolved()+ ]+ draft = _render_slot_draft(out_table, collected)+ _LOG.info(+ "[SlotResearch] round done — %d slot(s) filled, unresolved=%d, collected_answer=%s",+ sum(1 for s in out_table.get("slots", []) if s.get("candidate")),+ len(unresolved_slots),+ bool(collected),+ )+ return {+ "slot_table": out_table,+ "collected_answer": collected,+ "unresolved_slots": unresolved_slots,+ "slot_draft": draft,+ "rag_answer": draft or (state.get("rag_answer") or ""),+ "kbinfos": tools.kbinfos,+ "attempted": ledger,+ }+++def _merge_slot_patch(base, branch):+ """Fold a session's new-state branch into the shared slot table: for each+ variable, adopt the branch's candidate when present and stronger, else keep+ the base. Returns the merged State (base is not mutated)."""+ from rag.advanced_rag.harness.action_session import State, Variable++ if branch is None:+ return None+ merged_vars = []+ changed = False+ branch_by_id = {v.id: v for v in getattr(branch, "state", [])}+ for v in getattr(base, "state", []):+ bv = branch_by_id.get(v.id)+ if bv is None:+ merged_vars.append(v)+ continue+ cand = bv.candidate or v.candidate+ bstr = bv.candidate_strength if bv.candidate_strength is not None else None+ vstr = v.candidate_strength if v.candidate_strength is not None else None+ strength = None+ strength = None+ if cand is not None:+ strength = bstr if bstr is not None else (vstr if vstr is not None else 0.5)+ elif vstr is not None:+ strength = vstr+ clues = list(dict.fromkeys(list(v.discovered_clues) + list(bv.discovered_clues)))+ if cand != v.candidate or clues != v.discovered_clues:+ changed = True+ merged_vars.append(+ Variable(+ id=v.id,+ type=v.type,+ question_clues=list(v.question_clues),+ discovered_clues=clues,+ candidate=cand,+ candidate_strength=strength,+ )+ )+ if not changed:+ return None+ return State(+ state=merged_vars,+ depth=base.depth + 1,+ answer_variable=base.answer_variable,+ retrieved_evidence_ids=list(base.retrieved_evidence_ids),+ )+++async def _compose_fallback_draft(tools, state: AgenticState, answer_conf: dict) -> str:+ """Intermediate draft synthesized from the snippet pool.++ Used when a research pass produced no narration to reuse (this IS the+ 'intermediate draft'). Ranks the pool so the strongest hits lead,+ widens the context budget, and demands an explicit FOUND/MISSING split so+ the Sufficient Context Agent gets precise gaps.+ """+ from rag.advanced_rag.harness.tools.search import _base_chat_mdl, _chunk_text++ chunks = list((getattr(tools, "kbinfos", None) or {}).get("chunks") or [])+ # Rank: whatever relevance signal exists, else keep retrieval order.+ chunks.sort(key=lambda c: float(c.get("similarity") or c.get("score") or 0.0), reverse=True)+ per_chunk = 1200+ max_chunks = 16+ evidence = "\n".join(f"[{i + 1}] {_chunk_text(c)[:per_chunk]}" for i, c in enumerate(chunks[:max_chunks]))+ if not evidence:+ return ""+ mdl = _base_chat_mdl(tools)+ if mdl is None:+ return evidence[:4000]+ question = state.get("question") or ""++ sub_points = []+ for m in state.get("research_feedback") or []:+ content = str(m.get("content", "")) if isinstance(m, dict) else str(m)+ if content.strip():+ sub_points.append(content)+ focus = ("\n\nAdditional gaps you MUST cover:\n" + "\n".join(sub_points[-1:])) if sub_points else ""++ try:+ ans, _ = await mdl.async_chat(+ (+ "You are a research assistant writing an INTERMEDIATE DRAFT toward answering the user's "+ "question, using ONLY the retrieved evidence snippets below.\n"+ "Requirements:\n"+ "1. First list concrete FACTS FOUND in the snippets (exact numbers, dates, names preserved).\n"+ "2. Then output a line starting with 'MISSING:' naming precisely which part(s) of the "+ "question the snippets do NOT answer yet.\n"+ "3. No conclusions beyond the evidence; no general knowledge.\n"+ "Keep it under 250 words." + ("Write your draft in the same language as the question." if any(ord(ch) > 127 for ch in question) else "")+ ),+ [{"role": "user", "content": f"Question: {question}{focus}\n\nRetrieved evidence:\n{evidence}"}],+ dict(answer_conf or {}),+ )+ return (str(ans or "").strip() or evidence)[:6000]+ except Exception:+ _LOG.warning("[Draft] fallback composition failed; using snippet text", exc_info=True)+ return evidence[:4000]async def run_agentic_rag(tools, messages: list, max_loops: int = 3, gen_conf: dict | None = None):@@ -484,16 +1412,45 @@ async def run_agentic_rag(tools, messages: list, max_loops: int = 3, gen_conf: d)token_queue: asyncio.Queue = asyncio.Queue()- graph = build_agentic_graph(tools, token_queue, gen_conf=gen_conf)+ thinking_mode = str(getattr(tools, "thinking_mode", "") or "").lower()+ # One aligned graph for medium / high / ultra; only the planner+prefetch pair+ # (fan-out decomposition) and the SCA iteration separate the modes:+ # medium: no planner (raw question), SCA review loop on+ # high/ultra: planner + prefetch + SCA↔rewriter iteration+ use_graph = thinking_mode in ("medium", "high", "ultra")+ enable_sca = thinking_mode in ("medium", "high", "ultra")+ use_fanout = thinking_mode in ("high", "ultra")+ _LOG.info("[Agentic RAG] mode=%s sca=%s fanouts=%s", thinking_mode or "low", enable_sca, use_fanout)++ if use_graph:+ graph = build_agentic_graph(tools, token_queue, gen_conf=gen_conf, enable_sca=enable_sca, use_fanout=use_fanout)+ init_state: dict = {+ "messages": messages,+ "max_loops": max_loops,+ }+ else:+ # low (reasoning=1): lightweight formalize → direct_search → answer.+ graph = build_low_graph(tools, token_queue, gen_conf=gen_conf)+ init_state = {"messages": messages, "max_loops": max_loops}+_SENTINEL = object()holder: dict[str, Any] = {}+ # Node counts are bounded and static; the research loop consumes wrapper-side+ # turn budgets rather than graph recursion, so a modest limit always suffices.+ recursion_limit = 60 if use_graph else max(25, max_loops * 8)+async def _drive():+ # Last-resort wall clock: routing guards steer the graph to synthesis+ # BEFORE this fires; the hard cancel only trips if a node still hangs+ # (e.g. an unbounded HTTP read) so the client never hits its 300s+ # read-timeout and flags the question as an error.try:- holder["state"] = await graph.ainvoke(- {"messages": messages, "max_loops": max_loops},- {"recursion_limit": max(25, max_loops * 8)},- )+ async with asyncio.timeout(_TOTAL_BUDGET_S + 30.0):+ holder["state"] = await graph.ainvoke(init_state, {"recursion_limit": recursion_limit})+ except TimeoutError:+ logging.warning("run_agentic_rag: total research budget (%.0fs) exhausted — cutting off.", _TOTAL_BUDGET_S)+ holder["error"] = Trueexcept Exception:logging.exception("run_agentic_rag: graph execution failed")holder["error"] = True@@ -517,7 +1474,22 @@ async def run_agentic_rag(tools, messages: list, max_loops: int = 3, gen_conf: dif isinstance(final_kb, dict) and final_kb.get("chunks"):tools.kbinfos = final_kb- _LOG.info("[Agentic RAG] Research complete — %d passage(s) gathered after %d round(s).", len((state.get("kbinfos") or {}).get("chunks", [])), state.get("loop", 0))+ # Expose the sufficiency verdict to the outer loop so it can decide whether to+ # re-run `rag` (and how to rephrase the next question) from the reported gaps.+ verdict = state.get("verdict")+ try:+ tools._rag_verdict = verdict+ _LOG.info(+ "[Agentic RAG] Research complete — %d passage(s), %d SCA round(s), verdict=%s",+ len((state.get("kbinfos") or {}).get("chunks", [])),+ state.get("search_rounds", 0),+ (verdict or {}).get("status") if isinstance(verdict, dict) else verdict,+ )+ except Exception:+ _LOG.info(+ "[Agentic RAG] Research complete — %d passage(s) gathered.",+ len((state.get("kbinfos") or {}).get("chunks", [])),+ )if not produced and holder.get("error"):yield "I couldn't complete the search due to an internal error."diff --git a/rag/advanced_rag/harness/action_session.py b/rag/advanced_rag/harness/action_session.pynew file mode 100644index 0000000000..5213707319--- /dev/null+++ b/rag/advanced_rag/harness/action_session.py@@ -0,0 +1,1445 @@+#+# Copyright 2026 InfiniFlow, Inc. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""Slot-table research primitives + the graph-edge action session."""++from __future__ import annotations++import asyncio+import json+import logging+import re+import secrets+import time+from dataclasses import dataclass, field+from typing import Annotated, Any, TypedDict++from langchain_core.messages import HumanMessage, SystemMessage, convert_to_openai_messages+from langgraph.graph import END, START, StateGraph+from langgraph.graph.message import add_messages++_LOG = logging.getLogger(__name__)++_INIT_TIMEOUT_S = 45.0+_ACTION_MAX_TURNS = 4+_ACTION_MAX_TURNS_ULTRA = 6 # ultra runs deeper inside each action session+_ACTION_TIMEOUT_S = 75.0+_SNIPPETS_PER_QUERY = 4+_MAX_TOOL_RESPONSE_CHARS = 12000+# Near-duplicate search-query detection: the model re-issues the SAME intent with+# many paraphrase queries (observed: 20+ "Culdcept original creator/designer/+# OmiyaSoft/founder" variants in ONE session), each executed against ES even+# though the evidence is unchanged. We Jaccard-dedupe retrieval queries so a+# paraphrase (>= _NEAR_DUP_JACCARD token overlap with an already-run query) is+# short-circuited with a nudge instead of burning a turn on a redundant search.+_NEAR_DUP_JACCARD = 0.8+_RETRIEVAL_TOOLS = ("search_chunks", "grep_chunks", "grep_search")+++def _search_tokens(q: str) -> set[str]:+ """Lowercased alphanumeric tokens of a query for near-duplicate detection."""+ return set(re.findall(r"[a-z0-9]{2,}", (q or "").lower()))+++def _is_near_dup(q: str, seen: list[str]) -> bool:+ """True when q shares >= _NEAR_DUP_JACCARD of its tokens with any seen query."""+ if not q or not seen:+ return False+ toks = _search_tokens(q)+ if len(toks) < 2:+ return False+ for s in seen:+ other = _search_tokens(s)+ inter = len(toks & other)+ union = len(toks | other)+ if union and inter / union >= _NEAR_DUP_JACCARD:+ return True+ return False+++# ── Slot-table models (ex-tree/models.py) ─────────────────────────────────+@dataclass+class Variable:+ """One unknown entity to resolve. `id` is immutable across patches."""++ id: int+ type: str+ question_clues: list = field(default_factory=list)+ discovered_clues: list = field(default_factory=list)+ candidate: str | None = None+ candidate_strength: float | None = None++ def brief(self) -> str:+ if self.candidate:+ cs = f"{self.candidate_strength:.2f}" if self.candidate_strength is not None else "?"+ return f"[{self.id}] {self.type}: {self.candidate} ({cs})"+ return f"[{self.id}] {self.type}: EMPTY"++ def filled(self) -> bool:+ return bool(self.candidate)+++@dataclass+class State:+ state: list+ depth: int = 0+ id: str = ""+ answer_variable: int | None = None+ retrieved_evidence_ids: list = field(default_factory=list)++ def __post_init__(self):+ if not self.id:+ self.id = f"{self.depth:03x}_{int(time.time() * 1000) % 100000000:08x}{secrets.token_hex(1)}"++ def unresolved(self) -> list:+ return [v for v in self.state if not v.filled()]++ def by_id(self, vid):+ for v in self.state:+ if v.id == vid:+ return v+ return None++ def brief(self) -> str:+ marks = ["+" if v.filled() else "." for v in self.state]+ return f"d{self.depth}({''.join(marks)})"++ def render_slots(self) -> str:+ lines = []+ for v in self.state:+ line = f"- id={v.id} type={v.type}"+ if v.question_clues:+ line += "\n question_clues: " + "; ".join(v.question_clues)+ if v.discovered_clues:+ line += "\n discovered_clues: " + "; ".join(v.discovered_clues[-4:])+ if v.candidate:+ cs = f"{v.candidate_strength:.2f}" if v.candidate_strength is not None else "?"+ line += f"\n CANDIDATE: {v.candidate} (strength={cs})"+ lines.append(line)+ return "\n".join(lines)+++@dataclass+class Result:+ """Outcome of ONE run_action session (one of new_states / found_answer)."""++ messages: list = field(default_factory=list)+ new_states: list = field(default_factory=list)+ found_answer: str | None = None+ retrieved_evidence_ids: list = field(default_factory=list)+++# ── Tool surface ─────────────────────────────────────────────────────────+_RETRIEVE_TOOL_SPEC = {+ "type": "function",+ "function": {+ "name": "retrieve",+ "description": (+ "Keyword-first search of the fixed document corpus. Pass natural-"+ "language queries; returns SHORT snippets of the most relevant "+ "passages (exact-term matched where possible). Use multiple queries "+ "to cover different aspects. Supports 1-3 queries per call."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "query": {+ "type": "array",+ "items": {"type": "string"},+ "minItems": 1,+ "maxItems": 3,+ }+ },+ "required": ["query"],+ },+ },+}++_LIST_CHUNKS_TOOL_SPEC = {+ "type": "function",+ "function": {+ "name": "list_chunks",+ "description": (+ "Deep-read the FULL text of one document by doc_id (returned in "+ "retrieve snippets). Use for enumeration / count / arithmetic answers "+ "when snippets are insufficient. Returns all chunks of the document "+ "in reading order. One doc_id per call."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "doc_id": {+ "type": "string",+ "description": "document id seen in a retrieve snippet",+ }+ },+ "required": ["doc_id"],+ },+ },+}++_SEARCH_CHUNKS_TOOL_SPEC = {+ "type": "function",+ "function": {+ "name": "search_chunks",+ "description": (+ "SEMANTIC retrieval (hybrid vector+BM25) with COMPILED-STRUCTURE "+ "EXPANSION. Use as the PRIMARY recall tool when exact-term "+ "``retrieve`` returns nothing useful, or when the dataset is large and "+ "you are unsure which document holds the answer — the answer passage "+ "may share NO surface words with the query. "+ "Compiled expansion: automatically appends related chunks from the "+ "dataset's compiled structure (page index, tree/heading hierarchy, "+ "knowledge graph, wiki pages when present) so a semantic hit carries "+ "its structural neighbours (parent/child headings, sibling pages). "+ "If the dataset has NO compiled structure (incl. no wiki), expansion "+ "is a no-op — no error, just semantic hits. "+ "Returns snippet chunks ranked by relevance. 1-2 queries per call."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "query": {+ "type": "array",+ "items": {"type": "string"},+ "minItems": 1,+ "maxItems": 2,+ }+ },+ "required": ["query"],+ },+ },+}++# Multi-tool registry. ``execute_tool`` dispatches by name; add new tools by+# registering their callable + schema here (DeepSearch ToolNode equivalent).+_WEB_SEARCH_TOOL_SPEC = {+ "type": "function",+ "function": {+ "name": "web_search",+ "description": (+ "Search the open WEB. Use ONLY when the needed fact is world "+ "knowledge / recent event / not covered by the fixed corpus — e.g. "+ "a current event, a person's alive-now status, or a statistic newer "+ "than the corpus. If the fact plausibly lives in the documents, "+ "prefer corpus tools (retrieve/search_chunks) first. 1-2 queries per call."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "query": {+ "type": "array",+ "items": {"type": "string"},+ "minItems": 1,+ "maxItems": 2,+ }+ },+ "required": ["query"],+ },+ },+}++_NAVIGATE_TREE_TOOL_SPEC = {+ "type": "function",+ "function": {+ "name": "navigate_tree",+ "description": (+ "LOCATE the RIGHT DOCUMENT among MANY before deep-reading. Use it "+ "BEFORE search_chunks when the dataset is large and you have no "+ "doc_id yet — it routes by TOPIC/CLUSTERING similarity over the "+ "compiled document-navigation tree (not exact surface words), so it "+ "finds the document even when your query words differ from its text. "+ "Returns candidate doc_ids + a first-chunk summary of each. "+ "This is the FIRST hop of a navigation chain: "+ "navigate_tree(query) -> doc_id -> navigate_structure(doc_id, ...) "+ "-> list_chunks(doc_id, chunk_ids). "+ "Use when: the question names a topic/entity/alias but you do not "+ "know which document discusses it; search_chunks returned scattered "+ "hits across many docs and you must pick the source. "+ "Do NOT use if you already hold a doc_id (go straight to "+ "navigate_structure) or if the answer is likely a single exact "+ "passage (prefer retrieve/search_chunks). "+ "If the dataset has no compiled document navigation tree, it returns "+ "empty — fall back to search_chunks."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "query": {+ "type": "string",+ "description": "topic / entity / alias whose document(s) to locate",+ }+ },+ "required": ["query"],+ },+ },+}++_NAVIGATE_STRUCTURE_TOOL_SPEC = {+ "type": "function",+ "function": {+ "name": "navigate_structure",+ "description": (+ "PINPOINT A PASSAGE inside ONE document using its compiled structure "+ "(heading/catalog tree, concept mindmap, or entity graph) — the "+ "in-document counterpart of navigate_tree. "+ "Use AFTER you know the doc_id (from navigate_tree / search_chunks / "+ "retrieve) and need to find where the answer lives WITHOUT reading "+ "every chunk. Returns the structure outline annotated with matching "+ "chunk_ids (reading-order aware). Then call list_chunks(doc_id, "+ "chunk_ids) to read exactly those. "+ "kind: 'catalog' (default) for page-index/heading/timeline trees, "+ "'mindmap' for concept maps, 'graph' for entity-relation graphs. "+ "If the document has NO compiled structure, an empty <doc/> is "+ "returned — fall back to list_chunks to read the full document."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "doc_id": {"type": "string", "description": "document id seen from a prior tool result"},+ "query": {"type": "string", "description": "what to locate within the document"},+ "kind": {"type": "string", "enum": ["catalog", "mindmap", "graph"], "description": "compiled structure kind, default catalog"},+ },+ "required": ["doc_id"],+ },+ },+}++_CALCULATE_TOOL_SPEC = {+ "type": "function",+ "function": {+ "name": "calculate",+ "description": (+ "COMPUTE a numeric answer by generating and safely running code. "+ "MANDATORY whenever the question asks you to DERIVE a number by "+ "combining facts you found (sum/difference/percentage/ratio/sort/"+ "compare/difference in length/age, price, area, growth, etc.) — do "+ "NOT do arithmetic mentally. Language-neutral: the question and "+ "facts may be in ANY language (English, Chinese, ...); pass the "+ "numbers verbatim as written in the evidence regardless of language. "+ "Steps: (1) collect every needed number first (retrieve / "+ "search_chunks / navigate_* / list_chunks); (2) call calculate with "+ "the question + ALL those numbers; (3) report the computed result "+ "verbatim. If a needed number is missing, search for it first — do "+ "not estimate. If the answer IS one of the stated numbers (no "+ "combination needed), answer directly without this tool."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "question": {"type": "string", "description": "the user's question, verbatim"},+ "facts": {+ "type": "array",+ "items": {"type": "string"},+ "description": "numbers/facts found in the evidence, verbatim",+ },+ },+ "required": ["question", "facts"],+ },+ },+}++_GRAPH_EXPLORE_TOOL_SPEC = {+ "type": "function",+ "function": {+ "name": "graph_explore",+ "description": (+ "EXPLORE the compiled KNOWLEDGE GRAPH (entities + relations) for a "+ "RELATIONAL/multi-hop answer. Different from navigate_*: instead of "+ "locating a document or passage, it seeds entities for the query, "+ "hops along their RELATIONS, and returns either a direct answer or "+ "the source passages behind the relevant entities/relations. "+ "Use when the answer requires connecting several entities through "+ "their relations (e.g. who-was-related-to-whom, cause-effect chains, "+ "membership/ownership) and you already have a starting entity from a "+ "search result, a navigation outline, or a list_chunks reading. "+ "If the dataset has NO compiled knowledge graph, it returns empty — "+ "fall back to search_chunks / navigate_structure."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "query": {"type": "string", "description": "the relational question / starting entity"},+ "doc_scope": {+ "type": "array",+ "items": {"type": "string"},+ "description": "optional doc_ids to restrict the graph to (from prior navigation/list_chunks)",+ },+ },+ "required": ["query"],+ },+ },+}++# graph_explore is ULTRA-ONLY: relational graph exploration is the extra depth+# that distinguishes ultra from high (which stays on document/structure tools).+_ULTRA_ONLY_TOOLS = {"graph_explore"}++_TOOL_MAP = {+ "retrieve": _RETRIEVE_TOOL_SPEC,+ "search_chunks": _SEARCH_CHUNKS_TOOL_SPEC,+ "list_chunks": _LIST_CHUNKS_TOOL_SPEC,+ "navigate_tree": _NAVIGATE_TREE_TOOL_SPEC,+ "navigate_structure": _NAVIGATE_STRUCTURE_TOOL_SPEC,+ "calculate": _CALCULATE_TOOL_SPEC,+ "graph_explore": _GRAPH_EXPLORE_TOOL_SPEC,+ "web_search": _WEB_SEARCH_TOOL_SPEC,+}+++def _active_tool_specs(tools) -> list:+ """Tool schemas exposed to the model for THIS mode.++ Visibility rules:+ * ultra adds the relational ``graph_explore`` (high/medium do not see it).+ * ``web_search`` is hidden when NO web provider is configured — the model+ otherwise retries it up to 4× per session burning turns (40 empty calls+ observed in a 20-question run). Hiding beats any "do not use again" note.+ * compile-only tools (navigate_* / graph_explore) are kept visible even on+ datasets without compiled structure; their executors return an explicit+ empty-with-hint so the model falls back to plain search (see below).+ * RUNTIME: a compile-only tool is REMOVED from the surface after its first+ call proves the dataset has no such compiled structure (the executor marks+ it in ``tools._disabled_tools``). Without this the model keeps re-invoking+ a tool that always returns empty — e.g. 24 graph_explore calls in one+ benchmark run, every one "No compiled knowledge graph in scope".++ The returned list is what the model can call this session.+ """+ mode = str(getattr(tools, "thinking_mode", "") or "").lower()+ names = set(_TOOL_MAP.keys())+ if mode != "ultra":+ names -= _ULTRA_ONLY_TOOLS+ # Hide web_search without a provider — a hard visibility gate beats hoping+ # the model obeys a "do not use" note across sessions.+ if getattr(tools, "web_search", None) is None:+ names.discard("web_search")+ # Runtime: drop tools proven unavailable this session (no compiled structure).+ disabled = getattr(tools, "_disabled_tools", None) or set()+ if disabled:+ names -= set(disabled)+ return [spec for name, spec in _TOOL_MAP.items() if name in names]+++def _disable_tool(tools, name: str) -> None:+ """Mark a compile-only tool as unavailable for the REST of this session.++ Records the (best-effort, tools-object-scoped) fact that the dataset has no+ such compiled structure, so ``_active_tool_specs`` stops advertising it and+ ``execute_tool`` short-circuits it. Subsequent calls return a note instead of+ re-running a retrieval that would only come back empty.+ """+ if name not in _TOOL_MAP:+ return+ try:+ disabled = getattr(tools, "_disabled_tools", None)+ if disabled is None:+ disabled = set()+ try:+ tools._disabled_tools = disabled+ except Exception: # tools may be frozen/slots in tests+ return+ disabled.add(name)+ except Exception:+ _LOG.warning("[action_session] could not mark tool %r disabled", name, exc_info=True)+++def _is_empty_structure(text, xml_tag: str) -> bool:+ """True when a compiled-structure executor's payload has no usable content.++ A payload is "empty" not only when blank — a well-formed XML with+ ``count="0"`` or an ``error=`` marker also means "no structure compiled for+ this kind". Treating those as empty lets the executor disable the tool and+ steer the model back to plain retrieval instead of feeding it a count="0"+ outline it will keep probing.+ """+ text = (text or "").strip()+ if not text or text in ("[]", "{}"):+ return True+ if xml_tag and f"<{xml_tag}" in text and 'count="0"' in text:+ return True+ if "error=" in text:+ return True+ return False+++# ── JSON / terminal parsing helpers ──────────────────────────────────────+def extract_json(text: str):+ """Return the first parseable JSON object found in ``text``.++ Brace-matches to isolate ONE complete object (a greedy regex over-captures+ when the model emits a second object after the first, and ``json.loads``+ then fails with "Extra data"). Falls back to ``json_repair`` per candidate,+ then tries the next opening brace.+ """+ if not text:+ return None+ i = 0+ n = len(text)+ while i < n:+ start = text.find("{", i)+ if start < 0:+ return None+ depth = 0+ for j in range(start, n):+ ch = text[j]+ if ch == "{":+ depth += 1+ elif ch == "}":+ depth -= 1+ if depth == 0:+ candidate = text[start : j + 1]+ try:+ return json.loads(candidate, strict=False)+ except Exception:+ pass+ try:+ import json_repair++ return json_repair.loads(candidate)+ except Exception:+ break # invalid object; try the next "{"+ i = start + 1+ return None+++def extract_tag(text: str, tag: str):+ """Exact-tag extraction first; then lenient fallbacks for models that wrap+ the JSON in code fences or emit bare objects (observed with DeepSeek-class+ models ignoring the XML protocol)."""+ s, e = text.rfind(f"<{tag}>"), text.rfind(f"</{tag}>")+ if s != -1 and e != -1 and e > s:+ return text[s + len(tag) + 2 : e].strip()+ # fenced ```json ... ``` blocks mentioning the tag name+ fenced = re.findall(r"```(?:json)?\s*(\{.*?\}|\[.*?\])\s*```", text or "", re.DOTALL)+ if not fenced and re.search(rf"<{tag}\b", text or "") is None:+ return None+ for frag in reversed(fenced): # prefer the LAST block (most recent decision)+ try:+ obj = json.loads(frag)+ keys = set(obj.keys()) if isinstance(obj, dict) else set()+ want = {"new_states"} if tag == "state" else {"answer", "new_state"}+ if isinstance(obj, dict) and (keys & want):+ return frag.strip()+ except Exception:+ continue+ return None+++def apply_patch(base: State, branch_patches: list) -> State | None:+ """DeepSearch apply_patch: ONLY existing ids; mutable fields are candidate /+ candidate_strength / discovered_clues. `id` is immutable — patches may not+ add variables."""+ new_vars = []+ for v in base.state:+ new_vars.append(+ Variable(+ id=v.id,+ type=v.type,+ question_clues=list(v.question_clues),+ discovered_clues=list(v.discovered_clues),+ candidate=v.candidate,+ candidate_strength=v.candidate_strength,+ )+ )+ changed = False+ for pv in branch_patches or []:+ if not isinstance(pv, dict) or "id" not in pv:+ return None+ idx = next((i for i, nv in enumerate(new_vars) if nv.id == pv["id"]), None)+ if idx is None:+ continue+ nv = new_vars[idx]+ if "candidate" in pv:+ nv.candidate = str(pv["candidate"]) if pv.get("candidate") else None+ changed = True+ if pv.get("candidate_strength") is not None:+ try:+ nv.candidate_strength = min(max(float(pv["candidate_strength"]), 0.0), 1.0)+ changed = True+ except Exception:+ pass+ if isinstance(pv.get("discovered_clues"), list):+ nv.discovered_clues.extend(str(c)[:160] for c in pv["discovered_clues"][-4:])+ changed = True+ if not changed:+ return None+ return State(+ state=new_vars,+ depth=base.depth + 1,+ answer_variable=base.answer_variable,+ retrieved_evidence_ids=list(base.retrieved_evidence_ids),+ )+++# ── Tool executors ───────────────────────────────────────────────────────+def _kb_ids(tools):+ from rag.advanced_rag.harness.tools.search import _get_kb_ids++ return _get_kb_ids(tools) or None+++def _seed_evidence(tools):+ kbinfos = getattr(tools, "kbinfos", None)+ if kbinfos is None:+ kbinfos = {}+ try:+ tools.kbinfos = kbinfos+ except Exception:+ pass+ kbinfos.setdefault("chunks", [])+ return kbinfos+++def _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=True):+ """Register one chunk into the session output AND the shared evidence pool.++ Accumulating raw chunks into ``tools.kbinfos`` is what lets the downstream+ formalize_answer / compose stage cite them (DeepSearch search-context+ parity) — without it the research retrieves plenty but Composing reports+ "0 gathered passage(s)" → partial answer → outer loop re-invokes research.++ Imports the chunk helpers locally to keep ``tools.search`` (and its heavy+ deepdoc dependency chain) out of module-import time.+ """+ from rag.advanced_rag.harness.tools.search import _chunk_id, _chunk_text, _doc_id++ cid = _chunk_id(c)+ if cid in seen:+ return+ seen.add(cid)+ ids.append(cid)+ entry = {"id": str(cid), "content": (_chunk_text(c) or "")[:1200]}+ if include_doc_id:+ entry["doc_id"] = _doc_id(c)+ out.append(entry)+ if isinstance(c, dict) and cid not in kb_seen:+ kb_seen.add(cid)+ kbinfos["chunks"].append(c)+++async def _run_search(tools, search_fn, queries: list, top_n: int, max_q: int, **kw) -> tuple:+ """Run a corpus search fn per query and admit hits to output + evidence pool.++ Shared driver for every query-based tool (retrieve / search_chunks): the+ only differences are the search backend (grep vs hybrid), ``top_n``, the+ per-call query cap — all passed in. ``kw`` forwards backend kwargs such as+ ``use_compiled`` (search_chunks' compiled-structure expansion).+ """+ from rag.advanced_rag.harness.tools.search import _chunk_id++ kb_ids = _kb_ids(tools)+ out, ids = [], []+ seen = set()+ kbinfos = _seed_evidence(tools)+ kb_seen = {_chunk_id(c) for c in kbinfos["chunks"] if isinstance(c, dict)}+ for fq in queries[:max_q]:+ try:+ res = await search_fn(tools, fq, kb_ids=kb_ids, top_n=top_n, **kw)+ cands = res.get("chunks", []) or []+ except Exception:+ _LOG.warning("[action_session] %s failed for %r", getattr(search_fn, "__name__", "search"), fq, exc_info=True)+ continue+ for c in cands[:_SNIPPETS_PER_QUERY]:+ _admit_evidence(kbinfos, kb_seen, c, out, ids, seen)+ return out, ids+++async def _exec_retrieve(tools, queries: list) -> tuple:+ """Corpus search via grep_search (exact-term locate, compact snippets)."""+ from rag.advanced_rag.harness.tools.search import grep_search++ return await _run_search(tools, grep_search, queries, top_n=10, max_q=3)+++async def _exec_search_chunks(tools, queries: list, use_compiled: bool = False) -> tuple:+ """Semantic retrieval (hybrid vector+BM25, narrow bypass) — the react-style+ channel that finds passages sharing NO surface words with the query.++ ``use_compiled=True`` (high mode) turns on hybrid_search's COMPILED+ expansion: page-index / tree / knowledge-graph / wiki pages (when the+ dataset has them) are appended to a semantic hit so its structural+ neighbours (parent/child headings, sibling wiki pages) come along.+ Datasets with NO compiled structure are unaffected — expansion is a no-op.+ """+ from rag.advanced_rag.harness.tools.search import hybrid_search++ return await _run_search(tools, hybrid_search, queries, top_n=20, max_q=2, use_compiled=use_compiled)+++async def _exec_web_search(tools, queries: list) -> tuple:+ """Open-web retrieval via the configured provider (Tavily/QuerIt/Serply/+ YouCom). Results share the RAGFlow chunk shape so they merge into the same+ evidence pool as corpus hits."""+ from rag.advanced_rag.harness.tools.search import _chunk_id++ provider = getattr(tools, "web_search", None)+ if provider is None:+ # No web provider configured. DO NOT return an empty payload (the model+ # would just retry and burn turns). Return an explicit, actionable note+ # so the model moves on to the corpus tools — a web failure must never+ # block the Q&A.+ _LOG.warning("[action_session] web_search unavailable (no provider configured)")+ return [{"kind": "web_search", "note": "Web search is NOT configured for this session. Do not use this tool again; use the corpus tools (retrieve / search_chunks / navigate_*) instead."}], []++ out, ids = [], []+ seen = set()+ kbinfos = _seed_evidence(tools)+ kb_seen = {_chunk_id(c) for c in kbinfos["chunks"] if isinstance(c, dict)}+ for q in queries[:2]:+ try:+ web_res = provider.retrieve_chunks(q)+ if asyncio.iscoroutine(web_res) or hasattr(web_res, "__await__"):+ web_res = await web_res+ except Exception:+ _LOG.warning("[action_session] web_search failed for %r", q, exc_info=True)+ continue+ for c in ((web_res or {}).get("chunks") or [])[:8]:+ cid = _chunk_id(c)+ if not cid or cid in seen:+ continue+ _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=False)+ return out, ids+++async def _exec_list_chunks(tools, doc_id: str) -> tuple:+ """Deep-read one document's full text (list_chunks tool)."""+ from rag.advanced_rag.harness.tools.search import _chunk_id, list_chunks++ try:+ res = await list_chunks(tools, doc_id)+ except Exception:+ _LOG.warning("[action_session] list_chunks failed doc=%r", doc_id, exc_info=True)+ return [], []+ out, ids = [], []+ kbinfos = _seed_evidence(tools)+ kb_seen = {_chunk_id(c) for c in kbinfos["chunks"] if isinstance(c, dict)}+ seen = set()+ for c in (res.get("chunks") or [])[:30]:+ cid = _chunk_id(c)+ if not cid:+ continue+ _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=False)+ return out, ids+++def _arg_query_list(args, max_q: int) -> list:+ """Normalize a tool call's ``query`` argument (string | list | absent) and+ cap it — shared by every query-based tool."""+ q = args.get("query") if isinstance(args, dict) else None+ if isinstance(q, str):+ q = [q]+ return [str(x) for x in (q or [])][:max_q]+++def _inject_nav_tools_ref(tools) -> None:+ """navigation.py's _navigate_*_impl / graph_explore read the request-scoped+ tools instance from THEIR OWN module-level ``_tools_ref`` (for tenant /+ embed_mdl resolution). The old bind_dynamic_tools used to populate it; it+ is gone, so populate it here before any navigation/graph call — otherwise+ those tools crash on a None tools instance ('NoneType' has no attribute+ '_resolve_doc_tenant') or warn 'no embed_mdl available' and return empty.+ """+ try:+ import rag.advanced_rag.harness.tools.navigation as _nav++ _nav._tools_ref["tools"] = tools+ except Exception:+ _LOG.warning("[action_session] could not inject navigation tools ref", exc_info=True)+++async def _exec_navigate_tree(tools, args: dict) -> tuple:+ """Document-level embedding routing (navigate_tree). Returns doc candidates+ as a text outline; evidence ids = routed doc_ids for later deep-read."""+ from rag.advanced_rag.harness.tools.navigation import _navigate_tree_impl++ _inject_nav_tools_ref(tools)+ query = str(args.get("query") or "")+ text = await _navigate_tree_impl(query, keywords=str(args.get("keywords") or ""))+ # Empty / count="0" / error= payloads mean no compiled navigation tree —+ # this dataset has no such structure. Disable the tool for the session and+ # tell the model to fall back to plain retrieval (a count="0" XML is NOT a+ # useful payload: without this, the model keeps re-invoking it).+ text = (text or "").strip()+ if _is_empty_structure(text, "tree_navigation"):+ _disable_tool(tools, "navigate_tree")+ _LOG.info("[action_session] navigate_tree disabled: dataset has no compiled navigation tree (text=%r)", text[:120])+ return [{"kind": "navigate_tree", "note": "This dataset has NO compiled document navigation tree. navigate_tree is unavailable; use search_chunks / retrieve instead."}], []+ return [{"kind": "navigate_tree", "content": text[:8000]}], []+++async def _exec_navigate_structure(tools, args: dict) -> tuple:+ """In-document compiled-structure navigation (navigate_structure). Returns+ outline annotated with chunk_ids; those become evidence for list_chunks."""+ from rag.advanced_rag.harness.tools.navigation import _navigate_structure_impl++ _inject_nav_tools_ref(tools)+ doc_id = str(args.get("doc_id") or "")+ query = str(args.get("query") or "")+ kind = str(args.get("kind") or "catalog")+ text = await _navigate_structure_impl(query, doc_id=doc_id, kind=kind)+ # Empty / count="0" / error= payloads mean no compiled structure of this kind+ # in the dataset — the tool is a dead end. Disable it for the session so the+ # model stops re-invoking it (a count="0" XML is NOT a useful payload).+ text = (text or "").strip()+ if _is_empty_structure(text, "structure_navigation"):+ _disable_tool(tools, "navigate_structure")+ _LOG.info("[action_session] navigate_structure disabled: no compiled structure kind=%r (text=%r)", kind, text[:120])+ return [+ {+ "kind": "navigate_structure",+ "doc_id": doc_id,+ "note": "This document/dataset has NO compiled structure of the requested kind. navigate_structure is unavailable here; use search_chunks / retrieve / list_chunks instead.",+ }+ ], [doc_id] if doc_id else []+ return [{"kind": "navigate_structure", "doc_id": doc_id, "content": text[:8000]}], [doc_id] if doc_id else []+++async def _exec_calculate(tools, args: dict) -> tuple:+ """Arithmetic terminal: compute_from_facts decides whether the question+ needs a computed number and safely evaluates an expression over facts."""+ from rag.advanced_rag.harness.arithmetic import compute_from_facts++ question = str(args.get("question") or "")+ facts = [str(f) for f in (args.get("facts") or []) if str(f).strip()]+ # compute_from_facts needs BOTH ``max_length`` (message-fit budget) and+ # ``async_chat``. The OUTER chat_mdl (CountingChatModel/LLMBundle) carries+ # max_length; the innermost LiteLLMBase that _base_chat_mdl returns does+ # NOT. Pass the outer bundle.+ mdl = getattr(tools, "chat_mdl", None)+ if mdl is None:+ return [{"kind": "calculate", "error": "no model"}], []+ try:+ res = await compute_from_facts(mdl, question, facts)+ except Exception:+ _LOG.warning("[action_session] calculate failed", exc_info=True)+ res = None+ if not res:+ # nothing derivable -> tell the model so it doesn't loop on it+ return [{"kind": "calculate", "expression": None, "note": "no numeric answer derivable from given facts; answer directly or retrieve more numbers."}], []+ return [{"kind": "calculate", "expression": res.get("expression"), "result": res.get("result"), "computed": res.get("computed_value")}], []+++async def _exec_graph_explore(tools, args: dict) -> tuple:+ """Relational knowledge-graph exploration (ultra-only). Returns either a+ direct answer or source passages behind relevant entities/relations, so the+ model can continue with list_chunks/navigate on the returned chunk/doc ids."""+ from rag.advanced_rag.harness.tools.navigation import graph_explore++ _inject_nav_tools_ref(tools)+ query = str(args.get("query") or "")+ doc_scope = [str(d) for d in (args.get("doc_scope") or []) if str(d).strip()]+ try:+ res = await graph_explore(tools, query, doc_scope=doc_scope or None)+ except Exception:+ _LOG.warning("[action_session] graph_explore failed", exc_info=True)+ res = {}+ answer = str(res.get("answer") or "").strip()+ chunks = res.get("chunks") or []+ if answer:+ return [{"kind": "graph_explore", "answer": answer}], []+ if not chunks:+ # No knowledge graph compiled for this dataset/scope: relational+ # exploration is a dead end. Disable the tool for the session (it will+ # keep coming back empty otherwise — 24 empty calls in one run) and+ # guide the model back to retrieval.+ _disable_tool(tools, "graph_explore")+ _LOG.info("[action_session] graph_explore disabled: no compiled knowledge graph in scope")+ return [+ {+ "kind": "graph_explore",+ "note": "This dataset has NO compiled knowledge graph (or none in the given scope). graph_explore is unavailable; use search_chunks / navigate_structure / retrieve instead.",+ }+ ], []+ snippet = [{"id": c.get("id"), "content": (str(c.get("content") or "")[:1500])} for c in chunks[:6]]+ ids = [str(c.get("id")) for c in chunks[:6] if c.get("id")]+ return [{"kind": "graph_explore", "chunks": snippet}], ids+++async def execute_tool(tools, name: str, args: dict) -> tuple:+ """DeepSearch ToolNode: dispatch ONE native tool call by name.++ Returns ``(results, evidence_ids)`` where results is a JSON-serializable+ payload and evidence_ids are any doc/chunk ids worth tracking.+ """+ # Short-circuit a tool already proven unavailable this session (no compiled+ # structure of its kind). We still return a note, not an error, so the model+ # learns to switch to the corpus tools rather than loop.+ if name in (_TOOL_MAP and (getattr(tools, "_disabled_tools", None) or set())):+ _LOG.info("[action_session] tool %r disabled (no compiled structure); returning note", name)+ return [{"kind": name, "note": f"{name} is unavailable in this dataset (no compiled structure of its kind). Use search_chunks / retrieve / list_chunks instead."}], []+ if name == "retrieve":+ return await _exec_retrieve(tools, _arg_query_list(args, 3))+ if name == "search_chunks":+ # Compiled-structure expansion (page-index/tree/wiki/KG) inside+ # hybrid_search is enabled in ALL modes. Datasets with NO compiled+ # structure are unaffected — expansion is a no-op there.+ return await _exec_search_chunks(tools, _arg_query_list(args, 2), use_compiled=True)+ if name == "list_chunks":+ return await _exec_list_chunks(tools, str(args.get("doc_id") or ""))+ if name == "navigate_tree":+ return await _exec_navigate_tree(tools, args)+ if name == "navigate_structure":+ return await _exec_navigate_structure(tools, args)+ if name == "calculate":+ return await _exec_calculate(tools, args)+ if name == "graph_explore":+ return await _exec_graph_explore(tools, args)+ if name == "web_search":+ return await _exec_web_search(tools, _arg_query_list(args, 2))+ _LOG.warning("[action_session] unknown tool %r; ignored.", name)+ return [], []+++# ── LangGraph session graph ──────────────────────────────────────────────+async def _acompletion(mdl, messages: list, tools_list=None, temperature: float = 0.3, timeout_s: float = 60.0):+ """Provider-agnostic ONE raw completion (shared by the tool loop and the+ salvage call).++ ``messages`` is a list of langchain messages (reducer output); we convert+ them back to OpenAI-dict form for whichever provider path we take.+ ``tools_list`` is None for a plain completion (finalize salvage) or the tool+ schema list to advertise tool-calling. OpenAI-style wrappers expose+ ``async_client`` → native create(); LiteLLMBase reuses its own public+ ``_construct_completion_args`` and calls ``litellm.acompletion`` with our+ schema injected. ``timeout_s`` is the per-call wall-clock bound.+ """+ oai_messages = convert_to_openai_messages(messages)+ if getattr(mdl, "async_client", None) is not None:+ client = mdl.async_client+ chat_obj = getattr(client, "chat", client)+ completions = getattr(chat_obj, "completions", chat_obj)+ create = getattr(completions, "create")+ kwargs = {"model": mdl.model_name, "messages": oai_messages, "temperature": temperature}+ if tools_list:+ kwargs["tools"] = tools_list+ return await create(**kwargs)+ # LiteLLMBase path — construct args via its own public method so provider+ # prefix / api_key / retries all ride along; then swap OUR tool schema in.+ import litellm++ args = mdl._construct_completion_args(+ history=oai_messages,+ stream=False,+ tools=bool(tools_list),+ temperature=temperature,+ )+ if tools_list:+ args["tools"] = tools_list+ args["tool_choice"] = "auto"+ args.setdefault("num_retries", 0)+ return await litellm.acompletion(**args, drop_params=True, timeout=timeout_s)+++async def _llm_once_with_tools(tools, mdl, messages: list):+ """ONE native-tool completion with THIS mode's tool surface.++ The exposed tool set is mode-dependent: ultra adds the relational+ ``graph_explore``; high/medium keep the 7-tool document/structure surface.+ """+ return await _acompletion(mdl, messages, tools_list=_active_tool_specs(tools), temperature=0.3)+++def _parse_tool_calls(msg) -> list:+ """Normalize provider-native tool_calls to [{"id", "name", "args"}].++ ``name`` is the tool name the model emitted (retrieve / list_chunks);+ unknown names are filtered so execute_tool never sees them.++ Some providers (MiniMax) return tool_calls WITHOUT a stable ``id``; the+ assistant message must pair each tool_call with a unique id that the+ ``tool`` response then references, otherwise the API rejects with+ "tool call result does not follow tool call". We synthesize a stable+ ``call_N`` id whenever the provider omitted one.+ """+ calls = []+ for i, tc in enumerate(msg.tool_calls or []):+ fn = tc.function+ name = getattr(fn, "name", None) if fn else None+ raw_args = fn.arguments if fn else "{}"+ try:+ args = json.loads(raw_args) if isinstance(raw_args, str) and raw_args.strip() else (raw_args or {})+ except Exception:+ args = {}+ if not isinstance(args, dict):+ args = {}+ if name not in _TOOL_MAP:+ _LOG.warning("[action_session] tool_call to unknown tool %r ignored", name)+ continue+ calls.append({"id": tc.id or f"call_{i}", "name": name, "args": args})+ return calls+++def _parse_terminal(content: str, parent_state: State) -> tuple:+ """Parse the two DeepSearch terminal output blocks.++ Returns ``(new_states, found_answer)`` — exactly one of which may be+ non-empty. ``<state>`` patches produce new-state branches; ``<answer>``+ ends the tree with a final answer (optionally patching the state too).+ """+ if "<state>" in content:+ block = extract_tag(content, "state") or "{}"+ data = extract_json(block) or {}+ raw_branches = data.get("new_states") or []+ if raw_branches and all(isinstance(b, dict) and "state" not in b for b in raw_branches):+ raw_branches = [{"state": [b]} for b in raw_branches]+ branches = []+ for br in raw_branches:+ ns = apply_patch(parent_state, br.get("state", []) if isinstance(br, dict) else [])+ if ns is not None:+ branches.append(ns)+ return branches, None+ if "<answer>" in content:+ block = extract_tag(content, "answer") or "{}"+ data = extract_json(block) or {}+ answer = str(data.get("answer", "")).strip() or None+ final_state = parent_state+ if isinstance(data.get("new_state"), list):+ patched = apply_patch(parent_state, data["new_state"])+ if patched is not None:+ final_state = patched+ return [final_state], answer+ return [], None+++# ── LangGraph subgraph state ─────────────────────────────────────────────+class _SessionState(TypedDict, total=False):+ messages: Annotated[list, add_messages]+ parent_state: State+ tools: Any+ mdl: Any+ # routing signal+ _pending_calls: list+ _done: bool+ _tool_cache: dict+ # terminal outputs+ new_states: list+ found_answer: Any+ retrieved_evidence_ids: list+ # budget+ attempts: int+ deadline_left: float+ _ctx_budget: int+ _tool_chars: int+++async def _run_action_node(state: _SessionState) -> dict:+ """One native-tool LLM turn; route on tool_calls / terminal / nudge."""+ mdl = state["mdl"]+ wall = max(+ 15.0,+ min(_ACTION_TIMEOUT_S, state.get("deadline_left") or _ACTION_TIMEOUT_S),+ )+ attempts = state.get("attempts", 0) + 1+ try:+ async with asyncio.timeout(wall):+ resp = await _llm_once_with_tools(state["tools"], mdl, state["messages"])+ except TimeoutError:+ _LOG.warning("[action_session] turn timed out after %.0fs", wall)+ return {"_done": True, "new_states": [], "found_answer": None, "attempts": attempts}+ except Exception as e: # noqa: BLE001 - 422 content-moderation / conn errors+ _LOG.warning("[action_session] LLM call failed (%s); converging session empty", type(e).__name__)+ return {"_done": True, "new_states": [], "found_answer": None, "attempts": attempts}+ msg = resp.choices[0].message+ content = msg.content or ""++ calls = _parse_tool_calls(msg)+ if calls:+ # pass the model's native tool_calls through verbatim so langgraph pairs+ # them with the tool responses; _pending_calls drives the tool node+ return {+ "messages": [+ {+ "role": "assistant",+ "content": content,+ "tool_calls": [+ {+ "id": c["id"],+ "type": "function",+ "function": {+ "name": c["name"],+ "arguments": json.dumps(c["args"], ensure_ascii=False),+ },+ }+ for c in calls+ ],+ }+ ],+ "_pending_calls": calls,+ "attempts": attempts,+ }++ new_states, found_answer = _parse_terminal(content, state["parent_state"])+ if found_answer is not None or new_states:+ return {+ "new_states": new_states,+ "found_answer": found_answer,+ "_done": True,+ "attempts": attempts,+ }++ # Neither a call nor a terminal block: nudge once per turn.+ return {+ "messages": [+ {"role": "assistant", "content": content},+ {+ "role": "user",+ "content": "Call the retrieve tool, or output a <state> patch, or emit <answer>.",+ },+ ],+ "_pending_calls": [],+ "attempts": attempts,+ }+++async def _tool_node(state: _SessionState) -> dict:+ """Execute pending native tool calls via execute_tool; append ``tool``+ responses (DeepSearch ToolNode equivalent) with a context budget that keeps+ the running conversation within bounds (DeepSearch context_limit strategy)."""+ tools = state["tools"]+ tool_msgs = []+ evidence_ids = list(state.get("retrieved_evidence_ids") or [])+ # crude context budget: track total chars of tool payloads so far; when+ # over budget we shorten each response instead of growing unboundedly.+ budget_chars = int(state.get("_ctx_budget", _MAX_TOOL_RESPONSE_CHARS * 4))+ # O(1) running budget: cumulative tool-payload chars tracked in _tool_chars+ # (a per-node rescan of every tool message was O(n) each call).+ used = int(state.get("_tool_chars", 0))+ cache = state.get("_tool_cache") or {}+ # The assistant message declared EVERY tool_call verbatim (DeepSearch+ # run_action passes raw_tool_calls through), so we MUST return a matching+ # tool response for each and every one of them — a malformed/partial+ # history (assistant.tool_calls without a tool response) is exactly what+ # makes the model re-issue redundant retrieves. The same-session cache only+ # avoids RE-EXECUTING a repeated query, it still returns a response for it.+ pending = state.get("_pending_calls") or []+ seen_queries = list(state.get("_search_queries") or [])+ skipped = 0+ for c in pending:+ # Near-duplicate retrieval suppression: if the model re-issues the same+ # intent as an earlier search (paraphrase), do NOT re-run ES — return a+ # nudge so it patches / reframes instead of burning turns (Q30 burned+ # 20+ "Culdcept creator" paraphrases in one session).+ q = str((c.get("args") or {}).get("query") or "").strip()+ if c["name"] in _RETRIEVAL_TOOLS and q and _is_near_dup(q, seen_queries):+ skipped += 1+ _LOG.info("[action_session] skipping near-duplicate retrieval %r (already searched)", q[:80])+ tool_msgs.append(+ {+ "role": "tool",+ "tool_call_id": c["id"],+ "content": json.dumps(+ {+ "passages": [+ {+ "kind": c["name"],+ "note": "This query is a near-duplicate of an earlier retrieval and was skipped to avoid redundant searching. Patch the slot with what you have, or issue a genuinely NEW retrieval angle.",+ }+ ]+ },+ ensure_ascii=False,+ ),+ }+ )+ continue+ # same-session tool cache: avoid re-grep/list_chunks on the same target+ cache_key = (c["name"], json.dumps(c["args"], sort_keys=True, ensure_ascii=False))+ if cache_key in cache:+ chunks, ids = cache[cache_key]+ else:+ chunks, ids = await execute_tool(tools, c["name"], c["args"])+ cache[cache_key] = (chunks, ids)+ if q:+ seen_queries.append(q)+ evidence_ids.extend(ids)+ payload = json.dumps({"passages": chunks}, ensure_ascii=False, default=str)+ # if the session is already heavy, cut this payload proportionally+ if used + len(payload) > budget_chars:+ keep = max(800, budget_chars - used)+ payload = payload[:keep]+ used += len(payload)+ tool_msgs.append(+ {+ "role": "tool",+ "tool_call_id": c["id"],+ "content": payload,+ }+ )+ return {+ "messages": tool_msgs,+ "_pending_calls": [],+ "retrieved_evidence_ids": evidence_ids,+ "_tool_cache": cache,+ "_tool_chars": used,+ "_search_queries": seen_queries,+ "_skipped_dup": int(state.get("_skipped_dup", 0)) + skipped,+ }+++async def _finalize_node(state: _SessionState) -> dict:+ """Tool budget spent: ONE last call WITHOUT tools demanding the terminal+ JSON, salvaging whatever the session learned."""+ parent = state["parent_state"]+ new_states, found_answer = [], None+ evidence_ids = list(state.get("retrieved_evidence_ids") or [])++ budget_prompt = (+ "TOOL BUDGET EXHAUSTED. Based ONLY on the passages retrieved above, "+ "output now — no prose outside the block:\n"+ '<state>{"new_states": [{"state": [{"id": <slot_id>, "candidate": "<value>", '+ '"candidate_strength": <0..1>, "discovered_clues": ["..."]}]}]}</state>\n'+ 'If NOTHING was learned use: <state>{"new_states": []}</state>'+ )+ # Defensive: strip any assistant.tool_calls that never got a tool response+ # (e.g. tool execution failed), else the provider rejects the history.+ finalize_msgs = _strip_unpaired_tool_calls(list(state["messages"])) + [HumanMessage(content=budget_prompt)]+ try:+ async with asyncio.timeout(max(15.0, min(45.0, state.get("deadline_left") or 45.0))):+ fresp = await _acompletion(+ state["mdl"],+ finalize_msgs,+ tools_list=None,+ temperature=0.3,+ timeout_s=45.0,+ )+ fcontent = fresp.choices[0].message.content or ""+ new_states, found_answer = _parse_terminal(fcontent, parent)+ if found_answer:+ _LOG.info("[action_session] answer salvaged from exhausted session")+ elif new_states:+ _LOG.info("[action_session] %d branch(es) salvaged from exhausted session", len(new_states))+ except Exception:+ _LOG.exception("[action_session] salvage call failed")++ # Loose-clue harvest (deterministic, zero-LLM): even when every JSON+ # protocol attempt failed, the last narration often carries facts worth+ # keeping as breadcrumbs.+ if not new_states and not found_answer:+ loose_clues = []+ for m_ in reversed(state["messages"]):+ if getattr(m_, "type", "") == "ai" and str(getattr(m_, "content", "") or "").strip():+ txt = str(m_.content).strip()+ if len(txt) >= 24:+ loose_clues = [f"narrative: {txt[:220]}"]+ break+ if loose_clues:+ unresolved = parent.unresolved()+ target_id = unresolved[0].id if unresolved else (parent.state[0].id if parent.state else None)+ if target_id is not None:+ patched = apply_patch(+ parent,+ [{"id": target_id, "discovered_clues": loose_clues}],+ )+ if patched is not None:+ new_states = [patched]+ _LOG.warning("[action_session] loose-clue patch (narrative)")++ return {"new_states": new_states, "found_answer": found_answer, "_done": True, "retrieved_evidence_ids": evidence_ids}+++def _strip_unpaired_tool_calls(messages: list) -> list:+ """Remove assistant.tool_calls that never received a matching tool response+ so the conversation handed to the provider stays well-formed (MiniMax 400+ "tool call result does not follow tool call" otherwise). Keeps content."""+ responded = set()+ for m_ in messages:+ if getattr(m_, "type", "") == "tool":+ responded.add(getattr(m_, "tool_call_id", None))+ cleaned = []+ for m_ in messages:+ if getattr(m_, "type", "") == "ai" and getattr(m_, "tool_calls", None):+ paired = all(tc.get("id") in responded for tc in (m_.tool_calls or []))+ if not paired:+ # keep the assistant text, drop the unpaired tool_calls+ try:+ cleaned.append(m_.__class__(content=m_.content))+ except Exception: # noqa: BLE001 - best-effort strip+ cleaned.append(m_)+ continue+ cleaned.append(m_)+ return cleaned+++def _action_max_turns(state: _SessionState) -> int:+ """Per-mode action-session turn budget: ultra goes deeper inside each+ session (more tool calls before the finalize fallback) — the latency cost+ of the relational depth that distinguishes it from high."""+ tools = state.get("tools")+ mode = str(getattr(tools, "thinking_mode", "") or "").lower()+ return _ACTION_MAX_TURNS_ULTRA if mode == "ultra" else _ACTION_MAX_TURNS+++def _route(state: _SessionState) -> str:+ if state.get("_done"):+ return END+ # Run pending tool_calls FIRST, even at the turn budget: leaving an+ # assistant.tool_calls message without its tool response makes the provider+ # reject the next call ("tool call result does not follow tool call"). The+ # tool node clears _pending_calls, then the route re-checks the budget.+ if state.get("_pending_calls"):+ return "tool"+ if state.get("attempts", 0) >= _action_max_turns(state):+ return "finalize"+ # No-progress convergence: if the model keeps re-issuing near-duplicate+ # retrieval queries (skipped >=2), further turns are unlikely to surface new+ # evidence — converge to finalize instead of burning the remaining budget on+ # paraphrases (Q30/Q759 timeout root cause).+ if int(state.get("_skipped_dup", 0)) >= 2:+ _LOG.info("[action_session] %d near-duplicate retrieval(s) skipped; converging session early", int(state.get("_skipped_dup", 0)))+ return "finalize"+ return "run_action" # nudge retry+++def _build_session_graph():+ g = StateGraph(_SessionState)+ g.add_node("run_action", _run_action_node)+ g.add_node("tool", _tool_node)+ g.add_node("finalize", _finalize_node)+ g.add_edge(START, "run_action")+ g.add_conditional_edges(+ "run_action",+ _route,+ {"tool": "tool", "finalize": "finalize", END: END, "run_action": "run_action"},+ )+ g.add_edge("tool", "run_action")+ g.add_edge("finalize", END)+ return g.compile()+++_SESSION_GRAPH = _build_session_graph()+++async def run_action_session(+ tools,+ direction: str,+ parent_state: State,+ deadline_left: float | None = None,+ base_summary: str = "",+) -> Result:+ """Bounded graph-edge session pursuing ONE direction."""+ from rag.prompts.template import load_prompt++ system = load_prompt("action_run")+ seed_user = f"Direction: {direction}\n\nState:\n{parent_state.render_slots()}"+ if base_summary:+ seed_user += f"\n\nPrior round summary:\n{base_summary}"++ from rag.advanced_rag.harness.tools.search import _base_chat_mdl++ mdl = _base_chat_mdl(tools)+ if mdl is None:+ _LOG.warning("[action_session] no usable model resolved for action session")+ return Result(messages=[], new_states=[])++ initial: _SessionState = {+ "messages": [SystemMessage(content=system), HumanMessage(content=seed_user)],+ "parent_state": parent_state,+ "tools": tools,+ "mdl": mdl,+ "_pending_calls": [],+ "_done": False,+ "new_states": [],+ "found_answer": None,+ "retrieved_evidence_ids": [],+ "attempts": 0,+ "deadline_left": deadline_left or _ACTION_TIMEOUT_S,+ "_ctx_budget": _MAX_TOOL_RESPONSE_CHARS * 4,+ "_tool_chars": 0,+ "_tool_cache": {},+ "_search_queries": [],+ "_skipped_dup": 0,+ }+ try:+ final = await _SESSION_GRAPH.ainvoke(initial)+ except Exception:+ _LOG.exception("[action_session] session failed")+ return Result(messages=[], new_states=[])+ return Result(+ messages=final.get("messages", []),+ new_states=final.get("new_states", []),+ found_answer=final.get("found_answer"),+ retrieved_evidence_ids=final.get("retrieved_evidence_ids", []),+ )+++# ── Slot-table builder (ex-tree/runner.py initialize_state) ──────────────+async def _init_chat(tools, system: str, user: str, tmo: float) -> str:+ """ONE bounded LLM turn for the slot-table decomposition."""+ from rag.advanced_rag.harness.tools.search import _base_chat_mdl++ mdl = _base_chat_mdl(tools)+ if mdl is None:+ return ""+ try:+ async with asyncio.timeout(tmo):+ ans, _u = await mdl.async_chat(system, [{"role": "user", "content": user}], {"temperature": 0.3})+ return str(ans or "")+ except TimeoutError:+ _LOG.warning("[Action Session:init] timed out (%ds)", tmo)+ except Exception:+ _LOG.exception("[Action Session:init] failed")+ return ""+++async def initialize_state(tools, question, fanout_hint, deadline_left=None):+ from rag.prompts.template import load_prompt++ system = load_prompt("action_initialize_state")+ user = f"Question: {question}"+ if fanout_hint:+ user += "\n\nCandidate aspects already identified:\n" + "\n".join(f"- {h}" for h in fanout_hint)+ tmo = min(_INIT_TIMEOUT_S, deadline_left or _INIT_TIMEOUT_S)+ raw = await _init_chat(tools, system, user, tmo)+ data = extract_json(raw) or {}+ if not data:+ # one quick retry — transient provider stalls were observed (45s with+ # zero bytes); a second attempt succeeded in production logs.+ raw = await _init_chat(tools, system, user, tmo)+ data = extract_json(raw) or {}+ slots = []+ for i, s in enumerate(data.get("slots") or []):+ if isinstance(s, dict):+ slots.append(+ Variable(+ id=int(s.get("id", i)),+ type=str(s.get("type") or "entity"),+ question_clues=[str(c) for c in (s.get("clues") or [])][:4],+ )+ )+ first_queries = [str(q).strip() for q in (data.get("first_queries") or [])][:3]+ ans_var = data.get("answer_variable")+ if not slots:+ # Decomposition failed (timeout/parse): build the table from planner+ # fanouts so the first round still targets DISTINCT aspects instead of+ # one oversized query (observed: single-slot trees answered directly+ # and died, or grepped the raw question and missed).+ hint_slots = [Variable(id=i, type="aspect", question_clues=[str(h)[:120]]) for i, h in enumerate((fanout_hint or [])[:4])]+ if hint_slots:+ slots = hint_slots+ ans_var = 0+ if not first_queries:+ first_queries = [str(h) for h in (fanout_hint or [])[:3]]+ else:+ slots = [Variable(id=0, type="answer", question_clues=[question])]+ ans_var = 0+ if not first_queries:+ first_queries = [question]+ elif not first_queries:+ first_queries = [question]+ ans_slot = ans_var if isinstance(ans_var, int) and 0 <= ans_var < len(slots) else 0+ root = State(state=slots, depth=0, answer_variable=ans_slot)+ _LOG.info("[Action Session:init] %s", root.brief())+ return root, first_queriesdiff --git a/rag/advanced_rag/harness/agent.py b/rag/advanced_rag/harness/agent.pydeleted file mode 100644index 684291716d..0000000000--- a/rag/advanced_rag/harness/agent.py+++ /dev/null@@ -1,495 +0,0 @@-"""Research Agent — inner tool-calling loop for high/ultra modes.--Native tool-calling: a chat model cloned from ``tools.chat_mdl`` is bound-(via ``bind_tools``) to the phase-gated tool schemas plus ``think_tool`` /-``generate_report``, and a lightweight session routes each tool call to the-harness pipeline. Binding onto a *copy* keeps the shared ``tools.chat_mdl``-(used by the other graph nodes) free of any tool schema.--Models without native tool-calling fall back to prompt-based tool selection:-the tools are described in the prompt and the model emits ``<tool_call>`` JSON-that the loop parses.-"""--import json-import logging-import re--from rag.advanced_rag.harness.pipeline import Pipeline-from rag.advanced_rag.harness.prompts.research_agent_prompt import (- RESEARCH_AGENT_PROMPT,- RESEARCH_AGENT_TEXT_PROMPT,-)-from rag.advanced_rag.harness.stats import in_phase-from rag.advanced_rag.harness.tools.gating import (- SEARCH_PHASES,- determine_current_phase,- get_gated_tools,-)-from rag.advanced_rag.harness.tools.registry import _generate_report_schema, _think_schema-from rag.advanced_rag.harness.types import ClaimTarget, ExecutionStrategy, ToolResult--_LOG = logging.getLogger(__name__)---# Navigation tools that return passages (evidence chunks). After one of these-# runs, we judge sufficiency before the agent broadens into a full search.-_NAV_CHUNK_TOOLS = {"ontology_navigate", "mindmap_navigate"}---class ResearchToolSession:- """ToolCallSession adapter routing native tool calls to the harness pipeline.-- - regular tools run through :func:`execute_with_fallback`;- - ``think_tool`` is a no-op reasoning step that just lets the loop continue;- - ``generate_report`` is *captured* (not executed) so the agent loop can- return its structured arguments as the claim result.- """-- def __init__(self, pipeline: Pipeline, phase: str, claim: ClaimTarget | None = None):- self.pipeline = pipeline- self.phase = phase- self.claim = claim- self.report: dict | None = None- self.got_evidence = False- self.evidence_ids: list[int] = []- self._seen_evidence_ids: set[int] = set()-- async def tool_call_async(self, name: str, arguments: dict, request_timeout: float = 300):- arguments = arguments or {}- if name == "generate_report":- self.report = self._normalize_report(arguments)- return "Report received. Stop calling tools now."- if name == "think_tool":- return "Noted. Proceed with the next tool call."- result = await execute_with_fallback(self.pipeline, name, self.phase, **arguments)- if result.chunks:- self.got_evidence = True- self._record_evidence_ids(result.chunks)- message = _fmt_tool_result(result)- # Post-navigation sufficiency gate: once a navigation tool has returned- # passages, decide whether they already answer the task and, if so, steer- # the agent to finalize instead of running a broader (noisier) search.- if name in _NAV_CHUNK_TOOLS and result.chunks and self.claim is not None:- if await self._navigation_sufficient(result.chunks):- ev = ", ".join(str(i) for i in self.evidence_ids) or "the passages above"- message += f"\n\n[sufficiency check] These passages appear to answer the task. Call generate_report now with evidence_ids=[{ev}] — do not run further searches."- return message-- async def _navigation_sufficient(self, chunks: list[dict]) -> bool:- """One-shot LLM judge: do the given passages already answer the claim?"""- try:- passages = "\n\n".join((c.get("content_with_weight") or c.get("text") or "")[:500] for c in chunks[:8])- if not passages.strip():- return False- system = "Judge whether the passages already contain enough information to answer the task. Reply with a single word: YES or NO."- user = f"Task:\n{self.claim.description}\n\nPassages:\n{passages}\n\nAnswer (YES or NO):"- ans = await self.pipeline.tools.chat_mdl.async_chat(system, [{"role": "user", "content": user}], {"temperature": 0.0})- if isinstance(ans, tuple):- ans = ans[0]- ans = re.sub(r"^.*</think>", "", ans or "", flags=re.DOTALL)- _LOG.debug("[Navigation] sufficiency check: %s", ans)- return ans.strip().lower().startswith("yes")- except Exception:- _LOG.exception("[Navigation] sufficiency check failed")- return False-- def _normalize_report(self, report: dict) -> dict:- if not isinstance(report, dict):- # Unstrusted text-path parser output; never crash on it.- _LOG.warning("normalize_report: expected dict, got %s; using empty report", type(report).__name__)- report = {}- normalized = dict(report)- recorded = set(self.evidence_ids)- evidence_ids = []- # The schema requires evidence_ids to be a list of integers. A scalar or- # string would either raise or, worse, let the loop walk characters as- # IDs, so coerce anything else to an empty list first.- raw_evidence_ids = normalized.get("evidence_ids")- if not isinstance(raw_evidence_ids, list):- raw_evidence_ids = []- for eid in raw_evidence_ids:- # The schema permits integer IDs only. Reject booleans and floats- # before conversion: int(True) == 1 and int(1.9) == 1 would both- # silently become valid indexes, retaining evidence the model never- # referenced. Keep only ints and numeric strings.- if isinstance(eid, bool) or not isinstance(eid, (int, str)):- continue- try:- idx = int(eid)- except ValueError:- continue- # Restrict to chunks this claim actually recorded. Claims are- # researched concurrently over a *shared* kbinfos pool, so a model- # can otherwise cite another claim's chunk (or an out-of-range- # index) via an arbitrary evidence_id.- if idx in recorded:- if idx not in evidence_ids:- evidence_ids.append(idx)- elif recorded:- _LOG.warning(- "evidence_id %s not in claim's recorded set (size=%d); dropping",- idx,- len(recorded),- )- if not evidence_ids:- # Do NOT repurpose every recorded chunk as a citation: evidence_ids- # means the chunks the report actually references, and the schema- # defines it that way. When no valid ID remains, leave it empty and- # mark the report unverified so downstream verifiers (which combine- # evidence_ids + is_verified + confidence) cannot mistake unrelated- # chunks for supporting evidence.- normalized["is_verified"] = False- normalized["evidence_ids"] = evidence_ids- return normalized-- def _record_evidence_ids(self, chunks: list[dict]) -> None:- all_chunks = self.pipeline.tools.kbinfos.get("chunks", [])- index_by_key = {}- for idx, chunk in enumerate(all_chunks):- index_by_key[_chunk_key(chunk)] = idx-- for chunk in chunks:- idx = index_by_key.get(_chunk_key(chunk))- if idx is None:- idx = next((i for i, existing in enumerate(all_chunks) if existing is chunk), None)- if idx is None or idx in self._seen_evidence_ids:- continue- self._seen_evidence_ids.add(idx)- self.evidence_ids.append(idx)---def _chunk_key(chunk: dict) -> object:- return chunk.get("chunk_id") or chunk.get("id") or id(chunk)---def _build_tool_schemas(gated_defs: list[dict]) -> list[dict]:- """Phase-gated schemas (minus harness-only ``x_*`` keys) + the control tools."""- schemas: list[dict] = []- for d in gated_defs:- schemas.append({k: v for k, v in d.items() if not k.startswith("x_")})- schemas.append(_think_schema())- schemas.append(_generate_report_schema())- return schemas---@in_phase("claim_research")-async def research_agent_loop(- claim: ClaimTarget,- tools,- pipeline: Pipeline,- context,- mode: ExecutionStrategy,- compilation_map: dict,- followups: list[str] | None = None,-) -> dict:- """Inner loop for a single claim — native tool-calling with a text fallback.-- ``followups`` (Phase-2 LLM missing-pieces feedback from the previous- sufficiency round) is passed in explicitly by the orchestrator rather than- read from the shared ``context`` here. The orchestrator consumes and clears- ``context.pending_followups`` ONCE per round so every claim researched in a- parallel batch receives the SAME follow-up guidance (reading the shared list- per-claim would race: the first claim to execute would clear it, starving- the rest).- """- phase = determine_current_phase(context, claim=claim)- phase_config = SEARCH_PHASES.get(phase, {})- gated_defs = get_gated_tools(- phase=phase,- available_tools=mode.available_tools,- compilation_map=compilation_map,- context=context,- has_routed_scope=bool(getattr(pipeline, "_routed_docs", None)),- web_enabled=bool(getattr(tools, "has_web", lambda: False)()),- claim=claim,- )-- pipeline._active_phase = phase- pipeline._round_had_evidence = False- pipeline._round_had_routed_scope_progress = False-- # Clone so binding tools never leaks onto the shared chat model.- agent_mdl = tools.chat_mdl.clone()- if getattr(agent_mdl, "is_tools", False):- result = await _research_native(claim, agent_mdl, pipeline, phase, phase_config, gated_defs, mode, followups)- else:- _LOG.info("research_agent: model lacks native tool support; falling back to text-based tool selection")- result = await _research_text(claim, tools, pipeline, phase, phase_config, gated_defs, mode, followups)-- # Bookkeeping for locate-phase web fallback: a locate round that produced- # evidence chunks or newly routed document scope counts as progress and- # resets this claim's streak. Pre-existing request doc_scope does not- # count: only scope produced by this round should prevent the fallback.- # The phase stays `locate`; gating.py decides when repeated locate- # failures should admit `web_search` into the candidate tool set.- if pipeline._round_had_evidence or pipeline._round_had_routed_scope_progress:- claim.locate_empty_streak = 0- elif phase == "locate":- claim.locate_empty_streak += 1- return result---async def _research_native(- claim: ClaimTarget,- agent_mdl,- pipeline: Pipeline,- phase: str,- phase_config: dict,- gated_defs: list[dict],- mode: ExecutionStrategy,- followups: list[str] | None = None,-) -> dict:- """Bind tools onto ``agent_mdl`` and let its native tool loop drive research."""- schemas = _build_tool_schemas(gated_defs)- session = ResearchToolSession(pipeline, phase, claim)- agent_mdl.bind_tools(session, schemas)- # Bound the model's internal tool loop to the mode's agent-cycle budget.- base_rounds = max(1, mode.max_agent_cycles)- if hasattr(agent_mdl, "mdl") and hasattr(agent_mdl.mdl, "max_rounds"):- agent_mdl.mdl.max_rounds = base_rounds-- system = RESEARCH_AGENT_PROMPT.format(- claim_description=claim.description,- phase=phase,- phase_hint=phase_config.get("tool_hint", ""),- max_cycles=mode.max_agent_cycles,- )- history = [{"role": "user", "content": f"Research task: {claim.description}\nBegin."}]- # Phase-2 missing-pieces guidance: focus this round on the specific gaps the- # Sufficient Context AutoRater flagged, rather than re-searching broadly.- if followups:- history.append(- {- "role": "user",- "content": "Previous evidence was incomplete. Run targeted searches specifically for the following missing pieces:\n- " + "\n- ".join(followups),- }- )-- final_text = ""- try:- final_text = await agent_mdl.async_chat(system, history, {"temperature": 0.3})- if isinstance(final_text, tuple):- final_text = final_text[0]- except Exception:- _LOG.exception("research_agent(native): tool loop failed")-- if session.report is not None:- return session.report-- # The model finished without calling generate_report — synthesize a report- # from its final free-text turn so the claim still yields something usable.- _LOG.info("research_agent(native): no generate_report call; using final text as report")- return {- "report": (final_text or "").strip(),- "is_verified": session.got_evidence,- "confidence": 0.5 if session.got_evidence else 0.0,- "evidence_ids": list(session.evidence_ids),- "gaps": [] if session.got_evidence else ["no generate_report emitted"],- "discovered_claims": [],- }---async def _research_text(- claim: ClaimTarget,- tools,- pipeline: Pipeline,- phase: str,- phase_config: dict,- gated_defs: list[dict],- mode: ExecutionStrategy,- followups: list[str] | None = None,-) -> dict:- """Fallback: prompt-based tool selection for models without native tools."""- # Mirror the native path's cycle budget; the locate→web_search fallback is- # handled by gating.get_gated_tools injecting web_search into the tool set.- text_max_cycles = mode.max_agent_cycles- system = RESEARCH_AGENT_TEXT_PROMPT.format(- claim_description=claim.description,- phase=phase,- phase_hint=phase_config.get("tool_hint", ""),- tool_list=_fmt_tool_list(gated_defs),- max_cycles=text_max_cycles,- )-- history: list[dict] = []- if followups:- history.append(- {- "role": "user",- "content": "Previous evidence was incomplete. Run targeted searches specifically for the following missing pieces:\n- " + "\n- ".join(followups),- }- )-- # Use a session so evidence IDs are recorded and the final report is- # normalized against this claim's recorded chunks (same as the native path).- session = ResearchToolSession(pipeline, phase, claim)-- for cycle in range(text_max_cycles):- try:- ans = await tools.chat_mdl.async_chat(system, history, {"temperature": 0.3})- if isinstance(ans, tuple):- ans = ans[0]- except Exception:- _LOG.exception("research_agent(text): LLM call failed cycle %d", cycle)- continue-- history.append({"role": "assistant", "content": ans})-- tool_call = _parse_tool_call(ans)- if not tool_call:- history.append({"role": "user", "content": "Please call a tool. Do not output plain text."})- continue-- if tool_call.get("name") == "generate_report":- args = tool_call.get("arguments", {})- if not isinstance(args, dict):- _LOG.warning("generate_report: arguments not a dict (%s); using empty", type(args).__name__)- args = {}- report = session._normalize_report(args)- return report-- if tool_call.get("name") == "think_tool":- history.append({"role": "user", "content": "[continue]"})- continue-- args = tool_call.get("arguments", {})- # Text fallback bypasses ResearchToolSession.tool_call_async(), so it- result = await execute_with_fallback(pipeline, tool_call["name"], phase, **args)- if result.chunks:- session._record_evidence_ids(result.chunks)- history.append({"role": "user", "content": _fmt_tool_result(result)})-- return await _force_generate_report(history, tools, claim.claim_id, session)---def _parse_tool_call(text: str) -> dict | None:- """Parse tool call from LLM response text (text-fallback path)."""- m = re.search(r"<tool_call>(.*?)</tool_call>", text, re.DOTALL)- if m:- try:- return json.loads(m.group(1).strip())- except Exception:- pass-- m = re.search(r"```(?:json)?\s*({.*?})\s*```", text, re.DOTALL)- if m:- try:- return json.loads(m.group(1).strip())- except Exception:- pass-- m = re.search(r'\{\s*"name"\s*:', text)- if m:- try:- import json_repair-- return json_repair.loads(text)- except Exception:- pass-- return None---async def execute_with_fallback(- pipeline: Pipeline,- tool_name: str,- phase: str,- **kwargs,-) -> ToolResult:- """Execute tool; if empty, fall back along phase priority."""- result = await pipeline.execute(tool_name, **kwargs)-- if result.chunks or result.error:- return result-- phase_config = SEARCH_PHASES.get(phase, {})- priority = phase_config.get("tools_priority", [])- current_idx = next(- (i for i, t in enumerate(priority) if t == tool_name),- -1,- )- for fallback_name in priority[current_idx + 1 :]:- fallback_result = await pipeline.execute(fallback_name, **kwargs)- if fallback_result.chunks:- _LOG.info("fallback: %s empty → %s found %d chunks", tool_name, fallback_name, len(fallback_result.chunks))- fallback_result.metadata["was_fallback"] = True- fallback_result.metadata["fallback_from"] = tool_name- return fallback_result- if fallback_result.error:- break- return result---async def _force_generate_report(- history: list,- tools,- claim_id: str,- session: ResearchToolSession | None = None,-) -> dict:- """Force generate report when max cycles reached (text-fallback path)."""- try:- ans = await tools.chat_mdl.async_chat(- "",- history + [{"role": "user", "content": "We've reached the research limit. Please output a final report as JSON."}],- {"temperature": 0.3},- )- if isinstance(ans, tuple):- ans = ans[0]- text = re.sub(r"```(?:json)?\s*|\s*```", "", ans).strip()- import json_repair-- report = json_repair.loads(text)- if isinstance(report, dict) and session is not None:- normalized = session._normalize_report(report)- return normalized- return report if isinstance(report, dict) else {"report": str(report)}- except Exception:- _LOG.exception("force_generate_report failed")- return {- "report": "",- "is_verified": False,- "confidence": 0.0,- "evidence_ids": [],- "gaps": ["forced report — data may be incomplete"],- "discovered_claims": [],- }---def _fmt_tool_list(defs: list[dict]) -> str:- lines = []- for d in defs:- func = d.get("function", d)- name = func.get("name", "?")- desc = func.get("description", "")- params = func.get("parameters", {}).get("properties", {})- params_text = ", ".join(f"{k}: {v.get('description', '')}" for k, v in params.items())- lines.append(f"- {name}: {desc}")- if params_text:- lines.append(f" Parameters: {params_text}")- return "\n".join(lines)---def _fmt_tool_result(result: ToolResult) -> str:- if result.error:- return f"[tool error] {result.error}"- parts: list[str] = []- # Some tools (ontology_navigate, structured_query) answer directly rather than- # only returning passages — surface that, or the agent never sees it.- answer = (result.metadata or {}).get("answer") if isinstance(result.metadata, dict) else ""- if answer:- parts.append(f"Answer: {answer}")- # Show more chunks (6 vs the old 3) and order them by retrieval similarity,- # so the agent actually sees the passages that best match its query. The old- # "first 3 chunks, 300 chars each" made the agent miss the key evidence when- # it sat in chunk 4+ (5.log/6.log: it retrieved 48m/27m but reported- # "unrelated (Barack Obama)" and re-searched endlessly).- chunks = list(result.chunks)- chunks.sort(key=lambda c: float(c.get("similarity", 0.0) or 0.0), reverse=True)- for c in chunks[:6]:- text = c.get("content_with_weight") or c.get("text") or ""- if text:- parts.append(text[:300])- if not parts:- return "[no results found]"- return "\n\n".join(parts)diff --git a/rag/advanced_rag/harness/arithmetic.py b/rag/advanced_rag/harness/arithmetic.pynew file mode 100644index 0000000000..96043fd911--- /dev/null+++ b/rag/advanced_rag/harness/arithmetic.py@@ -0,0 +1,420 @@+"""Deterministic arithmetic over retrieved evidence, ported from the+agentic_search4 v8 keyword graph.++Some questions ask for a number no single source states — the combined+population of three counties, how many of the listed films won an award, the+years between two dates. Every input is in the evidence by then and only the+arithmetic is missing, which an LLM does by writing digits one at a time and+gets wrong often enough to matter. So the LLM writes ONE Python expression and+Python evaluates it.++The expression comes from a language model, so it is NOT trusted. It is parsed+and every node checked against an AST whitelist BEFORE evaluation; anything+unlisted — an attribute access, a subscript, a lambda, an f-string, a name that+is not one of the allowed functions — is rejected outright rather than sandboxed+at run time. Evaluation then runs with no builtins at all.++Public interface+----------------+``compute(expression)`` -> ``(rendered, error)`` evaluate one expression safely.+``compute_from_facts(question, facts)`` -> optional ``{label, value, expression, uses}``+ LLM decides whether the question asks for a derivable number, writes the+ expression, evaluates it, and returns a structured result for the caller to+ attach to its evidence list.+"""++import ast+import logging++_LOG = logging.getLogger(__name__)++_COMPUTE_MAX_CHARS = 400 # the whole expression; every figure is inline, none is long+++def _letters(*texts: object) -> int:+ """Count the alphabetic characters across the given names.++ "How many letters are in these names" is a real question, and every+ plain-Python way to answer it needs machinery this evaluator refuses — an+ attribute call (``"".join``, ``str.isalpha``) or a comprehension. So it is a+ function instead.++ Spaces, hyphens, apostrophes, digits and punctuation do NOT count; letters+ carrying diacritics DO ("José" is 4), because they are letters of the name.+ Takes any number of names, or a single list of them.+ """+ total = 0+ for text in texts:+ for item in text if isinstance(text, (list, tuple, set)) else [text]:+ if not isinstance(item, str):+ raise TypeError(f"letters() takes names, not {type(item).__name__}")+ total += sum(1 for ch in item if ch.isalpha())+ return total+++def _date_diff(*dates: str) -> int:+ """Days between two ISO dates (inclusive of the earlier, exclusive of the+ later — a calendar span, the same as "how many days between two dates" on a+ desk calendar).++ ``date_diff("1941-07-28", "1959-07-17")`` -> the number of days from the+ first date to the second. Exactly two ISO ``YYYY-MM-DD`` dates are required;+ anything else is refused. This is the arithmetic the old ``years between two+ dates -> 1998 - 1954`` example could NOT express, so a "how many days after+ his death did X die" question (Q317: 6563 days between two death dates) fell+ back to ``needed:false`` and the final answer stopped at the two dates.+ """+ if len(dates) != 2:+ raise TypeError("date_diff() takes exactly two ISO dates")+ from datetime import date as _date++ parsed = []+ for d in dates:+ if not isinstance(d, str):+ raise TypeError(f"date_diff() takes ISO date strings, not {type(d).__name__}")+ parts = d.strip().split("-")+ if len(parts) != 3:+ raise ValueError(f"not an ISO date: {d!r}")+ try:+ parsed.append(_date(int(parts[0]), int(parts[1]), int(parts[2])))+ except ValueError:+ raise ValueError(f"not a valid ISO date: {d!r}")+ return abs((parsed[1] - parsed[0]).days)+++def _digit_sum(*texts: object) -> int:+ """Add up the decimal digits inside the given values.++ "What do you get when you add up the numbers in the postcode" is a real+ question, and the plain-Python answer needs the comprehension this evaluator+ refuses — so, like ``letters``, it is a function instead of an expression.++ Every digit is added SEPARATELY: digit_sum("L7 7BN") is 14 and digit_sum("2020")+ is 4. A question that means whole numbers added together ("66 + 12") is written+ with those literals instead, because the value of a multi-digit number is not+ the sum of anything.++ Only ASCII digits count — a superscript or a fraction glyph is not a digit of+ the postcode. Takes any number of strings or whole numbers, or a list of them.+ """+ total = 0+ for text in texts:+ for item in text if isinstance(text, (list, tuple, set)) else [text]:+ if isinstance(item, bool) or not isinstance(item, (str, int)):+ raise TypeError(f"digit_sum() takes text or whole numbers, not {type(item).__name__}")+ total += sum(int(ch) for ch in str(item) if "0" <= ch <= "9")+ return total+++# Pure arithmetic on literals. Nothing here can reach an object, a module or a name.+_COMPUTE_FUNCTIONS = {+ "abs": abs,+ "round": round,+ "min": min,+ "max": max,+ "sum": sum,+ "len": len,+ "int": int,+ "float": float,+ "sorted": sorted,+ "letters": _letters,+ "digit_sum": _digit_sum,+ "date_diff": _date_diff,+}++_COMPUTE_NODES = (+ ast.Expression,+ ast.Constant,+ ast.Tuple,+ ast.List,+ ast.Set,+ ast.Load,+ ast.Name,+ ast.Call,+ ast.IfExp,+ ast.UnaryOp,+ ast.UAdd,+ ast.USub,+ ast.Not,+ ast.BinOp,+ ast.Add,+ ast.Sub,+ ast.Mult,+ ast.Div,+ ast.FloorDiv,+ ast.Mod,+ ast.Pow,+ ast.BoolOp,+ ast.And,+ ast.Or,+ ast.Compare,+ ast.Eq,+ ast.NotEq,+ ast.Lt,+ ast.LtE,+ ast.Gt,+ ast.GtE,+)+++# Functions whose result is a number whatever they are handed. `min`, `max` and+# `sum` are absent on purpose — min("b", "a") is a string — and `sorted` returns a+# list, so neither may stand where a number is required.+_COMPUTE_ALWAYS_NUMERIC = {"abs", "round", "int", "float", "len", "letters", "digit_sum", "date_diff"}+++def _is_numeric(node: ast.AST) -> bool:+ """True when ``node`` can ONLY evaluate to a number.++ Multiplication is the one operator that turns a short expression into an+ arbitrarily large object — ``"a" * 10**9``, ``[1] * 10**9`` — so both its+ operands must be provably numeric. Everything else in the whitelist either+ cannot grow (``+`` on literals is bounded by the expression length) or is+ already constrained (``**`` takes plain numbers only).+ """+ if isinstance(node, ast.Constant):+ return isinstance(node.value, (int, float)) # bool included: it IS an int+ if isinstance(node, ast.UnaryOp):+ return _is_numeric(node.operand)+ if isinstance(node, ast.BinOp):+ return _is_numeric(node.left) and _is_numeric(node.right)+ if isinstance(node, ast.IfExp):+ return _is_numeric(node.body) and _is_numeric(node.orelse)+ if isinstance(node, ast.Compare):+ return True # a comparison is a bool, and a bool is an int+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):+ if node.func.id in _COMPUTE_ALWAYS_NUMERIC:+ return True+ if node.func.id in {"sum", "min", "max"}:+ return all(_is_numeric(arg) or _is_numeric_sequence(arg) for arg in node.args)+ return False+++def _is_numeric_sequence(node: ast.AST) -> bool:+ """True for a literal list/tuple/set whose every element is provably numeric."""+ return isinstance(node, (ast.List, ast.Tuple, ast.Set)) and all(_is_numeric(element) for element in node.elts)+++def _check_expression(tree: ast.AST) -> str:+ """Reject anything outside the arithmetic whitelist. Returns "" when clean."""+ for node in ast.walk(tree):+ if not isinstance(node, _COMPUTE_NODES):+ return f"{type(node).__name__} is not allowed"+ if isinstance(node, ast.Name) and node.id not in _COMPUTE_FUNCTIONS:+ return f"unknown name {node.id!r}"+ if isinstance(node, ast.Call):+ if not isinstance(node.func, ast.Name) or node.func.id not in _COMPUTE_FUNCTIONS:+ return "only the listed functions may be called"+ if node.keywords:+ return "keyword arguments are not allowed"+ # `len("Ada Lovelace")` is 12 and the answer is 11. The gap is silent, so+ # the expression is refused rather than counted.+ if node.func.id == "len" and node.args and isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str):+ return "len() on a string literal is ambiguous; use letters()"+ for arg in node.args:+ if isinstance(arg, ast.Constant) and isinstance(arg.value, str):+ if len(arg.value) > 256:+ return "string literal is too long"+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Mult):+ if not (_is_numeric(node.left) and _is_numeric(node.right)):+ return "multiplication is only allowed on numbers"+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Pow):+ if not (_is_numeric(node.left) and _is_numeric(node.right)):+ return "exponentiation is only allowed on numbers"+ if isinstance(node.right, ast.Constant) and isinstance(node.right.value, (int, float)) and abs(node.right.value) > 64:+ return "exponent is too large"+ return ""+++def _format_number(value: float | int) -> str:+ """Render a computed number without float noise ("3.0" -> "3", 0.1+0.2 -> "0.3")."""+ if isinstance(value, int):+ return str(value)+ if value == int(value) and abs(value) < 10**15:+ return str(int(value))+ return f"{value:.6f}".rstrip("0").rstrip(".")+++def compute(expression: str) -> tuple[str, str]:+ """Evaluate an LLM-written arithmetic expression. Returns ``(rendered, error)``.++ Exactly one of the two is non-empty. Every rejection is a normal outcome — the+ caller simply carries on without the computed evidence.+ """+ expression = (expression or "").strip()+ if not expression:+ return "", "empty expression"+ if len(expression) > _COMPUTE_MAX_CHARS:+ return "", f"expression is longer than {_COMPUTE_MAX_CHARS} characters"+ try:+ tree = ast.parse(expression, mode="eval")+ except SyntaxError as exc:+ return "", f"does not parse ({exc.msg})"+ problem = _check_expression(tree)+ if problem:+ return "", problem+ try:+ value = eval(compile(tree, "<evidence-arithmetic>", "eval"), {"__builtins__": {}}, dict(_COMPUTE_FUNCTIONS))+ except Exception as exc:+ return "", f"failed to evaluate ({type(exc).__name__}: {exc})"+ if isinstance(value, bool) or not isinstance(value, (int, float)):+ return "", f"result is {type(value).__name__}, not a number"+ if isinstance(value, float) and (value != value or value in (float("inf"), float("-inf"))):+ return "", "result is not a finite number"+ return _format_number(value), ""+++_COMPUTE_SYSTEM = """You are given the ORIGINAL question and every fact discovered so far. Decide+whether that question asks for a NUMBER that NO fact states outright but that FOLLOWS ARITHMETICALLY+from figures the facts DO state — a sum, a difference, a count, an average, a percentage, a unit+conversion, an elapsed span.++If it does, compute it by writing ONE Python expression with every figure substituted as a literal.+The expression is evaluated on its own: no variables, no assignments, no imports, no attributes, no+subscripts. The only functions available are abs, round, min, max, sum, len, int, float, sorted,+letters, digit_sum and date_diff.+ combined population of three -> 12345 + 6789 + 101112+ how many of the listed items -> len(["Alpha", "Beta", "Gamma"])+ what percentage one figure is -> 100 * 4523 / 18092+ years between two dates -> 1998 - 1954+ days between two dates -> date_diff("1941-07-28", "1959-07-17")+ letters in a set of names -> letters("Ada Lovelace", "Alan Turing")+ digits of a postcode added up -> digit_sum("L7 7BN")++ADDING UP THE DIGITS of a postcode, a house number, a serial number, a year or an address: use+digit_sum(...), and never read the digits out by hand. It adds each digit separately, which is what+such a question means — digit_sum("L7 7BN") is 7+7 = 14, digit_sum("2020") is 2+0+2+0 = 4. Pass the+identifier EXACTLY as the facts write it, letters and spaces included; they are ignored. It is the+WRONG tool for whole numbers the facts state separately — two populations, two prices, two years are+added as plain literals (12345 + 6789), not fed to digit_sum.++COUNTING LETTERS: use letters(...), NEVER len(...) on a name. len counts spaces, hyphens and+apostrophes as though they were letters, so it is wrong by exactly the amount nobody notices+(len("Ada Lovelace") is 12; the name has 11 letters). letters(...) takes any number of names, or one+list of them, and counts alphabetic characters only. Spell each name EXACTLY as the facts give it,+including any middle name or accent — and if the facts do not show a name in full, that figure is+missing, so return "needed": false rather than counting a partial name.++DAYS BETWEEN TWO DATES: when the question asks "how many days after X did Y happen" / "how many days+between two dates", use date_diff("YYYY-MM-DD", "YYYY-MM-DD") with the two dates EXACTLY as the facts+write them. Do NOT subtract the years (1959 - 1941) — that is the wrong quantity for a days question+(18 is years, not days). If either date is not a full YYYY-MM-DD in the facts, the figure is missing,+so return "needed": false rather than approximating.++AGE (an age, or an age difference, at some event): the facts almost always give a birth YEAR and an+event YEAR; the age is `event_year - birth_year` (or `birth_year - event_year`, taken as the positive+difference). You do NOT need the birth month or day — the year is enough. If the facts give FULL dates+(YYYY-MM-DD), prefer date_diff(...) which handles the day correctly; otherwise subtract the years.+Example: "elected in 2010, born 1971" -> 2010 - 1971. If the event year is BEFORE the birth year, the+difference is `birth_year - event_year` (use abs(...)). Never refuse because the birthday is not a+full date — the YEAR is sufficient.++PERCENTAGE (what percent / what share / what proportion / what fraction): `100 * part / whole`, where+`part` and `whole` are the exact figures from the facts. Example:+ "2.7 million Tamazight speakers out of 556 million total" -> 100 * 2.7 / 556+Do not round to an integer unless the question asks for that; keep the source figures exact.++UNIT CONVERSION (a speed, rate, or span in mixed units): convert inside the expression. A speed in+km/h becomes m/s by dividing by 3.6. Example for a difference in m/s between a fish and a swimmer:+ fish_kmh / 3.6 - 50 / swimmer_seconds -> e.g. 132 / 3.6 - 50 / 21.07+Use the EXACT figures the facts state (do not round 21.07 to 21); if the facts give the speed already+in m/s, use it directly without dividing.++MULTIPLICATION (a rate times a count, e.g. dollars per day times days): multiply the RATE by the+COUNT exactly as the facts state them. Read the rate's NUMBER from the facts. Example: "a suggested+donation of $25 per day, kept up for 49 days" -> 25 * 49. If the facts state the rate as 1 but the+question calls it "a suggested donation", still use the exact figure the facts give — never substitute+a made-up base amount.++Prefer computing over giving up: when the question asks for a derivable number and the facts+provide the figures (even if in different units or spread across several facts), WRITE the+expression and compute it. In particular, questions asking for an AGE DIFFERENCE, a PERCENTAGE,+a SPEED DIFFERENCE (with unit conversion), or a MULTIPLICATION (a rate times a count) are exactly+what this tool is for.++Return "needed": false, with an empty expression, ONLY when:+- the ORIGINAL question does not ask for a number;+- a fact already states that number outright — a value you would only be restating is not a+ calculation;+- a figure the calculation needs is genuinely absent from the facts, or a list the count depends on+ is not shown to be complete. NEVER invent, estimate, recall or infer a figure. When input is+ missing, say so and return "needed": false — a wrong number is worse than none — but first check+ that the figure really is absent (e.g. the age's birth YEAR is enough; you do not need the month).++"label" names what the number IS, as a short noun phrase ("combined population of the three+counties"), so a later step can use the result without re-deriving it.+"uses" lists the INDEX NUMBERS of the facts whose figures you substituted.+Output ONLY JSON, no prose, no code fences:+{"needed": true/false, "expression": "<one Python expression, or empty>", "label": "<short noun phrase>", "uses": [<index number>, ...]}"""+++def _render_facts(facts: list[str]) -> str:+ """Render the fact list for the LLM: one fact per line, prefixed with its index."""+ return "\n".join(f"[{i}] {f}" for i, f in enumerate(facts))+++async def compute_from_facts(+ llm,+ question: str,+ facts: list[str],+ *,+ fit_budget: int | None = None,+) -> dict | None:+ """Ask the LLM whether ``question`` asks for a derivable number and, if so,+ write + safely evaluate the expression over ``facts``.++ ``llm`` exposes ``async_chat`` and ``max_length`` (a RAGFlow LLMBundle /+ CountingChatModel). ``facts`` is the list of discovered fact strings.++ Returns ``None`` when no derivation is needed or possible, otherwise a dict:+ ``{"needed": True, "label": str, "value": str, "expression": str, "uses": [int, ...]}``.+ The caller attaches ``value`` (with its expression and source references) to its+ evidence list.+ """+ if not question or not facts:+ return None+ from rag.prompts.generator import form_message, message_fit_in++ user = f"Facts discovered so far:\n{_render_facts(facts)}\n\nOriginal question:\n{question}\n\nOutput JSON:"+ try:+ budget = fit_budget or llm.max_length+ _, msg = message_fit_in(form_message(_COMPUTE_SYSTEM, user), budget)+ ans = await llm.async_chat(msg[0]["content"], msg[1:], {"temperature": 0.0})+ except Exception:+ _LOG.exception("[Compute] LLM call failed")+ return None+ if isinstance(ans, tuple):+ ans = ans[0]+ if not isinstance(ans, str):+ return None++ import re++ import json_repair++ cleaned = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL).strip()+ cleaned = re.sub(r"```(?:json)?\s*|\s*```", "", cleaned).strip()+ try:+ data = json_repair.loads(cleaned)+ except Exception:+ _LOG.info("[Compute] could not parse LLM JSON: %r", ans[:200])+ return None+ if not isinstance(data, dict) or not data.get("needed"):+ return None++ expression = str(data.get("expression") or "").strip()+ if not expression:+ return None+ label = str(data.get("label") or "").strip() or "Value calculated from the facts found"+ uses = []+ for n in data.get("uses") or []:+ try:+ uses.append(int(n))+ except (TypeError, ValueError):+ continue++ value, error = compute(expression)+ if error:+ _LOG.info("[Compute] refused `%s` — %s", expression[:120], error)+ return None+ return {"needed": True, "label": label, "value": value, "expression": expression, "uses": uses}diff --git a/rag/advanced_rag/harness/config.py b/rag/advanced_rag/harness/config.pyindex 651009998f..b239c9dff8 100644--- a/rag/advanced_rag/harness/config.py+++ b/rag/advanced_rag/harness/config.py@@ -1,4 +1,12 @@-"""Thinking mode configurations."""+"""Thinking mode configurations.++medium/high/ultra all run the single-agent SCA graph (see+``agentic_rag_graph.build_agentic_graph``); the ONLY thing the ReAct/SCA path+consumes from the strategy is ``available_tools`` (via ``get_mode("medium")``)+and ``label``. low runs the lightweight ``direct_search`` graph, which reads+``execution_strategy`` (=="direct_search") and ``available_tools``. All other+fields are legacy data and never read by any running path.+"""from rag.advanced_rag.harness.types import ExecutionStrategy@@ -6,101 +14,22 @@ THINKING_MODES: dict[str, ExecutionStrategy] = {"low": ExecutionStrategy(label="low",execution_strategy="direct_search",- requires_decomposition=False,- requires_agent_loop=False,- requires_sufficiency_judge=False,- requires_selective_gen=False,- allows_dynamic_claims=False,- allows_replan=False,- max_orchestrator_cycles=1,- max_agent_cycles=0,- max_parallel_agents=1,available_tools=["hybrid_search", "web_search", "bm25_search"],- sufficiency_threshold=0.85,- fallback_to_direct_llm=False,),"medium": ExecutionStrategy(label="medium",- execution_strategy="decompose_and_search",- requires_decomposition=True,- requires_agent_loop=False,- requires_sufficiency_judge=True,- requires_selective_gen=True,- allows_dynamic_claims=False,- allows_replan=False,- max_orchestrator_cycles=3,- max_agent_cycles=0,- max_parallel_agents=1,- available_tools=["hybrid_search", "web_search", "bm25_search"],- sufficiency_threshold=0.75,- fallback_to_direct_llm=False,- c_high=0.75,- c_low=0.45,- llm_floor=0.55,- allows_reconcile=False,+ execution_strategy="direct_search", # legacy; SCA path does not read it+ available_tools=["grep_search", "list_chunks", "web_search"],),"high": ExecutionStrategy(label="high",- execution_strategy="agentic_research",- requires_decomposition=True,- requires_agent_loop=False,- requires_sufficiency_judge=True,- requires_selective_gen=True,- allows_dynamic_claims=False,- allows_replan=False,- max_orchestrator_cycles=3,- max_agent_cycles=2,- max_parallel_agents=4,- available_tools=[- "hybrid_search",- "web_search",- "bm25_search",- "ontology_navigate",- "dataset_navigation_search",- "graph_explore",- "inspector_open_context",- "inspector_compare",- ],- sufficiency_threshold=0.65,- fallback_to_direct_llm=False,- c_high=0.70,- c_low=0.40,- llm_floor=0.50,- allows_reconcile=True,+ execution_strategy="direct_search", # legacy; SCA path does not read it+ available_tools=["grep_search", "list_chunks", "web_search"],),"ultra": ExecutionStrategy(label="ultra",- execution_strategy="deep_research",- requires_decomposition=True,- requires_agent_loop=True,- requires_sufficiency_judge=True,- requires_selective_gen=True,- allows_dynamic_claims=True,- allows_replan=True,- max_orchestrator_cycles=4,- max_agent_cycles=2,- max_parallel_agents=4,- available_tools=[- "hybrid_search",- "bm25_search",- "web_search",- "structured_query",- "ontology_navigate",- "dataset_navigation_search",- "mindmap_navigate",- "graph_explore",- "wiki_query",- "inspector_open_context",- "inspector_compare",- "inspector_grep_within",- "inspector_request_adjacent",- ],- sufficiency_threshold=0.55,- fallback_to_direct_llm=True,- c_high=0.65,- c_low=0.35,- llm_floor=0.45,- allows_reconcile=True,+ execution_strategy="direct_search", # legacy; SCA path does not read it+ available_tools=["grep_search", "list_chunks", "web_search"],),}diff --git a/rag/advanced_rag/harness/grep_sed_narrow.py b/rag/advanced_rag/harness/grep_sed_narrow.pynew file mode 100644index 0000000000..410eb83f46--- /dev/null+++ b/rag/advanced_rag/harness/grep_sed_narrow.py@@ -0,0 +1,449 @@+"""In-memory grep+sed narrowing engine (term-driven, zero extra LLM rounds).++Mirrors the Claude Code / Codex ``grep`` + ``sed`` workflow, but operates on+retrieval chunks held in memory rather than on the filesystem: grep terms already+produced by the main-analysis LLM (entities, numbers, key phrases) are turned into+word-boundary regexes for locating (grep), and simple string transforms narrow the+text (sed), dropping unrelated boilerplate instead of crude head-truncation.++Key design (Claude Code semantics):+- The model that writes grep words is the same model that reasons about the+ answer: terms come from main-analysis output, no extra LLM call.+- Grep words are lightweight byproducts, not a separate LLM round (no extra+ call, no timeout, no wasted tokens).+- The engine executes mechanically (regex match + string transform), no LLM.++Fallback chain (never drops the answer):+ narrow_by_terms (grep locate + sed transform)+ -> no hits / no terms+ -> _narrow_by_keywords (keyword sentence-level, zero LLM)+ -> original chunks returned as-is (upper _build_compact_evidence+ head-truncation as the final safety valve)++Safety: only ``re.compile`` + pure string ops, no eval, no arbitrary code;+grep-term count cap + context clamp.+"""++import logging+import re++from rag.advanced_rag.harness.tools.search import (+ _split_sentences,+ _is_fact_dense_sentence,+ _narrow_by_keywords,+)++_LOG = logging.getLogger(__name__)++# Cost / safety caps.+_MAX_GREP_TERMS = 16+_MAX_CONTEXT = 2+_DEFAULT_OUT_CHARS_PER_CHUNK = 1200+_DEFAULT_OUT_TOTAL_CHARS = 16000+# Head length kept per chunk when there is no match.+_HEAD_FALLBACK_CHARS = 400+# Absolute char budget per side during context expansion, so an over-long line+# (or a chunk that the line splitter folded into one line) cannot inflate the+# narrowed fragment.+_CONTEXT_CHAR_BUDGET = 600+# Short chunks (<= this many chars) are not narrowed: they are already 1-2 lines,+# keeping them whole is safer (answers often live in short chunks).+_MIN_NARROW_CHARS = 200+++def _escape_term(term: str) -> str:+ """Escape a plain grep term into a safe, word-boundary regex fragment.++ Numbers/entities are matched literally; very short terms (<=2 chars) and pure+ tokens are handled without \b so they don't vanish inside other words.+ """+ t = str(term).strip()+ if not t:+ return ""+ # Strip punctuation casing that could pollute the regex (inner digits/ hyphens+ # are kept).+ t = re.sub(r"^[\s.,:;!?'\"()\[\]{}]+|[\s.,:;!?'\"()\[\]{}]+$", "", t)+ if not t:+ return ""+ escaped = re.escape(t)+ # \b only works for ASCII word chars in Python re; it silently fails for CJK+ # (Chinese/Japanese/Korean), so never wrap CJK terms in \b.+ if re.search(r"[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]", t):+ return escaped+ # No \b for short terms (avoid failing to match "pop" inside "population");+ # use \b only for 3+ char words with alphanumeric edges.+ if len(t) >= 3 and t[0].isalnum() and t[-1].isalnum():+ return rf"\b{escaped}\b"+ return escaped+++def _terms_to_patterns(terms) -> list[re.Pattern]:+ """Turn grep terms into a list of compiled regexes (one per term)."""+ out: list[re.Pattern] = []+ for term in (terms or [])[:_MAX_GREP_TERMS]:+ frag = _escape_term(term)+ if not frag:+ continue+ try:+ out.append(re.compile(frag, re.IGNORECASE))+ except re.error:+ continue+ return out+++def _line_spans(content: str) -> list[tuple[int, int]]:+ """Line (start,end) spans, boundaries at ``\\n`` (grep semantics).++ Line boundaries are exact, unlike sentence splitting (which is lossy and may+ merge/trim whitespace). Start of line ``i`` is after the ``i``-th ``\\n``.+ """+ spans: list[tuple[int, int]] = []+ start = 0+ for nl in re.finditer(r"\n", content):+ spans.append((start, nl.start()))+ start = nl.end()+ if start <= len(content):+ spans.append((start, len(content)))+ if not spans:+ spans = [(0, len(content))]+ return spans+++def _exec_on_text(+ content: str,+ patterns: list[re.Pattern],+ context: dict,+ out_chars_per_chunk: int,+) -> tuple[str, bool]:+ """Run term-grep + line-context expansion against one chunk's text.++ Mirrors ``grep -n -C N``: matches are located by ``match.start()/end()`` (exact),+ then expanded to whole lines, with ``before``/``after`` extra lines of context.+ Returns ``(narrowed, matched)``. On no hit, falls back to fact-dense-sentence+ keeping (never drops everything).+ """+ if not content:+ return "", False+ before = context.get("before", 0)+ after = context.get("after", 0)++ # Step 1: locate matches (exact positions from the regex engine).+ hit_ranges: list[tuple[int, int]] = []+ for pattern in patterns:+ try:+ for m in pattern.finditer(content):+ hit_ranges.append((m.start(), m.end()))+ except re.error:+ continue+ if not hit_ranges:+ # Keep fact-dense sentences to avoid dropping numbers/entities.+ kept: list[str] = []+ for s in _split_sentences(content):+ if _is_fact_dense_sentence(s):+ kept.append(s)+ narrowed = "".join(kept).strip()+ if narrowed:+ return narrowed[: _HEAD_FALLBACK_CHARS * 4], False+ return content[:_HEAD_FALLBACK_CHARS], False++ # Step 2: merge overlapping/adjacent matches, expand to line range + context.+ hit_ranges.sort()+ merged: list[tuple[int, int]] = []+ for s, e in hit_ranges:+ if merged and s <= merged[-1][1]:+ merged[-1] = (merged[-1][0], max(merged[-1][1], e))+ else:+ merged.append((s, e))+ lines = _line_spans(content)+ expanded: list[tuple[int, int]] = []+ for s, e in merged:+ lo = hi = 0+ for i, (ls, le) in enumerate(lines):+ if s >= ls and s < le:+ lo = i+ if e > ls and e <= le:+ hi = i+ lo = max(0, lo - before)+ hi = min(len(lines) - 1, hi + after)+ frag_s, frag_e = lines[lo][0], lines[hi][1]+ # Per-side character budget fallback: if the expanded window exceeds the+ # budget on either side, clamp to a compact window around the match.+ if frag_e - frag_s > _CONTEXT_CHAR_BUDGET * 2 and (frag_e - frag_s) > (e - s):+ frag_s = max(0, s - _CONTEXT_CHAR_BUDGET)+ frag_e = min(len(content), e + _CONTEXT_CHAR_BUDGET)+ expanded.append((frag_s, frag_e))++ # Step 3: dedupe, join, truncate.+ seen: set[str] = set()+ out_parts: list[str] = []+ for s, e in expanded:+ p = content[s:e].strip()+ if not p:+ continue+ key = p[:200]+ if key in seen:+ continue+ seen.add(key)+ out_parts.append(p)+ narrowed = "\n\n".join(out_parts).strip()+ if len(narrowed) > out_chars_per_chunk:+ narrowed = narrowed[:out_chars_per_chunk]+ return narrowed or content[:_HEAD_FALLBACK_CHARS], True+++def _chunk_text(chunk) -> str:+ if isinstance(chunk, dict):+ return str(chunk.get("content_with_weight") or chunk.get("content") or chunk.get("text") or "")+ return str(chunk or "")+++def _apply_narrow(chunks: list[dict], kept_texts: list[str], matched: list[bool]) -> list[dict]:+ out: list[dict] = []+ for ck, text, ok in zip(chunks, kept_texts, matched):+ d = dict(ck)+ if ok:+ d["content_with_weight"] = text+ if "content" in d:+ d["content"] = text+ d.pop("highlight", None)+ out.append(d)+ return out+++def _fallback_narrow_by_keywords(chunks: list[dict], keywords: str) -> list[dict]:+ try:+ return _narrow_by_keywords(chunks, keywords) or chunks+ except Exception:+ return chunks+++def narrow_by_terms(+ chunks: list[dict],+ terms,+ *,+ fallback_terms=None,+ context: dict | None = None,+ keywords: str = "",+ max_out_chars_per_chunk: int = _DEFAULT_OUT_CHARS_PER_CHUNK,+ max_out_total_chars: int = _DEFAULT_OUT_TOTAL_CHARS,+) -> dict:+ """Narrow retrieval chunks by locating grep terms.++ ``terms`` are plain strings (entities / numbers / key phrases) used to grep the+ chunks. If the primary terms produce no hit at all, ``fallback_terms`` are tried+ once mechanically (zero extra LLM). Still no hit -> narrowing is abandoned and+ the original chunks are returned (matched=False); the caller must NOT treat a+ failed narrow as an answer failure. Never raises.+ """+ ctx = context or {"before": 0, "after": 0}+ try:+ before = max(0, min(int(ctx.get("before", 0)), _MAX_CONTEXT))+ after = max(0, min(int(ctx.get("after", 0)), _MAX_CONTEXT))+ except (TypeError, ValueError):+ before = after = 0+ context = {"before": before, "after": after}++ patterns = _terms_to_patterns(terms)+ stats = {+ "chunks_in": len(chunks),+ "chunks_kept": 0,+ "chars_in": sum(len(_chunk_text(c)) for c in chunks),+ "chars_out": 0,+ "matched": False,+ "used_terms": len(patterns),+ }+ if not chunks:+ return {"kept": [], "stats": stats}++ # No usable grep terms -> fall back to keyword narrowing (zero LLM).+ if not patterns:+ narrowed = _fallback_narrow_by_keywords(chunks, keywords)+ stats["chunks_kept"] = len(narrowed)+ stats["chars_out"] = sum(len(_chunk_text(c)) for c in narrowed)+ return {"kept": narrowed, "stats": stats}++ def _run(active_patterns) -> tuple[list[str], list[bool]]:+ texts: list[str] = []+ flags: list[bool] = []+ for c in chunks:+ raw = _chunk_text(c)+ if len(raw) <= _MIN_NARROW_CHARS:+ texts.append(raw)+ flags.append(True)+ continue+ text, ok = _exec_on_text(raw, active_patterns, context, max_out_chars_per_chunk)+ texts.append(text)+ flags.append(ok)+ return texts, flags++ kept_texts, matched_flags = _run(patterns)++ # Gentle retry: if the primary terms hit nothing, try the fallback terms once+ # (mechanical, no extra LLM). Mirrors Claude Code re-grepping with a different+ # word before giving up on a region.+ if fallback_terms and not any(matched_flags):+ fb_patterns = _terms_to_patterns(fallback_terms)+ if fb_patterns:+ kept_texts, matched_flags = _run(fb_patterns)+ stats["used_terms"] = max(stats["used_terms"], len(fb_patterns))++ kept = _apply_narrow(chunks, kept_texts, matched_flags)+ # Only apply the total-length cap when the grep actually matched. When matched+ # is False (no term hit, no narrowing happened), the chunks are returned+ # untouched so the caller's own compaction decides how to truncate. Truncating+ # here on a no-match would otherwise drop most chunks to a single one, losing+ # evidence needed for multi-hop/enumeration answers.+ if any(matched_flags):+ total_out = sum(len(_chunk_text(c)) for c in kept)+ if total_out > max_out_total_chars:+ # Distribute the total budget across as many matched chunks as+ # possible, instead of letting the FIRST chunk swallow the whole cap+ # and dropping every later chunk. Each chunk is capped to+ # max_out_chars_per_chunk, and chunks are kept while the running+ # total fits in max_out_total_chars. This preserves evidence spread+ # across chunks (needed for multi-hop/enumeration) instead of a+ # single 16K blob.+ per_chunk_cap = max(200, min(max_out_chars_per_chunk, max_out_total_chars // max(1, len(kept))))+ acc = 0+ trimmed = []+ for c in kept:+ t = _chunk_text(c)+ room = max_out_total_chars - acc+ if room <= 0:+ break+ take = min(len(t), per_chunk_cap, room)+ if take <= 0:+ break+ if take < len(t):+ c = dict(c)+ c["content_with_weight"] = t[:take]+ if "content" in c:+ c["content"] = t[:take]+ trimmed.append(c)+ acc += take+ kept = trimmed++ stats["chunks_kept"] = len(kept)+ stats["chars_out"] = sum(len(_chunk_text(c)) for c in kept)+ stats["matched"] = any(matched_flags)+ _LOG.info(+ "[grep-sed] chunks=%d->%d chars=%d->%d matched=%s terms=%d",+ stats["chunks_in"],+ stats["chunks_kept"],+ stats["chars_in"],+ stats["chars_out"],+ stats["matched"],+ stats["used_terms"],+ )+ return {"kept": kept, "stats": stats}+++def split_fallback_terms(*texts: str) -> list[str]:+ """Split free text into fallback grep terms (zero LLM).++ Used as the gentle-retry terms when the LLM-generated grep terms hit nothing.+ Any language: splits on sentence/comma boundaries, drops short/stopword-like+ tokens, keeps numbers and multi-word phrases as whole \b terms.+ """+ import re as _re++ terms: list[str] = []+ seen: set[str] = set()+ for v in texts:+ for part in _re.split(r"[\n。;;,.?!?]+", str(v or "")):+ part = part.strip().strip("'\"()[]{}")+ if not part or len(part) < 3:+ continue+ if part.lower() in _FALLBACK_STOPWORDS:+ continue+ if part in seen:+ continue+ seen.add(part)+ terms.append(part)+ return terms[:_MAX_GREP_TERMS]+++_FALLBACK_STOPWORDS = {+ "what",+ "which",+ "who",+ "where",+ "when",+ "how",+ "the",+ "a",+ "an",+ "of",+ "in",+ "on",+ "for",+ "to",+ "and",+ "or",+ "with",+ "is",+ "are",+ "was",+ "were",+ "list",+ "name",+ "give",+ "find",+ "tell",+ "me",+ "about",+ "from",+ "that",+ "this",+ "it",+ "its",+ "their",+ "they",+ "have",+ "has",+ "do",+ "does",+ "did",+ "based",+ "per",+ "according",+ "not",+}+++def grep_sed_narrow(+ chunks: list[dict],+ *,+ claim_sources: tuple[str, ...] = (),+ max_out_chars_per_chunk: int = _DEFAULT_OUT_CHARS_PER_CHUNK,+ max_out_total_chars: int = _DEFAULT_OUT_TOTAL_CHARS,+) -> dict:+ """Narrow chunks by grepping terms extracted directly from the claim (zero LLM).++ Grep terms are derived mechanically from the claim/question text (entities,+ numbers, proper nouns via ``split_fallback_terms``) — NO extra LLM call. The+ engine then greps+seds, with a gentle mechanical retry on a second-pass term+ set. Never raises; on any failure the original chunks are returned untouched+ so the caller's existing compaction is the final safety valve.+ """+ stats = {+ "chunks_in": len(chunks),+ "chunks_kept": 0,+ "chars_in": sum(len(_chunk_text(c)) for c in chunks),+ "chars_out": 0,+ "matched": False,+ "used_terms": 0,+ }+ if not chunks:+ return {"kept": chunks, "stats": stats}++ terms = split_fallback_terms(*claim_sources)+ stats["used_terms"] = len(terms)+ res = narrow_by_terms(+ chunks,+ terms,+ keywords=" ".join(claim_sources),+ max_out_chars_per_chunk=max_out_chars_per_chunk,+ max_out_total_chars=max_out_total_chars,+ )+ return resdiff --git a/rag/advanced_rag/harness/keywords.py b/rag/advanced_rag/harness/keywords.pynew file mode 100644index 0000000000..a866779629--- /dev/null+++ b/rag/advanced_rag/harness/keywords.py@@ -0,0 +1,169 @@+"""Four-aspect keyword extraction with entity weighting, ported from the+agentic_search4 v8 keyword graph.++A keyword search matches only the surface forms you give it, so a single flat+bag of terms is a poor retrieval driver: the model does not know how the corpus+phrases a fact. This extraction asks the LLM for FOUR aspects of the question —+``entity`` (what the fact is about), ``aliases`` (its surface variants),+``fact_type`` (words the corpus might use for this kind of fact, including table+column abbreviations) and ``qualifiers`` (year / edition / jurisdiction /+revision). ``entity`` is what discriminates, so it is repeated in the search+query to weight BM25 toward it; the plain deduped union of all four aspects is+used to narrow retrieved chunks to their keyword-bearing sentences.++Public interface+----------------+``extract_weighted_keywords(llm, question)`` -> ``(query, union)``+ ``query`` — entity terms repeated (x3) + qualifiers repeated (x3) + the other+ aspects once, comma-joined for the actual retrieval string (BM25+ weights the repeated entity/qualifier up; a year or jurisdiction is+ as discriminating as the entity and must dominate the ranking).+ ``union`` — every term once, comma-joined, for narrowing retrieved chunks.+"""++import logging++_LOG = logging.getLogger(__name__)++# Keywords are extracted into FOUR weighted aspects. ``entity`` terms are what+# discriminate and are repeated in the search query so BM25 weights them up;+# ``aliases`` find the document, ``fact_type`` and ``qualifiers`` only boost+# ranking. The plain deduped union (``keywords``) narrows retrieved chunks to+# their keyword-bearing sentences; the entity-weighted string (``query``) drives+# retrieval.+_KEYWORD_ASPECTS = ("entity", "aliases", "fact_type", "qualifiers")+_KEYWORD_ENTITY_REPEAT = 3 # copies of each entity term in the query, to weight it up+_KEYWORD_QUALIFIER_REPEAT = 3 # copies of each qualifier (year / version / jurisdiction), weighted up like entity+_KEYWORD_MAX_CHARS = 400 # hard cap on the weighted query so it never pollutes++_KEYWORDS_SYSTEM = """You turn ONE question into search terms for a keyword/BM25 search engine.++Emit the terms that would appear VERBATIM in a document that answers the question, sorted into FOUR+categories. Every term must come from the question itself or be a surface form of something in it.++A. "entity" — the specific thing the fact is ABOUT: proper nouns, titles, identifiers. Keep a+ multi-word entity whole, as ONE term ("Brown County", "Treaty of Versailles"); split across+ several terms its tokens match independently and drag in noise. A bare identifier — a serial,+ patent, catalogue or case number — is a complete entity on its own; never glue it to the words+ around it.+B. "aliases" — the engine matches ONLY the surface forms you supply, so emit the plausible variants+ of A: full vs. short name, native-language and transliterated forms, official vs. common name,+ acronym and its expansion, and the qualified form ("Brown County" -> "Brown County, Kansas").+C. "fact_type" — 3 to 6 words the corpus might use for this KIND of fact, since you cannot know how+ it is phrased. Spread them across registers:+ quantity of people -> population, inhabitants, residents, census, demographics, headcount+ time of an event -> founded, established, opened, dated, began+ role of a person -> served, appointed, elected, held, director+ SOURCES TABULATE WHAT QUESTIONS SPELL OUT: a statistic named in prose is usually written in a+ table as a column abbreviation, and the prose wording may not appear in the document at all. So+ include the abbreviation a table would use — "points per game" -> "PPG", "PTS"; "earnings per+ share" -> "EPS"; "games played" -> "GP" — and reach a superlative through its plain column too:+ "leading scorer" is found by looking for "PTS" and "PPG", not for the phrase itself.+D. "qualifiers" — year, edition, jurisdiction, revision. Worth emitting even when it looks+ redundant: the qualifier often sits in a table header or a document title that chunking has+ severed from the value. Include EVERY alternative expression of a DATE or NUMBER in the+ question — ordinals and their words ("21st" -> "twenty-first"), digits and their words+ ("2000000" -> "two million", "2 million"), and each common date format ("Aug 2nd" -> "August 2",+ "2 August", "08-02").++A and B are what FINDS the document; C and D only boost the ranking. So never withhold an entity+because you are unsure of it, and never pad C or D to reach a count.++DROP entirely: question words ("which", "who", "when", "how many"), relational scaffolding, and+generic high-frequency nouns ("year", "number", "city", "total", "list", "information"). They cost+ranking quality and retrieve nothing.++Output ONLY JSON, no prose, no code fences:+{"entity": ["<term>", ...], "aliases": ["<term>", ...], "fact_type": ["<term>", ...], "qualifiers": ["<term>", ...]}+Any category may be empty."""+++def _norm_keyword(s: str) -> str:+ """Normalise a keyword term for cross-category dedup."""+ return " ".join((s or "").lower().split())+++def _parse_aspects(raw: str) -> dict[str, list[str]]:+ """Parse the LLM's JSON into one deduped list per aspect.++ ONE dedup set spans all four categories: a term the model emits as both an+ entity and an alias must not collect a second share of the query's mass on+ the strength of having been named twice.+ """+ import re++ import json_repair++ data: dict = {}+ cleaned = re.sub(r"^.*</think>", "", raw or "", flags=re.DOTALL).strip()+ cleaned = re.sub(r"```(?:json)?\s*|\s*```", "", cleaned).strip()+ try:+ parsed = json_repair.loads(cleaned)+ if isinstance(parsed, dict):+ data = parsed+ except Exception:+ pass++ aspects: dict[str, list[str]] = {}+ seen: set[str] = set()+ for aspect in _KEYWORD_ASPECTS:+ terms: list[str] = []+ for k in data.get(aspect) or []:+ term = str(k).strip()+ key = _norm_keyword(term)+ if term and key and key not in seen:+ seen.add(key)+ terms.append(term)+ aspects[aspect] = terms+ return aspects+++async def extract_weighted_keywords(llm, question: str) -> tuple[str, str]:+ """Extract FOUR keyword aspects for ``question`` and return ``(query, keywords)``.++ ``query`` is the weighted search string: every entity term is repeated+ ``_KEYWORD_ENTITY_REPEAT`` times and every qualifier (year / version /+ jurisdiction) ``_KEYWORD_QUALIFIER_REPEAT`` times, so BM25 weights both up+ inside the same query; aliases and fact-type vocabulary appear once each.+ ``keywords`` is the plain deduped union of all four aspects (one copy each)+ used to narrow retrieved chunks to their keyword-bearing sentences.++ ``llm`` exposes ``async_chat`` and ``max_length``. Falls back to+ ``(question, question)`` when extraction fails.+ """+ if not question:+ return "", ""+ from rag.prompts.generator import form_message, message_fit_in++ aspects: dict[str, list[str]] = {}+ try:+ _, msg = message_fit_in(form_message(_KEYWORDS_SYSTEM, question), llm.max_length)+ ans = await llm.async_chat(msg[0]["content"], msg[1:], {"temperature": 0.1})+ if isinstance(ans, tuple):+ ans = ans[0]+ aspects = _parse_aspects(ans if isinstance(ans, str) else "")+ except Exception:+ _LOG.exception("extract_weighted_keywords failed")++ keywords = ", ".join(t for aspect in _KEYWORD_ASPECTS for t in aspects.get(aspect) or []) or question+ # Entity and qualifier terms are repeated so BM25 weights them up: the entity+ # is what the fact is about, and a year / edition / jurisdiction is just as+ # discriminating (often the only thing that separates two otherwise identical+ # chunks). Aliases and fact-type vocabulary appear once each — they find or+ # boost, but must not dominate the ranking.+ weighted = [t for t in (aspects.get("entity") or []) for _ in range(_KEYWORD_ENTITY_REPEAT)]+ weighted += [t for t in (aspects.get("qualifiers") or []) for _ in range(_KEYWORD_QUALIFIER_REPEAT)]+ weighted += [t for aspect in ("aliases", "fact_type") for t in (aspects.get(aspect) or [])]+ query = ", ".join(weighted) or keywords+ query = query[:_KEYWORD_MAX_CHARS]+ keywords = keywords[:_KEYWORD_MAX_CHARS]+ _LOG.info(+ "[Keywords] entity x%d: %s | aliases: %s | fact-type: %s | qualifiers x%d: %s",+ _KEYWORD_ENTITY_REPEAT,+ "; ".join(aspects.get("entity") or []) or "-",+ "; ".join(aspects.get("aliases") or []) or "-",+ "; ".join(aspects.get("fact_type") or []) or "-",+ _KEYWORD_QUALIFIER_REPEAT,+ "; ".join(aspects.get("qualifiers") or []) or "-",+ )+ return query, keywordsdiff --git a/rag/advanced_rag/harness/memory.py b/rag/advanced_rag/harness/memory.pynew file mode 100644index 0000000000..46ded2be86--- /dev/null+++ b/rag/advanced_rag/harness/memory.py@@ -0,0 +1,390 @@+"""Retrieval memory — a central store of every raw chunk any claim has retrieved.++Purpose+-------+Throughout a multi-hop answer the system retrieves many chunks (search is cheap)+but hands the LLM only a small narrowed slice (LLM calls are slow & token-heavy).+The raw chunks must not be thrown away: the LLM's per-claim report/grounded+extraction can compress away a fact the answer needs (e.g. a candidate set+"R, RMP, RA, RF, RIF" collapsed to "RIF"). ``RetrievalMemory`` keeps ALL raw+chunks that any search returned, so the finalizer can do a cheap, deterministic,+no-LLM, no-knowledge-base relevance search over the already-retrieved corpus to+recover the missing fact instead of re-querying.++Design (in-memory, language-agnostic)+-------------------------------------+- ``add`` stores every raw chunk exactly as returned by the search backend,+ de-duplicated by chunk identity, BEFORE any narrowing — it is the lossless+ store backing the (lossy) ``kbinfos["chunks"]`` that feeds the LLM.+- ``search`` is the primary consumer-facing primitive: a relevance-ranked lookup+ over memory for a query string, language-agnostic (Latin words + numbers + CJK+ character 3-grams), requiring a normalized overlap bar so Chinese and English+ queries behave alike. It never invokes the LLM and never triggers a knowledge-+ base search.+- ``grep`` is a lower-level loose-keyword primitive (word-boundary + prefix+ tolerance) used as a building block / for tests; production finalize uses the+ stricter ``search``.++The store lives on ``tools.kbinfos["memory"]`` so it is serialized / carried with+the rest of the research state and is visible to any tool that has ``tools``.+"""++import logging+import re++_LOG = logging.getLogger(__name__)++# How many memory chunks a single grep query may return at most (keeps the+# injected evidence bounded).+_GREP_MAX_CHUNKS = 6+# Max sentences kept per chunk (hit + context) so a long chunk does not blow up+# the injection.+_GREP_MAX_SENTENCES = 4+# Absolute char budget per side of a hit when expanding context.+_GREP_CONTEXT_CHARS = 400+# Short chunks are kept whole (answers often live in short chunks).+_SHORT_CHUNK_CHARS = 200+++def _chunk_key(ck: dict) -> str:+ return str(ck.get("chunk_id") or ck.get("id") or id(ck))+++def _chunk_text(ck: dict) -> str:+ """The searchable text of a chunk, preferring the raw original."""+ for k in ("content", "content_with_weight"):+ v = ck.get(k)+ if v:+ return str(v)+ return ""+++def _escape_term(term: str) -> str:+ t = str(term).strip()+ if not t:+ return ""+ t = re.sub(r"^[\s.,:;!?'\"()\[\]{}]+|[\s.,:;!?'\"()\[\]{}]+$", "", t)+ if not t:+ return ""+ escaped = re.escape(t)+ # CJK: no \b (Python re \b is ASCII-only and would never match).+ if re.search(r"[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]", t):+ return escaped+ if len(t) >= 3 and t[0].isalnum() and t[-1].isalnum():+ return rf"\b{escaped}\b"+ return escaped+++def _split_sentences(text: str) -> list[str]:+ """Split into sentences, treating block HTML/markdown tables as atomic."""+ # Reuse the shared sentence splitter from tools.search when available, else a+ # minimal one. Import lazily to avoid a heavy module graph at load time.+ try:+ from rag.advanced_rag.harness.tools.search import _split_sentences as _ss++ return _ss(text)+ except Exception:+ pass+ return [s for s in re.split(r"(?<=[.!?。!?])\s+", (text or "").strip()) if s]+++def _sentence_span_window(sents: list[str], idx: int) -> list[str]:+ """The hit sentence plus up to one neighbouring sentence on each side."""+ lo = max(0, idx - 1)+ hi = min(len(sents), idx + 2)+ window = sents[lo:hi]+ # Clamp total length.+ total = 0+ kept = []+ for s in window:+ total += len(s)+ if total > _GREP_CONTEXT_CHARS * 2:+ break+ kept.append(s)+ return kept or [sents[idx]]+++def add(tools, chunks) -> None:+ """Merge raw retrieved chunks into the central memory store (lossless)."""+ if not chunks:+ return+ mem = tools.kbinfos.setdefault("memory", [])+ seen = {_chunk_key(c) for c in mem}+ added = 0+ for c in chunks:+ if not isinstance(c, dict) or not _chunk_text(c):+ continue+ k = _chunk_key(c)+ if k in seen:+ continue+ seen.add(k)+ mem.append(c)+ added += 1+ if added:+ _LOG.info("[Memory] stored %d new raw chunk(s); memory now has %d.", added, len(mem))+++def grep(tools, terms, limit: int = _GREP_MAX_CHUNKS) -> list[dict]:+ """Return memory chunks that contain any of ``terms`` (word-boundary grep),+ narrowed to the matching sentence + small context.++ ``terms`` is a list of plain strings (entities / numbers / key phrases) as+ emitted by the analysis LLM. Returns a list of chunk dicts each carrying a+ narrowed ``content`` (a plain string) so the caller can splice them straight+ into an evidence list. Empty on no-hit / no-memory.+ """+ mem = tools.kbinfos.get("memory", []) or []+ if not mem or not terms:+ return []+ patterns = []+ # Prefix fallback: morphological tolerance. A gap term often differs from the+ # chunk's word by a suffix/prefix (gap "abbreviation" vs chunk "abbreviated";+ # gap "rifampin" vs chunk "Rifampicin"). We match the term's leading stem+ # (first 5 chars for terms >= 6) at the START of a chunk word, so a shared+ # root still hits without a short token over-matching. CJK terms are matched+ # verbatim (no stemming semantics).+ prefix_patterns = []+ for t in terms:+ frag = _escape_term(t)+ if frag:+ try:+ patterns.append(re.compile(frag, re.IGNORECASE))+ except re.error:+ continue+ _stripped = str(t).strip()+ _prefix = _stripped[:5] if len(_stripped) >= 6 else ""+ if _prefix and not re.search(r"[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]", _prefix):+ try:+ prefix_patterns.append(re.compile(rf"\b{re.escape(_prefix)}", re.IGNORECASE))+ except re.error:+ continue+ if not patterns and not prefix_patterns:+ return []++ def _match(text: str) -> bool:+ if any(p.search(text) for p in patterns):+ return True+ # Prefix fallback: the term's leading stem occurs at the START of a+ # chunk word (word boundary), tolerating inflectional suffixes.+ if prefix_patterns:+ for pp in prefix_patterns:+ if pp.search(text):+ return True+ return False++ hits = []+ for c in mem:+ text = _chunk_text(c)+ if len(text) <= _SHORT_CHUNK_CHARS:+ # Short chunk: keep whole (its answer may live anywhere in it).+ if _match(text):+ hits.append({"content": text, "doc_id": c.get("doc_id"), "chunk_id": c.get("chunk_id")})+ continue+ sents = _split_sentences(text)+ kept = []+ for i, s in enumerate(sents):+ if _match(s):+ for w in _sentence_span_window(sents, i):+ if w not in kept:+ kept.append(w)+ if len(kept) >= _GREP_MAX_SENTENCES:+ break+ if kept:+ hits.append({"content": "\n".join(kept), "doc_id": c.get("doc_id"), "chunk_id": c.get("chunk_id")})+ if len(hits) >= limit:+ break+ return hits+++def size(tools) -> int:+ return len(tools.kbinfos.get("memory", []) or [])+++def clear(tools) -> None:+ tools.kbinfos["memory"] = []+++# ─────────────────────────────────────────────────────────────────────────────+# Relevance-ranked retrieval over memory (used as a retrieval-reuse cache, NOT+# as a noise-injection source). Unlike ``grep`` (loose keyword hit), ``search``+# scores each memory chunk by how many SIGNIFICANT query terms it actually+# contains and returns only the chunks that clear a multi-term overlap bar —+# quality comparable to a knowledge-base retrieval, but served from memory so a+# fact retrieved earlier can be reused instead of re-querying the index.+# ─────────────────────────────────────────────────────────────────────────────++_STOPWORDS = {+ "what",+ "which",+ "how",+ "many",+ "much",+ "does",+ "did",+ "do",+ "the",+ "a",+ "an",+ "is",+ "are",+ "was",+ "were",+ "be",+ "been",+ "being",+ "of",+ "for",+ "to",+ "in",+ "on",+ "with",+ "and",+ "or",+ "by",+ "from",+ "at",+ "it",+ "its",+ "this",+ "that",+ "these",+ "those",+ "who",+ "when",+ "where",+ "why",+ "than",+ "then",+ "there",+ "their",+ "they",+ "them",+ "his",+ "her",+ "him",+ "she",+ "he",+ "we",+ "you",+ "your",+}++# CJK / other scripts have no word boundaries and no shared stopword list; match+# them as literal substrings. Script ranges: CJK unified ideographs, Hiragana,+# Katakana, Hangul, CJK punctuation is excluded.+_CJK_RE = re.compile(r"[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]+")+_LATIN_NUM_RE = re.compile(r"[A-Za-z0-9]+")+++def _is_cjk(s: str) -> bool:+ return bool(_CJK_RE.fullmatch(s))+++def _significant_terms(text: str, max_terms: int = 18) -> list[str]:+ """Language-agnostic significant-term extraction.++ - Latin alphanumeric runs → lowercased words (stopword-filtered, len >= 3).+ - CJK runs (Chinese/Japanese/Korean) → split into character 3-grams (no word+ boundaries exist, so n-grams are the language-agnostic way to match a+ substring like 利福平 inside a chunk); stopwords are NOT applied.+ - standalone numbers kept verbatim.+ De-duplicated, capped. Returns [] when nothing searchable.+ """+ out: list[str] = []+ seen: set[str] = set()++ def _push(tok: str) -> None:+ if tok and tok not in seen:+ seen.add(tok)+ out.append(tok)++ # Numbers anywhere.+ for m in re.finditer(r"\d+", text or ""):+ _push(m.group(0))+ if len(out) >= max_terms:+ return out+ # CJK runs → 3-grams (and the whole run if shorter than 3).+ for m in _CJK_RE.finditer(text or ""):+ run = m.group(0)+ if len(run) < 3:+ _push(run)+ else:+ for i in range(len(run) - 2):+ _push(run[i : i + 3])+ if len(out) >= max_terms:+ return out+ # Latin words (stopword-filtered).+ for m in _LATIN_NUM_RE.finditer(text or ""):+ raw = m.group(0)+ if raw.isdigit():+ continue # numbers already handled+ low = raw.lower()+ if len(low) >= 3 and low not in _STOPWORDS:+ _push(low)+ if len(out) >= max_terms:+ return out+ return out+++def _term_hits(text: str, terms: list[str]) -> int:+ """How many of ``terms`` occur in ``text``, language-agnostically.++ - CJK terms → literal substring (no word boundary).+ - Latin terms → word-boundary match with a short prefix fallback for+ inflectional variants ("abbreviation" also matches "abbreviated").+ - numbers → literal presence.+ """+ if not terms:+ return 0+ hits = 0+ for t in terms:+ if _is_cjk(t):+ if t in text:+ hits += 1+ elif t.isdigit():+ if t in text:+ hits += 1+ else:+ if re.search(rf"\b{re.escape(t)}\b", text, re.IGNORECASE):+ hits += 1+ elif len(t) >= 6:+ _prefix = re.escape(t[:5])+ if re.search(rf"\b{_prefix}", text, re.IGNORECASE):+ hits += 1+ return hits+++def search(tools, query: str, top_n: int = 6, min_overlap: int = 2, min_ratio: float = 0.12) -> list[dict]:+ """Relevance-ranked retrieval over memory for a query string (language-agnostic).++ Scores each memory chunk by how many of the query's significant terms+ (Latin words / numbers / CJK 3-grams) it matches, and keeps chunks that clear+ a *normalized* overlap bar so Chinese and English queries behave alike: a+ chunk is relevant when it shares ``>= 1`` term AND ``>= min_ratio`` of the+ query's terms (a long CJK query matches a short "利福平" via shared 3-grams;+ an English query matches "abbreviation → abbreviated" via word-boundary ++ prefix). Ranked by hit count. Cheap, deterministic, no-LLM — used to REUSE+ evidence already retrieved, never to inject loose noise. Returns [] when+ nothing clears the bar (caller falls back to the knowledge-base search).+ """+ mem = tools.kbinfos.get("memory", []) or []+ terms = _significant_terms(query)+ if not mem or not terms:+ return []+ _n = len(terms)+ scored = []+ for c in mem:+ text = _chunk_text(c)+ if not text:+ continue+ hits = _term_hits(text, terms)+ if hits >= 1 and (hits / _n) >= min_ratio:+ scored.append((hits, text, c))+ if not scored:+ return []+ scored.sort(key=lambda x: (-x[0], -len(x[1])))+ out = []+ for hits, text, c in scored[:top_n]:+ out.append({"content": text, "doc_id": c.get("doc_id"), "chunk_id": c.get("chunk_id"), "similarity": float(hits)})+ _LOG.info("[Memory.search] query=%r -> %d relevant chunk(s) (ratio>=%.2f, %d terms)", (query or "")[:60], len(out), min_ratio, _n)+ return outdiff --git a/rag/advanced_rag/harness/orchestrator/__init__.py b/rag/advanced_rag/harness/orchestrator/__init__.pyindex d4efbbadd6..87be91a1b2 100644--- a/rag/advanced_rag/harness/orchestrator/__init__.py+++ b/rag/advanced_rag/harness/orchestrator/__init__.py@@ -3,8 +3,6 @@import loggingfrom rag.advanced_rag.harness.config import get_mode-from rag.advanced_rag.harness.orchestrator.agentic import agentic_research-from rag.advanced_rag.harness.orchestrator.decompose import decompose_and_searchfrom rag.advanced_rag.harness.orchestrator.direct import direct_searchfrom rag.advanced_rag.harness.stats import in_phase@@ -27,11 +25,5 @@ async def orchestrator_loop(state: dict, tools) -> dict:if mode.execution_strategy == "direct_search":return await direct_search(state, tools)- if mode.execution_strategy == "decompose_and_search":- return await decompose_and_search(state, tools)-- if mode.execution_strategy in ("agentic_research", "deep_research"):- return await agentic_research(state, tools)-_LOG.warning("orchestrator: unknown strategy %s, fallback to direct", mode.execution_strategy)return await direct_search(state, tools)diff --git a/rag/advanced_rag/harness/orchestrator/agentic.py b/rag/advanced_rag/harness/orchestrator/agentic.pydeleted file mode 100644index e3a5d7fdf6..0000000000--- a/rag/advanced_rag/harness/orchestrator/agentic.py+++ /dev/null@@ -1,522 +0,0 @@-"""High/Ultra: two-level loop — orchestrator assigns claims, agent researches, sufficiency checks."""--import asyncio-import logging--from rag.advanced_rag.harness.agent import research_agent_loop-from rag.advanced_rag.harness.config import get_mode-from rag.advanced_rag.harness.orchestrator.sufficiency_llm import llm_sufficiency_boost-from rag.advanced_rag.harness.pipeline import Pipeline-from rag.advanced_rag.harness.stats import in_phase, record_round, record_round_claims-from rag.advanced_rag.harness.sufficiency import (- compute_fusion_score,- cross_check_claim,- route_sufficiency_verdict,-)-from rag.advanced_rag.harness.types import (- AgentResult,- ClaimTarget,- OrchestratorContext,-)--_LOG = logging.getLogger(__name__)-CLAIM_RESEARCH_TIMEOUT_SECONDS = 180---def _snip(text: str, limit: int = 160) -> str:- text = (text or "").replace("\n", " ").strip()- return text if len(text) <= limit else text[: limit - 3] + "..."---def _discovered_entity(tools) -> str | None:- """Pick a salient discovered name from the gathered evidence.-- Prefers an explicit entity/keyword tag on a chunk, falling back to a source- document name. Used only to gate ``graph_explore`` eligibility (its context- check needs ``context.last_entity``), so a coarse signal is enough.- """- chunks = (getattr(tools, "kbinfos", {}) or {}).get("chunks", []) or []- for c in chunks:- for key in ("entities_kwd", "important_kwd"):- val = c.get(key)- if isinstance(val, list) and val:- first = str(val[0]).strip()- if first:- return first- if isinstance(val, str) and val.strip():- return val.strip().split()[0]- for c in chunks:- name = str(c.get("docnm_kwd") or "").strip()- if name:- return name- return None---@in_phase("orchestrator")-async def agentic_research(state: dict, tools) -> dict:- """Two-level loop for high/ultra modes."""- question = state.get("question", "")- claims_raw = state.get("claims", [])- route = state.get("route", {})- mode_label = route.thinking_mode if route else "high"- mode = get_mode(mode_label)-- # Resolve compilation map- compilation_map = await _get_compilation_map(tools)-- claims = [ClaimTarget(**c) if isinstance(c, dict) else c for c in claims_raw]- ctx = OrchestratorContext(question=question, claims=claims, mode=mode_label)-- # Stagnation guard: if the fusion score stops improving across consecutive- # rounds, further searching is unlikely to help (e.g. the corpus simply lacks- # the data, and follow-ups keep returning nothing). Without this, a- # persistently INSUFFICIENT verdict burns every remaining cycle and, in the- # worst case, feeds a long empty loop (see check.log Q4: AutoRater said- # "not in corpus", follow-ups found nothing, yet the loop kept spinning).- prev_score: float | None = None- _STAGNATION_CYCLES = 2 # rounds with no meaningful gain before giving up- _STAGNATION_GAIN = 0.05 # minimum fusion-score improvement to count-- rounds_run = 0- for cycle in range(mode.max_orchestrator_cycles):- rounds_run = cycle + 1- ctx.iteration = cycle- record_round("orchestrator")- _LOG.info("[Agentic research] Research round %d of %d — %d step(s) still unanswered.", cycle + 1, mode.max_orchestrator_cycles, sum(1 for c in ctx.claims if not c.is_verified))-- # ── Step A: Research unverified claims (parallel if mode allows) ──- unverified = [c for c in ctx.claims if not c.is_verified]- record_round_claims("claim_research", len(unverified))-- if unverified:- # Consume Phase-2 follow-up queries (missing-pieces feedback) ONCE for- # this round and hand the SAME list to every claim in the batch.- # Reading the shared ``ctx.pending_followups`` inside- # research_agent_loop would race under ``asyncio.gather``: the first- # claim to execute would clear it, starving the parallel siblings.- # We only consume/clear here — inside ``if unverified`` — because a- # research task must actually dispatch to use them. When everything- # is already verified no task runs, so we retain the queries for the- # next cycle instead of discarding them.- followups: list[str] = []- if ctx.pending_followups:- followups = [str(q.get("query") or q.get("question") or "") for q in ctx.pending_followups if q]- followups = [q for q in followups if q.strip()]- ctx.pending_followups = []- if followups:- _LOG.info("[Agentic research] Round %d: injecting %d follow-up query(ies) to all claims: %s", cycle + 1, len(followups), followups)-- # Process in batches of max_parallel_agents- batch_size = mode.max_parallel_agents- for i in range(0, len(unverified), batch_size):- batch = unverified[i : i + batch_size]- _LOG.info(- "[Agentic research] Round %d: researching %d step(s) in parallel: %s",- cycle + 1,- len(batch),- "; ".join(f'"{c.description}"' for c in batch),- )- tasks = [_run_claim_research(c, tools, ctx, mode, compilation_map, followups=followups) for c in batch]- agent_results = await asyncio.gather(*tasks)- _LOG.info(- "[Agentic research] Round %d: finished researching %d step(s).",- cycle + 1,- len(agent_results),- )-- for c, result in zip(batch, agent_results):- is_verified = result.get("is_verified", False)- c.is_verified = is_verified- c.confidence = result.get("confidence", 0.0)- grounded = result.get("grounded", [])- numbers = result.get("numbers", [])- if "grounded" not in result or "numbers" not in result:- _LOG.warning(- "[Agentic research] claim=%s report omitted the schema-required grounded/numbers fields (grounded=%r numbers=%r) — verification for it is degraded",- c.claim_id,- grounded,- numbers,- )- c.agent_result = AgentResult(- claim_id=c.claim_id,- report=result.get("report", ""),- is_verified=is_verified,- confidence=c.confidence,- evidence_ids=result.get("evidence_ids", []),- gaps=result.get("gaps", []),- discovered_claims=result.get("discovered_claims", []),- grounded=grounded,- numbers=numbers,- )-- # Ultra: dynamic claim expansion- if mode.allows_dynamic_claims and result.get("discovered_claims"):- for dc in result["discovered_claims"]:- if dc and dc not in [cc.description for cc in ctx.claims]:- ctx.claims.append(- ClaimTarget(- claim_id=f"c_dyn_{len(ctx.claims)}",- description=dc,- )- )- _LOG.info('[Agentic research] Found a new angle worth researching: "%s"', dc)-- # ── Step A.5: note a discovered entity so graph_explore becomes eligible- # in the next round (its context gate requires context.last_entity). ──- ctx.note_entity(_discovered_entity(tools))-- # ── Step B: Sufficiency Check ──- all_chunks = {i: c for i, c in enumerate(tools.kbinfos.get("chunks", []))}- agent_results_list = [c.agent_result for c in ctx.claims if c.agent_result]- _LOG.info(- "[Sufficiency] Round %d: evidence pool=%d chunk(s), %d claim(s) with agent results: %s",- cycle + 1,- len(all_chunks),- len(agent_results_list),- [- {- "claim_id": r.claim_id,- "self_verified": r.is_verified,- "self_confidence": round(r.confidence, 3),- "evidence_ids": len(r.evidence_ids),- }- for r in agent_results_list- ],- )- cross_results = [cross_check_claim(r, all_chunks) for r in agent_results_list]-- verdict = compute_fusion_score(- agent_results_list,- cross_results,- mode,- question=ctx.question,- claims=ctx.claims,- all_chunks=all_chunks,- )- ctx.verdict = verdict-- # Decision ladder: the LLM Sufficient Context AutoRater is the primary- # sufficiency judge (invoked every round in high/ultra). Its verdict is- # combined with the code-level signals (hard vetoes + agent confidence)- # inside ``route_sufficiency_verdict`` → ``sufficiency_ladder``. The- # AutoRater's missing-pieces feedback is saved for the next round.- cited_ids: list[str] = []- for r in agent_results_list:- cited_ids.extend(r.evidence_ids or [])- boost = await llm_sufficiency_boost(tools, ctx.question, verdict, evidence_ids=cited_ids)- if boost and boost.get("followups"):- # Missing pieces → targeted follow-up searches for the next round.- ctx.pending_followups = boost.get("followups", [])- _LOG.info("[Agentic research] Stored %d follow-up query(ies) for next round.", len(ctx.pending_followups))- if boost:- _LOG.info("[Agentic research] Round %d: AutoRater is_sufficient=%s confidence=%.2f", cycle + 1, boost.get("is_sufficient"), boost.get("confidence", 1.0))-- # LLM groundedness review (Google "draft review" thought): check whether each- # claim's report is semantically supported by the cited evidence. Ungrounded- # claims (hallucinated / over-claimed drafts) are merged into hard_violations- # so the decision ladder forces a caveated answer — this catches relation/over-- # claim errors that the lexical code-level grounded check (cross_check_claim)- # cannot see.- from rag.advanced_rag.harness.orchestrator.grounded_llm import llm_grounded_verify-- grounded = await llm_grounded_verify(- tools,- ctx.question,- [(r.claim_id, r.report or "") for r in agent_results_list if r.report],- cited_ids,- )- # Treat a claim as violating when it is explicitly grounded=False OR has- # non-empty ungrounded assertions (covers the degenerate grounded=False /- # empty-ungrounded case too). Only accept IDs present in the original- # claims collection — the LLM may echo a bogus claim_id that must not leak- # into hard_violations.- valid_claim_ids = {r.claim_id for r in agent_results_list}- ungrounded_ids = [cid for cid, g in grounded.items() if cid in valid_claim_ids and (g.get("grounded") is False or g.get("ungrounded"))]- if ungrounded_ids:- existing = set(verdict.hard_violations or [])- verdict.hard_violations = list(existing | set(ungrounded_ids))- _LOG.info("[Agentic research] Round %d: %d claim(s) have ungrounded (draft-review) assertions: %s", cycle + 1, len(ungrounded_ids), ungrounded_ids)-- action, should_continue, caveat = route_sufficiency_verdict(- verdict,- mode_label,- cycle,- mode.max_orchestrator_cycles,- auto=boost,- )- if caveat:- _LOG.info("[Agentic research] Round %d: caveat=%s", cycle + 1, caveat)-- # Stagnation guard: when the verdict is not (yet) sufficient and the- # fusion score has not meaningfully improved for a couple of rounds,- # stop instead of burning the remaining cycle budget on unproductive- # re-searches. Override the CONTINUE decision with a partial answer.- if should_continue and verdict.status in ("INSUFFICIENT", "USEFUL_BUT_INCOMPLETE"):- if prev_score is not None and cycle >= _STAGNATION_CYCLES and verdict.score - prev_score < _STAGNATION_GAIN:- _LOG.info(- "[Agentic research] Round %d: score stagnant (%.3f → %.3f) — early-stopping to partial answer",- cycle + 1,- prev_score,- verdict.score,- )- action = "ANSWER_PARTIAL"- should_continue = False- else:- prev_score = verdict.score-- _LOG.info("[Agentic research] Round %d: evidence looks %s (confidence %.0f%%) — next: %s", cycle + 1, verdict.status, verdict.score * 100, action)-- if action == "ANSWER":- return _finalize(ctx, tools, partial=False, loop=rounds_run)- if action == "ANSWER_PARTIAL":- return _finalize(ctx, tools, partial=True, loop=rounds_run)- if action == "ABSTAIN":- tools.kbinfos["chunks"] = []- return {"verdict": verdict.__dict__, "abstain": True, "loop": rounds_run}- if action == "REPLAN":- # Ultra: re-plan on low score. Ground the new plan on the evidence- # gathered so far, and carry still-valid verified claims over so a- # replan doesn't re-research (and re-bill) work already done.- from rag.advanced_rag.harness.planner import planner_node-- state["feedback"] = verdict.feedback- state["route"] = route- state["seed_chunks"] = list(tools.kbinfos.get("chunks", []) or [])- new_plan = await planner_node(state, tools)- # Keep EVERY verified claim (even ones the new plan omitted — their- # evidence is still valid and shouldn't be re-researched), then- # append only the new plan's unverified claims.- verified = [c for c in ctx.claims if c.is_verified]- new_by_desc = {}- for c in new_plan.get("claims", ctx.claims):- if isinstance(c, ClaimTarget):- new_by_desc.setdefault(c.description, c)- seen = {c.description for c in verified}- ctx.claims = verified + [c for c in new_by_desc.values() if c.description not in seen]- if action == "FALLBACK_LLM":- return _finalize(ctx, tools, partial=True, fallback=True, loop=rounds_run)-- # Max cycles reached- return _finalize(ctx, tools, partial=True, loop=rounds_run)---async def _run_claim_research(- claim: ClaimTarget,- tools,- ctx: OrchestratorContext,- mode,- compilation_map: dict,- followups: list[str] | None = None,-) -> dict:- _LOG.info('[Agentic research] Researching: "%s"', _snip(claim.description))- # A dedicated pipeline per claim keeps the routing scope (``_routed_docs``)- # isolated: under asyncio.gather the shared single pipeline would let one- # claim's dataset_navigation_search leak its doc_scope into a sibling's- # follow-up searches (the doc_scope is set on the pipeline, not the claim).- # ``tools.kbinfos`` stays shared, so the citation pool still merges across- # claims via Pipeline._merge_into_kbinfos.- pipeline = Pipeline(tools, compilation_map)- try:- result = await asyncio.wait_for(- research_agent_loop(claim, tools, pipeline, ctx, mode, compilation_map, followups=followups),- timeout=CLAIM_RESEARCH_TIMEOUT_SECONDS,- )- except asyncio.CancelledError:- raise- except TimeoutError:- if getattr(pipeline, "_active_phase", None) == "locate":- if pipeline._round_had_evidence or pipeline._round_had_routed_scope_progress:- claim.locate_empty_streak = 0- else:- claim.locate_empty_streak += 1- _LOG.warning(- "[Agentic research] claim=%s timed out in locate (progress=%s, locate_empty_streak=%d).",- claim.claim_id,- pipeline._round_had_evidence or pipeline._round_had_routed_scope_progress,- claim.locate_empty_streak,- )- _LOG.warning(- '[Agentic research] Gave up on "%s" — it took longer than %ss.',- _snip(claim.description),- CLAIM_RESEARCH_TIMEOUT_SECONDS,- )- return {- "report": "",- "is_verified": False,- "confidence": 0.0,- "evidence_ids": [],- "gaps": [f"claim research timeout after {CLAIM_RESEARCH_TIMEOUT_SECONDS}s"],- "discovered_claims": [],- }- except Exception:- _LOG.exception('[Agentic research] Hit an error while researching "%s".', _snip(claim.description))- return {- "report": "",- "is_verified": False,- "confidence": 0.0,- "evidence_ids": [],- "gaps": ["claim research failed"],- "discovered_claims": [],- }-- _LOG.info(- '[Agentic research] Finished "%s" — %s, backed by %d passage(s) (confidence %.0f%%)%s.',- _snip(claim.description),- "answered" if result.get("is_verified") else "still unanswered",- len(result.get("evidence_ids") or []),- float(result.get("confidence") or 0.0) * 100,- f", {len(result.get('gaps') or [])} gap(s) remain" if result.get("gaps") else "",- )- return result---def _finalize(ctx: OrchestratorContext, tools, partial: bool = False, fallback: bool = False, loop: int = 0) -> dict:- """Merge agent results into kbinfos and return."""- _merge_agent_results(ctx, tools)- return {- "verdict": ctx.verdict.__dict__ if ctx.verdict else None,- "partial_answer": partial or fallback,- "loop": loop,- "kbinfos": tools.kbinfos,- }---def _merge_agent_results(ctx: OrchestratorContext, tools):- """Merge agent result reports into kbinfos as a pre_summary."""- combined = []- seen_evidence = set()-- for c in ctx.claims:- if c.agent_result and c.agent_result.report:- status = "✅" if c.is_verified else "❌"- combined.append(f"【{c.claim_id}】{status} {c.agent_result.report[:500]}")-- if combined:- tools.kbinfos["pre_summary"] = "\n\n".join(combined)-- # Collect the chunks the agents actually cited across all claims. These- # indices share the same positional space as kb_prompt's ``[ID:n]`` markers- # (both index tools.kbinfos["chunks"]).- for c in ctx.claims:- if c.agent_result and c.agent_result.evidence_ids:- for eid in c.agent_result.evidence_ids:- if isinstance(eid, int):- seen_evidence.add(eid)-- # Drop chunks no claim ever cited (e.g. pre_search recall that didn't pan- # out) so the final-answer LLM call only sees the useful evidence. Preserve- # order so the re-numbered [ID:n] citations stay stable. Defensive: never- # filter to empty — if nothing was cited, keep the full pool.- all_chunks = tools.kbinfos.get("chunks") or []- keep = sorted(i for i in seen_evidence if 0 <= i < len(all_chunks))- if keep and len(keep) < len(all_chunks):- _LOG.info("[Agentic research] Trimming evidence for the final answer: %d of %d chunk(s) were cited.", len(keep), len(all_chunks))- tools.kbinfos["chunks"] = [all_chunks[i] for i in keep]---async def _get_compilation_map(tools) -> dict[str, set[str]]:- """Build compilation map from RAGTools - check which KBs have compilation artifacts."""- result = {}- if not tools.kbs:- return result- for kb in tools.kbs:- comps = set()- parser_config = getattr(kb, "parser_config", None) or {}- if parser_config.get("toc"):- comps.add("toc")- if parser_config.get("knowledge_graph"):- comps.add("knowledge_graph")- if parser_config.get("wiki"):- comps.add("wiki")- if parser_config.get("mindmap"):- comps.add("mindmap")- if parser_config.get("page_index"):- comps.add("page_index")- await _add_template_group_compilations(comps, parser_config, getattr(kb, "tenant_id", ""))- if await _has_dataset_nav_rows(getattr(kb, "tenant_id", ""), getattr(kb, "id", "")):- comps.add("tree")- if comps:- result[kb.id] = comps- return result---async def _has_dataset_nav_rows(tenant_id: str, kb_id: str) -> bool:- if not tenant_id or not kb_id:- return False- try:- from common import settings- from common.doc_store.doc_store_base import OrderByExpr- from common.misc_utils import thread_pool_exec- from rag.nlp import search-- index_name = search.index_name(tenant_id)- if not settings.docStoreConn.index_exist(index_name, kb_id):- return False- res = await thread_pool_exec(- settings.docStoreConn.search,- ["id"],- [],- {"compile_kwd": ["dataset_nav"]},- [],- OrderByExpr(),- 0,- 1,- index_name,- [kb_id],- )- return bool(settings.docStoreConn.get_total(res))- except Exception:- _LOG.exception("[agentic] dataset-nav existence check failed for kb=%s", kb_id)- return False---async def _add_template_group_compilations(comps: set[str], parser_config: dict, tenant_id: str) -> None:- """Infer available compilation kinds from selected template groups."""- if not tenant_id:- return- try:- from api.db.services.compilation_template_group_service import CompilationTemplateGroupService- from common.misc_utils import thread_pool_exec- from rag.svr.task_executor_refactor.chunk_post_processor import (- _parser_config_compilation_template_group_ids,- )- except Exception:- _LOG.exception("[agentic] compilation-map helper import failed")- return-- try:- group_ids = _parser_config_compilation_template_group_ids(parser_config)- except Exception:- _LOG.exception("[agentic] compilation template group id resolution failed")- return-- for group_id in group_ids:- try:- group = await thread_pool_exec(CompilationTemplateGroupService.get_saved, group_id, tenant_id)- except Exception:- _LOG.exception("[agentic] compilation template group read failed id=%s", group_id)- continue- for template in (group or {}).get("templates") or []:- config = template.get("config") or {}- raw_kind = (config.get("kind") if isinstance(config, dict) else "") or template.get("kind") or ""- raw_norm = raw_kind.strip().lower().replace("-", "_") if isinstance(raw_kind, str) else ""- kind = _compilation_kind_for_agentic_map(raw_kind)- if raw_norm == "knowledge_graph":- comps.add("knowledge_graph")- if kind == "tree":- comps.add("tree")- elif kind in {"timeline", "page_index", "pageindex"}:- comps.add("page_index")- elif kind in {"mindmap", "mind_map"}:- comps.add("mindmap")- elif kind == "wiki":- comps.add("wiki")---def _compilation_kind_for_agentic_map(kind) -> str:- if not isinstance(kind, str):- return ""- normalized = kind.strip().lower().replace("-", "_")- if normalized in {"pageindex", "page_index"}:- return "timeline"- return normalizeddiff --git a/rag/advanced_rag/harness/orchestrator/decompose.py b/rag/advanced_rag/harness/orchestrator/decompose.pydeleted file mode 100644index 4e28cee33a..0000000000--- a/rag/advanced_rag/harness/orchestrator/decompose.py+++ /dev/null@@ -1,588 +0,0 @@-"""Medium mode: decompose -> parallel search -> evidence-guided follow-up."""--import asyncio-import json-import logging-import re--from rag.advanced_rag.harness.types import ClaimTarget, AgentResult, OrchestratorContext-from rag.advanced_rag.harness.config import get_mode-from rag.advanced_rag.harness.sufficiency import (- cross_check_claim,- compute_fusion_score,- route_sufficiency_verdict,-)-from rag.advanced_rag.harness.orchestrator.sufficiency_llm import llm_sufficiency_boost-from rag.advanced_rag.harness.stats import in_phase-from rag.advanced_rag.harness.tools.search import hybrid_search--_LOG = logging.getLogger(__name__)--_MAX_EVIDENCE_SNIPPETS = 6-_MAX_NEXT_QUERIES = 3--_EVIDENCE_ANALYSIS_SYSTEM = """You are controlling a multi-hop RAG retrieval loop.-Judge whether the retrieved passages verify the claim using only the provided evidence.-If the claim is not verified, produce targeted next search queries that use entities,-dates, names, or relationships discovered in the evidence and move closer to the-original question.-Distinguish final-answer entities from bridge entities. If the passages identify-only a clue node in the chain, keep the claim incomplete and search for the-remaining relation needed by the original question. Return JSON only."""--_EVIDENCE_ANALYSIS_USER = """Original question:-{question}--Claim to verify:-{claim}--Search query used this round:-{query}--Round: {cycle} of {max_cycles}--Retrieved evidence snippets:-{evidence}--Return JSON:-{{- "is_verified": true,- "confidence": 0.0,- "report": "Short evidence-backed finding, or what was learned so far.",- "gaps": ["specific missing fact or relationship"],- "next_queries": ["standalone follow-up search query"],- "grounded": ["key asserted facts that ARE directly supported by the cited evidence, atomically and verbatim enough to match"],- "numbers": ["for numerical/multi-hop answers: each figure used + its source, e.g. '2,161,000 from Wikipedia Demographics of Paris'; list ALL conflicting figures if several sources disagree"]-}}-Only list in grounded the facts you actually SAW in the evidence; prior-knowledge guesses go in gaps. If the claim is numerical or multi-hop and the evidence has multiple close-but-different figures, disclose all of them in numbers rather than silently picking one."""---@in_phase("decompose")-async def decompose_and_search(state: dict, tools) -> dict:- """Decompose, retrieve, analyze evidence, then iterate with next-hop queries."""- question = state.get("question", "")- keywords = state.get("keywords", "")- claims_raw = state.get("claims", [])- route = state.get("route")- mode_label = _mode_label(route)- mode = get_mode(mode_label)- max_cycles = _cycle_budget(state, mode.max_orchestrator_cycles)-- claims = [ClaimTarget(**c) if isinstance(c, dict) else c for c in claims_raw]- ctx = OrchestratorContext(question=question, claims=claims, mode=mode_label)- attempted_queries: dict[str, set[str]] = {c.claim_id: set() for c in ctx.claims}- pending_queries: dict[str, list[str]] = {c.claim_id: [] for c in ctx.claims}- completed_cycles = 0-- # Stagnation guard: stop when the fusion score stops improving across- # consecutive rounds (corpus lacks the data, follow-ups return nothing)- # instead of burning the remaining cycle budget unproductively.- prev_score: float | None = None- _STAGNATION_CYCLES = 2- _STAGNATION_GAIN = 0.05-- for cycle in range(max_cycles):- ctx.iteration = cycle- unverified = [c for c in ctx.claims if not c.is_verified]- if not unverified:- break-- _LOG.info(- "[Decompose search] Round %d of %d: researching %d unresolved claim(s).",- cycle + 1,- max_cycles,- len(unverified),- )-- tasks = []- searched_claims = []- for c in unverified:- query = _pick_next_query(- question,- c,- attempted_queries.setdefault(c.claim_id, set()),- pending_queries.setdefault(c.claim_id, []),- )- if not query:- _LOG.info("[Decompose search] No unused follow-up query remains for claim %s.", c.claim_id)- continue- attempted_queries[c.claim_id].add(_normalize_query(query))- searched_claims.append((c, query))- tasks.append(hybrid_search(tools, query=query, keywords=keywords, use_compiled=True))-- if not tasks:- break-- results = await asyncio.gather(*tasks, return_exceptions=True)- analysis_inputs = []-- for (c, query), result in zip(searched_claims, results):- if isinstance(result, Exception):- _LOG.exception("[Decompose search] Search failed for claim %s.", c.claim_id, exc_info=result)- result = {"chunks": [], "doc_aggs": []}-- chunks = result.get("chunks", []) or []- _merge_kbinfos(tools, result)- evidence_ids = _evidence_ids(tools, chunks)- analysis_inputs.append((c, query, result, evidence_ids))-- analyses = await asyncio.gather(- *[- _analyze_claim_evidence(- question=question,- claim=c,- query=query,- result=result,- evidence_ids=evidence_ids,- cycle=cycle,- max_cycles=max_cycles,- tools=tools,- )- for c, query, result, evidence_ids in analysis_inputs- ],- return_exceptions=True,- )-- for (c, query, result, evidence_ids), analysis in zip(analysis_inputs, analyses):- if isinstance(analysis, Exception):- _LOG.exception("[Decompose search] Evidence analysis failed for claim %s.", c.claim_id, exc_info=analysis)- analysis = _fallback_analysis(result, cycle, max_cycles)-- c.is_verified = analysis["is_verified"]- c.confidence = analysis["confidence"]- c.agent_result = AgentResult(- claim_id=c.claim_id,- report=analysis["report"],- is_verified=c.is_verified,- confidence=c.confidence,- evidence_ids=evidence_ids,- gaps=analysis["gaps"],- discovered_claims=[],- grounded=analysis.get("grounded", []),- numbers=analysis.get("numbers", []),- )-- next_queries = _new_queries(- analysis.get("next_queries", []),- attempted_queries.setdefault(c.claim_id, set()),- )- if not c.is_verified and next_queries:- pending_queries.setdefault(c.claim_id, []).extend(next_queries)- _LOG.info(- "[Decompose search] Claim %s needs another hop; queued %d targeted query/queries.",- c.claim_id,- len(next_queries),- )- _LOG.info(- '[Decompose search] Claim %s after "%s": %s (confidence %.0f%%).',- c.claim_id,- _snip(query),- "verified" if c.is_verified else "still incomplete",- c.confidence * 100,- )-- completed_cycles = cycle + 1-- all_chunks = {i: c for i, c in enumerate(tools.kbinfos.get("chunks", []))}- agent_results = [c.agent_result for c in ctx.claims if c.agent_result]- cross_results = [cross_check_claim(r, all_chunks) for r in agent_results]-- # HEAD sufficiency enhancements: pass the question + claims + evidence so- # the fusion activates required-entity AND-semantics veto, grounded-fact- # verification, and numeric multi-source conflict detection (not just the- # baseline ratio/mean that upstream's 3-arg call skipped).- verdict = compute_fusion_score(- agent_results,- cross_results,- mode,- question=ctx.question,- claims=ctx.claims,- all_chunks=all_chunks,- )- ctx.verdict = verdict-- # LLM Sufficient Context AutoRater (primary sufficiency judge). Medium- # keeps it gated to the borderline band for cost control; its verdict- # (`auto=boost`) is fed to the decision ladder inside- # ``route_sufficiency_verdict``, which replaces the old manual- # SUFFICIENT upgrade. Missing-piece follow-ups feed the next hop.- boost: dict = {}- if verdict.status in ("USEFUL_BUT_INCOMPLETE", "INSUFFICIENT", "CONFLICTING"):- boost = await llm_sufficiency_boost(tools, ctx.question, verdict, evidence_ids=_global_evidence_ids(tools, {"chunks": tools.kbinfos.get("chunks", [])}))- followups = boost.get("followups") or []- if followups:- ctx.pending_followups = followups- _LOG.info("[Decompose] Stored %d follow-up query(ies) for next round.", len(ctx.pending_followups))- if boost:- _LOG.info("[Decompose] AutoRater is_sufficient=%s confidence=%.2f", boost.get("is_sufficient"), boost.get("confidence", 1.0))-- # LLM groundedness review (Google "draft review"): runs unconditionally so every- # decomposed result — including a non-critical-band SUFFICIENT — is groundedness-- # validated before the status gate. (The lexical NER grounded check is disabled- # in favour of this LLM review, so it must not be skipped on any path.) Ungrounded- # claim drafts are merged into hard_violations → decision ladder caveat.- from rag.advanced_rag.harness.orchestrator.grounded_llm import llm_grounded_verify-- # Union of cited evidence IDs across all claim results (matches the- # agentic orchestrator's cited-evidence behavior) so the reviewer sees- # the exact evidence each claim referenced, not a global prefix.- cited_evidence_ids: list[str] = []- for r in agent_results:- cited_evidence_ids.extend(r.evidence_ids or [])- grounded = await llm_grounded_verify(- tools,- ctx.question,- [(r.claim_id, r.report or "") for r in agent_results if r.report],- cited_evidence_ids or None,- )- # Treat a claim as violating when it is explicitly grounded=False OR has- # non-empty ungrounded assertions (covers the degenerate grounded=False /- # empty-ungrounded case too). Only accept IDs that exist in the original- # claims collection — the LLM may echo a bogus claim_id, which must not- # leak into hard_violations.- valid_claim_ids = {r.claim_id for r in agent_results}- ungrounded_ids = [cid for cid, g in grounded.items() if cid in valid_claim_ids and (g.get("grounded") is False or g.get("ungrounded"))]- if ungrounded_ids:- existing = set(verdict.hard_violations or [])- verdict.hard_violations = list(existing | set(ungrounded_ids))- _LOG.info("[Decompose] %d claim(s) have ungrounded (draft-review) assertions: %s", len(ungrounded_ids), ungrounded_ids)-- action, should_continue, caveat = route_sufficiency_verdict(- verdict,- mode_label,- cycle,- max_cycles,- auto=boost,- )- if caveat:- _LOG.info("[Decompose] caveat=%s", caveat)-- # Stagnation guard: if the verdict is not (yet) sufficient and the score- # has not meaningfully improved across consecutive rounds, stop instead- # of burning the remaining cycle budget on unproductive re-searches.- if should_continue and verdict.status in ("INSUFFICIENT", "USEFUL_BUT_INCOMPLETE"):- if prev_score is not None and cycle >= _STAGNATION_CYCLES and verdict.score - prev_score < _STAGNATION_GAIN:- _LOG.info(- "[Decompose] Round %d: score stagnant (%.3f → %.3f) — early-stopping to partial answer",- cycle + 1,- prev_score,- verdict.score,- )- action = "ANSWER_PARTIAL"- should_continue = False- else:- prev_score = verdict.score-- if action in ("ANSWER", "ANSWER_PARTIAL"):- return _finalize(ctx, tools, partial=action == "ANSWER_PARTIAL", loop=completed_cycles)- if action == "ABSTAIN":- tools.kbinfos["chunks"] = []- return {"verdict": verdict.__dict__, "abstain": True, "loop": completed_cycles}- if action == "FALLBACK_LLM":- return _finalize(ctx, tools, partial=True, loop=completed_cycles)- if not should_continue:- break-- if not tools.kbinfos.get("chunks"):- return {"empty_result": True, "kbinfos": tools.kbinfos, "loop": completed_cycles}-- partial = any(not c.is_verified for c in ctx.claims)- return _finalize(ctx, tools, partial=partial, loop=completed_cycles)---async def _analyze_claim_evidence(- *,- question: str,- claim: ClaimTarget,- query: str,- result: dict,- evidence_ids: list[int],- cycle: int,- max_cycles: int,- tools,-) -> dict:- chunks = result.get("chunks", []) or []- if not chunks:- return {- "is_verified": False,- "confidence": 0.0,- "report": "",- "gaps": ["no evidence found"],- "next_queries": _fallback_queries(question, claim),- }-- try:- user = _EVIDENCE_ANALYSIS_USER.format(- question=question,- claim=claim.description,- query=query,- cycle=cycle + 1,- max_cycles=max_cycles,- evidence=_format_evidence(chunks),- )- msg = await tools._fit_messages(_EVIDENCE_ANALYSIS_SYSTEM, user)- ans = await tools.chat_mdl.async_chat(msg[0]["content"], msg[1:], {"temperature": 0.1})- if isinstance(ans, tuple):- ans = ans[0]- parsed = _extract_json(ans)- return _normalize_analysis(parsed, result, evidence_ids, question, claim, cycle, max_cycles)- except Exception:- _LOG.exception("[Decompose search] Evidence analysis LLM call failed.")- return _fallback_analysis(result, cycle, max_cycles, question, claim)---def _mode_label(route) -> str:- if not route:- return "medium"- if isinstance(route, dict):- return route.get("thinking_mode", "medium")- return getattr(route, "thinking_mode", "medium")---def _cycle_budget(state: dict, default_cycles: int) -> int:- try:- requested = int(state.get("max_loops") or default_cycles)- except (TypeError, ValueError):- requested = default_cycles- return max(1, min(default_cycles, requested))---def _extract_json(text: str) -> dict:- text = re.sub(r"^.*</think>", "", text or "", flags=re.DOTALL).strip()- text = re.sub(r"```(?:json)?\s*|\s*```", "", text).strip()- try:- import json_repair-- return json_repair.loads(text)- except Exception:- try:- return json.loads(text)- except Exception:- _LOG.warning("[Decompose search] Failed to parse evidence analysis output: %s", text[:200])- return {}---def _normalize_analysis(- parsed: dict,- result: dict,- evidence_ids: list[int],- question: str,- claim: ClaimTarget,- cycle: int,- max_cycles: int,-) -> dict:- confidence = _clamp_float(parsed.get("confidence"), 0.0, 1.0)- is_verified = bool(parsed.get("is_verified")) and bool(evidence_ids) and confidence >= 0.55- report = str(parsed.get("report") or "").strip() or _summarize(result)- gaps = _string_list(parsed.get("gaps"))- next_queries = _string_list(parsed.get("next_queries"))[:_MAX_NEXT_QUERIES]- grounded = _string_list(parsed.get("grounded"))- numbers = _string_list(parsed.get("numbers"))-- if is_verified:- gaps = []- next_queries = []- # NOTE: no optimistic floor here. The old ``max(confidence, 0.65)``- # inflated medium's agent confidence and distorted the decision-ladder- # gate (agent_confidence >= c_high/c_low). Keep the LLM's raw confidence- # so the ladder's thresholds behave as designed.- elif not next_queries and cycle + 1 < max_cycles:- next_queries = _fallback_queries(question, claim)-- return {- "is_verified": is_verified,- "confidence": confidence,- "report": report,- "gaps": gaps,- "next_queries": next_queries,- "grounded": grounded,- "numbers": numbers,- }---def _fallback_analysis(- result: dict,- cycle: int,- max_cycles: int,- question: str = "",- claim: ClaimTarget | None = None,-) -> dict:- chunks = result.get("chunks", []) or []- is_last_cycle = cycle + 1 >= max_cycles- is_verified = bool(chunks) and is_last_cycle- next_queries = [] if is_last_cycle or claim is None else _fallback_queries(question, claim)- return {- "is_verified": is_verified,- "confidence": 0.55 if is_verified else (0.35 if chunks else 0.0),- "report": _summarize(result),- "gaps": [] if is_verified else ["need more specific evidence"],- "next_queries": next_queries,- "grounded": [],- "numbers": [],- }---def _pick_next_query(- question: str,- claim: ClaimTarget,- attempted: set[str],- pending: list[str],-) -> str:- while pending:- query = (pending.pop(0) or "").strip()- normalized = _normalize_query(query)- if normalized and normalized not in attempted:- return query-- candidates = []- if not attempted:- candidates.append(claim.description)- candidates.extend(_fallback_queries(question, claim))-- for query in candidates:- query = (query or "").strip()- normalized = _normalize_query(query)- if normalized and normalized not in attempted:- return query- return ""---def _fallback_queries(question: str, claim: ClaimTarget) -> list[str]:- candidates = []- for gap in _agent_result_gaps(claim.agent_result):- candidates.append(f"{claim.description} {gap}")- if question:- candidates.append(f"{question} {claim.description}")- candidates.append(claim.description)- return candidates[:_MAX_NEXT_QUERIES]---def _new_queries(raw_queries: list[str], attempted: set[str]) -> list[str]:- queries = []- seen = set(attempted)- for query in raw_queries:- query = (query or "").strip()- normalized = _normalize_query(query)- if not normalized or normalized in seen:- continue- seen.add(normalized)- queries.append(query)- if len(queries) >= _MAX_NEXT_QUERIES:- break- return queries---def _agent_result_gaps(agent_result) -> list[str]:- if not agent_result:- return []- if isinstance(agent_result, dict):- return _string_list(agent_result.get("gaps"))- return _string_list(getattr(agent_result, "gaps", []))---def _normalize_query(query: str) -> str:- return " ".join((query or "").lower().split())---def _format_evidence(chunks: list[dict]) -> str:- snippets = []- for i, chunk in enumerate(chunks[:_MAX_EVIDENCE_SNIPPETS], start=1):- text = chunk.get("content_with_weight") or chunk.get("content") or chunk.get("text") or ""- source = chunk.get("docnm_kwd") or chunk.get("doc_name") or chunk.get("doc_id") or "source"- snippets.append(f"[{i}] {source}: {_snip(text, 900)}")- return "\n\n".join(snippets) or "(no evidence)"---def _string_list(value) -> list[str]:- if isinstance(value, str):- return [value.strip()] if value.strip() else []- if not isinstance(value, list):- return []- return [str(v).strip() for v in value if str(v).strip()]---def _clamp_float(value, lo: float, hi: float) -> float:- try:- number = float(value)- except (TypeError, ValueError):- number = 0.0- return min(hi, max(lo, number))---def _evidence_ids(tools, chunks: list[dict]) -> list[int]:- all_chunks = tools.kbinfos.get("chunks", [])- index_by_key = {_chunk_key(c): i for i, c in enumerate(all_chunks)}- ids = []- for chunk in chunks:- idx = index_by_key.get(_chunk_key(chunk))- if idx is not None and idx not in ids:- ids.append(idx)- return ids---def _finalize(ctx: OrchestratorContext, tools, partial: bool, loop: int) -> dict:- combined = []- for claim in ctx.claims:- if claim.agent_result and claim.agent_result.report:- status = "verified" if claim.is_verified else "incomplete"- combined.append(f"[{claim.claim_id}] {status} ({claim.description}): {claim.agent_result.report[:500]}")- if combined:- tools.kbinfos["pre_summary"] = "Research findings. These may include bridge entities; the final answer must still satisfy the original question's requested role.\n\n" + "\n\n".join(combined)-- return {- "verdict": ctx.verdict.__dict__ if ctx.verdict else None,- "partial_answer": partial,- "kbinfos": tools.kbinfos,- "loop": loop,- }---def _snip(text: str, limit: int = 160) -> str:- text = (text or "").replace("\n", " ").strip()- return text if len(text) <= limit else text[: limit - 3] + "..."---def _merge_kbinfos(tools, result: dict):- if not result or not result.get("chunks"):- return- seen = {_chunk_key(c) for c in tools.kbinfos.get("chunks", [])}- for c in result.get("chunks", []):- k = _chunk_key(c)- if k in seen:- continue- seen.add(k)- tools.kbinfos.setdefault("chunks", []).append(c)- dseen = {d.get("doc_id") for d in tools.kbinfos.get("doc_aggs", [])}- for d in result.get("doc_aggs", []):- if d.get("doc_id") in dseen:- continue- dseen.add(d.get("doc_id"))- tools.kbinfos.setdefault("doc_aggs", []).append(d)---def _chunk_key(ck: dict) -> str:- return ck.get("chunk_id") or ck.get("id") or str(id(ck))---def _global_evidence_ids(tools, result: dict) -> list[int]:- """Map a search result's chunks to their indices in ``tools.kbinfos``.-- The cross-check resolves evidence IDs against the shared kbinfos pool, so- the IDs must be global indices there — not positions within this result.- Must be called AFTER ``_merge_kbinfos`` so fresh chunks have indices.- """- index_by_key: dict[str, int] = {}- for idx, ck in enumerate(tools.kbinfos.get("chunks", [])):- index_by_key.setdefault(_chunk_key(ck), idx)- ids: list[int] = []- for ck in result.get("chunks", []):- idx = index_by_key.get(_chunk_key(ck))- if idx is not None and idx not in ids:- ids.append(idx)- return ids---def _summarize(result: dict) -> str:- chunks = result.get("chunks", [])- texts = [(c.get("content_with_weight") or c.get("content") or c.get("text") or "")[:200] for c in chunks[:3]]- return " | ".join(texts)diff --git a/rag/advanced_rag/harness/orchestrator/direct.py b/rag/advanced_rag/harness/orchestrator/direct.pyindex ed2b3126c5..befe6fd499 100644--- a/rag/advanced_rag/harness/orchestrator/direct.py+++ b/rag/advanced_rag/harness/orchestrator/direct.py@@ -13,9 +13,18 @@ async def direct_search(state: dict, tools) -> dict:"""Single hybrid search → merge into kbinfos."""question = state.get("question", "")keywords = state.get("keywords", "")+ # Entity/qualifier-weighted retrieval query: a problem-level search over the+ # bare question is exactly where the entity must dominate the ranking, so the+ # weighted query (entity x3, qualifier x3) is always attached.+ retrieval_query = ""+ try:+ if hasattr(tools, "_extract_keywords_weighted"):+ retrieval_query, _ = await tools._extract_keywords_weighted(question)+ except Exception:+ _LOG.exception("[Direct] entity-weighted keyword extraction failed")_LOG.info('[Direct search] Looking up the knowledge base for: "%s" (keywords: %s)', question, keywords)- result = await hybrid_search(tools, query=question, keywords=keywords, use_compiled=True)+ result = await hybrid_search(tools, query=question, keywords=keywords, retrieval_query=retrieval_query, use_compiled=True)_merge_kbinfos(tools, result)if not _has_chunks(tools):diff --git a/rag/advanced_rag/harness/orchestrator/grounded_llm.py b/rag/advanced_rag/harness/orchestrator/grounded_llm.pydeleted file mode 100644index 6cd64e48f1..0000000000--- a/rag/advanced_rag/harness/orchestrator/grounded_llm.py+++ /dev/null@@ -1,117 +0,0 @@-"""LLM groundedness review (draft review) for the orchestrator.--Inspired by Google's Sufficient Context Agent — it reviews the *intermediate-draft* (each claim's report) against the retrieved snippets to decide whether-the draft is actually grounded in the evidence.--This complements the code-level grounded check (``cross_check_claim`` /-``_grounded_hit``, which is lexical: it only catches missing entities / digits).-The LLM review is *semantic*: it catches assertions whose relation or claim is-not supported by the evidence even when the entity is present (e.g. evidence-says "Ithaca is a birthplace" but the report claims "hometown is Ithaca", or-the evidence only mentions a medication was prescribed while the report-over-claims "the patient is well").--Ungrounded assertions are fed back into the decision ladder as hard violations,-so a hallucinated / over-claimed draft forces a caveated answer or a re-search.-"""--from __future__ import annotations--import logging--from rag.advanced_rag.harness.stats import in_phase-from rag.prompts.generator import PROMPT_JINJA_ENV, gen_json-from rag.prompts.template import load_prompt--_LOG = logging.getLogger(__name__)--GROUNDED_REVIEW = load_prompt("grounded_select")---async def _llm_chat_json(chat_mdl, prompt_text: str):- try:- return await gen_json(prompt_text, "Output:\n", chat_mdl)- except Exception as exc: # noqa: BLE001- _LOG.info("[Grounded-draft] gen_json failed: %s", exc)- return {}---def _render_reports(reports: list[tuple[str, str]]) -> str:- """Render claim_id → report lines for the reviewer prompt."""- if not reports:- return "(no claims)"- return "\n".join(f"Claim {cid}: {rpt}" for cid, rpt in reports if rpt)---@in_phase("grounded")-async def llm_grounded_verify(- tools,- question: str,- reports: list[tuple[str, str]],- evidence_ids=None,-) -> dict:- """Draft review: is each claim's report grounded in the cited evidence?-- Parameters- ----------- tools : RAGTools- Must expose ``chat_mdl`` and ``kbinfos``.- reports : list[(claim_id, report)]- Each claim's draft text (``AgentResult.report``).- evidence_ids : list[str] | None- Union of cited evidence chunk IDs (falls back to a bounded prefix).-- Returns- -------- dict : ``{claim_id: {"grounded": bool, "ungrounded": [str, ...]}}``- Empty dict when the LLM review is unavailable (no chat model, no- evidence, or a failure) — callers treat that as "no new signal".- """- if not reports:- return {}- chat_mdl = getattr(tools, "chat_mdl", None)- if chat_mdl is None:- return {}-- from rag.advanced_rag.harness.orchestrator.sufficiency_llm import (- _evidence_md,- _narrow_keywords,- )-- evidence_md = _evidence_md(tools, evidence_ids, keywords=_narrow_keywords(question))- if not evidence_md:- return {}-- prompt_text = PROMPT_JINJA_ENV.from_string(GROUNDED_REVIEW).render(- question=question,- reports=_render_reports(reports),- evidence=evidence_md,- )- _LOG.info("[Grounded-draft] reviewing %d claim report(s) (evidence %d chars)", len(reports), len(evidence_md))- result = await _llm_chat_json(chat_mdl, prompt_text)-- # The LLM may return a malformed payload (array / scalar / null) instead of a- # dict. Log and fall back to the documented empty result so we never crash- # with AttributeError on .get("claims").- if not isinstance(result, dict):- _LOG.info("[Grounded-draft] unexpected LLM response type=%s (expected dict); treating as no groundedness signal", type(result).__name__)- return {}-- out: dict = {}- for item in result.get("claims") or []:- cid = str(item.get("claim_id") or "")- if not cid:- continue- ung = item.get("ungrounded_assertions") or []- ungrounded = []- for u in ung:- if isinstance(u, dict):- ungrounded.append(str(u.get("assertion") or u.get("reason") or ""))- elif u:- ungrounded.append(str(u))- out[cid] = {- "grounded": bool(item.get("grounded")),- "ungrounded": [a for a in ungrounded if a],- }- return outdiff --git a/rag/advanced_rag/harness/orchestrator/query_rewriter.py b/rag/advanced_rag/harness/orchestrator/query_rewriter.pynew file mode 100644index 0000000000..c186b323a5--- /dev/null+++ b/rag/advanced_rag/harness/orchestrator/query_rewriter.py@@ -0,0 +1,98 @@+"""Query Rewriter++Query Rewriter has two jobs:+ - Phase 1: break the long request into simple, searchable questions so the+ retriever finds relevant content more accurately.+ - Phase 4: turn the Sufficient Context Agent's missing-pieces feedback into a+ NEW targeted search query (e.g. "you missed allergies -> search 'rashes' /+ 'adverse events'").++This module implements the Phase 4 role for the orchestrator: it takes a+Sufficient Context Agent forward gap (what is missing + a search hint) and+rewrites it into a concrete, retrievable query. Unlike ``_try_replan``'s current+behavior — which reuses the gap text verbatim as a new claim description — this+produces a targeted query that actually names the missing entity + relation, so+the next search hits the gap instead of re-searching the same angle.++The rewriting is deliberately cheap and only invoked on replan (replan budget is+bounded), so it adds little latency.+"""++from __future__ import annotations++import logging++from rag.advanced_rag.harness.stats import in_phase+from rag.prompts.generator import PROMPT_JINJA_ENV, gen_json+from rag.prompts.template import load_prompt++_LOG = logging.getLogger(__name__)++REWRITE_PROMPT = load_prompt("sca_query_rewrite")+++@in_phase("rewrite")+async def rewrite_gap_to_query(+ tools,+ question: str,+ gaps: list[tuple[str, str]],+ bridge_values: list | None = None,+ research_context: str = "",+) -> list[dict]:+ """Rewrite forward gaps into targeted search queries (multi-hop aware).++ Parameters+ ----------+ tools : RAGTools+ Must expose ``chat_mdl``.+ question : str+ The original user question (context for the rewrite).+ gaps : list[(what, search_hint)]+ Forward gaps (what the answer still needs, plus a search hint).+ bridge_values : list | None+ Already-resolved bridge values across the claims (e.g. the confirmed shows+ in a "which finale ran longest" question). The rewriter anchors the new+ search query to these, so a missing hop is re-searched with the resolved+ upstream value instead of re-deriving it.+ research_context : str+ Rendered block describing what previous rounds already tried (queries ++ outcomes) and what the current evidence pool roughly contains. Gives the+ rewriter enough information to aim at UNCOVERED angles instead of+ paraphrasing dead ones — the information-augmented counterpart of the+ react loop's in-context visibility.++ Returns+ -------+ list[dict] : ``[{"query": "..."}]`` — targeted, retrievable queries. Empty+ when the rewrite is unavailable / fails, in which case the caller falls+ back to the original gap text.+ """+ chat_mdl = getattr(tools, "chat_mdl", None)+ if chat_mdl is None or not gaps:+ return []+ gaps_text = "\n".join(f"- what: {g[0] or ''}; hint: {g[1] or ''}" for g in gaps)+ bridge_text = "\n".join(f"- {b}" for b in (bridge_values or []) if str(b).strip())+ rendered = PROMPT_JINJA_ENV.from_string(REWRITE_PROMPT).render(+ question=question or "",+ gaps=gaps_text,+ bridge_values=bridge_text,+ research_context=research_context,+ )+ try:+ result = await gen_json(rendered, "Output:\n", chat_mdl)+ except Exception as exc: # noqa: BLE001+ _LOG.info("[QueryRewrite] failed: %s", exc)+ return []+ if not isinstance(result, dict):+ return []+ out = []+ for q in result.get("queries") or []:+ if isinstance(q, dict):+ query = str(q.get("query") or q.get("question") or "").strip()+ elif q:+ query = str(q).strip()+ else:+ continue+ if query and query not in out:+ out.append({"query": query})+ return outdiff --git a/rag/advanced_rag/harness/orchestrator/sufficiency_llm.py b/rag/advanced_rag/harness/orchestrator/sufficiency_llm.pydeleted file mode 100644index 50756eb74c..0000000000--- a/rag/advanced_rag/harness/orchestrator/sufficiency_llm.py+++ /dev/null@@ -1,266 +0,0 @@-"""LLM Sufficient Context AutoRater for the orchestrator.--This is the *primary* sufficiency judge in the decision-ladder design-(Sufficient Context paper, arXiv 2411.06037). We delegate to the LLM-AutoRater (``tools.judge_sufficiency``, backing prompt ``sufficiency_select``)-to decide whether the retrieved evidence actually supports a plausible answer.-- - AutoRater judges query + evidence semantically, not just keyword matching.- - On "insufficient" it returns concrete ``missing_information`` which we turn- into follow-up search queries (missing-pieces feedback) for the next round.- - It also returns ``confidence`` / ``contradictions`` / ``required_entities``- consumed by the decision ladder (``sufficiency_ladder``).--Call frequency: high/ultra invoke it every round; medium keeps it gated to the-critical band (token cost stays bounded for the default mode). The-``verdict.status in (...)"`` check below implements that medium gate.-"""--from __future__ import annotations--import logging-import re--from rag.advanced_rag.harness.stats import in_phase-from rag.advanced_rag.harness.types import SufficiencyVerdict--_LOG = logging.getLogger(__name__)---# Cap the AutoRater's evidence so a single judge call does not balloon to-# hundreds of KB (6.log showed 271KB from 80 accumulated passages). The judge-# only needs the *cited* evidence plus a bounded prefix — not the whole pool.-_MAX_EVIDENCE_CHUNKS = 24-_MAX_CHUNK_CHARS = 800-_MAX_EVIDENCE_CHARS = 24_000---def _narrow_keywords(question: str) -> list[str]:- """Language-agnostic keywords of ``question`` for snippet narrowing.-- No language-specific stopword list (RAGFlow supports en/zh/de/fr/es/pt/ja and- any hard-coded English stopwords would be wrong for the rest). Instead we use- a length heuristic that generalises across scripts:- - numeric tokens (years / figures) — universal, high-signal;- - latin tokens of length >= 4 (keeps content words like "population" /- "paris" / "statistics" while the short function words "the"/"of"/"was"- fall below the cut — a length rule, not a stopword list);- - CJK runs are converted to character bigrams (巴黎 → 巴黎; 人口 → 人口),- which is tokeniser-free and works for zh/ja without any word segmenter.-- Keeping a few extra tokens is safe: narrowing matches more sentences, so it- retains more of the chunk and is less likely to drop the answer.- """- tokens = re.findall(r"[a-zA-Z0-9]+|[\u4e00-\u9fff]+", (question or "").lower())- kw: list[str] = []- for t in tokens:- if t.isdigit():- kw.append(t)- elif re.search(r"[a-zA-Z]", t):- if len(t) >= 4:- kw.append(t)- else: # CJK run → character bigrams- if len(t) >= 2:- kw.extend(t[i : i + 2] for i in range(len(t) - 1))- return kw---def _clamp(value) -> float:- """Clamp the AutoRater's confidence field to [0, 1], defaulting to 1.0."""- try:- return max(0.0, min(1.0, float(value)))- except (TypeError, ValueError):- return 1.0---# A chunk with this many sentences or fewer is never narrowed — there is little-# to save and a real risk of losing the answer sentence.-_NARROW_MIN_SENTENCES = 3-# Number of neighbours kept around each keyword-bearing sentence.-_NARROW_NEIGHBOURS = 1---def _narrow_snippet_safe(content: str, kw_list: list[str]) -> str | None:- """Self-contained keyword narrowing of ``content`` to a compact snippet.-- This is a purpose-built replacement for ``_narrow_content`` (which returns- a re-formatted string full of ``...`` and ``*kw*`` markers that, when split- again downstream, corrupts sentence counts). Here we work directly on the- original sentences and re-emit plain text, so there is no intermediate- format to misinterpret.-- The only rule — a deliberately small one, because no rule set can cover- every phrasing: **narrow only when the keywords actually cover a meaningful- share of the chunk.** Concretely, we keep the keyword-bearing sentences plus- ``_NARROW_NEIGHBOURS`` around each, and refuse to narrow (return ``None``,- caller keeps the whole chunk) when:- * the chunk is short (<= ``_NARROW_MIN_SENTENCES`` sentences), or- * no sentence mentions a keyword, or- * keywords hit too few sentences for the narrowing to be safe- (< 2 hits, or fewer than half the sentences).-- That last case is the whole point: if the answer sentence is phrased without- the exact keyword (a bare figure, a synonym, a date), keyword hits will be- sparse and we keep the whole chunk so the answer is never dropped.-- Returns the narrowed plain text, or ``None`` to keep the chunk whole.- """- from rag.advanced_rag.harness.tools.search import _split_sentences-- sents = _split_sentences(content or "")- if len(sents) <= _NARROW_MIN_SENTENCES:- return None-- hit_idx = [i for i, s in enumerate(sents) if any(k in s.lower() for k in kw_list)]- # Refuse to narrow when keywords hit fewer than 2 sentences: with a single hit- # the narrowed snippet would keep just that sentence (+neighbours) and could- # drop most of the chunk, including the answer sentence. Two or more hits- # means the keywords genuinely relate to the passage, so narrowing is safe.- if len(hit_idx) < 2:- return None-- keep_idx: set[int] = set()- for i in hit_idx:- for j in range(max(0, i - _NARROW_NEIGHBOURS), min(len(sents), i + _NARROW_NEIGHBOURS + 1)):- keep_idx.add(j)- return " ".join(sents[i] for i in sorted(keep_idx)).strip()---def _evidence_md(tools, evidence_ids=None, keywords: str | None = None) -> str:- """Render cited evidence chunks with ``ID: n`` markers for the AutoRater.-- Prefers the chunks referenced by ``evidence_ids`` (the union of all claims'- evidence); falls back to a bounded prefix of the pool when no ids are- given. When ``keywords`` is provided each chunk is first narrowed to the- keyword-bearing sentences via ``_narrow_by_keywords`` (snippet evidence),- so the AutoRater sees compact relevant snippets instead of full passages.- Either way the output is capped (see module constants).- """- kb = getattr(tools, "kbinfos", None)- chunks = (kb or {}).get("chunks", [])- if not chunks:- return ""-- picked: list[tuple[int, dict]] = []- if evidence_ids:- seen = set()- for eid in evidence_ids:- try:- idx = int(eid)- except (TypeError, ValueError):- continue- if 0 <= idx < len(chunks) and idx not in seen:- picked.append((idx, chunks[idx]))- seen.add(idx)- if len(picked) >= _MAX_EVIDENCE_CHUNKS:- break- if not picked:- picked = [(i, c) for i, c in enumerate(chunks[:_MAX_EVIDENCE_CHUNKS])]-- # Narrow each kept chunk to its keyword-bearing sentences when keywords are- # available, preserving the original ``ID: n`` index (the answer cites these).- # If narrowing empties a chunk it is skipped; if everything narrows away we- # fall back to the original text so the AutoRater never sees nothing.- texts: list[tuple[int, str, str]] = []- # Normalise keywords: accept a comma/space-separated string OR an iterable of- # words. Narrowing operates on the raw text, independent of chunk structure.- if isinstance(keywords, str):- kw_list = [k.strip() for k in re.split(r"[,\s]+", keywords) if k.strip()]- else:- kw_list = [str(k) for k in (keywords or []) if str(k).strip()]- for idx, c in picked:- raw = c.get("content_with_weight") or c.get("text") or ""- title = c.get("docnm_kwd", "") or ""- if kw_list:- narrowed = _narrow_snippet_safe(raw, kw_list)- if narrowed:- texts.append((idx, title, narrowed))- continue- texts.append((idx, title, raw))-- blocks: list[str] = []- used = 0- for idx, title, text in texts:- text = text[:_MAX_CHUNK_CHARS]- if used + len(text) > _MAX_EVIDENCE_CHARS:- break- blocks.append(f"ID: {idx} | {title}\n{text}")- used += len(text) + 8- return "\n\n".join(blocks)---@in_phase("sufficiency")-async def llm_sufficiency_boost(- tools,- question: str,- verdict: SufficiencyVerdict,- evidence_ids=None,-) -> dict:- """Phase-2 LLM fallback, gated to the critical verdict band.-- Returns ``{}`` when no LLM boost is applicable (verdict already clear, or- the tools object lacks an LLM judge). Otherwise returns::-- {- "is_sufficient": bool, # LLM's call- "missing": [str, ...], # concrete gaps (Google missing pieces)- "followups": [ {question, query}, ... ], # next-round search queries- }- """- # Only boost ambiguous verdicts — clear SUFFICIENT/UNANSWERABLE need no LLM.- if verdict.status not in ("USEFUL_BUT_INCOMPLETE", "INSUFFICIENT", "CONFLICTING"):- return {}- if not hasattr(tools, "judge_sufficiency"):- return {}- chat_mdl = getattr(tools, "chat_mdl", None)- if chat_mdl is None:- return {}-- evidence_md = _evidence_md(tools, evidence_ids, keywords=_narrow_keywords(question))- if not evidence_md:- return {}-- _LOG.info("[LLM-sufficiency] verdict=%s → triggering LLM Sufficient Context AutoRater (evidence %d chars)", verdict.status, len(evidence_md))- try:- llm_result = await tools.judge_sufficiency(question, evidence_md) or {}- except Exception as exc:- _LOG.info("[LLM-sufficiency] judge_sufficiency failed: %s", exc)- return {}-- is_suff = bool(llm_result.get("is_sufficient") or llm_result.get("Sufficient Context"))- missing = [str(m) for m in (llm_result.get("missing_information") or llm_result.get("missing") or []) if str(m).strip()]- reasoning = (llm_result.get("reasoning") or "").strip()- # New Sufficient Context paper fields consumed by the decision ladder.- auto_confidence = _clamp(llm_result.get("confidence"))- contradictions = [str(c) for c in (llm_result.get("contradictions") or []) if str(c).strip()]- required_entities = [str(e) for e in (llm_result.get("required_entities") or []) if str(e).strip()]- coverage = llm_result.get("coverage") or {}- _LOG.info(- "[LLM-sufficiency] is_sufficient=%s confidence=%.2f contradictions=%s missing=%s reasoning=%s",- is_suff,- auto_confidence,- contradictions,- missing,- reasoning[:200],- )-- followups: list[dict] = []- if missing and hasattr(tools, "gen_followups"):- try:- followups = await tools.gen_followups(question, question, missing, evidence_md) or []- except Exception as exc:- _LOG.info("[LLM-sufficiency] gen_followups failed: %s", exc)-- if followups:- _LOG.info("[LLM-sufficiency] %d follow-up query(ie)s generated for next round: %s", len(followups), [q.get("question", "") for q in followups])-- return {- "is_sufficient": is_suff,- "confidence": auto_confidence,- "missing": missing,- "contradictions": contradictions,- "required_entities": required_entities,- "coverage": coverage,- "reasoning": reasoning,- "followups": followups,- }diff --git a/rag/advanced_rag/harness/orchestrator/sufficient_context.py b/rag/advanced_rag/harness/orchestrator/sufficient_context.pynew file mode 100644index 0000000000..70da66565a--- /dev/null+++ b/rag/advanced_rag/harness/orchestrator/sufficient_context.py@@ -0,0 +1,362 @@+"""Unified Sufficient Context Agent++Sufficient Context Agent performs ONE review pass that simultaneously+examines (1) the retrieved snippets, (2) the intermediate draft (each claim's+report), and (3) what is still missing (missing-pieces analysis). This replaces+the old two-call split — ``llm_sufficiency_boost`` (global verdict, no draft) ++``llm_grounded_verify`` (per-claim draft, no global verdict) — whose trigger+bands were complementary, so the three-part review rarely happened in one shot.++This module calls ONE LLM judge (backing prompt ``sca_select``) that sees the+question, the cited snippets (with ``ID: n`` markers), and each claim's draft,+and returns a unified verdict:++ {+ "is_sufficient": bool, # global sufficiency (Phase 5 stop signal)+ "confidence": float,+ "contradictions": [...],+ "reasoning": "...",+ "claims": [+ {"claim_id", "grounded", "ungrounded_assertions", "missing_information"}+ ]+ }++It also exposes adapter helpers ``to_boost`` / ``to_grounded`` so the caller can+feed the unified output into the EXISTING decision ladder (``boost``) and replan+(``grounded``) without changing their contracts.+"""++from __future__ import annotations++import json+import logging++from rag.advanced_rag.harness.stats import in_phase+from rag.prompts.generator import PROMPT_JINJA_ENV, gen_json+from rag.prompts.template import load_prompt++_LOG = logging.getLogger(__name__)++SCA_REVIEW = load_prompt("sca_select")++# The SCA now reviews ONLY the claims' reports (per-claim intermediate drafts),+# never the raw retrieved chunks. Rationale (Q2):+# - Each report is an "evidence-backed finding" produced by query_check from+# that claim's evidence, so it already distills the answer-bearing facts.+# - Sending raw chunks is what ballooned the prompt to 20k-100k chars and either+# degraded the LLM (empty claims) or timed it out; reports alone are tiny.+# - Groundedness is preserved because the report is generated FROM the cited+# evidence and query_check enforces grounded facts.+# Total cap for the rendered claims context (all per-claim reports + the overall+# intermediate draft) keeps the SCA prompt well inside the model window.+_SCA_CLAIMS_CONTEXT_MAX = 9000+# Max chars of each cited snippet's first line appended as an evidence anchor so+# the SCA can verify a draft against real retrieved text without a token blow-up.+_SCA_EVIDENCE_ANCHOR_CHARS = 300+++def _clamp(value, lo: float = 0.0, hi: float = 1.0) -> float:+ try:+ return max(lo, min(hi, float(value)))+ except (TypeError, ValueError):+ return 1.0+++def _coerce_dict(result) -> dict | None:+ """Coerce a ``gen_json`` response into a dict, tolerating model format drift.++ The reviewer model occasionally replies with a bare array (``[{...}]``) or a+ JSON string instead of a plain object. Previously any non-dict response was+ dropped, so the SCA produced NO signal on those rounds and replan/rewrite+ silently stopped. Recover the first dict when the response is a list of+ objects, and parse a string when it is valid JSON.+ """+ if isinstance(result, dict):+ return result+ if isinstance(result, list):+ for item in result:+ if isinstance(item, dict):+ return item+ return None+ if isinstance(result, str):+ try:+ parsed = json.loads(result)+ except Exception: # noqa: BLE001+ return None+ return _coerce_dict(parsed)+ return None+++def _render_reports(reports: list[tuple[str, str]]) -> str:+ """Render ``claim_id -> draft`` lines for the reviewer prompt."""+ if not reports:+ return "(no claim drafts)"+ return "\n".join(f"Claim {cid}: {rpt}" for cid, rpt in reports if rpt)+++def _render_claim_context(claims, question: str = "", kbinfos: dict | None = None) -> str:+ """Render per-claim context: each claim's report PLUS a brief evidence anchor.++ Q2 (reports only) + How6 (evidence anchor): the SCA reviews each claim's+ intermediate draft, and ALSO sees a short excerpt of the snippets THAT CLAIM+ cited, so it can verify the draft is grounded in real retrieved text and that+ any arithmetic operand actually appears in a snippet (rather than trusting the+ draft's numbers blindly). We send only the first line of each cited snippet+ (<= _SCA_EVIDENCE_ANCHOR_CHARS) so the prompt stays small (avoids the 20k-100k+ char blow-up that motivated Q2). Chunks whose id is not resolvable are skipped.+ """+ if not claims:+ return "(no claim drafts)"+ all_chunks = (kbinfos or {}).get("chunks") or []+ # ``evidence_ids`` are INDICES into ``kbinfos["chunks"]`` (see+ # decompose._evidence_ids), NOT the chunk's ``chunk_id`` hash. Key the+ # lookup by index so the cited-snippet anchors actually resolve — keying by+ # ``chunk_id`` made every anchor a miss (a hash string is never equal to an+ # index), silently disabling the "SCA reviews the retrieved snippets" guard.+ id2chunk = {str(i): c for i, c in enumerate(all_chunks)}+ blocks: list[str] = []+ used = 0+ for cid, rpt, eids in claims:+ if not rpt:+ continue+ block = f"Claim {cid} (draft):\n{rpt}"+ used += len(block) + 2+ # Evidence anchor: first line of each cited snippet (guards the draft).+ if eids:+ anchors: list[str] = []+ for eid in eids:+ ck = id2chunk.get(str(eid)) or id2chunk.get(eid)+ if not ck:+ continue+ txt = str(ck.get("chunk") or "").strip()+ if not txt:+ continue+ first = txt.split("\n")[0].strip()+ if len(first) > _SCA_EVIDENCE_ANCHOR_CHARS:+ first = first[:_SCA_EVIDENCE_ANCHOR_CHARS] + "…"+ anchors.append(first)+ if len(anchors) >= 2:+ break+ if anchors:+ block += "\n Evidence: " + " | ".join(anchors)+ used += len(anchors) * (_SCA_EVIDENCE_ANCHOR_CHARS + 4)+ blocks.append(block)+ if used >= _SCA_CLAIMS_CONTEXT_MAX:+ break+ return "\n\n".join(blocks) if blocks else "(no claim drafts)"+++def _render_overall_draft(claims, question: str = "") -> str:+ """Build the OVERALL intermediate draft ("rough draft") for the SCA.++ SCA reviews a single "rough draft" response for the WHOLE+ question, not just per-claim drafts — so it can judge whether the context lets+ the model answer the original question end-to-end, including cross-claim+ synthesis. We assemble a problem-level draft by concatenating each claim's+ report (which Q1 makes a concrete partial answer). The SCA then reviews this+ overall draft against the question to find cross-claim gaps that per-claim+ review would miss (e.g. two claims each grounded but the question needs them+ COMBINED into one derived answer).+ """+ if not claims:+ return "(no overall draft)"+ parts = [f"[Claim {cid}] {rpt.strip()}" for cid, rpt, _e in claims if rpt and rpt.strip()]+ if not parts:+ return "(no overall draft)"+ draft = "\n".join(parts)+ if len(draft) > _SCA_CLAIMS_CONTEXT_MAX:+ draft = draft[:_SCA_CLAIMS_CONTEXT_MAX]+ return draft+++@in_phase("sca")+async def sufficient_context_agent(+ tools,+ question: str,+ claims: list[tuple],+ evidence_ids=None,+) -> dict:+ """Unified SCA review: per-claim drafts + per-claim evidence + missing pieces.++ Parameters+ ----------+ tools : RAGTools+ Must expose ``chat_mdl`` and ``kbinfos``.+ claims : list[(claim_id, draft, evidence_ids)]+ Each claim's intermediate draft plus the evidence IDs THAT CLAIM cited+ (not the global union). Rendering per-claim evidence keeps the prompt+ small (~1-3 chunks per claim) so the LLM does not degrade, while the+ cited snippets still carry the answer-bearing facts.+ evidence_ids : list[str] | None+ Ignored for prompt rendering (kept for signature compatibility); per-claim+ evidence drives the review.++ Returns+ -------+ dict : the unified verdict above, or ``{}`` when unavailable (no chat model,+ no evidence, or a failure) — callers treat that as "no new signal".+ """+ if not claims:+ return {}+ chat_mdl = getattr(tools, "chat_mdl", None)+ if chat_mdl is None:+ return {}++ claims_context = _render_claim_context(claims, question, kbinfos=getattr(tools, "kbinfos", None))+ if not claims_context or claims_context == "(no claim drafts)":+ return {}+ overall_draft = _render_overall_draft(claims, question)++ prompt_text = PROMPT_JINJA_ENV.from_string(SCA_REVIEW).render(+ question=question,+ claims_context=claims_context,+ overall_draft=overall_draft,+ )+ _LOG.info(+ "[SCA] unified review of %d claim draft(s) (reports only, %d chars; overall draft %d chars)",+ len(claims),+ len(claims_context),+ len(overall_draft),+ )+ try:+ result = await gen_json(prompt_text, "Output:\n", chat_mdl)+ except Exception as exc: # noqa: BLE001+ _LOG.info("[SCA] unified review failed: %s", exc)+ return {}+ result = _coerce_dict(result)+ if not result:+ _LOG.info("[SCA] no usable response (type=%s); treating as no signal", type(result).__name__ if result is not None else "None")+ return {}++ claims_out: dict[str, dict] = {}+ for item in result.get("claims") or []:+ cid = str(item.get("claim_id") or "")+ if not cid:+ continue+ ungrounded = []+ for u in item.get("ungrounded_assertions") or []:+ if isinstance(u, dict):+ ungrounded.append(str(u.get("assertion") or u.get("reason") or ""))+ elif u:+ ungrounded.append(str(u))+ missing_info = []+ for m in item.get("missing_information") or []:+ if isinstance(m, dict):+ what = str(m.get("what") or "").strip()+ hint = str(m.get("search_hint") or "").strip()+ if what or hint:+ missing_info.append({"what": what, "search_hint": hint})+ elif m:+ missing_info.append({"what": str(m).strip(), "search_hint": ""})+ claims_out[cid] = {+ "grounded": bool(item.get("grounded")),+ "ungrounded": [a for a in ungrounded if a],+ "missing_information": missing_info,+ }++ # Parse the structured sub-query coverage set (Q-CARE): one entry per+ # step-by-step sub-question, each marked satisfied. Unsatisfied entries carry+ # the concrete missing_fact + search_hint — this is the precise "what is+ # missing / where to search next" signal the next round consumes, far more+ # targeted than per-claim free-text missing_information.+ sub_queries: list[dict] = []+ for sq in result.get("sub_queries") or []:+ if not isinstance(sq, dict):+ continue+ sq_text = str(sq.get("sub_query") or "").strip()+ if not sq_text:+ continue+ sq_out: dict = {+ "sub_query": sq_text,+ "satisfied": bool(sq.get("satisfied")),+ }+ if not sq_out["satisfied"]:+ sq_out["missing_fact"] = str(sq.get("missing_fact") or "").strip()+ sq_out["search_hint"] = str(sq.get("search_hint") or "").strip()+ sub_queries.append(sq_out)++ is_sufficient = bool(result.get("is_sufficient"))+ # Failsafe: when the SCA judges the context insufficient but returned an EMPTY+ # claims array (a known degradation on very long prompts), we must still give+ # the orchestrator something to re-search. Otherwise it abandons with+ # "I don't have enough information" despite having retrieved useful snippets.+ # Harvest any TOP-LEVEL missing_information the model may have put outside the+ # claims array, and fall back to the un-verified drafts themselves as gaps.+ if not is_sufficient and not claims_out:+ top_missing: list[dict] = []+ for m in result.get("missing_information") or []:+ if isinstance(m, dict):+ w = str(m.get("what") or "").strip()+ h = str(m.get("search_hint") or "").strip()+ if w or h:+ top_missing.append({"what": w, "search_hint": h})+ elif m:+ top_missing.append({"what": str(m).strip(), "search_hint": ""})+ if top_missing:+ claims_out["_global"] = {+ "grounded": False,+ "ungrounded": [],+ "missing_information": top_missing,+ }+ _LOG.info("[SCA] insufficient with empty claims; using %d top-level missing piece(s) as the re-search gap.", len(top_missing))+ elif claims:+ # No structured gap at all — derive a coarse one from the claim drafts+ # themselves so the loop still tries a re-search rather than abandoning.+ draft_gaps = [{"what": rpt.strip(), "search_hint": rpt.strip()} for _cid, rpt, _eids in claims if rpt and rpt.strip()]+ if draft_gaps:+ claims_out["_global"] = {+ "grounded": False,+ "ungrounded": [],+ "missing_information": draft_gaps,+ }+ _LOG.info("[SCA] insufficient with no structured gap; deriving %d coarse gap(s) from the drafts.", len(draft_gaps))++ return {+ "is_sufficient": is_sufficient,+ "confidence": _clamp(result.get("confidence")),+ "contradictions": [str(c) for c in (result.get("contradictions") or []) if str(c).strip()],+ "reasoning": str(result.get("reasoning") or "").strip(),+ "sub_queries": sub_queries,+ "claims": claims_out,+ }+++def to_boost(sca: dict, verdict, fallback_followups: list | None = None) -> dict:+ """Adapt the unified SCA output into the decision-ladder ``boost`` dict.++ Preserves the existing contract consumed by ``route_sufficiency_verdict``+ (``is_sufficient`` / ``confidence`` / ``missing`` / ``contradictions`` /+ ``feedback`` / ``followups``).+ """+ missing: list[str] = []+ for g in (sca.get("claims") or {}).values():+ for mi in g.get("missing_information") or []:+ w = str(mi.get("what") or "").strip()+ if w and w not in missing:+ missing.append(w)+ contradictions = list(sca.get("contradictions") or [])+ feedback = ""+ if missing:+ feedback = "missing: " + "; ".join(missing[:_FEEDBACK_MAX])+ return {+ "is_sufficient": bool(sca.get("is_sufficient")),+ "confidence": _clamp(sca.get("confidence")),+ "missing": missing,+ "contradictions": contradictions,+ "followups": fallback_followups or [],+ "feedback": feedback,+ # Structured sub-query coverage (Q-CARE): the precise "what is missing /+ # where to search next" signal consumed by _maybe_replan.+ "_sub_queries": list(sca.get("sub_queries") or []),+ }+++def to_grounded(sca: dict) -> dict:+ """Adapt the unified SCA output into the ``grounded`` dict consumed by+ ``_maybe_replan`` and the ungrounded-veto path:+ ``{claim_id: {grounded, ungrounded, missing_information}}``.+ """+ return dict(sca.get("claims") or {})+++_FEEDBACK_MAX = 4diff --git a/rag/advanced_rag/harness/pipeline.py b/rag/advanced_rag/harness/pipeline.pyindex c30afbe5ec..3da9f154c0 100644--- a/rag/advanced_rag/harness/pipeline.py+++ b/rag/advanced_rag/harness/pipeline.py@@ -31,11 +31,6 @@ class Pipeline:self.trace: list[dict] = []# Latest relevant-document set produced by a routing tool this run.self._routed_docs: list[str] = list(getattr(rag_tools, "doc_scope", None) or [])- # Per-claim round state, read by both normal completion and timeout handling.- self._active_phase: str | None = None- self._round_initial_routed_docs: tuple[str, ...] = tuple(self._routed_docs)- self._round_had_evidence = False- self._round_had_routed_scope_progress = Falseasync def execute(self, tool_name: str, **kwargs) -> ToolResult:"""Execute a registered tool by name."""@@ -59,19 +54,14 @@ class Pipeline:elapsed = time.time() - startself.trace.append({"tool": tool_name, "args": kwargs, "elapsed": elapsed, "success": True})result = self._normalize(raw)- if result.chunks:- self._round_had_evidence = True# A routing tool (e.g. dataset_navigation_search) yields the relevant# document IDs; remember them so the scope-consuming tools above can# inherit them on later turns.if result.docs:if hasattr(self.tools, "scoped_doc_ids"):- new_routed_docs = self.tools.scoped_doc_ids(list(result.docs)) or []+ self._routed_docs = self.tools.scoped_doc_ids(list(result.docs)) or []else:- new_routed_docs = list(result.docs)- if new_routed_docs and tuple(new_routed_docs) != self._round_initial_routed_docs:- self._round_had_routed_scope_progress = True- self._routed_docs = new_routed_docs+ self._routed_docs = list(result.docs)# Feed the shared citation pool: agent searches go through the# pipeline, so without this their evidence never reaches kbinfos and# the final answer has nothing to cite.diff --git a/rag/advanced_rag/harness/planner.py b/rag/advanced_rag/harness/planner.pydeleted file mode 100644index 126675aef7..0000000000--- a/rag/advanced_rag/harness/planner.py+++ /dev/null@@ -1,168 +0,0 @@-"""Planner node — question-type-aware claim decomposition."""--import json-import logging-import re--from common.token_utils import num_tokens_from_string-from rag.advanced_rag.agentic_rag_graph import _snip-from rag.advanced_rag.harness.config import get_mode-from rag.advanced_rag.harness.prompts.decompose_prompts import (- DECOMPOSE_COMPARATIVE,- DECOMPOSE_EXPLORATORY,- DECOMPOSE_FACTUAL,- DECOMPOSE_PROCEDURAL,-)-from rag.advanced_rag.harness.stats import in_phase-from rag.advanced_rag.harness.types import ClaimTarget, RouteDecision, WorkflowPlan--_LOG = logging.getLogger(__name__)---def _extract_json(text: str) -> dict:- text = re.sub(r"^.*</think>", "", text, flags=re.DOTALL).strip()- text = re.sub(r"```(?:json)?\s*|\s*```", "", text).strip()- try:- import json_repair-- return json_repair.loads(text)- except Exception:- try:- return json.loads(text)- except Exception:- _LOG.warning("planner: failed to parse LLM output: %s", text[:200])- return {}---@in_phase("planner")-async def planner_node(state: dict, tools) -> dict:- """Planner node — decompose question into claims based on question type."""- route: RouteDecision = state.get("route")- if not route:- _LOG.warning("planner: no route found, using defaults")- return _default_plan(state.get("question", ""))-- _LOG.info('[Planner] Working out how to research this %s question: "%s"', route.question_type, _snip(route.question))- if not route.requires_decomposition:- # Direct mode: single coarse claim- return _direct_plan(route.question)-- # Select decompose prompt by question type- prompt_map = {- "factual": DECOMPOSE_FACTUAL,- "comparative": DECOMPOSE_COMPARATIVE,- "procedural": DECOMPOSE_PROCEDURAL,- "analytical": DECOMPOSE_EXPLORATORY,- "exploratory": DECOMPOSE_EXPLORATORY,- }- decompose_prompt = prompt_map.get(route.question_type, DECOMPOSE_FACTUAL)-- mode = get_mode(route.thinking_mode)- max_claims = _get_max_claims(mode.label)- detail_level = _get_detail_level(mode.label)- retrieved = _format_seed_chunks(state.get("seed_chunks"), tools)-- try:- prompt = decompose_prompt.format(- question=route.question,- max_claims=max_claims,- detail_level=detail_level,- retrieved=retrieved,- )- system, user = prompt.split("Output format", 1)- system = system.strip()- user = "Output format" + user- # Replanning: the orchestrator sets ``feedback`` from the sufficiency- # verdict — the new plan must close those gaps instead of repeating- # the previous one.- feedback = (state.get("feedback") or "").strip()- if feedback:- system += (- "\n\nA previous research round already ran and left gaps. "- "Feedback from the sufficiency check — the new plan MUST address "- "these points with different, more targeted claims:\n" + feedback- )- msg = await tools._fit_messages(system, user)- ans = await tools.chat_mdl.async_chat(msg[0]["content"], msg[1:], {"temperature": 0.2})- if isinstance(ans, tuple):- ans = ans[0]- result = _extract_json(ans)- except Exception:- _LOG.exception("planner_node failed")- return _direct_plan(route.question)-- claims_raw = result.get("claims", [])- plan_type = {- "factual": "fact_decomposition",- "comparative": "comparative_decomposition",- "procedural": "procedural_decomposition",- }.get(route.question_type, "exploratory_decomposition")-- claims = []- for i, c in enumerate(claims_raw):- if isinstance(c, dict) and c.get("description"):- claims.append(- ClaimTarget(- claim_id=c.get("claim_id", f"c{i}"),- description=c["description"],- priority=c.get("priority", 0),- suggested_tools=c.get("suggested_tools", []),- )- )-- if not claims:- return _direct_plan(route.question)-- plan = WorkflowPlan(- plan_type=plan_type,- claims=claims,- max_iterations=mode.max_orchestrator_cycles,- )- _LOG.info("[Planner] Broke the question into %d research step(s): %s", len(plan.claims), "; ".join(f'"{c.description}"' for c in plan.claims))-- return {"plan": plan, "claims": plan.claims}---def _format_seed_chunks(seed_chunks, tools) -> str:- """Render preliminary-search chunks as grounding context for the planner."""- if not seed_chunks:- _LOG.info("[Planner] No preliminary passages — grounding the plan without seed chunks.")- return "(no preliminary results)"- try:- from rag.prompts.generator import kb_prompt-- blocks = kb_prompt({"chunks": seed_chunks, "doc_aggs": []}, tools.chat_mdl.max_length)- text = "\n".join(blocks).strip()- if not text:- return "(no preliminary results)"- _LOG.info(- "[Planner] Grounding the plan with %d preliminary passage(s) (~%d tokens of grounding context).",- len(seed_chunks),- num_tokens_from_string(text),- )- return text- except Exception:- _LOG.exception("planner: failed to format seed chunks")- return "(no preliminary results)"---def _direct_plan(question: str) -> dict:- """Single-claim plan for non-decomposed mode."""- plan = WorkflowPlan(- plan_type="direct",- claims=[ClaimTarget(claim_id="c0", description=question, priority=0)],- max_iterations=1,- )- return {"plan": plan, "claims": plan.claims}---def _default_plan(question: str) -> dict:- return _direct_plan(question)---def _get_max_claims(mode_label: str) -> int:- return {"low": 1, "medium": 3, "high": 5, "ultra": 8}.get(mode_label, 3)---def _get_detail_level(mode_label: str) -> str:- return {"low": "coarse", "medium": "normal", "high": "fine", "ultra": "extra_fine"}.get(mode_label, "normal")diff --git a/rag/advanced_rag/harness/prompts/decompose_prompts.py b/rag/advanced_rag/harness/prompts/decompose_prompts.pydeleted file mode 100644index a4dbda1cd8..0000000000--- a/rag/advanced_rag/harness/prompts/decompose_prompts.py+++ /dev/null@@ -1,165 +0,0 @@-"""Planner decompose prompts: one per question type.--Each prompt is grounded in a preliminary hybrid search: ``{retrieved}`` carries-the (keyword-narrowed) chunks retrieved for the user's question, so the-decomposition reflects what the corpus actually contains rather than guessing.-"""--DECOMPOSE_FACTUAL = """This is a factual question. List all atomic facts that need to be retrieved.-If there are multiple facts, list them one by one. If there is only one fact, output exactly one item.-Base the decomposition on BOTH the question and the preliminary retrieved context below.--Relevance rules (strict):-- Every claim MUST be directly necessary to answer the question. Do NOT add- background, biographical, or tangential facts about entities merely mentioned- in the corpus (e.g. an unrelated person's birthplace) unless the question- explicitly asks for them.-- Prefer fewer, atomic claims over many overlapping ones: merge facts about the- same entity/measure into one claim when they can be researched together.-- Each claim must be verifiable from the corpus; do not invent facts.-- OPEN QUERY (CRITICAL): Each claim's description must be an OPEN research target- (what/which/when/how many/what is X), NOT a pre-answered assertion. Never bake a- guessed value into the description. WRONG: "Dustin Brown's hometown is Ithaca".- RIGHT: "What is Dustin Brown's hometown?". The researcher verifies the OPEN target;- a pre-answered assertion biases it to merely confirm the guess and skips the real- answer. When a fact is genuinely unknown (e.g. a person's hometown), phrase the- claim as a question, never as "X is <guessed value>".--Question: {question}-Maximum number of claims: {max_claims}-Detail level: {detail_level}--Preliminary retrieved context for this question (use it to ground each claim in what the corpus actually contains; do not invent facts it cannot support):-{retrieved}--Output format (JSON):-{{- "claims": [- {{- "claim_id": "c1",- "description": "What year did Apple acquire Beats?",- "priority": 1- }}- ]-}}-"""---DECOMPOSE_COMPARATIVE = """This is a comparative question. It needs to be decomposed into:-1. Information about entity A for the comparison dimension.-2. Information about entity B for the comparison dimension.-3. Optional information that directly compares the two entities.-Base the decomposition on BOTH the question and the preliminary retrieved context below.--Relevance rules (strict):-- Every claim MUST be directly needed to answer the comparison. Do NOT add- tangential background about the entities (e.g. a third party's unrelated- details) unless the question explicitly asks for them.-- Merge overlapping facts about the same entity into one claim.-- OPEN QUERY (CRITICAL): Each claim's description must be an OPEN research target- (what/which/when/how many/what is X), NOT a pre-answered assertion. Never bake a- guessed value into the description. RIGHT: "What is the distance from Hangzhou to- Beijing?". WRONG: "The distance from Hangzhou to Beijing is 1,200 km". The- researcher verifies the OPEN target; an assertion only biases it to confirm the guess.--Question: {question}-Maximum number of claims: {max_claims}-Detail level: {detail_level}--Preliminary retrieved context for this question (use it to ground each claim in what the corpus actually contains; do not invent facts it cannot support):-{retrieved}--Output format (JSON):-{{- "claims": [- {{- "claim_id": "c1",- "description": "What is the distance from Hangzhou to Beijing?",- "priority": 1- }},- {{- "claim_id": "c2",- "description": "What is the distance from Shanghai to Beijing?",- "priority": 1- }},- {{- "claim_id": "c3",- "description": "Which city is closer to Beijing?",- "priority": 2- }}- ]-}}-"""---DECOMPOSE_PROCEDURAL = """This is a procedural question. Decompose it into the information needed for each step required to complete the operation.-Base the decomposition on BOTH the question and the preliminary retrieved context below.--Relevance rules (strict):-- Every claim MUST be information directly needed by one of the procedure's- steps. Skip any fact that is not required to carry out the operation.-- OPEN QUERY (CRITICAL): Each claim's description must be an OPEN research target- (what/which/when/how/what is X), NOT a pre-answered assertion. Never bake a- guessed value into the description. RIGHT: "What is the input format for step 2?".- WRONG: "The input format for step 2 is CSV".--Question: {question}-Maximum number of claims: {max_claims}-Detail level: {detail_level}--Preliminary retrieved context for this question (use it to ground each claim in what the corpus actually contains; do not invent facts it cannot support):-{retrieved}--Output format (JSON):-{{- "claims": [- {{- "claim_id": "c1",- "description": "What information is needed for the first step?",- "priority": 1- }},- {{- "claim_id": "c2",- "description": "What information is needed for the second step?",- "priority": 2- }}- ]-}}-"""---DECOMPOSE_EXPLORATORY = """This is an analytical or exploratory question. Decompose it into the main aspects or dimensions that need to be researched.-Base the decomposition on BOTH the question and the preliminary retrieved context below.--Relevance rules (strict):-- Every aspect/claim MUST be directly relevant to the question's core. Do NOT- list tangential or background aspects just because the corpus mentions them.-- Prefer fewer, well-scoped aspects over many overlapping ones.-- OPEN QUERY (CRITICAL): Each claim's description must be an OPEN research target- (what/which/when/how/what is X), NOT a pre-answered assertion. Never bake a- guessed value into the description. RIGHT: "What were the main factors behind X?".- WRONG: "The main factor behind X was Y".--Question: {question}-Maximum number of claims: {max_claims}-Detail level: {detail_level}--Preliminary retrieved context for this question (use it to ground each aspect in what the corpus actually contains; do not invent aspects it cannot support):-{retrieved}--Output format (JSON):-{{- "claims": [- {{- "claim_id": "c1",- "description": "What is the first aspect that needs to be researched?",- "priority": 1- }},- {{- "claim_id": "c2",- "description": "What is the second aspect that needs to be researched?",- "priority": 2- }}- ]-}}-"""diff --git a/rag/advanced_rag/harness/prompts/research_agent_prompt.py b/rag/advanced_rag/harness/prompts/research_agent_prompt.pydeleted file mode 100644index 74868e29c4..0000000000--- a/rag/advanced_rag/harness/prompts/research_agent_prompt.py+++ /dev/null@@ -1,106 +0,0 @@-"""Research Agent prompts.--``RESEARCH_AGENT_PROMPT`` — native tool-calling: the tool schemas are bound- onto the chat model via ``bind_tools``, so the- prompt only describes the task, not the tools.-``RESEARCH_AGENT_TEXT_PROMPT`` — fallback for models without native tool-calling:- the tools are described in-prompt and the model- emits ``<tool_call>`` JSON that the loop parses.-"""--RESEARCH_AGENT_PROMPT = """You are a research assistant. Investigate the given research task by calling the provided tools.--Research task: {claim_description}--Current phase: {phase}-Phase hint: {phase_hint}--Rules:-1. Go coarse-to-fine. First narrow the corpus with navigation tools (dataset_navigation_search,- then ontology_navigate / mindmap_navigate).-2. After a navigation tool returns passages, judge whether they already answer the task.- If they do, call generate_report immediately — do NOT search further.-3. Only call hybrid_search (or other broad search tools) when navigation did not yield- sufficient evidence.-4. Use think_tool to analyze results and plan the next step.-5. When you have gathered enough evidence, call generate_report with your findings- (report, is_verified, confidence, evidence_ids, gaps, discovered_claims).--ATTRIBUTE FIDELITY (CRITICAL):-Answer the EXACT attribute/relation the research task asks for. Do NOT substitute a-similar but different attribute, even when the substitution is semantically related and-tempting. For example, if the task asks for a person's HOMETOWN, do not report their-BIRTHPLACE (born in) as the answer — these are different facts. Likewise do not silently-swap "first" for "largest", "age at death" for "birth year", etc.-- In SEARCH queries you MAY use synonymous, translated, or corpus-specific terms (e.g.- "hometown", "place of residence", "ville natale") to find the evidence — retrieval- benefits from flexible wording. What matters is that the terms still TARGET the exact- attribute the task asks for; do not steer the search at a different fact.-- The REPORT must state the exact attribute the task asked for. Never present a different- attribute's value as the answer.-- If the exact attribute cannot be found in the evidence, mark the claim unverified and- list it in gaps — do NOT answer with a different attribute's value.--SOURCE ANCHORING (CRITICAL):-If the research task names a specific source (e.g. "In the Wikipedia article 'Demographics-of Paris', ..."), treat THAT named source as authoritative and retrieve the value from it.-Do NOT substitute another source's value (e.g. a statistical-agency estimate, a news site,-or a different webpage) as the answer. If the named source is found, its value wins. Only if-the named source cannot be located at all may you fall back, but then say so in the report-and set confidence accordingly.--You have at most {max_cycles} tool-calling rounds. Call exactly one tool per round,-and do not write a plain-text answer until you call generate_report.-"""---RESEARCH_AGENT_TEXT_PROMPT = """You are a research assistant. For the given research task, use the available tools to search for information.--Research task: {claim_description}--Current phase: {phase}-Phase hint: {phase_hint}--Available tools:-{tool_list}--Rules:-1. Go coarse-to-fine: first narrow the corpus with navigation tools, then search only if needed.-2. After a navigation tool returns passages, judge whether they already answer the task.- If they do, call generate_report immediately instead of searching further.-3. Only use hybrid_search / broad search tools when navigation did not yield sufficient evidence.-4. Use think_tool to analyze results after each step.-5. When you are confident enough to answer the research task, call generate_report.--ATTRIBUTE FIDELITY (CRITICAL):-Answer the EXACT attribute/relation the research task asks for. Do NOT substitute a similar-but different attribute (e.g. do not report HOMETOWN as BIRTHPLACE, do not swap "first" for-"largest", "age at death" for "birth year"). In SEARCH queries you MAY use synonymous,-translated, or corpus-specific terms (e.g. "hometown", "place of residence") as long as they-still TARGET the exact requested attribute — retrieval benefits from flexible wording. The-REPORT must state the exact attribute asked for and never present a different attribute's-value as the answer. If the exact attribute cannot be found in evidence, mark the claim-unverified and list it in gaps — do NOT answer with a different attribute's value.--SOURCE ANCHORING (CRITICAL):-If the research task names a specific source (e.g. "In the Wikipedia article 'Demographics of-Paris', ..."), treat THAT named source as authoritative and retrieve the value from it. Do-NOT substitute another source's value (statistical-agency estimate, news site, other webpage)-as the answer. If the named source is found, its value wins. Only if it cannot be located may-you fall back, and then say so in the report and lower confidence.--Tool call format: output exactly one JSON tool call per round:-<tool_call>{{"name": "tool_name", "arguments": {{"parameter_name": "value"}} }}</tool_call>--generate_report argument format:-{{- "report": "Research result report, factual and unformatted",- "is_verified": true/false,- "confidence": 0.0-1.0,- "evidence_ids": [0, 3],- "gaps": ["Information that was not found"],- "discovered_claims": ["New research directions discovered during research"]-}}--Maximum {max_cycles} rounds. Output one <tool_call> tag in each round and no other text.-"""diff --git a/rag/advanced_rag/harness/prompts/route_prompt.py b/rag/advanced_rag/harness/prompts/route_prompt.pydeleted file mode 100644index ea3af15b4e..0000000000--- a/rag/advanced_rag/harness/prompts/route_prompt.py+++ /dev/null@@ -1,19 +0,0 @@-"""Route node prompt: classify query type."""--ROUTE_PROMPT = """Analyze the following question and output a structured query analysis.--Question: {question}--Analyze it across these dimensions:-1. Question type: factual / comparative / analytical / procedural / exploratory / verification / summarization.-2. Whether it needs decomposition into atomic facts, meaning whether multiple independent pieces of information must be retrieved separately before answering: true/false.-3. Suggested knowledge compilation tool: null (none) / toc (document table of contents) / graph (knowledge graph) / wiki (compiled domain knowledge).--Output format (JSON):-{{- "question_type": "comparative",- "requires_decomposition": true,- "suggests_compilation": null,- "reasoning": "This is a comparative question, so it needs to be decomposed into two independent facts and one comparison relation."-}}-"""diff --git a/rag/advanced_rag/harness/route.py b/rag/advanced_rag/harness/route.pydeleted file mode 100644index fa5193fe05..0000000000--- a/rag/advanced_rag/harness/route.py+++ /dev/null@@ -1,97 +0,0 @@-"""Route node — query classification (one-time, no KB dependency)."""--import json-import logging-import re--from rag.advanced_rag.harness.types import RouteDecision-from rag.advanced_rag.harness.config import get_mode-from rag.advanced_rag.harness.prompts.route_prompt import ROUTE_PROMPT-from rag.advanced_rag.harness.stats import in_phase--_LOG = logging.getLogger(__name__)---def _extract_json(text: str) -> dict:- """Extract a JSON *object* from the LLM response.-- Handles markdown fences and think tags, then attempts ``json_repair``- followed by ``json.loads``. Robustness: the LLM can return a bare string,- a list, or a JSON primitive — anything that is not a ``dict`` is coerced to- ``{}`` (with a warning) so callers can safely use ``result.get(...)``. This- guards against the crash seen in check.log where ``route_node`` received a- string and ``result.get("question_type")`` raised AttributeError.- """- text = re.sub(r"^.*</think>", "", text, flags=re.DOTALL).strip()- text = re.sub(r"```(?:json)?\s*|\s*```", "", text).strip()- try:- import json_repair-- parsed = json_repair.loads(text)- except Exception:- try:- parsed = json.loads(text)- except Exception:- _LOG.warning("route: failed to parse LLM output: %s", text[:200])- return {}- if not isinstance(parsed, dict):- _LOG.warning("route: LLM returned non-object JSON (%s), coercing to {}: %s", type(parsed).__name__, str(parsed)[:200])- return {}- return parsed---@in_phase("route")-async def route_node(state: dict, tools) -> dict:- """Route node — analyze the question, produce RouteDecision."""- question = state.get("question", "")- if not question:- return _fallback_route(question)-- mode_label = getattr(tools, "thinking_mode", "medium")- mode = get_mode(mode_label)-- try:- system = ROUTE_PROMPT.format(question=question)- msg = await tools._fit_messages(system, question)- ans = await tools.chat_mdl.async_chat(msg[0]["content"], msg[1:], {"temperature": 0.1})- if isinstance(ans, tuple):- ans = ans[0]- result = _extract_json(ans)- except Exception:- _LOG.exception("route_node failed")- result = {}-- # Belt-and-suspenders: _extract_json already coerces to dict, but guard- # here too so a future code path can never crash on result.get().- if not isinstance(result, dict):- _LOG.warning("route_node: result not a dict (%s); falling back to defaults", type(result).__name__)- result = {}-- question_type = result.get("question_type", "factual")- requires_decomp = result.get("requires_decomposition", True)- suggests_comp = result.get("suggests_compilation")-- route = RouteDecision(- question=question,- thinking_mode=mode_label,- question_type=question_type,- requires_decomposition=mode.requires_decomposition and requires_decomp,- suggests_compilation=suggests_comp,- execution_strategy=mode.execution_strategy,- reasoning=result.get("reasoning", ""),- )-- return {"route": route}---def _fallback_route(question: str) -> dict:- route = RouteDecision(- question=question,- thinking_mode="medium",- question_type="factual",- requires_decomposition=False,- suggests_compilation=None,- execution_strategy="direct_search",- reasoning="fallback: empty question",- )- return {"route": route}diff --git a/rag/advanced_rag/harness/stats.py b/rag/advanced_rag/harness/stats.pyindex 12a5780b83..7e9ec8a923 100644--- a/rag/advanced_rag/harness/stats.py+++ b/rag/advanced_rag/harness/stats.py@@ -30,7 +30,6 @@ _ACTIVE_PHASES: ContextVar[tuple[str, ...]] = ContextVar("agentic_rag_active_pha_PHASE_ORDER = ["formalize","route",- "pre_search","planner","decompose","direct",diff --git a/rag/advanced_rag/harness/sufficiency.py b/rag/advanced_rag/harness/sufficiency.pydeleted file mode 100644index 81e8bbea2c..0000000000--- a/rag/advanced_rag/harness/sufficiency.py+++ /dev/null@@ -1,858 +0,0 @@-"""Sufficiency check — cross-check + fusion score + 5-way verdict."""--import logging--from rag.advanced_rag.harness.types import (- AgentResult,- ClaimCrossCheckResult,- SufficiencyVerdict,- ExecutionStrategy,-)-from rag.advanced_rag.harness.config import get_mode-from rag.advanced_rag.harness.sufficiency_ladder import (- ANSWER_WITH_CAVEAT,- RECONCILE,- UNANSWERABLE,-)--_LOG = logging.getLogger(__name__)--# Experimental switch: when True, the lexical NER grounded-fact check-# (``_grounded_hit`` below) runs in ``cross_check_claim``; when False it is-# disabled and groundedness is delegated entirely to the LLM draft review-# (``llm_grounded_verify`` in orchestrator/grounded_llm.py). Set to False to-# trial "LLM draft instead of NER".-_ENABLE_NER_GROUNDED = False---# ═══════════════════════════════════════════════════════════════-# Cross-check: code-only-# ═══════════════════════════════════════════════════════════════--import re---def extract_numbers(text: str) -> list[float]:- """Extract numeric values from text."""- return [float(m) for m in re.findall(r"\d+\.?\d*", text)]---def _filter_relevant_numbers(numbers: list[float]) -> list[float]:- """Drop numbers that carry no factual-claim signal.-- - Values in ``[0, 1]`` are overwhelmingly ratios / probabilities /- confidence scores the agent sprinkled into its prose, not facts to- verify against the evidence.- - Drop duplicates: "48 m" appearing three times should be checked once.- """- kept: list[float] = []- for n in numbers:- if 0.0 < n < 1.0:- continue- if n in kept:- continue- kept.append(n)- return kept---# ── Multilingual named-entity extraction ──────────────────────────────-# Cross-check previously only recognized English capitalized sequences, so-# Chinese/Japanese/Korean reports yielded zero entities and could never be-# verified. We now detect the report's language and route:-# - en/zh/de/fr/es/pt/ja: spaCy NER (models pre-loaded at build time; see-# pyproject.toml).-# - Any other language: langdetect returns the code; if no spaCy model is-# mapped the report still degrades gracefully to a no-op (never crashes).-# Language is detected with ``langdetect`` (Bayesian N-gram profiles, pure-# Python, 55+ languages — a product-grade alternative to hand-rolled-# heuristics), with a small Unicode-range heuristic as offline fallback.-# The spaCy pipeline is lazy-loaded once per process (singleton cache, like-# lightgraph) and degrades gracefully if a model is unavailable, so-# sufficiency never crashes on a missing model.--# RAGFlow language label / ISO code → spaCy model (mirrors lightgraph).-_LANG_TO_SPACY_MODEL = {- "en": "en_core_web_sm",- "english": "en_core_web_sm",- "zh": "zh_core_web_sm",- "chinese": "zh_core_web_sm",- "zh-cn": "zh_core_web_sm",- "de": "de_core_news_sm",- "german": "de_core_news_sm",- "fr": "fr_core_news_sm",- "french": "fr_core_news_sm",- "es": "es_core_news_sm",- "spanish": "es_core_news_sm",- "pt": "pt_core_news_sm",- "portuguese": "pt_core_news_sm",- "ja": "ja_core_news_sm",- "japanese": "ja_core_news_sm",-}--# spaCy NER labels that are not "evidence" (numbers, time, percentages).-_SPACY_SKIP_LABELS = {"ORDINAL", "CARDINAL", "DATE", "TIME", "PERCENT", "MONEY", "QUANTITY"}--_spacy_nlp_cache: dict = {}---def _is_cjk(text: str) -> bool:- """True if ``text`` contains Chinese/Japanese characters."""- return any("\u4e00" <= ch <= "\u9fff" for ch in text)---def _detect_language(text: str) -> str:- """Detect the report's language and return an ISO 639-1 code.-- Primary path uses ``langdetect`` (Bayesian N-gram profile classifier,- pure Python, ~55 languages). If langdetect is unavailable (or returns- nothing reliable for a very short / punctuation-only string), we fall- back to a small Unicode-range heuristic so the pipeline still works- offline without the extra dependency.- """- if not text:- return "en"- try:- from langdetect import detect, DetectorFactory-- # langdetect's global DetectorFactory is randomly seeded by default, so- # ambiguous/short inputs can flip between runs. Pin the seed once so- # detection (and therefore the spaCy NER model selected) is deterministic.- DetectorFactory.seed = 0- lang = detect(text[:500]) # long reports don't need the full body- if lang and lang != "unknown":- return lang- except Exception as exc: # langdetect not installed / detection error- _LOG.info(- "[multilingual-ner] langdetect unavailable/failed, falling back to Unicode heuristic: %s",- exc,- )- # Offline heuristic fallback.- if any("\u4e00" <= ch <= "\u9fff" for ch in text):- # Distinguish zh vs ja by presence of hiragana/katakana.- if any("\u3040" <= ch <= "\u30ff" for ch in text):- return "ja"- return "zh"- if any("\uac00" <= ch <= "\ud7af" for ch in text):- return "ko"- return "en"---def _resolve_spacy_model(language: str) -> str:- key = (language or "en").strip().lower()- return _LANG_TO_SPACY_MODEL.get(key, "en_core_web_sm")---def _spacy_ner_entities(text: str, language: str) -> list[str]:- """Extract named entities via spaCy NER, with per-process model caching.-- Returns [] on any failure (missing model, spacy import error) so the caller- can fall back to the regex path without crashing.- """- model_name = _resolve_spacy_model(language)- if model_name in _spacy_nlp_cache:- nlp = _spacy_nlp_cache[model_name]- else:- try:- import spacy-- nlp = spacy.load(model_name)- _spacy_nlp_cache[model_name] = nlp- except Exception as exc: # model not installed / spacy unavailable- _LOG.info("[multilingual-ner] spaCy model %s unavailable, falling back to regex: %s", model_name, exc)- return []- if nlp is None:- return []- try:- doc = nlp(text)- except Exception as exc:- _LOG.info("[multilingual-ner] spaCy inference failed for %s: %s", model_name, exc)- return []- seen: set[str] = set()- out: list[str] = []- for ent in doc.ents:- label = ent.label_ or ""- if label in _SPACY_SKIP_LABELS:- continue- name = ent.text.strip()- if not name or name in seen:- continue- seen.add(name)- out.append(name)- return out---def extract_named_entities(text: str) -> list[str]:- """Extract named entities via spaCy NER for any configured language.-- All languages (en/zh/de/fr/es/pt/ja) go through the spaCy NER pipeline —- no hand-rolled regexes or stopword lists. spaCy only returns recognized- named entities (PER/LOC/ORG/NORP/MISC/...), so connective tissue never- appears. Number/time/percentage labels are filtered (they are verified- separately via ``extract_numbers``). Returns [] if spaCy is unavailable.- """- if not text:- return []- lang = _detect_language(text)- return _spacy_ner_entities(text, lang)---def _detect_numeric_conflict(disclosed: list[str]) -> list[str]:- """Detect close-but-different figures among the agent's disclosed numbers.-- Extracts the leading number from each disclosure entry (e.g. "2,161,000 from- Wikipedia ..." → 2161000) and flags pairs that are numerically *close but not- equal* — the signature of a multi-source口径 conflict (same quantity, different- figure), which Q754 hit (population 2,161,000 vs 2,145,906 → 228 vs 227).- Numbers that are far apart are likely different quantities and not flagged.- """- import re as _re-- figures: list[tuple[float, str]] = []- for entry in disclosed:- m = _re.search(r"([\d][\d,]*(?:\.\d+)?)", entry)- if not m:- continue- try:- fig = float(m.group(1).replace(",", ""))- except ValueError:- continue- figures.append((fig, entry[:80]))-- conflicts: list[str] = []- for i in range(len(figures)):- for j in range(i + 1, len(figures)):- a, b = figures[i][0], figures[j][0]- if a <= 0 or b <= 0:- continue- ratio = max(a, b) / min(a, b)- # Close (within 30%) but not equal → same quantity, conflicting value.- if 1 < ratio <= 1.3:- conflicts.append(f"{figures[i][1]} vs {figures[j][1]}")- return conflicts---# Attribute/relationship descriptors: content words in a grounded fact that are-# neither named entities nor numbers, e.g. "hometown", "captain", "born", "age".-# Their presence in the evidence is what makes a grounded assertion authoritative-# (the fact's *relationship*, not just its entities, must be supported).-_PREDICATE_STOP = {- "the",- "a",- "an",- "of",- "in",- "on",- "at",- "to",- "for",- "and",- "or",- "is",- "was",- "are",- "were",- "by",- "with",- "from",- "his",- "her",- "their",- "its",- "that",- "this",- "he",- "she",- "they",- "it",- "also",- "as",- "when",- "who",- "what",- "which",- "there",- "have",- "has",- "had",- "be",- "been",- "being",-}---def _predicate_terms(fact: str, excluded: list[str]) -> list[str]:- """Return the predicate/attribute descriptors of ``fact`` (lowercased, ≤4 tokens).-- These are the content words that carry the relationship being asserted,- excluding proper entities (in ``excluded``) and common stop words. For- "hometown is Ithaca" → ["hometown"]; for "Dustin Brown captain of Los Angeles- Kings" → ["captain"]. Used to verify the fact's relationship is evidenced.- """- excluded_lower = {e.lower() for e in excluded}- tokens = re.findall(r"[A-Za-z][A-Za-z'-]{1,19}", fact.lower())- out: list[str] = []- for tok in tokens:- if tok in excluded_lower or tok in _PREDICATE_STOP or tok in out:- continue- out.append(tok)- return out[:4]---def _entity_present(ent: str, chunk_texts: list[str]) -> bool:- """Whether ``ent`` (lowercased) appears in any of the evidence chunk texts.-- CJK entities have no word boundaries (every Han char is ``\\w``) and are- commonly followed by function words ("的/是"), so they use a substring- match. Non-CJK entities use a bounded word/phrase match (Ann must not match- Annual). Mirrors the matching inside ``cross_check_claim``.- """- if _is_cjk(ent):- return any(ent.lower() in t for t in chunk_texts)- return any(re.search(rf"(?<![\w]){re.escape(ent.lower())}(?![\w])", t) for t in chunk_texts)---def required_entity_gaps(- question: str,- claims: list,- all_chunks: dict,-) -> dict[str, list[str]]:- """Find per-claim "required entities" missing from the evidence.-- Aligns with the Sufficient Context paper (arXiv 2411.06037): sufficiency is- anchored on *what the question needs*, not on what the agent *claims* in its- report. Extracting entities from the report is vulnerable to the agent- back-filling facts from prior knowledge (check1.log Q2: the agent injected- Tyson Fury's "age 35" from memory while the corpus had no Fury data, so the- report-based cross-check passed on Mike Tyson's numbers that padded the- score). Instead we extract entities from the *claim description + question*- (what the answer actually requires) and check each against the evidence.-- Returns {claim_id: [missing entities]}. A claim whose required entity is- absent from every cited/evidence chunk is flagged — under AND semantics one- missing required entity means that part of the question is unsupported.- """- # Union of all evidence chunk texts (independent of any claim's citation).- chunk_texts: list[str] = []- for chunk in (all_chunks or {}).values():- text = chunk.get("content_with_weight") or chunk.get("text") or ""- if text:- chunk_texts.append(text.lower())-- # Pre-extract question entities once (shared across claims) + seed per-claim.- q_entities = extract_named_entities(question or "")- gaps: dict[str, list[str]] = {}- for claim in claims or []:- cid = getattr(claim, "claim_id", None)- desc = getattr(claim, "description", None) or ""- if not cid or not desc:- continue- # Required entities = those in the claim description, plus any- # question-level entities also mentioned by the claim (so a composite- # question like "Mike Tyson AND Tyson Fury" keeps each named entity- # individually accountable).- desc_entities = extract_named_entities(desc)- desc_lower = {e.lower() for e in desc_entities}- required = [e for e in desc_entities] + [e for e in q_entities if e.lower() in desc_lower]- missing = [e for e in required if not _entity_present(e, chunk_texts)]- if missing:- gaps[cid] = missing- return gaps---def cross_check_claim(agent_result: AgentResult, all_chunks: dict) -> ClaimCrossCheckResult:- """Code-level cross-check: number matching + entity presence."""- report = agent_result.report- claimed = agent_result.is_verified- _LOG.info(- "[Cross-check] claim=%s entering — self_verified=%s, report_len=%d, evidence_ids=%s",- agent_result.claim_id,- claimed,- len(report or ""),- agent_result.evidence_ids,- )-- if not claimed:- _LOG.info("[Cross-check] claim=%s → FAILED (agent self-reported as unverified, score=0.0)", agent_result.claim_id)- return ClaimCrossCheckResult(- claim_id=agent_result.claim_id,- cross_check_passed=False,- cross_check_score=0.0,- mismatches=["agent self-reported as unverified"],- )-- raw_numbers = extract_numbers(report)- numbers = _filter_relevant_numbers(raw_numbers)- entities = extract_named_entities(report)- _LOG.info(- "[Cross-check] claim=%s extracted %d raw number(s) → %d relevant (noise filtered/deduped): %s, %d entity(ies)=%s from report",- agent_result.claim_id,- len(raw_numbers),- len(numbers),- numbers[:8],- len(entities),- entities[:8],- )-- # Existence check across the *union* of evidence chunks, not per-chunk.- # Verifying "does this fact appear anywhere in the cited evidence" is the- # right semantic — a number/entity supported by one chunk is verified.- # The old per-chunk loop demanded a fact appear in EVERY evidence chunk,- # so a fact confirmed in chunk A was recorded as a mismatch in chunks B..N- # and the score was diluted to near-zero even when the answer was correct- # (see benchmark/3.log: c4's 48/157/27/89 all matched chunk 0 yet scored- # 0.286; c5's 21/27/48/89 matched chunk 3 yet scored 0.168).- chunk_texts: list[str] = []- missing_ids: list[str] = []- for eid in agent_result.evidence_ids or []:- chunk = all_chunks.get(eid)- if not chunk:- missing_ids.append(str(eid))- continue- chunk_texts.append((chunk.get("content_with_weight") or chunk.get("text") or "").lower())- if missing_ids:- _LOG.info("[Cross-check] claim=%s %d evidence_id(s) MISSING from pool (index drift?): %s", agent_result.claim_id, len(missing_ids), missing_ids[:5])-- _LOG.info(- "[Cross-check] claim=%s %d evidence chunk(s) gathered (%d expected); verifying %d fact(s) against their union",- agent_result.claim_id,- len(chunk_texts),- len(agent_result.evidence_ids or []),- len(numbers) + len(entities),- )-- # ── Grounded-fact verification (answer↔evidence consistency) ──- # The agent explicitly lists the KEY assertions it claims are evidence-- # backed (schema field `grounded`). This is the ground truth to verify:- # the ratio-based number/entity check below can be gamed by padding facts- # (check1.log Q203: "hometown=Ithaca" scored 9/12=0.75 because Dustin Brown- # / Kings / Stanley Cup matched, while the answer-critical hometown fact- # was prior-injected and absent from evidence; Q665 likewise: "first solo- # album=1970" passed on Beatles padding). If any *grounded* fact is absent- # from the evidence, the claim is a prior-knowledge injection → hard-fail.- # A grounded fact is a natural-language assertion (e.g. "Dustin Brown- # captain of Los Angeles Kings"), which will never match evidence verbatim.- # Match on its *key content tokens* (entities + numbers) instead: the fact- # is considered grounded when at least half its key tokens appear in the- # evidence. "hometown is Ithaca" → key token [Ithaca] → absent → violation.- # "Dustin Brown captain of Los Angeles Kings" → [Dustin Brown, Los Angeles- # Kings] → both present → grounded. This is lenient against wording noise- # yet strict on the answer-critical named facts that prior-injection adds.- def _grounded_hit(fact: str) -> bool:- f = fact.lower()- if _is_cjk(f):- return any(f in t for t in chunk_texts)- # Key tokens = named entities + numbers mentioned in the fact.- key_tokens = [tok for tok in extract_named_entities(fact)]- key_tokens += [f"{int(n)}" for n in _filter_relevant_numbers(extract_numbers(fact))]- if not key_tokens:- # No extractable key content — fall back to whole-phrase match.- return any(re.search(rf"(?<![\w]){re.escape(f)}(?![\w])", t) for t in chunk_texts)- # Predicate/attribute co-occurrence: partial token overlap alone is not- # authoritative — the fact's *relationship* must also be evidenced. For- # "hometown is Ithaca", matching only "Ithaca" (present as a BIRTHPLACE)- # must NOT ground the claim. Extract the attribute descriptors (non-entity- # content words that carry the relation: "hometown", "captain of", "born")- # and require one of them to appear in the evidence. This downgrades a- # pure entity-overlap hit to non-authoritative when the predicate is absent.- predicate_tokens = _predicate_terms(fact, key_tokens)- entity_hits = sum(1 for tok in key_tokens if _entity_present(tok, chunk_texts))- if predicate_tokens:- pred_ok = any(re.search(rf"(?<![\w]){re.escape(pt)}(?![\w])", t) for pt in predicate_tokens for t in chunk_texts)- if not pred_ok:- return False # relationship not evidenced → prior-injection risk- return entity_hits / len(key_tokens) >= 0.5-- grounded_facts = [str(g) for g in (agent_result.grounded or []) if str(g).strip()]- if _ENABLE_NER_GROUNDED and grounded_facts:- ungounded = [g for g in grounded_facts if not _grounded_hit(g)]- if ungounded:- _LOG.warning(- "[Cross-check] claim=%s → GROUNDED-FACT VIOLATION: %d key fact(s) agent marked as evidence-backed are ABSENT from evidence: %s — hard-failing claim",- agent_result.claim_id,- len(ungounded),- ungounded[:6],- )- return ClaimCrossCheckResult(- claim_id=agent_result.claim_id,- cross_check_passed=False,- cross_check_score=0.0,- mismatches=[f"grounded fact not in evidence: {g}" for g in ungounded],- )-- # ── Numeric multi-source conflict detection (Q754: 225 vs 228口径) ──- # The agent may have used one figure while the evidence holds another value- # for the same quantity (e.g. Paris population from INSEE vs Wikipedia vs a- # news estimate). A ratio/mean cross-check passes as long as the used number- # matches somewhere, hiding the fact that a DIFFERENT authoritative figure- # exists. When the report discloses several distinct figures that are close- # but not equal (the classic "multiple sources, pick one" trap), surface them- # as conflicts and cap the claim below the pass floor so the caller does not- # blindly accept one口径.- disclosed = [str(n) for n in (agent_result.numbers or []) if str(n).strip()]- if disclosed:- conflict = _detect_numeric_conflict(disclosed)- if conflict:- _LOG.warning(- "[Cross-check] claim=%s → NUMERIC CONFLICT: multiple close-but-different figures for the same quantity: %s — capping below pass",- agent_result.claim_id,- conflict,- )- return ClaimCrossCheckResult(- claim_id=agent_result.claim_id,- cross_check_passed=False,- cross_check_score=0.0,- mismatches=[f"numeric source conflict: {c}" for c in conflict],- )-- def _anywhere(needle: str) -> bool:- return any(re.search(needle, t) for t in chunk_texts)-- matches: list[str] = []- mismatches: list[str] = []- for num in numbers:- # Numbers are extracted as floats ("1976" -> 1976.0) while chunk text- # spells them "1976" — match both raw and integral forms. Bounded so a- # number does not match inside a longer digit run (1976 vs 19760).- forms = {str(num), str(int(num))} if float(num).is_integer() else {str(num)}- found = any(_anywhere(rf"(?<![\w]){re.escape(f)}(?![\w])") for f in forms)- if found:- matches.append(f"number {num} found in evidence")- else:- mismatches.append(f"number {num} not found in any evidence chunk")-- for ent in entities:- if _is_cjk(ent):- # CJK entities have no word boundaries (every Han char is \w), and- # they are commonly followed by function words ("的/是"). Use a- # substring match on the lowercased text instead of a \b-like regex,- # which would fail on "清真寺的..." because "的" is \w.- found = any(ent.lower() in t for t in chunk_texts)- else:- # Bounded word/phrase match: Ann must not match Annual.- found = _anywhere(rf"(?<![\w]){re.escape(ent.lower())}(?![\w])")- if found:- matches.append(f"entity '{ent}' found in evidence")- else:- mismatches.append(f"entity '{ent}' not found in any evidence chunk")-- total = len(matches) + len(mismatches)- if total == 0:- # No evidence was actually examined — fail rather than pass neutrally:- # a claim with zero evidence IDs cannot be cross-checked at all.- if not agent_result.evidence_ids:- _LOG.info("[Cross-check] claim=%s → FAILED (no evidence ids, score=0.0)", agent_result.claim_id)- return ClaimCrossCheckResult(- claim_id=agent_result.claim_id,- cross_check_passed=False,- cross_check_score=0.0,- mismatches=["no evidence"],- )- # Evidence IDs exist but nothing extractable to verify against (e.g.- # Chinese reports yield no capitalized entities and no digits). We- # cannot confirm OR falsify — score it neutral (0.5) and do NOT mark it- # passed. The old "pass neutrally with score=1.0" treated unverifiable- # claims as fully verified, which let any entity-free, digit-free report- # sail through as SUFFICIENT.- _LOG.info(- "[Cross-check] claim=%s → NEUTRAL (evidence ids exist but nothing extractable to verify, score=0.5, not passed)",- agent_result.claim_id,- )- return ClaimCrossCheckResult(- claim_id=agent_result.claim_id,- cross_check_passed=False,- cross_check_score=0.5,- mismatches=["nothing extractable to cross-check"],- )- cross_score = len(matches) / total- # Pass when at least half the checked facts (numbers + entities) are- # confirmed in the evidence. The old ``mismatch < match*0.5`` required a- # 2/3 match rate and treated a single spurious number as fatal, which- # systematically failed otherwise-correct claims (see benchmark/2.log).- cross_passed = cross_score >= 0.5- _LOG.info(- "[Cross-check] claim=%s → %s (%d/%d matched, score=%.3f, pass>=0.50). matches=%s mismatches=%s",- agent_result.claim_id,- "PASSED" if cross_passed else "FAILED",- len(matches),- total,- cross_score,- matches[:5],- mismatches[:5],- )-- return ClaimCrossCheckResult(- claim_id=agent_result.claim_id,- cross_check_passed=cross_passed,- cross_check_score=cross_score,- evidence_matches=matches,- mismatches=mismatches,- )---# ═══════════════════════════════════════════════════════════════-# Fusion score-# ═══════════════════════════════════════════════════════════════---def compute_fusion_score(- agent_results: list[AgentResult],- cross_check_results: list[ClaimCrossCheckResult],- mode: ExecutionStrategy,- question: str = "",- claims: list | None = None,- all_chunks: dict | None = None,-) -> SufficiencyVerdict:- """Extract sufficiency *signals* — hard vetoes, agent confidence, conflicts.-- This is no longer a weighted fusion. The LLM AutoRater (invoked by the- orchestrator via ``llm_sufficiency_boost``) is the primary sufficiency- judge; this function only produces the *code-level* inputs the decision- ladder consumes:- - hard_violations: claims with a proven evidence gap (required entity- missing / grounded absent / numeric conflict) that must veto "good- enough" even if the AutoRater says sufficient;- - agent_confidence: mean self-confidence over the trusted subset;- - has_conflicts / missing_claims: surfaced for the ladder / caveat.-- ``question`` / ``claims`` / ``all_chunks`` are optional; when provided they- drive the *required-entity* AND-semantics veto from the Sufficient Context- paper (anchored on what the question needs, not what the agent claims), plus- suppression of self-confidence for claims whose required entities are- missing from the evidence.- """- # ── Required-entity gaps (Sufficient Context paper, AND semantics) ──- # Anchored on *what the question needs* rather than what the agent claims- # (see required_entity_gaps). Any claim whose required entity is absent from- # the evidence flags a localized gap: under AND semantics the whole verdict- # must not be SUFFICIENT, and the agent's self-confidence for that claim is- # suppressed (selective generation: confidence must not override missing- # evidence). Computed once here so it drives both Signal A and the veto.- required_gaps: dict[str, list[str]] = {}- if question or claims:- required_gaps = required_entity_gaps(question, claims, all_chunks)- if required_gaps:- _LOG.info(- "[Sufficiency] Required-entity gaps (AND semantics): %s",- {k: v for k, v in required_gaps.items()},- )- gapped_ids = set(required_gaps.keys())-- # Signal A: agent self-assessed confidence (continuous, per design doc).- # Only self-verified claims count toward "agent is confident" — an- # unverified claim's confidence is not trustworthy. This replaces the old- # boolean pass-rate (verified_count / n) which inflated the score to 1.0- # whenever the agent merely said "verified" (benchmark/2.log showed- # confidence 0.5-0.7 being reported as agent_score=1.0).- #- # A claim whose REQUIRED entity is missing from the evidence must not lend- # its self-confidence to Signal A — the agent is confident about a fact the- # corpus cannot support (prior-knowledge back-fill, see check1.log Q2). Its- # confidence is zeroed for the mean, so self-assessed confidence can never- # mask an evidence gap.- verified = [r for r in agent_results if r.is_verified and r.claim_id not in gapped_ids]- suppressed = [r.claim_id for r in agent_results if r.is_verified and r.claim_id in gapped_ids]- verified_count = len(verified)- agent_score = sum(r.confidence for r in verified) / verified_count if verified_count else 0.0- if suppressed:- _LOG.info(- "[Sufficiency] Signal A: suppressed %d self-verified claim(s) with missing required entities (confidence zeroed): %s",- len(suppressed),- suppressed,- )- _LOG.info(- "[Sufficiency] Signal A (self): %d/%d claims self-verified (and evidence-backed), mean confidence → agent_score=%.3f (raw confidence values=%s)",- verified_count,- len(agent_results),- agent_score,- [round(r.confidence, 3) for r in agent_results],- )-- # Signal B: cross-check score (continuous match rate), per design doc.- # Uses each claim's actual cross_check_score rather than a boolean- # pass/fail count, so partial-but-real evidence (e.g. 0.6) contributes- # proportionally instead of being zeroed.- #- # Unrelated-claim pollution (see 6.log/7.log): the planner occasionally- # invents a claim with no bearing on the question (e.g. "Suharto was born- # in Kemusuk" while the question asks about a mosque's heights). The agent- # finds no evidence for it (cross_check_score≈0), fails the cross-check,- # AND self-reports it as unverified. Such a claim drags Signal B down and- # pushes an otherwise sufficient fusion into the critical band that- # needlessly triggers the LLM fallback. We exclude "unanswerable + agent- # self-unverified" claims from the mean — they neither help nor should- # punish the verdict. We key on the agent's is_verified flag (not the- # confidence threshold) because a claim the agent itself calls unverified- # is the strongest unrelated/ungrounded signal (7.log's c2 reported- # confidence 0.35 but self-flagged unverified, which a confidence<0.2- # threshold would have missed).- cross_results = list(cross_check_results)- noise_threshold = 0.2- agent_verified = {r.claim_id: r.is_verified for r in agent_results}- noise_ids = [r.claim_id for r in cross_results if r.cross_check_score < noise_threshold and not r.cross_check_passed and not agent_verified.get(r.claim_id, False)]- kept = [r for r in cross_results if r.claim_id not in noise_ids]- if noise_ids and kept:- _LOG.info(- "[Sufficiency] Excluding %d unrelated/unverifiable claim(s) from Signal B: %s (cross<%.2f AND agent self-unverified)",- len(noise_ids),- noise_ids,- noise_threshold,- )- cross_results = kept-- # ── Hard-veto floor: a *localized* evidence gap must veto "good enough" ──- # Multi-claim questions (e.g. "Mike Tyson AND Tyson Fury") can average a- # genuinely weak claim up to the sufficient band. Q2 (check.log): c4/c5's- # "Tyson Fury" / "Usyk" entities matched 0 chunks, yet the 5-claim mean- # cross_check_score=0.822 crossed the threshold and produced an answer that- # could not back the Fury half. These claims become ``hard_violations`` that- # force the decision ladder to a caveated answer even if the LLM AutoRater- # says sufficient (code-proven evidence gap beats "roughly good enough").- min_cross_floor = getattr(mode, "fusion_min_cross", 0.5) or 0.5- self_verified_ids = {r.claim_id for r in agent_results if r.is_verified}- weak = [r.claim_id for r in cross_results if r.claim_id in self_verified_ids and r.cross_check_score < min_cross_floor]- # AND-semantics required-entity veto: a claim missing a required entity- # (even if its report-based cross-check passed — the numbers matched on- # unrelated padding) must veto. This catches the Q2 case that min-cross- # alone missed: c2/c3 self-verified with score 0.83/0.80 (padded by Mike- # Tyson's digits) yet Tyson Fury's entities were absent from every chunk.- weak += [cid for cid in gapped_ids if cid not in weak]- if weak:- _LOG.info(- "[Sufficiency] Hard-veto: %d self-verified claim(s) below floor %.2f OR missing a required entity: %s",- len(weak),- min_cross_floor,- weak,- )-- # Conflict detection — based on the kept (non-noisy) claims so an- # unrelated claim's mismatches don't manufacture a conflict.- has_conflicts = any(len(r.mismatches) > 0 for r in cross_results)- _LOG.info("[Sufficiency] Conflict detection: has_conflicts=%s", has_conflicts)-- # Cross-check status (code-only view, no AutoRater). This is a *preliminary*- # label used by the orchestrator to decide whether to call the LLM AutoRater- # (medium triggers it only in the borderline band); the final decision comes- # from the decision ladder with the AutoRater's verdict.- if not any(r.cross_check_passed for r in cross_results):- status = "UNANSWERABLE"- elif has_conflicts:- status = "CONFLICTING"- elif weak:- status = "INSUFFICIENT"- elif agent_score >= mode.sufficiency_threshold:- status = "SUFFICIENT"- else:- status = "USEFUL_BUT_INCOMPLETE"- _LOG.info(- "[Sufficiency] Code-level status=%s (agent_conf=%.3f, conflicts=%s, hard_veto=%s)",- status,- agent_score,- has_conflicts,- bool(weak),- )-- missing = [r.claim_id for r in cross_results if not r.cross_check_passed]- # Excluded unrelated claims are surfaced (not silently dropped) so the- # caller knows the planner invented unanswerable claims.- missing += noise_ids- # Thin-evidence claims that vetoed are surfaced too, so the caller sees- # exactly which part of the question still lacks support.- missing += [c for c in weak if c not in missing]-- return SufficiencyVerdict(- status=status,- # Reference score for logging/monitoring only — the final decision comes- # from the decision ladder, not from this scalar.- score=agent_score,- claim_assessments=[{"claim_id": r.claim_id, "is_verified": r.cross_check_passed, "score": r.cross_check_score, "mismatches": r.mismatches} for r in cross_results],- has_conflicts=has_conflicts,- missing_claims=missing,- feedback=_build_feedback(missing, cross_results),- hard_violations=weak,- agent_confidence=agent_score,- )---# ═══════════════════════════════════════════════════════════════-# Helpers-# ═══════════════════════════════════════════════════════════════---def _build_feedback(missing: list[str], results: list[ClaimCrossCheckResult]) -> str:- if not missing:- return "all claims verified"- hints = []- for r in results:- if not r.cross_check_passed:- hints.append(f"claim {r.claim_id}: {len(r.mismatches)} mismatch(es)")- return "missing: " + "; ".join(hints)---def route_sufficiency_verdict(- verdict: SufficiencyVerdict,- mode_label: str,- cycle: int,- max_cycles: int,- auto: dict | None = None,-) -> tuple:- """Decision-ladder routing → (action, should_continue, caveat).-- The LLM AutoRater (``auto``) is the primary sufficiency judge; the agent- confidence (``verdict.agent_confidence``) is the risk gate. ``auto`` is the- dict returned by ``llm_sufficiency_boost`` (``is_sufficient`` /- ``confidence`` / ``missing`` / ``contradictions``). When absent (e.g. the- medium mode did not trigger the AutoRater, or the tools lack an LLM judge),- we fall back to a code-only decision so the loop still terminates sensibly.- """- mode = get_mode(mode_label)- hard_violations = getattr(verdict, "hard_violations", []) or []- agent_confidence = getattr(verdict, "agent_confidence", getattr(verdict, "score", 0.0))-- # AutoRater signals, with sane defaults when it was not invoked.- auto_sufficient = bool(auto.get("is_sufficient")) if auto else (verdict.status == "SUFFICIENT")- auto_confidence = float(auto.get("confidence") or 1.0) if auto else 1.0- missing = list(auto.get("missing") or []) if auto else verdict.missing_claims- contradictions = list(auto.get("contradictions") or []) if auto else ([verdict.feedback] if verdict.has_conflicts else [])-- from rag.advanced_rag.harness.sufficiency_ladder import sufficiency_ladder-- out = sufficiency_ladder(- auto_sufficient=auto_sufficient,- auto_confidence=auto_confidence,- missing=missing,- contradictions=contradictions,- agent_confidence=agent_confidence,- c_high=mode.c_high,- c_low=mode.c_low,- llm_floor=mode.llm_floor,- allows_reconcile=mode.allows_reconcile,- cycle=cycle,- max_cycles=max_cycles,- hard_violations=hard_violations,- )- _LOG.info(- "[Sufficiency ladder] auto_sufficient=%s auto_conf=%.2f agent_conf=%.2f hard_violations=%s → %s",- auto_sufficient,- auto_confidence,- agent_confidence,- hard_violations,- out.action,- )-- # Map ladder action onto orchestrator actions.- action = out.action- if action == ANSWER_WITH_CAVEAT:- action = "ANSWER_PARTIAL"- elif action == RECONCILE:- # medium has no reconcile loop → degrade to CONTINUE (keep searching).- if mode.allows_reconcile:- return ("CONTINUE", True, out.caveat)- return ("CONTINUE", True, out.caveat)- elif action == UNANSWERABLE:- if mode.fallback_to_direct_llm:- action = "FALLBACK_LLM"- else:- action = "ABSTAIN"- return (action, False, out.caveat)- # ANSWER / GAP- return (action, out.should_continue, out.caveat)diff --git a/rag/advanced_rag/harness/sufficiency_ladder.py b/rag/advanced_rag/harness/sufficiency_ladder.pydeleted file mode 100644index cb38fb5aaf..0000000000--- a/rag/advanced_rag/harness/sufficiency_ladder.py+++ /dev/null@@ -1,176 +0,0 @@-"""Decision ladder for sufficiency — LLM AutoRater as primary judge, agent-confidence as a risk gate (Sufficient Context paper, arXiv 2411.06037).--This replaces the old "code cross-check + dual-signal weighted fusion + 5-way-verdict" pipeline. Because the operator cannot train/calibrate fusion weights-(the paper's logistic regression), we use a **monotonic threshold ladder**: the-LLM AutoRater decides *whether the evidence is sufficient to infer a plausible-answer*; the agent's self-assessed confidence only modulates *how the verdict is-presented* (full answer vs. caveated answer vs. reconcile). The only continuous-degree of freedom is conservativeness thresholds (``c_high`` / ``c_low`` /-``llm_floor``), which are product policy knobs, not learned parameters.--Semantics (paper §3 / §5.1):- - Sufficiency = "a plausible answer A' can be inferred from the context",- independent of whether that answer is correct, and without a ground truth.- - Selective generation = confidence does not re-score the LLM's sufficiency- verdict; it only decides whether to answer fully, caveat, or investigate.--Actions:- - ``ANSWER`` full answer (sufficient + high confidence)- - ``ANSWER_WITH_CAVEAT`` answer with an explicit caveat- - ``GAP`` follow-up search on the concrete missing pieces- - ``RECONCILE`` re-investigate a low-confidence / contradictory point- - ``UNANSWERABLE`` cannot answer (or fall back to a direct LLM)-"""--from __future__ import annotations--from dataclasses import dataclass, field-from typing import Any--# Action constants (shared with callers / orchestrators).-ANSWER = "ANSWER"-ANSWER_WITH_CAVEAT = "ANSWER_WITH_CAVEAT"-GAP = "GAP"-RECONCILE = "RECONCILE"-UNANSWERABLE = "UNANSWERABLE"---@dataclass-class LadderOutput:- """Result of the decision ladder."""-- action: str- should_continue: bool- caveat: str = ""- missing: list[str] = field(default_factory=list)---def aggregate_agent_confidence(- agent_results: list[Any],- hard_violations: set[str] | dict[str, list[str]] | None = None,-) -> float:- """Mean self-confidence over the trusted (self-verified, no hard veto) claims.-- No weights, no normalization — we only pick the trustworthy subset and take- the mean. Hard vetoes (required-entity gaps, grounded-fact violations,- numeric conflicts) suppress a claim's confidence entirely, so a confidence- signal can never mask a missing-evidence gap (selective generation).- """- violations: set[str] = set()- if hard_violations:- if isinstance(hard_violations, dict):- violations = set(hard_violations.keys())- else:- violations = set(hard_violations)-- trusted = [r for r in agent_results if getattr(r, "is_verified", False) and getattr(r, "claim_id", "") not in violations]- if not trusted:- return 0.0- total = 0.0- for r in trusted:- try:- total += float(getattr(r, "confidence", 0.0) or 0.0)- except (TypeError, ValueError):- continue- return total / len(trusted)---def sufficiency_ladder(- *,- auto_sufficient: bool,- auto_confidence: float,- missing: list[str],- contradictions: list[str],- agent_confidence: float,- c_high: float,- c_low: float,- llm_floor: float,- allows_reconcile: bool,- cycle: int,- max_cycles: int,- hard_violations: set[str] | dict[str, list[str]] | None = None,-) -> LadderOutput:- """Evaluate the decision ladder and return the action.-- Parameters- ----------- auto_sufficient : bool- LLM AutoRater's verdict (a plausible answer can be inferred).- auto_confidence : float- AutoRater's own confidence in its sufficiency judgment.- missing : list[str]- Concrete gaps when AutoRater says insufficient.- contradictions : list[str]- Evidence-internal contradictions (even if sufficient, caveat).- agent_confidence : float- Aggregated agent self-confidence (0..1).- c_high / c_low : float- Agent-confidence gates for full vs. caveated answer.- llm_floor : float- If AutoRater confidence < this, re-investigate regardless of verdict.- allows_reconcile : bool- Whether the mode can force a re-investigation (medium=False).- cycle / max_cycles : int- 0-based current cycle and the mode's budget.- hard_violations : set | dict- Claim IDs (or {id: gaps}) with a proven evidence gap (required entity- missing / grounded absent / numeric conflict). Any non-empty value- forces a caveated answer even if AutoRater says sufficient.- """- violations: set[str] = set()- if hard_violations:- violations = set(hard_violations.keys()) if isinstance(hard_violations, dict) else set(hard_violations)-- # 1. Hard veto floor: code-proven evidence gap beats the LLM's "good enough".- if violations:- return LadderOutput(- action=ANSWER_WITH_CAVEAT,- should_continue=False,- caveat=f"hard evidence gap in claim(s): {sorted(violations)[:6]}",- missing=missing,- )-- # 2. AutoRater says insufficient.- if not auto_sufficient:- if missing:- return LadderOutput(action=GAP, should_continue=True, missing=missing)- return LadderOutput(action=UNANSWERABLE, should_continue=False, missing=missing)-- # 3. AutoRater is not confident in its own sufficiency call.- if auto_confidence < llm_floor:- if allows_reconcile and cycle < max_cycles - 1:- return LadderOutput(- action=RECONCILE,- should_continue=True,- caveat="AutoRater itself is unsure; re-investigating",- )- return LadderOutput(- action=ANSWER_WITH_CAVEAT,- should_continue=False,- caveat="AutoRater sufficiency judgment is low-confidence",- )-- # 4. Sufficient + confident: agent confidence sets the presentation.- caveat = ""- should_continue = False- if agent_confidence >= c_high:- action = ANSWER- elif agent_confidence >= c_low:- action = ANSWER_WITH_CAVEAT- caveat = "evidence partially supports the answer"- elif allows_reconcile and cycle < max_cycles - 1:- action = RECONCILE- should_continue = True- else:- action = ANSWER_WITH_CAVEAT- caveat = "evidence partially supports the answer"-- if contradictions:- # Never silently pick one side of a contradiction; surface it.- caveat = "evidence contains conflicting figures"- action = ANSWER_WITH_CAVEAT- should_continue = False-- return LadderOutput(action=action, should_continue=should_continue, caveat=caveat, missing=missing)diff --git a/rag/advanced_rag/harness/tools/__init__.py b/rag/advanced_rag/harness/tools/__init__.pyindex c7402d25c3..e2dc4bc145 100644--- a/rag/advanced_rag/harness/tools/__init__.py+++ b/rag/advanced_rag/harness/tools/__init__.py@@ -4,7 +4,15 @@ from rag.advanced_rag.harness.tools.registry import _inspector_schema, _navigate# Register tools# Search tools-from rag.advanced_rag.harness.tools.search import bm25_search, hybrid_search, structured_query, vector_search, web_search+from rag.advanced_rag.harness.tools.search import (+ bm25_search,+ grep_search,+ hybrid_search,+ list_chunks,+ structured_query,+ vector_search,+ web_search,+)register_tool("hybrid_search", _search_schema("hybrid_search", "Embedding + Keywords search"), hybrid_search)register_tool("vector_search", _search_schema("vector_search", "Embedding search"), vector_search)@@ -12,6 +20,53 @@ register_tool("bm25_search", _search_schema("bm25_search", "Keywords search"), bregister_tool("web_search", _search_schema("web_search", "Internet search"), web_search)register_tool("structured_query", _search_schema("structured_query", "SQL search"), structured_query)+# Grep-style exact locate + deep-read (used by the sub-agent, mirrors dynamic's+# grep_chunks / list_chunks but returns the Pipeline {"chunks": [...]} contract).+register_tool(+ "grep_search",+ {+ "type": "function",+ "function": {+ "name": "grep_search",+ "description": (+ "Exact keyword/pattern locate: keyword-first (BM25) candidate retrieval, then a "+ "case-insensitive regex locate returning COMPACT snippet chunks (token-cheap). Use "+ "for exact names / ids / numbers. When nothing matches it returns the candidates unchanged."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "query": {"type": "string", "description": "the regex/keywords to locate (case-insensitive)."},+ },+ "required": ["query"],+ },+ },+ },+ grep_search,+)+register_tool(+ "list_chunks",+ {+ "type": "function",+ "function": {+ "name": "list_chunks",+ "description": (+ "Deep-read a document: return its COMPLETE chunk list in reading order. Call after "+ "grep_search / hybrid_search hits a document when you need the full text (enumeration / "+ "count / arithmetic answers)."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "doc_id": {"type": "string", "description": "the owning document id to read fully."},+ },+ "required": ["doc_id"],+ },+ },+ },+ list_chunks,+)+# Navigation tools (require compilation)from rag.advanced_rag.harness.tools.navigation import dataset_navigation_search, mindmap_navigate, ontology_navigatediff --git a/rag/advanced_rag/harness/tools/exploration.py b/rag/advanced_rag/harness/tools/exploration.pyindex 3b724c0946..3cc75763e8 100644--- a/rag/advanced_rag/harness/tools/exploration.py+++ b/rag/advanced_rag/harness/tools/exploration.py@@ -63,7 +63,7 @@ async def wiki_query(tools, query: str, keywords: str = "") -> dict:exprs = [match_text]if getattr(tools, "embed_mdl", None):try:- match_dense = await settings.retriever.get_vector(text, tools.embed_mdl, top_k=_WIKI_QUERY_TOP_N, num_candidates=_WIKI_QUERY_TOP_N * 2, similarity=0.1)+ match_dense = await settings.retriever.get_vector(text, tools.embed_mdl, _WIKI_QUERY_TOP_N, 0.1)exprs = [match_text, match_dense, FusionExpr("weighted_sum", _WIKI_QUERY_TOP_N, {"weights": "0.001, 1"})]except Exception:_LOG.exception("[Wiki lookup] dense expr build failed; BM25 only")diff --git a/rag/advanced_rag/harness/tools/gating.py b/rag/advanced_rag/harness/tools/gating.pyindex 6ec6f37b1a..63d240ddce 100644--- a/rag/advanced_rag/harness/tools/gating.py+++ b/rag/advanced_rag/harness/tools/gating.py@@ -1,11 +1,7 @@"""Tool selection gating: phase-based filtering and fallback chain."""-import logging-from rag.advanced_rag.harness.tools.registry import TOOL_REGISTRY-from rag.advanced_rag.harness.types import ClaimTarget, OrchestratorContext--_LOG = logging.getLogger(__name__)+from rag.advanced_rag.harness.types import OrchestratorContext# Search phase definitions@@ -18,23 +14,26 @@ SEARCH_PHASES = {"mindmap_navigate","hybrid_search","bm25_search",+ "grep_search","wiki_query",],"max_returned": 5,- "tool_hint": "Prefer navigation tools to locate document regions before directly searching keywords.",+ "tool_hint": "Prefer navigation tools to locate document regions before directly searching keywords; grep_search for exact names/ids/numbers.",},"explore": {"goal": "Explore deeply within the already located region.","tools_priority": ["hybrid_search","bm25_search",+ "grep_search",+ "list_chunks","web_search","graph_explore","inspector_open_context","inspector_request_adjacent",],- "max_returned": 4,- "tool_hint": "Prefer retrieval tools to gather detailed information within the located region; use web_search when the knowledge base lacks the answer or the question needs current/external facts.",+ "max_returned": 5,+ "tool_hint": "Prefer retrieval tools to gather detailed information within the located region; grep_search for exact locate and list_chunks to deep-read a hit document; use web_search when the knowledge base lacks the answer or the question needs current/external facts.",},"verify": {"goal": "Verify consistency across multiple sources.",@@ -94,7 +93,6 @@ def get_gated_tools(context: OrchestratorContext,has_routed_scope: bool = False,web_enabled: bool = True,- claim: ClaimTarget | None = None,) -> list[dict]:"""Filter, sort, and gate tools by phase priority and context."""phase_config = SEARCH_PHASES.get(phase)@@ -115,8 +113,6 @@ def get_gated_tools(sorted_tools.append(tool_name)selected = sorted_tools[: phase_config["max_returned"]]- if phase == "locate":- selected = _inject_locate_fallback_tools(selected, available_tools, claim, web_enabled)# Copy the registry schemas before annotating — the registry dicts are# shared process-wide, mutating them would leak phase hints across# concurrent requests.@@ -131,46 +127,10 @@ def _default_defs(tool_names: list[str], web_enabled: bool = True) -> list[dict]return [TOOL_REGISTRY[n]["function_schema"] for n in tool_names if n in TOOL_REGISTRY and (web_enabled or n != "web_search")]-# After this many consecutive `locate` rounds for a single claim that produced-# zero evidence chunks, keep the phase in `locate` but admit `web_search` into-# the gated tool set. This preserves the phase semantics ("still trying to-# locate an answer") while giving the agent an external-search escape hatch-# when the knowledge base lacks the fact entirely.-LOCATE_EMPTY_ADVANCE_THRESHOLD = 2---def determine_current_phase(- context: OrchestratorContext,- claim: ClaimTarget | None = None,-) -> str:+def determine_current_phase(context: OrchestratorContext) -> str:"""Determine the current search phase based on context."""- if claim is not None:- evidence_ids = claim.agent_result.evidence_ids if claim.agent_result else None- if not evidence_ids:- return "locate"- elif not context.has_any_chunks():+ if not context.has_any_chunks():return "locate"if context.verdict and context.verdict.has_conflicts:return "verify"return "explore"---def _inject_locate_fallback_tools(- selected: list[str],- available_tools: list[str],- claim: ClaimTarget | None,- web_enabled: bool,-) -> list[str]:- """Keep `locate` semantics but add external fallback when KB locate fails."""- if not web_enabled or claim is None:- return selected- if claim.locate_empty_streak < LOCATE_EMPTY_ADVANCE_THRESHOLD:- return selected- if "web_search" not in available_tools or "web_search" in selected:- return selected- _LOG.info(- "[Tool gating] claim=%s: injecting web_search after repeated locate misses (locate_empty_streak=%d)",- claim.claim_id,- claim.locate_empty_streak,- )- return [*selected, "web_search"]diff --git a/rag/advanced_rag/harness/tools/navigation.py b/rag/advanced_rag/harness/tools/navigation.pyindex 674ea6d6cb..cbbf063ac0 100644--- a/rag/advanced_rag/harness/tools/navigation.py+++ b/rag/advanced_rag/harness/tools/navigation.py@@ -20,9 +20,12 @@ back full, un-narrowed chunks.import jsonimport loggingimport re+from typing import Anyimport json_repair+from rag.llm.tool_decorator import tool+_LOG = logging.getLogger(__name__)# Compiled-structure kinds that describe a document's *layout*: a tree/outline@@ -76,7 +79,13 @@ async def _load_compiled_structure(tools, doc_id: str, kinds: set) -> dict:from common.misc_utils import thread_pool_execfrom rag.nlp import search- resolved = await thread_pool_exec(tools._resolve_doc_tenant, doc_id)+ # _resolve_doc_tenant is a short peewee (MySQL) lookup. Running it through+ # thread_pool_exec opens a connection on a short-lived thread; the peewee pool+ # tracks connections in a process-level _in_use set that is only released on+ # close(), so short-lived threads leak connections (MaxConnectionsExceeded+ # under fan-out parallelism). Call it directly on the event-loop thread to+ # reuse the pool's existing connection.+ resolved = tools._resolve_doc_tenant(doc_id)if not resolved:return {"entities": [], "relations": []}kb_id, tenant_id = resolved@@ -87,6 +96,8 @@ async def _load_compiled_structure(tools, doc_id: str, kinds: set) -> dict:"compile_kwd","compilation_template_ids","compilation_template_kind_kwd",+ "knowledge_graph_kwd",+ "doc_id",]async def _query(condition: dict, limit: int) -> dict:@@ -108,28 +119,54 @@ async def _load_compiled_structure(tools, doc_id: str, kinds: set) -> dict:_LOG.exception("ontology_navigate: failed reading compiled structure for doc=%s", doc_id)return {}- rows = dict(await _query({"doc_id": [doc_id], "knowledge_graph_kwd": ["graph"]}, 1000))+ # Two row shapes coexist for compiled structures:+ # * graph blob (``knowledge_graph_kwd="graph"``): a single compact row whose+ # content is ``{"entities": [...], "relations": [...]}``. This is what+ # RAPTOR/tree compilation (task_executor ``run_tree_templates`` →+ # ``_struct_upsert_graph_json``) writes.+ # * per-entity/relation rows (``knowledge_graph_kwd="entity"/"relation"``):+ # one row per node/edge, content is a single entity/relation dict. This is+ # what page_index (and pipeline Compiler tree via+ # ``_struct_upsert_tree_graph_rows``) writes.+ # ``compile_kwd`` (NOT ``knowledge_graph_kwd``) distinguishes the compile+ # TYPE (tree/page_index/timeline/...); ``knowledge_graph_kwd="graph"`` is a+ # legacy marker we still read for RAPTOR blobs, while the same ``compile_kwd``+ # may also have entity/relation rows. Read BOTH and merge so navigation works+ # regardless of which shape a given compile type produced.+ rows: dict = {}+ rows.update(await _query({"doc_id": [doc_id], "knowledge_graph_kwd": ["graph"]}, 1000))+ rows.update(+ await _query(+ {"doc_id": [doc_id], "knowledge_graph_kwd": ["entity", "relation"]},+ 3000,+ )+ )rows.update(await _query({"doc_id": [doc_id], "compile_kwd": ["raptor_graph"]}, 16))entities: list[dict] = []relations: list[dict] = []for row in rows.values():- try:- graph = json.loads(row.get("content_with_weight") or "{}")- except Exception:- continue- if not isinstance(graph, dict):- continue-compile_kwd = row.get("compile_kwd") or ""kind = _normalize_kind(row.get("compilation_template_kind_kwd") or compile_kwd)if compile_kwd == "raptor_graph":kind = "raptor"if kind not in kinds:continue-- entities.extend(graph.get("entities") or [])- relations.extend(graph.get("relations") or [])+ try:+ graph = json.loads(row.get("content_with_weight") or "{}")+ except Exception:+ continue+ if not isinstance(graph, dict):+ continue+ # graph blob: compact graph with nested entities/relations+ if row.get("knowledge_graph_kwd") == "graph":+ entities.extend(graph.get("entities") or [])+ relations.extend(graph.get("relations") or [])+ # per-entity/relation row: content is a single node/edge+ elif row.get("knowledge_graph_kwd") == "entity":+ entities.append(graph)+ elif row.get("knowledge_graph_kwd") == "relation":+ relations.append(graph)return {"entities": entities, "relations": relations}@@ -165,7 +202,8 @@ async def _load_chunks_by_ids(tools, doc_id: str, chunk_ids: list[str]) -> list[from common.misc_utils import thread_pool_execfrom rag.nlp import search- resolved = await thread_pool_exec(tools._resolve_doc_tenant, doc_id)+ # peewee MySQL lookup — call directly to reuse the pool's connection (see note).+ resolved = tools._resolve_doc_tenant(doc_id)if not resolved:return []kb_id, tenant_id = resolved@@ -723,14 +761,13 @@ async def _kg_scopes(tools, doc_scope: list[str] | None = None):With a ``doc_scope`` the graph is limited to those docs (grouped by theirKB); otherwise the whole bound KB graph is explored."""- from common.misc_utils import thread_pool_exec-if hasattr(tools, "scoped_doc_ids"):doc_scope = tools.scoped_doc_ids(doc_scope)if doc_scope:by_kb: dict[tuple, list[str]] = {}for doc_id in doc_scope:- resolved = await thread_pool_exec(tools._resolve_doc_tenant, doc_id)+ # peewee MySQL lookup — call directly to reuse the pool's connection.+ resolved = tools._resolve_doc_tenant(doc_id)if resolved:by_kb.setdefault(resolved, []).append(doc_id)return [(kb, tenant, docs) for (kb, tenant), docs in by_kb.items()]@@ -778,7 +815,7 @@ async def _kg_search(knn_topn = pool or top_nif getattr(tools, "embed_mdl", None):try:- exprs.append(await settings.retriever.get_vector(text, tools.embed_mdl, top_k=knn_topn, num_candidates=knn_topn * 2, similarity=similarity))+ exprs.append(await settings.retriever.get_vector(text, tools.embed_mdl, knn_topn, similarity))except Exception:_LOG.exception("[Graph exploration] vector build failed; using keyword match")if not exprs:@@ -1023,3 +1060,897 @@ async def graph_explore(tools, query: str, keywords: str = "", doc_scope: list[s_LOG.info("[Graph exploration] Insufficient; returning %d evidence passage(s) (%d before keyword filtering).", len(chunks), before)return {"answer": "", "chunks": chunks, "doc_aggs": _doc_aggs(chunks)}+++# ── Navigate-tree/structure tool set (migrated from harness/dynamic) ──+_NAV_TREE_MAX_DOCS = 8++# Cap on datasets scanned per tool call (avoids fan-out over too many KBs).+_NAV_TREE_MAX_DATASETS = 10++# Chars of a chunk's text returned by search_chunks in default (snippet) mode.+_SEARCH_SNIPPET_CHARS = 300++# ── XML helpers (uniform tag vocabulary shared by all retrieval tools) ──+++def _xml_escape(value: Any) -> str:+ s = "" if value is None else str(value)+ return s.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """)+++def _chunk_text(c: dict) -> str:+ return str(c.get("content_with_weight") or c.get("content") or c.get("text") or "")+++def _chunk_attr(c: dict, keys: tuple[str, ...]) -> str:+ for k in keys:+ v = c.get(k)+ if v not in (None, ""):+ return str(v)+ return ""+++def _doc_id(c: dict) -> str:+ return _chunk_attr(c, ("doc_id", "docid", "document_id"))+++def _dataset_id(c: dict) -> str:+ return _chunk_attr(c, ("dataset_id", "kb_id", "knowledgebase_id"))+++def _doc_title(c: dict) -> str:+ return _chunk_attr(c, ("docnm_kwd", "doc_title", "title", "document_name"))+++def _chunk_id(c: dict) -> str:+ return _chunk_attr(c, ("chunk_id", "id"))+++def _rank_chunks_by_terms(candidates: list[dict], queries: list[str]) -> list[dict]:+ """Rank candidate chunks by how many query terms overlap with their text.++ Zero-LLM keyword relevance for the precise ``chunk_ids`` search path (the+ scope is a handful of chunks, so a cheap overlap score suffices). Returns+ chunks sorted most-relevant-first.+ """+ # Collect significant terms from all queries.+ terms: list[str] = []+ for q in queries:+ for tok in re.findall(r"[A-Za-z0-9_]{2,}", (q or "").lower()):+ if tok not in terms:+ terms.append(tok)+ if not terms:+ return list(candidates)+ scored = []+ for c in candidates:+ text = _chunk_text(c).lower()+ hits = sum(1 for t in terms if t in text)+ if hits:+ scored.append((hits, c))+ scored.sort(key=lambda x: x[0], reverse=True)+ return [c for _, c in scored]+++# The rag-agent runner sets this module slot to the active RAGTools instance for+# the current request, so the migrated navigation tools (defined once at import)+# read the request-scoped retrieval context.+_tools_ref: dict[str, Any] = {}+++def _tools_slot():+ return _tools_ref.get("tools")+++def _get_kb_ids(tools_slot) -> list[str]:+ if tools_slot is None:+ return []+ ids = getattr(tools_slot, "kb_ids", None) or []+ return list(ids)+++# ── Tool: navigate_tree (compiled tree structure, high/ultra modes) ──+++@tool(timeout=120)+async def _navigate_tree_impl(+ query: str,+ keywords: str = "",+ doc_scope: list[str] | None = None,+) -> str:+ """Locate the document(s) most likely to hold the answer, via the dataset's+ compiled navigation tree (``dataset_nav`` / ``nav_doc`` layer).++ This tool routes the question by searching the compiled navigation-tree+ document leaves (nav_doc) — which cover EVERY document of the dataset+ (2400+ here), organized into clusters — then returns each routed document+ with a short excerpt (its first chunk) so you can decide which to deep-read.++ Use it when search_chunks / grep_chunks did not clearly locate the answer and+ you need the document-level view to guide you (e.g. an entity spans many+ docs, or the question mentions a general topic that lives across a document's+ body). After it returns doc_id values, deep-read the relevant one(s) with+ list_chunks / navigate_structure.++ Returns an XML <tree_navigation> document. Each routed document is a <doc>+ element with doc_id, doc_title attributes and a <snippet> element (a short+ excerpt, NOT the full text). If nothing routes, an empty <tree_navigation/>+ is returned.++ :param query: REQUIRED: the question / topic to route.+ :param keywords: Optional terms kept only for prompt hints; document routing+ is driven by the nav-tree doc leaves and does not require keyword hits.+ :param doc_scope: Optional document ids to restrict routing to (at most 8).+ :return: XML <tree_navigation> document.+ """+ from common import settings++ if not getattr(settings, "retriever", None):+ return '<tree_navigation count="0" error="no retriever">\n</tree_navigation>'+ query = str(query or "").strip()+ if not query:+ return '<tree_navigation count="0" error="query is required">\n</tree_navigation>'++ tools_slot = _tools_slot()+ # Route via the compiled navigation tree (dataset_nav / nav_doc layer): it+ # covers ALL documents of the dataset organized into clusters, so a query can+ # reach documents that pure embedding routing would miss. Fall back to pure+ # embedding routing only when the dataset has NO compiled navigation tree.+ routed = await dataset_navigation_search(tools_slot, query, keywords, doc_scope)+ if not routed:+ routed = await _route_docs_via_embedding(tools_slot, query, doc_scope)+ if not routed:+ return '<tree_navigation count="0">\n</tree_navigation>'++ parts = [f'<tree_navigation count="{len(routed)}" query="{_xml_escape(query)}">']+ for i, doc_id in enumerate(routed[:_NAV_TREE_MAX_DOCS]):+ title = ""+ snippet = ""+ try:+ full = await tools_slot.fetch_full_document(doc_id)+ doc_chunks = full.get("chunks", []) or []+ if doc_chunks:+ title = _doc_title(doc_chunks[0]) or doc_id+ snippet = _chunk_text(doc_chunks[0])+ if len(snippet) > 500:+ snippet = snippet[:500]+ else:+ title = doc_id+ except Exception:+ title = doc_id+ parts.append(f' <doc rank="{i + 1}" doc_id="{_xml_escape(doc_id)}" doc_title="{_xml_escape(title)}">')+ if snippet:+ parts.append(f" <snippet>{_xml_escape(snippet)}</snippet>")+ parts.append(" </doc>")+ parts.append("</tree_navigation>")+ return "\n".join(parts)+++async def _route_docs_via_embedding(tools_slot, query: str, doc_scope: list[str] | None = None, top_n: int = 12) -> list[str]:+ """Route ``query`` to documents by PURE embedding similarity — ZERO keywords.++ Encodes the query with ``tools.embed_mdl`` and retrieves the most+ vector-similar real document chunks (compiled products excluded), aggregated+ to documents via ``doc_aggs``. There is NO nav-tree ``_text_score > 0``+ keyword-literal gate and no hybrid/BM25 leg: routing is driven purely by+ vector similarity, so a question that matches a document's body only+ semantically (e.g. an entity alias the summary never spells out) still+ routes to the right document.++ Returns a deduplicated most-relevant-first ``doc_id`` list, or ``[]`` on+ failure / when nothing hits. This function only routes — it does not+ retrieve structure.+ """+ if not query or not str(query).strip():+ return []+ if not getattr(tools_slot, "embed_mdl", None):+ _LOG.warning("[navigate_structure] no embed_mdl available; cannot route by embedding")+ return []+ from common import settings++ if not getattr(settings, "retriever", None):+ return []+ target_ids = _get_kb_ids(tools_slot)+ if not target_ids:+ return []+ tenant_ids = getattr(tools_slot, "tenant_ids", None) or []+ if not tenant_ids:+ tid = _first_tenant_id(tools_slot)+ if tid:+ tenant_ids = [tid]+ if not tenant_ids:+ return []+ try:+ kbinfos = await settings.retriever.retrieval(+ str(query).strip(),+ tools_slot.embed_mdl,+ tenant_ids,+ target_ids,+ 1,+ top_n,+ 0.2,+ vector_similarity_weight=1.0, # pure embedding — no keyword/BM25 leg+ aggs=True,+ highlight=False,+ doc_ids=doc_scope,+ must_not={"exists": "compile_kwd"}, # real doc chunks only; compiled products have their own tools+ )+ except Exception:+ _LOG.exception("[navigate_structure] embedding doc routing failed")+ return []+ doc_ids: list[str] = []+ seen: set[str] = set()+ for agg in kbinfos.get("doc_aggs") or []:+ did = str(agg.get("doc_id") or "").strip()+ if did and did not in seen:+ seen.add(did)+ doc_ids.append(did)+ _LOG.info("[navigate_structure] Embedding doc routing found %d candidate document(s).", len(doc_ids))+ return doc_ids+++def _first_tenant_id(tools_slot) -> str:+ """Best-effort tenant id from the RAGTools context."""+ try:+ for attr in ("tenant_id", "tenant_ids"):+ v = getattr(tools_slot, attr, None)+ if isinstance(v, list):+ for t in v:+ if str(t or "").strip():+ return str(t)+ elif v:+ return str(v)+ except Exception:+ pass+ try:+ kbs = getattr(tools_slot, "kbs", None) or []+ for kb in kbs:+ tid = getattr(kb, "tenant_id", None)+ if tid:+ return str(tid)+ except Exception:+ pass+ return ""+++# ── Tool: navigate_structure (compiled structure of a document, high/ultra) ──+++@tool(timeout=120)+async def _navigate_structure_impl(+ query: str,+ doc_id: str = "",+ kind: str = "catalog",+ keywords: str = "",+ doc_scope: list[str] | None = None,+) -> str:+ """Read a document's COMPILED STRUCTURE (tree / page-index catalog, concept+ mindmap, or entity graph) and return its entities, relations, and the chunks+ behind them — so you can see how the document is organized internally.++ This is the "inside-the-doc" counterpart to navigate_tree. Use it when you+ have located a document (via search_chunks / grep_chunks / navigate_tree) and+ want to understand its internal structure — the catalog tree of headings, the+ concept mindmap, or the entity graph — to find where an answer lives without+ reading every chunk.++ Returns an XML <structure_navigation> document. Each analyzed document is a+ <doc> element with doc_id / doc_title and a <structure> element — a compact+ OUTLINE of the compiled entities and relations, with each line annotating the+ source-chunk ids that back it (e.g. ``[chunks: c100,c101]``). This is an+ outline only (low token cost) — it does NOT include the chunk text. To read+ the actual content behind a line, call list_chunks with its doc_id and+ chunk_ids to deep-read precisely those chunks. If the document has no compiled+ structure of the requested kind, an empty <doc/> is returned (no error) — fall+ back to list_chunks / search_chunks.++ This tool performs NO chat-LLM calls: it only reads the compiled structure+ rows from the doc store.++ :param query: REQUIRED: the question/topic (used to route to a document when+ ``doc_id`` is not given).+ :param doc_id: Optional: read the structure of this specific document. When+ omitted, documents are located by PURE embedding similarity (no keyword+ gate), not the compiled nav tree.+ :param kind: Which compiled structure to read: "catalog" (tree / page-index /+ timeline, default), "mindmap" (concept mindmap), or "graph" (entity graph).+ :param keywords: Optional terms kept only for prompt hints; document routing+ is embedding-driven and does not require keyword hits.+ :param doc_scope: Optional document ids to restrict routing to (at most 8).+ :return: XML <structure_navigation> document (outline only).+ """+ from common import settings++ if not getattr(settings, "retriever", None):+ return '<structure_navigation count="0" error="no retriever">\n</structure_navigation>'+ query = str(query or "").strip()+ if not query:+ return '<structure_navigation count="0" error="query is required">\n</structure_navigation>'++ # Determine which compiled kinds to read.+ kinds = _structure_kinds_for(kind)++ tools_slot = _tools_slot()+ if doc_id:+ doc_ids = [str(doc_id).strip()] if str(doc_id).strip() else []+ else:+ # Locate the document by PURE embedding similarity — no nav-tree+ # keyword-literal gate, no hybrid/BM25 leg (see _route_docs_via_embedding).+ doc_ids = await _route_docs_via_embedding(tools_slot, query, doc_scope)+ if not doc_ids:+ return '<structure_navigation count="0" error="no document located">\n</structure_navigation>'++ structures = await _read_structures(tools_slot, query, doc_ids[:_NAV_TREE_MAX_DOCS], kinds)++ parts = [f'<structure_navigation count="{len(structures)}" query="{_xml_escape(query)}" kind="{_xml_escape(kind)}">']+ for i, s in enumerate(structures):+ parts.append(f' <doc rank="{i + 1}" doc_id="{_xml_escape(s["doc_id"])}" doc_title="{_xml_escape(s["title"])}" entities="{len(s["entities"])}" relations="{len(s["relations"])}">')+ outline = s["outline"]+ if outline:+ parts.append(f" <structure>{_xml_escape(outline)}</structure>")+ parts.append(" </doc>")+ parts.append("</structure_navigation>")+ return "\n".join(parts)+++def _structure_kinds_for(kind: str) -> set:+ """Map a ``navigate_structure`` kind string to the compiled kinds set."""++ k = (kind or "catalog").strip().lower()+ if k in ("mindmap", "mind_map", "concept"):+ return set(_MINDMAP_KINDS)+ if k in ("graph", "kg", "entity", "ontology"):+ return {"graph", "ontology", "entity", "raptor"}+ return set(_CATALOG_KINDS)+++# --- navigate_structure hierarchical drill-down (zero LLM) ---+# Caps controlling the drill-down scope.+_STRUCT_MAX_DEPTH = 3 # max TOC levels drilled.+_STRUCT_BRANCH_K = 2 # keep top-K most relevant nodes per level.+_STRUCT_RELEVANCE_MIN = 1 # a node must match at least this many query terms to descend.+_STRUCT_VEC_BEAM_RATIO = 0.5 # vector beam: drop nodes below best*this similarity.+_STRUCT_MAX_NODES = 10 # cap on nodes rendered in the outline.+_STRUCT_MAX_CHUNKS = 4 # cap on chunk snippets returned in the outline.+_STRUCT_DESC_SNIPPET = 180 # cap on a node's description snippet in the outline.+# Snippet length / per-doc cap for the related chunks appended after a+# search_chunks hit (from the document's compiled structure, not a full read).+_STRUCT_RELATED_SNIPPET_CHARS = 300+_STRUCT_RELATED_MAX_PER_DOC = 4+++async def _embed_query(tools_slot, query: str):+ """Encode ``query`` into a vector via the embedding model.++ Returns ``(qvec, dim)`` where ``qvec`` is a list/np-array and ``dim`` is its+ length (used to pick the ``q_<dim>_vec`` doc-store field). Returns ``(None, 0)``+ when no embedding model is available (caller falls back to keyword scoring).+ """+ try:+ embd = getattr(tools_slot, "embed_mdl", None)+ if embd is None or not callable(getattr(embd, "encode_queries", None)):+ return None, 0+ # ``encode_queries`` is a SYNCHRONOUS method returning ``(vector, token_count)``+ # on the embedding model; do NOT await it (it is not a coroutine).+ qvec, _tok = embd.encode_queries(query)+ if qvec is None:+ return None, 0+ import numpy as np++ arr = np.asarray(qvec, dtype=float)+ if arr.ndim != 1 or arr.size == 0:+ return None, 0+ return arr, int(arr.size)+ except Exception:+ _LOG.exception("[navigate_structure] query embedding failed; falling back to keyword")+ return None, 0+++async def _load_entities_with_vectors(tools_slot, doc_id: str, kinds: set, vec_field: str) -> list[dict]:+ """Load a document's compiled-structure entities WITH their embeddings.++ Like ``_load_compiled_structure`` but also reads ``vec_field`` (the ``q_<dim>_vec``+ column) for each entity row, so vector beam descent can score nodes by cosine+ similarity to the query. Returns entities with an added ``"_vec"`` key.+ """+ from common import settings+ from common.doc_store.doc_store_base import OrderByExpr+ from common.misc_utils import thread_pool_exec+ from rag.nlp import search++ # peewee MySQL lookup — call directly to reuse the pool's connection (see note).+ resolved = tools_slot._resolve_doc_tenant(doc_id)+ if not resolved:+ return []+ kb_id, tenant_id = resolved+ index_name = search.index_name(tenant_id)+ fields = ["content_with_weight", "compile_kwd", "compilation_template_kind_kwd", "knowledge_graph_kwd", "doc_id"]+ if vec_field:+ fields.append(vec_field)++ try:+ # Same dual-shape logic as _load_compiled_structure: RAPTOR/tree writes a+ # compact graph blob (knowledge_graph_kwd="graph"), while page_index /+ # pipeline-Compiler tree write per-entity rows (knowledge_graph_kwd="entity").+ # Read BOTH so vector beam descent works for either shape.+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ fields,+ [],+ {"doc_id": [doc_id], "knowledge_graph_kwd": ["graph"]},+ [],+ OrderByExpr(),+ 0,+ 1000,+ index_name,+ [kb_id],+ )+ res2 = await thread_pool_exec(+ settings.docStoreConn.search,+ fields,+ [],+ {"doc_id": [doc_id], "knowledge_graph_kwd": ["entity"]},+ [],+ OrderByExpr(),+ 0,+ 3000,+ index_name,+ [kb_id],+ )+ rows = settings.docStoreConn.get_fields(res, fields) or {}+ rows.update(settings.docStoreConn.get_fields(res2, fields) or {})+ except Exception:+ _LOG.exception("[navigate_structure] _load_entities_with_vectors failed for doc=%s", doc_id)+ return []++ out: list[dict] = []+ for row in rows.values():+ try:+ graph = json.loads(row.get("content_with_weight") or "{}")+ except Exception:+ continue+ if not isinstance(graph, dict):+ continue+ kind = _normalize_kind(row.get("compilation_template_kind_kwd") or row.get("compile_kwd") or "")+ if kind not in kinds:+ continue+ vec = row.get(vec_field) if vec_field else None+ # graph blob: content nests entities under "entities"; entity row: content+ # is a single entity dict.+ candidates = graph.get("entities") or [] if row.get("knowledge_graph_kwd") == "graph" else [graph]+ for e in candidates:+ if not isinstance(e, dict) or not (e.get("name") or "").strip():+ continue+ e = dict(e)+ if vec is not None:+ e["_vec"] = vec+ out.append(e)+ return out+++async def _read_structures(tools_slot, query: str, doc_ids: list[str], kinds: set) -> list[dict]:+ """Read compiled structures for ``doc_ids`` — VECTOR BEAM DRILL-DOWN.++ The compiled structure (esp. tree_node / page_index TOC) is a HIERARCHY whose+ nodes already carry embeddings (``q_<dim>_vec``). We exploit that: embed the+ query once, then for each document follow the pre-built tree from its roots,+ at every level keeping the top-K nodes by COSINE similarity to the query+ (beam search) and descending to their children.++ Collect the union of ``source_chunk_ids`` over the drilled path — a SET of+ relevant chunks (not a single one) — then return short snippets plus node+ pointers, so the model follows the pointers with grep_chunks / search_chunks+ instead of receiving large chunks. Zero chat-LLM.+ """++ qvec, dim = await _embed_query(tools_slot, query)+ vec_field = f"q_{dim}_vec" if dim else ""++ out: list[dict] = []+ for doc_id in doc_ids:+ # Entities with their embeddings (for cosine scoring) + relations (for the+ # tree edges). ``kinds`` gates which compiled buckets we read.+ entities: list[dict] = []+ relations: list[dict] = []+ try:+ if vec_field:+ entities = await _load_entities_with_vectors(tools_slot, doc_id, kinds, vec_field)+ if not entities:+ structure = await _load_compiled_structure(tools_slot, doc_id, kinds)+ entities = structure.get("entities") or []+ relations = structure.get("relations") or []+ else:+ structure = await _load_compiled_structure(tools_slot, doc_id, kinds)+ relations = structure.get("relations") or []+ except Exception:+ _LOG.exception("[navigate_structure] structure load failed for doc=%s", doc_id)+ outline = await _render_toc_drilldown(query, qvec, entities, relations)+ out.append(+ {+ "doc_id": doc_id,+ "title": "",+ "entities": entities,+ "relations": relations,+ "outline": outline,+ }+ )+ return out+++def _build_toc_tree(entities: list[dict], relations: list[dict]):+ """Build parent→children map, child→parent map, and the root node names.++ Tree relations are interpreted as ``from`` = parent, ``to`` = child. Roots are+ tree nodes with no parent (isolated nodes are their own roots).+ """+ by_name: dict[str, dict] = {}+ for e in entities:+ name = (e.get("name") or "").strip()+ if name and name not in by_name:+ by_name[name] = e+ children: dict[str, list[str]] = {}+ parents: dict[str, str] = {}+ for r in relations:+ p = (r.get("from") or "").strip()+ c = (r.get("to") or "").strip()+ if not p or not c or p == c:+ continue+ children.setdefault(p, [])+ if c not in children[p]:+ children[p].append(c)+ parents[c] = p+ roots = [n for n in by_name if n not in parents]+ if not roots:+ # Every node has a parent? Use nodes with no children (isolated) as roots.+ roots = [n for n in by_name if n not in children] or list(by_name)+ return by_name, children, parents, roots+++def _node_relevance(query_terms: list[str], entity: dict) -> int:+ """Keyword-overlap relevance of a TOC node against the query."""+ if not query_terms:+ return 0+ text = f"{entity.get('name') or ''} {entity.get('description') or ''}".lower()+ return sum(1 for t in query_terms if t in text)+++def _collect_chunk_ids(nodes: list[dict], cap: int = 32) -> list[str]:+ """Union of source_chunk_ids across nodes, deduped and bounded."""+ seen: list[str] = []+ for n in nodes:+ for cid in n.get("source_chunk_ids") or []:+ if isinstance(cid, str) and cid and cid not in seen:+ seen.append(cid)+ if len(seen) >= cap:+ return seen+ return seen+++def _cosine(a, b):+ """Cosine similarity between two arrays (skip zero vectors)."""+ import numpy as np++ try:+ aa = np.asarray(a, dtype=float).reshape(-1)+ bb = np.asarray(b, dtype=float).reshape(-1)+ if aa.size == 0 or bb.size == 0 or aa.size != bb.size:+ return 0.0+ denom = float(np.linalg.norm(aa) * np.linalg.norm(bb))+ if denom == 0.0:+ return 0.0+ return float(np.dot(aa, bb) / denom)+ except Exception:+ return 0.0+++def _node_score(qvec, query_terms, entity: dict) -> float:+ """Score a TOC node: cosine similarity (if the node has an embedding and we+ have a query vector), else keyword-overlap. Higher is better."""+ if qvec is not None:+ v = entity.get("_vec")+ if v is not None:+ return _cosine(qvec, v)+ return float(_node_relevance(query_terms, entity))+++def _drill_kept_nodes(query_terms: list[str], qvec, entities: list[dict], relations: list[dict]) -> tuple[list[dict], dict, set]:+ """VECTOR BEAM drill-down over the compiled TOC hierarchy.++ From the roots, at each level keep the top-K nodes by cosine similarity to the+ query (``_node_score``) and descend to their children; keyword-overlap is the+ fallback when no vectors are available. Returns ``(kept_nodes, parents, kept_names)``+ so callers can both render the outline AND collect the related chunks behind the+ drilled nodes' ``source_chunk_ids``.+ """+ by_name, children, parents, roots = _build_toc_tree(entities, relations)+ if not roots:+ return [], {}, set()++ frontier = list(roots)+ kept_names: set[str] = set()+ depth = 0+ # ``<=`` so the deepest relevant leaf (at depth MAX_DEPTH) is also collected,+ # not just the intermediate levels.+ while frontier and depth <= _STRUCT_MAX_DEPTH:+ scored = [(_node_score(qvec, query_terms, by_name[n]), n) for n in frontier if n in by_name]+ if not scored:+ break+ scored.sort(key=lambda x: x[0], reverse=True)+ if not scored:+ break+ best = scored[0][0]+ # If even the best node has no relevance, stop descending.+ if qvec is None and best < _STRUCT_RELEVANCE_MIN:+ break+ # Beam: keep top-K, but drop nodes far below the best (avoids pulling in+ # low-similarity sibling sub-sections). For vector scoring use a relative+ # threshold; for keyword scoring keep nodes above the absolute minimum.+ if qvec is not None:+ top = [s for s in scored if s[0] >= best * _STRUCT_VEC_BEAM_RATIO][:_STRUCT_BRANCH_K]+ else:+ top = [s for s in scored if s[0] >= _STRUCT_RELEVANCE_MIN][:_STRUCT_BRANCH_K]+ if not top:+ break+ new_frontier: list[str] = []+ for _score, name in top:+ if name not in kept_names:+ kept_names.add(name)+ new_frontier.extend(children.get(name, []))+ frontier = new_frontier+ depth += 1++ # Include ancestor path of the kept nodes so the outline shows the TOC chain.+ for name in list(kept_names):+ cur = parents.get(name)+ guard = 0+ while cur and cur not in kept_names and guard < _STRUCT_MAX_DEPTH:+ kept_names.add(cur)+ cur = parents.get(cur)+ guard += 1++ kept_nodes: list[dict] = []+ for n in kept_names:+ e = by_name.get(n)+ if e is not None:+ kept_nodes.append(e)+ return kept_nodes, parents, kept_names+++async def _render_toc_drilldown(query: str, qvec, entities: list[dict], relations: list[dict]) -> str:+ """Drill down the pre-built TOC hierarchy toward the query and render an outline.++ Uses VECTOR BEAM descent when the nodes carry embeddings (``_vec``) and a query+ vector is available: from the roots, at each level keep the top-K nodes by+ cosine similarity to the query and descend to their children. Falls back to+ keyword-overlap when no vectors are available.++ Returns lines like:+ - Paris Demographics (tree_node): <desc> [chunks: c1,c2]+ - Immigration since 1945 (tree_node): <desc> [chunks: c3]+ - [chunk c3]: <snippet>+ """+ query_terms = [t for t in re.findall(r"[A-Za-z0-9_]{2,}", (query or "").lower()) if len(t) >= 2]+ if not query_terms and qvec is None:+ return _render_outline(entities[:_STRUCT_MAX_NODES], relations[:_STRUCT_MAX_NODES])++ kept_nodes, parents, kept_names = _drill_kept_nodes(query_terms, qvec, entities, relations)+ if not kept_nodes:+ return _render_outline(entities[:_STRUCT_MAX_NODES], relations[:_STRUCT_MAX_NODES])++ # Render the drilled nodes (indented by depth via ancestor count).+ depth_of: dict[str, int] = {}+ for n in kept_names:+ d = 0+ cur = parents.get(n)+ while cur and cur in kept_names:+ d += 1+ cur = parents.get(cur)+ depth_of[n] = d++ lines: list[str] = []+ for e in kept_nodes[:_STRUCT_MAX_NODES]:+ name = (e.get("name") or "").strip()+ etype = (e.get("type") or "other").strip()+ desc = (e.get("description") or "").strip()+ chunks = _chunk_ptrs(e)+ indent = " " * depth_of.get(name, 0)+ line = f"{indent}- {name} ({etype})"+ if desc:+ line += f": {_snippet(desc, _STRUCT_DESC_SNIPPET)}"+ if chunks:+ line += f" [chunks: {chunks}]"+ lines.append(line)++ # Rank the chunks behind the drilled nodes and add short snippets.+ wanted = _collect_chunk_ids(kept_nodes)+ if wanted:+ chunks = await _load_chunks_for_ids(_tools_slot(), wanted)+ if chunks:+ ranked = _rank_chunks_by_terms(chunks, [query])+ for c in ranked[:_STRUCT_MAX_CHUNKS]:+ cid = _chunk_id(c)+ text = _chunk_text(c).strip()+ lines.append(f"- [chunk {cid}]: {_snippet(text, 300)}")+ return "\n".join(lines)+++def _snippet(s: str, n: int) -> str:+ """Truncate a string to ``n`` chars on a char boundary with an ellipsis."""+ s = s.strip()+ if len(s) <= n:+ return s+ return s[:n].rstrip() + "..."+++def _render_outline(entities: list[dict], relations: list[dict]) -> str:+ """Render a compact flat outline (fallback when no query terms are usable)."""+ lines: list[str] = []+ cap_e, cap_r = 40, 40+ for e in entities[:cap_e]:+ name = (e.get("name") or "").strip()+ if not name:+ continue+ etype = (e.get("type") or "other").strip()+ desc = (e.get("description") or "").strip()+ chunks = _chunk_ptrs(e)+ line = f"- {name} ({etype})"+ if desc:+ line += f": {_snippet(desc, _STRUCT_DESC_SNIPPET)}"+ if chunks:+ line += f" [chunks: {chunks}]"+ lines.append(line)+ for r in relations[:cap_r]:+ frm = (r.get("from") or "").strip()+ to = (r.get("to") or "").strip()+ if not frm or not to:+ continue+ chunks = _chunk_ptrs(r)+ line = f"- {frm} -[{r.get('type') or 'related_to'}]-> {to}"+ if chunks:+ line += f" [chunks: {chunks}]"+ lines.append(line)+ return "\n".join(lines)+++def _chunk_ptrs(item: dict) -> str:+ """Comma-join the source_chunk_ids of a structure entity/relation."""+ cids = [c for c in (item.get("source_chunk_ids") or []) if isinstance(c, str) and c]+ if not cids:+ return ""+ # Dedup + bound to avoid a huge pointer list.+ seen: list[str] = []+ for c in cids:+ if c not in seen:+ seen.append(c)+ if len(seen) >= 8:+ break+ return ",".join(seen)+++async def _load_chunks_for_ids(tools_slot, chunk_ids: list[str]) -> list[dict]:+ """Load chunks by id (across any owning document). Zero LLM.++ Unlike ``_load_specific_chunks`` (which requires a doc_scope), this scans the+ bound documents' chunks for the requested ids — used to fetch the chunks behind+ the relevant entities of a navigate_structure outline.+ """+ if not chunk_ids:+ return []+ wanted = {str(c).strip() for c in chunk_ids if str(c).strip()}+ if not wanted:+ return []+ # Resolve the owning document(s) by scanning the conversation's bound docs is+ # not available here, so fall back to a per-kb chunk query by id via the+ # retriever (zero LLM). If unavailable, return empty (model uses pointers).+ try:+ from common import settings+ from rag.nlp import search as _rag_search++ fields = ["content_with_weight", "docnm_kwd", "doc_id"]+ kb_ids = _get_kb_ids(tools_slot) or []+ found: dict[str, dict] = {}+ tenant_ids = getattr(tools_slot, "tenant_ids", None) or []+ if not tenant_ids:+ tid = _first_tenant_id(tools_slot)+ if tid:+ tenant_ids = [tid]+ from common.misc_utils import thread_pool_exec++ for tid in tenant_ids[:4]:+ for kb_id in kb_ids[:_NAV_TREE_MAX_DATASETS]:+ index = _rag_search.index_name(tid)+ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ fields,+ [],+ {"id": list(wanted)[:128]},+ [],+ None,+ 0,+ 128,+ index,+ [kb_id],+ )+ rows = settings.docStoreConn.get_fields(res, fields) or {}+ except Exception:+ continue+ for cid, row in rows.items():+ if cid in wanted:+ found[cid] = {+ "chunk_id": cid,+ "content_with_weight": row.get("content_with_weight") or "",+ "docnm_kwd": row.get("docnm_kwd") or "",+ "doc_id": row.get("doc_id") or "",+ }+ return [found[c] for c in chunk_ids if c in found]+ except Exception:+ _LOG.exception("[navigate_structure] _load_chunks_for_ids failed")+ return []+++async def _expand_related_via_structure(+ tools_slot,+ query: str,+ doc_ids: list[str],+ exclude: set[str],+ max_per_doc: int = _STRUCT_RELATED_MAX_PER_DOC,+) -> list[dict]:+ """After ``search_chunks`` hits a document, read its compiled structure and+ return OTHER chunks related to the query (behind the beam-drilled entities).++ Uses the same VECTOR BEAM drill-down as ``navigate_structure`` (zero chat-LLM):+ embed the query, descend the compiled tree/catalog hierarchy by cosine+ similarity, collect the drilled entities' ``source_chunk_ids``, load those+ chunks, and reduce each to a short snippet. Chunks already returned by the+ search (``exclude``) are skipped so only genuinely NEW related chunks appear.++ Returns a list of chunk dicts with ``content_with_weight`` set to the snippet,+ ordered most-related-first. ``[]`` when the document has no compiled structure+ with embeddings, nothing is relevant, or nothing is left after ``exclude``.+ """+ if not doc_ids or not query or not str(query).strip():+ return []+ from common import settings++ if not getattr(settings, "retriever", None):+ return []++ qvec, dim = await _embed_query(tools_slot, query)+ vec_field = f"q_{dim}_vec" if dim else ""+ query_terms = [t for t in re.findall(r"[A-Za-z0-9_]{2,}", (query or "").lower()) if len(t) >= 2]++ out: list[dict] = []+ for doc_id in doc_ids[:_NAV_TREE_MAX_DOCS]:+ try:+ entities: list[dict] = []+ relations: list[dict] = []+ if vec_field:+ entities = await _load_entities_with_vectors(tools_slot, doc_id, _CATALOG_KINDS, vec_field)+ if not entities:+ continue # no compiled tree/catalog with embeddings -> nothing to drill+ structure = await _load_compiled_structure(tools_slot, doc_id, _CATALOG_KINDS)+ relations = structure.get("relations") or []++ kept_nodes, _parents, _kept = _drill_kept_nodes(query_terms, qvec, entities, relations)+ if not kept_nodes:+ continue+ wanted = _collect_chunk_ids(kept_nodes)+ wanted = [c for c in wanted if c not in exclude]+ if not wanted:+ continue+ chunks = await _load_chunks_for_ids(tools_slot, wanted)+ if not chunks:+ continue+ ranked = _rank_chunks_by_terms(chunks, [query])+ for c in ranked[:max_per_doc]:+ cid = _chunk_id(c)+ if cid in exclude:+ continue+ exclude.add(cid)+ c["content_with_weight"] = _snippet(_chunk_text(c), _STRUCT_RELATED_SNIPPET_CHARS)+ c["related_via_structure"] = True+ out.append(c)+ except Exception:+ _LOG.exception("[search_chunks] compiled-structure related-chunk expansion failed for doc=%s", doc_id)+ return outdiff --git a/rag/advanced_rag/harness/tools/search.py b/rag/advanced_rag/harness/tools/search.pyindex 1307f5f67a..6a09cddfec 100644--- a/rag/advanced_rag/harness/tools/search.py+++ b/rag/advanced_rag/harness/tools/search.py@@ -2,20 +2,99 @@import loggingimport re+from typing import Any+from rag.llm.tool_decorator import toolimport hashlib+from functools import lru_cachefrom common import settings-from .navigation import _kg_scopes+from .navigation import _expand_related_via_structure, _kg_scopes_LOG = logging.getLogger(__name__)+def _compact_keywords(kw: str, max_terms: int = 15) -> str:+ """Deduplicate and cap a formalize / extract_keywords keyword string.++ The extraction prompt asks the model for 3-10 terms *plus* 2-3 synonyms each,+ which models answer with a 40-60 word redundant synonym run (e.g. "average+ distance left field line MLB retractable roof stadiums 2024 ... retractable+ dome covered stadium mean distance outfield" — ~350 chars). Appending that+ whole run onto the query diluted the vector leg and dragged BM25 onto+ unrelated docs. This keeps the recall terms but drops the redundancy:+ dedupe (preserving order) and cap at ``max_terms`` so it stays a compact+ hint instead of a pollution source. Accepts both space- and comma-separated+ input (single-turn extract_keywords emits spaces; multi-turn formalize+ emits commas).+ """+ if not kw:+ return ""+ tokens = re.split(r"[,\s]+", (kw or "").strip())+ seen: list[str] = []+ for t in tokens:+ t = t.strip()+ if not t:+ continue+ if t not in seen:+ seen.append(t)+ if len(seen) >= max_terms:+ break+ return " ".join(seen)++# Sentence terminators: Chinese 。!?;, English ! ? ;, newline, and a# digit-guarded English period (so "3.14" / "v1.2" don't split)._SENT_END = re.compile(r"[。!?;!?;]+|(?<!\d)\.(?!\d)")-# Table blocks are kept ATOMIC — never split by sentence terminators — so a-# whole table counts as one "sentence" for keyword matching / narrowing.-_HTML_TABLE = re.compile(r"<table\b[^>]*>.*?</table>", re.IGNORECASE | re.DOTALL)+# Block-level HTML elements and markdown tables are kept ATOMIC — never split by+# sentence terminators — so a whole table / list / block counts as ONE "sentence"+# for keyword matching and narrowing (a keyword inside one keeps the whole block).+_HTML_TAG = re.compile(r"<(/?)([a-zA-Z][a-zA-Z0-9]*)\b([^>]*)>")++# Only BLOCK-level containers are protected. Inline tags (<b>, <i>, <a>, <span>,+# <em>, <strong>, <code>, ...) are deliberately excluded so ordinary prose that+# contains inline formatting still splits into sentences normally.+_HTML_BLOCK_TAGS = {+ "table",+ "thead",+ "tbody",+ "tfoot",+ "tr",+ "td",+ "th",+ "caption",+ "colgroup",+ "ul",+ "ol",+ "li",+ "dl",+ "dt",+ "dd",+ "div",+ "p",+ "pre",+ "blockquote",+ "section",+ "article",+ "aside",+ "nav",+ "main",+ "figure",+ "figcaption",+ "header",+ "footer",+ "address",+ "details",+ "summary",+ "form",+ "fieldset",+ "h1",+ "h2",+ "h3",+ "h4",+ "h5",+ "h6",+}+# Markdown table: a header row with a pipe, a separator row of dashes/colons/# pipes, then zero+ body rows with a pipe._MD_TABLE = re.compile(@@ -26,15 +105,51 @@ _MD_TABLE = re.compile()+def _html_block_spans(text: str) -> list[tuple[int, int]]:+ """Outermost balanced block-level HTML element spans (nesting-aware).++ Uses a tag stack (not a regex) so nested elements (e.g. a ``<table>`` with+ ``<td>``s, or nested ``<div>``s) yield ONE span for the outermost element and+ are never truncated at the first close tag the way a non-greedy regex would.+ Unclosed / stray tags are ignored (that region just falls back to plain+ sentence splitting).+ """+ spans: list[tuple[int, int]] = []+ stack: list[tuple[str, int]] = []+ for m in _HTML_TAG.finditer(text):+ name = m.group(2).lower()+ if name not in _HTML_BLOCK_TAGS:+ continue+ if m.group(1): # closing tag </name>+ for i in range(len(stack) - 1, -1, -1):+ if stack[i][0] == name:+ start = stack[i][1]+ del stack[i:]+ if not stack: # closed an outermost block+ spans.append((start, m.end()))+ break+ # a stray </name> with no matching open is ignored+ elif not m.group(3).rstrip().endswith("/"): # opening (skip self-closing)+ stack.append((name, m.start()))+ return spans++def _protected_spans(text: str) -> list[tuple[int, int]]:- """Non-overlapping ``(start, end)`` spans of table blocks, in order."""- spans = [(m.start(), m.end()) for m in _HTML_TABLE.finditer(text)]+ """Non-overlapping ``(start, end)`` spans kept atomic, in order.++ Covers block-level HTML elements and markdown tables; overlapping spans are+ merged (unioned) so a match that straddles two is never split.+ """+ spans = _html_block_spans(text)spans += [(m.start(), m.end()) for m in _MD_TABLE.finditer(text)]spans.sort()merged: list[tuple[int, int]] = []last_end = -1for s, e in spans:- if s < last_end: # overlaps an already-kept span -> skip+ if s < last_end: # overlaps an already-kept span -> union it in+ if e > last_end:+ merged[-1] = (merged[-1][0], e)+ last_end = econtinuemerged.append((s, e))last_end = e@@ -61,8 +176,10 @@ def _split_plain(text: str) -> list[str]:def _split_sentences(text: str) -> list[str]:"""Split ``text`` into sentences, keeping each terminator attached.- Table blocks — HTML ``<table>...</table>`` and markdown tables — are treated- as a single atomic sentence and are never split internally.+ Block-level HTML elements (``<table>``, ``<div>``, ``<p>``, ``<ul>``, ... —+ see :data:`_HTML_BLOCK_TAGS`) and markdown tables are treated as a single+ atomic sentence and are never split internally, so a keyword falling inside+ one keeps the whole block together."""if not text:return []@@ -84,25 +201,172 @@ def _split_sentences(text: str) -> list[str]:return sents+# ---------------------------------------------------------------------------+# Stem-tolerant keyword matching (ported from agentic_search4 v8)+#+# Substring matching misses inflected forms: "nominations" misses "nominated",+# "company" misses "companies". Keywords are derived from the question, which+# states things in the inflected form ("which band HEADLINED", "was NOMINATED+# three times"), so the failing direction is the common one. Both sides are+# therefore reduced to a stem before comparison.+# ---------------------------------------------------------------------------+try: # available at runtime — nltk already backs rag/nlp/synonym.py+ from nltk.stem import PorterStemmer as _PorterStemmer++ _porter_stem = _PorterStemmer().stem+except Exception: # pragma: no cover - exercised only where nltk is absent+ _porter_stem = None++# Longest first: "nominations" must lose "ations", not just the trailing "s".+_STEM_SUFFIXES = (+ ("ations", ""),+ ("ation", ""),+ ("ated", ""),+ ("ates", ""),+ ("ate", ""),+ ("ings", ""),+ ("ing", ""),+ ("ies", "i"),+ ("ied", "i"),+ ("ed", ""),+ ("es", ""),+ ("s", ""),+)++_WORD_RE = re.compile(r"[a-z0-9]+")+++def _fallback_stem(word: str) -> str:+ """Suffix stripper used when nltk is unavailable. Approximates Porter."""+ w = word+ for suffix, replacement in _STEM_SUFFIXES:+ if w.endswith(suffix) and len(w) - len(suffix) >= 3:+ w = w[: len(w) - len(suffix)] + replacement+ break+ if len(w) > 3 and w.endswith("y"):+ w = w[:-1] + "i"+ if len(w) > 3 and w.endswith("e"):+ w = w[:-1]+ if len(w) > 3 and w[-1] == w[-2] and w[-1] not in "aeiou":+ w = w[:-1] # running -> runn -> run+ return w+++@lru_cache(maxsize=8192)+def _stem(word: str) -> str:+ return _porter_stem(word) if _porter_stem else _fallback_stem(word)+++def _stemmable(token: str) -> bool:+ """Only plain ASCII words are stemmed.++ Identifiers ("1344259", "2020-21"), notation ("PPG") and CJK text must match+ verbatim — stemming would corrupt them, and it has no meaning for Chinese.+ """+ return len(token) >= 4 and token.isascii() and token.isalpha()+++def _keyword_forms(kwds: list[str]) -> tuple[list[str], list[tuple[str, ...]]]:+ """Split keywords into verbatim substrings and stem sequences.++ A keyword whose tokens are ALL stemmable becomes a stem sequence (matched+ anywhere as a contiguous run of stems); anything containing an identifier,+ notation or CJK falls back to a verbatim substring match.+ """+ verbatim: list[str] = []+ stemmed: list[tuple[str, ...]] = []+ for kw in kwds or []:+ k = (kw or "").strip().lower()+ if not k:+ continue+ tokens = _WORD_RE.findall(k)+ if tokens and all(_stemmable(t) for t in tokens):+ stemmed.append(tuple(_stem(t) for t in tokens))+ else:+ verbatim.append(k)+ return verbatim, stemmed+++def _sentence_stems(sentence: str) -> list[str]:+ return [_stem(t) if _stemmable(t) else t for t in _WORD_RE.findall(sentence.lower())]+++def _sentence_matches(low: str, stems: list[str], verbatim: list[str], stemmed: list[tuple[str, ...]]) -> bool:+ """True when a sentence contains a verbatim keyword or a contiguous stem run."""+ if any(v in low for v in verbatim):+ return True+ for seq in stemmed:+ width = len(seq)+ for start in range(len(stems) - width + 1):+ if tuple(stems[start : start + width]) == seq:+ return True+ return False+++_FACT_RE = re.compile(+ r"(\d[\d,\.]*(?:st|nd|rd|th)?%?)"+ r"|(19|20)\d{2}" # years+ r"|\b(percent|percentage|million|billion|thousand|km|km2|sq\s*km|m\s*above|m)"+ r"\b",+ re.IGNORECASE,+)+_PROPER_NOUN_RE = re.compile(r"(?<![.!?]\.)\b[A-Z][a-z]{2,}\b")+++def _is_fact_dense_sentence(sent: str) -> bool:+ """Heuristically flag a sentence that carries a fact the answer may hinge on+ but which does not necessarily contain the query keywords — a number, a year,+ a percentage, or a proper noun / named entity. Such sentences are kept during+ narrowing even when they sit far from any keyword hit, so a numeric or+ entity answer is never dropped just because it lacks the keyword phrasing.+ """+ low = sent.lower()+ if _FACT_RE.search(sent) or _FACT_RE.search(low):+ return True+ if _PROPER_NOUN_RE.search(sent):+ return True+ return False++def _narrow_content(content: str, kwds: list[str]) -> str | None:- """Return ``content`` narrowed to keyword sentences +/- 1 neighbour.+ """Return ``content`` narrowed to keyword sentences +/- 2 neighbours.- Returns ``None`` when no keyword occurs anywhere in ``content``.+ Matching is stem-tolerant: a keyword matches any word sharing its stem, so+ "nominations" finds "nominated". Sentences that are fact-dense (numbers /+ years / percentages / proper nouns) are kept regardless of keyword distance,+ so numeric or named-entity answers survive narrowing. Returns ``None`` when+ no keyword occurs anywhere in ``content``."""+ # Structured tables must be returned whole. A keyword hit anywhere in a big+ # table (e.g. the capitals-by-latitude table) otherwise narrows to the hit+ # sentence +/- neighbours and DROPS the far end of the table — precisely the+ # "table truncated at -4.58°N, Maseru (-29.3°) missing" bug on FRAMES Q408.+ # A table row is one data point, not a sentence, so keyword-window narrowing is+ # wrong here; keep the full table (it is already rank-sorted by the retriever).+ if "<table" in content.lower() or "<tr" in content.lower() or "<td" in content.lower():+ return "..." + _highlight_keywords(content, kwds) + "..."+sents = _split_sentences(content)if not sents:return None+ # Stem-tolerant matching: a keyword matches any word sharing its stem, so+ # "nominations" finds "nominated" and "company" finds "companies".+ verbatim, stemmed = _keyword_forms(kwds)+ if not verbatim and not stemmed:+ return Nonekeep: set[int] = set()matched = Falsefor i, s in enumerate(sents):low = s.lower()- if any(kw in low for kw in kwds):+ if _sentence_matches(low, _sentence_stems(s), verbatim, stemmed):matched = True- if i > 0:- keep.add(i - 1)- keep.add(i)- if i + 1 < len(sents):- keep.add(i + 1)+ for j in range(max(0, i - 2), min(len(sents), i + 3)):+ keep.add(j)+ elif _is_fact_dense_sentence(s):+ # Keep fact-dense sentences even without a keyword hit so the answer+ # value (a bare figure, a date, a proper noun) is never lost.+ for j in range(max(0, i - 1), min(len(sents), i + 2)):+ keep.add(j)if not matched:return Nonenarrowed = "".join(sents[i] for i in sorted(keep)).strip()@@ -110,10 +374,29 @@ def _narrow_content(content: str, kwds: list[str]) -> str | None:def _highlight_keywords(text: str, kwds: list[str]) -> str:- terms = sorted({kw for kw in kwds if kw}, key=len, reverse=True)+ """Star the verbatim keyword phrases AND any word sharing a keyword's stem.++ Full keyword phrases are matched first and starred as ONE contiguous span, so+ a multi-word entity like "Atlanta Braves" becomes ``*Atlanta Braves*`` — never+ ``*Atlanta* *Braves*`` — because the downstream cross-check matches entities+ with a bounded contiguous regex that a per-word star would break.+ """+ phrases = sorted({(kw or "").strip().lower() for kw in kwds or [] if (kw or "").strip()}, key=len, reverse=True)+ terms: list[str] = list(phrases)+ # Add stem-matched words NOT already inside a phrase, so "nominated" still+ # gets starred for keyword "nominations" while "Atlanta Braves" stays whole.+ verbatim, stemmed = _keyword_forms(kwds)+ stem_set = {s for seq in stemmed for s in seq}+ if stem_set:+ for word in re.findall(r"[A-Za-z]+", text):+ low = word.lower()+ if _stemmable(low) and _stem(low) in stem_set and not any(low in p for p in phrases):+ terms.append(low)if not terms:return text- pattern = re.compile("|".join(re.escape(term) for term in terms), re.IGNORECASE)+ # Longest first so a phrase wins over a word it contains; one pass, so an+ # already-starred span is never starred again.+ pattern = re.compile("|".join(re.escape(t) for t in sorted(terms, key=len, reverse=True)), re.IGNORECASE)return pattern.sub(lambda m: f"*{m.group(0)}*", text)@@ -122,7 +405,8 @@ def _narrow_by_keywords(chunks: list[dict], keywords: str) -> list[dict]:drop keyword-less chunks.Keywords are the comma-separated terms (with close synonyms) produced by- ``formalize``; matching is case-insensitive substring.+ ``formalize``; matching is stem-tolerant (a keyword matches any word sharing+ its stem)."""kwds = [k.strip().lower() for k in (keywords or "").split(",") if k.strip()]if not kwds or not chunks:@@ -200,7 +484,9 @@ def _normalize(kbinfos: dict, tenant_ids: list[str] | str | None) -> dict:return kbinfos-async def hybrid_search(tools, query: str, kb_ids: list[str] | None = None, top_n: int = 12, doc_scope: list[str] | None = None, keywords: str = "", use_compiled: bool = False) -> dict:+async def hybrid_search(+ tools, query: str, kb_ids: list[str] | None = None, top_n: int = 12, doc_scope: list[str] | None = None, keywords: str = "", retrieval_query: str = "", use_compiled: bool = False+) -> dict:target_ids = kb_ids or list(dict.fromkeys(tools.kb_ids + [kb.id for kb in tools.sql_kbs]))if not target_ids:return {"chunks": [], "doc_aggs": []}@@ -208,9 +494,14 @@ async def hybrid_search(tools, query: str, kb_ids: list[str] | None = None, top_doc_scope = tools.scoped_doc_ids(doc_scope)_LOG.info(f'[Hybrid search] Searching the knowledge base for "{query}" (keywords: {keywords})')- # Query expansion: append the formalized-question keywords + close synonyms- # so hybrid/BM25 retrieval gets extra recall signal.- effective_query = f"{query} {keywords}".strip() if keywords else query+ # Query expansion: append the entity-weighted ``retrieval_query`` (entity+ # terms repeated so BM25 weights them up inside the same query) — the plain+ # keyword union or the compact formalize keywords fall back when none is+ # supplied. ``keywords`` is always used only to narrow retrieved chunks.+ if retrieval_query:+ effective_query = f"{query} {retrieval_query}".strip()[:400]+ else:+ effective_query = f"{query} {keywords}".strip() if keywords else query# Per-request dedup: an identical query+scope is retrieved at most once, so# e.g. pre_search and a claim search asking the same question don't repeat@@ -240,7 +531,22 @@ async def hybrid_search(tools, query: str, kb_ids: list[str] | None = None, top_must_not={"exists": "compile_kwd"}, # plain retrieval = document chunks only; compiled products have their own tools)kbinfos = _normalize(kbinfos, tools.tenant_ids)- kbinfos["chunks"] = _narrow_or_keep(kbinfos.get("chunks", []), keywords, "Hybrid search")+ # Preserve the RAW retrieved chunks in the central memory store BEFORE any+ # narrowing. search is cheap and the raw corpus may hold a fact the LLM's+ # report/grounded extraction later compresses away — a gap-driven grep over+ # memory recovers it without re-querying the knowledge base.+ try:+ from rag.advanced_rag.harness.memory import add as _memory_add++ _memory_add(tools, kbinfos.get("chunks", []) or [])+ except Exception:+ pass # memory is best-effort; never fail the search over it.+ # Narrow-or-keep: if the query keywords match any chunk, keep only the+ # matching passages (shrinking the evidence handed to the LLM so a single+ # full-context call stays small and fast); if nothing matches, keep ALL+ # chunks intact so no evidence is silently dropped. This bounds per-claim+ # analysis size without losing the numeric/entity rows when keywords hit.+ kbinfos["chunks"] = _narrow_or_keep(kbinfos.get("chunks", []), keywords, "hybrid_search")if use_compiled and kbinfos.get("chunks"):_LOG.info("[Hybrid search] Compiled expansion enabled — enriching with page_index/tree/KG navigation.")await _expand_with_compiled(tools, query, keywords, kbinfos, doc_scope)@@ -249,13 +555,13 @@ async def hybrid_search(tools, query: str, kb_ids: list[str] | None = None, top_return kbinfos-async def vector_search(tools, query: str, kb_ids: list[str] | None = None, top_n: int = 12, keywords: str = "", doc_scope: list[str] | None = None) -> dict:+async def vector_search(tools, query: str, kb_ids: list[str] | None = None, top_n: int = 12, keywords: str = "", retrieval_query: str = "", doc_scope: list[str] | None = None) -> dict:if not tools.embed_mdl:_LOG.warning("vector_search: no embed_mdl available")return {"chunks": [], "doc_aggs": []}_LOG.info(f'[Vector search] Searching by meaning for "{query}" (keywords: {keywords})')- effective_query = f"{query} {keywords}".strip() if keywords else query+ effective_query = f"{query} {retrieval_query}".strip()[:400] if retrieval_query else f"{query} {keywords}".strip() if keywords else querytarget_ids = kb_ids or tools.kb_idsif hasattr(tools, "scoped_doc_ids"):doc_scope = tools.scoped_doc_ids(doc_scope)@@ -274,14 +580,20 @@ async def vector_search(tools, query: str, kb_ids: list[str] | None = None, top_must_not={"exists": "compile_kwd"},)kbinfos = _normalize(kbinfos, tools.tenant_ids)+ try:+ from rag.advanced_rag.harness.memory import add as _memory_add++ _memory_add(tools, kbinfos.get("chunks", []) or [])+ except Exception:+ passkbinfos["chunks"] = _narrow_or_keep(kbinfos.get("chunks", []), keywords, "Vector search")return kbinfos-async def bm25_search(tools, query: str, kb_ids: list[str] | None = None, top_n: int = 12, keywords: str = "", doc_scope: list[str] | None = None) -> dict:+async def bm25_search(tools, query: str, kb_ids: list[str] | None = None, top_n: int = 12, keywords: str = "", retrieval_query: str = "", doc_scope: list[str] | None = None) -> dict:_LOG.info(f'[BM25 search] Searching by keyword for "{query}" (keywords: {keywords})')target_ids = kb_ids or tools.kb_ids- effective_query = f"{query} {keywords}".strip() if keywords else query+ effective_query = f"{query} {retrieval_query}".strip()[:400] if retrieval_query else f"{query} {keywords}".strip() if keywords else queryif hasattr(tools, "scoped_doc_ids"):doc_scope = tools.scoped_doc_ids(doc_scope)kbinfos = await settings.retriever.retrieval(@@ -299,6 +611,12 @@ async def bm25_search(tools, query: str, kb_ids: list[str] | None = None, top_n:must_not={"exists": "compile_kwd"},)kbinfos = _normalize(kbinfos, tools.tenant_ids)+ try:+ from rag.advanced_rag.harness.memory import add as _memory_add++ _memory_add(tools, kbinfos.get("chunks", []) or [])+ except Exception:+ passkbinfos["chunks"] = _narrow_or_keep(kbinfos.get("chunks", []), keywords, "BM25 search")return kbinfos@@ -437,7 +755,7 @@ async def _search_compiled_rows(embd_mdl = getattr(tools, "embed_mdl", None)if embd_mdl:try:- exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_k=top_n, num_candidates=top_n * 2, similarity=0.1))+ exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_n, 0.1))except Exception:_LOG.exception("[Compiled expand] vector build failed; using keyword match")if not exprs:@@ -477,7 +795,7 @@ async def _load_chunks_for_doc(tools, doc_id: str, chunk_ids: list[str]) -> listfrom common.misc_utils import thread_pool_execfrom rag.nlp import search- resolved = await thread_pool_exec(tools._resolve_doc_tenant, doc_id)+ resolved = tools._resolve_doc_tenant(doc_id)if not resolved:return []kb_id, tenant_id = resolved@@ -681,7 +999,7 @@ async def _search_synthesis_pages(embd_mdl = getattr(tools, "embed_mdl", None)if embd_mdl:try:- exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_k=top_n, num_candidates=top_n * 2, similarity=0.1))+ exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_n, 0.1))except Exception:_LOG.exception("[Wiki expand] vector build failed; using keyword match")if not exprs:@@ -770,7 +1088,7 @@ async def _expand_wiki_page_strategy(return new_chunks-async def web_search(tools, query: str, keywords: str = "") -> dict:+async def web_search(tools, query: str, keywords: str = "", retrieval_query: str = "") -> dict:if not tools.has_web():return {"chunks": [], "doc_aggs": []}@@ -778,7 +1096,7 @@ async def web_search(tools, query: str, keywords: str = "") -> dict:try:from common.misc_utils import thread_pool_exec- effective_query = f"{query} {keywords}".strip() if keywords else query+ effective_query = f"{query} {retrieval_query}".strip()[:400] if retrieval_query else f"{query} {keywords}".strip() if keywords else queryweb_res = await thread_pool_exec(tools.web_search.retrieve_chunks, effective_query)return {"chunks": web_res.get("chunks", []), "doc_aggs": web_res.get("doc_aggs", [])}except Exception:@@ -816,3 +1134,835 @@ async def structured_query(tools, query: str, keywords: str = "", kb_ids: list[s"chunks": ref.get("chunks") or [],"doc_aggs": ref.get("doc_aggs") or [],}+++# ─── Grep-style exact search (BM25 candidate pool + regex locate), and ───+# ─── list_chunks (deep-read a document) — used by the sub-agent so it can ───+# ─── do exact keyword/pattern locate + full-document deep-read like dynamic. ───++_GREP_TERMS_MAX = 10+_GREP_OUT_CHARS_PER_CHUNK = 700+_GREP_OUT_TOTAL_CHARS = 8000+_LIST_CHUNKS_MAX_CHUNKS = 80+++def _grep_terms_from_query(query: str, max_terms: int = _GREP_TERMS_MAX) -> list[str]:+ """Extract compact grep terms from a query: bare alnum words of length>=2,+ deduped (order-preserving) and capped. Numbers/ids are preserved as-is."""+ if not query:+ return []+ terms: list[str] = []+ seen: set[str] = set()+ for m in re.findall(r"[A-Za-z0-9][A-Za-z0-9_.\-]{1,}", query):+ t = m.strip("._-")+ if len(t) < 2:+ continue+ low = t.lower()+ if low in seen:+ continue+ seen.add(low)+ terms.append(t)+ if len(terms) >= max_terms:+ break+ return terms+++async def grep_search(+ tools,+ query: str,+ kb_ids: list[str] | None = None,+ top_n: int = 60,+ doc_scope: list[str] | None = None,+) -> dict:+ """Exact keyword/pattern locate: BM25-first candidate pool, then a+ case-insensitive regex locate with a short context window (like dynamic's+ grep_chunks, but returning the ``Pipeline`` ``{"chunks": [...]}`` contract).++ Returns compact snippet chunks (token-cheap) for the sub-agent. When grep+ matches nothing, the BM25 candidates are returned unchanged so evidence is+ never dropped (enumeration / multi-hop answers must not lose candidates).+ """+ from rag.advanced_rag.harness.grep_sed_narrow import narrow_by_terms++ _LOG.info('[Grep search] Keyword-first locate for "%s"', query)+ if not query or not str(query).strip():+ return {"chunks": [], "doc_aggs": []}+ res = await bm25_search(tools, query=str(query).strip(), kb_ids=kb_ids, top_n=top_n, doc_scope=doc_scope)+ chunks = res.get("chunks", []) or []+ terms = _grep_terms_from_query(str(query).strip())+ if not chunks or not terms:+ return res+ try:+ out = narrow_by_terms(+ chunks,+ terms,+ keywords=str(query).strip(),+ context={"before": 1, "after": 0},+ max_out_chars_per_chunk=_GREP_OUT_CHARS_PER_CHUNK,+ max_out_total_chars=_GREP_OUT_TOTAL_CHARS,+ )+ kept = out.get("kept") or []+ if kept:+ _LOG.info(+ "[Grep search] narrowed %d->%d chunk(s), %.1fK chars.",+ len(chunks),+ len(kept),+ sum(len(str(c.get("content_with_weight") or c.get("content") or "")) for c in kept) / 1000.0,+ )+ res["chunks"] = kept+ except Exception:+ _LOG.exception("[Grep search] narrow failed; using raw BM25 candidates.")+ return res+++async def list_chunks(tools, doc_id: str) -> dict:+ """Deep-read a document: return its COMPLETE chunk list in reading order.++ The sub-agent calls this after grep_search / hybrid_search hit a document+ when it needs the full text (enumeration / count / arithmetic answers), like+ dynamic's ``list_chunks`` tool. Returns ``{"chunks": [...], "doc_aggs": [...]}``.+ """+ if not doc_id or not str(doc_id).strip():+ return {"chunks": [], "doc_aggs": []}+ if not callable(getattr(tools, "fetch_full_document", None)):+ return {"chunks": [], "doc_aggs": []}+ _LOG.info("[List chunks] Deep-reading document %s", doc_id)+ try:+ full = await tools.fetch_full_document(str(doc_id).strip())+ except Exception:+ _LOG.exception("[List chunks] fetch_full_document failed doc=%s", doc_id)+ return {"chunks": [], "doc_aggs": []}+ chunks = (full.get("chunks") or [])[:_LIST_CHUNKS_MAX_CHUNKS]+ return {"chunks": chunks, "doc_aggs": full.get("doc_aggs") or []}+++# ── Rag-agent tool set (migrated from harness/dynamic) ──+_NAV_TREE_MAX_DOCS = 8++# Cap on datasets scanned per tool call (avoids fan-out over too many KBs).+_NAV_TREE_MAX_DATASETS = 10++# Chars of a chunk's text returned by search_chunks in default (snippet) mode.+_SEARCH_SNIPPET_CHARS = 300++# ── XML helpers (uniform tag vocabulary shared by all retrieval tools) ──+++def _xml_escape(value: Any) -> str:+ s = "" if value is None else str(value)+ return s.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """)+++def _chunk_text(c: dict) -> str:+ return str(c.get("content_with_weight") or c.get("content") or c.get("text") or "")+++def _chunk_attr(c: dict, keys: tuple[str, ...]) -> str:+ for k in keys:+ v = c.get(k)+ if v not in (None, ""):+ return str(v)+ return ""+++def _doc_id(c: dict) -> str:+ return _chunk_attr(c, ("doc_id", "docid", "document_id"))+++def _dataset_id(c: dict) -> str:+ return _chunk_attr(c, ("dataset_id", "kb_id", "knowledgebase_id"))+++def _doc_title(c: dict) -> str:+ return _chunk_attr(c, ("docnm_kwd", "doc_title", "title", "document_name"))+++def _chunk_id(c: dict) -> str:+ return _chunk_attr(c, ("chunk_id", "id"))+++def _snippet(s: str, n: int) -> str:+ """Truncate a string to ``n`` chars on a char boundary with an ellipsis."""+ s = (s or "").strip()+ if len(s) <= n:+ return s+ return s[:n].rstrip() + "..."+++# ── Tool: think ──+++@tool+async def _think_impl(+ thought: str,+ next_thought_needed: bool = True,+ thought_number: int | None = None,+ total_thoughts: int | None = None,+ is_revision: bool = False,+ revises_thought: int | None = None,+ branch_from_thought: int | None = None,+ branch_id: str = "",+ needs_more_thoughts: bool = False,+) -> str:+ """A stateless reflective-thinking tool that records a reasoning step.++ Use it to plan, reflect on retrieved content, and reason step by step+ before answering. It does not persist history — keep your plan in the+ conversation context. When your thinking is complete, deliver your answer+ by writing it as your plain reply and stopping (no further tool calls).+ NEVER include the final answer directly in a thought.++ IMPORTANT: The ``thought`` field must be plain natural-language reasoning+ ONLY. Do NOT embed XML tags, ``<invoke ...>``, tool names, JSON, or any+ pseudo tool-call syntax inside it. To call another tool (e.g. search_chunks),+ emit a separate tool call for that tool instead of describing it in text+ inside ``thought``.++ All arguments are optional for robustness: if the model omits the counters,+ the tool records the thought and, by default, signals that thinking continues+ (so the model keeps retrieving rather than stopping early).++ :param thought: Your current thinking step, in natural user-friendly+ language. Focus on WHAT you're trying to find and WHY, not HOW.+ :param next_thought_needed: True if you need more thinking. Defaults to True+ (conservative: keep thinking) when omitted.+ :param thought_number: Current thought number (>= 1). Optional.+ :param total_thoughts: Current estimate of thoughts needed (>= 1). Optional.+ :param is_revision: Whether this thought revises previous thinking.+ :param revises_thought: If a revision, which thought number is reconsidered.+ :param branch_from_thought: If branching, which thought number is the branch point.+ :param branch_id: Identifier for the current branch (if any).+ :param needs_more_thoughts: If reaching the end but realising more thoughts are needed.+ :return: A completion marker; does not affect the conversation content.+ """+ if not thought or not str(thought).strip():+ return "Thought process invalid: thought must be a non-empty string"+ if thought_number is not None and thought_number < 1:+ return "Thought process invalid: thought_number must be >= 1"+ if total_thoughts is not None and total_thoughts < 1:+ return "Thought process invalid: total_thoughts must be >= 1"+ # Conservative by default: without explicit counters we prefer to keep going+ # rather than stop early (avoids partial answers on multi-hop questions).+ if thought_number is None or total_thoughts is None:+ incomplete = bool(next_thought_needed or needs_more_thoughts)+ else:+ incomplete = bool(next_thought_needed or needs_more_thoughts or thought_number < total_thoughts)+ if incomplete:+ return "Thought process recorded - unfinished steps remain, continue exploring and calling tools"+ return "Thought process recorded"+++# ── Tool: todo_write ──+++@tool+async def _todo_write_impl(task: str = "", steps: list | None = None) -> str:+ """Create and manage a structured task list for retrieval and research tasks.++ Use it to track progress across 3+ retrieval/research steps. It is+ stateless: the model maintains the plan within the conversation context.++ CRITICAL - Focus on Retrieval Tasks Only: track RETRIEVAL and RESEARCH+ tasks (searching datasets, retrieving documents, gathering information).+ Do NOT include summary or synthesis tasks here — those are for the think+ tool. Examples: "Search for X in dataset", "Retrieve information about Y".+ Exclude: "Summarize findings", "Generate final answer".++ :param task: Short task title.+ :param steps: List of {"id","description","status"} where status is one of+ "pending" / "in_progress" / "completed". Limit to ONE in_progress at a+ time.+ :return: The current task list echoed back.+ """+ entries = steps if isinstance(steps, list) else []+ rendered = [str(task).strip()] if (task or "").strip() else []+ for i, s in enumerate(entries):+ if not isinstance(s, dict):+ continue+ sid = str(s.get("id") or i + 1)+ desc = str(s.get("description") or "").strip()+ status = str(s.get("status") or "pending")+ rendered.append(f" [{status}] #{sid}: {desc}")+ if not rendered:+ return "No tasks recorded."+ return "\n".join(rendered)+++# ── Tool: grep_chunks ──+++@tool(timeout=120)+async def _grep_chunks_impl(+ query: str,+ chunk_ids: list[str] | None = None,+ dataset_ids: list[str] | None = None,+ doc_scope: list[str] | None = None,+ tools=None,+) -> str:+ """Search dataset chunk content with a single POSIX regular expression,+ case-insensitive (behaves like grep -E -i).++ Pack multiple concepts into ONE regex using | alternation — do not call+ this tool repeatedly for synonyms. Keep the regex BROAD: use bare keywords+ and names combined with |, and do NOT anchor it to a specific subject/verb+ chain (e.g. prefer "何进|董太后|鸩杀|斩|诛" over "何进.*斩").++ When navigate_structure returned an outline with ``[chunks: c100,c101]``+ pointers, pass those chunk_ids here to grep ONLY within those chunks (precise+ locate inside a document's compiled-structure spans) instead of the whole+ dataset. doc_scope must then carry the owning document id(s).++ Returns an XML <grep_results> document. Each matching chunk is a <chunk>+ element with chunk_id, doc_id (owning document id), doc_title, dataset_id,+ page_num, chunk_index and score attributes, and a <match_snippet> element —+ a SHORT window (~80 chars on each side of the first match), NOT the full+ chunk text. The snippet is for fast relevance judgement only. To read a+ located document's complete text, call list_chunks with the returned doc_id.++ :param query: REQUIRED: a single POSIX regex applied to chunk content+ (case-insensitive). Combine multiple concepts with | in ONE regex.+ :param chunk_ids: Optional list of chunk ids to restrict the grep to (from a+ navigate_structure outline's ``[chunks: ...]`` pointers). When given,+ grep runs only inside those chunks of doc_scope's documents.+ :param dataset_ids: Optional dataset ids to restrict the search to (at most+ 10). When omitted, the conversation's bound datasets are used.+ :param doc_scope: Optional document ids to restrict the search to (at most 10).+ :return: XML <grep_results> document.+ """+ if not query or not str(query).strip():+ return '<grep_results count="0">\n</grep_results>'+ q = str(query).strip()+ # Validate the regex up front so a malformed pattern is reported, not silently skipped.+ try:+ re.compile("(?i)" + q)+ except re.error as e:+ return f'<grep_results count="0" error="invalid regex: {_xml_escape(str(e))}">\n</grep_results>'++ from common import settings++ if not getattr(settings, "retriever", None):+ return '<grep_results count="0" error="no retriever">\n</grep_results>'++ _t = tools or _tools_slot()+ target_ids = dataset_ids or list(dict.fromkeys(_get_kb_ids(_t)))+ if not target_ids:+ return '<grep_results count="0" error="no bound datasets">\n</grep_results>'++ # Precise path: grep ONLY within the chunk_ids flagged by navigate_structure's+ # outline pointers (inside doc_scope's documents). No full-dataset retrieval.+ if chunk_ids:+ candidates = await _load_specific_chunks(_t, chunk_ids, doc_scope)+ if not candidates:+ return '<grep_results count="0" error="none of the given chunk_ids found">\n</grep_results>'+ else:+ # Candidate pool: keyword-first retrieval (BM25) gives the widest recall+ # for exact identifiers / names / error codes, then narrow_by_terms does+ # the actual regex locate + context-window extraction (zero extra LLM).+ candidates = []+ try:+ res = await bm25_search(_t, q, kb_ids=target_ids, top_n=60, doc_scope=doc_scope)+ candidates = res.get("chunks", []) or []+ except Exception:+ _LOG.exception("[grep_chunks] bm25 candidate retrieval failed; falling back to hybrid")+ try:+ res = await hybrid_search(_t, q, kb_ids=target_ids, top_n=60, doc_scope=doc_scope)+ candidates = res.get("chunks", []) or []+ except Exception:+ _LOG.exception("[grep_chunks] hybrid fallback failed")+ return '<grep_results count="0" error="retrieval failed">\n</grep_results>'++ terms = _query_to_terms(q)+ regex = re.compile("(?i)" + q)+ if chunk_ids:+ # Strict precise grep: only chunks whose text actually matches the regex+ # are reported. (narrow_by_terms may otherwise keep non-matching context+ # chunks, which is wrong when the scope is exactly the flagged chunks.)+ matched = [c for c in candidates if regex.search(_chunk_text(c))]+ else:+ from rag.advanced_rag.harness.grep_sed_narrow import narrow_by_terms++ narrowed = narrow_by_terms(+ candidates,+ terms,+ fallback_terms=None,+ context={"before": 0, "after": 1},+ keywords=q,+ max_out_chars_per_chunk=1200,+ max_out_total_chars=16000,+ )+ matched = narrowed.get("kept", candidates)++ parts = [f'<grep_results count="{len(matched)}" query="{_xml_escape(q)}">']+ for i, c in enumerate(matched):+ cid = _chunk_id(c)+ did = _doc_id(c)+ dsid = _dataset_id(c)+ title = _doc_title(c)+ score = c.get("similarity", 0.0) or c.get("score", 0.0) or 0.0+ parts.append(f' <chunk rank="{i + 1}" chunk_id="{_xml_escape(cid)}" doc_id="{_xml_escape(did)}" dataset_id="{_xml_escape(dsid)}" doc_title="{_xml_escape(title)}" score="{float(score):.3f}">')+ snippet = _chunk_text(c)+ if len(snippet) > 240:+ snippet = snippet[:240]+ if snippet:+ parts.append(f" <match_snippet>{_xml_escape(snippet)}</match_snippet>")+ parts.append(" </chunk>")+ parts.append("</grep_results>")+ return "\n".join(parts)+++# ── Tool: search_chunks ──+++@tool(timeout=120)+async def _search_chunks_impl(+ queries: list[str],+ chunk_ids: list[str] | None = None,+ dataset_ids: list[str] | None = None,+ doc_scope: list[str] | None = None,+ top_n: int = 8,+ full_content: bool = False,+ similarity_threshold: float = 0.2,+ keywords_similarity_weight: float = 0.3,+ tools=None,+) -> str:+ """Semantic/vector search tool for retrieving knowledge by meaning, intent,+ and conceptual relevance.++ This tool uses embeddings to understand the query and find semantically+ similar content across dataset chunks. It searches by MEANING rather than+ exact text. Do NOT use it for exact keyword matching (use grep_chunks for+ that); do NOT pass long raw text or user messages as queries.++ When navigate_structure returned an outline with ``[chunks: c100,c101]``+ pointers, pass those chunk_ids here to search ONLY within those chunks+ (precise locate inside a document's compiled-structure spans). doc_scope must+ then carry the owning document id(s).++ By default returns a COMPACT snippet per hit (token-cheap) so you can judge+ relevance. When you need the full original chunk text, read it precisely with+ list_chunks (pass the doc_id + chunk_id), or set ``full_content=True`` only if+ you must have the full text in one call.++ Returns an XML <search_results> document. Each hit is a <chunk> element with+ attributes rank, chunk_id, doc_id, page_num, chunk_index, dataset_id,+ doc_title and score, and a <content> element carrying either a short snippet+ (default) or the FULL chunk text (full_content=True).++ :param queries: REQUIRED: 1-5 short, well-formed semantic questions or+ conceptual statements.+ :param chunk_ids: Optional list of chunk ids to restrict the search to (from+ a navigate_structure outline's ``[chunks: ...]`` pointers).+ :param dataset_ids: Optional dataset ids to restrict the search.+ :param doc_scope: Optional document ids to restrict the search.+ :param top_n: Per-query result count, default 8 (kept small to avoid flooding+ the history with large chunks).+ :param full_content: When True, return the FULL chunk text; default False+ returns a compact snippet per hit.+ :param similarity_threshold: Similarity threshold, default 0.2.+ :param keywords_similarity_weight: Keyword-vs-vector weight, default 0.3.+ :return: XML <search_results> document.+ """+ from common import settings++ if not getattr(settings, "retriever", None):+ return '<search_results count="0" error="no retriever">\n</search_results>'++ if not isinstance(queries, list):+ queries = [queries]+ queries = [str(q).strip() for q in queries if str(q).strip()]+ if not queries or len(queries) > 5:+ queries = queries[:5]+ if not queries:+ return '<search_results count="0" error="no queries">\n</search_results>'++ tools_slot = tools or _tools_slot()+ target_ids = dataset_ids or list(dict.fromkeys(_get_kb_ids(tools_slot)))+ if not target_ids:+ return '<search_results count="0" error="no bound datasets">\n</search_results>'++ # Precise path: search ONLY within the chunk_ids flagged by navigate_structure+ # outline pointers (inside doc_scope's documents). Rank by keyword overlap+ # (zero LLM) — the scope is small, so full hybrid search is unnecessary.+ if chunk_ids:+ candidates = await _load_specific_chunks(tools_slot, chunk_ids, doc_scope)+ if not candidates:+ return '<search_results count="0" error="none of the given chunk_ids found">\n</search_results>'+ ranked = _rank_chunks_by_terms(candidates, queries)+ merged = ranked[: max(1, int(top_n or 12)) * 2]+ else:+ seen: set[str] = set()+ merged: list[dict] = []+ for q in queries:+ try:+ res = await hybrid_search(+ tools_slot,+ q,+ kb_ids=target_ids,+ top_n=max(1, int(top_n or 12)),+ doc_scope=doc_scope,+ )+ except Exception:+ _LOG.exception("[search_chunks] hybrid_search failed for %r", q)+ continue+ for c in res.get("chunks", []) or []:+ k = _chunk_id(c)+ if k and k in seen:+ continue+ if k:+ seen.add(k)+ merged.append(c)++ # After the semantic hits, enrich with RELATED chunks discovered via each hit+ # document's COMPILED structure (zero chat-LLM, vector beam drill-down). For the+ # docs that surfaced, follow the tree/catalog hierarchy to OTHER chunks related+ # to the same query and append their snippets, so a search that lands on one+ # spot in a document also surfaces the surrounding sub-sections that answer it.+ # gated to high/ultra (the same modes that bind navigate_tree/navigate_structure)+ # so medium keeps its plain hybrid behavior.+ mode = str(getattr(tools_slot, "thinking_mode", "") or "").strip().lower()+ if merged and mode in _COMPILED_TOOL_MODES:+ hit_docs = [d for d in dict.fromkeys(_doc_id(c) for c in merged) if d]+ if hit_docs:+ for q in queries:+ related = await _expand_related_via_structure(tools_slot, q, hit_docs, seen)+ merged.extend(related)++ # Cap total chunks returned to keep the tool token-cheap (default snippet mode).+ merged = merged[: max(1, int(top_n or 8))]++ parts = [f'<search_results count="{len(merged)}">']+ for i, c in enumerate(merged):+ cid = _chunk_id(c)+ did = _doc_id(c)+ dsid = _dataset_id(c)+ title = _doc_title(c)+ page = c.get("page_num", 0) or c.get("page", 0) or 0+ cidx = c.get("chunk_index", 0) or 0+ score = c.get("similarity", 0.0) or c.get("score", 0.0) or 0.0+ related = ' related="true"' if c.get("related_via_structure") else ""+ parts.append(+ f' <chunk rank="{i + 1}" chunk_id="{_xml_escape(cid)}" doc_id="{_xml_escape(did)}" '+ f'page_num="{page}" chunk_index="{cidx}" dataset_id="{_xml_escape(dsid)}" '+ f'doc_title="{_xml_escape(title)}" score="{float(score):.3f}"{related}>'+ )+ content = _chunk_text(c)+ if content:+ if full_content:+ parts.append(f" <content>{_xml_escape(content)}</content>")+ else:+ parts.append(f" <content>{_xml_escape(_snippet(content, _SEARCH_SNIPPET_CHARS))}</content>")+ parts.append(" </chunk>")+ parts.append("</search_results>")+ return "\n".join(parts)+++# ── Tool: list_chunks ──+++@tool(timeout=120)+async def _list_chunks_impl(doc_id: str, chunk_ids: list[str] | None = None, limit: int = 20, offset: int = 0, tools=None) -> str:+ """Read the FULL original text of ONE dataset document in reading order+ (Deep Read).++ Use this AFTER grep_chunks / search_chunks / navigate_structure locate a+ document: pass its doc_id to read the complete chunk text — including+ surrounding context that grep snippets omit. When navigate_structure returned+ an outline with ``[chunks: c100,c101]`` pointers, pass those chunk_ids here to+ read ONLY those chunks (fast, precise deep-read) instead of the whole document.++ Returns an XML <chunks> document with doc_id, offset, limit and fetched+ attributes. Each chunk is a <chunk> element carrying chunk_id, doc_id,+ page_num, chunk_index, dataset_id, doc_title and a <content> element with+ the FULL original chunk text, ordered by reading order. A+ <pagination next_offset=.../> element signals that more pages remain.++ :param doc_id: REQUIRED: the document id to read (from grep_chunks /+ search_chunks / navigate_structure output).+ :param chunk_ids: Optional list of specific chunk ids to read (from a+ navigate_structure outline's ``[chunks: ...]`` pointers). When given, only+ those chunks are returned and limit/offset are ignored.+ :param limit: Page size, default 20, max 100.+ :param offset: Page offset, default 0.+ :return: XML <chunks> document.+ """+ from common import settings++ if not getattr(settings, "retriever", None):+ return f'<chunks doc_id="{_xml_escape(doc_id)}" error="no retriever">\n</chunks>'+ doc_id = str(doc_id or "").strip()+ if not doc_id:+ return '<chunks doc_id="" error="doc_id is required">\n</chunks>'+ limit = max(1, min(int(limit or 20), 100))+ offset = max(0, int(offset or 0))++ tools_slot = tools or _tools_slot()+ try:+ full = await tools_slot.fetch_full_document(doc_id)+ except Exception:+ _LOG.exception("[list_chunks] fetch_full_document failed for doc_id=%s", doc_id)+ return f'<chunks doc_id="{_xml_escape(doc_id)}" offset="{offset}" limit="{limit}" fetched="0">\n</chunks>'++ chunks = full.get("chunks", []) or []++ # Precise deep-read: only the requested chunk ids (from navigate_structure+ # outline pointers). Ignores limit/offset.+ if chunk_ids:+ wanted = [str(c).strip() for c in chunk_ids if str(c).strip()]+ by_id = {(_chunk_id(c) or ""): c for c in chunks}+ page = [by_id[cid] for cid in wanted if cid in by_id]+ if not page:+ return f'<chunks doc_id="{_xml_escape(doc_id)}" chunk_ids="{_xml_escape(",".join(wanted))}" fetched="0">\n</chunks>'+ parts = [f'<chunks doc_id="{_xml_escape(doc_id)}" chunk_ids="{_xml_escape(",".join(wanted))}" fetched="{len(page)}">']+ for c in page:+ parts.append(_render_chunk_block(c, doc_id))+ parts.append("</chunks>")+ return "\n".join(parts)++ page = chunks[offset : offset + limit]++ parts = [f'<chunks doc_id="{_xml_escape(doc_id)}" offset="{offset}" limit="{limit}" fetched="{len(page)}">']+ for c in page:+ parts.append(_render_chunk_block(c, doc_id))+ if len(page) == limit and (offset + limit) < len(chunks):+ parts.append(f' <pagination next_offset="{offset + limit}" remaining="true" />')+ parts.append("</chunks>")+ return "\n".join(parts)+++# ── Tool: calculate (safe numeric arithmetic over retrieved facts) ──+++@tool(timeout=30)+async def _calculate_impl(expression: str) -> str:+ """Safely evaluate ONE Python arithmetic expression to compute a number the+ retrieved facts imply but no source states outright (an age, an age+ difference, a percentage, a difference of speeds after unit conversion, a+ rate times a count).++ Use it for numeric answers. When the answer needs arithmetic (e.g. "how+ much younger", "what percentage", "difference in m/s", "total dollars over+ a fast"), write ONE Python expression with every figure as a literal pulled+ EXACTLY from the retrieved facts, and let this tool compute it precisely —+ do NOT do the arithmetic by hand in prose (LLMs get it wrong often enough+ to matter). The expression is evaluated in a sandbox: only plain arithmetic+ on numbers is allowed, plus the functions abs, round, min, max, sum, len,+ int, float, sorted, letters, digit_sum and date_diff. No variables, imports,+ attributes, subscripts, or non-numeric operations.++ Examples:+ - age difference -> 2010 - 1971+ - percentage -> 100 * 2.7 / 556+ - speed difference in m/s (km/h ÷ 3.6) -> 132 / 3.6 - 50 / 21.07+ - rate × count -> 25 * 49+ - days between two dates -> date_diff("1941-07-28", "1959-07-17")+ - letters in names -> letters("Ada Lovelace", "Alan Turing")++ :param expression: A single Python arithmetic expression (numbers + operators+ + the whitelisted functions above). Every figure must come from the+ retrieved evidence — never invent or round inputs.+ :return: XML <calculation> document with the exact computed value, or an+ <error> describing why the expression was refused.+ """+ from rag.advanced_rag.harness.arithmetic import compute++ expr = str(expression or "").strip()+ if not expr:+ return '<calculation error="empty expression">\n</calculation>'+ value, err = compute(expr)+ if err:+ return f'<calculation error="{_xml_escape(err)}">\n</calculation>'+ return f'<calculation value="{_xml_escape(value)}" expression="{_xml_escape(expr)}">\n</calculation>'+++def _render_chunk_block(c: dict, doc_id: str) -> str:+ """Render one chunk as an XML <chunk> block with full <content>."""+ cid = _chunk_id(c)+ dsid = _dataset_id(c)+ title = _doc_title(c)+ page_num = c.get("page_num", 0) or c.get("page", 0) or 0+ cidx = c.get("chunk_index", 0) or 0+ lines = [f' <chunk chunk_id="{_xml_escape(cid)}" doc_id="{_xml_escape(doc_id)}" page_num="{page_num}" chunk_index="{cidx}" dataset_id="{_xml_escape(dsid)}" doc_title="{_xml_escape(title)}">']+ content = _chunk_text(c)+ if content:+ lines.append(f" <content>{_xml_escape(content)}</content>")+ lines.append(" </chunk>")+ return "\n".join(lines)+++async def _load_specific_chunks(tools_slot, chunk_ids: list[str], doc_scope: list[str] | None = None) -> list[dict]:+ """Load specific chunks (from navigate_structure outline pointers) by id.++ Fetches each ``doc_scope`` document's full chunk list and filters to the+ requested ``chunk_ids``. When ``doc_scope`` is omitted, scans all bound+ documents (expensive) — so prefer passing doc_scope with the owning doc(s).+ Zero LLM.+ """+ wanted = {str(c).strip() for c in chunk_ids if str(c).strip()}+ if not wanted:+ return []+ if not doc_scope:+ # Derive candidate docs from the tools' kb_ids' documents is not available+ # here; require doc_scope for the precise path.+ return []+ found: list[dict] = []+ seen: set[str] = set()+ for doc_id in doc_scope[:8]:+ try:+ full = await tools_slot.fetch_full_document(doc_id)+ except Exception:+ _LOG.exception("[grep_chunks] fetch_full_document failed for doc_id=%s", doc_id)+ continue+ for c in full.get("chunks", []) or []:+ cid = _chunk_id(c)+ if cid in wanted and cid not in seen:+ seen.add(cid)+ found.append(c)+ return found+++def _rank_chunks_by_terms(candidates: list[dict], queries: list[str]) -> list[dict]:+ """Rank candidate chunks by how many query terms overlap with their text.++ Zero-LLM keyword relevance for the precise ``chunk_ids`` search path (the+ scope is a handful of chunks, so a cheap overlap score suffices). Returns+ chunks sorted most-relevant-first.+ """+ # Collect significant terms from all queries.+ terms: list[str] = []+ for q in queries:+ for tok in re.findall(r"[A-Za-z0-9_]{2,}", (q or "").lower()):+ if tok not in terms:+ terms.append(tok)+ if not terms:+ return list(candidates)+ scored = []+ for c in candidates:+ text = _chunk_text(c).lower()+ hits = sum(1 for t in terms if t in text)+ if hits:+ scored.append((hits, c))+ scored.sort(key=lambda x: x[0], reverse=True)+ return [c for _, c in scored]+++# The dynamic runner sets this module slot to the active RAGTools instance for+# the current request, so the tools above (defined once at import) read the+# request-scoped retrieval context.+_tools_ref: dict[str, Any] = {}+++def _tools_slot():+ return _tools_ref.get("tools")+++def _get_kb_ids(tools_slot) -> list[str]:+ if tools_slot is None:+ return []+ ids = getattr(tools_slot, "kb_ids", None) or []+ return list(ids)+++def _query_to_terms(query: str) -> list[str]:+ """Derive plain grep terms from a regex query for narrow_by_terms.++ Splits the alternation into its constituents (capped) so the regex locate+ and the term-based context window both fire on the same broad set of names.+ """+ q = (query or "").strip()+ if not q:+ return []+ # Strip regex anchors/quantifiers that are meaningless as grep terms.+ q = re.sub(r"^\(\?i\)", "", q)+ q = q.replace("\\b", "").replace("(?i)", "")+ parts = re.split(r"\|", q)+ terms = []+ for p in parts:+ p = re.sub(r"[.*+?^$()\[\]{}]", " ", p).strip()+ if not p:+ continue+ for tok in re.split(r"\s+", p):+ tok = tok.strip()+ if tok and len(tok) >= 2 and tok not in terms:+ terms.append(tok)+ if len(terms) >= 16:+ break+ return terms+++def _base_chat_mdl(tools_slot):+ """Return the concrete model instance that runs the tool-calling loop.++ The chain is ``RAGTools.chat_mdl`` → ``CountingChatModel`` (proxy) →+ ``LLMBundle`` (tenant bundle) → ``mdl`` (a ``rag.llm.chat_model.Base``+ instance). We must bind and run the loop on the *innermost* ``Base`` object,+ because ``LLMBundle.bind_tools`` is the legacy ``(toolcall_session, tools)``+ style (gated on ``is_tools``) and does NOT expose the decorator+ ``bind_tools(tools=[...])`` style nor ``async_chat_streamly_with_tools``.+ """+ try:+ chat_mdl = getattr(tools_slot, "chat_mdl", None)+ if chat_mdl is None:+ return None+ # CountingChatModel proxy → LLMBundle bundle.+ raw = getattr(chat_mdl, "_chat_mdl", None) or chat_mdl+ # LLMBundle → innermost Base/ChatModel instance.+ mdl = getattr(raw, "mdl", None) or raw+ if mdl is None:+ return None+ # Ensure we actually landed on a tool-calling-capable object; otherwise+ # surface the chain for debugging.+ if not callable(getattr(mdl, "bind_tools", None)) or not callable(getattr(mdl, "async_chat_streamly_with_tools", None)):+ _LOG.error(+ "[dynamic] resolved model %r lacks tool loop methods (chain: chat_mdl=%r raw=%r)",+ mdl,+ chat_mdl,+ raw,+ )+ return None+ return mdl+ except Exception:+ _LOG.exception("[dynamic] failed to resolve base chat model")+ return None+++# Public tool name the model / prompt refers to, mapped from the internal+# ``@tool``-decorated function name. ``_build_openai_schema`` uses ``fn.__name__``+# (e.g. ``_grep_chunks_impl``), but the model should call the clean, documented+# names below (matching the Go port and the prompt's tool-selection guidelines).+_TOOL_NAME_BY_FUNC = {+ "_think_impl": "think",+ "_todo_write_impl": "todo_write",+ "_grep_chunks_impl": "grep_chunks",+ "_search_chunks_impl": "search_chunks",+ "_list_chunks_impl": "list_chunks",+ "_calculate_impl": "calculate",+ "_navigate_tree_impl": "navigate_tree",+ "_navigate_structure_impl": "navigate_structure",+}++# Thinking modes that enable the knowledge-compilation tools (``navigate_tree``+# + ``navigate_structure``). Mirrors the high/ultra static tool lists in+# ``harness/config.py``.+_COMPILED_TOOL_MODES = {"high", "ultra"}+++def _with_clean_names(callables: list) -> list:+ """Return ``callables`` with their schema ``function.name`` set to the clean+ public name. Re-uses the cached schema to avoid re-deriving param docs."""+ for fn in callables:+ fname = fn.__name__+ clean = _TOOL_NAME_BY_FUNC.get(fname)+ if not clean:+ _LOG.warning("[dynamic] no public name mapped for tool function %r; leaving as-is", fname)+ continue+ schema = fn.openai_schema+ schema["function"]["name"] = clean+ return callables+++async def _only_strings(stream):+ """Pass through only string items from a tool-loop stream, silently dropping+ non-string metadata (int token sentinels, provider-specific tuples)."""+ async for item in stream:+ if isinstance(item, str):+ yield itemdiff --git a/rag/advanced_rag/harness/types.py b/rag/advanced_rag/harness/types.pyindex 60d735c84d..09f976745a 100644--- a/rag/advanced_rag/harness/types.py+++ b/rag/advanced_rag/harness/types.py@@ -29,19 +29,24 @@ class RouteDecision:@dataclassclass ExecutionStrategy:label: Literal["low", "medium", "high", "ultra"]- execution_strategy: Literal["direct_search", "decompose_and_search", "agentic_research", "deep_research"]- requires_decomposition: bool- requires_agent_loop: bool- requires_sufficiency_judge: bool- requires_selective_gen: bool- allows_dynamic_claims: bool- allows_replan: bool- max_orchestrator_cycles: int- max_agent_cycles: int- max_parallel_agents: int- available_tools: list[str]- sufficiency_threshold: float- fallback_to_direct_llm: bool+ # legacy dispatch field — only ``low``'s "direct_search" is consumed by the+ # (remaining) orchestrator path; medium/high/ultra run the SCA react graph and+ # never read it.+ execution_strategy: Literal["direct_search", "decompose_and_search", "agentic_research", "deep_research"] = "direct_search"+ # Legacy research-agent / decomposition flags — not consumed by any running+ # path (medium/high/ultra run the single-agent SCA graph; low runs direct_search).+ requires_decomposition: bool = False+ requires_agent_loop: bool = False+ requires_sufficiency_judge: bool = False+ requires_selective_gen: bool = False+ allows_dynamic_claims: bool = False+ allows_replan: bool = False+ max_orchestrator_cycles: int = 1+ max_agent_cycles: int = 0+ max_parallel_agents: int = 1+ available_tools: list[str] = field(default_factory=list)+ sufficiency_threshold: float = 0.5+ fallback_to_direct_llm: bool = False# Min cross-check floor for a self-verified claim. Any claim scoring below# this becomes a hard veto (its localized evidence gap must not be averaged# away by stronger sibling claims). Default 0.5 == the cross-check pass bar.@@ -56,6 +61,14 @@ class ExecutionStrategy:# Whether the mode can force a re-investigation (RECONCILE). medium=False so# RECONCILE degrades to CONTINUE; high/ultra=True.allows_reconcile: bool = False+ # Terminal-tool shortcut: when True, the outer tool loop treats ``rag`` as a+ # terminal tool and short-circuits after its first successful call — the+ # produced (cited) answer is returned immediately instead of being fed back+ # for another outer round. Mirrors the ``_terminal`` short-circuit that+ # already exists in the streaming tool loops. When False, the outer loop+ # keeps its multi-round re-ask behaviour (research more evidence between+ # rounds). Default False keeps legacy behaviour unless a mode opts in.+ terminal_tool_shortcut: bool = False# ═══════════════════════════════════════════════════════════════@@ -63,6 +76,39 @@ class ExecutionStrategy:# ═══════════════════════════════════════════════════════════════+@dataclass+class ClaimPlan:+ """Cross-round, per-claim plan state driving iterative, plan-anchored search.++ Unlike the old `while cycle < max_cycles` re-search loop, this keeps what a+ claim has learned and what it still needs across rounds, so each round only+ searches the missing part instead of blindly re-searching everything.+ """++ # Evidence accumulated across rounds (grounded facts + numbers).+ evidence: list[str] = field(default_factory=list)+ # Structured missing information (which entity / time / enumeration item).+ missing: list[str] = field(default_factory=list)+ # Bridge entities already resolved from prior hops (e.g. ["Tulsa", "1921"]).+ resolved_bridge: list[str] = field(default_factory=list)+ # Ordered hops still to resolve (each an atomic open query; later hops may+ # reference earlier resolved values via {0}, {1}, ...).+ pending_hops: list[str] = field(default_factory=list)+ # Next retrieval target for this round (entity+time anchored), or "".+ next_target: str = ""+ # Structurally accumulated member values for an AGGREGATE enumeration claim+ # (e.g. ["Minute Maid Park 330", "American Family Field 315"]). Each round the+ # claim's query_check appends newly-confirmed members here; ``missing`` records+ # which member(s) are still outstanding. combine claims read this as the+ # complete member set to synthesize from. Enables code-level completeness+ # checking ("no members missing") instead of trusting free-text reports.+ enumerated_members: list[str] = field(default_factory=list)++ # Whether any retrieval target remains (refined / hop / next_target / missing).+ def actionable(self) -> bool:+ return bool(self.missing or self.pending_hops or self.next_target)++@dataclassclass ClaimTarget:claim_id: str@@ -72,14 +118,36 @@ class ClaimTarget:confidence: float = 0.0suggested_tools: list[str] = field(default_factory=list)agent_result: AgentResult | None = None- # Consecutive research rounds for THIS claim that stayed in `locate` and- # produced neither evidence chunks nor routed document scope. Claim-scoped- # on purpose: claims are researched in parallel over one shared- # OrchestratorContext, so a global counter would let one claim's empty- # locate rounds influence a sibling. Once it reaches- # LOCATE_EMPTY_ADVANCE_THRESHOLD, the phase still stays `locate`, but- # gating may admit `web_search` for facts the corpus lacks.- locate_empty_streak: int = 0+ # Multi-hop: an OPEN query for a bridge entity/relationship that must be+ # retrieved BEFORE this claim can be answered (e.g. "Who is Brian Bergstein's+ # employer?"). decompose resolves the prerequisite first, then uses the found+ # entity to target this claim. Empty when not multi-hop.+ prerequisite: str = ""+ # Ordered list of hops for multi-hop claims (each an atomic open query, in+ # dependency order). When present, bridge resolution walks these hops and+ # stores resolved values in `plan.resolved_bridge`. Backward compatible: when+ # empty, we fall back to the single `prerequisite`.+ prerequisites: list[str] = field(default_factory=list)+ # Reasoning structure the planner assigned to this claim:+ # flat — single-hop fact, independent+ # chain — depends on a prerequisite (bridge entity/relationship)+ # aggregate — requires enumerating all members of a set and combining+ # them (count/sum/average/max/min)+ # temporal — answer depends on a specific year/period or cross-time link+ # comparative/procedural — retained for compatibility+ claim_type: str = "flat"+ # For aggregate claims: the full set of members to enumerate (e.g. "all MLB+ # stadiums with a retractable roof"), used to guide exhaustive retrieval.+ target: str = ""+ # Pure-synthesis marker: this claim is NOT a retrievable fact — it is a+ # combine/synthesis node whose value is produced by formalize_answer from the+ # enumerated member values (via WorkflowPlan.synthesis). It must never be fed+ # to hybrid_search / query_check (a "Combine the enumerated values..." query+ # retrieves nothing meaningful). The orchestrator excludes it from search and+ # treats its completeness as dependent on its source enumeration claim.+ synth_only: bool = False+ # Cross-round plan state (evidence / missing / bridge / hops).+ plan: ClaimPlan = field(default_factory=ClaimPlan)@dataclass@@ -87,6 +155,14 @@ class WorkflowPlan:plan_type: str # direct | fact_decomposition | comparative_decomposition | procedural_decomposition | exploratory_decompositionclaims: list[ClaimTarget]max_iterations: int+ # One-sentence cross-claim synthesis instruction from the planner: how the+ # individual claim findings must be combined into the final answer (e.g. for a+ # difference question "answer = c2.year - c1.year"; for a comparison "select+ # the youngest from c1's enumerated list"; for an aggregate "sum c1's counts").+ # Empty for a single-claim / flat question where no synthesis is needed. It is+ # injected into pre_summary so formalize_answer performs the composition+ # explicitly instead of guessing it from juxtaposed claim blocks.+ synthesis: str = ""# ═══════════════════════════════════════════════════════════════@@ -172,19 +248,91 @@ class ToolResult:# ═══════════════════════════════════════════════════════════════+@dataclass+class ExecutionBook:+ """Per-run execution ledger (L1 state).++ Converges the previously-scattered per-claim bookkeeping variables+ (``attempted_queries``, ``attempted_evidence``, ``pending_queries``,+ ``refined_queries``, follow-up pool, frozen set) into ONE object that lives+ with the orchestrator run, so replan / full-redo / claim-frozen never need+ fragile ``dict.update(...)`` patches to stay in sync. Each key is a claim_id+ (or the sentinel ``""`` for the global follow-up pool).++ Lifecycle:+ * Inner-loop retry and incremental replan MUTATE it in place (same plan).+ * FULL plan re-do REPLACES the per-claim entries for dropped claims while+ preserving ``frozen`` and the global follow-up pool (verified evidence+ lives in ``kbinfos``/``claim.agent_result``, not here).+ """++ # Queries already issued per claim (dedup key for the inner retry loop).+ attempted_queries: dict[str, set] = field(default_factory=dict)+ # Evidence chunk ids already analyzed per claim.+ attempted_evidence: dict[str, set] = field(default_factory=dict)+ # Targeted queries waiting to be searched for a claim (next-hops / replan).+ pending_queries: dict[str, list] = field(default_factory=dict)+ # query_check's structured "refined next query" per claim (smartsearch).+ refined_queries: dict[str, str] = field(default_factory=dict)+ # SCA / AutoRater global follow-up pool (consumed then cleared).+ followup_pool: list[dict] = field(default_factory=list)+ # Claim ids frozen by claim-level stagnation guard (never re-searched; their+ # already-collected evidence stays available for finalize).+ frozen: set = field(default_factory=set)+ # Normalized text of queries injected by the Query Rewriter / replan+ # (``_inject_replan_queries``). These are *precise, self-contained* targeted+ # queries (one-to-one for a reported gap), so the orchestrator must NOT+ # re-append the global formalize keywords when searching them — appending+ # them re-introduces the query dilution/pollution the rewrite was meant to+ # fix (see decompose._ref_kw).+ replan_query_seen: set = field(default_factory=set)++ def init_claim(self, claim_id: str) -> None:+ """Idempotently create an empty ledger entry for ``claim_id``."""+ self.attempted_queries.setdefault(claim_id, set())+ self.attempted_evidence.setdefault(claim_id, set())+ self.pending_queries.setdefault(claim_id, [])+ self.refined_queries.setdefault(claim_id, "")++ def drop_claim(self, claim_id: str) -> None:+ """Remove a claim's ledger entries (used by full plan re-do)."""+ for m in (self.attempted_queries, self.attempted_evidence, self.pending_queries, self.refined_queries):+ m.pop(claim_id, None)++@dataclassclass OrchestratorContext:question: strclaims: list[ClaimTarget]mode: striteration: int = 0+ current_phase: str = "locate"verdict: SufficiencyVerdict | None = Nonehistory: list[dict] = field(default_factory=list)- # Follow-up search queries produced by the Phase-2 LLM Sufficient Context+ # Follow-up search queries produced by the LLM Sufficient Context# AutoRater when evidence was deemed insufficient. They are consumed by the- # next research round to guide targeted follow-up search (Google's- # missing-pieces feedback), then cleared.+ # next research round to guide targeted follow-up search, then cleared.pending_followups: list[dict] = field(default_factory=list)+ # Cross-claim synthesis instruction from the planner (how claim findings+ # combine into the final answer). Injected into pre_summary by _finalize.+ synthesis: str = ""+ # Deterministic arithmetic result derived from the gathered facts (label,+ # value, expression) when the question asks for a number that follows+ # arithmetically from them. Computed by _compute_final_arithmetic and+ # injected into pre_summary so formalize_answer uses the exact value.+ computed_answer: str = ""+ # Targeted search queries produced by the Query Rewriter+ # from the SCA's forward gaps, as (claim_id, query) pairs recorded by+ # ``_try_replan`` for the newly-added claims. The orchestrator injects them+ # into the new claims' pending-query pool so they reach the retriever+ # verbatim — NOT re-synthesized by _pick_next_query — closing the "rewrite+ # got diluted" gap. Cleared after a successful replan.+ replan_queries: list[tuple[str, str]] = field(default_factory=list)+ # Number of FULL plan re-do's performed (the outer-loop-equivalent: re-running+ # the planner with SCA feedback to correct the overall decomposition direction,+ # as opposed to an incremental replan that only adds missing-claim search).+ # Bounded so a persistently-wrong plan cannot re-decompose forever.+ full_replan_count: int = 0_last_entity: str | None = None@propertydiff --git a/rag/advanced_rag/knowlege_compile/structure.py b/rag/advanced_rag/knowlege_compile/structure.pyindex 099ced6572..663149dfee 100644--- a/rag/advanced_rag/knowlege_compile/structure.py+++ b/rag/advanced_rag/knowlege_compile/structure.py@@ -2396,6 +2396,7 @@ async def _struct_upsert_graph_json(row_id = _struct_graph_row_id(doc_id, compile_kwd, compilation_template_id)row = {"id": row_id,+ "content_with_weight": json.dumps(graph, ensure_ascii=False),"compile_kwd": compile_kwd,"knowledge_graph_kwd": "graph","doc_id": doc_id,diff --git a/rag/advanced_rag/knowlege_compile/wiki.py b/rag/advanced_rag/knowlege_compile/wiki.pyindex 81c0b6d087..3a71d98fbc 100644--- a/rag/advanced_rag/knowlege_compile/wiki.py+++ b/rag/advanced_rag/knowlege_compile/wiki.py@@ -844,7 +844,7 @@ async def _wiki_scan_current_chunk_state(while True:res = await thread_pool_exec(settings.docStoreConn.search,- ["id", "doc_id", "content_with_weight", "compile_kwd"],+ ["id", "doc_id", "content_with_weight"],[],{"doc_id": [doc_id],@@ -858,18 +858,8 @@ async def _wiki_scan_current_chunk_state(index,[kb_id],)- rows = settings.docStoreConn.get_fields(res, ["id", "doc_id", "content_with_weight", "compile_kwd"]) or {}+ rows = settings.docStoreConn.get_fields(res, ["id", "doc_id", "content_with_weight"]) or {}for row_id, row in rows.items():- # The source-chunk query also has a backend-level- # ``must_not exists compile_kwd`` guard. Keep this explicit- # check because some doc-store implementations do not apply- # that nested filter consistently. Derived Wiki/structure- # rows must never become MAP inputs or chunk deltas.- compile_kwd = row.get("compile_kwd")- if isinstance(compile_kwd, (list, tuple, set)):- compile_kwd = next((str(value).strip() for value in compile_kwd if value), "")- if str(compile_kwd or "").strip():- continuechunk_id = str(row.get("id") or row_id or "")if chunk_id:state[chunk_id] = {@@ -922,39 +912,6 @@ async def _wiki_load_active_map_state(if len(rows) < 1000:breakoffset += 1000- # Active snapshots are normally source-only because they are committed- # from _wiki_scan_current_chunk_state. Older snapshots, however, may have- # been created before derived rows were excluded. If a historical state ID- # still exists in the index and carries compile_kwd, it is a compiled- # product rather than an original chunk and must not be counted as a- # deleted source chunk. Missing rows are intentionally retained: they are- # the legitimate deleted/removed source chunks we need to report.- chunk_ids = list(state)- for start in range(0, len(chunk_ids), 500):- batch_ids = chunk_ids[start : start + 500]- try:- res = await thread_pool_exec(- settings.docStoreConn.search,- ["id", "compile_kwd"],- [],- {"id": batch_ids},- [],- OrderByExpr(),- 0,- len(batch_ids),- index,- [kb_id],- )- rows = settings.docStoreConn.get_fields(res, ["id", "compile_kwd"]) or {}- except Exception:- logging.exception("wiki_map: failed to validate historical source snapshot")- continue- for row_id, row in rows.items():- compile_kwd = row.get("compile_kwd")- if isinstance(compile_kwd, (list, tuple, set)):- compile_kwd = next((str(value).strip() for value in compile_kwd if value), "")- if str(compile_kwd or "").strip():- state.pop(str(row.get("id") or row_id), None)return statediff --git a/rag/advanced_rag/tree_structured_query_decomposition_retrieval.py b/rag/advanced_rag/tree_structured_query_decomposition_retrieval.pydeleted file mode 100644index 4f2bf6461b..0000000000--- a/rag/advanced_rag/tree_structured_query_decomposition_retrieval.py+++ /dev/null@@ -1,141 +0,0 @@-#-# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-#-import asyncio-import logging-from functools import partial-from api.db.services.llm_service import LLMBundle-from common.misc_utils import hashable_key-from rag.prompts import kb_prompt-from rag.prompts.generator import sufficiency_check, multi_queries_gen-from rag.utils.web_search_conn import create_web_search_provider-from timeit import default_timer as timer---class TreeStructuredQueryDecompositionRetrieval:- def __init__(- self,- chat_mdl: LLMBundle,- prompt_config: dict,- kb_retrieve: partial = None,- kg_retrieve: partial = None,- internet_enabled: bool = False,- ):- self.chat_mdl = chat_mdl- self.prompt_config = prompt_config- self._kb_retrieve = kb_retrieve- self._kg_retrieve = kg_retrieve- self.internet_enabled = internet_enabled- self._lock = asyncio.Lock()-- async def _retrieve_information(self, search_query):- """Retrieve information from different sources"""- # 1. Knowledge base retrieval- kbinfos = {"total": 0, "chunks": [], "doc_aggs": []}- try:- kbinfos = await self._kb_retrieve(question=search_query) if self._kb_retrieve else {"total": 0, "chunks": [], "doc_aggs": []}- kbinfos.setdefault("total", 0)- except Exception as e:- logging.error(f"Knowledge base retrieval error: {e}")-- # 2. Web retrieval (if a web search provider is configured)- try:- web_search = create_web_search_provider(self.prompt_config) if self.internet_enabled else None- if web_search:- web_res = web_search.retrieve_chunks(search_query)- kbinfos["chunks"].extend(web_res["chunks"])- kbinfos["doc_aggs"].extend(web_res["doc_aggs"])- except Exception as e:- logging.error(f"Web retrieval error: {e}")-- # 3. Knowledge graph retrieval (if configured)- try:- if self.prompt_config.get("use_kg") and self._kg_retrieve:- ck = await self._kg_retrieve(question=search_query)- if ck["content_with_weight"]:- kbinfos["chunks"].insert(0, ck)- except Exception as e:- logging.error(f"Knowledge graph retrieval error: {e}")-- return kbinfos-- async def _async_update_chunk_info(self, chunk_info, kbinfos):- async with self._lock:- """Update chunk information for citations"""- if not chunk_info["chunks"]:- # If this is the first retrieval, use the retrieval results directly- for k in chunk_info.keys():- chunk_info[k] = kbinfos[k]- else:- # Merge newly retrieved information, avoiding duplicates- cids = {hashable_key(c["chunk_id"]) for c in chunk_info["chunks"]}- for c in kbinfos["chunks"]:- key = hashable_key(c["chunk_id"])- if key not in cids:- chunk_info["chunks"].append(c)- cids.add(key)-- dids = {hashable_key(d["doc_id"]) for d in chunk_info["doc_aggs"]}- for d in kbinfos["doc_aggs"]:- key = hashable_key(d["doc_id"])- if key not in dids:- chunk_info["doc_aggs"].append(d)- dids.add(key)-- chunk_info["total"] = chunk_info.get("total", 0) + kbinfos.get("total", 0)-- async def research(self, chunk_info, question, query, depth=3, callback=None):- if callback:- await callback("<START_DEEP_RESEARCH>")- try:- await self._research(chunk_info, question, query, depth, callback)- except Exception:- logging.exception("Unhandled exception in deep research for query: %s", query)- finally:- if callback:- await callback("<END_DEEP_RESEARCH>")-- async def _research(self, chunk_info, question, query, depth=3, callback=None):- if depth == 0:- # if callback:- # await callback("Reach the max search depth.")- return ""- if callback:- await callback(f"Searching by `{query}`...")- st = timer()- ret = await self._retrieve_information(query)- if callback:- await callback("Retrieval %d results in %.1fms" % (len(ret["chunks"]), (timer() - st) * 1000))- await self._async_update_chunk_info(chunk_info, ret)- ret = kb_prompt(ret, self.chat_mdl.max_length * 0.5)-- if callback:- await callback("Checking the sufficiency for retrieved information.")- suff = await sufficiency_check(self.chat_mdl, question, ret)- if suff.get("is_sufficient"):- if callback:- await callback(f"Yes, the retrieved information is sufficient for '{question}'.")- return ret-- # if callback:- # await callback("The retrieved information is not sufficient. Planing next steps...")- succ_question_info = await multi_queries_gen(self.chat_mdl, question, query, suff.get("missing_information", []), ret)- if callback:- await callback("Next step is to search for the following questions:</br> - " + "</br> - ".join(step["question"] for step in succ_question_info["questions"]))- steps = []- for step in succ_question_info["questions"]:- steps.append(asyncio.create_task(self._research(chunk_info, step["question"], step["query"], depth - 1, callback)))- results = await asyncio.gather(*steps, return_exceptions=True)- return "\n".join([str(r) for r in results])diff --git a/rag/llm/chat_model.py b/rag/llm/chat_model.pyindex 982309af11..07468d295f 100644--- a/rag/llm/chat_model.py+++ b/rag/llm/chat_model.py@@ -549,6 +549,23 @@ class Base(ABC):logging.info(f"Response tool_calls={response.choices[0].message.tool_calls}")results = await asyncio.gather(*[_exec_tool(tc) for tc in response.choices[0].message.tool_calls])+ # Terminal-tool short-circuit (mirror of the streaming+ # variant at the top of this file): a terminal tool already+ # produces the final answer, so return its result instead of+ # feeding it back for another LLM round. Without this the+ # non-streaming react loop keeps re-invoking `rag` every+ # round (Q654 spun 17 tree passes → 15 min). `rag` is always+ # terminal, so default to {"rag"} even if a probe wrapper+ # dropped the configured terminal_tools.+ _terminal = getattr(self, "terminal_tools", None) or {"rag"}+ for tc, name, args, result, err in results:+ if name in _terminal and not err:+ logging.info("[Tool loop] The %s tool produced the final answer — done.", name)+ out = result if isinstance(result, str) else json.dumps(result, ensure_ascii=False)+ if out:+ ans += out+ self.last_usage = dict(agg_usage)+ return ans, tk_counthistory = self._append_history_batch(history, results)for tc, name, args, result, err in results:ans += self._verbose_tool_use(name, args, err if err else result)@@ -2437,16 +2454,17 @@ class LiteLLMBase(ABC):# Terminal-tool short-circuit: a terminal tool already# produces the final answer, so stream its result and stop# instead of feeding it back for another LLM round.- _terminal = getattr(self, "terminal_tools", None)- if _terminal:- for tc, name, args, result, err in results:- if name in _terminal and not err:- logging.info(f"[Tool loop] The {name} tool produced the final answer — done.")- out = result if isinstance(result, str) else json.dumps(result, ensure_ascii=False)- if out:- yield out- yield total_tokens- return+ # `rag` is always terminal (dialog_service sets terminal_tools,+ # but a probe wrapper may drop it, so default to {"rag"}).+ _terminal = getattr(self, "terminal_tools", None) or {"rag"}+ for tc, name, args, result, err in results:+ if name in _terminal and not err:+ logging.info(f"[Tool loop] The {name} tool produced the final answer — done.")+ out = result if isinstance(result, str) else json.dumps(result, ensure_ascii=False)+ if out:+ yield out+ yield total_tokens+ returnhistory = self._append_history_batch(history,diff --git a/rag/nlp/search.py b/rag/nlp/search.pyindex 3839a38659..c211a17e83 100644--- a/rag/nlp/search.py+++ b/rag/nlp/search.py@@ -16,6 +16,8 @@import jsonimport loggingimport re+import threading+import timefrom collections import OrderedDict, defaultdictfrom dataclasses import dataclass@@ -46,9 +48,18 @@ def index_name(uid):class Dealer:+ # Short-lived cache of "doc_id exists in MySQL" used by _prune_deleted_chunks.+ # Every retrieval would otherwise hit MySQL per query (fan-out searches and the+ # ReAct native loop hammer the same doc_ids repeatedly), which exhausts the+ # connection pool under concurrency. Doc existence is stable within seconds, so+ # a short TTL lets us skip the DB round-trip for repeats.+ _DOC_EXISTS_TTL = 120.0+def __init__(self, dataStore: DocStoreConnection):self.qryr = query.FulltextQueryer()self.dataStore = dataStore+ self._doc_exists_cache: OrderedDict = OrderedDict()+ self._doc_exists_lock = threading.Lock()@dataclassclass SearchResult:@@ -75,13 +86,37 @@ class Dealer:return set()unique_doc_ids = list(dict.fromkeys(doc_ids))-- def _load():- from api.db.services.document_service import DocumentService-- return {row["id"] for row in DocumentService.get_by_ids(unique_doc_ids).dicts()}-- return await thread_pool_exec(_load)+ now = time.time()++ # Fast path: serve every doc_id from the short-lived cache if it is fresh.+ with self._doc_exists_lock:+ cached = {d: v for d, v in self._doc_exists_cache.items() if now - v[0] < self._DOC_EXISTS_TTL}+ hit = {d for d in unique_doc_ids if d in cached}+ miss = [d for d in unique_doc_ids if d not in cached]++ if not miss:+ return hit++ # Run the existence check on the MAIN thread against the shared peewee pool.+ # Doing it through ``thread_pool_exec`` spins up a fresh thread per call; the+ # pooled MySQL connection gets bound to that (short-lived) thread's local pool+ # and, once the thread is torn down, stays locked in the dead thread — a leak+ # that exhausts the connection pool under fan-out / ReAct concurrency (hundreds+ # of MaxConnectionsExceeded). Reusing the main thread's pool keeps connections+ # returning properly; the query itself is small and the cache makes this rare.+ from api.db.services.document_service import DocumentService++ found = {row["id"] for row in DocumentService.get_by_ids(miss).dicts()}++ # Merge results; a missing doc is recorded as False so repeat queries skip it too.+ with self._doc_exists_lock:+ for d in miss:+ self._doc_exists_cache[d] = (now, d in found)+ # Bound the cache so it cannot grow unbounded across many documents.+ while len(self._doc_exists_cache) > 4096:+ self._doc_exists_cache.popitem(last=False)++ return hit.union(found)async def _prune_deleted_chunks(self, sres: SearchResult) -> SearchResult:# Temporary safety net:diff --git a/rag/prompts/action_initialize_state.md b/rag/prompts/action_initialize_state.mdnew file mode 100644index 0000000000..2b614df083--- /dev/null+++ b/rag/prompts/action_initialize_state.md@@ -0,0 +1,24 @@+You are a research strategist. Decompose the user's question into a table of FACT SLOTS that must be filled to answer it.++The question may span MULTIPLE DATASETS and/or the open WEB. The available data sources are listed in the user message:++- "Dataset '<name>'" — slots whose facts live in that corpus. If more than one dataset is listed, create at least one slot PER DATASET (its facts may need corpus-specific phrasing); cross-referencing between datasets is encouraged when the question spans them.+- "Web" — when listed, the open web is a source: create a dedicated slot (type "web") for facts that are current-world knowledge, recent events, or simply not covered by the listed datasets.++Output ONLY a JSON object:+{+ "slots": [+ {"id": 0, "type": "<entity|person|date|duration|count|number|place|web|dataset>", "clues": ["<what identifies this slot from the question>", "..."], "source": "<dataset name or 'web'>"},+ ...+ ],+ "answer_slot": <id of the slot the FINAL answer goes to>,+ "first_queries": ["<concrete searchable query for the first retrieval round>", ...]+}++Rules:+- 2-6 slots; each slot ONE fact (a name, a date, a count...), never a clause.+- "answer_slot" holds the top-level requested fact; other slots are its dependencies.+- Cover EVERY listed source: one or more slots per dataset, plus a "web" slot when Web is available and the question touches world knowledge or recent events.+- clues must be self-contained phrases usable as retrieval hints.+- 1-4 first_queries: direct keyword-style searches against the listed sources.+- No prose outside JSON.diff --git a/rag/prompts/action_run.md b/rag/prompts/action_run.mdnew file mode 100644index 0000000000..39bfabaf00--- /dev/null+++ b/rag/prompts/action_run.md@@ -0,0 +1,32 @@+You are a deep research assistant working INSIDE a bounded search tree.++Input: the user message contains ONE research `Direction`, plus the current+`State` (slot table with immutable ids and mutable candidate fields).++Execute this single direction. Return EXACTLY ONE of:++1) TOOL CALL MODE — call `retrieve` (the only bound tool) with 1-3 corpus queries.+ Results arrive in the next turn.++2) STATE PATCH MODE — when you have partial updates, emit:+<state>+{"new_states": [+ {"state": [{"id": <int>, "candidate": "<value>", "candidate_strength": <0..1>, "discovered_clues": ["..."]}, ...]},+ ...more branches allowed...+]}+</state>+Rules: patch ONLY existing ids; include ONLY changed variables; every change must trace to retrieved evidence; candidate_strength semantics: proven >0.9, strong 0.7-0.9, tentative 0.4-0.7, weak <0.4. An EMPTY branch list (`"new_states": []`) signals no progress — emit it rather than calling tools forever.++3) FINAL ANSWER MODE — only when ALL slots can be filled consistently:+<answer>+{"answer": "<final answer text>", "new_state": [{"id": ..., "candidate": ..., "candidate_strength": ...}]}+</answer>++CRITICAL RULES+- Think before choosing a mode, but output exactly ONE mode per response.+- Strength >0.7 on the answer slot means you MUST emit final answer instead of another state patch.+- ALWAYS end this action with a state patch: a patch with your updates, or `<state>{"new_states": []}</state>` if you found nothing new.+- **Do NOT keep calling tools once the direction is reasonably exhausted.** If further searches return repetitive, irrelevant, or empty results, immediately return a state patch (with updates or empty). Extra redundant searches waste the session — stop after 1-2 useful tool calls per direction unless a NEW fact is actually emerging.+- ACTION COMPLETION IS MANDATORY: when you have what you need (or hit a dead end), output the state patch now. Do not ask to continue searching.+- Unverifiable candidates must be eliminated (set candidate null) with a clue documenting why.+- Partial verification is OK: record a candidate at tentative strength (0.4-0.7) if you can't fully verify it yet, and move on.diff --git a/rag/prompts/grounded_select.md b/rag/prompts/grounded_select.mddeleted file mode 100644index eddc5bf9bb..0000000000--- a/rag/prompts/grounded_select.md+++ /dev/null@@ -1,44 +0,0 @@-You are an answer-groundedness reviewer. For each claim's report (the draft answer), determine whether every assertion is supported by the provided evidence.--A claim's report is GROUNDED only if each of its assertions can be inferred from the evidence. The assertion does NOT need to use the same words as the evidence (semantic paraphrase is fine), but it must NOT:-- assert a fact that is absent from the evidence (likely model prior-injection / hallucination);-- assert a relation or value that contradicts the evidence;-- over-claim beyond what the evidence supports (e.g. the evidence only says a medication was prescribed, but the report claims "the patient is well").--Question: {{ question }}--Claim reports to verify:-{{ reports }}--Evidence (each chunk labeled with an integer ID):-{{ evidence }}--For each claim, classify its assertions:-- SUPPORTED: the evidence explicitly supports it, including a semantic paraphrase.-- UNGROUNDED: the evidence lacks the content, or the claimed relation/value contradicts the evidence, or the assertion over-claims beyond the evidence.--Output format (JSON):-```json-{- "claims": [- {- "claim_id": "c1",- "grounded": true,- "ungrounded_assertions": []- },- {- "claim_id": "c2",- "grounded": false,- "ungrounded_assertions": [- {"assertion": "the patient had no adverse reactions", "reason": "the evidence only mentions the medication was prescribed, not the patient's reaction"}- ]- }- ]-}-```--Requirements:-1. Include EVERY claim in the output (do not skip any claim_id).-2. `grounded` is true only if ALL of the claim's assertions are supported.-3. `ungrounded_assertions` is empty when `grounded` is true; otherwise list each ungrounded assertion with a one-line `reason`.-4. Prefer identifying genuine over-claims or contradictions over surface-level wording differences — a semantic paraphrase IS supported.diff --git a/rag/prompts/sca_query_rewrite.md b/rag/prompts/sca_query_rewrite.mdnew file mode 100644index 0000000000..4bac290600--- /dev/null+++ b/rag/prompts/sca_query_rewrite.md@@ -0,0 +1,35 @@+You are a Query Rewriter for a multi-hop RAG system. Your job is to turn the Sufficient Context Agent's missing-pieces feedback into targeted, retrievable search queries.++Original user question:+{{ question }}++Already-resolved bridge values (facts confirmed from earlier searches — use these to anchor the new query instead of re-deriving them):+{{ bridge_values }}++Research history and evidence at hand (from previous retrieval rounds):+{{ research_context }}++Missing pieces identified by the Sufficient Context Agent (each has "what" = what the answer still needs, and "hint" = a suggested search hint):+{{ gaps }}++Rewrite each missing piece into a concrete search query that the retriever can hit directly. Rules:+1. The query must name the specific missing entity + the relation/property needed (e.g. instead of "the patient's allergies", write "allergic reactions adverse events discharge John Doe").+2. MULTI-HOP: when the question is multi-hop and the missing piece is the NEXT hop's value, ANCHOR the query to the already-resolved bridge value. E.g. if the bridge value is "M*A*S*H and Cheers are the two most-watched finales" and the gap is "their runtimes", the query must be "MASH finale run time minutes" / "Cheers finale run time minutes" — not "finale run times" alone, which would lose the resolved anchor.+3. If the "what" names a specific entity (person / work / place / year) whose property is missing, anchor the query to that entity + the missing property (e.g. "MASH finale run time minutes", "Brian Bergstein employer company").+4. If a disambiguation is needed, add the distinguishing qualifier ONLY when the evidence actually suggests it (e.g. a disambiguation "the heritage 341 London Broncos player" when the claim is about that specific player). CRITICAL: NEVER introduce an entity/relation/attribute that the evidence and the missing piece do NOT suggest, and NEVER re-infer a category from the question alone (e.g. do NOT write "Ron Hutchinson football player" unless the evidence actually identifies him as a footballer — he may be a hockey player; the query must stay faithful to what the missing piece says).+5. Keep each query standalone and searchable — do NOT use pronouns ("he", "it", "this") — repeat the key entity explicitly.+6. ONE QUERY PER MISSING PIECE. Do not emit multiple near-duplicate queries for the same gap ("What teams"/"For each team"/"Complete enumeration" of the same thing are duplicates — keep exactly one, the most concretely anchored). The number of output queries should equal the number of distinct, genuinely-different missing pieces.+7. Drop any missing piece that cannot be turned into a searchable query.+8. DIVERSITY: consult the "Research history and evidence at hand" section. Do NOT output any query that paraphrases an already-tried one (listed there WITH its outcome — a previous search that yielded nothing new means that angle is dead). Aim each new query at aspects the current evidence does NOT yet cover, possibly combining the bridge values with different entity/relation combinations.++Output format (JSON):+```json+{+ "queries": [+ {"query": "concrete search query 1"},+ {"query": "concrete search query 2"}+ ]+}+```++Return a strict JSON object with no commentary before or after.diff --git a/rag/prompts/sca_select.md b/rag/prompts/sca_select.mdnew file mode 100644index 0000000000..2e70cc72d0--- /dev/null+++ b/rag/prompts/sca_select.md@@ -0,0 +1,106 @@+You are Google-style "Sufficient Context Agent" — a quality-control inspector for a multi-hop RAG retrieval loop. In ONE pass you perform a unified three-part review of (1) the retrieved snippets, (2) each claim's intermediate draft, and (3) what is still missing — then you decide whether the context is sufficient to answer the original question.++## Input++Question:+{{ question }}++OVERALL INTERMEDIATE DRAFT — a problem-level candidate answer assembled from the claims' reports (each claim already answered its sub-question as concretely as the evidence allows). Review THIS as the thing that would become the final answer:+{{ overall_draft }}++Per-claim reports (the intermediate drafts that produced the overall draft above):+{{ claims_context }}++## Part A — Sufficient Context (global)++Determine whether the retrieved content is sufficient to answer the user's question, using the Sufficient Context paper's autorater criterion:++Sufficient Context = 1 IF the CONTEXT is sufficient to INFER the answer to the question, and 0 IF the CONTEXT cannot be used to INFER the answer to the question. "A diligent reader could craft a definitive answer using only the supplied text." The answer does NOT need to be proven correct or match a ground truth — only that a reasonable answer can be constructed from the context. Multi-hop reasoning is allowed, but leaps of faith are not; ambiguities must be resolved inside the snippet bundle.++The CONTEXT here is the OVERALL INTERMEDIATE DRAFT (the problem-level candidate answer assembled from the claims) plus the per-claim reports. Judge whether THIS draft lets a diligent reader construct the answer to the ORIGINAL question end-to-end — including any cross-claim synthesis (e.g. two claims that must be COMBINED into one derived answer, or an enumeration the claims only partially cover). If the draft answers the question (even partially), mark sufficient; if the question cannot be answered from the draft at all, mark insufficient and say exactly what is missing (Part C).++MANDATORY three-stage reasoning (this is what makes the judgement reliable — follow it strictly):+1. FIRST, write down the STEP-BY-STEP SUB-QUESTIONS you would need to answer in order to arrive at the label. Make sure to include questions about any ASSUMPTIONS implicit in the QUESTION, and include questions about any MATHEMATICAL CALCULATIONS or ARITHMETIC that would be required.+2. THEN, answer each of those sub-questions step by step, working through any required mathematical calculations or arithmetic explicitly.+3. FINALLY, use these answers to evaluate the criterion and decide `is_sufficient`. If the sub-question answers let you construct a definitive answer from the context, it is sufficient — even if the answer is not perfectly complete. Only when the sub-questions cannot be answered from the context (so NO plausible answer can be inferred) is it insufficient, and then you must fill `missing_information` with exactly what is missing (Part C).++Computed answers: when a sub-question requires arithmetic (e.g. difference, count, "how many years"), WORK THE CALCULATION yourself from the values in the context and only mark sufficient if it resolves to a definite result with the RIGHT operands. If it cannot be completed, mark insufficient and list what is missing.++The STEP-BY-STEP SUB-QUESTIONS from stage 1 MUST ALSO be emitted in a structured `sub_queries` array (one entry per sub-question), because they are the precise "what is missing" signal the next search round consumes. Decomposition rules (mutually-exclusive + complete coverage — every sub-question is an atomic fact needed to answer the ORIGINAL question, sub-questions do not overlap, and together they fully cover the question):+- MULTI-STEP / multi-hop: emit ONE sub-query per logical hop, and the LAST hop is the entity/role the question actually asks for. E.g. Q="Who is the president of the team whose name was inspired by the Boston Braves?" -> [sub-query "Which team's name was inspired by the Boston Braves?", sub-query "Who is that team's president?"].+- MULTI-ASPECT / enumeration+aggregate: emit ONE sub-query for the enumeration (values of EVERY member) and ONE for the aggregate (the combination over those members). E.g. Q="average left-field distance of every stadium" -> [sub-query "left-field distance of every stadium", sub-query "average of those left-field distances"].+- ARITHMETIC / temporal: the arithmetic itself is a sub-query, and every OPERAND must have a value. If an operand is absent, that sub-query is `satisfied: false` and `missing_fact` names the absent operand specifically. E.g. Q="how many days between the two deaths?" -> [sub-query "McLean death date" (satisfied), sub-query "Meyer death date" (satisfied:false, missing_fact:"Eugene Meyer's exact death date", search_hint:"Eugene Meyer Washington Post died date"), sub-query "days between them"].++Each entry in `sub_queries` is `{"sub_query": "...", "satisfied": true/false}`; when `satisfied` is false it MUST additionally carry `missing_fact` (the concrete absent fact) and `search_hint` (a searchable query anchoring entity+attribute+time). `satisfied: false` marks exactly WHAT is missing and WHERE to search next.++## Part B — Claim-draft groundedness (per claim)++For each claim's report (the intermediate draft), determine whether it is grounded and self-consistent. Each report was generated from that claim's cited evidence by the search agent, so the main risk is NOT absent-evidence hallucination but (a) prior-knowledge padding sneaking past the "saw it in evidence" rule, (b) internal contradictions between a claim's own numbers/entities, and (c) over-claiming a derived value.++A report is GROUNDED only if it reads as a faithful, internally consistent, evidence-backed finding. Flag as UNGROUNDED anything that:+- asserts a specific number/entity/relation but contradicts another part of the same report (internal conflict);+- over-claims a computed/derived result beyond what the underlying values support;+- reads like prior-knowledge padding rather than an evidence-backed finding.++For each claim, classify the report as GROUNDED or UNGROUNDED. List each ungrounded assertion with a one-line reason.++Derived/computed assertions — APPLY ONLY IF the question asks for a DERIVED result AND the report states that computed result. Then VERIFY THE CALCULATION YOURSELF from the values in the evidence: every operand must have an explicit value in the evidence AND be the RIGHT entity; recompute and check the report's stated result matches. If any operand's figure is absent, or the operands are the wrong entities, or the recomputed result disagrees, the computed assertion is UNGROUNDED (name the missing/wrong operand or the recomputed vs stated value).++## Part C — Missing pieces (forward gap)++`missing_information` = what the answer STILL needs but the evidence does NOT contain, with a searchable hint. This is the targeted re-search signal (Google Query Rewriter input). It is distinct from `ungrounded_assertions`:+- `ungrounded_assertions` = the draft asserted something the evidence does NOT support -> drop/correct it.+- `missing_information` = the answer REQUIRES a fact/entity/value the evidence lacks -> search for it.++Populate `missing_information` when, even if every draft is grounded, the evidence does NOT cover a part the question asks for. Examples:+- the question asks for a specific property (age / year / employer / distance / count) and the evidence has the entity but not that property;+- a disambiguation is needed (evidence has a same-named but wrong entity);+- the evidence covers only part of an enumeration (some list members present, others absent).++Each `missing_information` item MUST carry a concrete `search_hint`: a searchable query (keywords / qualifiers / the specific entity+relation) that would retrieve the missing fact. Leave empty when the evidence is complete.++## Output format (JSON) — ONE object, no commentary before/after++```json+{+ "is_sufficient": true,+ "confidence": 0.0,+ "required_entities": ["Entity 1"],+ "contradictions": ["conflicting figures if any"],+ "reasoning": "step-by-step sufficiency judgment",+ "sub_queries": [+ {+ "sub_query": "Which team's name was inspired by the Boston Braves?",+ "satisfied": true+ },+ {+ "sub_query": "Who is that team's president?",+ "satisfied": false,+ "missing_fact": "the team's current president",+ "search_hint": "Washington Commanders president 2024"+ }+ ],+ "claims": [+ {+ "claim_id": "c1",+ "grounded": true,+ "ungrounded_assertions": [],+ "missing_information": [+ {"what": "the specific missing fact the answer needs", "search_hint": "searchable query to retrieve it"}+ ]+ }+ ]+}+```++## Requirements++0. **The TOP-LEVEL output is ONE single JSON object.** `claims` is an array FIELD inside that object — never wrap the whole output in an array, and never output just the `claims` array. The output must start with `{` and end with `}`.+1. `is_sufficient` true iff a plausible answer can be inferred from the context (Part A). Use the paper's SUFFICIENT-CONTEXT criterion: sufficient means a plausible answer can be constructed from the context — it does NOT need to be provably correct or complete. **A missing detail that a rough/partial answer can still accommodate does NOT make the context insufficient** — if the context lets you infer ANY plausible answer, mark sufficient (the presentation layer decides whether to caveat). Only when the context cannot support ANY plausible answer is it insufficient, and then your job is to say exactly what is missing (Part C).+2. `confidence` (0-1): how confident you are in the sufficiency decision. 0.9-1.0 if clearly sufficient/fails; 0.5-0.7 if partial/ambiguous; below 0.5 if you cannot tell.+3. `contradictions`: list internally conflicting figures/statements that make a single answer ambiguous. Empty array when none.+4. `claims` is MANDATORY and must NEVER be empty when there is at least one claim draft. Include EVERY claim_id exactly once. For each claim you must output `grounded`, `ungrounded_assertions`, AND `missing_information`. Do NOT omit any claim or any field.+5. `missing_information` is the MOST IMPORTANT field when `is_sufficient` is false. It is what drives the next search. When the context is insufficient, you MUST list, per claim, the concrete facts/entities/relations that are absent but required — each with a searchable `search_hint`. An empty `missing_information` is only acceptable when the claim is fully covered. **Never output an empty `claims` array while also saying `is_sufficient: false`** — that is a contradiction and aborts the whole search. If you judge the context insufficient, your primary job is to say exactly WHAT is missing and WHERE to search next.+6. `sub_queries` is MANDATORY (may be an empty array only when there is nothing to decompose). Emit the full step-by-step sub-query set from Part A stage 1, each marked `satisfied`. A `satisfied: false` entry MUST carry `missing_fact` + `search_hint` and is the precise "what to search next" signal — the Query Rewriter turns exactly those into the next round's queries. Keep sub-queries mutually-exclusive and fully-covering (no overlap, no gap beyond the unsatisfied ones).+7. `ungrounded_assertions` empty when grounded; otherwise list each with a one-line `reason`.+8. `reasoning` should be concise and clear.diff --git a/rag/prompts/sufficiency_select.md b/rag/prompts/sufficiency_select.mdindex b4fbfc43d3..9ddb2b82cb 100644--- a/rag/prompts/sufficiency_select.md+++ b/rag/prompts/sufficiency_select.md@@ -18,6 +18,15 @@ Reasoning procedure (do this step-by-step before answering):5. Note any internally conflicting figures/statements in the context ("contradictions").6. Decide whether a plausible answer can be inferred; give your confidence in that decision.+Derived/computed answers — APPLY ONLY IF the question actually asks for a computed/derived result:+- The rules below are STRICTLY CONDITIONAL. They apply ONLY when the question explicitly asks for a DERIVED result (e.g. "how many more/fewer", "what is the difference", "how many years between", "X% of", "how much older/younger", "how many more letters", "how many times larger").+- If the question is a PLAIN FACTUAL question (e.g. "what year", "what is the name", "who was X", "when did X happen") — NOT a computation — do NOT apply these rules. Answer the sufficiency judgment normally, and do NOT mark the context insufficient merely because two figures are present without being combined. Over-conservative abstaining on factual questions must be avoided: a context that supports a single, unambiguous factual answer is SUFFICIENT even if it also contains unrelated numbers.+- HOW TO TELL: scan the question for computation intent (a difference/ratio/percentage/timespan/letter-count being requested). No computation intent → skip this section entirely.+- WHEN APPLICABLE, you MUST actually WORK THE CALCULATION yourself, step by step, from the values present in the context, and only mark the context sufficient if the computation can be carried out AND resolves to a definite result.+- WORK STEP BY STEP: (a) identify the exact operands and their source values in the context (the specific entities/figures the question references); (b) run the arithmetic; (c) confirm the result is well-defined and not ambiguous.+- This is the crux: a context that contains the numbers is NOT sufficient if the operands are the WRONG entities (e.g. using Providence/Park City when the question needs a different city pair), or if the calculation cannot be resolved. If the computation cannot be completed to a definite answer, mark `is_sufficient` false and list exactly what is missing or wrong in `missing_information`.+- Also surface ASSUMPTIONS implicit in the question: if the question implies a specific pairing/interpretation (e.g. "the director's birthplace city vs the premiere city"), check the context supports that exact interpretation before declaring sufficient.+Output format (JSON):```json{diff --git a/test/unit_test/api/db/services/test_gaussdb_dialog_sql.py b/test/unit_test/api/db/services/test_gaussdb_dialog_sql.pyindex a9b4c36ba6..aad44677da 100644--- a/test/unit_test/api/db/services/test_gaussdb_dialog_sql.py+++ b/test/unit_test/api/db/services/test_gaussdb_dialog_sql.py@@ -239,7 +239,7 @@ def _install_settings_import_stubs(monkeypatch):resolve_reference_metadata_preferences=lambda *_args, **_kwargs: {},)install_module("rag.graphrag.general.mind_map_extractor", MindMapExtractor=_Dummy)- advanced_rag = install_module("rag.advanced_rag", DeepResearcher=_Dummy)+ advanced_rag = install_module("rag.advanced_rag")advanced_rag.__path__ = []install_module("rag.advanced_rag.agentic_rag", RAGTools=_Dummy)install_module("rag.advanced_rag.knowlege_compile")diff --git a/test/unit_test/common/test_hashable_key.py b/test/unit_test/common/test_hashable_key.pyindex 851629ba2d..2554517889 100644--- a/test/unit_test/common/test_hashable_key.py+++ b/test/unit_test/common/test_hashable_key.py@@ -14,7 +14,7 @@# limitations under the License.#"""Tests for ``hashable_key``, the set-backed dedup key used by the RAG merge-paths (dataset_structure_merger, tree_structured_query_decomposition_retrieval).+paths (dataset_structure_merger).Malformed provenance values (lists/dicts where a string was expected) must beusable as set members without raising, and without merging values that aren'tactually equal."""diff --git a/test/unit_test/rag/advanced_rag/test_tree_structured_query_decomposition_retrieval.py b/test/unit_test/rag/advanced_rag/test_tree_structured_query_decomposition_retrieval.pydeleted file mode 100644index 82612188a2..0000000000--- a/test/unit_test/rag/advanced_rag/test_tree_structured_query_decomposition_retrieval.py+++ /dev/null@@ -1,195 +0,0 @@-#-# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-#-"""Regression tests for TreeStructuredQueryDecompositionRetrieval._async_update_chunk_info.--The historical implementation rebuilt ``cids``/``dids`` as a plain list on-every merge round and checked membership with ``in`` — O(n) per item, O(n^2)-per round. The fix tracks a set alongside the ordered list. These tests pin-the observable behavior (dedup + relative order across rounds) so the-quadratic pattern cannot silently come back.-"""--import pytest--from rag.advanced_rag.tree_structured_query_decomposition_retrieval import (- TreeStructuredQueryDecompositionRetrieval,-)---def _retrieval() -> TreeStructuredQueryDecompositionRetrieval:- return TreeStructuredQueryDecompositionRetrieval(chat_mdl=None, prompt_config={})---def _chunk(cid: str) -> dict:- return {"chunk_id": cid, "content_with_weight": f"content-{cid}"}---def _doc(did: str) -> dict:- return {"doc_id": did, "doc_name": f"name-{did}"}---async def test_first_round_assigns_results_directly():- retrieval = _retrieval()- chunk_info = {"total": 0, "chunks": [], "doc_aggs": []}- kbinfos = {"total": 2, "chunks": [_chunk("c1"), _chunk("c2")], "doc_aggs": [_doc("d1")]}-- await retrieval._async_update_chunk_info(chunk_info, kbinfos)-- assert chunk_info["chunks"] == kbinfos["chunks"]- assert chunk_info["doc_aggs"] == kbinfos["doc_aggs"]- assert chunk_info["total"] == 2---async def test_merge_round_dedups_against_existing_ids_and_preserves_order():- retrieval = _retrieval()- chunk_info = {- "total": 2,- "chunks": [_chunk("c1"), _chunk("c2")],- "doc_aggs": [_doc("d1")],- }- # c2 already exists (existing-ID overlap), c2 also repeats within this- # same incoming batch (duplicate-input overlap) — only c3/c4 and d2 are new.- kbinfos = {- "total": 3,- "chunks": [_chunk("c2"), _chunk("c3"), _chunk("c2"), _chunk("c4")],- "doc_aggs": [_doc("d1"), _doc("d2")],- }-- await retrieval._async_update_chunk_info(chunk_info, kbinfos)-- assert [c["chunk_id"] for c in chunk_info["chunks"]] == ["c1", "c2", "c3", "c4"]- assert [d["doc_id"] for d in chunk_info["doc_aggs"]] == ["d1", "d2"]- assert chunk_info["total"] == 5---async def test_merge_round_tolerates_unhashable_chunk_and_doc_ids():- """Malformed provenance (e.g. a chunk_id/doc_id that's a list/dict instead- of a string) must not raise TypeError just because dedup is set-backed."""- retrieval = _retrieval()- chunk_info = {- "total": 1,- "chunks": [{"chunk_id": ["nested", "id"], "content_with_weight": "c1"}],- "doc_aggs": [{"doc_id": {"bad": "id"}, "doc_name": "d1"}],- }- kbinfos = {- "total": 1,- "chunks": [{"chunk_id": ["nested", "id"], "content_with_weight": "dup"}, _chunk("c2")],- "doc_aggs": [{"doc_id": {"bad": "id"}, "doc_name": "dup"}, _doc("d2")],- }-- await retrieval._async_update_chunk_info(chunk_info, kbinfos)-- # the malformed existing entries are deduped (canonical-key based), new valid ones are appended- assert len(chunk_info["chunks"]) == 2- assert chunk_info["chunks"][1]["chunk_id"] == "c2"- assert len(chunk_info["doc_aggs"]) == 2- assert chunk_info["doc_aggs"][1]["doc_id"] == "d2"---async def test_merge_round_dedups_dicts_regardless_of_key_order():- """dict equality ignores key order, so a dict doc_id/chunk_id with the same- keys/values in a different order must still be treated as a duplicate."""- retrieval = _retrieval()- chunk_info = {- "total": 1,- "chunks": [{"chunk_id": {"a": 1, "b": 2}, "content_with_weight": "c1"}],- "doc_aggs": [],- }- kbinfos = {- "total": 1,- # same dict, keys reordered -> must dedup, not be treated as new- "chunks": [{"chunk_id": {"b": 2, "a": 1}, "content_with_weight": "dup"}, _chunk("c2")],- "doc_aggs": [],- }-- await retrieval._async_update_chunk_info(chunk_info, kbinfos)-- assert len(chunk_info["chunks"]) == 2- assert chunk_info["chunks"][1]["chunk_id"] == "c2"---async def test_merge_round_does_not_collide_string_with_malformed_repr():- """A real string chunk_id that happens to equal repr(<malformed value>)- must not be treated as a duplicate of that malformed value."""- retrieval = _retrieval()- malformed = ["nested", "id"]- lookalike = repr(malformed) # e.g. "['nested', 'id']"- chunk_info = {- "total": 1,- "chunks": [{"chunk_id": malformed, "content_with_weight": "c1"}],- "doc_aggs": [],- }- kbinfos = {- "total": 1,- "chunks": [{"chunk_id": lookalike, "content_with_weight": "c2"}],- "doc_aggs": [],- }-- await retrieval._async_update_chunk_info(chunk_info, kbinfos)-- # both must be kept -> they are not the same chunk despite the repr() collision- assert len(chunk_info["chunks"]) == 2- assert chunk_info["chunks"][1]["chunk_id"] == lookalike---async def test_merge_round_treats_nested_list_and_tuple_as_distinct():- """A list and a tuple with equal elements are never equal in Python, so a- chunk_id nesting one of each must not be deduped against the other."""- retrieval = _retrieval()- chunk_info = {- "total": 1,- "chunks": [{"chunk_id": {"ids": ["a", "b"]}, "content_with_weight": "c1"}],- "doc_aggs": [],- }- kbinfos = {- "total": 1,- "chunks": [{"chunk_id": {"ids": ("a", "b")}, "content_with_weight": "c2"}],- "doc_aggs": [],- }-- await retrieval._async_update_chunk_info(chunk_info, kbinfos)-- assert [chunk["chunk_id"]["ids"] for chunk in chunk_info["chunks"]] == [- ["a", "b"],- ("a", "b"),- ]---async def test_multiple_merge_rounds_keep_accumulating_without_duplicates():- retrieval = _retrieval()- chunk_info = {"total": 0, "chunks": [], "doc_aggs": []}-- await retrieval._async_update_chunk_info(chunk_info, {"total": 1, "chunks": [_chunk("c1")], "doc_aggs": [_doc("d1")]})- await retrieval._async_update_chunk_info(chunk_info, {"total": 1, "chunks": [_chunk("c1"), _chunk("c2")], "doc_aggs": [_doc("d1"), _doc("d2")]})- await retrieval._async_update_chunk_info(chunk_info, {"total": 1, "chunks": [_chunk("c2"), _chunk("c3")], "doc_aggs": [_doc("d2"), _doc("d3")]})-- assert [c["chunk_id"] for c in chunk_info["chunks"]] == ["c1", "c2", "c3"]- assert [d["doc_id"] for d in chunk_info["doc_aggs"]] == ["d1", "d2", "d3"]- assert chunk_info["total"] == 3