RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Refactor agentic search (#19046)
Yingfeng4c1cb6481df1
diff --git a/rag/advanced_rag/agentic_rag.py b/rag/advanced_rag/agentic_rag.pyindex 51e0bafeb2..79cf1082fd 100644--- a/rag/advanced_rag/agentic_rag.py+++ b/rag/advanced_rag/agentic_rag.py@@ -787,12 +787,21 @@ class RAGTools:)if not chunks:break+ budget_hit = Falsefor ck in chunks:num = num_tokens_from_string(str(ck["content_with_weight"]))if tokens + num > self.chat_mdl.max_length:+ budget_hit = Truebreaktokens += numcks.append(ck)+ if budget_hit:+ # Break the OUTER paging loop too. The bare `break` above only+ # exits the inner loop, so a document that hits the token budget+ # still ran all ~79 pages — pure waste, and enough of it (one+ # navigate_tree call used to fan this out over 8 documents) to+ # overwhelm the doc store.+ breakif not cks:return {"chunks": [], "doc_aggs": []}doc_name = next((c.get("docnm_kwd") or "" for c in cks if c.get("docnm_kwd")), "")diff --git a/rag/advanced_rag/agentic_rag_graph.py b/rag/advanced_rag/agentic_rag_graph.pyindex 34a6320698..b8fc238235 100644--- a/rag/advanced_rag/agentic_rag_graph.py+++ b/rag/advanced_rag/agentic_rag_graph.py@@ -489,21 +489,26 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: in# touch ``kbinfos``. All mutation happens in the main coroutine below,# so concurrent fan-out searches cannot race on the shared list.# Channel A: exact-surface collector.+ #+ # Feed the BM25 candidate pool the CONCISE entity terms, not just the full+ # sentence: a long question buries the discriminating entities (e.g.+ # "Culdect Saga") under stopwords, so generic docs outrank the specific+ # one and the proper-noun chunk never even enters the pool. Passing the+ # keyed terms as ``keywords`` makes bm25_search build+ # ``effective_query = "sentence + entity terms"`` so the discriminating+ # noun gets its own score mass.+ terms = _query_to_terms(fq)+ keyed = [t for t in terms if len(t) >= 3 and t.lower() not in _FANOUT_STOPWORDS and not t.isdigit() or (len(t) >= 4 and t.isdigit())]try:- res = await bm25_search(tools, fq, kb_ids=kb_ids, top_n=60)+ res = await bm25_search(tools, fq, kb_ids=kb_ids, top_n=60, keywords=" ".join(keyed or terms))candidates = res.get("chunks", []) or []except Exception:_LOG.warning("[rag_agent] BM25 search failed for %r", fq, exc_info=True)candidates = []kept_a: list = []- terms = _query_to_terms(fq)if candidates:try:- # Feed the matcher CONCISE entity terms instead of the full sentence:- # a long question buries the discriminating entities under stopwords,- # so generic docs outrank the specific one. Numbers are kept.- keyed = [t for t in terms if len(t) >= 3 and t.lower() not in _FANOUT_STOPWORDS and not t.isdigit() or (len(t) >= 4 and t.isdigit())]narrowed = narrow_by_terms(candidates,keyed or terms,@@ -672,8 +677,33 @@ async def _compose_answer_from_evidence(state: AgenticState, tools, token_queue:system = FINAL_ANSWER_SYSTEM.format(cite_rules=rules)# Honor the dialog-level system prompt (UI-configured) the same way the# reasoning-disabled path does — merge from upstream/main.+ #+ # The configured prompt is appended AFTER the agentic contract so that+ # presentational instructions actually take effect: FINAL_ANSWER_SYSTEM ends+ # with a "# Language" rule ("answer in the same language as the question"),+ # and when the configured prompt was prepended instead, that rule won on+ # later-write-priority and silently overrode "answer in English", "be+ # concise" and similar settings. Behaviour then differed from the+ # reasoning-disabled path, where the same prompt is the only system prompt.+ #+ # Appending alone would let a configured prompt break the contract, so the+ # closing clause re-states precedence. Note the split: LANGUAGE IS DELIBERATELY+ # LEFT OVERRIDABLE — "answer in the same language as the question" is exactly+ # the rule a user setting "answer in English" means to replace, so it must not+ # be listed as protected. What stays protected is the evidence contract:+ # citing sources, answering the exact attribute asked for, and never+ # substituting prior knowledge for missing evidence.if getattr(tools, "system_prompt", "").strip():- system = f"{tools.system_prompt.strip()}\n\n{system}"+ system = (+ f"{system}\n\n"+ "# Assistant configuration (set by the user)\n"+ f"{tools.system_prompt.strip()}\n\n"+ "Follow the configuration above for language, tone, style, format and "+ "any other presentational instruction, including where it overrides "+ "the language rule above. Where it conflicts with the citation rules, "+ "attribute fidelity, or the requirement to answer only from the "+ "provided evidence, those three take precedence."+ )parts.append(f"Evidence:\n{evidence}")user_content = "\n".join(parts)@@ -1080,8 +1110,13 @@ def build_agentic_graph(g.add_node("query_rewrite", query_rewrite)g.add_node("formalize_answer", formalize_answer)- # Slot mode owns its first-hop queries (slot-table first_queries) — the- # planning prefetch would duplicate them at full BM25+hybrid cost for ~20s+ # Prefetch is DISABLED. In slot mode the planner emits a slot table whose+ # first-hop queries are already retrieved by rag_agent's action_session, so a+ # programmatic prefetch would duplicate that retrieval at full BM25+hybrid+ # cost (~20s). Worse, the prefetch previously FILLED the snippet pool with+ # broad-fanout chunks first, so the model-driven drill had nothing left to+ # admit (observed "Drill admitted 0 for entire runs"). rag_agent retrieves on+ # its own, so the pool fills naturally from targeted per-slot searches.use_prefetch = use_fanoutg.add_edge(START, "formalize_question")diff --git a/rag/advanced_rag/harness/action_session.py b/rag/advanced_rag/harness/action_session.pyindex 23e38a2aa3..eaa32138d9 100644--- a/rag/advanced_rag/harness/action_session.py+++ b/rag/advanced_rag/harness/action_session.py@@ -39,6 +39,12 @@ _INIT_TIMEOUT_S = 45.0_ACTION_TIMEOUT_S = 75.0_SNIPPETS_PER_QUERY = 4_MAX_TOOL_RESPONSE_CHARS = 12000+# Dataset-level empty results (reason="no_structure") a compiled-structure tool+# must accumulate before it is disabled for the rest of the session. Kept above+# 1: a single empty can be SCOPED — graph_explore over a doc_scope that has no+# knowledge graph says nothing about the other documents, and disabling on the+# first empty is exactly the over-eager behaviour this replaced.+_EMPTY_STRIKES = 2# Near-duplicate search-query detection: the model re-issues the SAME intent with# many paraphrase queries (observed: 20+ "Culdcept original creator/designer/# OmiyaSoft/founder" variants in ONE session), each executed against ES even@@ -141,6 +147,40 @@ class Result:retrieved_evidence_ids: list = field(default_factory=list)+# ── Unified tool-result contract ─────────────────────────────────────────+# Every executor returns a :class:`ToolOutcome` instead of a bare+# ``(chunks, ids)`` tuple, so ``_tool_node`` can inspect WHAT happened and act+# on it. Previously the node only saw a JSON string and counted characters —+# no policy (timeout, budget, routing, disable) had anything to hook into.+OK = "ok"+EMPTY = "empty" # dataset-level: no such compiled structure exists here+MISS = "miss" # query-level: nothing matched THIS query; the tool is still valid+POOR = "poor" # produced output, but too weak to be useful+REDUNDANT = "redundant" # ran fine, but added no NEW evidence+ERROR = "error" # infra / provider failure+++@dataclass+class ToolOutcome:+ """Result of ONE tool call.++ :param payload: what the model sees (a list of passage dicts).+ :param evidence_ids: doc/chunk ids worth tracking.+ :param status: one of the constants above — the signal ``_tool_node`` acts on.+ :param reason: machine-readable cause: ``""`` / ``no_structure`` / ``no_doc``+ / ``infra`` / ``bad_args``. Only ``no_structure`` is DATASET-level and may+ disable a tool; ``no_doc`` is QUERY-level and must NOT.+ :param metrics: numeric signals for policy decisions (``hits``,+ ``new_evidence``, ``top_score``, ...).+ """++ payload: list+ evidence_ids: list = field(default_factory=list)+ status: str = OK+ reason: str = ""+ metrics: dict = field(default_factory=dict)++# ── Tool surface ─────────────────────────────────────────────────────────_RETRIEVE_TOOL_SPEC = {"type": "function",@@ -447,26 +487,23 @@ def _disable_tool(tools, name: str) -> None:returndisabled.add(name)except Exception:- _LOG.warning("[action_session] could not mark tool %r disabled", name, exc_info=True)+ _LOG.warning("[Action Session] could not mark tool %r disabled", name, exc_info=True)-def _is_empty_structure(text, xml_tag: str) -> bool:- """True when a compiled-structure executor's payload has no usable content.+def _reason_status(reason: str) -> str:+ """Map a machine-readable failure cause to a :class:`ToolOutcome` status.- A payload is "empty" not only when blank — a well-formed XML with- ``count="0"`` or an ``error=`` marker also means "no structure compiled for- this kind". Treating those as empty lets the executor disable the tool and- steer the model back to plain retrieval instead of feeding it a count="0"- outline it will keep probing.+ Single source of truth for the cause→status mapping so a tool cannot+ disagree with itself about what its own ``empty_reason`` means."""- text = (text or "").strip()- if not text or text in ("[]", "{}"):- return True- if xml_tag and f"<{xml_tag}" in text and 'count="0"' in text:- return True- if "error=" in text:- return True- return False+ if reason == "no_structure":+ return EMPTY+ if reason == "bad_args":+ return ERROR+ if reason == "infra":+ return ERROR+ # no_doc: this query reached nothing, the tool itself is fine.+ return MISS# ── JSON / terminal parsing helpers ──────────────────────────────────────@@ -596,7 +633,7 @@ def _seed_evidence(tools):return kbinfos-def _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=True):+def _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=True) -> bool:"""Register one chunk into the session output AND the shared evidence pool.Accumulating raw chunks into ``tools.kbinfos`` is what lets the downstream@@ -604,6 +641,12 @@ def _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=True):parity) — without it the research retrieves plenty but Composing reports"0 gathered passage(s)" → partial answer → outer loop re-invokes research.+ Returns True when the chunk was NEW to the shared pool. A retrieval whose+ every hit was already known surfaces as ``redundant`` — the model otherwise+ receives a normal-looking passage list and believes the search succeeded,+ which is one cause of it re-searching the same ground (the near-dup guard+ only catches paraphrased QUERIES, not fresh queries over known chunks).+Imports the chunk helpers locally to keep ``tools.search`` (and its heavydeepdoc dependency chain) out of module-import time."""@@ -611,7 +654,7 @@ def _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=True):cid = _chunk_id(c)if cid in seen:- return+ return Falseseen.add(cid)ids.append(cid)entry = {"id": str(cid), "content": (_chunk_text(c) or "")[:1200]}@@ -621,6 +664,8 @@ def _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=True):if isinstance(c, dict) and cid not in kb_seen:kb_seen.add(cid)kbinfos["chunks"].append(c)+ return True+ return Falseasync def _run_search(tools, search_fn, queries: list, top_n: int, max_q: int, **kw) -> tuple:@@ -636,6 +681,7 @@ async def _run_search(tools, search_fn, queries: list, top_n: int, max_q: int, *kb_ids = _kb_ids(tools)out, ids = [], []seen = set()+ new_evidence = 0kbinfos = _seed_evidence(tools)kb_seen = {_chunk_id(c) for c in kbinfos["chunks"] if isinstance(c, dict)}for fq in queries[:max_q]:@@ -643,21 +689,55 @@ async def _run_search(tools, search_fn, queries: list, top_n: int, max_q: int, *res = await search_fn(tools, fq, kb_ids=kb_ids, top_n=top_n, **kw)cands = res.get("chunks", []) or []except Exception:- _LOG.warning("[action_session] %s failed for %r", getattr(search_fn, "__name__", "search"), fq, exc_info=True)+ _LOG.warning("[Action Session] %s failed for %r", getattr(search_fn, "__name__", "search"), fq, exc_info=True)continuefor c in cands[:_SNIPPETS_PER_QUERY]:- _admit_evidence(kbinfos, kb_seen, c, out, ids, seen)- return out, ids+ if _admit_evidence(kbinfos, kb_seen, c, out, ids, seen):+ new_evidence += 1+ return out, ids, new_evidence++def _search_outcome(payload: list, ids: list, new_evidence: int) -> ToolOutcome:+ """Wrap a query-based retrieval result into a :class:`ToolOutcome`.++ A run that surfaced hits but admitted NOTHING NEW to the shared pool is+ ``redundant`` rather than ``ok``: the model is told so it stops re-issuing+ the same ground. The near-dup guard cannot catch this case — it compares+ QUERY text, while this compares the EVIDENCE the query produced.+ """+ if not payload:+ return ToolOutcome(payload=[], evidence_ids=[], status=MISS, reason="no_doc", metrics={"hits": 0, "new_evidence": 0})+ return ToolOutcome(+ payload=payload,+ evidence_ids=ids,+ status=REDUNDANT if new_evidence == 0 else OK,+ metrics={"hits": len(payload), "new_evidence": new_evidence},+ )-async def _exec_retrieve(tools, queries: list) -> tuple:- """Corpus search via grep_search (exact-term locate, compact snippets)."""++async def _exec_retrieve(tools, queries: list, doc_scope: list | None = None, nav_hint: str = "") -> ToolOutcome:+ """Corpus search via grep_search (exact-term locate, compact snippets).++ ``doc_scope`` restricts the search to the given documents.+ ``nav_hint`` is the nav-routed docs' summary fed as a SOFT retrieval hint+ (BM25 keywords) — it boosts routed docs WITHOUT excluding the rest of the+ corpus, realising "nav is a hint, not a constraint".+ """from rag.advanced_rag.harness.tools.search import grep_search- return await _run_search(tools, grep_search, queries, top_n=10, max_q=3)+ out, ids, new_ev = await _run_search(+ tools,+ grep_search,+ queries,+ top_n=10,+ max_q=3,+ doc_scope=doc_scope or None,+ keywords=nav_hint or None,+ )+ return _search_outcome(out, ids, new_ev)-async def _exec_search_chunks(tools, queries: list, use_compiled: bool = False) -> tuple:+async def _exec_search_chunks(tools, queries: list, use_compiled: bool = False) -> ToolOutcome:"""Semantic retrieval (hybrid vector+BM25, narrow bypass) — the react-stylechannel that finds passages sharing NO surface words with the query.@@ -669,10 +749,11 @@ async def _exec_search_chunks(tools, queries: list, use_compiled: bool = False)"""from rag.advanced_rag.harness.tools.search import hybrid_search- return await _run_search(tools, hybrid_search, queries, top_n=20, max_q=2, use_compiled=use_compiled)+ out, ids, new_ev = await _run_search(tools, hybrid_search, queries, top_n=20, max_q=2, use_compiled=use_compiled)+ return _search_outcome(out, ids, new_ev)-async def _exec_web_search(tools, queries: list) -> tuple:+async def _exec_web_search(tools, queries: list) -> ToolOutcome:"""Open-web retrieval via the configured provider (Tavily/QuerIt/Serply/YouCom). Results share the RAGFlow chunk shape so they merge into the sameevidence pool as corpus hits."""@@ -684,10 +765,16 @@ async def _exec_web_search(tools, queries: list) -> tuple:# would just retry and burn turns). Return an explicit, actionable note# so the model moves on to the corpus tools — a web failure must never# block the Q&A.- _LOG.warning("[action_session] web_search unavailable (no provider configured)")- return [{"kind": "web_search", "note": "Web search is NOT configured for this session. Do not use this tool again; use the corpus tools (retrieve / search_chunks / navigate_*) instead."}], []+ _LOG.warning("[Action Session] web_search unavailable (no provider configured)")+ return ToolOutcome(+ payload=[+ {"kind": "web_search", "note": "Web search is NOT configured for this session. Do not use this tool again; use the corpus tools (retrieve / search_chunks / navigate_*) instead."}+ ],+ status=ERROR,+ reason="infra",+ )- out, ids = [], []+ out, ids, new_ev = [], [], 0seen = set()kbinfos = _seed_evidence(tools)kb_seen = {_chunk_id(c) for c in kbinfos["chunks"] if isinstance(c, dict)}@@ -697,26 +784,32 @@ async def _exec_web_search(tools, queries: list) -> tuple:if asyncio.iscoroutine(web_res) or hasattr(web_res, "__await__"):web_res = await web_resexcept Exception:- _LOG.warning("[action_session] web_search failed for %r", q, exc_info=True)+ _LOG.warning("[Action Session] web_search failed for %r", q, exc_info=True)continuefor c in ((web_res or {}).get("chunks") or [])[:8]:cid = _chunk_id(c)if not cid or cid in seen:continue- _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=False)- return out, ids+ if _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=False):+ new_ev += 1+ return _search_outcome(out, ids, new_ev)++async def _exec_list_chunks(tools, doc_id: str) -> ToolOutcome:+ """Deep-read one document's full text (list_chunks tool).-async def _exec_list_chunks(tools, doc_id: str) -> tuple:- """Deep-read one document's full text (list_chunks tool)."""+ An unknown/blank ``doc_id`` yields no chunks rather than raising — that is a+ QUERY-level miss (``miss``), never a dataset fact, so ``_tool_node`` must not+ disable the tool over it.+ """from rag.advanced_rag.harness.tools.search import _chunk_id, list_chunkstry:res = await list_chunks(tools, doc_id)except Exception:- _LOG.warning("[action_session] list_chunks failed doc=%r", doc_id, exc_info=True)- return [], []- out, ids = [], []+ _LOG.warning("[Action Session] list_chunks failed doc=%r", doc_id, exc_info=True)+ return ToolOutcome(payload=[], status=ERROR, reason="infra")+ out, ids, new_ev = [], [], 0kbinfos = _seed_evidence(tools)kb_seen = {_chunk_id(c) for c in kbinfos["chunks"] if isinstance(c, dict)}seen = set()@@ -724,8 +817,9 @@ async def _exec_list_chunks(tools, doc_id: str) -> tuple:cid = _chunk_id(c)if not cid:continue- _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=False)- return out, ids+ if _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=False):+ new_ev += 1+ return _search_outcome(out, ids, new_ev)def _arg_query_list(args, max_q: int) -> list:@@ -750,57 +844,84 @@ def _inject_nav_tools_ref(tools) -> None:_nav._tools_ref["tools"] = toolsexcept Exception:- _LOG.warning("[action_session] could not inject navigation tools ref", exc_info=True)+ _LOG.warning("[Action Session] could not inject navigation tools ref", exc_info=True)-async def _exec_navigate_tree(tools, args: dict) -> tuple:+async def _exec_navigate_tree(tools, args: dict) -> ToolOutcome:"""Document-level embedding routing (navigate_tree). Returns doc candidates- as a text outline; evidence ids = routed doc_ids for later deep-read."""+ as a text outline; evidence ids = routed doc_ids for later deep-read.++ Classifies the payload but does NOT disable the tool — that decision belongs+ to ``_tool_node``, which owns the strike counter. Disabling here (the old+ behaviour) killed the tool for the whole session on a single hard query:+ routing falls through at ``_NAV_MIN_DOC_SCORE``, which is query-dependent,+ not a statement about the dataset.+ """from rag.advanced_rag.harness.tools.navigation import _navigate_tree_impl_inject_nav_tools_ref(tools)query = str(args.get("query") or "")- text = await _navigate_tree_impl(query, keywords=str(args.get("keywords") or ""))- # Empty / count="0" / error= payloads mean no compiled navigation tree —- # this dataset has no such structure. Disable the tool for the session and- # tell the model to fall back to plain retrieval (a count="0" XML is NOT a- # useful payload: without this, the model keeps re-invoking it).- text = (text or "").strip()- if _is_empty_structure(text, "tree_navigation"):- _disable_tool(tools, "navigate_tree")- _LOG.info("[action_session] navigate_tree disabled: dataset has no compiled navigation tree (text=%r)", text[:120])- return [{"kind": "navigate_tree", "note": "This dataset has NO compiled document navigation tree. navigate_tree is unavailable; use search_chunks / retrieve instead."}], []- return [{"kind": "navigate_tree", "content": text[:8000]}], []---async def _exec_navigate_structure(tools, args: dict) -> tuple:+ res = await _navigate_tree_impl(query, keywords=str(args.get("keywords") or ""))+ if res.empty_reason:+ # Structured signal — no XML parsing. The routed doc_ids feed the caller's+ # known-doc set, which navigate_structure / list_chunks are validated against.+ return ToolOutcome(+ payload=[{"kind": "navigate_tree", "note": "navigate_tree routed to no document for this query. Rephrase the query, or use search_chunks / retrieve instead."}],+ status=_reason_status(res.empty_reason),+ reason=res.empty_reason,+ )+ return ToolOutcome(+ payload=[{"kind": "navigate_tree", "content": (res.text or "")[:8000], "doc_ids": res.doc_ids}],+ status=OK,+ metrics={"docs": len(res.doc_ids), "routed_docs": list(res.routed_docs)},+ )+++async def _exec_navigate_structure(tools, args: dict) -> ToolOutcome:"""In-document compiled-structure navigation (navigate_structure). Returns- outline annotated with chunk_ids; those become evidence for list_chunks."""+ outline annotated with chunk_ids; those become evidence for list_chunks.++ As with navigate_tree, classification only — ``_tool_node`` decides whether+ to disable.+ """from rag.advanced_rag.harness.tools.navigation import _navigate_structure_impl_inject_nav_tools_ref(tools)doc_id = str(args.get("doc_id") or "")query = str(args.get("query") or "")kind = str(args.get("kind") or "catalog")- text = await _navigate_structure_impl(query, doc_id=doc_id, kind=kind)- # Empty / count="0" / error= payloads mean no compiled structure of this kind- # in the dataset — the tool is a dead end. Disable it for the session so the- # model stops re-invoking it (a count="0" XML is NOT a useful payload).- text = (text or "").strip()- if _is_empty_structure(text, "structure_navigation"):- _disable_tool(tools, "navigate_structure")- _LOG.info("[action_session] navigate_structure disabled: no compiled structure kind=%r (text=%r)", kind, text[:120])- return [- {- "kind": "navigate_structure",- "doc_id": doc_id,- "note": "This document/dataset has NO compiled structure of the requested kind. navigate_structure is unavailable here; use search_chunks / retrieve / list_chunks instead.",- }- ], [doc_id] if doc_id else []- return [{"kind": "navigate_structure", "doc_id": doc_id, "content": text[:8000]}], [doc_id] if doc_id else []+ res = await _navigate_structure_impl(query, doc_id=doc_id, kind=kind)+ metrics = {+ "entities": res.entities,+ "chunk_ptrs": res.chunk_ptrs,+ "top_score": res.top_score,+ "chunk_paths": res.chunk_paths,+ }+ if res.empty_reason:+ return ToolOutcome(+ payload=[+ {+ "kind": "navigate_structure",+ "doc_id": doc_id,+ "note": f"No compiled structure of kind={kind!r} reachable for this document. Try another doc_id or kind, or use search_chunks / retrieve / list_chunks.",+ }+ ],+ status=_reason_status(res.empty_reason),+ reason=res.empty_reason,+ metrics=metrics,+ )+ # Reached a structure but drilled to nothing usable: not an error, just weak —+ # the orchestrator may fall back to retrieval on ``status == poor``.+ status = POOR if res.chunk_ptrs == 0 else OK+ return ToolOutcome(+ payload=[{"kind": "navigate_structure", "doc_id": doc_id, "content": (res.text or "")[:8000]}],+ evidence_ids=res.doc_ids or ([doc_id] if doc_id else []),+ status=status,+ metrics=metrics,+ )-async def _exec_calculate(tools, args: dict) -> tuple:+async def _exec_calculate(tools, args: dict) -> ToolOutcome:"""Arithmetic terminal: compute_from_facts decides whether the questionneeds a computed number and safely evaluates an expression over facts."""from rag.advanced_rag.harness.arithmetic import compute_from_facts@@ -813,22 +934,31 @@ async def _exec_calculate(tools, args: dict) -> tuple:# NOT. Pass the outer bundle.mdl = getattr(tools, "chat_mdl", None)if mdl is None:- return [{"kind": "calculate", "error": "no model"}], []+ return ToolOutcome(payload=[{"kind": "calculate", "error": "no model"}], status=ERROR, reason="infra")try:res = await compute_from_facts(mdl, question, facts)except Exception:- _LOG.warning("[action_session] calculate failed", exc_info=True)+ _LOG.warning("[Action Session] calculate failed", exc_info=True)res = Noneif not res:# nothing derivable -> tell the model so it doesn't loop on it- return [{"kind": "calculate", "expression": None, "note": "no numeric answer derivable from given facts; answer directly or retrieve more numbers."}], []- return [{"kind": "calculate", "expression": res.get("expression"), "result": res.get("value")}], []+ return ToolOutcome(+ payload=[{"kind": "calculate", "expression": None, "note": "no numeric answer derivable from given facts; answer directly or retrieve more numbers."}],+ status=POOR,+ reason="no_doc",+ )+ return ToolOutcome(payload=[{"kind": "calculate", "expression": res.get("expression"), "result": res.get("value")}], status=OK)-async def _exec_graph_explore(tools, args: dict) -> tuple:+async def _exec_graph_explore(tools, args: dict) -> ToolOutcome:"""Relational knowledge-graph exploration (ultra-only). Returns either adirect answer or source passages behind relevant entities/relations, so the- model can continue with list_chunks/navigate on the returned chunk/doc ids."""+ model can continue with list_chunks/navigate on the returned chunk/doc ids.++ An empty result means no compiled KG in scope — a DATASET-level fact+ (``no_structure``), which is the one class of failure that may disable the+ tool. The disable itself is applied by ``_tool_node``.+ """from rag.advanced_rag.harness.tools.navigation import graph_explore_inject_nav_tools_ref(tools)@@ -837,44 +967,48 @@ async def _exec_graph_explore(tools, args: dict) -> tuple:try:res = await graph_explore(tools, query, doc_scope=doc_scope or None)except Exception:- _LOG.warning("[action_session] graph_explore failed", exc_info=True)+ _LOG.warning("[Action Session] graph_explore failed", exc_info=True)res = {}answer = str(res.get("answer") or "").strip()chunks = res.get("chunks") or []if answer:- return [{"kind": "graph_explore", "answer": answer}], []+ return ToolOutcome(payload=[{"kind": "graph_explore", "answer": answer}], status=OK, metrics={"hits": 0})if not chunks:- # No knowledge graph compiled for this dataset/scope: relational- # exploration is a dead end. Disable the tool for the session (it will- # keep coming back empty otherwise — 24 empty calls in one run) and- # guide the model back to retrieval.- _disable_tool(tools, "graph_explore")- _LOG.info("[action_session] graph_explore disabled: no compiled knowledge graph in scope")- return [- {- "kind": "graph_explore",- "note": "This dataset has NO compiled knowledge graph (or none in the given scope). graph_explore is unavailable; use search_chunks / navigate_structure / retrieve instead.",- }- ], []+ return ToolOutcome(+ payload=[+ {+ "kind": "graph_explore",+ "note": "This dataset has NO compiled knowledge graph (or none in the given scope). graph_explore is unavailable; use search_chunks / navigate_structure / retrieve instead.",+ }+ ],+ status=EMPTY,+ reason="no_structure",+ )snippet = [{"id": c.get("id"), "content": (str(c.get("content") or "")[:1500])} for c in chunks[:6]]ids = [str(c.get("id")) for c in chunks[:6] if c.get("id")]- return [{"kind": "graph_explore", "chunks": snippet}], ids+ return ToolOutcome(payload=[{"kind": "graph_explore", "chunks": snippet}], evidence_ids=ids, status=OK, metrics={"hits": len(ids)})-async def execute_tool(tools, name: str, args: dict) -> tuple:+async def execute_tool(tools, name: str, args: dict) -> ToolOutcome:"""DeepSearch ToolNode: dispatch ONE native tool call by name.- Returns ``(results, evidence_ids)`` where results is a JSON-serializable- payload and evidence_ids are any doc/chunk ids worth tracking.+ Returns a :class:`ToolOutcome` whose ``status`` / ``reason`` let the caller+ act on WHAT happened — empty payload, query miss, infra failure, or a run+ that added no new evidence — instead of only counting characters."""# Short-circuit a tool already proven unavailable this session (no compiled# structure of its kind). We still return a note, not an error, so the model# learns to switch to the corpus tools rather than loop.if name in (_TOOL_MAP and (getattr(tools, "_disabled_tools", None) or set())):- _LOG.info("[action_session] tool %r disabled (no compiled structure); returning note", name)- return [{"kind": name, "note": f"{name} is unavailable in this dataset (no compiled structure of its kind). Use search_chunks / retrieve / list_chunks instead."}], []+ _LOG.info("[Action Session] tool %r disabled (no compiled structure); returning note", name)+ return ToolOutcome(+ payload=[{"kind": name, "note": f"{name} is unavailable in this dataset (no compiled structure of its kind). Use search_chunks / retrieve / list_chunks instead."}],+ status=EMPTY,+ reason="no_structure",+ )if name == "retrieve":- return await _exec_retrieve(tools, _arg_query_list(args, 3))+ scope = [str(d) for d in (args.get("doc_scope") or []) if str(d).strip()]+ return await _exec_retrieve(tools, _arg_query_list(args, 3), doc_scope=scope or None)if name == "search_chunks":# Compiled-structure expansion (page-index/tree/wiki/KG) inside# hybrid_search is enabled in ALL modes. Datasets with NO compiled@@ -892,8 +1026,8 @@ async def execute_tool(tools, name: str, args: dict) -> tuple:return await _exec_graph_explore(tools, args)if name == "web_search":return await _exec_web_search(tools, _arg_query_list(args, 2))- _LOG.warning("[action_session] unknown tool %r; ignored.", name)- return [], []+ _LOG.warning("[Action Session] unknown tool %r; ignored.", name)+ return ToolOutcome(payload=[], status=ERROR, reason="bad_args")# ── LangGraph session graph ──────────────────────────────────────────────@@ -977,7 +1111,7 @@ def _parse_tool_calls(msg) -> list:## Models most often emit the XML protocol tags (state / answer) as# tool names; the hint below steers them back to plain-text output.- _LOG.warning("[action_session] tool_call to unknown tool %r; replying with a hint", name)+ _LOG.warning("[Action Session] tool_call to unknown tool %r; replying with a hint", name)calls.append({"id": tc.id or f"call_{i}", "name": name, "args": args, "unknown": True})continuecalls.append({"id": tc.id or f"call_{i}", "name": name, "args": args})@@ -1035,6 +1169,15 @@ class _SessionState(TypedDict, total=False):deadline_left: float_ctx_budget: int_tool_chars: int+ # per-tool policy bookkeeping: how many dataset-level empties each tool has+ # accrued, and the (status, reason, metrics) log of every executed call.+ _tool_strikes: dict+ _tool_outcomes: list+ # navigation ladder: the ladder spans the prefix AND the session (the model+ # drives the LLM rung in between), so its state has to outlive the prefix.+ _direction: str # this slot's question — needed to re-run retrieval later+ _routed_docs: list # navigate_tree's top-n: scope for the `scoped` rung+ _nav_rule_id: str # which rung control currently rests on ("" = finished)async def _run_action_node(state: _SessionState) -> dict:@@ -1049,10 +1192,10 @@ async def _run_action_node(state: _SessionState) -> dict:async with asyncio.timeout(wall):resp = await _llm_once_with_tools(state["tools"], mdl, state["messages"])except TimeoutError:- _LOG.warning("[action_session] turn timed out after %.0fs", wall)+ _LOG.warning("[Action Session] turn timed out after %.0fs", wall)return {"_done": True, "new_states": [], "found_answer": None, "attempts": attempts}except Exception as e: # noqa: BLE001 - 422 content-moderation / conn errors- _LOG.warning("[action_session] LLM call failed (%s); converging session empty", type(e).__name__)+ _LOG.warning("[Action Session] LLM call failed (%s); converging session empty", type(e).__name__)return {"_done": True, "new_states": [], "found_answer": None, "attempts": attempts}msg = resp.choices[0].messagecontent = msg.content or ""@@ -1129,6 +1272,13 @@ async def _tool_node(state: _SessionState) -> dict:pending = state.get("_pending_calls") or []seen_queries = list(state.get("_search_queries") or [])skipped = 0+ strikes = dict(state.get("_tool_strikes") or {})+ outcomes = list(state.get("_tool_outcomes") or [])+ # Where the nav ladder currently rests. Bound BEFORE the loop so it survives+ # the empty-pending case (UnboundLocalError) and is the input for every+ # tool_call that comes in — a call only advances its own rung, and a later+ # ladder continuation must continue from the SAME resting point.+ pending_rule = state.get("_nav_rule_id", "")for c in pending:# Near-duplicate retrieval suppression: if the model re-issues the same# intent as an earlier search (paraphrase), do NOT re-run ES — return a@@ -1137,7 +1287,7 @@ async def _tool_node(state: _SessionState) -> dict:q = str((c.get("args") or {}).get("query") or "").strip()if c["name"] in _RETRIEVAL_TOOLS and q and _is_near_dup(q, seen_queries):skipped += 1- _LOG.info("[action_session] skipping near-duplicate retrieval %r (already searched)", q[:80])+ _LOG.info("[Action Session] skipping near-duplicate retrieval %r (already searched)", q[:80])tool_msgs.append({"role": "tool",@@ -1177,13 +1327,42 @@ async def _tool_node(state: _SessionState) -> dict:# same-session tool cache: avoid re-grep/list_chunks on the same targetcache_key = (c["name"], json.dumps(c["args"], sort_keys=True, ensure_ascii=False))if cache_key in cache:- chunks, ids = cache[cache_key]+ oc = cache[cache_key]else:- chunks, ids = await execute_tool(tools, c["name"], c["args"])- cache[cache_key] = (chunks, ids)+ oc = await execute_tool(tools, c["name"], c["args"])+ cache[cache_key] = ocif q:seen_queries.append(q)- evidence_ids.extend(ids)+ evidence_ids.extend(oc.evidence_ids)+ chunks = list(oc.payload or [])+ # ── Policy: act on WHAT happened, not just on payload size ──────────+ if oc.status == OK:+ # A real hit clears the tool's strike record: it demonstrably works.+ strikes.pop(c["name"], None)+ elif oc.status == EMPTY and oc.reason == "no_structure":+ # Dataset-level dead end. Strike it; disable once the strikes pile up+ # so one unlucky scope cannot kill the tool (graph_explore over a+ # doc_scope without a KG says nothing about the other documents).+ n = strikes.get(c["name"], 0) + 1+ strikes[c["name"]] = n+ if n >= _EMPTY_STRIKES:+ _disable_tool(tools, c["name"])+ _LOG.info("[Action Session] %s disabled after %d dataset-level empty results", c["name"], n)+ elif oc.status == REDUNDANT:+ # Ran fine, but every hit was already in the shared evidence pool.+ # Say so explicitly — otherwise the model sees a normal passage list+ # and concludes the search succeeded, then re-searches the same ground.+ chunks.append(+ {+ "kind": c["name"],+ "note": "All hits from this call were ALREADY in your evidence. Re-reading them will not add anything — issue a genuinely new angle, or output a <state> patch with what you have.",+ }+ )+ # ── Ladder: did this call satisfy the rung the model was asked to do? ──+ # (drill is AUTO now — it runs its own retrieve+structure+merge chain in+ # the prefix — so there is no model-driven navigate_structure rung here.)+ status, reason = oc.status, oc.reason+ outcomes.append({"name": c["name"], "status": status, "reason": reason, "metrics": oc.metrics})payload = json.dumps({"passages": chunks}, ensure_ascii=False, default=str)# if the session is already heavy, cut this payload proportionallyif used + len(payload) > budget_chars:@@ -1197,6 +1376,49 @@ async def _tool_node(state: _SessionState) -> dict:"content": payload,})+ # Continue the ladder in code when the model's rung came back weak. Keep+ # going on the rule's own verdict so one weak step can cascade through the+ # remaining (cheaper, wider) rungs instead of leaving the model to rediscover+ # the fallback one turn at a time.+ if pending_rule:+ rule = _NAV_RULE_BY_ID.get(pending_rule)+ if rule is not None:+ nxt = rule.next.get(status, "")+ if nxt:+ ctx = _NavContext(+ direction=state.get("_direction", ""),+ known_docs=list(state.get("_routed_docs") or []),+ )+ ladder_budget = max(5.0, (state.get("deadline_left") or _ACTION_TIMEOUT_S) * _NAV_PREFIX_BUDGET_RATIO)+ ladder_msgs: list = []+ stepped_to = await _run_nav_chain(+ tools,+ ctx,+ nxt,+ ladder_budget,+ _nav_tool_surface(tools),+ ladder_msgs,+ evidence_ids,+ outcomes,+ id_prefix="ladder",+ # Respect the session's remaining context budget: these+ # pairs are appended outside the node's own accounting.+ max_chars=max(800, budget_chars - used),+ )+ # The ladder keeps advancing: a later tool_call in the SAME+ # batch (or the next turn) resumes from where this call left+ # it, not from the original resting point. ``""`` means the+ # ladder finished — also correct to record.+ pending_rule = stepped_to+ used += sum(len(m.get("content") or "") for m in ladder_msgs if m.get("role") == "tool")+ tool_msgs.extend(ladder_msgs)+ if status != OK:+ _LOG.info(+ "[Action Session] ladder %r -> %r (reason=%s)",+ rule.id,+ nxt,+ reason,+ )return {"messages": tool_msgs,"_pending_calls": [],@@ -1205,6 +1427,11 @@ async def _tool_node(state: _SessionState) -> dict:"_tool_chars": used,"_search_queries": seen_queries,"_skipped_dup": int(state.get("_skipped_dup", 0)) + skipped,+ "_tool_strikes": strikes,+ "_tool_outcomes": outcomes,+ "_routed_docs": state.get("_routed_docs") or [],+ "_direction": state.get("_direction", ""),+ "_nav_rule_id": pending_rule,}@@ -1237,11 +1464,11 @@ async def _finalize_node(state: _SessionState) -> dict:fcontent = fresp.choices[0].message.content or ""new_states, found_answer = _parse_terminal(fcontent, parent)if found_answer:- _LOG.info("[action_session] answer salvaged from exhausted session")+ _LOG.info("[Action Session] answer salvaged from exhausted session")elif new_states:- _LOG.info("[action_session] %d branch(es) salvaged from exhausted session", len(new_states))+ _LOG.info("[Action Session] %d branch(es) salvaged from exhausted session", len(new_states))except Exception:- _LOG.exception("[action_session] salvage call failed")+ _LOG.exception("[Action Session] salvage call failed")# Loose-clue harvest (deterministic, zero-LLM): even when every JSON# protocol attempt failed, the last narration often carries facts worth@@ -1264,7 +1491,7 @@ async def _finalize_node(state: _SessionState) -> dict:)if patched is not None:new_states = [patched]- _LOG.warning("[action_session] loose-clue patch (narrative)")+ _LOG.warning("[Action Session] loose-clue patch (narrative)")return {"new_states": new_states, "found_answer": found_answer, "_done": True, "retrieved_evidence_ids": evidence_ids}@@ -1315,7 +1542,7 @@ def _route(state: _SessionState) -> str:# evidence — converge to finalize instead of burning the remaining budget on# paraphrases (Q30/Q759 timeout root cause).if int(state.get("_skipped_dup", 0)) >= 2:- _LOG.info("[action_session] %d near-duplicate retrieval(s) skipped; converging session early", int(state.get("_skipped_dup", 0)))+ _LOG.info("[Action Session] %d near-duplicate retrieval(s) skipped; converging session early", int(state.get("_skipped_dup", 0)))return "finalize"return "run_action" # nudge retry@@ -1336,6 +1563,328 @@ def _build_session_graph():return g.compile()+# ── Deterministic navigation prefix (code-driven tool chain) ─────────────+# The ReAct loop lets the MODEL decide which tool to call next, so ordering rules+# ("route with navigate_tree first, then drill with navigate_structure, fall back+# to retrieve when the drill is weak") can only be *suggested* in the prompt and+# are routinely ignored. This prefix runs those rules IN CODE before the loop+# starts, and injects the resulting exchange as completed assistant/tool messages:+# the model then explores on top of a guaranteed-correct opening, and cannot skip+# a step. It keeps the loop's freedom for everything after the chain.+AUTO = "auto" # code-driven: the orchestrator runs this step itself+LLM = "llm" # model-driven: the code only constrains it, the model calls it+++@dataclass+class _NavRule:+ """One step of the navigation chain.++ :param mode: ``AUTO`` — the orchestrator runs this step itself (no model+ round-trip). ``LLM`` — the step needs a decision only the model can make+ (e.g. WHICH of the routed documents to drill into), so the chain stops+ here and hands control back; the remaining rules still describe what to+ do with the outcome once the model reports back.+ :param run: optional async ``run(tools, ctx, available, budget_s) ->+ ToolOutcome`` that replaces the single-tool path. A step that composes+ several tools (drill: scoped retrieve + navigate_structure + merge ++ LLM quality verdict) uses this instead of ``tool``+``args``.+ :param args: builds the tool arguments from the running context (AUTO only,+ and used for display when ``run`` composes internally).+ :param when: optional guard; the step is skipped when it returns False.+ :param next: ``status -> next rule id``. ``""`` (or a missing key) ends the+ chain. Routing on ``status`` is what makes "quality poor → fall back"+ a code decision rather than a prompt suggestion.+ """++ id: str+ tool: str+ mode: str = AUTO+ run: Any = None+ args: Any = None+ when: Any = None+ next: dict = field(default_factory=dict)+++@dataclass+class _NavContext:+ """Mutable state threaded through the navigation chain.++ ``known_docs`` is the routed scope (doc_ids, for validate-against checks).+ ``routed_docs`` carries each routed doc's OVERALL SUMMARY — the hint that+ feeds retrieval as a soft boost instead of a hard filter ("nav is a hint, not+ a constraint"). ``nav_hint`` is the joined summaries used as retrieval+ keywords so routed docs rank up WITHOUT excluding the rest of the corpus.+ """++ direction: str+ known_docs: list = field(default_factory=list)+ routed_docs: list = field(default_factory=list) # [(doc_id, summary), ...]+ nav_hint: str = ""+++async def _run_drill_merge(tools, ctx: _NavContext, available: set, budget_s: float) -> ToolOutcome:+ """The ``drill`` rung (AUTO): corpus retrieve for REAL chunks + the+ root->chunk structure paths from navigate_structure, MERGED on chunk_id so+ every retrieved chunk carries its hierarchy context.++ **Nav is a hint, not a constraint.** Retrieval runs over the WHOLE corpus+ (no ``doc_scope`` filter), but the nav-routed docs' summaries are fed back+ as BM25 ``keywords`` (soft boost), and the returned chunks are re-ranked so+ routed-doc chunks float to the top while non-routed chunks stay below. A+ multi-hop / enumeration answer living OUTSIDE the routed docs therefore+ survives — it just ranks lower — instead of being filtered out entirely.++ There is deliberately NO LLM verdict here: since retrieval is no longer+ scope-locked, there is no "did the scoped search miss?" signal to grade, and+ no ``POOR -> global`` fallback to drive. The merged evidence is handed to the+ model, whose judgement plus the outer SCA decides sufficiency.+ """+ merged: list = []+ evidence_ids: list = []+ seen_ids: set = set()++ # 1. Skeleton A: whole-corpus retrieval, softly boosted by the nav hints.+ ret_oc = await _exec_retrieve(tools, [ctx.direction], nav_hint=ctx.nav_hint)+ for cid in ret_oc.evidence_ids:+ if cid not in seen_ids:+ seen_ids.add(cid)+ evidence_ids.append(cid)+ routed_ids = set(ctx.known_docs)++ # 2. Supplement B: root->chunk paths from EVERY routed doc's structure.+ paths: dict[str, str] = {}+ if "navigate_structure" in available and ctx.known_docs:+ for doc_id in ctx.known_docs:+ try:+ async with asyncio.timeout(max(5.0, budget_s or _NAV_PREFIX_CALL_TIMEOUT_S)):+ s_oc = await execute_tool(tools, "navigate_structure", {"doc_id": doc_id, "query": ctx.direction, "kind": "catalog"})+ except Exception:+ _LOG.warning("[Action Session] drill structure load failed doc=%s", doc_id, exc_info=True)+ continue+ for d in s_oc.evidence_ids:+ if d not in seen_ids:+ seen_ids.add(d)+ evidence_ids.append(d)+ for cid, path in ((s_oc.metrics or {}).get("chunk_paths") or {}).items():+ if cid and cid not in paths:+ paths[cid] = path++ # 3. Merge: skeleton A is the base; attach structure_path where ids align.+ # Re-rank so routed-doc chunks float to the top WITHOUT deleting the rest.+ for p in ret_oc.payload or []:+ if not isinstance(p, dict):+ merged.append({"content": str(p)})+ continue+ e = dict(p)+ cid = e.get("id")+ sp = paths.get(str(cid)) if cid else None+ if sp:+ e["structure_path"] = sp+ doc = e.get("doc_id") or e.get("document_id") or ""+ e["nav_rank"] = 0 if str(doc) in routed_ids else 1 # 0 = routed (top)+ merged.append(e)+ merged.sort(key=lambda e: (e.get("nav_rank", 1), 0))++ if not merged:+ return ToolOutcome(payload=[], evidence_ids=evidence_ids, status=MISS, reason="no_doc", metrics={"hits": 0})+ return ToolOutcome(payload=merged, evidence_ids=evidence_ids, status=OK, metrics={"hits": len(merged), "routed": len(routed_ids)})+++#: Per-slot strategy ladder. "Nav is a hint, not a constraint": neither rung+#: filters the corpus down to the routed docs — the tree ROUTES (locate), then+#: drill retrieves over the WHOLE corpus while softly boosting + re-ranking the+#: routed-doc chunks, so a multi-hop/enumeration answer living outside the+#: routed docs survives (just ranks lower) instead of being filtered out.+#:+#: locate navigate_tree route to top-n documents, exposing their summaries+#: drill retrieve+merge whole-corpus retrieve (nav summaries as BM25 soft+#: hint) + navigate_structure paths merged on chunk_id+#: + routed-doc chunks re-ranked to the top+#: global retrieve safety net only if drill itself came back empty+#:+#: There is deliberately NO LLM verdict on the drill evidence: retrieval is not+#: scope-locked, so there is no "did the scoped search miss?" signal to grade and+#: no ``POOR -> global`` fallback to drive. Sufficiency is the model's + outer+#: SCA's job, not the ladder's.+#:+#: OFF while we isolate the routing regression. The run-time cost of the ladder is+#: not the issue — the issue is what it does to retrieval scope: with the ladder+#: armed, routing collapsed to 1-5 docs (6 outright zero-doc failures per 20-question+#: run) versus main's 7-12, which let the downstream retrieve degenerate into a+#: corpus-wide search, filled the evidence pool with unrelated chunks (gathered+#: passages 85 -> 105, max 213 -> 356) and drove SCA INSUFFICIENT from 6 to 27 per+#: run. Keep it off until routing quality is back at main's level.+_NAV_RULES_ENABLED = True++_NAV_RULES: tuple = (+ _NavRule(+ id="locate",+ tool="navigate_tree",+ mode=AUTO,+ args=lambda ctx: {"query": ctx.direction},+ # Tree missed => no routed hints to merge against; the only useful step is+ # an unscoped search.+ next={OK: "drill", MISS: "global", EMPTY: "global", POOR: "global", ERROR: "global"},+ ),+ _NavRule(+ id="drill",+ tool="navigate_structure",+ mode=AUTO,+ run=_run_drill_merge,+ args=lambda ctx: {"query": ctx.direction},+ # drill returns OK with the merged, re-ranked evidence; only an empty+ # whole-corpus result (MISS) or an infra failure falls through to global.+ next={OK: "", MISS: "global", EMPTY: "global", ERROR: "global"},+ ),+ _NavRule(+ id="global",+ tool="retrieve",+ mode=AUTO,+ args=lambda ctx: {"query": [ctx.direction]},+ next={},+ ),+)+_NAV_RULE_BY_ID: dict = {r.id: r for r in _NAV_RULES}+_NAV_START_RULE = "locate"++# Share of the session deadline the prefix may spend; the ReAct loop keeps the rest.+_NAV_PREFIX_BUDGET_RATIO = 0.35+# Per-call ceiling so one slow navigation cannot swallow the whole prefix.+_NAV_PREFIX_CALL_TIMEOUT_S = 25.0+# Cap on the nav-hint text fed back into retrieval as BM25 keywords.+_NAV_HINT_CHARS = 600+++def _nav_tool_surface(tools) -> set:+ """Tools this session may call, or an empty set when it cannot be resolved."""+ try:+ return {s["function"]["name"] for s in _active_tool_specs(tools)}+ except Exception:+ _LOG.warning("[Action Session] could not resolve the tool surface", exc_info=True)+ return set()+++def _emit_nav_pair(messages: list, call_id: str, tool: str, args: dict, oc: ToolOutcome, max_chars: int = 0) -> int:+ """Append one completed assistant/tool exchange.++ The pair MUST be well formed — every ``tool_call`` needs its ``tool``+ response, or the provider rejects the history (the reason+ ``_strip_unpaired_tool_calls`` exists).++ ``max_chars`` caps the serialized payload; these pairs bypass the tool node's+ own budget accounting, so without a cap a wide fallback search could inject an+ arbitrarily large message. Returns the characters actually added.+ """+ payload = json.dumps({"passages": oc.payload}, ensure_ascii=False, default=str)+ if max_chars and len(payload) > max_chars:+ payload = payload[: max(max_chars, 800)]+ messages.append(+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [{"id": call_id, "type": "function", "function": {"name": tool, "arguments": json.dumps(args, ensure_ascii=False)}}],+ }+ )+ messages.append({"role": "tool", "tool_call_id": call_id, "content": payload})+ return len(payload)+++async def _run_nav_chain(+ tools,+ ctx: _NavContext,+ start_id: str,+ budget_s: float,+ available: set,+ messages: list,+ evidence_ids: list,+ outcomes: list,+ id_prefix: str = "nav",+ max_chars: int = 0,+) -> str:+ """Run :data:`_NAV_RULES` from ``start_id``, stopping at the first LLM step.++ AUTO steps are executed here; an LLM step is returned without being run,+ because only the model can supply its arguments (e.g. which routed document+ to drill). Shared by the pre-session prefix and by the in-session fallback,+ so both consume exactly the same ladder.++ Returns the id of the rule control now rests on (``""`` when the ladder is+ finished or was abandoned).+ """+ started = time.monotonic()+ spent = 0+ rule_id = start_id+ while rule_id:+ rule = _NAV_RULE_BY_ID.get(rule_id)+ if rule is None or rule.tool not in available:+ break+ if rule.mode == LLM:+ return rule_id # hand control back: the model must decide+ if rule.when is not None and not rule.when(ctx):+ break+ remaining = budget_s - (time.monotonic() - started)+ if remaining <= 1.0:+ _LOG.info("[Action Session] nav chain out of budget before %r", rule_id)+ break+ try:+ if rule.run is not None:+ # A composed step (e.g. drill = retrieve + structure + merge ++ # LLM verdict) owns its own tool calls; it only needs the budget.+ oc: ToolOutcome = await rule.run(tools, ctx, available, max(5.0, remaining))+ else:+ async with asyncio.timeout(min(_NAV_PREFIX_CALL_TIMEOUT_S, remaining)):+ oc = await execute_tool(tools, rule.tool, rule.args(ctx))+ except Exception:+ _LOG.warning("[Action Session] nav chain step %r failed", rule_id, exc_info=True)+ break++ outcomes.append({"name": rule.tool, "status": oc.status, "reason": oc.reason, "metrics": oc.metrics})+ if rule.tool == "navigate_tree" and oc.status == OK:+ # Routed docs become the validated scope for every later step.+ for d in (oc.payload or [{}])[0].get("doc_ids") or []:+ if d and d not in ctx.known_docs:+ ctx.known_docs.append(d)+ # Keep the routed docs WITH their summaries: the hint used to boost+ # (not filter) retrieval for the drilled docs.+ ctx.routed_docs = list((oc.metrics or {}).get("routed_docs") or [])+ hints = [str(s) for _d, s in ctx.routed_docs if s]+ if hints:+ ctx.nav_hint = " ".join(hints)[:_NAV_HINT_CHARS]+ evidence_ids.extend(oc.evidence_ids)+ spent += _emit_nav_pair(messages, f"{id_prefix}_{rule_id}", rule.tool, rule.args(ctx), oc, max_chars=max_chars)+ rule_id = rule.next.get(oc.status, "")+ return rule_id+++async def run_nav_prefix(tools, direction: str, deadline_left: float, ctx: _NavContext | None = None) -> tuple[list, list, list, str]:+ """Run the navigation ladder up to the first model-driven step.++ Returns ``(messages, evidence_ids, outcomes, pending_rule_id)`` — the+ messages are completed assistant/tool pairs ready to seed the session+ history, so the model sees the chain as work already done and cannot skip a+ step. ``pending_rule_id`` is the rule the model is expected to act on ("" when+ the ladder already finished).++ ``ctx`` may be supplied by the caller to keep the ladder's accumulated state+ (notably ``known_docs``, the routed scope the later rungs search within)+ after the prefix ends.++ Returns empty results when navigation is unavailable — the ladder is an+ optimisation, never a precondition for the session to run.+ """+ available = _nav_tool_surface(tools)+ if not available or "navigate_tree" not in available:+ return [], [], [], ""++ ctx = ctx or _NavContext(direction=direction)+ messages: list = []+ evidence_ids: list = []+ outcomes: list = []+ budget = max(5.0, (deadline_left or _ACTION_TIMEOUT_S) * _NAV_PREFIX_BUDGET_RATIO)+ pending = await _run_nav_chain(tools, ctx, _NAV_START_RULE, budget, available, messages, evidence_ids, outcomes)+ return messages, evidence_ids, outcomes, pending++_SESSION_GRAPH = _build_session_graph()@@ -1358,11 +1907,38 @@ async def run_action_session(mdl = _base_chat_mdl(tools)if mdl is None:- _LOG.warning("[action_session] no usable model resolved for action session")+ _LOG.warning("[Action Session] no usable model resolved for action session")return Result(messages=[], new_states=[])+ budget_left = deadline_left or _ACTION_TIMEOUT_S+ # Walk the navigation ladder up to its first model-driven rung: the routing+ # rules below are guaranteed in code, so the model cannot skip navigate_tree+ # and jump straight to navigate_structure the way it does when the order is+ # only a prompt hint. The remaining rungs stay armed in session state so a+ # weak drill falls through to scoped/global retrieval in code.+ #+ # Skipped entirely when :data:`_NAV_RULES_ENABLED` is off: ``pending_rule``+ # then stays "" so the ladder in :func:`_tool_node` never arms either, and the+ # session degenerates to the plain model-driven ReAct loop main runs.+ prefix_started = time.monotonic()+ nav_ctx = _NavContext(direction=direction)+ prefix_msgs, prefix_ids, prefix_outcomes, pending_rule = [], [], [], ""+ if _NAV_RULES_ENABLED:+ try:+ prefix_msgs, prefix_ids, prefix_outcomes, pending_rule = await run_nav_prefix(tools, direction, budget_left, ctx=nav_ctx)+ except Exception:+ _LOG.warning("[Action Session] navigation prefix failed; continuing with the plain loop", exc_info=True)+ if prefix_msgs:+ _LOG.info(+ "[Action Session] nav prefix completed %d step(s), pending=%r, routed=%d doc(s)",+ len(prefix_outcomes),+ pending_rule,+ len(nav_ctx.known_docs),+ )+ spent = time.monotonic() - prefix_started+initial: _SessionState = {- "messages": [SystemMessage(content=system), HumanMessage(content=seed_user)],+ "messages": [SystemMessage(content=system), HumanMessage(content=seed_user)] + prefix_msgs,"parent_state": parent_state,"tools": tools,"mdl": mdl,@@ -1370,19 +1946,24 @@ async def run_action_session("_done": False,"new_states": [],"found_answer": None,- "retrieved_evidence_ids": [],+ "retrieved_evidence_ids": list(prefix_ids),"attempts": 0,- "deadline_left": deadline_left or _ACTION_TIMEOUT_S,+ "deadline_left": max(10.0, budget_left - spent),"_ctx_budget": _MAX_TOOL_RESPONSE_CHARS * 4,"_tool_chars": 0,"_tool_cache": {},"_search_queries": [],"_skipped_dup": 0,+ "_tool_strikes": {},+ "_tool_outcomes": list(prefix_outcomes),+ "_direction": direction,+ "_routed_docs": list(nav_ctx.known_docs),+ "_nav_rule_id": pending_rule,}try:final = await _SESSION_GRAPH.ainvoke(initial)except Exception:- _LOG.exception("[action_session] session failed")+ _LOG.exception("[Action Session] session failed")return Result(messages=[], new_states=[])return Result(messages=final.get("messages", []),diff --git a/rag/advanced_rag/harness/tools/navigation.py b/rag/advanced_rag/harness/tools/navigation.pyindex 17fb1927f7..b6dddf36e9 100644--- a/rag/advanced_rag/harness/tools/navigation.py+++ b/rag/advanced_rag/harness/tools/navigation.py@@ -27,6 +27,7 @@ to ``hypergraph``) and walks it breadth-first from seed entities.import jsonimport loggingimport re+from dataclasses import dataclass, fieldfrom typing import Anyimport json_repair@@ -34,7 +35,6 @@ import json_repairfrom rag.advanced_rag.harness.chunk_utils import (_chunk_id,_chunk_text,- _doc_title,_snippet,_xml_escape,)@@ -58,6 +58,46 @@ _MAX_EVIDENCE_CHUNKS = 24_MAX_ENTITIES = 300+@dataclass+class NavResult:+ """Structured outcome of ONE compiled-navigation call.++ ``text`` is what the MODEL sees (unchanged XML). The remaining fields are the+ signals the ORCHESTRATOR needs to route on: does this dataset have the+ structure at all, did THIS query reach anything, and was the result worth+ using? Without them the caller can only regex the XML — which is how a+ ``count="1" entities="0"`` empty shell used to read as a successful hit, and+ how a single unrouted query used to disable the tool for a whole session.++ :param doc_ids: documents reached (the routing result).+ :param routed_docs: ``[(doc_id, summary), ...]`` — the routed documents WITH+ their overall summaries. The summary is the "hint" the orchestrator feeds+ back into retrieval: nav is a hint, not a constraint, so these become a+ soft boost/rerank signal instead of a hard ``doc_scope`` filter.+ :param entities: compiled entities discovered across those documents.+ :param chunk_ptrs: chunk pointers the outline points the model at.+ :param top_score: best beam score reached while drilling (0.0 when the+ keyword fallback scored instead of vectors).+ :param chunk_paths: ``{chunk_id: "root -> ... -> node"}`` — the full+ hierarchy path (from the tree root down) of the node that owns each+ drilled chunk. Lets a caller merge structure context onto retrieved+ evidence: the snippet says WHAT a chunk contains, the path says WHERE it+ sits in the document.+ :param empty_reason: ``""`` when usable, else ``no_structure`` /+ ``no_doc`` / ``infra`` / ``bad_args``. Only ``no_structure`` is a+ statement about the DATASET; ``no_doc`` is about the query.+ """++ text: str = ""+ doc_ids: list = field(default_factory=list)+ routed_docs: list = field(default_factory=list)+ entities: int = 0+ chunk_ptrs: int = 0+ top_score: float = 0.0+ chunk_paths: dict = field(default_factory=dict)+ empty_reason: str = ""++def _normalize_kind(kind) -> str:"""Mirror the API's kind normalization (page_index/knowledge_graph -> timeline)."""if not isinstance(kind, str):@@ -616,18 +656,32 @@ _NAV_SEARCH_MAX_DOCS = 12 # documents the hybrid search routes to_NAV_MIN_DOC_SCORE = 0.2 # drop docs below this score-async def dataset_navigation_search(tools, topic: str, keywords: str = "", doc_scope: list[str] | None = None) -> list[str]:- """Return the ``doc_id``s most relevant to the question / keywords by- searching the dataset's navigation-tree document leaves (``nav_doc`` layer).-- Runs ``search_dataset_layers`` with ``mode="nav_doc"``: a direct hybrid- search over the nav-tree doc leaves, so it only sees documents that have a- compiled nav-tree node. Faster, no LLM cost, but less precise for ambiguous- queries.-- Returns the routed ``doc_id`` list (capped at ``_NAV_SEARCH_MAX_DOCS``), or- ``[]`` when no question/keywords are given or the search returns nothing.- This function only routes — it does not retrieve.+async def _nav_search_titled(tools, topic: str, keywords: str = "", doc_scope: list[str] | None = None) -> list[tuple[str, str]]:+ """Descend the compiled nav TREE and return ``(doc_id, summary)`` pairs.++ Uses ``mode="navigation_tree"`` (BFS beam descent: root cluster → sub-clusters+ → nav_doc leaves) rather than a flat sweep over every nav_doc row. Routing+ through the hierarchy narrows the field level by level, which is what makes it+ accurate on large datasets where many document summaries look alike in a flat+ comparison.++ HYBRID descent (vector + BM25): both the KNN legs and the ``_text_score``+ gates stay armed. A pure-vector descent was tried — it dropped the text legs+ and the ``_text_score > 0`` gate, on the theory that routing cares about+ semantic matches a summary never spells out. It regressed routing badly:+ scores collapsed onto the dense leg alone, the descent fell below+ ``_NAV_TREE_MIN_SCORE`` far more often, and routing returned 1-5 docs instead+ of hybrid's 7-12 — with 6 outright zero-doc failures per 20-question run.+ Losing that scope let the downstream retrieval degenerate into a corpus-wide+ search, which filled the evidence pool with unrelated chunks (gathered+ passages 85 -> 105, max 213 -> 356) and drove SCA INSUFFICIENT from 6 to 27+ per run. Routing needs both legs; the vector-only path was removed.++ Each nav row already carries the document's overall summary (``description``+ on the leaf row), so labelling the route costs ZERO extra queries — the+ caller must never load the document for a label.++ Routing only — this never retrieves document content."""query = " ".join(part for part in ((topic or "").strip(), (keywords or "").strip()) if part).strip()if not query:@@ -642,7 +696,7 @@ async def dataset_navigation_search(tools, topic: str, keywords: str = "", doc_skbs = getattr(tools, "kbs", []) or []allowed_docs = set(doc_scope or [])- candidates: dict[str, float] = {}+ candidates: dict[str, tuple[float, str]] = {}for kb in kbs:# ``doc_scope`` is forwarded query-time (search_dataset_layers applies# it as a store filter on every mode), so the top_k truncation never@@ -652,7 +706,7 @@ async def dataset_navigation_search(tools, topic: str, keywords: str = "", doc_skb.id,kb.tenant_id,query,- "nav_doc",+ "navigation_tree",top_k=_NAV_SEARCH_MAX_DOCS,doc_scope=list(allowed_docs) or None,)@@ -668,14 +722,37 @@ async def dataset_navigation_search(tools, topic: str, keywords: str = "", doc_sdid = str(item.get("doc_id") or "").strip()if not did:continue- candidates[did] = max(candidates.get(did, float("-inf")), score)+ # The nav leaf row carries the document's OVERALL summary. Using it+ # here is the whole point: the route is labelled for free, instead of+ # loading the document to read its first chunk.+ nav = item.get("_nav") or {}+ summary = str(nav.get("description") or nav.get("name") or "").strip()+ prev = candidates.get(did)+ if prev is None or score > prev[0]:+ candidates[did] = (score, summary)- routed = [did for did, _ in sorted(candidates.items(), key=lambda pair: pair[1], reverse=True)[:_NAV_SEARCH_MAX_DOCS]]+ routed = [(did, summary) for did, (score, summary) in sorted(candidates.items(), key=lambda pair: pair[1][0], reverse=True)[:_NAV_SEARCH_MAX_DOCS]]_LOG.info("[Dataset navigation search] Routed to %d document(s) (min_score=%.1f).", len(routed), _NAV_MIN_DOC_SCORE)return routed[:_NAV_SEARCH_MAX_DOCS]+async def dataset_navigation_search(tools, topic: str, keywords: str = "", doc_scope: list[str] | None = None) -> list[str]:+ """Return the ``doc_id``s most relevant to the question / keywords by+ descending the dataset's compiled navigation tree.++ Routes through :func:`_nav_search_titled`, which runs+ ``search_dataset_layers`` with ``mode="navigation_tree"``: a hybrid BFS beam+ descent (vector + BM25) from the root clusters down to the ``nav_doc``+ leaves, with the ``_text_score`` gates armed.++ Returns the routed ``doc_id`` list (capped at ``_NAV_SEARCH_MAX_DOCS``), or+ ``[]`` when no question/keywords are given or the search returns nothing.+ This function only routes — it does not retrieve.+ """+ return [did for did, _title in await _nav_search_titled(tools, topic, keywords, doc_scope)]++# Knowledge-graph and wiki exploration live in ``exploration`` and are# re-exported here: compiled-product expansion imports ``_kg_scopes`` from this# module, and the action-session tool surface exposes ``graph_explore`` by this@@ -754,136 +831,81 @@ async def _navigate_tree_impl(query: str,keywords: str = "",doc_scope: list[str] | None = None,-) -> str:- """Locate the document(s) most likely to hold the answer, via the dataset's- compiled navigation tree (``dataset_nav`` / ``nav_doc`` layer).+) -> NavResult:+ """Locate the document(s) most likely to hold the answer, by descending the+ dataset's compiled navigation tree (``dataset_nav``).- This tool routes the question by searching the compiled navigation-tree- document leaves (nav_doc) — which cover EVERY document of the dataset- (2400+ here), organized into clusters — then returns each routed document- with a short excerpt (its first chunk) so you can decide which to deep-read.+ This tool ROUTES, it does not retrieve: it walks the compiled cluster+ hierarchy — root cluster → sub-clusters → ``nav_doc`` leaves — with BFS beam+ pruning, and returns the ``doc_id`` list reached along the most relevant+ branches. Descending the hierarchy beats a flat sweep over every document+ summary when the dataset is large and many summaries look alike.Use it when search_chunks / grep_chunks did not clearly locate the answer andyou need the document-level view to guide you (e.g. an entity spans manydocs, or the question mentions a general topic that lives across a document's- body). After it returns doc_id values, deep-read the relevant one(s) with- list_chunks / navigate_structure.+ body). After it returns doc_id values, use navigate_structure / list_chunks on+ the relevant one(s) to get the actual passages.++ It deliberately does NOT fetch document content. Loading full documents here+ cost ~79 ES queries per document (``fetch_full_document`` pages 128 chunks at+ a time) only to render a 500-char snippet that navigate_structure supersedes —+ under the per-pass session fan-out that alone was enough to knock ES over.Returns an XML <tree_navigation> document. Each routed document is a <doc>- element with doc_id, doc_title attributes and a <snippet> element (a short- excerpt, NOT the full text). If nothing routes, an empty <tree_navigation/>- is returned.+ element with a ``doc_id`` attribute and, when the nav leaf has one, a+ <summary> child holding the document's overall summary. No <snippet>: the+ content decision belongs to navigate_structure. If nothing routes, an empty+ <tree_navigation/> is returned.:param query: REQUIRED: the question / topic to route.- :param keywords: Optional terms kept only for prompt hints; document routing- is driven by the nav-tree doc leaves and does not require keyword hits.+ :param keywords: Optional terms kept only for prompt hints; routing descends+ on vector similarity and does not require keyword hits.:param doc_scope: Optional document ids to restrict routing to (at most 8).- :return: XML <tree_navigation> document.+ :return: :class:`NavResult` — the XML in ``text`` plus the routed ``doc_ids``."""from common import settingsif not getattr(settings, "retriever", None):- return '<tree_navigation count="0" error="no retriever">\n</tree_navigation>'+ return NavResult(text='<tree_navigation count="0" error="no retriever">\n</tree_navigation>', empty_reason="infra")query = str(query or "").strip()if not query:- return '<tree_navigation count="0" error="query is required">\n</tree_navigation>'+ return NavResult(text='<tree_navigation count="0" error="query is required">\n</tree_navigation>', empty_reason="bad_args")tools_slot = _tools_slot()- # Route via the compiled navigation tree (dataset_nav / nav_doc layer): it- # covers ALL documents of the dataset organized into clusters, so a query can- # reach documents that pure embedding routing would miss. Fall back to pure- # embedding routing only when the dataset has NO compiled navigation tree.- routed = await dataset_navigation_search(tools_slot, query, keywords, doc_scope)+ # PURE VECTOR descent over the compiled nav tree (dataset_nav), so a query+ # reaches documents that term matching would miss. The climb narrows the+ # candidate set level by level instead of comparing every document summary at+ # once — accuracy at scale, at the cost of a bounded number of extra KNN+ # queries (beam_width per depth).+ #+ # There is deliberately NO chunk-retrieval fallback here. Routing used to fall+ # back to a full retriever pass over every chunk whenever the nav search came+ # back empty — the generic retriever reinventing what dataset_nav already+ # does, and by far the most expensive thing this file could do. When routing+ # misses, the caller falls back to retrieve/search_chunks: the same work, done+ # once and owned by the orchestrator instead of hidden inside a "route" call.+ routed = await _nav_search_titled(tools_slot, query, keywords, doc_scope)if not routed:- routed = await _route_docs_via_embedding(tools_slot, query, doc_scope)- if not routed:- return '<tree_navigation count="0">\n</tree_navigation>'+ # THIS query routed to nothing. The dataset may still have a nav tree that+ # a better-formed query would hit — a query-level miss, not a dataset fact.+ return NavResult(text='<tree_navigation count="0">\n</tree_navigation>', empty_reason="no_doc")+ routed = routed[:_NAV_TREE_MAX_DOCS]parts = [f'<tree_navigation count="{len(routed)}" query="{_xml_escape(query)}">']- for i, doc_id in enumerate(routed[:_NAV_TREE_MAX_DOCS]):- title = ""- snippet = ""- try:- full = await tools_slot.fetch_full_document(doc_id)- doc_chunks = full.get("chunks", []) or []- if doc_chunks:- title = _doc_title(doc_chunks[0]) or doc_id- snippet = _chunk_text(doc_chunks[0])- if len(snippet) > 500:- snippet = snippet[:500]- else:- title = doc_id- except Exception:- title = doc_id- parts.append(f' <doc rank="{i + 1}" doc_id="{_xml_escape(doc_id)}" doc_title="{_xml_escape(title)}">')- if snippet:- parts.append(f" <snippet>{_xml_escape(snippet)}</snippet>")- parts.append(" </doc>")+ for i, (doc_id, summary) in enumerate(routed):+ if summary:+ parts.append(f' <doc rank="{i + 1}" doc_id="{_xml_escape(doc_id)}">')+ parts.append(f" <summary>{_xml_escape(summary)}</summary>")+ parts.append(" </doc>")+ else:+ parts.append(f' <doc rank="{i + 1}" doc_id="{_xml_escape(doc_id)}"/>')parts.append("</tree_navigation>")- return "\n".join(parts)---async def _route_docs_via_embedding(tools_slot, query: str, doc_scope: list[str] | None = None, top_n: int = 12) -> list[str]:- """Route ``query`` to documents by PURE embedding similarity — ZERO keywords.-- Encodes the query with ``tools.embed_mdl`` and retrieves the most- vector-similar real document chunks (compiled products excluded), aggregated- to documents via ``doc_aggs``. There is NO nav-tree ``_text_score > 0``- keyword-literal gate and no hybrid/BM25 leg: routing is driven purely by- vector similarity, so a question that matches a document's body only- semantically (e.g. an entity alias the summary never spells out) still- routes to the right document.-- Returns a deduplicated most-relevant-first ``doc_id`` list, or ``[]`` on- failure / when nothing hits. This function only routes — it does not- retrieve structure.- """- if not query or not str(query).strip():- return []- if not getattr(tools_slot, "embed_mdl", None):- _LOG.warning("[navigate_structure] no embed_mdl available; cannot route by embedding")- return []- from common import settings-- if not getattr(settings, "retriever", None):- return []- target_ids = _get_kb_ids(tools_slot)- if not target_ids:- return []- tenant_ids = getattr(tools_slot, "tenant_ids", None) or []- if not tenant_ids:- tid = _first_tenant_id(tools_slot)- if tid:- tenant_ids = [tid]- if not tenant_ids:- return []- try:- kbinfos = await settings.retriever.retrieval(- str(query).strip(),- tools_slot.embed_mdl,- tenant_ids,- target_ids,- 1,- top_n,- 0.2,- vector_similarity_weight=1.0, # pure embedding — no keyword/BM25 leg- aggs=True,- highlight=False,- doc_ids=doc_scope,- must_not={"exists": "compile_kwd"}, # real doc chunks only; compiled products have their own tools- )- except Exception:- _LOG.exception("[navigate_structure] embedding doc routing failed")- return []- doc_ids: list[str] = []- seen: set[str] = set()- for agg in kbinfos.get("doc_aggs") or []:- did = str(agg.get("doc_id") or "").strip()- if did and did not in seen:- seen.add(did)- doc_ids.append(did)- _LOG.info("[navigate_structure] Embedding doc routing found %d candidate document(s).", len(doc_ids))- return doc_ids+ return NavResult(+ text="\n".join(parts),+ doc_ids=[did for did, _ in routed],+ routed_docs=[(did, summary) for did, summary in routed],+ )def _first_tenant_id(tools_slot) -> str:@@ -959,10 +981,10 @@ async def _navigate_structure_impl(from common import settingsif not getattr(settings, "retriever", None):- return '<structure_navigation count="0" error="no retriever">\n</structure_navigation>'+ return NavResult(text='<structure_navigation count="0" error="no retriever">\n</structure_navigation>', empty_reason="infra")query = str(query or "").strip()if not query:- return '<structure_navigation count="0" error="query is required">\n</structure_navigation>'+ return NavResult(text='<structure_navigation count="0" error="query is required">\n</structure_navigation>', empty_reason="bad_args")# Determine which compiled kinds to read.kinds = _structure_kinds_for(kind)@@ -971,23 +993,47 @@ async def _navigate_structure_impl(if doc_id:doc_ids = [str(doc_id).strip()] if str(doc_id).strip() else []else:- # Locate the document by PURE embedding similarity — no nav-tree- # keyword-literal gate, no hybrid/BM25 leg (see _route_docs_via_embedding).- doc_ids = await _route_docs_via_embedding(tools_slot, query, doc_scope)+ # Route by PURE VECTOR similarity over the nav tree — the same seam+ # navigate_tree uses. This used to fall back to a full chunk-retriever+ # pass, which reinvented nav routing with the most expensive tool in the+ # file: a hybrid sweep over every chunk of the dataset to answer "which+ # document?", a question dataset_nav already answers with one KNN query.+ doc_ids = [did for did, _summary in await _nav_search_titled(tools_slot, query, "", doc_scope)]if not doc_ids:- return '<structure_navigation count="0" error="no document located">\n</structure_navigation>'+ # The QUERY routed to nothing (or the caller passed a doc_id no route ever+ # produced). Not a statement about the dataset's compiled structures.+ return NavResult(text='<structure_navigation count="0" error="no document located">\n</structure_navigation>', empty_reason="no_doc")structures = await _read_structures(tools_slot, query, doc_ids[:_NAV_TREE_MAX_DOCS], kinds)parts = [f'<structure_navigation count="{len(structures)}" query="{_xml_escape(query)}" kind="{_xml_escape(kind)}">']+ total_entities = 0+ total_ptrs = 0+ best_score = 0.0+ all_chunk_paths: dict[str, str] = {}for i, s in enumerate(structures):+ stats = s.get("stats") or {}+ total_entities += len(s.get("entities") or [])+ total_ptrs += int(stats.get("chunk_ptrs") or 0)+ best_score = max(best_score, float(stats.get("top_score") or 0.0))+ for cid, path in (s.get("chunk_paths") or {}).items():+ if cid and cid not in all_chunk_paths:+ all_chunk_paths[cid] = pathparts.append(f' <doc rank="{i + 1}" doc_id="{_xml_escape(s["doc_id"])}" doc_title="{_xml_escape(s["title"])}" entities="{len(s["entities"])}" relations="{len(s["relations"])}">')outline = s["outline"]if outline:parts.append(f" <structure>{_xml_escape(outline)}</structure>")parts.append(" </doc>")parts.append("</structure_navigation>")- return "\n".join(parts)+ return NavResult(+ text="\n".join(parts),+ doc_ids=list(structures and [s["doc_id"] for s in structures] or []),+ entities=total_entities,+ chunk_ptrs=total_ptrs,+ top_score=round(best_score, 4),+ chunk_paths=all_chunk_paths,+ empty_reason="no_structure" if total_entities == 0 else "",+ )def _structure_kinds_for(kind: str) -> set:@@ -1161,7 +1207,7 @@ async def _read_structures(tools_slot, query: str, doc_ids: list[str], kinds: serelations = structure.get("relations") or []except Exception:_LOG.exception("[navigate_structure] structure load failed for doc=%s", doc_id)- outline = await _render_toc_drilldown(query, qvec, entities, relations)+ outline, stats, chunk_paths = await _render_toc_drilldown(query, qvec, entities, relations)out.append({"doc_id": doc_id,@@ -1169,6 +1215,8 @@ async def _read_structures(tools_slot, query: str, doc_ids: list[str], kinds: se"entities": entities,"relations": relations,"outline": outline,+ "stats": stats,+ "chunk_paths": chunk_paths,})return out@@ -1250,22 +1298,28 @@ def _node_score(qvec, query_terms, entity: dict) -> float:return float(_node_relevance(query_terms, entity))-def _drill_kept_nodes(query_terms: list[str], qvec, entities: list[dict], relations: list[dict]) -> tuple[list[dict], dict, set]:+def _drill_kept_nodes(query_terms: list[str], qvec, entities: list[dict], relations: list[dict]) -> tuple[list[dict], dict, set, float]:"""VECTOR BEAM drill-down over the compiled TOC hierarchy.From the roots, at each level keep the top-K nodes by cosine similarity to thequery (``_node_score``) and descend to their children; keyword-overlap is the- fallback when no vectors are available. Returns ``(kept_nodes, parents, kept_names)``- so callers can both render the outline AND collect the related chunks behind the- drilled nodes' ``source_chunk_ids``.+ fallback when no vectors are available. Returns+ ``(kept_nodes, parents, kept_names, best_score)`` so callers can both render+ the outline AND collect the related chunks behind the drilled nodes'+ ``source_chunk_ids``.++ ``best_score`` is the highest score reached at any level — the quality signal+ the orchestrator routes on. It was computed here and thrown away; surfacing it+ is what lets callers tell a shallow drift from a solid hit without re-scoring."""by_name, children, parents, roots = _build_toc_tree(entities, relations)if not roots:- return [], {}, set()+ return [], {}, set(), 0.0frontier = list(roots)kept_names: set[str] = set()depth = 0+ best_overall = 0.0# ``<=`` so the deepest relevant leaf (at depth MAX_DEPTH) is also collected,# not just the intermediate levels.while frontier and depth <= _STRUCT_MAX_DEPTH:@@ -1276,6 +1330,8 @@ def _drill_kept_nodes(query_terms: list[str], qvec, entities: list[dict], relatiif not scored:breakbest = scored[0][0]+ if best > best_overall:+ best_overall = best# If even the best node has no relevance, stop descending.if qvec is None and best < _STRUCT_RELEVANCE_MIN:break@@ -1310,10 +1366,10 @@ def _drill_kept_nodes(query_terms: list[str], qvec, entities: list[dict], relatie = by_name.get(n)if e is not None:kept_nodes.append(e)- return kept_nodes, parents, kept_names+ return kept_nodes, parents, kept_names, best_overall-async def _render_toc_drilldown(query: str, qvec, entities: list[dict], relations: list[dict]) -> str:+async def _render_toc_drilldown(query: str, qvec, entities: list[dict], relations: list[dict]) -> tuple[str, dict, dict]:"""Drill down the pre-built TOC hierarchy toward the query and render an outline.Uses VECTOR BEAM descent when the nodes carry embeddings (``_vec``) and a query@@ -1325,14 +1381,19 @@ async def _render_toc_drilldown(query: str, qvec, entities: list[dict], relation- Paris Demographics (tree_node): <desc> [chunks: c1,c2]- Immigration since 1945 (tree_node): <desc> [chunks: c3]- [chunk c3]: <snippet>++ Also returns a stats dict (``nodes``, ``chunk_ptrs``, ``top_score``) and a+ ``chunk_paths`` map ``{chunk_id: "root -> ... -> node"}`` — the full+ hierarchy path of the node owning each drilled chunk — so a caller can merge+ structure context onto retrieved evidence without re-deriving the tree."""query_terms = [t for t in re.findall(r"[A-Za-z0-9_]{2,}", (query or "").lower()) if len(t) >= 2]if not query_terms and qvec is None:- return _render_outline(entities[:_STRUCT_MAX_NODES], relations[:_STRUCT_MAX_NODES])+ return _render_outline(entities[:_STRUCT_MAX_NODES], relations[:_STRUCT_MAX_NODES]), _outline_stats(entities), {}- kept_nodes, parents, kept_names = _drill_kept_nodes(query_terms, qvec, entities, relations)+ kept_nodes, parents, kept_names, best = _drill_kept_nodes(query_terms, qvec, entities, relations)if not kept_nodes:- return _render_outline(entities[:_STRUCT_MAX_NODES], relations[:_STRUCT_MAX_NODES])+ return _render_outline(entities[:_STRUCT_MAX_NODES], relations[:_STRUCT_MAX_NODES]), _outline_stats(entities), {}# Render the drilled nodes (indented by depth via ancestor count).depth_of: dict[str, int] = {}@@ -1344,6 +1405,25 @@ async def _render_toc_drilldown(query: str, qvec, entities: list[dict], relationcur = parents.get(cur)depth_of[n] = d+ # root -> ... -> node path for every drilled node, mapped onto its chunks so+ # the caller can annotate retrieved evidence with where it sits structurally.+ chunk_paths: dict[str, str] = {}+ for e in kept_nodes:+ name = (e.get("name") or "").strip()+ if not name:+ continue+ segs = [name]+ cur = parents.get(name)+ guard = 0+ while cur and guard < _STRUCT_MAX_DEPTH:+ segs.append(cur)+ cur = parents.get(cur)+ guard += 1+ path = " -> ".join(reversed(segs))+ for cid in e.get("source_chunk_ids") or []:+ if isinstance(cid, str) and cid and cid not in chunk_paths:+ chunk_paths[cid] = path+lines: list[str] = []for e in kept_nodes[:_STRUCT_MAX_NODES]:name = (e.get("name") or "").strip()@@ -1368,7 +1448,21 @@ async def _render_toc_drilldown(query: str, qvec, entities: list[dict], relationcid = _chunk_id(c)text = _chunk_text(c).strip()lines.append(f"- [chunk {cid}]: {_snippet(text, 300)}")- return "\n".join(lines)+ stats = {+ "nodes": len(kept_nodes),+ "chunk_ptrs": len(wanted),+ "top_score": round(float(best), 4),+ }+ return "\n".join(lines), stats, chunk_paths+++def _outline_stats(entities: list[dict]) -> dict:+ """Stats for the flat fallback outline: no drill happened, so the beam score+ is unknown (0.0) and only the entity/chunk counts are meaningful."""+ ptrs = 0+ for e in entities[:_STRUCT_MAX_NODES]:+ ptrs += len(_collect_chunk_ids([e]))+ return {"nodes": len(entities[:_STRUCT_MAX_NODES]), "chunk_ptrs": ptrs, "top_score": 0.0}def _render_outline(entities: list[dict], relations: list[dict]) -> str:diff --git a/rag/advanced_rag/harness/tools/search.py b/rag/advanced_rag/harness/tools/search.pyindex f001742cd3..e19e90efd1 100644--- a/rag/advanced_rag/harness/tools/search.py+++ b/rag/advanced_rag/harness/tools/search.py@@ -372,11 +372,17 @@ async def grep_search(kb_ids: list[str] | None = None,top_n: int = 60,doc_scope: list[str] | None = None,+ keywords: str | None = None,) -> dict:"""Exact keyword/pattern locate: BM25-first candidate pool, then acase-insensitive regex locate with a short context window (like dynamic'sgrep_chunks, but returning the ``Pipeline`` ``{"chunks": [...]}`` contract).+ ``keywords`` (optional) is a SOFT retrieval hint fed to the BM25 candidate+ pool — e.g. the nav-routed docs' summaries ("nav is a hint, not a+ constraint"). When empty, the query's own terms are extracted instead. It+ never acts as a hard filter; the candidate pool still spans the whole corpus.+Returns compact snippet chunks (token-cheap) for the sub-agent. When grepmatches nothing, the BM25 candidates are returned unchanged so evidence isnever dropped (enumeration / multi-hop answers must not lose candidates).@@ -386,9 +392,15 @@ async def grep_search(_LOG.info('[Grep search] Keyword-first locate for "%s"', query)if not query or not str(query).strip():return {"chunks": [], "doc_aggs": []}- res = await bm25_search(tools, query=str(query).strip(), kb_ids=kb_ids, top_n=top_n, doc_scope=doc_scope)- chunks = res.get("chunks", []) or []terms = _grep_terms_from_query(str(query).strip())+ # Pass the extracted terms (or the explicit nav hint) as BM25 keywords so the+ # candidate pool is built from "sentence + discriminating terms". A long+ # question buries its proper nouns (e.g. "Culdect Saga") under stopwords;+ # without the keywords boost the noun chunk never enters the pool and grep+ # has nothing to locate.+ hint = keywords if keywords else " ".join(terms)+ res = await bm25_search(tools, query=str(query).strip(), kb_ids=kb_ids, top_n=top_n, doc_scope=doc_scope, keywords=hint)+ chunks = res.get("chunks", []) or []if not chunks or not terms:return restry:diff --git a/test/unit_test/rag/advanced_rag/knowlege_compile/conftest.py b/test/unit_test/rag/advanced_rag/knowlege_compile/conftest.pyindex bbf2996f73..f7434c9814 100644--- a/test/unit_test/rag/advanced_rag/knowlege_compile/conftest.py+++ b/test/unit_test/rag/advanced_rag/knowlege_compile/conftest.py@@ -73,6 +73,28 @@ for name in _stub_only:if name not in sys.modules:sys.modules[name] = types.ModuleType(name)+# dataset_nav imports RedisDistributedLock at module level but only exercises it+# on the build/upsert paths, so a no-op class is enough to import the search half.+if not hasattr(sys.modules["rag.utils.redis_conn"], "RedisDistributedLock"):++ class _NoOpLock:+ def __init__(self, *args, **kwargs):+ pass++ async def spin_acquire(self):+ return True++ async def __aenter__(self):+ return self++ async def __aexit__(self, *exc):+ return False++ async def release(self):+ return True++ sys.modules["rag.utils.redis_conn"].RedisDistributedLock = _NoOpLock+# message_fit_in is imported by wiki_incremental at module levelif not hasattr(sys.modules["rag.prompts.generator"], "message_fit_in"):@@ -155,6 +177,18 @@ _common_mod.knowledge_compile_gen_conf = lambda *a, **k: {}_common_mod.run_chunked_pipeline = MagicMock(return_value={})_common_mod.stable_row_id = lambda *a, **k: ""++async def _encode(embd_mdl, texts: list):+ """Mirror of ``_common.encode``: run the model in the pool, drop the token+ count. Bound late so the conftest's ``thread_pool_exec`` stub is honoured."""+ if not texts:+ return []+ embeddings, _ = await _fake_thread_pool_exec(embd_mdl.encode, texts)+ return list(embeddings)+++_common_mod.encode = _encode+# ---- Test helper constants (same values as structure.py) ----sys.modules["rag.advanced_rag.knowlege_compile.structure"].CONCEPT_MIN_CLAIMS = 3sys.modules["rag.advanced_rag.knowlege_compile.structure"].CONCEPT_MIN_SOURCES = 2diff --git a/test/unit_test/rag/advanced_rag/test_action_session.py b/test/unit_test/rag/advanced_rag/test_action_session.pyindex 3f10fa041d..0dde58cb6f 100644--- a/test/unit_test/rag/advanced_rag/test_action_session.py+++ b/test/unit_test/rag/advanced_rag/test_action_session.py@@ -22,7 +22,7 @@ async def test_exec_calculate_returns_computed_value(monkeypatch):monkeypatch.setattr("rag.advanced_rag.harness.arithmetic.compute_from_facts", fake_compute_from_facts)tools = SimpleNamespace(chat_mdl=object())- results, evidence_ids = await _exec_calculate(+ oc = await _exec_calculate(tools,{"question": "What is the total?",@@ -30,5 +30,6 @@ async def test_exec_calculate_returns_computed_value(monkeypatch):},)- assert results == [{"kind": "calculate", "expression": "2 + 3", "result": "5"}]- assert evidence_ids == []+ assert oc.status == "ok"+ assert oc.payload == [{"kind": "calculate", "expression": "2 + 3", "result": "5"}]+ assert oc.evidence_ids == []diff --git a/test/unit_test/rag/advanced_rag/test_dataset_navigation_search.py b/test/unit_test/rag/advanced_rag/test_dataset_navigation_search.pyindex b9c0236128..fccca4100f 100644--- a/test/unit_test/rag/advanced_rag/test_dataset_navigation_search.py+++ b/test/unit_test/rag/advanced_rag/test_dataset_navigation_search.py@@ -20,7 +20,10 @@ async def test_dataset_navigation_search_ranks_across_all_bound_kbs(monkeypatch)async def fake_search_dataset_layers(kb_id, tenant_id, query, mode, top_k, doc_scope):assert query == "topic keywords"- assert mode == "nav_doc"+ # Diverges from main's "nav_doc" on purpose: routing now descends the+ # compiled nav TREE (BFS beam, `search_nav_tree_descent`) instead of+ # flat-sweeping every nav_doc row. Same hybrid ranking underneath.+ assert mode == "navigation_tree"assert top_k == _NAV_SEARCH_MAX_DOCSassert doc_scope is Noneif kb_id == kb1.id: