Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Refactor agentic search (#19000)

Yingfengfc62487e3978
diff --git a/rag/advanced_rag/__init__.py b/rag/advanced_rag/__init__.py
index cf3a6b3366..e20b0c948c 100644
--- a/rag/advanced_rag/__init__.py
+++ b/rag/advanced_rag/__init__.py
@@ -14,16 +14,14 @@
# limitations under the License.
#
-from .harness.config import THINKING_MODES, get_mode
-from .harness.types import RouteDecision, ExecutionStrategy, ClaimTarget, WorkflowPlan, SufficiencyVerdict
+# Mode configuration is the public entry point: every mode-dependent decision in
+# the harness reads from here.
+from .harness.config import NAIVE, THINKING_MODES, get_mode, resolve_mode
__all__ = [
"THINKING_MODES",
"get_mode",
- "RouteDecision",
- "ExecutionStrategy",
- "ClaimTarget",
- "WorkflowPlan",
- "SufficiencyVerdict",
+ "resolve_mode",
+ "NAIVE",
]
diff --git a/rag/advanced_rag/agentic_rag_graph.py b/rag/advanced_rag/agentic_rag_graph.py
index 1fb084c373..482a141014 100644
--- a/rag/advanced_rag/agentic_rag_graph.py
+++ b/rag/advanced_rag/agentic_rag_graph.py
@@ -60,6 +60,7 @@ from typing import Annotated, Any, TypedDict
from langgraph.graph import END, START, StateGraph
from langgraph.graph.message import add_messages
+from rag.advanced_rag.harness.config import NAIVE, resolve_mode
from rag.advanced_rag.harness.stats import in_phase
from rag.prompts.generator import form_message, kb_prompt, message_fit_in
@@ -200,7 +201,7 @@ class AgenticState(TypedDict, total=False):
partial_answer: bool
abstain: bool
empty_result: bool
- verdict: dict # SufficiencyVerdict serialized
+ verdict: dict # sufficiency verdict as a plain dict (status/score/gaps/...)
sca: dict # raw SCA payload (its gaps feed the Query Rewriter)
# ── budgets & counters ──
@@ -761,8 +762,7 @@ def build_agentic_graph(
# more research rounds before giving up, at the cost of latency — this is
# one of the two differentiators from high (the other is the ultra-only
# graph_explore relational tool). high/medium stay at 3.
- _mode_label = str(getattr(tools, "thinking_mode", "") or "").lower()
- sca_max_rounds = 5 if _mode_label == "ultra" else 3
+ sca_max_rounds = resolve_mode(tools).sca_max_rounds
# ── Node: formalize_question ──
@in_phase("formalize")
@@ -1402,6 +1402,60 @@ async def _compose_fallback_draft(tools, state: AgenticState, answer_conf: dict)
return evidence[:4000]
+async def _naive_rag(tools, messages: list, gen_conf: dict | None = None):
+ """Answer with one retrieve pass — no agentic graph at all.
+
+ Used when the thinking mode is unrecognised. Instead of failing the request
+ (the label comes from user input) we degrade to plain retrieval + one
+ composed answer, matching the ``run_agentic_rag`` yield contract so callers
+ are unaffected.
+ """
+ question = ""
+ for m in reversed(messages or []):
+ if m.get("role") == "user":
+ question = str(m.get("content") or "").strip()
+ break
+
+ _LOG.info("[Naive RAG] single-pass retrieval for question_len=%d", len(question))
+ try:
+ res = await tools.retrieve(question) if question else {"chunks": [], "doc_aggs": []}
+ except Exception:
+ _LOG.exception("[Naive RAG] retrieval failed")
+ res = {"chunks": [], "doc_aggs": []}
+
+ chunks = res.get("chunks") or []
+ if not chunks:
+ yield str(getattr(tools, "empty_response", "") or "")
+ return
+
+ # Accumulate onto the shared pool so the composed answer can be cited, and so
+ # callers reading tools.kbinfos see the same shape as the agentic path.
+ from rag.advanced_rag.harness.tools.search import _chunk_id
+
+ kbinfos = getattr(tools, "kbinfos", None)
+ if isinstance(kbinfos, dict):
+ existing = {_chunk_id(c) for c in (kbinfos.get("chunks") or [])}
+ for c in chunks:
+ if _chunk_id(c) not in existing:
+ kbinfos["chunks"].append(c)
+ tools.kbinfos = kbinfos
+
+ evidence = "\n\n".join(f"[{i}] {str(c.get('content_with_weight') or c.get('content') or '')[:1500]}" for i, c in enumerate(chunks[:8], 1))
+ answer_conf = dict(gen_conf) if gen_conf else {"temperature": 0.3}
+ try:
+ from rag.prompts.generator import form_message, message_fit_in
+
+ system = "Answer the question using ONLY the numbered evidence below. Cite with [n] markers. If the evidence does not answer it, say so plainly — do not use outside knowledge."
+ _, msg = message_fit_in(form_message(system, f"Question: {question}\n\nEvidence:\n{evidence}"), tools.chat_mdl.max_length)
+ ans = await tools.chat_mdl.async_chat(msg[0]["content"], msg[1:], answer_conf)
+ if isinstance(ans, tuple):
+ ans = ans[0]
+ yield str(ans or "").strip() or str(getattr(tools, "empty_response", "") or "")
+ except Exception:
+ _LOG.exception("[Naive RAG] composition failed; returning evidence")
+ yield evidence[:4000]
+
+
async def run_agentic_rag(tools, messages: list, max_loops: int = 3, gen_conf: dict | None = None):
"""Drive the agentic-search graph, yielding answer-token strings."""
_LOG.info(
@@ -1412,15 +1466,27 @@ async def run_agentic_rag(tools, messages: list, max_loops: int = 3, gen_conf: d
)
token_queue: asyncio.Queue = asyncio.Queue()
- thinking_mode = str(getattr(tools, "thinking_mode", "") or "").lower()
+ mode = resolve_mode(tools)
+ thinking_mode = mode.label
# One aligned graph for medium / high / ultra; only the planner+prefetch pair
# (fan-out decomposition) and the SCA iteration separate the modes:
# medium: no planner (raw question), SCA review loop on
# high/ultra: planner + prefetch + SCA↔rewriter iteration
- use_graph = thinking_mode in ("medium", "high", "ultra")
- enable_sca = thinking_mode in ("medium", "high", "ultra")
- use_fanout = thinking_mode in ("high", "ultra")
- _LOG.info("[Agentic RAG] mode=%s sca=%s fanouts=%s", thinking_mode or "low", enable_sca, use_fanout)
+ # low has no agentic graph at all (direct_search), and an unrecognised label
+ # resolves to the naive spec — also non-agentic.
+ use_graph = mode.agentic
+ enable_sca = mode.enable_sca
+ use_fanout = mode.use_fanout
+
+ # An unrecognised mode label degrades to plain retrieval instead of failing:
+ # answer from one retrieve pass with no agentic graph whatsoever.
+ if mode is NAIVE:
+ _LOG.warning("[Agentic RAG] unrecognised thinking mode; falling back to naive (non-agentic) retrieval")
+ async for tok in _naive_rag(tools, messages, gen_conf=gen_conf):
+ yield tok
+ return
+
+ _LOG.info("[Agentic RAG] mode=%s sca=%s fanouts=%s", thinking_mode, enable_sca, use_fanout)
if use_graph:
graph = build_agentic_graph(tools, token_queue, gen_conf=gen_conf, enable_sca=enable_sca, use_fanout=use_fanout)
diff --git a/rag/advanced_rag/harness/action_session.py b/rag/advanced_rag/harness/action_session.py
index 5213707319..3ccf72f6ff 100644
--- a/rag/advanced_rag/harness/action_session.py
+++ b/rag/advanced_rag/harness/action_session.py
@@ -31,11 +31,11 @@ from langchain_core.messages import HumanMessage, SystemMessage, convert_to_open
from langgraph.graph import END, START, StateGraph
from langgraph.graph.message import add_messages
+from rag.advanced_rag.harness.config import resolve_mode
+
_LOG = logging.getLogger(__name__)
_INIT_TIMEOUT_S = 45.0
-_ACTION_MAX_TURNS = 4
-_ACTION_MAX_TURNS_ULTRA = 6 # ultra runs deeper inside each action session
_ACTION_TIMEOUT_S = 75.0
_SNIPPETS_PER_QUERY = 4
_MAX_TOOL_RESPONSE_CHARS = 12000
@@ -384,10 +384,6 @@ _GRAPH_EXPLORE_TOOL_SPEC = {
},
}
-# graph_explore is ULTRA-ONLY: relational graph exploration is the extra depth
-# that distinguishes ultra from high (which stays on document/structure tools).
-_ULTRA_ONLY_TOOLS = {"graph_explore"}
-
_TOOL_MAP = {
"retrieve": _RETRIEVE_TOOL_SPEC,
"search_chunks": _SEARCH_CHUNKS_TOOL_SPEC,
@@ -404,7 +400,8 @@ def _active_tool_specs(tools) -> list:
"""Tool schemas exposed to the model for THIS mode.
Visibility rules:
- * ultra adds the relational ``graph_explore`` (high/medium do not see it).
+ * the per-mode tool set is declared in ``config.THINKING_MODES`` — ultra is
+ the only mode that sees the relational ``graph_explore``.
* ``web_search`` is hidden when NO web provider is configured — the model
otherwise retries it up to 4× per session burning turns (40 empty calls
observed in a 20-question run). Hiding beats any "do not use again" note.
@@ -419,10 +416,7 @@ def _active_tool_specs(tools) -> list:
The returned list is what the model can call this session.
"""
- mode = str(getattr(tools, "thinking_mode", "") or "").lower()
- names = set(_TOOL_MAP.keys())
- if mode != "ultra":
- names -= _ULTRA_ONLY_TOOLS
+ names = set(resolve_mode(tools).tools) & set(_TOOL_MAP.keys())
# Hide web_search without a provider — a hard visibility gate beats hoping
# the model obeys a "do not use" note across sessions.
if getattr(tools, "web_search", None) is None:
@@ -977,7 +971,16 @@ def _parse_tool_calls(msg) -> list:
if not isinstance(args, dict):
args = {}
if name not in _TOOL_MAP:
- _LOG.warning("[action_session] tool_call to unknown tool %r ignored", name)
+ # Do NOT drop it. OpenAI's protocol requires every assistant
+ # tool_call to be answered by a matching ``tool`` message — dropping
+ # one leaves a dangling tool_call and the next request is rejected
+ # ("tool call result does not follow tool call"). Keep it as a
+ # "unknown" call so _tool_node replies with a correction instead.
+ #
+ # Models most often emit the XML protocol tags (state / answer) as
+ # tool names; the hint below steers them back to plain-text output.
+ _LOG.warning("[action_session] tool_call to unknown tool %r; replying with a hint", name)
+ calls.append({"id": tc.id or f"call_{i}", "name": name, "args": args, "unknown": True})
continue
calls.append({"id": tc.id or f"call_{i}", "name": name, "args": args})
return calls
@@ -1155,6 +1158,24 @@ async def _tool_node(state: _SessionState) -> dict:
}
)
continue
+ # Unknown tool name — never execute it; answer with a correction so the
+ # model can recover. Models usually emit the XML protocol tags (state /
+ # answer) as tool names; they belong in the reply body as plain text.
+ if c.get("unknown"):
+ hint = (
+ f"'{c['name']}' is not a tool. State patches and final answers are "
+ "plain TEXT in your reply body, wrapped in <state>...</state> or "
+ "<answer>...</answer> XML tags — do not emit them as tool calls. "
+ f"Available tools: {', '.join(sorted(_TOOL_MAP))}."
+ )
+ tool_msgs.append(
+ {
+ "role": "tool",
+ "tool_call_id": c["id"],
+ "content": json.dumps({"passages": [{"kind": "error", "note": hint}]}, ensure_ascii=False),
+ }
+ )
+ continue
# same-session tool cache: avoid re-grep/list_chunks on the same target
cache_key = (c["name"], json.dumps(c["args"], sort_keys=True, ensure_ascii=False))
if cache_key in cache:
@@ -1277,9 +1298,7 @@ def _action_max_turns(state: _SessionState) -> int:
"""Per-mode action-session turn budget: ultra goes deeper inside each
session (more tool calls before the finalize fallback) — the latency cost
of the relational depth that distinguishes it from high."""
- tools = state.get("tools")
- mode = str(getattr(tools, "thinking_mode", "") or "").lower()
- return _ACTION_MAX_TURNS_ULTRA if mode == "ultra" else _ACTION_MAX_TURNS
+ return resolve_mode(state.get("tools")).action_max_turns
def _route(state: _SessionState) -> str:
diff --git a/rag/advanced_rag/harness/chunk_utils.py b/rag/advanced_rag/harness/chunk_utils.py
new file mode 100644
index 0000000000..a2a919e63c
--- /dev/null
+++ b/rag/advanced_rag/harness/chunk_utils.py
@@ -0,0 +1,55 @@
+"""Shared chunk field accessors for the agentic-RAG harness.
+
+Chunk rows reach the harness from several places (hybrid search, grep, compiled
+structure expansion, navigation outlines) and carry the same fields under a few
+legacy aliases. These accessors are the single definition used by the search and
+navigation tools — previously each module carried its own byte-identical copy,
+which had already drifted for ``_chunk_id`` (str vs int) and silently broke
+de-duplication when the two were mixed.
+"""
+
+from typing import Any
+
+
+def _xml_escape(value: Any) -> str:
+ """Escape a value for embedding in an XML attribute/element."""
+ s = "" if value is None else str(value)
+ return s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
+
+
+def _chunk_text(c: dict) -> str:
+ """Chunk body text across the historical content field aliases."""
+ return str(c.get("content_with_weight") or c.get("content") or c.get("text") or "")
+
+
+def _chunk_attr(c: dict, keys: tuple[str, ...]) -> str:
+ """First non-empty value among ``keys`` (legacy field aliases)."""
+ for k in keys:
+ v = c.get(k)
+ if v not in (None, ""):
+ return str(v)
+ return ""
+
+
+def _doc_id(c: dict) -> str:
+ return _chunk_attr(c, ("doc_id", "docid", "document_id"))
+
+
+def _dataset_id(c: dict) -> str:
+ return _chunk_attr(c, ("dataset_id", "kb_id", "knowledgebase_id"))
+
+
+def _doc_title(c: dict) -> str:
+ return _chunk_attr(c, ("docnm_kwd", "doc_title", "title", "document_name"))
+
+
+def _chunk_id(c: dict) -> str:
+ return _chunk_attr(c, ("chunk_id", "id"))
+
+
+def _snippet(s: str, n: int) -> str:
+ """Truncate a string to ``n`` chars on a char boundary with an ellipsis."""
+ s = (s or "").strip()
+ if len(s) <= n:
+ return s
+ return s[:n].rstrip() + "..."
diff --git a/rag/advanced_rag/harness/config.py b/rag/advanced_rag/harness/config.py
index b239c9dff8..4d906a7e85 100644
--- a/rag/advanced_rag/harness/config.py
+++ b/rag/advanced_rag/harness/config.py
@@ -1,41 +1,133 @@
-"""Thinking mode configurations.
-
-medium/high/ultra all run the single-agent SCA graph (see
-``agentic_rag_graph.build_agentic_graph``); the ONLY thing the ReAct/SCA path
-consumes from the strategy is ``available_tools`` (via ``get_mode("medium")``)
-and ``label``. low runs the lightweight ``direct_search`` graph, which reads
-``execution_strategy`` (=="direct_search") and ``available_tools``. All other
-fields are legacy data and never read by any running path.
+"""Thinking-mode configuration: the single authority for mode behaviour.
+
+Every mode-dependent decision in the harness reads from this module instead of
+re-deriving it from a ``thinking_mode`` string. The decisions that used to be
+hard-coded in four places are:
+
+===================== ========================================== =========================
+decision previously hard-coded in now
+===================== ========================================== =========================
+graph + SCA + fanout ``agentic_rag_graph.run_agentic_rag`` ``ModeSpec.graph`` /
+ ``enable_sca`` /
+ ``use_fanout``
+SCA iteration budget ``agentic_rag_graph.build_agentic_graph`` ``sca_max_rounds``
+tool surface ``action_session._active_tool_specs`` ``tools``
+session turn budget ``action_session._action_max_turns`` ``action_max_turns``
+===================== ========================================== =========================
+
+The values below reproduce the previous behaviour exactly, so this is a
+structural change only — tune a mode by editing one table.
+
+Unknown labels: ``resolve_mode`` falls back to ``NAIVE`` rather than raising.
+The mode comes from user input at request time, and raising would fail the whole
+request; a naive (non-agentic) answer degrades gracefully instead.
"""
-from rag.advanced_rag.harness.types import ExecutionStrategy
+from dataclasses import dataclass, field
+
+# Tools the action session can bind, in the order they are declared. Kept here
+# so a mode's tool set is data, not an if-chain in the runtime.
+_ALL_TOOLS = (
+ "retrieve",
+ "search_chunks",
+ "list_chunks",
+ "navigate_tree",
+ "navigate_structure",
+ "calculate",
+ "web_search",
+)
+# Relational exploration is the extra depth reserved for ultra.
+_GRAPH_EXPLORE = "graph_explore"
+
+
+@dataclass(frozen=True)
+class ModeSpec:
+ """Everything that varies between thinking modes.
+
+ :param label: the mode's own name.
+ :param agentic: run the agentic graph. When False the caller must answer
+ without it (naive retrieval) — see :data:`NAIVE`.
+ :param enable_sca: run the sufficiency-check review loop.
+ :param sca_max_rounds: SCA→rewriter iteration budget (0 when disabled).
+ :param use_fanout: planner + prefetch decomposition (multi-slot research).
+ :param action_max_turns: turns inside one action session.
+ :param tools: tool names visible to the model in this mode. Empty means the
+ model gets no tool loop at all.
+ """
-THINKING_MODES: dict[str, ExecutionStrategy] = {
- "low": ExecutionStrategy(
+ label: str
+ agentic: bool = True
+ enable_sca: bool = True
+ sca_max_rounds: int = 3
+ use_fanout: bool = False
+ action_max_turns: int = 4
+ tools: frozenset[str] = field(default_factory=lambda: frozenset(_ALL_TOOLS))
+
+
+def _tools(*names: str) -> frozenset[str]:
+ return frozenset(names)
+
+
+THINKING_MODES: dict[str, ModeSpec] = {
+ # low: one hybrid-search pass through ``direct_search``. No action session,
+ # so no tool loop — the model never sees tools in this mode.
+ "low": ModeSpec(
label="low",
- execution_strategy="direct_search",
- available_tools=["hybrid_search", "web_search", "bm25_search"],
+ agentic=False,
+ enable_sca=False,
+ sca_max_rounds=0,
+ use_fanout=False,
+ action_max_turns=4,
+ tools=frozenset(),
),
- "medium": ExecutionStrategy(
+ # medium: agentic, SCA review on, no planner decomposition.
+ "medium": ModeSpec(
label="medium",
- execution_strategy="direct_search", # legacy; SCA path does not read it
- available_tools=["grep_search", "list_chunks", "web_search"],
+ action_max_turns=4,
+ sca_max_rounds=3,
+ use_fanout=False,
+ tools=_tools(*_ALL_TOOLS),
),
- "high": ExecutionStrategy(
+ # high: adds planner + prefetch fan-out over the same tool surface.
+ "high": ModeSpec(
label="high",
- execution_strategy="direct_search", # legacy; SCA path does not read it
- available_tools=["grep_search", "list_chunks", "web_search"],
+ action_max_turns=4,
+ sca_max_rounds=3,
+ use_fanout=True,
+ tools=_tools(*_ALL_TOOLS),
),
- "ultra": ExecutionStrategy(
+ # ultra: deeper sessions, more SCA rounds, and the relational tool.
+ "ultra": ModeSpec(
label="ultra",
- execution_strategy="direct_search", # legacy; SCA path does not read it
- available_tools=["grep_search", "list_chunks", "web_search"],
+ action_max_turns=6,
+ sca_max_rounds=5,
+ use_fanout=True,
+ tools=_tools(*_ALL_TOOLS, _GRAPH_EXPLORE),
),
}
+#: Fallback for an unrecognised mode label. Not agentic — the caller answers
+#: with plain retrieval rather than failing the request.
+NAIVE = ModeSpec(
+ label="naive",
+ agentic=False,
+ enable_sca=False,
+ sca_max_rounds=0,
+ use_fanout=False,
+ action_max_turns=4,
+ tools=frozenset(),
+)
+
+
+def get_mode(label: str) -> ModeSpec:
+ """Return the spec for ``label``.
+
+ Unknown labels fall back to :data:`NAIVE` (non-agentic) — the label arrives
+ from user input, and raising here would fail the request outright.
+ """
+ return THINKING_MODES.get(str(label or "").strip().lower(), NAIVE)
+
-def get_mode(label: str) -> ExecutionStrategy:
- mode = THINKING_MODES.get(label)
- if not mode:
- raise ValueError(f"Unknown thinking mode: {label}. Available: {list(THINKING_MODES.keys())}")
- return mode
+def resolve_mode(tools) -> ModeSpec:
+ """Read a ``RAGTools``-like object's ``thinking_mode`` into its spec."""
+ return get_mode(getattr(tools, "thinking_mode", ""))
diff --git a/rag/advanced_rag/harness/orchestrator/__init__.py b/rag/advanced_rag/harness/orchestrator/__init__.py
index 87be91a1b2..9066d26600 100644
--- a/rag/advanced_rag/harness/orchestrator/__init__.py
+++ b/rag/advanced_rag/harness/orchestrator/__init__.py
@@ -1,29 +1,12 @@
-"""Orchestrator loop — dispatches to execution strategy based on thinking mode."""
+"""Orchestration strategies for the agentic graph.
-import logging
+Each module here is one strategy the graph can run; they are imported lazily by
+``agentic_rag_graph`` at the point of use:
-from rag.advanced_rag.harness.config import get_mode
-from rag.advanced_rag.harness.orchestrator.direct import direct_search
-from rag.advanced_rag.harness.stats import in_phase
+* :mod:`direct` — single-pass retrieval (the low path).
+* :mod:`sufficient_context` — sufficiency review + gap-driven rewrites.
+* :mod:`query_rewriter` — turns a reported gap into a follow-up query.
-_LOG = logging.getLogger(__name__)
-
-
-@in_phase("orchestrator")
-async def orchestrator_loop(state: dict, tools) -> dict:
- """Main orchestrator — dispatch to strategy based on thinking mode."""
- route = state.get("route")
- if not route:
- _LOG.warning("orchestrator: no route, using direct_search")
- return await direct_search(state, tools)
-
- mode_label = route.thinking_mode if isinstance(route, dict) else route.thinking_mode
- mode = get_mode(mode_label)
-
- _LOG.info('[Orchestrator] Researching with the "%s" approach (%s thinking).', mode.execution_strategy, mode_label)
-
- if mode.execution_strategy == "direct_search":
- return await direct_search(state, tools)
-
- _LOG.warning("orchestrator: unknown strategy %s, fallback to direct", mode.execution_strategy)
- return await direct_search(state, tools)
+Which strategies a mode runs is decided in ``harness.config.THINKING_MODES``, not
+here — see ``ModeSpec.enable_sca`` / ``use_fanout``.
+"""
diff --git a/rag/advanced_rag/harness/pipeline.py b/rag/advanced_rag/harness/pipeline.py
deleted file mode 100644
index 3da9f154c0..0000000000
--- a/rag/advanced_rag/harness/pipeline.py
+++ /dev/null
@@ -1,155 +0,0 @@
-"""Pipeline — unified tool execution dispatcher."""
-
-import logging
-import time
-from typing import Any
-
-from rag.advanced_rag.harness.tools.registry import TOOL_REGISTRY
-from rag.advanced_rag.harness.types import ToolResult
-
-_LOG = logging.getLogger(__name__)
-
-# Tools that retrieve *within* a set of documents. When a routing tool
-# (``dataset_navigation_search``) has produced a relevant-document set, these
-# inherit it as their ``doc_scope`` unless the caller passed one explicitly, so
-# a follow-up search stays within the routed docs instead of re-scanning the KB.
-_DOC_SCOPE_CONSUMERS = {"ontology_navigate", "mindmap_navigate", "graph_explore", "hybrid_search", "vector_search", "bm25_search", "structured_query", "dataset_navigation_search"}
-
-
-class Pipeline:
- """Unified tool execution layer.
-
- - execute(tool_name, **kwargs): dispatch to registered tool, normalize result
- - available_tools(mode_tools): return LLM-visible tool definitions (compilation-filtered)
- - get_chunks(evidence_ids): retrieve raw chunks for sufficiency cross-check
- - trace: execution history for auditing
- """
-
- def __init__(self, rag_tools, compilation_map: dict[str, set[str]] | None = None):
- self.tools = rag_tools
- self.compilation_map = compilation_map or {}
- self.trace: list[dict] = []
- # Latest relevant-document set produced by a routing tool this run.
- self._routed_docs: list[str] = list(getattr(rag_tools, "doc_scope", None) or [])
-
- async def execute(self, tool_name: str, **kwargs) -> ToolResult:
- """Execute a registered tool by name."""
- tool = TOOL_REGISTRY.get(tool_name)
- if not tool:
- return ToolResult(chunks=[], metadata={}, error=f"Unknown tool: {tool_name}")
-
- fn = tool.get("fn")
- if not fn:
- return ToolResult(chunks=[], metadata={}, error=f"Tool {tool_name} has no executor")
-
- # Downstream scoping: a within-document tool inherits the doc IDs a prior
- # router (dataset_navigation_search) produced, unless the caller passed
- # an explicit doc_scope.
- if tool_name in _DOC_SCOPE_CONSUMERS and self._routed_docs and not kwargs.get("doc_scope"):
- kwargs["doc_scope"] = list(self._routed_docs)
-
- start = time.time()
- try:
- raw = await fn(self.tools, **kwargs)
- elapsed = time.time() - start
- self.trace.append({"tool": tool_name, "args": kwargs, "elapsed": elapsed, "success": True})
- result = self._normalize(raw)
- # A routing tool (e.g. dataset_navigation_search) yields the relevant
- # document IDs; remember them so the scope-consuming tools above can
- # inherit them on later turns.
- if result.docs:
- if hasattr(self.tools, "scoped_doc_ids"):
- self._routed_docs = self.tools.scoped_doc_ids(list(result.docs)) or []
- else:
- self._routed_docs = list(result.docs)
- # Feed the shared citation pool: agent searches go through the
- # pipeline, so without this their evidence never reaches kbinfos and
- # the final answer has nothing to cite.
- self._merge_into_kbinfos(result)
- return result
- except Exception as e:
- elapsed = time.time() - start
- _LOG.exception("Pipeline.execute(%s) failed", tool_name)
- self.trace.append({"tool": tool_name, "args": kwargs, "elapsed": elapsed, "success": False, "error": str(e)})
- return ToolResult(chunks=[], metadata={}, error=str(e))
-
- def available_tools(self, mode_tools: list[str]) -> list[dict]:
- """Return LLM-visible tool definitions, filtered by compilation availability."""
- names = filter_available_tools(mode_tools, self.compilation_map)
- defs = []
- for name in names:
- tool = TOOL_REGISTRY.get(name)
- if tool and tool.get("function_schema"):
- defs.append(tool["function_schema"])
- return defs
-
- def get_chunks(self, evidence_ids: list[int]) -> dict[int, dict]:
- """Retrieve raw chunks by ID from current kbinfos."""
- result = {}
- chunks = self.tools.kbinfos.get("chunks", [])
- for eid in evidence_ids:
- if 0 <= eid < len(chunks):
- result[eid] = chunks[eid]
- return result
-
- def get_trace(self) -> list[dict]:
- return list(self.trace)
-
- # ── Private ──
-
- def _merge_into_kbinfos(self, result: ToolResult) -> None:
- """Merge a tool result's chunks/doc_aggs into ``tools.kbinfos``, deduped."""
- if not result or not result.chunks:
- return
- kb = self.tools.kbinfos
- seen = {c.get("chunk_id") or c.get("id") or id(c) for c in kb.get("chunks", [])}
- for c in result.chunks:
- k = c.get("chunk_id") or c.get("id") or id(c)
- if k in seen:
- continue
- seen.add(k)
- kb.setdefault("chunks", []).append(c)
- aggs = result.metadata.get("aggs") if isinstance(result.metadata, dict) else None
- if aggs:
- dseen = {d.get("doc_id") for d in kb.get("doc_aggs", [])}
- for d in aggs:
- if d.get("doc_id") in dseen:
- continue
- dseen.add(d.get("doc_id"))
- kb.setdefault("doc_aggs", []).append(d)
-
- @staticmethod
- def _normalize(raw: Any) -> ToolResult:
- if isinstance(raw, ToolResult):
- return raw
- if isinstance(raw, dict):
- return ToolResult(
- chunks=raw.get("chunks", []),
- docs=raw.get("docs") or None,
- metadata={"aggs": raw.get("doc_aggs", []), "answer": raw.get("answer", "")},
- )
- if isinstance(raw, list):
- # A list of doc-id strings is a document-routing result (e.g.
- # dataset_navigation_search); a list of dicts is chunks.
- if raw and all(isinstance(x, str) for x in raw):
- return ToolResult(docs=list(raw), metadata={})
- return ToolResult(chunks=raw, metadata={})
- return ToolResult(chunks=[], metadata={"raw": str(raw)})
-
-
-def filter_available_tools(tool_names: list[str], compilation_map: dict[str, set[str]]) -> list[str]:
- """Filter tool list by compilation artifact availability."""
- available = []
- for name in tool_names:
- tool = TOOL_REGISTRY.get(name)
- if not tool:
- continue
- if tool.get("requires_compilation"):
- comp_type = tool.get("compilation_type")
- # ``compilation_type`` may name one artifact or several (a tool that
- # reads either one is available when ANY of them is compiled).
- wanted = {comp_type} if isinstance(comp_type, str) else set(comp_type or ())
- if wanted and not any(wanted & comps for comps in compilation_map.values()):
- continue
- available.append(name)
- return available
diff --git a/rag/advanced_rag/harness/structure_qa.py b/rag/advanced_rag/harness/structure_qa.py
new file mode 100644
index 0000000000..f440464de2
--- /dev/null
+++ b/rag/advanced_rag/harness/structure_qa.py
@@ -0,0 +1,101 @@
+"""Ask the chat model to answer a question from a compiled-structure outline.
+
+Shared by the two navigation paths that read compiled rows:
+
+* document structure navigation (``navigation._navigate_within_doc``) — the
+ document's catalog / mindmap outline;
+* knowledge-graph exploration (``navigation.graph_explore``) — the compiled
+ knowledge graph.
+
+Both render entities + relations into the same compact outline, ask the model
+whether that outline alone answers the question, and use the returned
+``relevant_entities`` to pull the underlying source chunks even when the outline
+is NOT sufficient (so evidence still flows back to the caller).
+
+Lives apart from ``navigation`` because it is pure prompt/render work with no
+store access, and because both callers would otherwise import it from a module
+that in turn imports them.
+"""
+
+import logging
+import re
+
+import json_repair
+
+_LOG = logging.getLogger(__name__)
+
+# Cap how much compiled structure we render into the prompt.
+_MAX_ENTITIES = 300
+_MAX_RELATIONS = 300
+
+_NAV_SYSTEM = """You are given {noun} of one or more documents — an outline of entities and their relations — and a question.
+
+Decide whether that outline alone already answers the question.
+
+Rules:
+1. Answer ONLY from the outline below. Do not invent facts.
+2. Set "is_sufficient" to true only when the outline genuinely answers the question; otherwise false with an empty answer.
+3. Always fill "relevant_entities" with the exact `name` values of the entities most related to the question (up to 10), even when the outline is not sufficient — they are used to pull the underlying source text.
+
+Output ONLY JSON, no prose, no code fences:
+{{"is_sufficient": true/false, "answer": "<answer, or empty>", "relevant_entities": ["<entity name>", ...]}}"""
+
+
+def _render_structure(entities: list[dict], relations: list[dict]) -> str:
+ """Render the compiled structure as a compact outline for the prompt."""
+ lines: list[str] = []
+ if entities:
+ lines.append("Entities:")
+ for e in entities[:_MAX_ENTITIES]:
+ name = (e.get("name") or "").strip()
+ if not name:
+ continue
+ typ = (e.get("type") or "other").strip()
+ desc = " ".join((e.get("description") or "").split())
+ lines.append(f"- {name} ({typ})" + (f": {desc}" if desc else ""))
+ if relations:
+ lines.append("\nRelations:")
+ for r in relations[:_MAX_RELATIONS]:
+ src, tgt = (r.get("from") or "").strip(), (r.get("to") or "").strip()
+ if not src or not tgt:
+ continue
+ lines.append(f"- {src} -[{(r.get('type') or 'related').strip()}]-> {tgt}")
+ return "\n".join(lines)
+
+
+async def _ask_structure(tools, topic: str, entities: list[dict], relations: list[dict], noun: str, label: str) -> tuple[str, list[str]]:
+ """Ask the chat model to answer ``topic`` from the rendered outline.
+
+ Returns ``(answer, relevant_entity_names)`` — ``answer`` is empty unless the
+ model judged the outline sufficient; the names are always returned so the
+ caller can pull the underlying source chunks.
+ """
+ verdict = {}
+ try:
+ from rag.prompts.generator import form_message, message_fit_in
+
+ user = f"Question:\n{topic}\n\n{noun.capitalize()}:\n{_render_structure(entities, relations)}\n\nOutput JSON:"
+ _, msg = message_fit_in(form_message(_NAV_SYSTEM.format(noun=f"the {noun}"), user), tools.chat_mdl.max_length)
+ ans = await tools.chat_mdl.async_chat(msg[0]["content"], msg[1:], {"temperature": 0.2})
+ if isinstance(ans, tuple):
+ ans = ans[0]
+ cleaned = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
+ cleaned = re.sub(r"```(?:json)?\s*|\s*```", "", cleaned).strip()
+ verdict = json_repair.loads(cleaned) or {}
+ if not isinstance(verdict, dict):
+ verdict = {}
+ except Exception:
+ _LOG.exception(f"[{label}] Could not read the outline with the model.")
+
+ is_sufficient = bool(verdict.get("is_sufficient"))
+ answer = (verdict.get("answer") or "").strip() if is_sufficient else ""
+ relevant = [n for n in (verdict.get("relevant_entities") or []) if isinstance(n, str)]
+ _LOG.info(
+ "[%s] The %s %s the question; %d relevant entity(ies): %s",
+ label,
+ noun,
+ "answers" if is_sufficient else "does not fully answer",
+ len(relevant),
+ ", ".join(relevant[:10]) or "none",
+ )
+ return answer, relevant
diff --git a/rag/advanced_rag/harness/tools/__init__.py b/rag/advanced_rag/harness/tools/__init__.py
index e2dc4bc145..79321f0e1f 100644
--- a/rag/advanced_rag/harness/tools/__init__.py
+++ b/rag/advanced_rag/harness/tools/__init__.py
@@ -1,126 +1,20 @@
-"""Tool system: register all tools with the registry on import."""
-
-from rag.advanced_rag.harness.tools.registry import _inspector_schema, _navigate_schema, _search_schema, register_tool
-
-# Register tools
-# Search tools
-from rag.advanced_rag.harness.tools.search import (
- bm25_search,
- grep_search,
- hybrid_search,
- list_chunks,
- structured_query,
- vector_search,
- web_search,
-)
-
-register_tool("hybrid_search", _search_schema("hybrid_search", "Embedding + Keywords search"), hybrid_search)
-register_tool("vector_search", _search_schema("vector_search", "Embedding search"), vector_search)
-register_tool("bm25_search", _search_schema("bm25_search", "Keywords search"), bm25_search)
-register_tool("web_search", _search_schema("web_search", "Internet search"), web_search)
-register_tool("structured_query", _search_schema("structured_query", "SQL search"), structured_query)
-
-# Grep-style exact locate + deep-read (used by the sub-agent, mirrors dynamic's
-# grep_chunks / list_chunks but returns the Pipeline {"chunks": [...]} contract).
-register_tool(
- "grep_search",
- {
- "type": "function",
- "function": {
- "name": "grep_search",
- "description": (
- "Exact keyword/pattern locate: keyword-first (BM25) candidate retrieval, then a "
- "case-insensitive regex locate returning COMPACT snippet chunks (token-cheap). Use "
- "for exact names / ids / numbers. When nothing matches it returns the candidates unchanged."
- ),
- "parameters": {
- "type": "object",
- "properties": {
- "query": {"type": "string", "description": "the regex/keywords to locate (case-insensitive)."},
- },
- "required": ["query"],
- },
- },
- },
- grep_search,
-)
-register_tool(
- "list_chunks",
- {
- "type": "function",
- "function": {
- "name": "list_chunks",
- "description": (
- "Deep-read a document: return its COMPLETE chunk list in reading order. Call after "
- "grep_search / hybrid_search hits a document when you need the full text (enumeration / "
- "count / arithmetic answers)."
- ),
- "parameters": {
- "type": "object",
- "properties": {
- "doc_id": {"type": "string", "description": "the owning document id to read fully."},
- },
- "required": ["doc_id"],
- },
- },
- },
- list_chunks,
-)
-
-# Navigation tools (require compilation)
-from rag.advanced_rag.harness.tools.navigation import dataset_navigation_search, mindmap_navigate, ontology_navigate
-
-# ontology_navigate covers both the tree/TOC outline and the page index.
-register_tool(
- "ontology_navigate",
- _navigate_schema("ontology_navigate", "Get question-related chunks from the given documents catalog (tree / page index / mind map). It always follows dataset navigation."),
- ontology_navigate,
- requires_compilation=True,
- compilation_type=("toc", "page_index", "mindmap"),
-)
-register_tool(
- "mindmap_navigate", _navigate_schema("mindmap_navigate", "Get question-related chunks from the document's mindmap"), mindmap_navigate, requires_compilation=True, compilation_type="mindmap"
-)
-register_tool(
- "dataset_navigation_search",
- _navigate_schema(
- "dataset_navigation_search",
- "Find the documents most relevant to the question by hybrid-searching the dataset's navigation-tree document leaves (nav_doc layer) — only documents that have a compiled nav-tree node are reachable. Returns the most relevant document IDs.",
- ),
- dataset_navigation_search,
- requires_compilation=True,
- compilation_type="tree",
-)
-
-# Exploration tools (require compilation)
-from rag.advanced_rag.harness.tools.exploration import graph_explore, wiki_query
-
-register_tool("graph_explore", _search_schema("graph_explore", "Knowledge graph exploration"), graph_explore, requires_compilation=True, compilation_type="knowledge_graph")
-register_tool("wiki_query", _search_schema("wiki_query", "Wiki search"), wiki_query, requires_compilation=True, compilation_type="wiki")
-
-# Inspector tools
-from rag.advanced_rag.harness.tools.inspector import compare_sources, grep_within, open_context, request_adjacent
-
-register_tool(
- "inspector_open_context",
- _inspector_schema("open_context", "Expand the original context around a chunk", {"chunk_id": {"type": "string"}, "width": {"type": "integer", "description": "Number of characters to expand"}}),
- open_context,
-)
-register_tool(
- "inspector_compare",
- _inspector_schema("compare_sources", "Compare multiple chunks to find common points and contradictions", {"chunk_ids": {"type": "array", "items": {"type": "string"}}}),
- compare_sources,
-)
-register_tool(
- "inspector_grep_within", _inspector_schema("grep_within", "Search for an exact keyword or pattern within a document", {"doc_id": {"type": "string"}, "pattern": {"type": "string"}}), grep_within
-)
-register_tool(
- "inspector_request_adjacent",
- _inspector_schema(
- "request_adjacent", "Get adjacent entries before or after a chunk", {"chunk_id": {"type": "string"}, "direction": {"type": "string", "enum": ["prev", "next"]}, "count": {"type": "integer"}}
- ),
- request_adjacent,
-)
-
-# Built-in agent tools
-# (generate_report and think_tool are handled by the agent loop itself, not by Pipeline)
+"""Retrieval and navigation tool implementations.
+
+This package holds the *implementations* the action-session runtime calls:
+
+* :mod:`search` — hybrid / BM25 / grep retrieval plus compiled-structure expansion.
+* :mod:`navigation` — dataset navigation tree, document structure navigation and
+ knowledge-graph exploration.
+* :mod:`exploration` — re-exports ``graph_explore`` and hosts ``wiki_query``.
+
+Tool *schemas* and per-turn dispatch live in
+:mod:`rag.advanced_rag.harness.action_session` (``_TOOL_MAP`` / ``execute_tool``),
+which is what the model actually sees.
+
+Note: a declarative ``TOOL_REGISTRY`` (registry.py + gating.py + pipeline.py)
+previously lived here and registered 17 tools on import. It was never read by any
+live path — the only consumers were each other and a pipeline with no external
+callers — while advertising tool names (``ontology_navigate``, ``mindmap_navigate``)
+that no longer existed in the code. It has been removed; ``_active_tool_specs``
+is now the single place that decides the visible tool surface.
+"""
diff --git a/rag/advanced_rag/harness/tools/compiled_expansion.py b/rag/advanced_rag/harness/tools/compiled_expansion.py
new file mode 100644
index 0000000000..f4f1bbd9f3
--- /dev/null
+++ b/rag/advanced_rag/harness/tools/compiled_expansion.py
@@ -0,0 +1,489 @@
+"""Compiled-product expansion for hybrid search (zero-LLM).
+
+When ``hybrid_search`` runs with ``use_compiled=True`` this module layers the
+dataset's *compiled* products on top of the retrieved chunks: per-kind compiled
+structure rows (page_index / timeline / mind_map / knowledge_graph / tree), the
+synthesised pages (wiki / artifact / essence), and wiki page drill-downs.
+
+Datasets with no compiled products are unaffected — every strategy short-circuits
+when its rows are absent.
+
+Lives apart from ``search`` because it is a self-contained retrieval strategy
+with its own row shapes, not part of the core hybrid/vector/BM25 legs.
+"""
+
+import logging
+
+# ``_kg_scopes`` resolves the (kb_id, tenant_id) pairs to scan; it lives in
+# ``navigation`` because that module owns the compiled-row lookup helpers.
+# Everything else (settings / thread pool / OrderByExpr) is imported lazily
+# inside each strategy, matching the original module's style.
+from rag.advanced_rag.harness.tools.navigation import _kg_scopes
+
+_LOG = logging.getLogger(__name__)
+
+# ─── Compiled product expansion (zero-LLM, used by hybrid_search with use_compiled=True) ───
+
+
+async def _expand_with_compiled(tools, query: str, keywords: str, kbinfos: dict, doc_scope: list[str] | None = None) -> None:
+ """Zero-LLM compiled-product expansion: page_index → tree → KG.
+
+ For each bound KB, searches compiled entity rows matching the query,
+ hops 1-hop via relations to find neighbour entities, then appends
+ their source passages to ``kbinfos["chunks"]``.
+ """
+ before = len(kbinfos.get("chunks", []))
+ seen_ids = {c.get("chunk_id") or c.get("id") for c in kbinfos.get("chunks", [])}
+
+ scopes = await _kg_scopes(tools, doc_scope)
+ if not scopes:
+ return
+
+ for kb_id, tenant_id, doc_ids in scopes:
+ # 1-hop entity-graph expansion per template kind.
+ # Each template writes entity/relation rows tagged with
+ # ``compilation_template_kind_kwd`` — search them independently.
+ for label, template_kind in (
+ ("knowledge_graph", "knowledge_graph"),
+ ("mind_map", "mind_map"),
+ ("timeline", "timeline"),
+ ("page_index", "page_index"),
+ ):
+ chunks = await _expand_compiled_strategy(
+ tools,
+ kb_id,
+ tenant_id,
+ doc_ids,
+ query,
+ seen_ids,
+ template_kind=template_kind,
+ max_chunks=5,
+ )
+ if chunks:
+ kbinfos.setdefault("chunks", []).extend(chunks)
+ _LOG.debug("[Compiled expand] %s: +%d chunks", label, len(chunks))
+
+ # Tree structure graph (uses ``compile_kwd``, not template kind).
+ chunks = await _expand_compiled_strategy(
+ tools,
+ kb_id,
+ tenant_id,
+ doc_ids,
+ query,
+ seen_ids,
+ compile_kwd="tree",
+ max_chunks=5,
+ )
+ if chunks:
+ kbinfos.setdefault("chunks", []).extend(chunks)
+ _LOG.debug("[Compiled expand] tree: +%d chunks", len(chunks))
+
+ # Synthesis pages — standalone rendered articles from wiki / session
+ # graph / session essence templates. Searched directly (no entity-graph nav).
+ for label, ckwd in (
+ ("wiki_page", "wiki_page"),
+ ("artifact_page", "artifact_page"),
+ ("essence", "essence"),
+ ):
+ chunks = await _expand_wiki_page_strategy(
+ tools,
+ kb_id,
+ tenant_id,
+ doc_ids,
+ query,
+ seen_ids,
+ compile_kwd=ckwd,
+ max_chunks=5,
+ )
+ if chunks:
+ kbinfos.setdefault("chunks", []).extend(chunks)
+ _LOG.debug("[Compiled expand] %s: +%d chunks", label, len(chunks))
+
+ # Re-sort so compiled-expansion chunks blend by similarity with regular ones.
+ chunks = kbinfos.get("chunks", [])
+ if chunks:
+ chunks.sort(key=lambda c: c.get("similarity", 0.0), reverse=True)
+
+ after = len(chunks)
+ _LOG.info("[Hybrid search] Compiled expansion added %d chunks.", after - before)
+
+
+async def _search_compiled_rows(
+ tools,
+ kb_id: str,
+ tenant_id: str,
+ doc_ids: list[str] | None,
+ kind: str,
+ *,
+ text: str = "",
+ top_n: int = 8,
+ extra: dict | None = None,
+ compile_kwd: str | None = None,
+ template_kind: str | None = None,
+) -> dict:
+ """Search compiled KG rows in one KB, returning raw field maps.
+
+ *compile_kwd* filters by the ``compile_kwd`` field (e.g. "tree" for tree
+ structure nodes). *template_kind* filters by ``compilation_template_kind_kwd``
+ (e.g. "knowledge_graph", "mind_map"). Leave both ``None`` to scan all rows.
+ """
+ from common import settings
+ from common.doc_store.doc_store_base import MatchTextExpr, OrderByExpr
+ from common.misc_utils import thread_pool_exec
+ from rag.nlp import search
+
+ condition: dict = {"knowledge_graph_kwd": [kind]}
+ if compile_kwd:
+ condition["compile_kwd"] = compile_kwd
+ if template_kind:
+ condition["compilation_template_kind_kwd"] = template_kind
+ if doc_ids:
+ condition["doc_id"] = list(doc_ids)
+ if extra:
+ condition.update(extra)
+
+ fields = [
+ "content_with_weight",
+ "source_chunk_ids",
+ "doc_id",
+ "docnm_kwd",
+ "from_entity_kwd",
+ "to_entity_kwd",
+ "name_kwd",
+ ]
+ exprs = []
+ if text:
+ embd_mdl = getattr(tools, "embed_mdl", None)
+ if embd_mdl:
+ try:
+ exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_n, 0.1))
+ except Exception:
+ _LOG.exception("[Compiled expand] vector build failed; using keyword match")
+ if not exprs:
+ exprs.append(
+ MatchTextExpr(
+ ["content_ltks", "content_sm_ltks"],
+ text,
+ top_n,
+ )
+ )
+
+ try:
+ res = await thread_pool_exec(
+ settings.docStoreConn.search,
+ fields,
+ [],
+ condition,
+ exprs,
+ OrderByExpr(),
+ 0,
+ top_n,
+ search.index_name(tenant_id),
+ [kb_id],
+ )
+ return settings.docStoreConn.get_fields(res, fields) or {}
+ except Exception:
+ _LOG.exception("[Compiled expand] search failed (kind=%s compile_kwd=%s)", kind, compile_kwd)
+ return {}
+
+
+async def _load_chunks_for_doc(tools, doc_id: str, chunk_ids: list[str]) -> list[dict]:
+ """Load chunks by their IDs from the doc store."""
+ if not chunk_ids:
+ return []
+ from common import settings
+ from common.doc_store.doc_store_base import OrderByExpr
+ from common.misc_utils import thread_pool_exec
+ from rag.nlp import search
+
+ resolved = tools._resolve_doc_tenant(doc_id)
+ if not resolved:
+ return []
+ kb_id, tenant_id = resolved
+
+ fields = ["content_with_weight", "docnm_kwd", "doc_id", "id"]
+ try:
+ res = await thread_pool_exec(
+ settings.docStoreConn.search,
+ fields,
+ [],
+ {"id": list(chunk_ids)},
+ [],
+ OrderByExpr(),
+ 0,
+ len(chunk_ids),
+ search.index_name(tenant_id),
+ [kb_id],
+ )
+ rows = settings.docStoreConn.get_fields(res, fields)
+ if not rows:
+ return []
+ return [{**v, "chunk_id": k} for k, v in rows.items()]
+ except Exception:
+ _LOG.exception("[Compiled expand] failed to load chunks for doc_id=%s", doc_id)
+ return []
+
+
+async def _expand_compiled_strategy(
+ tools,
+ kb_id: str,
+ tenant_id: str,
+ doc_ids: list[str] | None,
+ query: str,
+ seen_ids: set[str],
+ *,
+ compile_kwd: str | None = None,
+ template_kind: str | None = None,
+ max_chunks: int = 5,
+) -> list[dict]:
+ """Generic 1-hop compiled expansion: entity search → relation nav → chunk load.
+
+ 1. Embedding-match seed entities (filtered by *compile_kwd* or *template_kind*).
+ 2. Fetch relations adjacent to seed entities (forward + backward).
+ 3. Collect neighbour entity names (1-hop away).
+ 4. Look up neighbour entities to get ``source_chunk_ids``.
+ 5. Load actual chunks, deduplicate, respect *max_chunks*.
+
+ *compile_kwd* is used for structure graphs (e.g. "tree").
+ *template_kind* is used for entity extraction rows (e.g. "knowledge_graph").
+ """
+ import json
+
+ # -- 1. Seed entities --
+ seed_rows = await _search_compiled_rows(
+ tools,
+ kb_id,
+ tenant_id,
+ doc_ids,
+ "entity",
+ text=query,
+ top_n=5,
+ compile_kwd=compile_kwd,
+ template_kind=template_kind,
+ )
+ if not seed_rows:
+ return []
+
+ seed_names: set[str] = set()
+ for r in seed_rows.values():
+ try:
+ payload = json.loads(r.get("content_with_weight") or "{}")
+ except Exception:
+ continue
+ name = (payload.get("name") or payload.get("title") or "").strip()
+ if name:
+ seed_names.add(name)
+ if not seed_names:
+ return []
+
+ # -- 2. Adjacent relations (outgoing + incoming) --
+ # Provide both original and lowercased names — dataset_structure_merger
+ # lowercases merged-row endpoints while per-doc rows keep original case.
+ seed_list = sorted({n.lower() for n in seed_names} | seed_names)
+ fwd = await _search_compiled_rows(
+ tools,
+ kb_id,
+ tenant_id,
+ doc_ids,
+ "relation",
+ top_n=50,
+ compile_kwd=compile_kwd,
+ template_kind=template_kind,
+ extra={"from_entity_kwd": seed_list},
+ )
+ bwd = await _search_compiled_rows(
+ tools,
+ kb_id,
+ tenant_id,
+ doc_ids,
+ "relation",
+ top_n=50,
+ compile_kwd=compile_kwd,
+ template_kind=template_kind,
+ extra={"to_entity_kwd": seed_list},
+ )
+ all_rels = {**fwd, **bwd}
+
+ # -- 3. Neighbour names (1-hop, exclude seeds) --
+ seed_lower = {n.lower() for n in seed_names}
+ neighbour_names: set[str] = set()
+ for r in all_rels.values():
+ frm = (r.get("from_entity_kwd") or "").strip()
+ frm_lower = frm.lower()
+ to = (r.get("to_entity_kwd") or "").strip()
+ to_lower = to.lower()
+ if frm_lower in seed_lower and to and to_lower not in seed_lower:
+ neighbour_names.add(to)
+ if to_lower in seed_lower and frm and frm_lower not in seed_lower:
+ neighbour_names.add(frm)
+ if not neighbour_names:
+ return []
+
+ # -- 4. Neighbour entity source_chunk_ids --
+ # Provide both original and lowercased — same as seed_list above.
+ neigh_list = sorted({n.lower() for n in neighbour_names} | neighbour_names)
+ if len(neigh_list) > 100:
+ neigh_list = neigh_list[:100] # reasonable cap for name_kwd search
+ neigh_rows = await _search_compiled_rows(
+ tools,
+ kb_id,
+ tenant_id,
+ doc_ids,
+ "entity",
+ top_n=len(neigh_list),
+ compile_kwd=compile_kwd,
+ template_kind=template_kind,
+ extra={"name_kwd": neigh_list},
+ )
+
+ # Group chunk IDs by doc
+ by_doc: dict[str, set[str]] = {}
+ for r in neigh_rows.values():
+ doc_id = r.get("doc_id") or ""
+ for cid in r.get("source_chunk_ids") or []:
+ if cid and cid not in seen_ids:
+ by_doc.setdefault(doc_id, set()).add(cid)
+
+ # -- 5. Load and return --
+ new_chunks: list[dict] = []
+ for doc_id, cids in by_doc.items():
+ if len(new_chunks) >= max_chunks:
+ break
+ limit = max_chunks - len(new_chunks)
+ chunks = await _load_chunks_for_doc(tools, doc_id, list(cids)[:limit])
+ for c in chunks:
+ cid = c.get("chunk_id") or c.get("id")
+ if cid and cid not in seen_ids:
+ seen_ids.add(cid)
+ new_chunks.append(c)
+
+ return new_chunks
+
+
+async def _search_synthesis_pages(
+ tools,
+ kb_id: str,
+ tenant_id: str,
+ doc_ids: list[str] | None,
+ text: str,
+ *,
+ compile_kwd: str = "wiki_page",
+ top_n: int = 8,
+) -> dict:
+ """Search synthesis-compiled page rows (no knowledge_graph_kwd filter).
+
+ Synthesis pages are standalone articles (wiki_page, artifact_page,
+ essence, etc.) with ``content_with_weight``, keyword index, and
+ vector. They do NOT carry the ``knowledge_graph_kwd`` field (unlike
+ entity/relation rows from extraction).
+ """
+ from common import settings
+ from common.doc_store.doc_store_base import MatchTextExpr, OrderByExpr
+ from common.misc_utils import thread_pool_exec
+ from rag.nlp import search
+
+ condition: dict = {"compile_kwd": compile_kwd, "available_int": 1}
+ if doc_ids:
+ condition["source_doc_ids"] = list(doc_ids)
+
+ fields = [
+ "content_with_weight",
+ "summary_with_weight",
+ "source_chunk_ids",
+ "doc_id",
+ "title_kwd",
+ "topic_kwd",
+ ]
+
+ exprs = []
+ if text:
+ embd_mdl = getattr(tools, "embed_mdl", None)
+ if embd_mdl:
+ try:
+ exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_n, 0.1))
+ except Exception:
+ _LOG.exception("[Wiki expand] vector build failed; using keyword match")
+ if not exprs:
+ exprs.append(
+ MatchTextExpr(
+ ["content_ltks", "content_sm_ltks"],
+ text,
+ top_n,
+ )
+ )
+
+ try:
+ res = await thread_pool_exec(
+ settings.docStoreConn.search,
+ fields,
+ [],
+ condition,
+ exprs,
+ OrderByExpr(),
+ 0,
+ top_n,
+ search.index_name(tenant_id),
+ [kb_id],
+ )
+ return settings.docStoreConn.get_fields(res, fields) or {}
+ except Exception:
+ _LOG.exception("[Wiki expand] search failed for kb=%s", kb_id)
+ return {}
+
+
+async def _expand_wiki_page_strategy(
+ tools,
+ kb_id: str,
+ tenant_id: str,
+ doc_ids: list[str] | None,
+ query: str,
+ seen_ids: set[str],
+ *,
+ compile_kwd: str = "wiki_page",
+ max_chunks: int = 5,
+) -> list[dict]:
+ """Expand synthesis-compiled pages: semantic search → load source chunks.
+
+ Unlike ``_expand_compiled_strategy`` (which does 1-hop entity-graph
+ navigation), synthesis pages are standalone rendered articles — we search
+ them directly and load the referenced source chunks as context.
+
+ *compile_kwd* selects which synthesis type: "wiki_page" (Wiki template),
+ "artifact_page" (Session Graph synthesis), "essence" (Session Essence).
+ """
+ # -- 1. Search synthesis pages --
+ wiki_rows = await _search_synthesis_pages(
+ tools,
+ kb_id,
+ tenant_id,
+ doc_ids,
+ query,
+ compile_kwd=compile_kwd,
+ top_n=5,
+ )
+ if not wiki_rows:
+ return []
+
+ # -- 2. Collect source_chunk_ids from matching pages --
+ by_doc: dict[str, set[str]] = {}
+ for r in wiki_rows.values():
+ doc_id = r.get("doc_id") or ""
+ for cid in r.get("source_chunk_ids") or []:
+ if cid and cid not in seen_ids:
+ by_doc.setdefault(doc_id, set()).add(cid)
+
+ # -- 3. Load chunks, assign high similarity for priority ranking --
+ new_chunks: list[dict] = []
+ for doc_id, cids in by_doc.items():
+ if len(new_chunks) >= max_chunks:
+ break
+ limit = max_chunks - len(new_chunks)
+ chunks = await _load_chunks_for_doc(tools, doc_id, list(cids)[:limit])
+ for c in chunks:
+ cid = c.get("chunk_id") or c.get("id")
+ if cid and cid not in seen_ids:
+ seen_ids.add(cid)
+ c.setdefault("similarity", 0.9) # wiki pages rank high
+ new_chunks.append(c)
+
+ return new_chunks
diff --git a/rag/advanced_rag/harness/tools/exploration.py b/rag/advanced_rag/harness/tools/exploration.py
index 3cc75763e8..88db170eea 100644
--- a/rag/advanced_rag/harness/tools/exploration.py
+++ b/rag/advanced_rag/harness/tools/exploration.py
@@ -1,26 +1,373 @@
-"""Exploration tools: knowledge graph and wiki lookup.
+"""Exploration tools: knowledge-graph walks and wiki page drill-downs.
-``graph_explore`` lives in :mod:`navigation` (it shares the compiled-structure
-machinery) and is re-exported here so the tool registry keeps one import point.
+Two ways of reaching evidence that is *not* a bag of similar chunks:
-``wiki_query`` runs a hybrid (BM25 + dense) search over the searchable wiki draft
-rows written by ``_wiki_persist_draft`` (``compile_kwd="wiki_page_draft"``) and
-returns each page's markdown as a chunk. It takes the same ``keywords`` the other
-search tools do — the keywords drive the keyword-sentence narrowing. Parameter
-names must match the registered ``_search_schema`` (``query`` + ``keywords``),
-otherwise every LLM tool call fails with a TypeError.
+1. **``graph_explore``** — seeds entities by dense similarity to the question,
+ expands BFS over compiled ``relation`` rows for a bounded number of hops, then
+ asks the model whether the resulting subgraph answers the question outright;
+ when it does not, the subgraph is returned as evidence passages so the caller
+ can keep researching.
+
+ Reads the compiled knowledge graph only (``knowledge_graph_kwd``
+ entity/relation rows whose ``compile_kwd`` resolves to ``hypergraph``).
+ Datasets without a compiled graph short-circuit — every caller treats an empty
+ subgraph as "no graph here", not as an error.
+
+2. **``wiki_query``** — asks a question of one dataset's compiled *wiki* pages,
+ which are already synthesised narratives rather than raw chunks.
+
+``_kg_scopes`` is the one piece the rest of the harness needs from here:
+compiled-product expansion uses it to resolve the (kb_id, tenant_id) pairs to
+scan.
"""
import json
import logging
-# graph_explore is implemented alongside catalog/mindmap navigation because it
-# reuses their outline-answering + evidence-pulling helpers.
-from rag.advanced_rag.harness.tools.navigation import graph_explore # noqa: F401
+from rag.advanced_rag.harness.structure_qa import _ask_structure
+
+# Everything else (settings / doc-store exprs / thread pool / index name) is
+# imported lazily inside each helper, matching the original module's style.
_LOG = logging.getLogger(__name__)
-# compile_kwd of the searchable wiki draft rows (see _wiki_persist_draft).
+# ── Knowledge-graph exploration ─────────────────────────────────────────────
+#
+# Unlike catalog/mindmap (which read the merged "graph" JSON of one doc), the KG
+# store keeps one searchable row per entity/relation, so graph_explore *searches*
+# its way to a small subgraph: seed entities by the question, hop out over
+# relations to the 2nd-degree neighbours, then answer from that subgraph.
+
+# Scope of the compiled KG rows we search. Without a doc_scope we read the
+# dataset-merged rows (one graph per dataset); with a doc_scope we read the
+# per-document rows and filter by doc_id. Kept local rather than imported from
+# the task-executor layer so the harness has no dependency on it.
+_SCOPE_KWD_DATASET = "dataset"
+_SCOPE_KWD_DOC = "doc"
+
+_KG_SEEDS = 2 # top-N entities matched directly to the question
+_KG_SEED_POOL = 64 # KNN candidate pool before the mention_count_int re-sort
+_KG_SEED_SIM = 0.8 # dense-similarity floor for seed entities
+_KG_HOPS = 2 # relation hops out from the seeds (1 => "2nd degree")
+_KG_NEIGHBORS = 128 # cap on neighbour entity rows resolved per hop
+_KG_REL_LIMIT = 32 # relations fetched per endpoint filter
+
+
+async def _kg_scopes(tools, doc_scope: list[str] | None = None):
+ """Resolve the (kb_id, tenant_id, doc_ids|None) groups to search.
+
+ With a ``doc_scope`` the graph is limited to those docs (grouped by their
+ KB); otherwise the whole bound KB graph is explored.
+ """
+ if hasattr(tools, "scoped_doc_ids"):
+ doc_scope = tools.scoped_doc_ids(doc_scope)
+ if doc_scope:
+ by_kb: dict[tuple, list[str]] = {}
+ for doc_id in doc_scope:
+ # peewee MySQL lookup — call directly to reuse the pool's connection.
+ resolved = tools._resolve_doc_tenant(doc_id)
+ if resolved:
+ by_kb.setdefault(resolved, []).append(doc_id)
+ return [(kb, tenant, docs) for (kb, tenant), docs in by_kb.items()]
+ return [(kb.id, kb.tenant_id, None) for kb in getattr(tools, "kbs", []) or []]
+
+
+async def _kg_search(
+ tools,
+ kb_id: str,
+ tenant_id: str,
+ doc_ids,
+ kind: str,
+ text: str = "",
+ top_n: int = 8,
+ extra: dict | None = None,
+ scope_kwd: str | None = None,
+ order_desc: str | None = None,
+ pool: int | None = None,
+ similarity: float = 0.6,
+) -> list[dict]:
+ """Search the compiled KG rows of one KB and return the raw field maps.
+
+ ``scope_kwd`` narrows to dataset-merged (``"dataset"``) or per-doc (``"doc"``)
+ rows. ``order_desc`` sorts the hits by that field descending; when combined
+ with a dense ``text`` match, ``pool`` sets the KNN candidate count so the
+ re-sort ranks a wider pool than the ``top_n`` finally kept. ``similarity`` is
+ the dense-match floor.
+ """
+ from common import settings
+ from common.doc_store.doc_store_base import MatchTextExpr, OrderByExpr
+ from common.misc_utils import thread_pool_exec
+ from rag.nlp import search
+
+ condition: dict = {"knowledge_graph_kwd": [kind]}
+ if scope_kwd:
+ condition["scope_kwd"] = [scope_kwd]
+ if doc_ids:
+ condition["doc_id"] = list(doc_ids)
+ if extra:
+ condition.update(extra)
+
+ fields = ["content_with_weight", "source_chunk_ids", "doc_id", "docnm_kwd", "name_kwd", "mention_count_int", "from_entity_kwd", "to_entity_kwd"]
+ exprs = []
+ if text:
+ knn_topn = pool or top_n
+ if getattr(tools, "embed_mdl", None):
+ try:
+ exprs.append(await settings.retriever.get_vector(text, tools.embed_mdl, knn_topn, similarity))
+ except Exception:
+ _LOG.exception("[Graph exploration] vector build failed; using keyword match")
+ if not exprs:
+ exprs.append(MatchTextExpr(["content_ltks", "content_sm_ltks"], text, knn_topn))
+
+ order_by = OrderByExpr()
+ if order_desc:
+ try:
+ order_by.desc(order_desc)
+ except Exception:
+ order_by = OrderByExpr()
+
+ try:
+ res = await thread_pool_exec(
+ settings.docStoreConn.search,
+ fields,
+ [],
+ condition,
+ exprs,
+ order_by,
+ 0,
+ top_n,
+ search.index_name(tenant_id),
+ [kb_id],
+ )
+ rows = settings.docStoreConn.get_fields(res, fields) or {}
+ except Exception:
+ _LOG.exception("[Graph exploration] KG search failed (kind=%s)", kind)
+ return {}
+ return rows
+
+
+def _kg_parse_entity(row: dict) -> dict | None:
+ try:
+ payload = json.loads(row.get("content_with_weight") or "{}")
+ except Exception:
+ payload = {}
+ name = (payload.get("name") or payload.get("term") or payload.get("title") or "").strip()
+ if not name:
+ return None
+ aliases = [str(a).strip() for a in (payload.get("aliases") or []) if str(a).strip()]
+ return {
+ "name": name,
+ "type": (payload.get("type") or "other"),
+ "description": (payload.get("description") or payload.get("description") or ""),
+ "aliases": aliases,
+ "source_chunk_ids": list(row.get("source_chunk_ids") or []),
+ "doc_id": row.get("doc_id") or "",
+ "docnm_kwd": row.get("docnm_kwd") or "",
+ }
+
+
+def _kg_parse_relation(row: dict) -> dict | None:
+ src = (row.get("from_entity_kwd") or "").strip()
+ tgt = (row.get("to_entity_kwd") or "").strip()
+ if not src or not tgt:
+ return None
+ typ = "related"
+ try:
+ payload = json.loads(row.get("content_with_weight") or "{}")
+ typ = payload.get("type") or payload.get("relation") or "related"
+ except Exception:
+ pass
+ return {
+ "from": src,
+ "to": tgt,
+ "type": typ,
+ "source_chunk_ids": list(row.get("source_chunk_ids") or []),
+ "doc_id": row.get("doc_id") or "",
+ }
+
+
+def _endpoint_terms(names) -> list[str]:
+ """Case variants for matching relation endpoints.
+
+ ``dataset_structure_merger`` lowercases ``from_entity_kwd``/``to_entity_kwd``
+ on merged rows while entity names keep their original case, so hop queries
+ must try both forms. Accepts a single name or an iterable and returns the
+ sorted union of each name's original and lowercased form.
+ """
+ if isinstance(names, str):
+ names = [names]
+ terms: set[str] = set()
+ for n in names or []:
+ n = (n or "").strip()
+ if n:
+ terms.add(n)
+ terms.add(n.lower())
+ return sorted(terms)
+
+
+def _collect_evidence_ids(entities: list[dict], relations: list[dict], relevant_names: list[str]) -> dict:
+ """Group the source_chunk_ids of the relevant entities AND relations by doc.
+
+ An entity is relevant when its name/alias was named by the model; a relation
+ is relevant when either endpoint is.
+ """
+ wanted = {n.strip().lower() for n in relevant_names if isinstance(n, str) and n.strip()}
+ by_doc: dict[str, list[str]] = {}
+ seen: set[tuple[str, str]] = set()
+
+ def _add(doc_id: str, ids):
+ for cid in ids or []:
+ if not (isinstance(cid, str) and cid):
+ continue
+ key = (doc_id, cid)
+ if key in seen:
+ continue
+ seen.add(key)
+ by_doc.setdefault(doc_id, []).append(cid)
+
+ for e in entities:
+ names = {(e.get("name") or "").strip().lower(), *[(a or "").strip().lower() for a in (e.get("aliases") or [])]}
+ if names & wanted:
+ _add(e.get("doc_id") or "", e.get("source_chunk_ids"))
+ for r in relations:
+ if {(r.get("from") or "").strip().lower(), (r.get("to") or "").strip().lower()} & wanted:
+ _add(r.get("doc_id") or "", r.get("source_chunk_ids"))
+ return by_doc
+
+
+async def graph_explore(tools, query: str, keywords: str = "", doc_scope: list[str] | None = None) -> dict:
+ """Explore the compiled knowledge graph to answer ``query``.
+
+ Seeds the top-``_KG_SEEDS`` entities for the question (dense match above
+ ``_KG_SEED_SIM``, ranked by ``mention_count_int``), hops ``_KG_HOPS`` out over
+ their relations, then asks the chat model whether that subgraph answers the
+ question. When it does, the answer is returned directly; when it doesn't, the
+ source passages behind the entities/relations the model found relevant are
+ returned as evidence (narrowed by ``keywords``) so the caller can continue.
+
+ Without ``doc_scope`` the dataset-merged graph is searched
+ (``scope_kwd="dataset"``); with it, the per-document rows of those docs
+ (``scope_kwd="doc"``, filtered by ``doc_id``).
+
+ :returns: ``{"answer": str, "chunks": [...], "doc_aggs": [...]}`` — exactly one
+ of ``answer`` / ``chunks`` is populated.
+ """
+ from rag.advanced_rag.harness.tools.search import _narrow_by_keywords
+
+ _empty = {"answer": "", "chunks": [], "doc_aggs": []}
+ if hasattr(tools, "scoped_doc_ids"):
+ doc_scope = tools.scoped_doc_ids(doc_scope)
+ _LOG.info(f'[Graph exploration] Exploring the knowledge graph for "{query}" (keywords: {keywords})')
+
+ scopes = await _kg_scopes(tools, doc_scope)
+ if not scopes:
+ _LOG.info("[Graph exploration] No knowledge base in scope.")
+ return _empty
+
+ scope_kwd = _SCOPE_KWD_DOC if doc_scope else _SCOPE_KWD_DATASET
+ text = f"{query} {keywords}".strip()
+ entities: list[dict] = []
+ relations: list[dict] = []
+ ent_names: set[str] = set()
+
+ def _add_entities(new: list[dict], scope_key: str = "") -> list[str]:
+ added = []
+ for e in new:
+ key = f"{scope_key}:{e['name'].lower()}"
+ if key in ent_names:
+ continue
+ ent_names.add(key)
+ entities.append(e)
+ added.append(e["name"])
+ return added
+
+ for kb_id, tenant_id, doc_ids in scopes:
+ # (1) Seeds: condition C — dense match (>= _KG_SEED_SIM) over the scoped
+ # entity rows, ranked by mention_count_int desc, top _KG_SEEDS.
+ seed_rows = await _kg_search(
+ tools,
+ kb_id,
+ tenant_id,
+ doc_ids,
+ "entity",
+ text=text,
+ top_n=_KG_SEEDS,
+ scope_kwd=scope_kwd,
+ order_desc="mention_count_int",
+ pool=_KG_SEED_POOL,
+ similarity=_KG_SEED_SIM,
+ )
+ seeds = [e for e in (_kg_parse_entity(r) for r in seed_rows.values()) if e]
+ frontier = _add_entities(seeds, kb_id)
+ _LOG.info("[Graph exploration] Seeded %d entity(ies): %s", len(frontier), ", ".join(frontier) or "none")
+
+ # (2) Expand _KG_HOPS out, collecting relations and neighbour entities.
+ for _hop in range(_KG_HOPS):
+ if not frontier:
+ break
+ terms = _endpoint_terms(frontier) # case variants — merged rows lowercase endpoints
+ rel_rows: dict = {}
+ rel_rows.update(await _kg_search(tools, kb_id, tenant_id, doc_ids, "relation", top_n=_KG_REL_LIMIT, scope_kwd=scope_kwd, extra={"from_entity_kwd": terms}))
+ rel_rows.update(await _kg_search(tools, kb_id, tenant_id, doc_ids, "relation", top_n=_KG_REL_LIMIT, scope_kwd=scope_kwd, extra={"to_entity_kwd": terms}))
+ hop_relations = [r for r in (_kg_parse_relation(x) for x in rel_rows.values()) if r]
+ relations.extend(hop_relations)
+
+ seen_lower = {k.split(":", 1)[1] for k in ent_names if k.startswith(f"{kb_id}:")}
+ neigh_names = {n.strip() for r in hop_relations for n in (r["from"], r["to"]) if n and n.strip()}
+ neigh_lower_set = {n.lower() for n in neigh_names} - seen_lower
+ if not neigh_lower_set:
+ break
+ neigh_filtered = {n for n in neigh_names if n.lower() in neigh_lower_set}
+ neigh_rows = await _kg_search(
+ tools,
+ kb_id,
+ tenant_id,
+ doc_ids,
+ "entity",
+ top_n=min(max(len(neigh_filtered), 1), _KG_NEIGHBORS),
+ scope_kwd=scope_kwd,
+ extra={"name_kwd": _endpoint_terms(neigh_filtered)},
+ )
+ neighbours = [e for e in (_kg_parse_entity(r) for r in neigh_rows.values()) if e]
+ frontier = _add_entities(neighbours, kb_id)
+ _LOG.info("[Graph exploration] Hop %d reached %d neighbour entity(ies).", _hop + 1, len(frontier))
+
+ if not entities and not relations:
+ _LOG.info("[Graph exploration] No compiled knowledge graph in scope.")
+ return _empty
+
+ _LOG.info("[Graph exploration] Built a subgraph of %d entity(ies) and %d relation(s).", len(entities), len(relations))
+
+ # (3) Does the subgraph answer the question?
+ answer, relevant = await _ask_structure(tools, query, entities, relations, "knowledge graph", "Graph exploration")
+
+ # (4a) Sufficient — return the answer, no chunks.
+ if answer:
+ _LOG.info("[Graph exploration] The subgraph answered the question directly.")
+ return {"answer": answer, "chunks": [], "doc_aggs": []}
+
+ # (4b) Insufficient — return the source passages behind the relevant nodes.
+ # Imported lazily: both live in ``navigation``, which imports this module.
+ from rag.advanced_rag.harness.tools.navigation import _doc_aggs, _load_chunks_by_ids
+
+ evidence = _collect_evidence_ids(entities, relations, relevant)
+ chunks: list[dict] = []
+ for doc_id, ids in evidence.items():
+ if doc_id and ids:
+ chunks.extend(await _load_chunks_by_ids(tools, doc_id, ids))
+
+ before = len(chunks)
+ chunks = _narrow_by_keywords(chunks, keywords)
+ _LOG.info("[Graph exploration] Insufficient; returning %d evidence passage(s) (%d before keyword filtering).", len(chunks), before)
+
+ return {"answer": "", "chunks": chunks, "doc_aggs": _doc_aggs(chunks)}
+
+
+# ── Wiki page drill-down ──────────────────────────────────────────────────
+#
+# Compiled wiki pages are already synthesised narratives rather than raw chunks,
+# so a question can often be answered straight from them.
+
_WIKI_DRAFT_COMPILE_KWD = "wiki_page_draft"
_WIKI_QUERY_TOP_N = 12
diff --git a/rag/advanced_rag/harness/tools/gating.py b/rag/advanced_rag/harness/tools/gating.py
deleted file mode 100644
index 63d240ddce..0000000000
--- a/rag/advanced_rag/harness/tools/gating.py
+++ /dev/null
@@ -1,136 +0,0 @@
-"""Tool selection gating: phase-based filtering and fallback chain."""
-
-from rag.advanced_rag.harness.tools.registry import TOOL_REGISTRY
-from rag.advanced_rag.harness.types import OrchestratorContext
-
-# Search phase definitions
-
-SEARCH_PHASES = {
- "locate": {
- "goal": "Locate documents or regions that may contain the answer.",
- "tools_priority": [
- "dataset_navigation_search",
- "ontology_navigate",
- "mindmap_navigate",
- "hybrid_search",
- "bm25_search",
- "grep_search",
- "wiki_query",
- ],
- "max_returned": 5,
- "tool_hint": "Prefer navigation tools to locate document regions before directly searching keywords; grep_search for exact names/ids/numbers.",
- },
- "explore": {
- "goal": "Explore deeply within the already located region.",
- "tools_priority": [
- "hybrid_search",
- "bm25_search",
- "grep_search",
- "list_chunks",
- "web_search",
- "graph_explore",
- "inspector_open_context",
- "inspector_request_adjacent",
- ],
- "max_returned": 5,
- "tool_hint": "Prefer retrieval tools to gather detailed information within the located region; grep_search for exact locate and list_chunks to deep-read a hit document; use web_search when the knowledge base lacks the answer or the question needs current/external facts.",
- },
- "verify": {
- "goal": "Verify consistency across multiple sources.",
- "tools_priority": [
- "inspector_open_context",
- "inspector_compare",
- "inspector_grep_within",
- "web_search",
- "hybrid_search",
- ],
- "max_returned": 4,
- "tool_hint": "Prefer inspector tools to compare existing evidence; use web_search to corroborate against external sources.",
- },
- "cross_domain": {
- "goal": "Explore cross-domain relationships for discovered entities.",
- "tools_priority": [
- "graph_explore",
- "wiki_query",
- "hybrid_search",
- "web_search",
- ],
- "max_returned": 3,
- "tool_hint": "Prefer walking the graph to discover cross-domain relationships.",
- },
-}
-
-
-def compilation_available(tool_name: str, compilation_map: dict) -> bool:
- """Check if any KB provides the required compilation artifact."""
- tool = TOOL_REGISTRY.get(tool_name)
- if not tool or not tool.get("requires_compilation"):
- return True
- comp_type = tool["compilation_type"]
- if not compilation_map:
- return False
- comp_types = set(comp_type) if isinstance(comp_type, (list, tuple, set)) else {comp_type}
- return any(bool(comp_types & set(comps)) for comps in compilation_map.values())
-
-
-def tool_fits_context(tool_name: str, context: OrchestratorContext, has_routed_scope: bool = False) -> bool:
- """Check if a tool is sensible given current search context."""
- if tool_name.startswith("inspector_") and not context.has_any_chunks():
- return False
- if tool_name in {"ontology_navigate", "mindmap_navigate"} and not has_routed_scope:
- return False
- if tool_name == "dataset_navigation_search" and not context.current_claim:
- return False
- if tool_name == "graph_explore" and not context.last_entity:
- return False
- return True
-
-
-def get_gated_tools(
- phase: str,
- available_tools: list[str],
- compilation_map: dict[str, set[str]],
- context: OrchestratorContext,
- has_routed_scope: bool = False,
- web_enabled: bool = True,
-) -> list[dict]:
- """Filter, sort, and gate tools by phase priority and context."""
- phase_config = SEARCH_PHASES.get(phase)
- if not phase_config:
- return _default_defs(available_tools, web_enabled)
-
- sorted_tools = []
- for tool_name in phase_config["tools_priority"]:
- if tool_name not in available_tools:
- continue
- if tool_name == "web_search" and not web_enabled:
- # No web provider configured — don't bind a tool that no-ops.
- continue
- if not compilation_available(tool_name, compilation_map):
- continue
- if not tool_fits_context(tool_name, context, has_routed_scope):
- continue
- sorted_tools.append(tool_name)
-
- selected = sorted_tools[: phase_config["max_returned"]]
- # Copy the registry schemas before annotating — the registry dicts are
- # shared process-wide, mutating them would leak phase hints across
- # concurrent requests.
- defs = []
- for n in selected:
- if n in TOOL_REGISTRY:
- defs.append({**TOOL_REGISTRY[n]["function_schema"], "x_phase": phase, "x_phase_hint": phase_config["tool_hint"]})
- return defs
-
-
-def _default_defs(tool_names: list[str], web_enabled: bool = True) -> list[dict]:
- return [TOOL_REGISTRY[n]["function_schema"] for n in tool_names if n in TOOL_REGISTRY and (web_enabled or n != "web_search")]
-
-
-def determine_current_phase(context: OrchestratorContext) -> str:
- """Determine the current search phase based on context."""
- if not context.has_any_chunks():
- return "locate"
- if context.verdict and context.verdict.has_conflicts:
- return "verify"
- return "explore"
diff --git a/rag/advanced_rag/harness/tools/inspector.py b/rag/advanced_rag/harness/tools/inspector.py
deleted file mode 100644
index cc7fcaf067..0000000000
--- a/rag/advanced_rag/harness/tools/inspector.py
+++ /dev/null
@@ -1,87 +0,0 @@
-"""Inspector tools: operate on already-returned results."""
-
-import logging
-from copy import deepcopy
-
-_LOG = logging.getLogger(__name__)
-
-
-async def open_context(tools, chunk_id: str, width: int = 500) -> dict:
- """Expand context around a chunk by looking up adjacent chunks in kbinfos."""
- chunks = tools.kbinfos.get("chunks", [])
- idx = _find_chunk_index(chunks, chunk_id)
- if idx is None:
- return {"chunks": [], "doc_aggs": []}
- start = max(0, idx - 2)
- end = min(len(chunks), idx + 2)
- context = chunks[start:end]
- _LOG.info("open_context: chunk=%s index=%d context=%d chunks", chunk_id, idx, len(context))
- return {"chunks": context, "doc_aggs": _collect_doc_aggs(tools, context)}
-
-
-async def compare_sources(tools, chunk_ids: list[str]) -> dict:
- """Find chunks in kbinfos and list the document sources they come from."""
- if not chunk_ids:
- return {"chunks": [], "doc_aggs": []}
- chunks = tools.kbinfos.get("chunks", [])
- matched = [c for c in chunks if _chunk_id(c) in chunk_ids]
- return {"chunks": matched, "doc_aggs": _collect_doc_aggs(tools, matched)}
-
-
-async def grep_within(tools, doc_id: str, pattern: str) -> dict:
- """Find a keyword within a document and return its chunks narrowed to the
- matching sentences (+/- 1 neighbour).
-
- ``pattern`` is treated as the keyword string (comma-separate for several).
- Delegates to :func:`_narrow_by_keywords`, which keeps only keyword-bearing
- sentences and drops chunks with no match. Operates on copies so the shared
- ``kbinfos`` citation pool is never mutated.
- """
- from rag.advanced_rag.harness.tools.search import _narrow_by_keywords
-
- chunks = [deepcopy(c) for c in tools.kbinfos.get("chunks", []) if c.get("doc_id") == doc_id]
- matched = _narrow_by_keywords(chunks, pattern)
- return {"chunks": matched, "doc_aggs": _collect_doc_aggs(tools, matched)}
-
-
-async def request_adjacent(tools, chunk_id: str, direction: str = "next", count: int = 3) -> dict:
- """Get adjacent entries before or after a chunk."""
- chunks = tools.kbinfos.get("chunks", [])
- idx = _find_chunk_index(chunks, chunk_id)
- if idx is None:
- return {"chunks": [], "doc_aggs": []}
-
- if direction == "prev":
- start = max(0, idx - count)
- end = idx
- else:
- start = idx + 1
- end = min(len(chunks), start + count)
-
- adjacent = chunks[start:end]
- return {"chunks": adjacent, "doc_aggs": _collect_doc_aggs(tools, adjacent)}
-
-
-# Helpers
-
-
-def _find_chunk_index(chunks: list[dict], chunk_id: str) -> int | None:
- for i, c in enumerate(chunks):
- if _chunk_id(c) == chunk_id:
- return i
- return None
-
-
-def _chunk_id(ck: dict) -> str:
- return str(ck.get("chunk_id") or ck.get("id") or "")
-
-
-def _collect_doc_aggs(tools, chunks: list[dict]) -> list[dict]:
- seen = set()
- aggs = []
- for c in chunks:
- doc_id = c.get("doc_id")
- if doc_id and doc_id not in seen:
- seen.add(doc_id)
- aggs.append({"doc_id": doc_id, "doc_name": c.get("docnm_kwd", "")})
- return aggs
diff --git a/rag/advanced_rag/harness/tools/navigation.py b/rag/advanced_rag/harness/tools/navigation.py
index cbbf063ac0..17fb1927f7 100644
--- a/rag/advanced_rag/harness/tools/navigation.py
+++ b/rag/advanced_rag/harness/tools/navigation.py
@@ -1,20 +1,27 @@
-"""Navigation tools: document catalog and concept mindmap.
-
-Both answer from a document's *compiled structure* rather than going straight to
-retrieval: they read the compiled entities+relations for the documents in scope,
-let the chat model answer from that outline, and pull the underlying chunks back
-via each relevant entity's ``source_chunk_ids``.
-
-Knowledge compilation writes every template kind into the same graph rows
-(``{"entities": [...], "relations": [...]}``), so the two tools share one
-implementation and differ only in which *kinds* they read:
-
-* ``ontology_navigate`` — the document's layout: tree / TOC, page index, RAPTOR.
-* ``mindmap_navigate`` — the concept mindmap.
-
-Both take the same ``keywords`` the search tools do — keywords drive query
-expansion and the keyword-sentence narrowing; without them navigation would hand
-back full, un-narrowed chunks.
+"""Navigation tools over a dataset's compiled structures.
+
+Two families live here, answering different questions about a dataset:
+
+1. **Dataset navigation tree** (``dataset_nav``, KB-level). Knowledge
+ compilation builds a cluster tree covering EVERY document of the dataset
+ (``compile_kwd="dataset_nav"`` with ``type_kwd="nav_cluster"/"nav_doc"`` rows
+ linked by ``parent_kwd``). ``_navigate_tree_impl`` routes a question through
+ that tree's document leaves to pick which documents to deep-read.
+
+2. **Document structure navigation** (``compile_kwd="tree"/"page_index"/...``,
+ document-level). Reads a document's compiled entities+relations and renders a
+ drill-down outline, letting the chat model answer from the structure and pull
+ the underlying chunks back via each entity's ``source_chunk_ids``.
+
+ Two row shapes coexist for these and are read together: a compact *graph
+ blob* (``knowledge_graph_kwd="graph"``, content ``{"entities": [...],
+ "relations": [...]}``) written by RAPTOR/tree compilation, and *per-row*
+ ``knowledge_graph_kwd="entity"/"relation"`` documents written by page_index
+ (and pipeline-Compiler tree). ``compile_kwd`` — NOT ``knowledge_graph_kwd`` —
+ is what distinguishes the compile type.
+
+``graph_explore`` reads the compiled knowledge graph (``compile_kwd`` resolving
+to ``hypergraph``) and walks it breadth-first from seed entities.
"""
import json
@@ -24,6 +31,13 @@ from typing import Any
import json_repair
+from rag.advanced_rag.harness.chunk_utils import (
+ _chunk_id,
+ _chunk_text,
+ _doc_title,
+ _snippet,
+ _xml_escape,
+)
from rag.llm.tool_decorator import tool
_LOG = logging.getLogger(__name__)
@@ -37,22 +51,11 @@ _CATALOG_KINDS = {"tree", "timeline", "raptor", "page_index", "pageindex"}
# Compiled-structure kinds that describe the document's *concepts*.
_MINDMAP_KINDS = {"mindmap", "mind_map"}
-# Cap how much compiled structure we render into the prompt.
-_MAX_ENTITIES = 300
-_MAX_RELATIONS = 300
+# Cap on evidence chunks pulled from a compiled-structure outline.
_MAX_EVIDENCE_CHUNKS = 24
-_NAV_SYSTEM = """You are given {noun} of one or more documents — an outline of entities and their relations — and a question.
-
-Decide whether that outline alone already answers the question.
-
-Rules:
-1. Answer ONLY from the outline below. Do not invent facts.
-2. Set "is_sufficient" to true only when the outline genuinely answers the question; otherwise false with an empty answer.
-3. Always fill "relevant_entities" with the exact `name` values of the entities most related to the question (up to 10), even when the outline is not sufficient — they are used to pull the underlying source text.
-
-Output ONLY JSON, no prose, no code fences:
-{{"is_sufficient": true/false, "answer": "<answer, or empty>", "relevant_entities": ["<entity name>", ...]}}"""
+# Cap on entities offered to the nav-tree entity selector.
+_MAX_ENTITIES = 300
def _normalize_kind(kind) -> str:
@@ -171,28 +174,6 @@ async def _load_compiled_structure(tools, doc_id: str, kinds: set) -> dict:
return {"entities": entities, "relations": relations}
-def _render_structure(entities: list[dict], relations: list[dict]) -> str:
- """Render the compiled structure as a compact outline for the prompt."""
- lines: list[str] = []
- if entities:
- lines.append("Entities:")
- for e in entities[:_MAX_ENTITIES]:
- name = (e.get("name") or "").strip()
- if not name:
- continue
- typ = (e.get("type") or "other").strip()
- desc = " ".join((e.get("description") or "").split())
- lines.append(f"- {name} ({typ})" + (f": {desc}" if desc else ""))
- if relations:
- lines.append("\nRelations:")
- for r in relations[:_MAX_RELATIONS]:
- src, tgt = (r.get("from") or "").strip(), (r.get("to") or "").strip()
- if not src or not tgt:
- continue
- lines.append(f"- {src} -[{(r.get('type') or 'related').strip()}]-> {tgt}")
- return "\n".join(lines)
-
-
async def _load_chunks_by_ids(tools, doc_id: str, chunk_ids: list[str]) -> list[dict]:
"""Fetch chunks by their ids from the doc store."""
if not chunk_ids:
@@ -250,44 +231,6 @@ def _doc_aggs(chunks: list[dict]) -> list[dict]:
return aggs
-async def _ask_structure(tools, topic: str, entities: list[dict], relations: list[dict], noun: str, label: str) -> tuple[str, list[str]]:
- """Ask the chat model to answer ``topic`` from the rendered outline.
-
- Returns ``(answer, relevant_entity_names)`` — ``answer`` is empty unless the
- model judged the outline sufficient; the names are always returned so the
- caller can pull the underlying source chunks.
- """
- verdict = {}
- try:
- from rag.prompts.generator import form_message, message_fit_in
-
- user = f"Question:\n{topic}\n\n{noun.capitalize()}:\n{_render_structure(entities, relations)}\n\nOutput JSON:"
- _, msg = message_fit_in(form_message(_NAV_SYSTEM.format(noun=f"the {noun}"), user), tools.chat_mdl.max_length)
- ans = await tools.chat_mdl.async_chat(msg[0]["content"], msg[1:], {"temperature": 0.2})
- if isinstance(ans, tuple):
- ans = ans[0]
- cleaned = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
- cleaned = re.sub(r"```(?:json)?\s*|\s*```", "", cleaned).strip()
- verdict = json_repair.loads(cleaned) or {}
- if not isinstance(verdict, dict):
- verdict = {}
- except Exception:
- _LOG.exception(f"[{label}] Could not read the outline with the model.")
-
- is_sufficient = bool(verdict.get("is_sufficient"))
- answer = (verdict.get("answer") or "").strip() if is_sufficient else ""
- relevant = [n for n in (verdict.get("relevant_entities") or []) if isinstance(n, str)]
- _LOG.info(
- "[%s] The %s %s the question; %d relevant entity(ies): %s",
- label,
- noun,
- "answers" if is_sufficient else "does not fully answer",
- len(relevant),
- ", ".join(relevant[:10]) or "none",
- )
- return answer, relevant
-
-
async def _navigate_within_doc(
tools,
topic: str,
@@ -733,334 +676,21 @@ async def dataset_navigation_search(tools, topic: str, keywords: str = "", doc_s
return routed[:_NAV_SEARCH_MAX_DOCS]
-# ── Knowledge-graph exploration ─────────────────────────────────────────────
-#
-# Unlike catalog/mindmap (which read the merged "graph" JSON of one doc), the KG
-# store keeps one searchable row per entity/relation, so graph_explore *searches*
-# its way to a small subgraph: seed entities by the question, hop out over
-# relations to the 2nd-degree neighbours, then answer from that subgraph.
-
-# Scope of the compiled KG rows we search. Without a doc_scope we read the
-# dataset-merged rows (one graph per dataset); with a doc_scope we read the
-# per-document rows and filter by doc_id. Kept local rather than imported from
-# the task-executor layer so the harness has no dependency on it.
-_SCOPE_KWD_DATASET = "dataset"
-_SCOPE_KWD_DOC = "doc"
-
-_KG_SEEDS = 2 # top-N entities matched directly to the question
-_KG_SEED_POOL = 64 # KNN candidate pool before the mention_count_int re-sort
-_KG_SEED_SIM = 0.8 # dense-similarity floor for seed entities
-_KG_HOPS = 2 # relation hops out from the seeds (1 => "2nd degree")
-_KG_NEIGHBORS = 128 # cap on neighbour entity rows resolved per hop
-_KG_REL_LIMIT = 32 # relations fetched per endpoint filter
-
-
-async def _kg_scopes(tools, doc_scope: list[str] | None = None):
- """Resolve the (kb_id, tenant_id, doc_ids|None) groups to search.
-
- With a ``doc_scope`` the graph is limited to those docs (grouped by their
- KB); otherwise the whole bound KB graph is explored.
- """
- if hasattr(tools, "scoped_doc_ids"):
- doc_scope = tools.scoped_doc_ids(doc_scope)
- if doc_scope:
- by_kb: dict[tuple, list[str]] = {}
- for doc_id in doc_scope:
- # peewee MySQL lookup — call directly to reuse the pool's connection.
- resolved = tools._resolve_doc_tenant(doc_id)
- if resolved:
- by_kb.setdefault(resolved, []).append(doc_id)
- return [(kb, tenant, docs) for (kb, tenant), docs in by_kb.items()]
- return [(kb.id, kb.tenant_id, None) for kb in getattr(tools, "kbs", []) or []]
-
-
-async def _kg_search(
- tools,
- kb_id: str,
- tenant_id: str,
- doc_ids,
- kind: str,
- text: str = "",
- top_n: int = 8,
- extra: dict | None = None,
- scope_kwd: str | None = None,
- order_desc: str | None = None,
- pool: int | None = None,
- similarity: float = 0.6,
-) -> list[dict]:
- """Search the compiled KG rows of one KB and return the raw field maps.
-
- ``scope_kwd`` narrows to dataset-merged (``"dataset"``) or per-doc (``"doc"``)
- rows. ``order_desc`` sorts the hits by that field descending; when combined
- with a dense ``text`` match, ``pool`` sets the KNN candidate count so the
- re-sort ranks a wider pool than the ``top_n`` finally kept. ``similarity`` is
- the dense-match floor.
- """
- from common import settings
- from common.doc_store.doc_store_base import MatchTextExpr, OrderByExpr
- from common.misc_utils import thread_pool_exec
- from rag.nlp import search
-
- condition: dict = {"knowledge_graph_kwd": [kind]}
- if scope_kwd:
- condition["scope_kwd"] = [scope_kwd]
- if doc_ids:
- condition["doc_id"] = list(doc_ids)
- if extra:
- condition.update(extra)
-
- fields = ["content_with_weight", "source_chunk_ids", "doc_id", "docnm_kwd", "name_kwd", "mention_count_int", "from_entity_kwd", "to_entity_kwd"]
- exprs = []
- if text:
- knn_topn = pool or top_n
- if getattr(tools, "embed_mdl", None):
- try:
- exprs.append(await settings.retriever.get_vector(text, tools.embed_mdl, knn_topn, similarity))
- except Exception:
- _LOG.exception("[Graph exploration] vector build failed; using keyword match")
- if not exprs:
- exprs.append(MatchTextExpr(["content_ltks", "content_sm_ltks"], text, knn_topn))
-
- order_by = OrderByExpr()
- if order_desc:
- try:
- order_by.desc(order_desc)
- except Exception:
- order_by = OrderByExpr()
-
- try:
- res = await thread_pool_exec(
- settings.docStoreConn.search,
- fields,
- [],
- condition,
- exprs,
- order_by,
- 0,
- top_n,
- search.index_name(tenant_id),
- [kb_id],
- )
- rows = settings.docStoreConn.get_fields(res, fields) or {}
- except Exception:
- _LOG.exception("[Graph exploration] KG search failed (kind=%s)", kind)
- return {}
- return rows
-
-
-def _kg_parse_entity(row: dict) -> dict | None:
- try:
- payload = json.loads(row.get("content_with_weight") or "{}")
- except Exception:
- payload = {}
- name = (payload.get("name") or payload.get("term") or payload.get("title") or "").strip()
- if not name:
- return None
- aliases = [str(a).strip() for a in (payload.get("aliases") or []) if str(a).strip()]
- return {
- "name": name,
- "type": (payload.get("type") or "other"),
- "description": (payload.get("description") or payload.get("description") or ""),
- "aliases": aliases,
- "source_chunk_ids": list(row.get("source_chunk_ids") or []),
- "doc_id": row.get("doc_id") or "",
- "docnm_kwd": row.get("docnm_kwd") or "",
- }
-
-
-def _kg_parse_relation(row: dict) -> dict | None:
- src = (row.get("from_entity_kwd") or "").strip()
- tgt = (row.get("to_entity_kwd") or "").strip()
- if not src or not tgt:
- return None
- typ = "related"
- try:
- payload = json.loads(row.get("content_with_weight") or "{}")
- typ = payload.get("type") or payload.get("relation") or "related"
- except Exception:
- pass
- return {
- "from": src,
- "to": tgt,
- "type": typ,
- "source_chunk_ids": list(row.get("source_chunk_ids") or []),
- "doc_id": row.get("doc_id") or "",
- }
-
-
-def _endpoint_terms(names) -> list[str]:
- """Case variants for matching relation endpoints.
-
- ``dataset_structure_merger`` lowercases ``from_entity_kwd``/``to_entity_kwd``
- on merged rows while entity names keep their original case, so hop queries
- must try both forms. Accepts a single name or an iterable and returns the
- sorted union of each name's original and lowercased form.
- """
- if isinstance(names, str):
- names = [names]
- terms: set[str] = set()
- for n in names or []:
- n = (n or "").strip()
- if n:
- terms.add(n)
- terms.add(n.lower())
- return sorted(terms)
-
-
-def _collect_evidence_ids(entities: list[dict], relations: list[dict], relevant_names: list[str]) -> dict:
- """Group the source_chunk_ids of the relevant entities AND relations by doc.
-
- An entity is relevant when its name/alias was named by the model; a relation
- is relevant when either endpoint is.
- """
- wanted = {n.strip().lower() for n in relevant_names if isinstance(n, str) and n.strip()}
- by_doc: dict[str, list[str]] = {}
- seen: set[tuple[str, str]] = set()
-
- def _add(doc_id: str, ids):
- for cid in ids or []:
- if not (isinstance(cid, str) and cid):
- continue
- key = (doc_id, cid)
- if key in seen:
- continue
- seen.add(key)
- by_doc.setdefault(doc_id, []).append(cid)
-
- for e in entities:
- names = {(e.get("name") or "").strip().lower(), *[(a or "").strip().lower() for a in (e.get("aliases") or [])]}
- if names & wanted:
- _add(e.get("doc_id") or "", e.get("source_chunk_ids"))
- for r in relations:
- if {(r.get("from") or "").strip().lower(), (r.get("to") or "").strip().lower()} & wanted:
- _add(r.get("doc_id") or "", r.get("source_chunk_ids"))
- return by_doc
-
-
-async def graph_explore(tools, query: str, keywords: str = "", doc_scope: list[str] | None = None) -> dict:
- """Explore the compiled knowledge graph to answer ``query``.
-
- Seeds the top-``_KG_SEEDS`` entities for the question (dense match above
- ``_KG_SEED_SIM``, ranked by ``mention_count_int``), hops ``_KG_HOPS`` out over
- their relations, then asks the chat model whether that subgraph answers the
- question. When it does, the answer is returned directly; when it doesn't, the
- source passages behind the entities/relations the model found relevant are
- returned as evidence (narrowed by ``keywords``) so the caller can continue.
-
- Without ``doc_scope`` the dataset-merged graph is searched
- (``scope_kwd="dataset"``); with it, the per-document rows of those docs
- (``scope_kwd="doc"``, filtered by ``doc_id``).
-
- :returns: ``{"answer": str, "chunks": [...], "doc_aggs": [...]}`` — exactly one
- of ``answer`` / ``chunks`` is populated.
- """
- from rag.advanced_rag.harness.tools.search import _narrow_by_keywords
-
- _empty = {"answer": "", "chunks": [], "doc_aggs": []}
- if hasattr(tools, "scoped_doc_ids"):
- doc_scope = tools.scoped_doc_ids(doc_scope)
- _LOG.info(f'[Graph exploration] Exploring the knowledge graph for "{query}" (keywords: {keywords})')
-
- scopes = await _kg_scopes(tools, doc_scope)
- if not scopes:
- _LOG.info("[Graph exploration] No knowledge base in scope.")
- return _empty
-
- scope_kwd = _SCOPE_KWD_DOC if doc_scope else _SCOPE_KWD_DATASET
- text = f"{query} {keywords}".strip()
- entities: list[dict] = []
- relations: list[dict] = []
- ent_names: set[str] = set()
-
- def _add_entities(new: list[dict], scope_key: str = "") -> list[str]:
- added = []
- for e in new:
- key = f"{scope_key}:{e['name'].lower()}"
- if key in ent_names:
- continue
- ent_names.add(key)
- entities.append(e)
- added.append(e["name"])
- return added
-
- for kb_id, tenant_id, doc_ids in scopes:
- # (1) Seeds: condition C — dense match (>= _KG_SEED_SIM) over the scoped
- # entity rows, ranked by mention_count_int desc, top _KG_SEEDS.
- seed_rows = await _kg_search(
- tools,
- kb_id,
- tenant_id,
- doc_ids,
- "entity",
- text=text,
- top_n=_KG_SEEDS,
- scope_kwd=scope_kwd,
- order_desc="mention_count_int",
- pool=_KG_SEED_POOL,
- similarity=_KG_SEED_SIM,
- )
- seeds = [e for e in (_kg_parse_entity(r) for r in seed_rows.values()) if e]
- frontier = _add_entities(seeds, kb_id)
- _LOG.info("[Graph exploration] Seeded %d entity(ies): %s", len(frontier), ", ".join(frontier) or "none")
-
- # (2) Expand _KG_HOPS out, collecting relations and neighbour entities.
- for _hop in range(_KG_HOPS):
- if not frontier:
- break
- terms = _endpoint_terms(frontier) # case variants — merged rows lowercase endpoints
- rel_rows: dict = {}
- rel_rows.update(await _kg_search(tools, kb_id, tenant_id, doc_ids, "relation", top_n=_KG_REL_LIMIT, scope_kwd=scope_kwd, extra={"from_entity_kwd": terms}))
- rel_rows.update(await _kg_search(tools, kb_id, tenant_id, doc_ids, "relation", top_n=_KG_REL_LIMIT, scope_kwd=scope_kwd, extra={"to_entity_kwd": terms}))
- hop_relations = [r for r in (_kg_parse_relation(x) for x in rel_rows.values()) if r]
- relations.extend(hop_relations)
-
- seen_lower = {k.split(":", 1)[1] for k in ent_names if k.startswith(f"{kb_id}:")}
- neigh_names = {n.strip() for r in hop_relations for n in (r["from"], r["to"]) if n and n.strip()}
- neigh_lower_set = {n.lower() for n in neigh_names} - seen_lower
- if not neigh_lower_set:
- break
- neigh_filtered = {n for n in neigh_names if n.lower() in neigh_lower_set}
- neigh_rows = await _kg_search(
- tools,
- kb_id,
- tenant_id,
- doc_ids,
- "entity",
- top_n=min(max(len(neigh_filtered), 1), _KG_NEIGHBORS),
- scope_kwd=scope_kwd,
- extra={"name_kwd": _endpoint_terms(neigh_filtered)},
- )
- neighbours = [e for e in (_kg_parse_entity(r) for r in neigh_rows.values()) if e]
- frontier = _add_entities(neighbours, kb_id)
- _LOG.info("[Graph exploration] Hop %d reached %d neighbour entity(ies).", _hop + 1, len(frontier))
-
- if not entities and not relations:
- _LOG.info("[Graph exploration] No compiled knowledge graph in scope.")
- return _empty
-
- _LOG.info("[Graph exploration] Built a subgraph of %d entity(ies) and %d relation(s).", len(entities), len(relations))
-
- # (3) Does the subgraph answer the question?
- answer, relevant = await _ask_structure(tools, query, entities, relations, "knowledge graph", "Graph exploration")
-
- # (4a) Sufficient — return the answer, no chunks.
- if answer:
- _LOG.info("[Graph exploration] The subgraph answered the question directly.")
- return {"answer": answer, "chunks": [], "doc_aggs": []}
-
- # (4b) Insufficient — return the source passages behind the relevant nodes.
- evidence = _collect_evidence_ids(entities, relations, relevant)
- chunks: list[dict] = []
- for doc_id, ids in evidence.items():
- if doc_id and ids:
- chunks.extend(await _load_chunks_by_ids(tools, doc_id, ids))
-
- before = len(chunks)
- chunks = _narrow_by_keywords(chunks, keywords)
- _LOG.info("[Graph exploration] Insufficient; returning %d evidence passage(s) (%d before keyword filtering).", len(chunks), before)
-
- return {"answer": "", "chunks": chunks, "doc_aggs": _doc_aggs(chunks)}
-
+# Knowledge-graph and wiki exploration live in ``exploration`` and are
+# re-exported here: compiled-product expansion imports ``_kg_scopes`` from this
+# module, and the action-session tool surface exposes ``graph_explore`` by this
+# name.
+from rag.advanced_rag.harness.tools.exploration import ( # noqa: F401
+ _collect_evidence_ids,
+ _endpoint_terms,
+ _kg_parse_entity,
+ _kg_parse_relation,
+ _kg_scopes,
+ _kg_search,
+ _SCOPE_KWD_DATASET,
+ _SCOPE_KWD_DOC,
+ graph_explore,
+)
# ── Navigate-tree/structure tool set (migrated from harness/dynamic) ──
_NAV_TREE_MAX_DOCS = 8
@@ -1074,39 +704,6 @@ _SEARCH_SNIPPET_CHARS = 300
# ── XML helpers (uniform tag vocabulary shared by all retrieval tools) ──
-def _xml_escape(value: Any) -> str:
- s = "" if value is None else str(value)
- return s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
-
-
-def _chunk_text(c: dict) -> str:
- return str(c.get("content_with_weight") or c.get("content") or c.get("text") or "")
-
-
-def _chunk_attr(c: dict, keys: tuple[str, ...]) -> str:
- for k in keys:
- v = c.get(k)
- if v not in (None, ""):
- return str(v)
- return ""
-
-
-def _doc_id(c: dict) -> str:
- return _chunk_attr(c, ("doc_id", "docid", "document_id"))
-
-
-def _dataset_id(c: dict) -> str:
- return _chunk_attr(c, ("dataset_id", "kb_id", "knowledgebase_id"))
-
-
-def _doc_title(c: dict) -> str:
- return _chunk_attr(c, ("docnm_kwd", "doc_title", "title", "document_name"))
-
-
-def _chunk_id(c: dict) -> str:
- return _chunk_attr(c, ("chunk_id", "id"))
-
-
def _rank_chunks_by_terms(candidates: list[dict], queries: list[str]) -> list[dict]:
"""Rank candidate chunks by how many query terms overlap with their text.
@@ -1774,14 +1371,6 @@ async def _render_toc_drilldown(query: str, qvec, entities: list[dict], relation
return "\n".join(lines)
-def _snippet(s: str, n: int) -> str:
- """Truncate a string to ``n`` chars on a char boundary with an ellipsis."""
- s = s.strip()
- if len(s) <= n:
- return s
- return s[:n].rstrip() + "..."
-
-
def _render_outline(entities: list[dict], relations: list[dict]) -> str:
"""Render a compact flat outline (fallback when no query terms are usable)."""
lines: list[str] = []
diff --git a/rag/advanced_rag/harness/tools/registry.py b/rag/advanced_rag/harness/tools/registry.py
deleted file mode 100644
index b4b4a4c746..0000000000
--- a/rag/advanced_rag/harness/tools/registry.py
+++ /dev/null
@@ -1,159 +0,0 @@
-"""Tool registry: all available tools with metadata and function schemas."""
-
-from typing import Any
-
-# Tool registry: tool_name -> {metadata, function_schema, fn}
-# 'fn' filled at registration time; schema used for LLM tool definitions.
-TOOL_REGISTRY: dict[str, dict[str, Any]] = {}
-
-# Executor interface
-# Each tool registers a callable with signature:
-# async def fn(tools, **kwargs) -> dict # {"chunks": [...], ...}
-
-
-def register_tool(name: str, schema: dict, fn: callable, requires_compilation: bool = False, compilation_type: str | tuple[str, ...] | None = None, processing_time: str = "fast") -> None:
- TOOL_REGISTRY[name] = {
- "name": name,
- "function_schema": schema,
- "fn": fn,
- "requires_compilation": requires_compilation,
- "compilation_type": compilation_type,
- "processing_time": processing_time,
- }
-
-
-def get_tool(tool_name: str) -> dict | None:
- return TOOL_REGISTRY.get(tool_name)
-
-
-def get_function_schemas(tool_names: list[str]) -> list[dict]:
- """Return function schemas for the given tool names, if registered."""
- return [TOOL_REGISTRY[n]["function_schema"] for n in tool_names if n in TOOL_REGISTRY]
-
-
-# Common schema builders
-
-
-def _search_schema(name: str, desc: str) -> dict:
- return {
- "type": "function",
- "function": {
- "name": name,
- "description": desc,
- "parameters": {
- "type": "object",
- "properties": {
- "query": {"type": "string", "description": "the original user's question."},
- "keywords": {"type": "string", "description": "the keywords used for searching split by space or ','."},
- },
- "required": ["query"],
- },
- },
- }
-
-
-def _navigate_schema(name: str, desc: str) -> dict:
- return {
- "type": "function",
- "function": {
- "name": name,
- "description": desc,
- "parameters": {
- "type": "object",
- "properties": {
- "topic": {"type": "string", "description": "the query/keywords to navigate to."},
- "keywords": {"type": "string", "description": "the query/keywords used for searching split by space or ','."},
- "doc_scope": {"type": "array", "items": {"type": "string"}, "description": "The doc ID list to narrow down the searching scope. None or empty value means no limitation at all."},
- },
- "required": ["topic"],
- },
- },
- }
-
-
-def _inspector_schema(name: str, desc: str, props: dict = None) -> dict:
- schema = {
- "type": "function",
- "function": {
- "name": name,
- "description": desc,
- "parameters": {
- "type": "object",
- "properties": props
- or {
- "chunk_id": {"type": "string", "description": "chunk ID"},
- },
- "required": list((props or {"chunk_id": {}}).keys()),
- },
- },
- }
- return schema
-
-
-def _think_schema() -> dict:
- return {
- "type": "function",
- "function": {
- "name": "think_tool",
- "description": "Internal reasoning. Analyze the collected results and plan the next step. Do not output final user-facing content while reasoning.",
- "parameters": {
- "type": "object",
- "properties": {
- "reasoning": {
- "type": "string",
- "description": "Reasoning content: what has been found, what is still missing, and what to do next.",
- },
- },
- "required": ["reasoning"],
- },
- },
- }
-
-
-def _generate_report_schema() -> dict:
- return {
- "type": "function",
- "function": {
- "name": "generate_report",
- "description": "Call when the research is complete. Output the research report and claim-level verification results.",
- "parameters": {
- "type": "object",
- "properties": {
- "report": {"type": "string", "description": "Research result report, factual and unformatted."},
- "is_verified": {"type": "boolean", "description": "Whether sufficient evidence was found."},
- "confidence": {"type": "number", "description": "Confidence from 0 to 1."},
- "evidence_ids": {
- "type": "array",
- "items": {"type": "integer"},
- "description": "Referenced chunk IDs.",
- },
- "gaps": {
- "type": "array",
- "items": {"type": "string"},
- "description": "Information that was not found.",
- },
- "grounded": {
- "type": "array",
- "items": {"type": "string"},
- "description": "The KEY factual assertions in the report that ARE directly supported by the cited evidence chunks (evidence_ids). List each atomic fact you saw in the evidence, verbatim enough to be matched. This must include the answer-critical facts — e.g. 'hometown is Ithaca', 'first solo album was 1970'. Facts you inferred from prior knowledge but did NOT see in the evidence must be omitted from this list (and instead placed in gaps). Leaving a key asserted fact out of grounded, or listing a fact not actually in the evidence, will be caught and the claim marked unverified.",
- },
- "numbers": {
- "type": "array",
- "items": {"type": "string"},
- "description": "For numerical / multi-hop questions, list each number you used in the final answer together with its exact figure and source. Format each as '<figure> from <source/context>', e.g. '2,161,000 from Wikipedia Demographics of Paris 2019 table', '9,508 from Brown County 2020 census'. This makes the numeric口径 transparent and verifiable. If the evidence contained multiple conflicting values for the same quantity, list ALL of them and say which one you chose and why. Do NOT silently pick one of several conflicting figures; disclose the conflict.",
- },
- "discovered_claims": {
- "type": "array",
- "items": {"type": "string"},
- "description": "New research directions discovered during research.",
- },
- },
- # grounded and numbers are required: the agent must explicitly
- # declare which facts are evidence-backed (grounded) and which
- # figures+source it used (numbers, [] for non-numeric). These feed
- # the cross-check's grounded-fact and numeric-conflict detection,
- # so silently omitting them would bypass verification.
- "required": ["report", "is_verified", "confidence", "grounded", "numbers"],
- },
- },
- }
diff --git a/rag/advanced_rag/harness/tools/search.py b/rag/advanced_rag/harness/tools/search.py
index 6a09cddfec..91e6302d9e 100644
--- a/rag/advanced_rag/harness/tools/search.py
+++ b/rag/advanced_rag/harness/tools/search.py
@@ -1,458 +1,48 @@
-"""Search tools: hybrid, vector, BM25, web, structured."""
+"""Search tools: hybrid, vector, BM25, web, structured.
+
+Retrieval legs plus the compiled-structure expansion layered on top of hybrid
+search. The tool-facing schemas and dispatch live in ``action_session``; this
+module owns the retrieval implementations they call.
+"""
import logging
import re
from typing import Any
-from rag.llm.tool_decorator import tool
-import hashlib
-from functools import lru_cache
from common import settings
-from .navigation import _expand_related_via_structure, _kg_scopes
-
-_LOG = logging.getLogger(__name__)
-
-
-def _compact_keywords(kw: str, max_terms: int = 15) -> str:
- """Deduplicate and cap a formalize / extract_keywords keyword string.
-
- The extraction prompt asks the model for 3-10 terms *plus* 2-3 synonyms each,
- which models answer with a 40-60 word redundant synonym run (e.g. "average
- distance left field line MLB retractable roof stadiums 2024 ... retractable
- dome covered stadium mean distance outfield" — ~350 chars). Appending that
- whole run onto the query diluted the vector leg and dragged BM25 onto
- unrelated docs. This keeps the recall terms but drops the redundancy:
- dedupe (preserving order) and cap at ``max_terms`` so it stays a compact
- hint instead of a pollution source. Accepts both space- and comma-separated
- input (single-turn extract_keywords emits spaces; multi-turn formalize
- emits commas).
- """
- if not kw:
- return ""
- tokens = re.split(r"[,\s]+", (kw or "").strip())
- seen: list[str] = []
- for t in tokens:
- t = t.strip()
- if not t:
- continue
- if t not in seen:
- seen.append(t)
- if len(seen) >= max_terms:
- break
- return " ".join(seen)
-
-
-# Sentence terminators: Chinese 。!?;, English ! ? ;, newline, and a
-# digit-guarded English period (so "3.14" / "v1.2" don't split).
-_SENT_END = re.compile(r"[。!?;!?;]+|(?<!\d)\.(?!\d)")
-
-# Block-level HTML elements and markdown tables are kept ATOMIC — never split by
-# sentence terminators — so a whole table / list / block counts as ONE "sentence"
-# for keyword matching and narrowing (a keyword inside one keeps the whole block).
-_HTML_TAG = re.compile(r"<(/?)([a-zA-Z][a-zA-Z0-9]*)\b([^>]*)>")
-
-# Only BLOCK-level containers are protected. Inline tags (<b>, <i>, <a>, <span>,
-# <em>, <strong>, <code>, ...) are deliberately excluded so ordinary prose that
-# contains inline formatting still splits into sentences normally.
-_HTML_BLOCK_TAGS = {
- "table",
- "thead",
- "tbody",
- "tfoot",
- "tr",
- "td",
- "th",
- "caption",
- "colgroup",
- "ul",
- "ol",
- "li",
- "dl",
- "dt",
- "dd",
- "div",
- "p",
- "pre",
- "blockquote",
- "section",
- "article",
- "aside",
- "nav",
- "main",
- "figure",
- "figcaption",
- "header",
- "footer",
- "address",
- "details",
- "summary",
- "form",
- "fieldset",
- "h1",
- "h2",
- "h3",
- "h4",
- "h5",
- "h6",
-}
-
-# Markdown table: a header row with a pipe, a separator row of dashes/colons/
-# pipes, then zero+ body rows with a pipe.
-_MD_TABLE = re.compile(
- r"^[ \t]*\|?[^\n]*\|[^\n]*\r?\n"
- r"[ \t]*\|?[ \t]*:?-{1,}:?[ \t]*(?:\|[ \t]*:?-{1,}:?[ \t]*)+\|?[ \t]*\r?\n"
- r"(?:[ \t]*\|?[^\n]*\|[^\n]*\r?\n?)*",
- re.MULTILINE,
+from rag.advanced_rag.harness.chunk_utils import ( # noqa: F401
+ _chunk_attr,
+ _chunk_id,
+ _chunk_text,
+ _dataset_id,
+ _doc_id,
+ _doc_title,
+ _snippet,
+ _xml_escape,
)
+# ``_expand_related_via_structure`` is kept imported (not currently called here)
+# so the compiled-structure related-chunk expansion stays reachable without
+# re-editing imports; the @tool front-end that used it was removed.
+from .navigation import _expand_related_via_structure, _kg_scopes # noqa: F401
-def _html_block_spans(text: str) -> list[tuple[int, int]]:
- """Outermost balanced block-level HTML element spans (nesting-aware).
-
- Uses a tag stack (not a regex) so nested elements (e.g. a ``<table>`` with
- ``<td>``s, or nested ``<div>``s) yield ONE span for the outermost element and
- are never truncated at the first close tag the way a non-greedy regex would.
- Unclosed / stray tags are ignored (that region just falls back to plain
- sentence splitting).
- """
- spans: list[tuple[int, int]] = []
- stack: list[tuple[str, int]] = []
- for m in _HTML_TAG.finditer(text):
- name = m.group(2).lower()
- if name not in _HTML_BLOCK_TAGS:
- continue
- if m.group(1): # closing tag </name>
- for i in range(len(stack) - 1, -1, -1):
- if stack[i][0] == name:
- start = stack[i][1]
- del stack[i:]
- if not stack: # closed an outermost block
- spans.append((start, m.end()))
- break
- # a stray </name> with no matching open is ignored
- elif not m.group(3).rstrip().endswith("/"): # opening (skip self-closing)
- stack.append((name, m.start()))
- return spans
-
-
-def _protected_spans(text: str) -> list[tuple[int, int]]:
- """Non-overlapping ``(start, end)`` spans kept atomic, in order.
-
- Covers block-level HTML elements and markdown tables; overlapping spans are
- merged (unioned) so a match that straddles two is never split.
- """
- spans = _html_block_spans(text)
- spans += [(m.start(), m.end()) for m in _MD_TABLE.finditer(text)]
- spans.sort()
- merged: list[tuple[int, int]] = []
- last_end = -1
- for s, e in spans:
- if s < last_end: # overlaps an already-kept span -> union it in
- if e > last_end:
- merged[-1] = (merged[-1][0], e)
- last_end = e
- continue
- merged.append((s, e))
- last_end = e
- return merged
-
-
-def _split_plain(text: str) -> list[str]:
- """Terminator-based sentence split, keeping each terminator attached."""
- sents: list[str] = []
- start = 0
- for m in _SENT_END.finditer(text):
- end = m.end()
- seg = text[start:end]
- if seg.strip():
- sents.append(seg)
- start = end
- if start < len(text):
- tail = text[start:]
- if tail.strip():
- sents.append(tail)
- return sents
-
-
-def _split_sentences(text: str) -> list[str]:
- """Split ``text`` into sentences, keeping each terminator attached.
-
- Block-level HTML elements (``<table>``, ``<div>``, ``<p>``, ``<ul>``, ... —
- see :data:`_HTML_BLOCK_TAGS`) and markdown tables are treated as a single
- atomic sentence and are never split internally, so a keyword falling inside
- one keeps the whole block together.
- """
- if not text:
- return []
- spans = _protected_spans(text)
- if not spans:
- return _split_plain(text)
-
- sents: list[str] = []
- pos = 0
- for s, e in spans:
- if s > pos:
- sents.extend(_split_plain(text[pos:s]))
- block = text[s:e]
- if block.strip():
- sents.append(block)
- pos = e
- if pos < len(text):
- sents.extend(_split_plain(text[pos:]))
- return sents
-
-
-# ---------------------------------------------------------------------------
-# Stem-tolerant keyword matching (ported from agentic_search4 v8)
-#
-# Substring matching misses inflected forms: "nominations" misses "nominated",
-# "company" misses "companies". Keywords are derived from the question, which
-# states things in the inflected form ("which band HEADLINED", "was NOMINATED
-# three times"), so the failing direction is the common one. Both sides are
-# therefore reduced to a stem before comparison.
-# ---------------------------------------------------------------------------
-try: # available at runtime — nltk already backs rag/nlp/synonym.py
- from nltk.stem import PorterStemmer as _PorterStemmer
-
- _porter_stem = _PorterStemmer().stem
-except Exception: # pragma: no cover - exercised only where nltk is absent
- _porter_stem = None
-
-# Longest first: "nominations" must lose "ations", not just the trailing "s".
-_STEM_SUFFIXES = (
- ("ations", ""),
- ("ation", ""),
- ("ated", ""),
- ("ates", ""),
- ("ate", ""),
- ("ings", ""),
- ("ing", ""),
- ("ies", "i"),
- ("ied", "i"),
- ("ed", ""),
- ("es", ""),
- ("s", ""),
-)
-
-_WORD_RE = re.compile(r"[a-z0-9]+")
-
-
-def _fallback_stem(word: str) -> str:
- """Suffix stripper used when nltk is unavailable. Approximates Porter."""
- w = word
- for suffix, replacement in _STEM_SUFFIXES:
- if w.endswith(suffix) and len(w) - len(suffix) >= 3:
- w = w[: len(w) - len(suffix)] + replacement
- break
- if len(w) > 3 and w.endswith("y"):
- w = w[:-1] + "i"
- if len(w) > 3 and w.endswith("e"):
- w = w[:-1]
- if len(w) > 3 and w[-1] == w[-2] and w[-1] not in "aeiou":
- w = w[:-1] # running -> runn -> run
- return w
-
-
-@lru_cache(maxsize=8192)
-def _stem(word: str) -> str:
- return _porter_stem(word) if _porter_stem else _fallback_stem(word)
-
-
-def _stemmable(token: str) -> bool:
- """Only plain ASCII words are stemmed.
-
- Identifiers ("1344259", "2020-21"), notation ("PPG") and CJK text must match
- verbatim — stemming would corrupt them, and it has no meaning for Chinese.
- """
- return len(token) >= 4 and token.isascii() and token.isalpha()
-
+_LOG = logging.getLogger(__name__)
-def _keyword_forms(kwds: list[str]) -> tuple[list[str], list[tuple[str, ...]]]:
- """Split keywords into verbatim substrings and stem sequences.
- A keyword whose tokens are ALL stemmable becomes a stem sequence (matched
- anywhere as a contiguous run of stems); anything containing an identifier,
- notation or CJK falls back to a verbatim substring match.
- """
- verbatim: list[str] = []
- stemmed: list[tuple[str, ...]] = []
- for kw in kwds or []:
- k = (kw or "").strip().lower()
- if not k:
- continue
- tokens = _WORD_RE.findall(k)
- if tokens and all(_stemmable(t) for t in tokens):
- stemmed.append(tuple(_stem(t) for t in tokens))
- else:
- verbatim.append(k)
- return verbatim, stemmed
-
-
-def _sentence_stems(sentence: str) -> list[str]:
- return [_stem(t) if _stemmable(t) else t for t in _WORD_RE.findall(sentence.lower())]
-
-
-def _sentence_matches(low: str, stems: list[str], verbatim: list[str], stemmed: list[tuple[str, ...]]) -> bool:
- """True when a sentence contains a verbatim keyword or a contiguous stem run."""
- if any(v in low for v in verbatim):
- return True
- for seq in stemmed:
- width = len(seq)
- for start in range(len(stems) - width + 1):
- if tuple(stems[start : start + width]) == seq:
- return True
- return False
-
-
-_FACT_RE = re.compile(
- r"(\d[\d,\.]*(?:st|nd|rd|th)?%?)"
- r"|(19|20)\d{2}" # years
- r"|\b(percent|percentage|million|billion|thousand|km|km2|sq\s*km|m\s*above|m)"
- r"\b",
- re.IGNORECASE,
+# Text processing (sentence split / stemming / keyword narrowing) lives in
+# ``text_processing`` and is re-exported here: callers across the harness
+# import these names from this module.
+from rag.advanced_rag.harness.tools.text_processing import ( # noqa: F401
+ _compact_keywords,
+ _highlight_keywords,
+ _is_fact_dense_sentence,
+ _keyword_forms,
+ _narrow_by_keywords,
+ _narrow_content,
+ _narrow_or_keep,
+ _sentence_matches,
+ _split_sentences,
+ _stem,
)
-_PROPER_NOUN_RE = re.compile(r"(?<![.!?]\.)\b[A-Z][a-z]{2,}\b")
-
-
-def _is_fact_dense_sentence(sent: str) -> bool:
- """Heuristically flag a sentence that carries a fact the answer may hinge on
- but which does not necessarily contain the query keywords — a number, a year,
- a percentage, or a proper noun / named entity. Such sentences are kept during
- narrowing even when they sit far from any keyword hit, so a numeric or
- entity answer is never dropped just because it lacks the keyword phrasing.
- """
- low = sent.lower()
- if _FACT_RE.search(sent) or _FACT_RE.search(low):
- return True
- if _PROPER_NOUN_RE.search(sent):
- return True
- return False
-
-
-def _narrow_content(content: str, kwds: list[str]) -> str | None:
- """Return ``content`` narrowed to keyword sentences +/- 2 neighbours.
-
- Matching is stem-tolerant: a keyword matches any word sharing its stem, so
- "nominations" finds "nominated". Sentences that are fact-dense (numbers /
- years / percentages / proper nouns) are kept regardless of keyword distance,
- so numeric or named-entity answers survive narrowing. Returns ``None`` when
- no keyword occurs anywhere in ``content``.
- """
- # Structured tables must be returned whole. A keyword hit anywhere in a big
- # table (e.g. the capitals-by-latitude table) otherwise narrows to the hit
- # sentence +/- neighbours and DROPS the far end of the table — precisely the
- # "table truncated at -4.58°N, Maseru (-29.3°) missing" bug on FRAMES Q408.
- # A table row is one data point, not a sentence, so keyword-window narrowing is
- # wrong here; keep the full table (it is already rank-sorted by the retriever).
- if "<table" in content.lower() or "<tr" in content.lower() or "<td" in content.lower():
- return "..." + _highlight_keywords(content, kwds) + "..."
-
- sents = _split_sentences(content)
- if not sents:
- return None
- # Stem-tolerant matching: a keyword matches any word sharing its stem, so
- # "nominations" finds "nominated" and "company" finds "companies".
- verbatim, stemmed = _keyword_forms(kwds)
- if not verbatim and not stemmed:
- return None
- keep: set[int] = set()
- matched = False
- for i, s in enumerate(sents):
- low = s.lower()
- if _sentence_matches(low, _sentence_stems(s), verbatim, stemmed):
- matched = True
- for j in range(max(0, i - 2), min(len(sents), i + 3)):
- keep.add(j)
- elif _is_fact_dense_sentence(s):
- # Keep fact-dense sentences even without a keyword hit so the answer
- # value (a bare figure, a date, a proper noun) is never lost.
- for j in range(max(0, i - 1), min(len(sents), i + 2)):
- keep.add(j)
- if not matched:
- return None
- narrowed = "".join(sents[i] for i in sorted(keep)).strip()
- return "..." + _highlight_keywords(narrowed, kwds) + "..."
-
-
-def _highlight_keywords(text: str, kwds: list[str]) -> str:
- """Star the verbatim keyword phrases AND any word sharing a keyword's stem.
-
- Full keyword phrases are matched first and starred as ONE contiguous span, so
- a multi-word entity like "Atlanta Braves" becomes ``*Atlanta Braves*`` — never
- ``*Atlanta* *Braves*`` — because the downstream cross-check matches entities
- with a bounded contiguous regex that a per-word star would break.
- """
- phrases = sorted({(kw or "").strip().lower() for kw in kwds or [] if (kw or "").strip()}, key=len, reverse=True)
- terms: list[str] = list(phrases)
- # Add stem-matched words NOT already inside a phrase, so "nominated" still
- # gets starred for keyword "nominations" while "Atlanta Braves" stays whole.
- verbatim, stemmed = _keyword_forms(kwds)
- stem_set = {s for seq in stemmed for s in seq}
- if stem_set:
- for word in re.findall(r"[A-Za-z]+", text):
- low = word.lower()
- if _stemmable(low) and _stem(low) in stem_set and not any(low in p for p in phrases):
- terms.append(low)
- if not terms:
- return text
- # Longest first so a phrase wins over a word it contains; one pass, so an
- # already-starred span is never starred again.
- pattern = re.compile("|".join(re.escape(t) for t in sorted(terms, key=len, reverse=True)), re.IGNORECASE)
- return pattern.sub(lambda m: f"*{m.group(0)}*", text)
-
-
-def _narrow_by_keywords(chunks: list[dict], keywords: str) -> list[dict]:
- """Narrow each chunk to its keyword-bearing sentences (+/- 1 neighbour) and
- drop keyword-less chunks.
-
- Keywords are the comma-separated terms (with close synonyms) produced by
- ``formalize``; matching is stem-tolerant (a keyword matches any word sharing
- its stem).
- """
- kwds = [k.strip().lower() for k in (keywords or "").split(",") if k.strip()]
- if not kwds or not chunks:
- return chunks
- if len(kwds) < 3:
- kwds = [k.strip().lower() for k in (keywords or "").split(" ") if k.strip()]
- _kwds = []
- for i in range(len(kwds) - 1):
- _kwds.append(kwds[i] + " " + kwds[i + 1])
- kwds = _kwds
-
- scored = [(ck, _narrow_content(ck.get("content_with_weight") or ck.get("content") or "", kwds)) for ck in chunks]
- out: list[dict] = []
- dedup: set[str] = set()
- for ck, nc in scored:
- if nc is not None:
- nc_hash = hashlib.md5(nc.encode("utf-8")).hexdigest()
- if nc_hash in dedup:
- continue
- dedup.add(nc_hash)
- ck["content_with_weight"] = nc
- if "content" in ck:
- ck["content"] = nc
- ck.pop("highlight", None)
- out.append(ck)
- return out
-
-
-def _narrow_or_keep(chunks: list[dict], keywords: str, label: str) -> list[dict]:
- """Narrow chunks to keyword sentences, but keep the originals when
- narrowing would drop everything.
-
- No keyword overlap does not mean irrelevant — the retriever already ranked
- these chunks, and a sub-question's wording need not contain the parent
- question's keywords. Dropping them all produced empty results, unverified
- claims and pointless retry cycles.
- """
- if not keywords or not chunks:
- return chunks
- length = len(chunks)
- narrowed = _narrow_by_keywords(chunks, keywords)
- if narrowed:
- _LOG.info(f"[{label}] Kept {len(narrowed)} of {length} passage(s) that actually mention the keywords.")
- return narrowed
- _LOG.info(f"[{label}] Keyword narrowing matched nothing — keeping all {length} retrieved passage(s).")
- return chunks
def _search_cache_key(effective_query: str, target_ids, top_n: int, doc_scope) -> tuple:
@@ -621,471 +211,17 @@ async def bm25_search(tools, query: str, kb_ids: list[str] | None = None, top_n:
return kbinfos
-# ─── Compiled product expansion (zero-LLM, used by hybrid_search with use_compiled=True) ───
-
-
-async def _expand_with_compiled(tools, query: str, keywords: str, kbinfos: dict, doc_scope: list[str] | None = None) -> None:
- """Zero-LLM compiled-product expansion: page_index → tree → KG.
-
- For each bound KB, searches compiled entity rows matching the query,
- hops 1-hop via relations to find neighbour entities, then appends
- their source passages to ``kbinfos["chunks"]``.
- """
- before = len(kbinfos.get("chunks", []))
- seen_ids = {c.get("chunk_id") or c.get("id") for c in kbinfos.get("chunks", [])}
-
- scopes = await _kg_scopes(tools, doc_scope)
- if not scopes:
- return
-
- for kb_id, tenant_id, doc_ids in scopes:
- # 1-hop entity-graph expansion per template kind.
- # Each template writes entity/relation rows tagged with
- # ``compilation_template_kind_kwd`` — search them independently.
- for label, template_kind in (
- ("knowledge_graph", "knowledge_graph"),
- ("mind_map", "mind_map"),
- ("timeline", "timeline"),
- ("page_index", "page_index"),
- ):
- chunks = await _expand_compiled_strategy(
- tools,
- kb_id,
- tenant_id,
- doc_ids,
- query,
- seen_ids,
- template_kind=template_kind,
- max_chunks=5,
- )
- if chunks:
- kbinfos.setdefault("chunks", []).extend(chunks)
- _LOG.debug("[Compiled expand] %s: +%d chunks", label, len(chunks))
-
- # Tree structure graph (uses ``compile_kwd``, not template kind).
- chunks = await _expand_compiled_strategy(
- tools,
- kb_id,
- tenant_id,
- doc_ids,
- query,
- seen_ids,
- compile_kwd="tree",
- max_chunks=5,
- )
- if chunks:
- kbinfos.setdefault("chunks", []).extend(chunks)
- _LOG.debug("[Compiled expand] tree: +%d chunks", len(chunks))
-
- # Synthesis pages — standalone rendered articles from wiki / session
- # graph / session essence templates. Searched directly (no entity-graph nav).
- for label, ckwd in (
- ("wiki_page", "wiki_page"),
- ("artifact_page", "artifact_page"),
- ("essence", "essence"),
- ):
- chunks = await _expand_wiki_page_strategy(
- tools,
- kb_id,
- tenant_id,
- doc_ids,
- query,
- seen_ids,
- compile_kwd=ckwd,
- max_chunks=5,
- )
- if chunks:
- kbinfos.setdefault("chunks", []).extend(chunks)
- _LOG.debug("[Compiled expand] %s: +%d chunks", label, len(chunks))
-
- # Re-sort so compiled-expansion chunks blend by similarity with regular ones.
- chunks = kbinfos.get("chunks", [])
- if chunks:
- chunks.sort(key=lambda c: c.get("similarity", 0.0), reverse=True)
-
- after = len(chunks)
- _LOG.info("[Hybrid search] Compiled expansion added %d chunks.", after - before)
-
-
-async def _search_compiled_rows(
- tools,
- kb_id: str,
- tenant_id: str,
- doc_ids: list[str] | None,
- kind: str,
- *,
- text: str = "",
- top_n: int = 8,
- extra: dict | None = None,
- compile_kwd: str | None = None,
- template_kind: str | None = None,
-) -> dict:
- """Search compiled KG rows in one KB, returning raw field maps.
-
- *compile_kwd* filters by the ``compile_kwd`` field (e.g. "tree" for tree
- structure nodes). *template_kind* filters by ``compilation_template_kind_kwd``
- (e.g. "knowledge_graph", "mind_map"). Leave both ``None`` to scan all rows.
- """
- from common import settings
- from common.doc_store.doc_store_base import MatchTextExpr, OrderByExpr
- from common.misc_utils import thread_pool_exec
- from rag.nlp import search
-
- condition: dict = {"knowledge_graph_kwd": [kind]}
- if compile_kwd:
- condition["compile_kwd"] = compile_kwd
- if template_kind:
- condition["compilation_template_kind_kwd"] = template_kind
- if doc_ids:
- condition["doc_id"] = list(doc_ids)
- if extra:
- condition.update(extra)
-
- fields = [
- "content_with_weight",
- "source_chunk_ids",
- "doc_id",
- "docnm_kwd",
- "from_entity_kwd",
- "to_entity_kwd",
- "name_kwd",
- ]
- exprs = []
- if text:
- embd_mdl = getattr(tools, "embed_mdl", None)
- if embd_mdl:
- try:
- exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_n, 0.1))
- except Exception:
- _LOG.exception("[Compiled expand] vector build failed; using keyword match")
- if not exprs:
- exprs.append(
- MatchTextExpr(
- ["content_ltks", "content_sm_ltks"],
- text,
- top_n,
- )
- )
-
- try:
- res = await thread_pool_exec(
- settings.docStoreConn.search,
- fields,
- [],
- condition,
- exprs,
- OrderByExpr(),
- 0,
- top_n,
- search.index_name(tenant_id),
- [kb_id],
- )
- return settings.docStoreConn.get_fields(res, fields) or {}
- except Exception:
- _LOG.exception("[Compiled expand] search failed (kind=%s compile_kwd=%s)", kind, compile_kwd)
- return {}
-
-
-async def _load_chunks_for_doc(tools, doc_id: str, chunk_ids: list[str]) -> list[dict]:
- """Load chunks by their IDs from the doc store."""
- if not chunk_ids:
- return []
- from common import settings
- from common.doc_store.doc_store_base import OrderByExpr
- from common.misc_utils import thread_pool_exec
- from rag.nlp import search
-
- resolved = tools._resolve_doc_tenant(doc_id)
- if not resolved:
- return []
- kb_id, tenant_id = resolved
-
- fields = ["content_with_weight", "docnm_kwd", "doc_id", "id"]
- try:
- res = await thread_pool_exec(
- settings.docStoreConn.search,
- fields,
- [],
- {"id": list(chunk_ids)},
- [],
- OrderByExpr(),
- 0,
- len(chunk_ids),
- search.index_name(tenant_id),
- [kb_id],
- )
- rows = settings.docStoreConn.get_fields(res, fields)
- if not rows:
- return []
- return [{**v, "chunk_id": k} for k, v in rows.items()]
- except Exception:
- _LOG.exception("[Compiled expand] failed to load chunks for doc_id=%s", doc_id)
- return []
-
-
-async def _expand_compiled_strategy(
- tools,
- kb_id: str,
- tenant_id: str,
- doc_ids: list[str] | None,
- query: str,
- seen_ids: set[str],
- *,
- compile_kwd: str | None = None,
- template_kind: str | None = None,
- max_chunks: int = 5,
-) -> list[dict]:
- """Generic 1-hop compiled expansion: entity search → relation nav → chunk load.
-
- 1. Embedding-match seed entities (filtered by *compile_kwd* or *template_kind*).
- 2. Fetch relations adjacent to seed entities (forward + backward).
- 3. Collect neighbour entity names (1-hop away).
- 4. Look up neighbour entities to get ``source_chunk_ids``.
- 5. Load actual chunks, deduplicate, respect *max_chunks*.
-
- *compile_kwd* is used for structure graphs (e.g. "tree").
- *template_kind* is used for entity extraction rows (e.g. "knowledge_graph").
- """
- import json
-
- # -- 1. Seed entities --
- seed_rows = await _search_compiled_rows(
- tools,
- kb_id,
- tenant_id,
- doc_ids,
- "entity",
- text=query,
- top_n=5,
- compile_kwd=compile_kwd,
- template_kind=template_kind,
- )
- if not seed_rows:
- return []
-
- seed_names: set[str] = set()
- for r in seed_rows.values():
- try:
- payload = json.loads(r.get("content_with_weight") or "{}")
- except Exception:
- continue
- name = (payload.get("name") or payload.get("title") or "").strip()
- if name:
- seed_names.add(name)
- if not seed_names:
- return []
-
- # -- 2. Adjacent relations (outgoing + incoming) --
- # Provide both original and lowercased names — dataset_structure_merger
- # lowercases merged-row endpoints while per-doc rows keep original case.
- seed_list = sorted({n.lower() for n in seed_names} | seed_names)
- fwd = await _search_compiled_rows(
- tools,
- kb_id,
- tenant_id,
- doc_ids,
- "relation",
- top_n=50,
- compile_kwd=compile_kwd,
- template_kind=template_kind,
- extra={"from_entity_kwd": seed_list},
- )
- bwd = await _search_compiled_rows(
- tools,
- kb_id,
- tenant_id,
- doc_ids,
- "relation",
- top_n=50,
- compile_kwd=compile_kwd,
- template_kind=template_kind,
- extra={"to_entity_kwd": seed_list},
- )
- all_rels = {**fwd, **bwd}
-
- # -- 3. Neighbour names (1-hop, exclude seeds) --
- seed_lower = {n.lower() for n in seed_names}
- neighbour_names: set[str] = set()
- for r in all_rels.values():
- frm = (r.get("from_entity_kwd") or "").strip()
- frm_lower = frm.lower()
- to = (r.get("to_entity_kwd") or "").strip()
- to_lower = to.lower()
- if frm_lower in seed_lower and to and to_lower not in seed_lower:
- neighbour_names.add(to)
- if to_lower in seed_lower and frm and frm_lower not in seed_lower:
- neighbour_names.add(frm)
- if not neighbour_names:
- return []
-
- # -- 4. Neighbour entity source_chunk_ids --
- # Provide both original and lowercased — same as seed_list above.
- neigh_list = sorted({n.lower() for n in neighbour_names} | neighbour_names)
- if len(neigh_list) > 100:
- neigh_list = neigh_list[:100] # reasonable cap for name_kwd search
- neigh_rows = await _search_compiled_rows(
- tools,
- kb_id,
- tenant_id,
- doc_ids,
- "entity",
- top_n=len(neigh_list),
- compile_kwd=compile_kwd,
- template_kind=template_kind,
- extra={"name_kwd": neigh_list},
- )
-
- # Group chunk IDs by doc
- by_doc: dict[str, set[str]] = {}
- for r in neigh_rows.values():
- doc_id = r.get("doc_id") or ""
- for cid in r.get("source_chunk_ids") or []:
- if cid and cid not in seen_ids:
- by_doc.setdefault(doc_id, set()).add(cid)
-
- # -- 5. Load and return --
- new_chunks: list[dict] = []
- for doc_id, cids in by_doc.items():
- if len(new_chunks) >= max_chunks:
- break
- limit = max_chunks - len(new_chunks)
- chunks = await _load_chunks_for_doc(tools, doc_id, list(cids)[:limit])
- for c in chunks:
- cid = c.get("chunk_id") or c.get("id")
- if cid and cid not in seen_ids:
- seen_ids.add(cid)
- new_chunks.append(c)
-
- return new_chunks
-
-
-async def _search_synthesis_pages(
- tools,
- kb_id: str,
- tenant_id: str,
- doc_ids: list[str] | None,
- text: str,
- *,
- compile_kwd: str = "wiki_page",
- top_n: int = 8,
-) -> dict:
- """Search synthesis-compiled page rows (no knowledge_graph_kwd filter).
-
- Synthesis pages are standalone articles (wiki_page, artifact_page,
- essence, etc.) with ``content_with_weight``, keyword index, and
- vector. They do NOT carry the ``knowledge_graph_kwd`` field (unlike
- entity/relation rows from extraction).
- """
- from common import settings
- from common.doc_store.doc_store_base import MatchTextExpr, OrderByExpr
- from common.misc_utils import thread_pool_exec
- from rag.nlp import search
-
- condition: dict = {"compile_kwd": compile_kwd, "available_int": 1}
- if doc_ids:
- condition["source_doc_ids"] = list(doc_ids)
-
- fields = [
- "content_with_weight",
- "summary_with_weight",
- "source_chunk_ids",
- "doc_id",
- "title_kwd",
- "topic_kwd",
- ]
-
- exprs = []
- if text:
- embd_mdl = getattr(tools, "embed_mdl", None)
- if embd_mdl:
- try:
- exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_n, 0.1))
- except Exception:
- _LOG.exception("[Wiki expand] vector build failed; using keyword match")
- if not exprs:
- exprs.append(
- MatchTextExpr(
- ["content_ltks", "content_sm_ltks"],
- text,
- top_n,
- )
- )
-
- try:
- res = await thread_pool_exec(
- settings.docStoreConn.search,
- fields,
- [],
- condition,
- exprs,
- OrderByExpr(),
- 0,
- top_n,
- search.index_name(tenant_id),
- [kb_id],
- )
- return settings.docStoreConn.get_fields(res, fields) or {}
- except Exception:
- _LOG.exception("[Wiki expand] search failed for kb=%s", kb_id)
- return {}
-
-
-async def _expand_wiki_page_strategy(
- tools,
- kb_id: str,
- tenant_id: str,
- doc_ids: list[str] | None,
- query: str,
- seen_ids: set[str],
- *,
- compile_kwd: str = "wiki_page",
- max_chunks: int = 5,
-) -> list[dict]:
- """Expand synthesis-compiled pages: semantic search → load source chunks.
-
- Unlike ``_expand_compiled_strategy`` (which does 1-hop entity-graph
- navigation), synthesis pages are standalone rendered articles — we search
- them directly and load the referenced source chunks as context.
-
- *compile_kwd* selects which synthesis type: "wiki_page" (Wiki template),
- "artifact_page" (Session Graph synthesis), "essence" (Session Essence).
- """
- # -- 1. Search synthesis pages --
- wiki_rows = await _search_synthesis_pages(
- tools,
- kb_id,
- tenant_id,
- doc_ids,
- query,
- compile_kwd=compile_kwd,
- top_n=5,
- )
- if not wiki_rows:
- return []
-
- # -- 2. Collect source_chunk_ids from matching pages --
- by_doc: dict[str, set[str]] = {}
- for r in wiki_rows.values():
- doc_id = r.get("doc_id") or ""
- for cid in r.get("source_chunk_ids") or []:
- if cid and cid not in seen_ids:
- by_doc.setdefault(doc_id, set()).add(cid)
-
- # -- 3. Load chunks, assign high similarity for priority ranking --
- new_chunks: list[dict] = []
- for doc_id, cids in by_doc.items():
- if len(new_chunks) >= max_chunks:
- break
- limit = max_chunks - len(new_chunks)
- chunks = await _load_chunks_for_doc(tools, doc_id, list(cids)[:limit])
- for c in chunks:
- cid = c.get("chunk_id") or c.get("id")
- if cid and cid not in seen_ids:
- seen_ids.add(cid)
- c.setdefault("similarity", 0.9) # wiki pages rank high
- new_chunks.append(c)
-
- return new_chunks
+# Compiled-product expansion lives in ``compiled_expansion`` and is
+# re-exported here: navigation and the dynamic runner import these names
+# from this module.
+from rag.advanced_rag.harness.tools.compiled_expansion import ( # noqa: F401
+ _expand_compiled_strategy,
+ _expand_wiki_page_strategy,
+ _expand_with_compiled,
+ _load_chunks_for_doc,
+ _search_compiled_rows,
+ _search_synthesis_pages,
+)
async def web_search(tools, query: str, keywords: str = "", retrieval_query: str = "") -> dict:
@@ -1245,549 +381,6 @@ _NAV_TREE_MAX_DATASETS = 10
# Chars of a chunk's text returned by search_chunks in default (snippet) mode.
_SEARCH_SNIPPET_CHARS = 300
-# ── XML helpers (uniform tag vocabulary shared by all retrieval tools) ──
-
-
-def _xml_escape(value: Any) -> str:
- s = "" if value is None else str(value)
- return s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
-
-
-def _chunk_text(c: dict) -> str:
- return str(c.get("content_with_weight") or c.get("content") or c.get("text") or "")
-
-
-def _chunk_attr(c: dict, keys: tuple[str, ...]) -> str:
- for k in keys:
- v = c.get(k)
- if v not in (None, ""):
- return str(v)
- return ""
-
-
-def _doc_id(c: dict) -> str:
- return _chunk_attr(c, ("doc_id", "docid", "document_id"))
-
-
-def _dataset_id(c: dict) -> str:
- return _chunk_attr(c, ("dataset_id", "kb_id", "knowledgebase_id"))
-
-
-def _doc_title(c: dict) -> str:
- return _chunk_attr(c, ("docnm_kwd", "doc_title", "title", "document_name"))
-
-
-def _chunk_id(c: dict) -> str:
- return _chunk_attr(c, ("chunk_id", "id"))
-
-
-def _snippet(s: str, n: int) -> str:
- """Truncate a string to ``n`` chars on a char boundary with an ellipsis."""
- s = (s or "").strip()
- if len(s) <= n:
- return s
- return s[:n].rstrip() + "..."
-
-
-# ── Tool: think ──
-
-
-@tool
-async def _think_impl(
- thought: str,
- next_thought_needed: bool = True,
- thought_number: int | None = None,
- total_thoughts: int | None = None,
- is_revision: bool = False,
- revises_thought: int | None = None,
- branch_from_thought: int | None = None,
- branch_id: str = "",
- needs_more_thoughts: bool = False,
-) -> str:
- """A stateless reflective-thinking tool that records a reasoning step.
-
- Use it to plan, reflect on retrieved content, and reason step by step
- before answering. It does not persist history — keep your plan in the
- conversation context. When your thinking is complete, deliver your answer
- by writing it as your plain reply and stopping (no further tool calls).
- NEVER include the final answer directly in a thought.
-
- IMPORTANT: The ``thought`` field must be plain natural-language reasoning
- ONLY. Do NOT embed XML tags, ``<invoke ...>``, tool names, JSON, or any
- pseudo tool-call syntax inside it. To call another tool (e.g. search_chunks),
- emit a separate tool call for that tool instead of describing it in text
- inside ``thought``.
-
- All arguments are optional for robustness: if the model omits the counters,
- the tool records the thought and, by default, signals that thinking continues
- (so the model keeps retrieving rather than stopping early).
-
- :param thought: Your current thinking step, in natural user-friendly
- language. Focus on WHAT you're trying to find and WHY, not HOW.
- :param next_thought_needed: True if you need more thinking. Defaults to True
- (conservative: keep thinking) when omitted.
- :param thought_number: Current thought number (>= 1). Optional.
- :param total_thoughts: Current estimate of thoughts needed (>= 1). Optional.
- :param is_revision: Whether this thought revises previous thinking.
- :param revises_thought: If a revision, which thought number is reconsidered.
- :param branch_from_thought: If branching, which thought number is the branch point.
- :param branch_id: Identifier for the current branch (if any).
- :param needs_more_thoughts: If reaching the end but realising more thoughts are needed.
- :return: A completion marker; does not affect the conversation content.
- """
- if not thought or not str(thought).strip():
- return "Thought process invalid: thought must be a non-empty string"
- if thought_number is not None and thought_number < 1:
- return "Thought process invalid: thought_number must be >= 1"
- if total_thoughts is not None and total_thoughts < 1:
- return "Thought process invalid: total_thoughts must be >= 1"
- # Conservative by default: without explicit counters we prefer to keep going
- # rather than stop early (avoids partial answers on multi-hop questions).
- if thought_number is None or total_thoughts is None:
- incomplete = bool(next_thought_needed or needs_more_thoughts)
- else:
- incomplete = bool(next_thought_needed or needs_more_thoughts or thought_number < total_thoughts)
- if incomplete:
- return "Thought process recorded - unfinished steps remain, continue exploring and calling tools"
- return "Thought process recorded"
-
-
-# ── Tool: todo_write ──
-
-
-@tool
-async def _todo_write_impl(task: str = "", steps: list | None = None) -> str:
- """Create and manage a structured task list for retrieval and research tasks.
-
- Use it to track progress across 3+ retrieval/research steps. It is
- stateless: the model maintains the plan within the conversation context.
-
- CRITICAL - Focus on Retrieval Tasks Only: track RETRIEVAL and RESEARCH
- tasks (searching datasets, retrieving documents, gathering information).
- Do NOT include summary or synthesis tasks here — those are for the think
- tool. Examples: "Search for X in dataset", "Retrieve information about Y".
- Exclude: "Summarize findings", "Generate final answer".
-
- :param task: Short task title.
- :param steps: List of {"id","description","status"} where status is one of
- "pending" / "in_progress" / "completed". Limit to ONE in_progress at a
- time.
- :return: The current task list echoed back.
- """
- entries = steps if isinstance(steps, list) else []
- rendered = [str(task).strip()] if (task or "").strip() else []
- for i, s in enumerate(entries):
- if not isinstance(s, dict):
- continue
- sid = str(s.get("id") or i + 1)
- desc = str(s.get("description") or "").strip()
- status = str(s.get("status") or "pending")
- rendered.append(f" [{status}] #{sid}: {desc}")
- if not rendered:
- return "No tasks recorded."
- return "\n".join(rendered)
-
-
-# ── Tool: grep_chunks ──
-
-
-@tool(timeout=120)
-async def _grep_chunks_impl(
- query: str,
- chunk_ids: list[str] | None = None,
- dataset_ids: list[str] | None = None,
- doc_scope: list[str] | None = None,
- tools=None,
-) -> str:
- """Search dataset chunk content with a single POSIX regular expression,
- case-insensitive (behaves like grep -E -i).
-
- Pack multiple concepts into ONE regex using | alternation — do not call
- this tool repeatedly for synonyms. Keep the regex BROAD: use bare keywords
- and names combined with |, and do NOT anchor it to a specific subject/verb
- chain (e.g. prefer "何进|董太后|鸩杀|斩|诛" over "何进.*斩").
-
- When navigate_structure returned an outline with ``[chunks: c100,c101]``
- pointers, pass those chunk_ids here to grep ONLY within those chunks (precise
- locate inside a document's compiled-structure spans) instead of the whole
- dataset. doc_scope must then carry the owning document id(s).
-
- Returns an XML <grep_results> document. Each matching chunk is a <chunk>
- element with chunk_id, doc_id (owning document id), doc_title, dataset_id,
- page_num, chunk_index and score attributes, and a <match_snippet> element —
- a SHORT window (~80 chars on each side of the first match), NOT the full
- chunk text. The snippet is for fast relevance judgement only. To read a
- located document's complete text, call list_chunks with the returned doc_id.
-
- :param query: REQUIRED: a single POSIX regex applied to chunk content
- (case-insensitive). Combine multiple concepts with | in ONE regex.
- :param chunk_ids: Optional list of chunk ids to restrict the grep to (from a
- navigate_structure outline's ``[chunks: ...]`` pointers). When given,
- grep runs only inside those chunks of doc_scope's documents.
- :param dataset_ids: Optional dataset ids to restrict the search to (at most
- 10). When omitted, the conversation's bound datasets are used.
- :param doc_scope: Optional document ids to restrict the search to (at most 10).
- :return: XML <grep_results> document.
- """
- if not query or not str(query).strip():
- return '<grep_results count="0">\n</grep_results>'
- q = str(query).strip()
- # Validate the regex up front so a malformed pattern is reported, not silently skipped.
- try:
- re.compile("(?i)" + q)
- except re.error as e:
- return f'<grep_results count="0" error="invalid regex: {_xml_escape(str(e))}">\n</grep_results>'
-
- from common import settings
-
- if not getattr(settings, "retriever", None):
- return '<grep_results count="0" error="no retriever">\n</grep_results>'
-
- _t = tools or _tools_slot()
- target_ids = dataset_ids or list(dict.fromkeys(_get_kb_ids(_t)))
- if not target_ids:
- return '<grep_results count="0" error="no bound datasets">\n</grep_results>'
-
- # Precise path: grep ONLY within the chunk_ids flagged by navigate_structure's
- # outline pointers (inside doc_scope's documents). No full-dataset retrieval.
- if chunk_ids:
- candidates = await _load_specific_chunks(_t, chunk_ids, doc_scope)
- if not candidates:
- return '<grep_results count="0" error="none of the given chunk_ids found">\n</grep_results>'
- else:
- # Candidate pool: keyword-first retrieval (BM25) gives the widest recall
- # for exact identifiers / names / error codes, then narrow_by_terms does
- # the actual regex locate + context-window extraction (zero extra LLM).
- candidates = []
- try:
- res = await bm25_search(_t, q, kb_ids=target_ids, top_n=60, doc_scope=doc_scope)
- candidates = res.get("chunks", []) or []
- except Exception:
- _LOG.exception("[grep_chunks] bm25 candidate retrieval failed; falling back to hybrid")
- try:
- res = await hybrid_search(_t, q, kb_ids=target_ids, top_n=60, doc_scope=doc_scope)
- candidates = res.get("chunks", []) or []
- except Exception:
- _LOG.exception("[grep_chunks] hybrid fallback failed")
- return '<grep_results count="0" error="retrieval failed">\n</grep_results>'
-
- terms = _query_to_terms(q)
- regex = re.compile("(?i)" + q)
- if chunk_ids:
- # Strict precise grep: only chunks whose text actually matches the regex
- # are reported. (narrow_by_terms may otherwise keep non-matching context
- # chunks, which is wrong when the scope is exactly the flagged chunks.)
- matched = [c for c in candidates if regex.search(_chunk_text(c))]
- else:
- from rag.advanced_rag.harness.grep_sed_narrow import narrow_by_terms
-
- narrowed = narrow_by_terms(
- candidates,
- terms,
- fallback_terms=None,
- context={"before": 0, "after": 1},
- keywords=q,
- max_out_chars_per_chunk=1200,
- max_out_total_chars=16000,
- )
- matched = narrowed.get("kept", candidates)
-
- parts = [f'<grep_results count="{len(matched)}" query="{_xml_escape(q)}">']
- for i, c in enumerate(matched):
- cid = _chunk_id(c)
- did = _doc_id(c)
- dsid = _dataset_id(c)
- title = _doc_title(c)
- score = c.get("similarity", 0.0) or c.get("score", 0.0) or 0.0
- parts.append(f' <chunk rank="{i + 1}" chunk_id="{_xml_escape(cid)}" doc_id="{_xml_escape(did)}" dataset_id="{_xml_escape(dsid)}" doc_title="{_xml_escape(title)}" score="{float(score):.3f}">')
- snippet = _chunk_text(c)
- if len(snippet) > 240:
- snippet = snippet[:240]
- if snippet:
- parts.append(f" <match_snippet>{_xml_escape(snippet)}</match_snippet>")
- parts.append(" </chunk>")
- parts.append("</grep_results>")
- return "\n".join(parts)
-
-
-# ── Tool: search_chunks ──
-
-
-@tool(timeout=120)
-async def _search_chunks_impl(
- queries: list[str],
- chunk_ids: list[str] | None = None,
- dataset_ids: list[str] | None = None,
- doc_scope: list[str] | None = None,
- top_n: int = 8,
- full_content: bool = False,
- similarity_threshold: float = 0.2,
- keywords_similarity_weight: float = 0.3,
- tools=None,
-) -> str:
- """Semantic/vector search tool for retrieving knowledge by meaning, intent,
- and conceptual relevance.
-
- This tool uses embeddings to understand the query and find semantically
- similar content across dataset chunks. It searches by MEANING rather than
- exact text. Do NOT use it for exact keyword matching (use grep_chunks for
- that); do NOT pass long raw text or user messages as queries.
-
- When navigate_structure returned an outline with ``[chunks: c100,c101]``
- pointers, pass those chunk_ids here to search ONLY within those chunks
- (precise locate inside a document's compiled-structure spans). doc_scope must
- then carry the owning document id(s).
-
- By default returns a COMPACT snippet per hit (token-cheap) so you can judge
- relevance. When you need the full original chunk text, read it precisely with
- list_chunks (pass the doc_id + chunk_id), or set ``full_content=True`` only if
- you must have the full text in one call.
-
- Returns an XML <search_results> document. Each hit is a <chunk> element with
- attributes rank, chunk_id, doc_id, page_num, chunk_index, dataset_id,
- doc_title and score, and a <content> element carrying either a short snippet
- (default) or the FULL chunk text (full_content=True).
-
- :param queries: REQUIRED: 1-5 short, well-formed semantic questions or
- conceptual statements.
- :param chunk_ids: Optional list of chunk ids to restrict the search to (from
- a navigate_structure outline's ``[chunks: ...]`` pointers).
- :param dataset_ids: Optional dataset ids to restrict the search.
- :param doc_scope: Optional document ids to restrict the search.
- :param top_n: Per-query result count, default 8 (kept small to avoid flooding
- the history with large chunks).
- :param full_content: When True, return the FULL chunk text; default False
- returns a compact snippet per hit.
- :param similarity_threshold: Similarity threshold, default 0.2.
- :param keywords_similarity_weight: Keyword-vs-vector weight, default 0.3.
- :return: XML <search_results> document.
- """
- from common import settings
-
- if not getattr(settings, "retriever", None):
- return '<search_results count="0" error="no retriever">\n</search_results>'
-
- if not isinstance(queries, list):
- queries = [queries]
- queries = [str(q).strip() for q in queries if str(q).strip()]
- if not queries or len(queries) > 5:
- queries = queries[:5]
- if not queries:
- return '<search_results count="0" error="no queries">\n</search_results>'
-
- tools_slot = tools or _tools_slot()
- target_ids = dataset_ids or list(dict.fromkeys(_get_kb_ids(tools_slot)))
- if not target_ids:
- return '<search_results count="0" error="no bound datasets">\n</search_results>'
-
- # Precise path: search ONLY within the chunk_ids flagged by navigate_structure
- # outline pointers (inside doc_scope's documents). Rank by keyword overlap
- # (zero LLM) — the scope is small, so full hybrid search is unnecessary.
- if chunk_ids:
- candidates = await _load_specific_chunks(tools_slot, chunk_ids, doc_scope)
- if not candidates:
- return '<search_results count="0" error="none of the given chunk_ids found">\n</search_results>'
- ranked = _rank_chunks_by_terms(candidates, queries)
- merged = ranked[: max(1, int(top_n or 12)) * 2]
- else:
- seen: set[str] = set()
- merged: list[dict] = []
- for q in queries:
- try:
- res = await hybrid_search(
- tools_slot,
- q,
- kb_ids=target_ids,
- top_n=max(1, int(top_n or 12)),
- doc_scope=doc_scope,
- )
- except Exception:
- _LOG.exception("[search_chunks] hybrid_search failed for %r", q)
- continue
- for c in res.get("chunks", []) or []:
- k = _chunk_id(c)
- if k and k in seen:
- continue
- if k:
- seen.add(k)
- merged.append(c)
-
- # After the semantic hits, enrich with RELATED chunks discovered via each hit
- # document's COMPILED structure (zero chat-LLM, vector beam drill-down). For the
- # docs that surfaced, follow the tree/catalog hierarchy to OTHER chunks related
- # to the same query and append their snippets, so a search that lands on one
- # spot in a document also surfaces the surrounding sub-sections that answer it.
- # gated to high/ultra (the same modes that bind navigate_tree/navigate_structure)
- # so medium keeps its plain hybrid behavior.
- mode = str(getattr(tools_slot, "thinking_mode", "") or "").strip().lower()
- if merged and mode in _COMPILED_TOOL_MODES:
- hit_docs = [d for d in dict.fromkeys(_doc_id(c) for c in merged) if d]
- if hit_docs:
- for q in queries:
- related = await _expand_related_via_structure(tools_slot, q, hit_docs, seen)
- merged.extend(related)
-
- # Cap total chunks returned to keep the tool token-cheap (default snippet mode).
- merged = merged[: max(1, int(top_n or 8))]
-
- parts = [f'<search_results count="{len(merged)}">']
- for i, c in enumerate(merged):
- cid = _chunk_id(c)
- did = _doc_id(c)
- dsid = _dataset_id(c)
- title = _doc_title(c)
- page = c.get("page_num", 0) or c.get("page", 0) or 0
- cidx = c.get("chunk_index", 0) or 0
- score = c.get("similarity", 0.0) or c.get("score", 0.0) or 0.0
- related = ' related="true"' if c.get("related_via_structure") else ""
- parts.append(
- f' <chunk rank="{i + 1}" chunk_id="{_xml_escape(cid)}" doc_id="{_xml_escape(did)}" '
- f'page_num="{page}" chunk_index="{cidx}" dataset_id="{_xml_escape(dsid)}" '
- f'doc_title="{_xml_escape(title)}" score="{float(score):.3f}"{related}>'
- )
- content = _chunk_text(c)
- if content:
- if full_content:
- parts.append(f" <content>{_xml_escape(content)}</content>")
- else:
- parts.append(f" <content>{_xml_escape(_snippet(content, _SEARCH_SNIPPET_CHARS))}</content>")
- parts.append(" </chunk>")
- parts.append("</search_results>")
- return "\n".join(parts)
-
-
-# ── Tool: list_chunks ──
-
-
-@tool(timeout=120)
-async def _list_chunks_impl(doc_id: str, chunk_ids: list[str] | None = None, limit: int = 20, offset: int = 0, tools=None) -> str:
- """Read the FULL original text of ONE dataset document in reading order
- (Deep Read).
-
- Use this AFTER grep_chunks / search_chunks / navigate_structure locate a
- document: pass its doc_id to read the complete chunk text — including
- surrounding context that grep snippets omit. When navigate_structure returned
- an outline with ``[chunks: c100,c101]`` pointers, pass those chunk_ids here to
- read ONLY those chunks (fast, precise deep-read) instead of the whole document.
-
- Returns an XML <chunks> document with doc_id, offset, limit and fetched
- attributes. Each chunk is a <chunk> element carrying chunk_id, doc_id,
- page_num, chunk_index, dataset_id, doc_title and a <content> element with
- the FULL original chunk text, ordered by reading order. A
- <pagination next_offset=.../> element signals that more pages remain.
-
- :param doc_id: REQUIRED: the document id to read (from grep_chunks /
- search_chunks / navigate_structure output).
- :param chunk_ids: Optional list of specific chunk ids to read (from a
- navigate_structure outline's ``[chunks: ...]`` pointers). When given, only
- those chunks are returned and limit/offset are ignored.
- :param limit: Page size, default 20, max 100.
- :param offset: Page offset, default 0.
- :return: XML <chunks> document.
- """
- from common import settings
-
- if not getattr(settings, "retriever", None):
- return f'<chunks doc_id="{_xml_escape(doc_id)}" error="no retriever">\n</chunks>'
- doc_id = str(doc_id or "").strip()
- if not doc_id:
- return '<chunks doc_id="" error="doc_id is required">\n</chunks>'
- limit = max(1, min(int(limit or 20), 100))
- offset = max(0, int(offset or 0))
-
- tools_slot = tools or _tools_slot()
- try:
- full = await tools_slot.fetch_full_document(doc_id)
- except Exception:
- _LOG.exception("[list_chunks] fetch_full_document failed for doc_id=%s", doc_id)
- return f'<chunks doc_id="{_xml_escape(doc_id)}" offset="{offset}" limit="{limit}" fetched="0">\n</chunks>'
-
- chunks = full.get("chunks", []) or []
-
- # Precise deep-read: only the requested chunk ids (from navigate_structure
- # outline pointers). Ignores limit/offset.
- if chunk_ids:
- wanted = [str(c).strip() for c in chunk_ids if str(c).strip()]
- by_id = {(_chunk_id(c) or ""): c for c in chunks}
- page = [by_id[cid] for cid in wanted if cid in by_id]
- if not page:
- return f'<chunks doc_id="{_xml_escape(doc_id)}" chunk_ids="{_xml_escape(",".join(wanted))}" fetched="0">\n</chunks>'
- parts = [f'<chunks doc_id="{_xml_escape(doc_id)}" chunk_ids="{_xml_escape(",".join(wanted))}" fetched="{len(page)}">']
- for c in page:
- parts.append(_render_chunk_block(c, doc_id))
- parts.append("</chunks>")
- return "\n".join(parts)
-
- page = chunks[offset : offset + limit]
-
- parts = [f'<chunks doc_id="{_xml_escape(doc_id)}" offset="{offset}" limit="{limit}" fetched="{len(page)}">']
- for c in page:
- parts.append(_render_chunk_block(c, doc_id))
- if len(page) == limit and (offset + limit) < len(chunks):
- parts.append(f' <pagination next_offset="{offset + limit}" remaining="true" />')
- parts.append("</chunks>")
- return "\n".join(parts)
-
-
-# ── Tool: calculate (safe numeric arithmetic over retrieved facts) ──
-
-
-@tool(timeout=30)
-async def _calculate_impl(expression: str) -> str:
- """Safely evaluate ONE Python arithmetic expression to compute a number the
- retrieved facts imply but no source states outright (an age, an age
- difference, a percentage, a difference of speeds after unit conversion, a
- rate times a count).
-
- Use it for numeric answers. When the answer needs arithmetic (e.g. "how
- much younger", "what percentage", "difference in m/s", "total dollars over
- a fast"), write ONE Python expression with every figure as a literal pulled
- EXACTLY from the retrieved facts, and let this tool compute it precisely —
- do NOT do the arithmetic by hand in prose (LLMs get it wrong often enough
- to matter). The expression is evaluated in a sandbox: only plain arithmetic
- on numbers is allowed, plus the functions abs, round, min, max, sum, len,
- int, float, sorted, letters, digit_sum and date_diff. No variables, imports,
- attributes, subscripts, or non-numeric operations.
-
- Examples:
- - age difference -> 2010 - 1971
- - percentage -> 100 * 2.7 / 556
- - speed difference in m/s (km/h ÷ 3.6) -> 132 / 3.6 - 50 / 21.07
- - rate × count -> 25 * 49
- - days between two dates -> date_diff("1941-07-28", "1959-07-17")
- - letters in names -> letters("Ada Lovelace", "Alan Turing")
-
- :param expression: A single Python arithmetic expression (numbers + operators
- + the whitelisted functions above). Every figure must come from the
- retrieved evidence — never invent or round inputs.
- :return: XML <calculation> document with the exact computed value, or an
- <error> describing why the expression was refused.
- """
- from rag.advanced_rag.harness.arithmetic import compute
-
- expr = str(expression or "").strip()
- if not expr:
- return '<calculation error="empty expression">\n</calculation>'
- value, err = compute(expr)
- if err:
- return f'<calculation error="{_xml_escape(err)}">\n</calculation>'
- return f'<calculation value="{_xml_escape(value)}" expression="{_xml_escape(expr)}">\n</calculation>'
-
-
-def _render_chunk_block(c: dict, doc_id: str) -> str:
- """Render one chunk as an XML <chunk> block with full <content>."""
- cid = _chunk_id(c)
- dsid = _dataset_id(c)
- title = _doc_title(c)
- page_num = c.get("page_num", 0) or c.get("page", 0) or 0
- cidx = c.get("chunk_index", 0) or 0
- lines = [f' <chunk chunk_id="{_xml_escape(cid)}" doc_id="{_xml_escape(doc_id)}" page_num="{page_num}" chunk_index="{cidx}" dataset_id="{_xml_escape(dsid)}" doc_title="{_xml_escape(title)}">']
- content = _chunk_text(c)
- if content:
- lines.append(f" <content>{_xml_escape(content)}</content>")
- lines.append(" </chunk>")
- return "\n".join(lines)
-
async def _load_specific_chunks(tools_slot, chunk_ids: list[str], doc_scope: list[str] | None = None) -> list[dict]:
"""Load specific chunks (from navigate_structure outline pointers) by id.
diff --git a/rag/advanced_rag/harness/tools/text_processing.py b/rag/advanced_rag/harness/tools/text_processing.py
new file mode 100644
index 0000000000..cf0e97d12e
--- /dev/null
+++ b/rag/advanced_rag/harness/tools/text_processing.py
@@ -0,0 +1,460 @@
+"""Keyword-driven text processing shared by the retrieval tools.
+
+Sentence splitting, light stemming, and the keyword narrowing/highlighting that
+keeps chunk payloads token-cheap: retrieval returns full chunks, and narrowing
+cuts each one down to the sentences that actually carry the query terms.
+
+Lives in its own module because it is pure text work — no retrieval, no store
+access — and is reused well beyond ``search`` (grep/sed narrowing, memory,
+navigation).
+"""
+
+import hashlib
+import logging
+import re
+from functools import lru_cache
+
+_LOG = logging.getLogger(__name__)
+
+
+def _compact_keywords(kw: str, max_terms: int = 15) -> str:
+ """Deduplicate and cap a formalize / extract_keywords keyword string.
+
+ The extraction prompt asks the model for 3-10 terms *plus* 2-3 synonyms each,
+ which models answer with a 40-60 word redundant synonym run (e.g. "average
+ distance left field line MLB retractable roof stadiums 2024 ... retractable
+ dome covered stadium mean distance outfield" — ~350 chars). Appending that
+ whole run onto the query diluted the vector leg and dragged BM25 onto
+ unrelated docs. This keeps the recall terms but drops the redundancy:
+ dedupe (preserving order) and cap at ``max_terms`` so it stays a compact
+ hint instead of a pollution source. Accepts both space- and comma-separated
+ input (single-turn extract_keywords emits spaces; multi-turn formalize
+ emits commas).
+ """
+ if not kw:
+ return ""
+ tokens = re.split(r"[,\s]+", (kw or "").strip())
+ seen: list[str] = []
+ for t in tokens:
+ t = t.strip()
+ if not t:
+ continue
+ if t not in seen:
+ seen.append(t)
+ if len(seen) >= max_terms:
+ break
+ return " ".join(seen)
+
+
+# Sentence terminators: Chinese 。!?;, English ! ? ;, newline, and a
+# digit-guarded English period (so "3.14" / "v1.2" don't split).
+_SENT_END = re.compile(r"[。!?;!?;]+|(?<!\d)\.(?!\d)")
+
+# Block-level HTML elements and markdown tables are kept ATOMIC — never split by
+# sentence terminators — so a whole table / list / block counts as ONE "sentence"
+# for keyword matching and narrowing (a keyword inside one keeps the whole block).
+_HTML_TAG = re.compile(r"<(/?)([a-zA-Z][a-zA-Z0-9]*)\b([^>]*)>")
+
+# Only BLOCK-level containers are protected. Inline tags (<b>, <i>, <a>, <span>,
+# <em>, <strong>, <code>, ...) are deliberately excluded so ordinary prose that
+# contains inline formatting still splits into sentences normally.
+_HTML_BLOCK_TAGS = {
+ "table",
+ "thead",
+ "tbody",
+ "tfoot",
+ "tr",
+ "td",
+ "th",
+ "caption",
+ "colgroup",
+ "ul",
+ "ol",
+ "li",
+ "dl",
+ "dt",
+ "dd",
+ "div",
+ "p",
+ "pre",
+ "blockquote",
+ "section",
+ "article",
+ "aside",
+ "nav",
+ "main",
+ "figure",
+ "figcaption",
+ "header",
+ "footer",
+ "address",
+ "details",
+ "summary",
+ "form",
+ "fieldset",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+}
+
+# Markdown table: a header row with a pipe, a separator row of dashes/colons/
+# pipes, then zero+ body rows with a pipe.
+_MD_TABLE = re.compile(
+ r"^[ \t]*\|?[^\n]*\|[^\n]*\r?\n"
+ r"[ \t]*\|?[ \t]*:?-{1,}:?[ \t]*(?:\|[ \t]*:?-{1,}:?[ \t]*)+\|?[ \t]*\r?\n"
+ r"(?:[ \t]*\|?[^\n]*\|[^\n]*\r?\n?)*",
+ re.MULTILINE,
+)
+
+
+def _html_block_spans(text: str) -> list[tuple[int, int]]:
+ """Outermost balanced block-level HTML element spans (nesting-aware).
+
+ Uses a tag stack (not a regex) so nested elements (e.g. a ``<table>`` with
+ ``<td>``s, or nested ``<div>``s) yield ONE span for the outermost element and
+ are never truncated at the first close tag the way a non-greedy regex would.
+ Unclosed / stray tags are ignored (that region just falls back to plain
+ sentence splitting).
+ """
+ spans: list[tuple[int, int]] = []
+ stack: list[tuple[str, int]] = []
+ for m in _HTML_TAG.finditer(text):
+ name = m.group(2).lower()
+ if name not in _HTML_BLOCK_TAGS:
+ continue
+ if m.group(1): # closing tag </name>
+ for i in range(len(stack) - 1, -1, -1):
+ if stack[i][0] == name:
+ start = stack[i][1]
+ del stack[i:]
+ if not stack: # closed an outermost block
+ spans.append((start, m.end()))
+ break
+ # a stray </name> with no matching open is ignored
+ elif not m.group(3).rstrip().endswith("/"): # opening (skip self-closing)
+ stack.append((name, m.start()))
+ return spans
+
+
+def _protected_spans(text: str) -> list[tuple[int, int]]:
+ """Non-overlapping ``(start, end)`` spans kept atomic, in order.
+
+ Covers block-level HTML elements and markdown tables; overlapping spans are
+ merged (unioned) so a match that straddles two is never split.
+ """
+ spans = _html_block_spans(text)
+ spans += [(m.start(), m.end()) for m in _MD_TABLE.finditer(text)]
+ spans.sort()
+ merged: list[tuple[int, int]] = []
+ last_end = -1
+ for s, e in spans:
+ if s < last_end: # overlaps an already-kept span -> union it in
+ if e > last_end:
+ merged[-1] = (merged[-1][0], e)
+ last_end = e
+ continue
+ merged.append((s, e))
+ last_end = e
+ return merged
+
+
+def _split_plain(text: str) -> list[str]:
+ """Terminator-based sentence split, keeping each terminator attached."""
+ sents: list[str] = []
+ start = 0
+ for m in _SENT_END.finditer(text):
+ end = m.end()
+ seg = text[start:end]
+ if seg.strip():
+ sents.append(seg)
+ start = end
+ if start < len(text):
+ tail = text[start:]
+ if tail.strip():
+ sents.append(tail)
+ return sents
+
+
+def _split_sentences(text: str) -> list[str]:
+ """Split ``text`` into sentences, keeping each terminator attached.
+
+ Block-level HTML elements (``<table>``, ``<div>``, ``<p>``, ``<ul>``, ... —
+ see :data:`_HTML_BLOCK_TAGS`) and markdown tables are treated as a single
+ atomic sentence and are never split internally, so a keyword falling inside
+ one keeps the whole block together.
+ """
+ if not text:
+ return []
+ spans = _protected_spans(text)
+ if not spans:
+ return _split_plain(text)
+
+ sents: list[str] = []
+ pos = 0
+ for s, e in spans:
+ if s > pos:
+ sents.extend(_split_plain(text[pos:s]))
+ block = text[s:e]
+ if block.strip():
+ sents.append(block)
+ pos = e
+ if pos < len(text):
+ sents.extend(_split_plain(text[pos:]))
+ return sents
+
+
+# ---------------------------------------------------------------------------
+# Stem-tolerant keyword matching (ported from agentic_search4 v8)
+#
+# Substring matching misses inflected forms: "nominations" misses "nominated",
+# "company" misses "companies". Keywords are derived from the question, which
+# states things in the inflected form ("which band HEADLINED", "was NOMINATED
+# three times"), so the failing direction is the common one. Both sides are
+# therefore reduced to a stem before comparison.
+# ---------------------------------------------------------------------------
+try: # available at runtime — nltk already backs rag/nlp/synonym.py
+ from nltk.stem import PorterStemmer as _PorterStemmer
+
+ _porter_stem = _PorterStemmer().stem
+except Exception: # pragma: no cover - exercised only where nltk is absent
+ _porter_stem = None
+
+# Longest first: "nominations" must lose "ations", not just the trailing "s".
+_STEM_SUFFIXES = (
+ ("ations", ""),
+ ("ation", ""),
+ ("ated", ""),
+ ("ates", ""),
+ ("ate", ""),
+ ("ings", ""),
+ ("ing", ""),
+ ("ies", "i"),
+ ("ied", "i"),
+ ("ed", ""),
+ ("es", ""),
+ ("s", ""),
+)
+
+_WORD_RE = re.compile(r"[a-z0-9]+")
+
+
+def _fallback_stem(word: str) -> str:
+ """Suffix stripper used when nltk is unavailable. Approximates Porter."""
+ w = word
+ for suffix, replacement in _STEM_SUFFIXES:
+ if w.endswith(suffix) and len(w) - len(suffix) >= 3:
+ w = w[: len(w) - len(suffix)] + replacement
+ break
+ if len(w) > 3 and w.endswith("y"):
+ w = w[:-1] + "i"
+ if len(w) > 3 and w.endswith("e"):
+ w = w[:-1]
+ if len(w) > 3 and w[-1] == w[-2] and w[-1] not in "aeiou":
+ w = w[:-1] # running -> runn -> run
+ return w
+
+
+@lru_cache(maxsize=8192)
+def _stem(word: str) -> str:
+ return _porter_stem(word) if _porter_stem else _fallback_stem(word)
+
+
+def _stemmable(token: str) -> bool:
+ """Only plain ASCII words are stemmed.
+
+ Identifiers ("1344259", "2020-21"), notation ("PPG") and CJK text must match
+ verbatim — stemming would corrupt them, and it has no meaning for Chinese.
+ """
+ return len(token) >= 4 and token.isascii() and token.isalpha()
+
+
+def _keyword_forms(kwds: list[str]) -> tuple[list[str], list[tuple[str, ...]]]:
+ """Split keywords into verbatim substrings and stem sequences.
+
+ A keyword whose tokens are ALL stemmable becomes a stem sequence (matched
+ anywhere as a contiguous run of stems); anything containing an identifier,
+ notation or CJK falls back to a verbatim substring match.
+ """
+ verbatim: list[str] = []
+ stemmed: list[tuple[str, ...]] = []
+ for kw in kwds or []:
+ k = (kw or "").strip().lower()
+ if not k:
+ continue
+ tokens = _WORD_RE.findall(k)
+ if tokens and all(_stemmable(t) for t in tokens):
+ stemmed.append(tuple(_stem(t) for t in tokens))
+ else:
+ verbatim.append(k)
+ return verbatim, stemmed
+
+
+def _sentence_stems(sentence: str) -> list[str]:
+ return [_stem(t) if _stemmable(t) else t for t in _WORD_RE.findall(sentence.lower())]
+
+
+def _sentence_matches(low: str, stems: list[str], verbatim: list[str], stemmed: list[tuple[str, ...]]) -> bool:
+ """True when a sentence contains a verbatim keyword or a contiguous stem run."""
+ if any(v in low for v in verbatim):
+ return True
+ for seq in stemmed:
+ width = len(seq)
+ for start in range(len(stems) - width + 1):
+ if tuple(stems[start : start + width]) == seq:
+ return True
+ return False
+
+
+_FACT_RE = re.compile(
+ r"(\d[\d,\.]*(?:st|nd|rd|th)?%?)"
+ r"|(19|20)\d{2}" # years
+ r"|\b(percent|percentage|million|billion|thousand|km|km2|sq\s*km|m\s*above|m)"
+ r"\b",
+ re.IGNORECASE,
+)
+_PROPER_NOUN_RE = re.compile(r"(?<![.!?]\.)\b[A-Z][a-z]{2,}\b")
+
+
+def _is_fact_dense_sentence(sent: str) -> bool:
+ """Heuristically flag a sentence that carries a fact the answer may hinge on
+ but which does not necessarily contain the query keywords — a number, a year,
+ a percentage, or a proper noun / named entity. Such sentences are kept during
+ narrowing even when they sit far from any keyword hit, so a numeric or
+ entity answer is never dropped just because it lacks the keyword phrasing.
+ """
+ low = sent.lower()
+ if _FACT_RE.search(sent) or _FACT_RE.search(low):
+ return True
+ if _PROPER_NOUN_RE.search(sent):
+ return True
+ return False
+
+
+def _narrow_content(content: str, kwds: list[str]) -> str | None:
+ """Return ``content`` narrowed to keyword sentences +/- 2 neighbours.
+
+ Matching is stem-tolerant: a keyword matches any word sharing its stem, so
+ "nominations" finds "nominated". Sentences that are fact-dense (numbers /
+ years / percentages / proper nouns) are kept regardless of keyword distance,
+ so numeric or named-entity answers survive narrowing. Returns ``None`` when
+ no keyword occurs anywhere in ``content``.
+ """
+ # Structured tables must be returned whole. A keyword hit anywhere in a big
+ # table (e.g. the capitals-by-latitude table) otherwise narrows to the hit
+ # sentence +/- neighbours and DROPS the far end of the table — precisely the
+ # "table truncated at -4.58°N, Maseru (-29.3°) missing" bug on FRAMES Q408.
+ # A table row is one data point, not a sentence, so keyword-window narrowing is
+ # wrong here; keep the full table (it is already rank-sorted by the retriever).
+ if "<table" in content.lower() or "<tr" in content.lower() or "<td" in content.lower():
+ return "..." + _highlight_keywords(content, kwds) + "..."
+
+ sents = _split_sentences(content)
+ if not sents:
+ return None
+ # Stem-tolerant matching: a keyword matches any word sharing its stem, so
+ # "nominations" finds "nominated" and "company" finds "companies".
+ verbatim, stemmed = _keyword_forms(kwds)
+ if not verbatim and not stemmed:
+ return None
+ keep: set[int] = set()
+ matched = False
+ for i, s in enumerate(sents):
+ low = s.lower()
+ if _sentence_matches(low, _sentence_stems(s), verbatim, stemmed):
+ matched = True
+ for j in range(max(0, i - 2), min(len(sents), i + 3)):
+ keep.add(j)
+ elif _is_fact_dense_sentence(s):
+ # Keep fact-dense sentences even without a keyword hit so the answer
+ # value (a bare figure, a date, a proper noun) is never lost.
+ for j in range(max(0, i - 1), min(len(sents), i + 2)):
+ keep.add(j)
+ if not matched:
+ return None
+ narrowed = "".join(sents[i] for i in sorted(keep)).strip()
+ return "..." + _highlight_keywords(narrowed, kwds) + "..."
+
+
+def _highlight_keywords(text: str, kwds: list[str]) -> str:
+ """Star the verbatim keyword phrases AND any word sharing a keyword's stem.
+
+ Full keyword phrases are matched first and starred as ONE contiguous span, so
+ a multi-word entity like "Atlanta Braves" becomes ``*Atlanta Braves*`` — never
+ ``*Atlanta* *Braves*`` — because the downstream cross-check matches entities
+ with a bounded contiguous regex that a per-word star would break.
+ """
+ phrases = sorted({(kw or "").strip().lower() for kw in kwds or [] if (kw or "").strip()}, key=len, reverse=True)
+ terms: list[str] = list(phrases)
+ # Add stem-matched words NOT already inside a phrase, so "nominated" still
+ # gets starred for keyword "nominations" while "Atlanta Braves" stays whole.
+ verbatim, stemmed = _keyword_forms(kwds)
+ stem_set = {s for seq in stemmed for s in seq}
+ if stem_set:
+ for word in re.findall(r"[A-Za-z]+", text):
+ low = word.lower()
+ if _stemmable(low) and _stem(low) in stem_set and not any(low in p for p in phrases):
+ terms.append(low)
+ if not terms:
+ return text
+ # Longest first so a phrase wins over a word it contains; one pass, so an
+ # already-starred span is never starred again.
+ pattern = re.compile("|".join(re.escape(t) for t in sorted(terms, key=len, reverse=True)), re.IGNORECASE)
+ return pattern.sub(lambda m: f"*{m.group(0)}*", text)
+
+
+def _narrow_by_keywords(chunks: list[dict], keywords: str) -> list[dict]:
+ """Narrow each chunk to its keyword-bearing sentences (+/- 1 neighbour) and
+ drop keyword-less chunks.
+
+ Keywords are the comma-separated terms (with close synonyms) produced by
+ ``formalize``; matching is stem-tolerant (a keyword matches any word sharing
+ its stem).
+ """
+ kwds = [k.strip().lower() for k in (keywords or "").split(",") if k.strip()]
+ if not kwds or not chunks:
+ return chunks
+ if len(kwds) < 3:
+ kwds = [k.strip().lower() for k in (keywords or "").split(" ") if k.strip()]
+ _kwds = []
+ for i in range(len(kwds) - 1):
+ _kwds.append(kwds[i] + " " + kwds[i + 1])
+ kwds = _kwds
+
+ scored = [(ck, _narrow_content(ck.get("content_with_weight") or ck.get("content") or "", kwds)) for ck in chunks]
+ out: list[dict] = []
+ dedup: set[str] = set()
+ for ck, nc in scored:
+ if nc is not None:
+ nc_hash = hashlib.md5(nc.encode("utf-8")).hexdigest()
+ if nc_hash in dedup:
+ continue
+ dedup.add(nc_hash)
+ ck["content_with_weight"] = nc
+ if "content" in ck:
+ ck["content"] = nc
+ ck.pop("highlight", None)
+ out.append(ck)
+ return out
+
+
+def _narrow_or_keep(chunks: list[dict], keywords: str, label: str) -> list[dict]:
+ """Narrow chunks to keyword sentences, but keep the originals when
+ narrowing would drop everything.
+
+ No keyword overlap does not mean irrelevant — the retriever already ranked
+ these chunks, and a sub-question's wording need not contain the parent
+ question's keywords. Dropping them all produced empty results, unverified
+ claims and pointless retry cycles.
+ """
+ if not keywords or not chunks:
+ return chunks
+ length = len(chunks)
+ narrowed = _narrow_by_keywords(chunks, keywords)
+ if narrowed:
+ _LOG.info(f"[{label}] Kept {len(narrowed)} of {length} passage(s) that actually mention the keywords.")
+ return narrowed
+ _LOG.info(f"[{label}] Keyword narrowing matched nothing — keeping all {length} retrieved passage(s).")
+ return chunks
diff --git a/rag/advanced_rag/harness/types.py b/rag/advanced_rag/harness/types.py
deleted file mode 100644
index 09f976745a..0000000000
--- a/rag/advanced_rag/harness/types.py
+++ /dev/null
@@ -1,372 +0,0 @@
-"""Data types for Agentic RAG harness."""
-
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from typing import Literal
-
-# ═══════════════════════════════════════════════════════════════
-# Route
-# ═══════════════════════════════════════════════════════════════
-
-
-@dataclass
-class RouteDecision:
- question: str
- thinking_mode: str
- question_type: str # factual | comparative | analytical | procedural | exploratory | verification | summarization
- requires_decomposition: bool
- suggests_compilation: str | None
- execution_strategy: str
- reasoning: str = ""
-
-
-# ═══════════════════════════════════════════════════════════════
-# Thinking Mode
-# ═══════════════════════════════════════════════════════════════
-
-
-@dataclass
-class ExecutionStrategy:
- label: Literal["low", "medium", "high", "ultra"]
- # legacy dispatch field — only ``low``'s "direct_search" is consumed by the
- # (remaining) orchestrator path; medium/high/ultra run the SCA react graph and
- # never read it.
- execution_strategy: Literal["direct_search", "decompose_and_search", "agentic_research", "deep_research"] = "direct_search"
- # Legacy research-agent / decomposition flags — not consumed by any running
- # path (medium/high/ultra run the single-agent SCA graph; low runs direct_search).
- requires_decomposition: bool = False
- requires_agent_loop: bool = False
- requires_sufficiency_judge: bool = False
- requires_selective_gen: bool = False
- allows_dynamic_claims: bool = False
- allows_replan: bool = False
- max_orchestrator_cycles: int = 1
- max_agent_cycles: int = 0
- max_parallel_agents: int = 1
- available_tools: list[str] = field(default_factory=list)
- sufficiency_threshold: float = 0.5
- fallback_to_direct_llm: bool = False
- # Min cross-check floor for a self-verified claim. Any claim scoring below
- # this becomes a hard veto (its localized evidence gap must not be averaged
- # away by stronger sibling claims). Default 0.5 == the cross-check pass bar.
- fusion_min_cross: float = 0.5
- # ── Decision-ladder operating points (Sufficient Context redesign) ──
- # These are monotonic product-policy thresholds — NOT trained weights. They
- # express "how conservative to be" (higher = investigate more before
- # answering), replacing the old weighted fusion.
- c_high: float = 0.75 # agent confidence >= this and AutoRater sufficient → ANSWER
- c_low: float = 0.45 # agent confidence >= this → ANSWER_WITH_CAVEAT; below → reconcile
- llm_floor: float = 0.55 # AutoRater confidence < this → re-investigate regardless of verdict
- # Whether the mode can force a re-investigation (RECONCILE). medium=False so
- # RECONCILE degrades to CONTINUE; high/ultra=True.
- allows_reconcile: bool = False
- # Terminal-tool shortcut: when True, the outer tool loop treats ``rag`` as a
- # terminal tool and short-circuits after its first successful call — the
- # produced (cited) answer is returned immediately instead of being fed back
- # for another outer round. Mirrors the ``_terminal`` short-circuit that
- # already exists in the streaming tool loops. When False, the outer loop
- # keeps its multi-round re-ask behaviour (research more evidence between
- # rounds). Default False keeps legacy behaviour unless a mode opts in.
- terminal_tool_shortcut: bool = False
-
-
-# ═══════════════════════════════════════════════════════════════
-# Plan & Claims
-# ═══════════════════════════════════════════════════════════════
-
-
-@dataclass
-class ClaimPlan:
- """Cross-round, per-claim plan state driving iterative, plan-anchored search.
-
- Unlike the old `while cycle < max_cycles` re-search loop, this keeps what a
- claim has learned and what it still needs across rounds, so each round only
- searches the missing part instead of blindly re-searching everything.
- """
-
- # Evidence accumulated across rounds (grounded facts + numbers).
- evidence: list[str] = field(default_factory=list)
- # Structured missing information (which entity / time / enumeration item).
- missing: list[str] = field(default_factory=list)
- # Bridge entities already resolved from prior hops (e.g. ["Tulsa", "1921"]).
- resolved_bridge: list[str] = field(default_factory=list)
- # Ordered hops still to resolve (each an atomic open query; later hops may
- # reference earlier resolved values via {0}, {1}, ...).
- pending_hops: list[str] = field(default_factory=list)
- # Next retrieval target for this round (entity+time anchored), or "".
- next_target: str = ""
- # Structurally accumulated member values for an AGGREGATE enumeration claim
- # (e.g. ["Minute Maid Park 330", "American Family Field 315"]). Each round the
- # claim's query_check appends newly-confirmed members here; ``missing`` records
- # which member(s) are still outstanding. combine claims read this as the
- # complete member set to synthesize from. Enables code-level completeness
- # checking ("no members missing") instead of trusting free-text reports.
- enumerated_members: list[str] = field(default_factory=list)
-
- # Whether any retrieval target remains (refined / hop / next_target / missing).
- def actionable(self) -> bool:
- return bool(self.missing or self.pending_hops or self.next_target)
-
-
-@dataclass
-class ClaimTarget:
- claim_id: str
- description: str
- priority: int = 0
- is_verified: bool = False
- confidence: float = 0.0
- suggested_tools: list[str] = field(default_factory=list)
- agent_result: AgentResult | None = None
- # Multi-hop: an OPEN query for a bridge entity/relationship that must be
- # retrieved BEFORE this claim can be answered (e.g. "Who is Brian Bergstein's
- # employer?"). decompose resolves the prerequisite first, then uses the found
- # entity to target this claim. Empty when not multi-hop.
- prerequisite: str = ""
- # Ordered list of hops for multi-hop claims (each an atomic open query, in
- # dependency order). When present, bridge resolution walks these hops and
- # stores resolved values in `plan.resolved_bridge`. Backward compatible: when
- # empty, we fall back to the single `prerequisite`.
- prerequisites: list[str] = field(default_factory=list)
- # Reasoning structure the planner assigned to this claim:
- # flat — single-hop fact, independent
- # chain — depends on a prerequisite (bridge entity/relationship)
- # aggregate — requires enumerating all members of a set and combining
- # them (count/sum/average/max/min)
- # temporal — answer depends on a specific year/period or cross-time link
- # comparative/procedural — retained for compatibility
- claim_type: str = "flat"
- # For aggregate claims: the full set of members to enumerate (e.g. "all MLB
- # stadiums with a retractable roof"), used to guide exhaustive retrieval.
- target: str = ""
- # Pure-synthesis marker: this claim is NOT a retrievable fact — it is a
- # combine/synthesis node whose value is produced by formalize_answer from the
- # enumerated member values (via WorkflowPlan.synthesis). It must never be fed
- # to hybrid_search / query_check (a "Combine the enumerated values..." query
- # retrieves nothing meaningful). The orchestrator excludes it from search and
- # treats its completeness as dependent on its source enumeration claim.
- synth_only: bool = False
- # Cross-round plan state (evidence / missing / bridge / hops).
- plan: ClaimPlan = field(default_factory=ClaimPlan)
-
-
-@dataclass
-class WorkflowPlan:
- plan_type: str # direct | fact_decomposition | comparative_decomposition | procedural_decomposition | exploratory_decomposition
- claims: list[ClaimTarget]
- max_iterations: int
- # One-sentence cross-claim synthesis instruction from the planner: how the
- # individual claim findings must be combined into the final answer (e.g. for a
- # difference question "answer = c2.year - c1.year"; for a comparison "select
- # the youngest from c1's enumerated list"; for an aggregate "sum c1's counts").
- # Empty for a single-claim / flat question where no synthesis is needed. It is
- # injected into pre_summary so formalize_answer performs the composition
- # explicitly instead of guessing it from juxtaposed claim blocks.
- synthesis: str = ""
-
-
-# ═══════════════════════════════════════════════════════════════
-# Agent Result
-# ═══════════════════════════════════════════════════════════════
-
-
-@dataclass
-class AgentResult:
- claim_id: str
- report: str
- is_verified: bool
- confidence: float
- evidence_ids: list[int] = field(default_factory=list)
- gaps: list[str] = field(default_factory=list)
- discovered_claims: list[str] = field(default_factory=list)
- # Key factual assertions the agent claims are directly supported by the
- # cited evidence. Used by cross-check as the *ground-truth* list to verify:
- # if a grounded fact is absent from the evidence, the claim is a
- # prior-knowledge-injection risk (see check1.log Q203 hometown=Ithaca,
- # Q665 first-solo-album=1970) and must not count as sufficient.
- grounded: list[str] = field(default_factory=list)
- # For numerical / multi-hop questions: the figures used in the answer with
- # their sources ("2,161,000 from Wikipedia Demographics of Paris 2019").
- # Used to detect multi-source numeric conflicts that a ratio/mean cross-check
- # would otherwise paper over (see check.log Q754: 225 vs 228 population口径).
- numbers: list[str] = field(default_factory=list)
-
-
-# ═══════════════════════════════════════════════════════════════
-# Cross Check
-# ═══════════════════════════════════════════════════════════════
-
-
-@dataclass
-class ClaimCrossCheckResult:
- claim_id: str
- cross_check_passed: bool
- cross_check_score: float
- evidence_matches: list[str] = field(default_factory=list)
- mismatches: list[str] = field(default_factory=list)
-
-
-# ═══════════════════════════════════════════════════════════════
-# Sufficiency Verdict
-# ═══════════════════════════════════════════════════════════════
-
-
-@dataclass
-class SufficiencyVerdict:
- status: str # code-level preliminary label (decision ladder decides final action)
- score: float
- claim_assessments: list[dict]
- has_conflicts: bool
- missing_claims: list[str]
- feedback: str
- # Decision-ladder inputs (Sufficient Context redesign):
- # - hard_violations: claim IDs with a code-proven evidence gap (required
- # entity missing / grounded absent / numeric conflict) that must veto a
- # full answer even if the LLM AutoRater says sufficient.
- # - agent_confidence: mean self-confidence over the trusted subset.
- hard_violations: list[str] = field(default_factory=list)
- agent_confidence: float = 0.0
-
-
-# ═══════════════════════════════════════════════════════════════
-# Pipeline
-# ═══════════════════════════════════════════════════════════════
-
-
-@dataclass
-class ToolResult:
- chunks: list[dict] = field(default_factory=list)
- # Doc-id list from routing tools (e.g. dataset_navigation_search) that
- # narrow the corpus to the relevant documents instead of returning chunks.
- docs: list[str] | None = None
- metadata: dict = field(default_factory=dict)
- error: str | None = None
-
-
-# ═══════════════════════════════════════════════════════════════
-# Orchestrator Context
-# ═══════════════════════════════════════════════════════════════
-
-
-@dataclass
-class ExecutionBook:
- """Per-run execution ledger (L1 state).
-
- Converges the previously-scattered per-claim bookkeeping variables
- (``attempted_queries``, ``attempted_evidence``, ``pending_queries``,
- ``refined_queries``, follow-up pool, frozen set) into ONE object that lives
- with the orchestrator run, so replan / full-redo / claim-frozen never need
- fragile ``dict.update(...)`` patches to stay in sync. Each key is a claim_id
- (or the sentinel ``""`` for the global follow-up pool).
-
- Lifecycle:
- * Inner-loop retry and incremental replan MUTATE it in place (same plan).
- * FULL plan re-do REPLACES the per-claim entries for dropped claims while
- preserving ``frozen`` and the global follow-up pool (verified evidence
- lives in ``kbinfos``/``claim.agent_result``, not here).
- """
-
- # Queries already issued per claim (dedup key for the inner retry loop).
- attempted_queries: dict[str, set] = field(default_factory=dict)
- # Evidence chunk ids already analyzed per claim.
- attempted_evidence: dict[str, set] = field(default_factory=dict)
- # Targeted queries waiting to be searched for a claim (next-hops / replan).
- pending_queries: dict[str, list] = field(default_factory=dict)
- # query_check's structured "refined next query" per claim (smartsearch).
- refined_queries: dict[str, str] = field(default_factory=dict)
- # SCA / AutoRater global follow-up pool (consumed then cleared).
- followup_pool: list[dict] = field(default_factory=list)
- # Claim ids frozen by claim-level stagnation guard (never re-searched; their
- # already-collected evidence stays available for finalize).
- frozen: set = field(default_factory=set)
- # Normalized text of queries injected by the Query Rewriter / replan
- # (``_inject_replan_queries``). These are *precise, self-contained* targeted
- # queries (one-to-one for a reported gap), so the orchestrator must NOT
- # re-append the global formalize keywords when searching them — appending
- # them re-introduces the query dilution/pollution the rewrite was meant to
- # fix (see decompose._ref_kw).
- replan_query_seen: set = field(default_factory=set)
-
- def init_claim(self, claim_id: str) -> None:
- """Idempotently create an empty ledger entry for ``claim_id``."""
- self.attempted_queries.setdefault(claim_id, set())
- self.attempted_evidence.setdefault(claim_id, set())
- self.pending_queries.setdefault(claim_id, [])
- self.refined_queries.setdefault(claim_id, "")
-
- def drop_claim(self, claim_id: str) -> None:
- """Remove a claim's ledger entries (used by full plan re-do)."""
- for m in (self.attempted_queries, self.attempted_evidence, self.pending_queries, self.refined_queries):
- m.pop(claim_id, None)
-
-
-@dataclass
-class OrchestratorContext:
- question: str
- claims: list[ClaimTarget]
- mode: str
- iteration: int = 0
- current_phase: str = "locate"
- verdict: SufficiencyVerdict | None = None
- history: list[dict] = field(default_factory=list)
- # Follow-up search queries produced by the LLM Sufficient Context
- # AutoRater when evidence was deemed insufficient. They are consumed by the
- # next research round to guide targeted follow-up search, then cleared.
- pending_followups: list[dict] = field(default_factory=list)
- # Cross-claim synthesis instruction from the planner (how claim findings
- # combine into the final answer). Injected into pre_summary by _finalize.
- synthesis: str = ""
- # Deterministic arithmetic result derived from the gathered facts (label,
- # value, expression) when the question asks for a number that follows
- # arithmetically from them. Computed by _compute_final_arithmetic and
- # injected into pre_summary so formalize_answer uses the exact value.
- computed_answer: str = ""
- # Targeted search queries produced by the Query Rewriter
- # from the SCA's forward gaps, as (claim_id, query) pairs recorded by
- # ``_try_replan`` for the newly-added claims. The orchestrator injects them
- # into the new claims' pending-query pool so they reach the retriever
- # verbatim — NOT re-synthesized by _pick_next_query — closing the "rewrite
- # got diluted" gap. Cleared after a successful replan.
- replan_queries: list[tuple[str, str]] = field(default_factory=list)
- # Number of FULL plan re-do's performed (the outer-loop-equivalent: re-running
- # the planner with SCA feedback to correct the overall decomposition direction,
- # as opposed to an incremental replan that only adds missing-claim search).
- # Bounded so a persistently-wrong plan cannot re-decompose forever.
- full_replan_count: int = 0
- _last_entity: str | None = None
-
- @property
- def last_entity(self) -> str | None:
- return self._last_entity
-
- def note_entity(self, name: str | None) -> None:
- """Record the most recently discovered entity/document name.
-
- Gates ``graph_explore`` in ``tool_fits_context``: the tool is only
- offered once research has surfaced something to expand from. Ignores
- empty values so a fruitless round can't clear a prior discovery.
- """
- if isinstance(name, str) and name.strip():
- self._last_entity = name.strip()
-
- @property
- def current_claim(self) -> str | None:
- unverified = [c for c in self.claims if not c.is_verified]
- return unverified[0].description if unverified else None
-
- def has_any_chunks(self) -> bool:
- """True once any claim's research produced evidence passages.
-
- Reads the claims (whose ``agent_result`` is populated by both
- orchestrators) — the ``agent_results`` dict is never written to, so
- reading it would leave the search phase stuck at "locate" forever and
- gate off every inspector tool.
- """
- for c in self.claims:
- r = c.agent_result
- if r is None:
- continue
- ids = r.get("evidence_ids") if isinstance(r, dict) else r.evidence_ids
- if ids:
- return True
- return False
diff --git a/rag/prompts/action_run.md b/rag/prompts/action_run.md
index 39bfabaf00..696b966e08 100644
--- a/rag/prompts/action_run.md
+++ b/rag/prompts/action_run.md
@@ -3,12 +3,16 @@ You are a deep research assistant working INSIDE a bounded search tree.
Input: the user message contains ONE research `Direction`, plus the current
`State` (slot table with immutable ids and mutable candidate fields).
-Execute this single direction. Return EXACTLY ONE of:
+Execute this single direction. Reply with EXACTLY ONE of the three below.
-1) TOOL CALL MODE — call `retrieve` (the only bound tool) with 1-3 corpus queries.
+The three differ in HOW they are delivered — read this carefully, because only
+the first one is a tool call:
+
+1) TOOL CALL MODE — a real tool call: call `retrieve` with 1-3 corpus queries.
Results arrive in the next turn.
-2) STATE PATCH MODE — when you have partial updates, emit:
+2) STATE PATCH MODE — NOT a tool call. Write this XML as plain TEXT in your
+ reply body (do not call any tool named "state"):
<state>
{"new_states": [
{"state": [{"id": <int>, "candidate": "<value>", "candidate_strength": <0..1>, "discovered_clues": ["..."]}, ...]},
@@ -17,7 +21,9 @@ Execute this single direction. Return EXACTLY ONE of:
</state>
Rules: patch ONLY existing ids; include ONLY changed variables; every change must trace to retrieved evidence; candidate_strength semantics: proven >0.9, strong 0.7-0.9, tentative 0.4-0.7, weak <0.4. An EMPTY branch list (`"new_states": []`) signals no progress — emit it rather than calling tools forever.
-3) FINAL ANSWER MODE — only when ALL slots can be filled consistently:
+3) FINAL ANSWER MODE — NOT a tool call either. Write this XML as plain TEXT in
+ your reply body (do not call any tool named "answer"). Use it only when ALL
+ slots can be filled consistently:
<answer>
{"answer": "<final answer text>", "new_state": [{"id": ..., "candidate": ..., "candidate_strength": ...}]}
</answer>