RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(agentic_rag): slot-table harness, table rendering + pre-search/metadata channels (#19780)
Caobaining197018a5e6017
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.ymlindex daf7d5004d..ae8b465c37 100644--- a/.github/workflows/sep-tests.yml+++ b/.github/workflows/sep-tests.yml@@ -1018,11 +1018,24 @@ jobs:- name: Collect ragflow log Infinityif: ${{ !cancelled() }}+ # Log collection is diagnostics only: rebuild the artifacts path here+ # (the directory created by "Prepare function test environment" may be+ # gone by now, which made `cp` fail with ENOENT) and never fail the job+ # over a log copy error.run: |+ set -uo pipefail+ RUNNER_WORKSPACE_PREFIX="${RUNNER_WORKSPACE_PREFIX:-${HOME}}"+ ARTIFACTS_DIR="${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}/${API_PROXY_SCHEME}"+ LOG_DEST="${ARTIFACTS_DIR}/ragflow-logs-infinity"+ mkdir -p "${ARTIFACTS_DIR}"if [ -d docker/ragflow-logs ]; then- destination="${ARTIFACTS_DIR}/ragflow-logs-infinity"- sudo cp -r docker/ragflow-logs "${destination}"- sudo chown -R "$(id -u):$(id -g)" "${destination}"+ sudo rm -rf "${LOG_DEST}"+ if sudo cp -a docker/ragflow-logs "${LOG_DEST}"; then+ sudo chown -R "$(id -u):$(id -g)" "${LOG_DEST}" || true+ echo "Copied ragflow logs to ${LOG_DEST}"+ else+ echo "::warning::Failed to copy ragflow logs to ${LOG_DEST}; continuing"+ fiecho "ragflow log" && sudo tail -n 200 docker/ragflow-logs/ragflow_server.log || trueelseecho "No docker/ragflow-logs directory found; skipping log collection"@@ -1697,11 +1710,23 @@ jobs:- name: Collect ragflow log Elasticsearchif: ${{ !cancelled() }}+ # Keep this step self-sufficient: rebuild the artifacts path instead of+ # relying on the directory created earlier, and downgrade copy failures+ # to warnings so a diagnostics step can never fail the job.run: |+ set -uo pipefail+ RUNNER_WORKSPACE_PREFIX="${RUNNER_WORKSPACE_PREFIX:-${HOME}}"+ ARTIFACTS_DIR="${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}/${API_PROXY_SCHEME}"+ LOG_DEST="${ARTIFACTS_DIR}/ragflow-logs-es"+ mkdir -p "${ARTIFACTS_DIR}"if [ -d docker/ragflow-logs ]; then- destination="${ARTIFACTS_DIR}/ragflow-logs-es"- sudo cp -r docker/ragflow-logs "${destination}"- sudo chown -R "$(id -u):$(id -g)" "${destination}"+ sudo rm -rf "${LOG_DEST}"+ if sudo cp -a docker/ragflow-logs "${LOG_DEST}"; then+ sudo chown -R "$(id -u):$(id -g)" "${LOG_DEST}" || true+ echo "Copied ragflow logs to ${LOG_DEST}"+ else+ echo "::warning::Failed to copy ragflow logs to ${LOG_DEST}; continuing"+ fiecho "ragflow log" && sudo tail -n 200 docker/ragflow-logs/ragflow_server.log || trueelseecho "No docker/ragflow-logs directory found; skipping log collection"diff --git a/rag/advanced_rag/agentic_rag_graph.py b/rag/advanced_rag/agentic_rag_graph.pyindex d671467d84..d1be162074 100644--- a/rag/advanced_rag/agentic_rag_graph.py+++ b/rag/advanced_rag/agentic_rag_graph.py@@ -494,7 +494,7 @@ async def _expand_fanouts(tools, question: str, answer_conf: dict) -> list[str]:_u = usage if isinstance(usage, dict) else getattr(mdl, "last_usage", None)record_external_call(_u if isinstance(_u, dict) else None)except Exception: # noqa: BLE001- pass+ _LOG.debug("[Planner] external-call usage booking skipped", exc_info=True)fanouts = _parse_fanouts(str(ans or ""))if not fanouts:# The model answered the question instead of decomposing it (no JSON, or@@ -514,7 +514,7 @@ async def _expand_fanouts(tools, question: str, answer_conf: dict) -> list[str]:_u2 = getattr(mdl, "last_usage", None)record_external_call(_u2 if isinstance(_u2, dict) else None)except Exception: # noqa: BLE001- pass+ _LOG.debug("[Planner] retry external-call usage booking skipped", exc_info=True)fanouts = _parse_fanouts(str(ans2 or ""))except Exception: # noqa: BLE001_LOG.warning("[Planner] strict fan-out retry failed", exc_info=True)@@ -527,6 +527,118 @@ async def _expand_fanouts(tools, question: str, answer_conf: dict) -> list[str]:return [question] if question else []+# Pre-search metadata channel: the LLM extracts the short NAMED ENTITIES each+# sub-question targets, so the metadata (title) filter can pre-select the document+# subset before retrieval. Entities use SPACES, never underscores.+_METADATA_ENTITY_PROMPT = (+ "For each numbered search sub-question below, extract up to 3 short named entities "+ "it targets — the kind of name a document index stores as a title (an article/report "+ "name, an organisation, a place, an event, a person). Use the exact surface form with "+ "SPACES, never underscores. If a sub-question has no clear title-like entity, use an "+ "empty string for it. Keep each entity under 10 words; never copy the whole "+ "sub-question, never answer it. "+ 'Respond with JSON only: {"entities": [["<e1>", "<e2>", ...], ...]} — a list of '+ "entity-arrays in the SAME ORDER as the sub-questions, JSON only, no prose."+)++# A usable metadata entity filter is a SHORT name — a long string is a copied+# sub-question or an answer, which would make the title filter match nothing.+_METADATA_ENTITY_MAX_WORDS = 10+_METADATA_ENTITY_MAX_CHARS = 80+_METADATA_MAX_ENTITIES = 3+++def _entity_looks_usable(entity: str) -> bool:+ """Guard: the entity fed to the metadata title filter must be a short name."""+ s = (entity or "").strip()+ if not s:+ return False+ low = s.lower()+ if any(mark in low for mark in _FANOUT_ANSWER_MARKS):+ return False+ return not (len(s) > _METADATA_ENTITY_MAX_CHARS or len(s.split()) > _METADATA_ENTITY_MAX_WORDS)+++def _parse_fanout_entities(text: str, fanouts: list[str]) -> list[list[str]]:+ """Extract up to 3 short entities per sub-question from a model reply.++ Order-aligned with ``fanouts``; each element is a (deduped, guarded) entity+ list. Accepts ``{"entities": [["e1","e2"], ...]}`` (preferred), a flat+ ``{"entities": ["e1","e2",...]}`` (treated as one group for every sub-question),+ or ``{"entities": [{"sub_question": ..., "entities": [...]}]}`` matched by text.+ Entities failing :func:`_entity_looks_usable` are dropped.+ """+ out: list[list[str]] = [[] for _ in fanouts]+ data = _extract_json_object(text)+ if not isinstance(data, dict):+ return out+ items = data.get("entities")+ if not isinstance(items, list) or not items:+ return out+ by_text: dict[str, list[str]] = {}+ positional: list[list[str]] = []+ for it in items:+ if isinstance(it, dict):+ ents = [str(e).strip() for e in (it.get("entities") or it.get("titles") or [])]+ sq = str(it.get("sub_question") or it.get("fanout") or "").strip()+ if sq:+ by_text[sq] = ents+ positional.append(ents)+ elif isinstance(it, str):+ positional.append([str(it).strip()])+ elif isinstance(it, list):+ positional.append([str(e).strip() for e in it])+ for i, fq in enumerate(fanouts):+ raw = by_text.get(fq, []) if by_text else (positional[i] if i < len(positional) else [])+ seen: set[str] = set()+ cleaned: list[str] = []+ for e in raw:+ e = (e or "").strip()+ if e and e.lower() not in seen and _entity_looks_usable(e):+ seen.add(e.lower())+ cleaned.append(e)+ if len(cleaned) >= _METADATA_MAX_ENTITIES:+ break+ out[i] = cleaned+ return out+++async def _extract_fanout_entities(tools, fanouts: list[str], answer_conf: dict) -> dict[str, list[str]]:+ """Map each sub-question to 1-3 short entities for metadata (title) filtering.++ ONE chat call for the whole batch. Returns ``{fanout: [entities]}`` containing+ only entries with at least one usable entity; any failure yields ``{}`` so the+ metadata channel is simply skipped for this round — it never blocks pre-search.+ """+ queries = [str(f).strip() for f in (fanouts or []) if isinstance(f, str) and str(f).strip()]+ if not queries:+ return {}+ try:+ from rag.advanced_rag.harness.tools.search import _base_chat_mdl++ mdl = _base_chat_mdl(tools)+ except Exception: # noqa: BLE001+ _LOG.warning("[Prefetch] could not resolve base chat model for entity extraction", exc_info=True)+ return {}+ if mdl is None:+ return {}+ listed = "\n".join(f"{i + 1}. {q}" for i, q in enumerate(queries))+ try:+ ans, _ = await mdl.async_chat(+ _METADATA_ENTITY_PROMPT,+ [{"role": "user", "content": f"Sub-questions:\n{listed}"}],+ dict(answer_conf or {}),+ )+ entities = _parse_fanout_entities(str(ans or ""), queries)+ except Exception: # noqa: BLE001+ _LOG.warning("[Prefetch] entity extraction failed; skipping metadata channel", exc_info=True)+ return {}+ mapping = {q: es for q, es in zip(queries, entities) if es}+ if mapping:+ _LOG.info("[Prefetch] metadata entities: %s", mapping)+ return mapping++# Storage ceiling of the snippet pool across ALL rounds. Storage and REVIEW# are decoupled: the SCA only reads a ranked view (``_SCA_VIEW_CAP``), so the# pool may accumulate freely while prompts stay bounded. The former single@@ -558,17 +670,33 @@ _MAX_SLOT_DEPTH = 3# the research tree, not the gap-promotion rounds), so the depth test alone# cannot stop the table from growing one slot per round — this bound can._MAX_SLOTS_TOTAL = 8+_PRESEARCH_MAX_POOL = 18 # pre-search (first prefetch) hard ceiling on TOTAL chunks admitted+# Channel A (BM25 + narrow) keeps this many chunks per fan-out query. It is the+# retrieval-window size, NOT the pool share: the pool is divided equally between+# the channels that have chunks to offer, so A's quota no longer decides how much+# of the pool it gets (measured 2026-09-16: 3 fan-outs x 5 = 15 chunks used to+# take 15 of the 18 pre-search slots, starving the metadata channel).+_CHANNEL_A_QUERY_QUOTA = 5+# Channel B (semantic hybrid, narrow bypass) per-query quota. It was switched off+# outright in pre-search by 2b2ae7a; re-enabled 2026-09-16. Two things changed+# since: the pool share is now split EQUALLY between the channels that actually+# have chunks, so a third channel no longer steals A's/C's slots, and the+# zero-score attribution showed the surviving failures are passages whose surface+# words do not overlap the query at all (a father's occupation, a section buried+# in a long document) — exactly what only the semantic channel can reach.+_PRESEARCH_HYBRID_ENABLED = True+_CHANNEL_B_QUERY_QUOTA = 4_DRILL_RESERVE = 12 # slots kept free after the FIRST prefetch so the research# executor can top up evidence_SCA_VIEW_CAP = 60 # chunks shown to the Sufficient Context Agent per review (24 -> 60: 24 of 225 hid the answer-bearing table chunk from the SCA)-async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: int | None = None) -> int:+async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: int | None = None, answer_conf: dict | None = None, use_metadata: bool = False) -> int:"""Programmatic fan-out retrieval: fetch all queries and fill ``kbinfos``.Phase 2 ("The RAG Agent searches ... all the query fanouts at once")for the SNIPPET pool the Sufficient Context Agent reviews (Phase 3 reads- actual retrieved text). Two collector channels run per query:+ actual retrieved text). Up to three collector channels run per query:* Channel A (BM25 + ``narrow_by_terms``): exact-name recall, rewritten to ashort match window.@@ -576,6 +704,9 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: intext does NOT share surface words with the query. The 2026-08-27 hybridexperiment proved these blocks are valuable but are ALL filtered out by the keyword narrow step whilecrowding candidates out of top_n — so they now bypass it entirely.+ * Channel C (metadata title pre-filter, only when ``use_metadata``): the LLM+ distils each sub-question to a short title/entity, documents are pre-selected+ by their ``title`` metadata, and hybrid retrieval runs INSIDE that subset.Called after planning and again after every Query-Rewriter pass. Returnshow many NEW chunks were added; failures are swallowed per-query.@@ -589,6 +720,7 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: in_query_to_terms,bm25_search,hybrid_search,+ metadata_search,)except Exception: # noqa: BLE001_LOG.warning("[rag_agent] could not import fan-out search helpers", exc_info=True)@@ -634,7 +766,16 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: in_LOG.warning("[Prefetch] unexpected queries payload of type %s; skipping fan-out search", type(fanouts).__name__)return 0- async def _search_one(fq: str) -> tuple[list, list]:+ # Channel C setup: one batched LLM call maps each sub-question to 1-3 short+ # entities used to pre-filter documents by their ``title`` metadata (OR'ed).+ # Best-effort: an empty mapping simply disables the channel for this round.+ entities_by_query: dict[str, list[str]] = {}+ if use_metadata:+ entities_by_query = await _extract_fanout_entities(tools, fanouts, answer_conf or {})+ if not entities_by_query:+ _LOG.info("[Prefetch] no usable metadata entities; metadata channel disabled this round")++ async def _search_one(fq: str) -> tuple[list, list, list]:# Each coroutine ONLY retrieves and returns chunk lists; it does NOT# touch ``kbinfos``. All mutation happens in the main coroutine below,# so concurrent fan-out searches cannot race on the shared list.@@ -650,7 +791,7 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: interms = _query_to_terms(fq)keyed = [t for t in terms if len(t) >= 3 and t.lower() not in _FANOUT_STOPWORDS and not t.isdigit() or (len(t) >= 4 and t.isdigit())]try:- res = await bm25_search(tools, fq, kb_ids=kb_ids, top_n=60, keywords=" ".join(keyed or terms))+ res = await bm25_search(tools, fq, kb_ids=kb_ids, top_n=48, keywords=" ".join(keyed or terms))candidates = res.get("chunks", []) or []except Exception: # noqa: BLE001_LOG.warning("[rag_agent] BM25 search failed for %r", fq, exc_info=True)@@ -663,31 +804,63 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: incandidates,keyed or terms,fallback_terms=None,- context={"before": 0, "after": 1},+ # Match-centred window: the answer-bearing row of a long+ # section is rarely its first line, and the old (0,1) window+ # showed the match plus one line only.+ context={"before": 2, "after": 4},keywords=fq,- max_out_chars_per_chunk=1200,+ max_out_chars_per_chunk=1500,max_out_total_chars=16000,)- kept_a = (narrowed.get("kept", []) or [])[: max(1, top_n)]+ kept_a = (narrowed.get("kept", []) or [])[: max(1, _CHANNEL_A_QUERY_QUOTA)]except Exception: # noqa: BLE001_LOG.warning("[rag_agent] narrowing failed for %r; using raw BM25 head", fq, exc_info=True)- kept_a = candidates[: max(1, top_n)]+ kept_a = candidates[: max(1, _CHANNEL_A_QUERY_QUOTA)]- # Channel B: semantic collector, narrow BYPASS. Keep only hits that- # Channel A did not already surface (dedup happens again at merge).+ # Channel B: semantic collector (hybrid, narrow bypass). Controlled by+ # ``_PRESEARCH_HYBRID_ENABLED`` — when off, kept_b stays empty and+ # Channel C's dedup (seen_ab = seen_ids_a | kept_b) still works unchanged.kept_b: list = []seen_ids_a = {_chunk_id(c) for c in kept_a}- try:- hres = await hybrid_search(tools, fq, kb_ids=kb_ids, top_n=30)- for c in hres.get("chunks", []) or []:- if _chunk_id(c) in seen_ids_a:- continue- kept_b.append(c)- if len(kept_b) >= 4: # modest semantic quota per query- break- except Exception: # noqa: BLE001- _LOG.warning("[rag_agent] hybrid channel failed for %r; skipping", fq, exc_info=True)- return kept_a, kept_b+ if _PRESEARCH_HYBRID_ENABLED:+ try:+ hres = await hybrid_search(tools, fq, kb_ids=kb_ids, top_n=30)+ for c in hres.get("chunks", []) or []:+ if _chunk_id(c) in seen_ids_a:+ continue+ kept_b.append(c)+ if len(kept_b) >= _CHANNEL_B_QUERY_QUOTA: # modest semantic quota per query+ break+ except Exception: # noqa: BLE001+ _LOG.warning("[rag_agent] hybrid channel failed for %r; skipping", fq, exc_info=True)++ # Channel C: metadata title pre-filter (only when usable entities were+ # extracted for this sub-question). Documents are pre-selected by their+ # ``title`` metadata matching ANY of the entities (OR), then hybrid+ # retrieval runs INSIDE that subset; keep only hits A/B did not already+ # surface (dedup happens again at merge).+ kept_c: list = []+ entities = entities_by_query.get(fq)+ if entities:+ seen_ab = seen_ids_a | {_chunk_id(c) for c in kept_b}+ try:+ mres = await metadata_search(+ tools,+ fq,+ [{"key": "title", "op": "contains", "value": e} for e in entities],+ logic="or",+ kb_ids=kb_ids,+ top_n=top_n,+ )+ for c in mres.get("chunks", []) or []:+ if _chunk_id(c) in seen_ab:+ continue+ kept_c.append(c)+ if len(kept_c) >= 4: # modest metadata quota per query+ break+ except Exception: # noqa: BLE001+ _LOG.warning("[rag_agent] metadata channel failed for %r (entities=%r); skipping", fq, entities, exc_info=True)+ return kept_a, kept_b, kept_casync def _collect_evidence() -> list:"""Channel 0: compiled evidence rows, folded into pool-shaped entries.@@ -773,11 +946,77 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: inraw_added += 1return stop+ def _channel(pair, idx: int) -> list:+ if isinstance(pair, Exception) or not isinstance(pair, tuple):+ return []+ return pair[idx] if idx < len(pair) else []++ # ── Equal pool share for every channel that actually has chunks ────────+ # Admitting the collectors strictly in order let channel A (up to+ # ``_CHANNEL_A_QUERY_QUOTA`` chunks per fan-out query) consume the whole+ # pre-search pool before the metadata channel was reached: measured+ # 2026-09-16, 3 fan-outs -> A took 15 of the 18 slots and the metadata+ # channel was truncated to 3 of the 8 chunks it offered; with 4 fan-outs it+ # admitted nothing at all while its entity-extraction LLM call had already+ # been paid for. Every channel that has something to offer now gets an equal+ # share of the remaining room, so a claim-less dataset (FRAMES has no+ # ``entity_type_kwd`` rows, so the evidence channel is inactive) with channel+ # B disabled splits A/C 50/50 instead of A taking everything.+ counts: dict = {"evidence": 0, "A": 0, "B": 0, "C": 0}+ offered: dict = {name: sum(len(_channel(p, i)) for p in results) for i, name in ((0, "A"), (1, "B"), (2, "C"))}+ active: list = (["evidence"] if evidence_chunks else []) + [n for n in ("A", "B", "C") if offered[n]]+ share = max(1, -(-room // len(active))) if (active and room > 0) else 0+ _LOG.info(+ "[Prefetch] pool shares: room=%d active=%s share=%d (offered evidence=%d A=%d B=%d C=%d)",+ room,+ active or "-",+ share,+ len(evidence_chunks or []),+ offered["A"],+ offered["B"],+ offered["C"],+ )++ def _log_admission() -> None:+ _LOG.info(+ "[Prefetch] admission by channel: evidence=%d A=%d B=%d C=%d (offered A=%d B=%d C=%d) total=%d room=%d",+ counts["evidence"],+ counts["A"],+ counts["B"],+ counts["C"],+ offered["A"],+ offered["B"],+ offered["C"],+ added,+ room,+ )++ def _run_channel(name: str, idx: int, cap: int) -> bool:+ # ``_admit`` consumes a whole per-query batch at once, so slice each+ # batch to the channel's remaining share — otherwise one batch can+ # overshoot the cap by up to (batch size - 1) and re-create the very+ # imbalance the share is meant to remove.+ before = added+ stop = False+ for pair in results:+ remaining = cap - (added - before)+ if remaining <= 0:+ break+ if _admit(_channel(pair, idx)[:remaining]):+ stop = True+ break+ counts[name] += added - before+ return stop+# Evidence rows first: they are answer material, so they must not be- # crowded out by the chunk channels.+ # crowded out by the chunk channels. Bounded by its equal share like the+ # chunk channels are.+ ev_before = addedif evidence_chunks:- evidence_chunks = evidence_chunks[:_EVIDENCE_POOL_QUOTA]+ evidence_chunks = evidence_chunks[: min(_EVIDENCE_POOL_QUOTA, share)]if _admit(evidence_chunks):+ counts["evidence"] = added - ev_before+ _log_admission()return added# Directional top-up (external gathering): an evidence row carries a# verbatim quote but not its surrounding passage, so pull exactly the@@ -793,23 +1032,24 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: infrom rag.advanced_rag.harness.tools.navigation import _load_chunks_for_idsfetched = await _load_chunks_for_ids(tools, wanted[:_EVIDENCE_TOP_UP])- if fetched and _admit(fetched):+ room_for_evidence = max(0, share - (added - ev_before))+ if fetched and _admit(fetched[:room_for_evidence]):+ counts["evidence"] = added - ev_before+ _log_admission()return addedexcept Exception: # noqa: BLE001_LOG.warning("[rag_agent] evidence top-up failed; continuing", exc_info=True)- # Channel A first (exact matches earn their slots), then semantic extras.- for pair in results:- if isinstance(pair, Exception):- continue- kept_a, kept_b = pair if isinstance(pair, tuple) else ([], [])- if _admit(kept_a):- break- for pair in results:- if isinstance(pair, Exception):- continue- _, kept_b = pair if isinstance(pair, tuple) else ([], [])- if _admit(kept_b):- break+ counts["evidence"] = added - ev_before+ # Channel order still decides who is served first in the only remaining+ # unequal case: the ceil() rounding of ``share`` can make the shares sum to+ # slightly more than ``room``, and the global room guard then trims whichever+ # channel is served last. Each channel is otherwise capped at exactly its+ # own share, so the pool split stays even and a channel with nothing to offer+ # costs nobody else a slot.+ _run_channel("A", 0, share)+ _run_channel("B", 1, share)+ _run_channel("C", 2, share)+ _log_admission()if room == 0:_LOG.info("[Prefetch] snippet pool FULL (%d chunks); nothing new admitted", max_total)@@ -859,8 +1099,53 @@ async def _compose_answer_from_evidence(state: AgenticState, tools, token_queue:)from rag.advanced_rag.agentic_rag import _EVIDENCE_BUDGET_TOKENS- _CITE_CHUNK_CAP = 6- cite_chunks = ranked[:_CITE_CHUNK_CAP] or all_chunks+ _CITE_CHUNK_CAP = 12+ # The answer model's evidence pool is assembled from three prioritised+ # groups, deduped by chunk id, then trimmed to the cap (``kb_prompt`` still+ # enforces the token budget on top of this):+ # 1. slot-cited chunks — the passages that actually produced each slot's+ # draft/candidate (``state["slot_evidence"]``). This is evidence the+ # research already committed to, so it belongs in the pool by default+ # instead of being re-selected by similarity from scratch.+ # 2. table chunks — aggregation answers ("how many times / how many+ # children / which year") are tallied from rows, so the table has to+ # reach the answer model; a pure top-k-by-similarity slice dropped the+ # deciding election-result table (FRAMES Q673/Q483).+ # 3. top-similarity chunks — fallback citation reference.+ # The previous pool was a hard top-6-by-similarity: too few once several+ # slots each cite distinct evidence, and it starved groups 1 and 2.+ from rag.advanced_rag.harness.tools.search import _chunk_id, _is_table_chunk++ by_chunk_id: dict[str, dict] = {}+ for _c in all_chunks:+ _cid = _chunk_id(_c)+ if _cid and _cid not in by_chunk_id:+ by_chunk_id[_cid] = _c+ _slot_evidence = state.get("slot_evidence") or {}+ _slot_order: list[str] = []+ for _meta in _slot_evidence.values():+ for _eid in _meta.get("evidence_ids") or []:+ _eid = str(_eid)+ if _eid not in _slot_order:+ _slot_order.append(_eid)+ slot_cite = [by_chunk_id[eid] for eid in _slot_order if eid in by_chunk_id]+ _TABLE_CITE_CAP = 4+ table_cite = [c for c in ranked if _is_table_chunk(c)][:_TABLE_CITE_CAP]++ cite_chunks: list = []+ _seen_cite: set[str] = set()+ for _group in (slot_cite, table_cite, ranked):+ for _c in _group:+ _key = _chunk_id(_c) or str(id(_c))+ if _key in _seen_cite:+ continue+ _seen_cite.add(_key)+ cite_chunks.append(_c)+ if len(cite_chunks) >= _CITE_CHUNK_CAP:+ break+ if len(cite_chunks) >= _CITE_CHUNK_CAP:+ break+ cite_chunks = cite_chunks or all_chunks# DESIGN ENHANCEMENT (Go parity: RunResponse.SlotCitations): expose the# slot evidence ids and the citation-pool chunk ids so the rag tool's# post-processing can rewrite unresolvable [ID:Slot N] markers into real@@ -1093,8 +1378,20 @@ def build_agentic_graph(tools.kbinfos = dict(getattr(tools, "kbinfos", None) or state.get("kbinfos") or {"chunks": [], "doc_aggs": []})t = min(_PREFETCH_TIMEOUT_S, max(10.0, _remaining_s(state) - _MIN_ROUND_HEADROOM_S))# First-round prefetch leaves drill slots free (see _DRILL_RESERVE).+ # This is the pre-search stage right after fan-out: besides BM25 + hybrid,+ # run the metadata (title) channel so sub-questions whose target document+ # is identifiable by title reach it directly.+ # The ceiling stays fixed; fairness between the channels is enforced+ # inside _fanout_search by sharing the pool equally (see "pool shares"+ # there), instead of letting channel A's per-query quota eat it all.added = await _bounded(- _fanout_search(tools, queries, capacity=_MAX_SNIPPET_POOL - _DRILL_RESERVE),+ _fanout_search(+ tools,+ queries,+ capacity=_PRESEARCH_MAX_POOL,+ answer_conf=answer_conf,+ use_metadata=True,+ ),t,"programmatic prefetch",)@@ -1484,6 +1781,25 @@ def build_agentic_graph(return g.compile()+_NEGATIVE_CANDIDATES = {"0", "0.0", "zero", "none", "no", "null", "n/a", "na", "无", "没有", "沒有"}+++def _looks_like_negative_candidate(cand) -> bool:+ """True when a slot candidate is a bare zero/negative (e.g. 0, none, no).++ Such values are frequently produced from "no record was retrieved" rather+ than from a positive source. The draft renderer flags them when they carry no+ evidence_ids, so the SCA / finalize treat them as absence-based (unverified)+ instead of as a positively-evidenced value.+ """+ s = str(cand or "").strip().lower()+ if not s:+ return False+ if set(s) <= set("0."): # bare zeros, e.g. "0", "0.0"+ return True+ return s in _NEGATIVE_CANDIDATES++def _render_slot_draft(slot_table, collected_answer: str | None = None, slot_evidence: dict | None = None) -> str:"""Render a slot table into a fact-preserving draft for the SCA.@@ -1528,6 +1844,11 @@ def _render_slot_draft(slot_table, collected_answer: str | None = None, slot_evidetails.append(f"terminal={terminal}")if evidence_ids:details.append(f"evidence_ids={evidence_ids}")+ elif _looks_like_negative_candidate(cand):+ # Absence-based: a bare 0/none that no retrieved passage supports.+ # Flag it so the SCA / finalize treat it as unverified rather than a+ # positively-evidenced value (guards "no record found ⇒ 0").+ details.append("absence-based — unverified, no positive evidence_ids")suffix = " [" + ", ".join(details) + "]" if details else ""lines.append(f"- slot {vid} [{vtype}]: {cand} (strength={strength}){suffix}" + (f" — {tail}" if tail else ""))else:diff --git a/rag/advanced_rag/harness/action_session.py b/rag/advanced_rag/harness/action_session.pyindex 5a1215265c..ed967c0fd9 100644--- a/rag/advanced_rag/harness/action_session.py+++ b/rag/advanced_rag/harness/action_session.py@@ -19,6 +19,7 @@from __future__ import annotationsimport asyncio+import copyimport jsonimport loggingimport re@@ -38,7 +39,7 @@ _LOG = logging.getLogger(__name__)_INIT_TIMEOUT_S = 45.0_ACTION_TIMEOUT_S = 75.0_SNIPPETS_PER_QUERY = 4-_MAX_TOOL_RESPONSE_CHARS = 12000+_MAX_TOOL_RESPONSE_CHARS = 100000# Dataset-level empty results (reason="no_structure") a compiled-structure tool# must accumulate before it is disabled for the rest of the session. Kept above# 1: a single empty can be SCOPED — graph_explore over a doc_scope that has no@@ -406,6 +407,66 @@ _GRAPH_EXPLORE_TOOL_SPEC = {},}+_METADATA_SEARCH_TOOL_SPEC = {+ "type": "function",+ "function": {+ "name": "metadata_search",+ "description": (+ "WHEN TO CALL: PRE-FILTER the document set by title BEFORE retrieving chunks — the question names a document by title or recognizable name; or you need a named document subset; or the corpus is large and a title filter would sharpen recall. Prefer 'contains' with a distinctive substring. "+ "CALL AT MOST ONCE PER DIRECTION: then use search_chunks / retrieve inside those documents. "+ "DO NOT CALL: nothing names a document/subset; you already hold a doc_id (use list_chunks); counting or enumerating. "+ "ARGUMENTS: query — 1-2 strings. filters — [{key, value, op}]; key is only 'title'; op — see enum; logic 'and'|'or'. For the string ops the value MUST be ONE keyword, never a list — one call per keyword; 'in' takes a list; 'empty' takes no value. Example: [{key: 'title', op: 'contains', value: 'New York'}]. Titles use spaces, not underscores. "+ "OUTPUT: ranked chunks from ONLY the matching documents. ok = new evidence; redundant = already seen; miss = nothing matched. "+ "IF IT FAILS: 'no documents match' — shorten the substring, or drop the filter and use search_chunks. Do NOT retry the same filter."+ ),+ "parameters": {+ "type": "object",+ "properties": {+ "query": {+ "type": "array",+ "items": {"type": "string"},+ "minItems": 1,+ "maxItems": 2,+ },+ "filters": {+ "type": "array",+ "minItems": 1,+ "items": {+ "type": "object",+ "properties": {+ # Only `title` is exposed for now. Extending to other+ # metadata fields is a matter of adding to this enum —+ # the executor below is already field-agnostic.+ "key": {+ "type": "string",+ "enum": ["title"],+ "description": "the document title",+ },+ "value": {+ "type": ["string", "array", "null"],+ "description": (+ "The keyword/value to match. For contains/=/start with/end with/not "+ "contains this MUST be a SINGLE string keyword (e.g. 'Bowling'), never "+ "a list — to match several keywords, call metadata_search once per "+ "keyword. 'in'/'not in' MAY be a list of strings; 'empty'/'not empty' "+ "take no value."+ ),+ },+ "op": {+ "type": "string",+ "enum": ["=", "contains", "not contains", "start with", "end with", "in", "empty", "not empty"],+ },+ },+ "required": ["key", "op"],+ },+ },+ "logic": {"type": "string", "enum": ["and", "or"], "default": "and"},+ },+ "required": ["query", "filters"],+ },+ },+}+_TOOL_MAP = {"retrieve": _RETRIEVE_TOOL_SPEC,"search_chunks": _SEARCH_CHUNKS_TOOL_SPEC,@@ -415,6 +476,7 @@ _TOOL_MAP = {"calculate": _CALCULATE_TOOL_SPEC,"graph_explore": _GRAPH_EXPLORE_TOOL_SPEC,"web_search": _WEB_SEARCH_TOOL_SPEC,+ "metadata_search": _METADATA_SEARCH_TOOL_SPEC,}@@ -447,7 +509,12 @@ def _active_tool_specs(tools) -> list:disabled = getattr(tools, "_disabled_tools", None) or set()if disabled:names -= set(disabled)- return [spec for name, spec in _TOOL_MAP.items() if name in names]+ specs = []+ for name, spec in _TOOL_MAP.items():+ if name not in names:+ continue+ specs.append(copy.deepcopy(spec))+ return specsdef _disable_tool(tools, name: str) -> None:@@ -676,16 +743,23 @@ def _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=True) ->return Falseseen.add(cid)ids.append(cid)- # Table chunks pass through UN-truncated: the 1200-char cap hides answer+ # Table chunks pass through UN-truncated: the 5000-char cap hides answer# rows in the mid/late table (Q86: rank-19 row at char 5181 of a 8274-char# standings table was cut, so the session model guessed the athlete). The# pool (kbinfos) already stores the full chunk; only the model-facing# session output was truncated here._ct = _chunk_text(c) or ""if _is_table_chunk(c):- entry = {"id": str(cid), "content": _ct}+ # Table chunks are shown to the model as a Markdown view (key-value for+ # infoboxes, a full-row pipe table for ranked/result tables) instead of+ # raw <table> markup: same rows, a fraction of the tokens, and the format+ # comparison over 11 serializations ranks Markdown-KV/Markdown above raw+ # HTML. The shared pool keeps the RAW chunk for citation and re-reads.+ from rag.advanced_rag.harness.tools.table_view import table_view_or_raw++ entry = {"id": str(cid), "content": table_view_or_raw(_ct)}else:- entry = {"id": str(cid), "content": _ct[:1200]}+ entry = {"id": str(cid), "content": _ct[:5000]}if include_doc_id:entry["doc_id"] = _doc_id(c)out.append(entry)@@ -770,9 +844,16 @@ async def _run_search(tools, search_fn, queries: list, top_n: int, max_q: int, *only differences are the search backend (grep vs hybrid), ``top_n``, theper-call query cap — all passed in. ``kw`` forwards backend kwargs such as``use_compiled`` (search_chunks' compiled-structure expansion).++ ``fail_hard`` (default False) controls exception handling: most tools want a+ soft per-query skip so a single backend blip does not kill the whole turn;+ ``metadata_search`` passes True so a real retrieval failure surfaces as+ ``ERROR/infra`` instead of being silently downgraded to ``MISS/no_doc``+ (which would mislead the model into thinking no document matched)."""from rag.advanced_rag.harness.tools.search import _chunk_id+ fail_hard = kw.pop("fail_hard", False)kb_ids = _kb_ids(tools)out, ids = [], []seen = set()@@ -794,7 +875,9 @@ async def _run_search(tools, search_fn, queries: list, top_n: int, max_q: int, *try:res = await search_fn(tools, fq, kb_ids=kb_ids, top_n=top_n, **kw)cands = res.get("chunks", []) or []- except Exception: # noqa: BLE001+ except Exception:+ if fail_hard:+ raise_LOG.warning("[Action Session] %s failed for %r", getattr(search_fn, "__name__", "search"), fq, exc_info=True)continuefor c in cands[:_SNIPPETS_PER_QUERY]:@@ -901,6 +984,183 @@ async def _exec_web_search(tools, queries: list) -> ToolOutcome:return _search_outcome(out, ids, new_ev)+def _normalize_metadata_value(value, op):+ """Coerce a model-supplied filter value to the single value the executor+ expects. String ops (contains, =, start with, end with, not contains) take+ ONE keyword; if a list sneaks in, collapse it to the first non-empty element+ (one metadata_search call = one keyword) and log it. 'in'/'not in' keep+ their list (the value set), and empty/not empty take no value."""+ if op in ("in", "not in"):+ return value+ if isinstance(value, list):+ flat = [str(v) for v in value if v not in (None, "")]+ if not flat:+ return None+ chosen = flat[0]+ if len(flat) > 1:+ _LOG.info(+ "[metadata search] value list collapsed to single keyword %r (ignored: %s); to match all, call metadata_search once per keyword",+ chosen,+ flat[1:],+ )+ return chosen+ return value+++async def _exec_metadata_search(tools, args: dict) -> ToolOutcome:+ """Metadata-filtered hybrid retrieval (metadata_search tool).++ Pipeline: validate the requested metadata keys against the dataset's real+ fields -> resolve matching doc_ids via ES/Infinity push-down (in-memory+ ``meta_filter`` fallback) -> hybrid_search scoped to exactly those+ documents.++ The executor is field-agnostic: which keys are reachable is decided by the+ ``key`` enum in :data:`_METADATA_SEARCH_TOOL_SPEC` (currently ``title``+ only), so exposing another metadata field later needs no change here.+ """+ from api.db.services.doc_metadata_service import DocMetadataService+ from common.metadata_utils import meta_filter+ from common.misc_utils import thread_pool_exec+ from rag.advanced_rag.harness.tools.search import hybrid_search++ queries = _arg_query_list(args, 2)+ filters = args.get("filters") or []+ logic = str(args.get("logic") or "and")++ if not queries:+ _LOG.info("[metadata search] skipped — no query provided")+ return ToolOutcome(payload=[], status=ERROR, reason="bad_args", metrics={"hits": 0, "new_evidence": 0})+ if not filters:+ _LOG.info("[metadata search] skipped — no metadata filters supplied (need ≥1 {key, op, value})")+ return ToolOutcome(+ payload=[+ {+ "kind": "metadata_search",+ "note": "No metadata conditions given. metadata_search needs at least one {key, value, op} filter — otherwise use search_chunks / retrieve.",+ }+ ],+ status=MISS,+ reason="bad_args",+ metrics={"hits": 0, "new_evidence": 0},+ )++ kb_ids = _kb_ids(tools)+ if not kb_ids:+ return ToolOutcome(payload=[], status=ERROR, reason="infra", metrics={"hits": 0, "new_evidence": 0})++ # 1) Key validation against the dataset's REAL metadata fields. A dataset+ # without the requested key (e.g. no title) must degrade to a hint, not+ # to an empty retrieval the model would retry forever.+ try:+ known_keys = await thread_pool_exec(DocMetadataService.get_metadata_keys_by_kbs, kb_ids)+ except Exception: # noqa: BLE001+ _LOG.warning("[metadata search] get_metadata_keys_by_kbs failed", exc_info=True)+ known_keys = []+ known_set = {str(k) for k in (known_keys or [])}+ bad = [str(f.get("key")) for f in filters if str(f.get("key")) not in known_set]+ if bad:+ _LOG.info(+ "[metadata search] bad key(s) %s — not in dataset metadata (available: %s)",+ bad,+ sorted(known_set)[:30],+ )+ return ToolOutcome(+ payload=[+ {+ "kind": "metadata_search",+ "note": f"Metadata key(s) {bad} do not exist in this dataset. Available: {sorted(known_set)[:30] or 'NONE — this dataset has no metadata, use search_chunks / retrieve'}.",+ }+ ],+ status=MISS,+ reason="bad_args",+ metrics={"hits": 0, "new_evidence": 0},+ )++ # Normalize filter values BEFORE they reach meta_filter / ES push-down, so a+ # model that mistakenly emits a list where one keyword is expected can never+ # silently produce a no_doc TypeError again. 'in'/'not in' keep their list+ # semantics; string ops collapse a list to its first keyword (one call = one+ # keyword) and log it so the model can re-call per keyword if needed.+ normalized_filters = [{**f, "value": _normalize_metadata_value(f.get("value"), f.get("op"))} for f in filters]++ # 2) Resolve matching doc_ids: ES/Infinity push-down, in-memory fallback.+ try:+ doc_ids = await thread_pool_exec(DocMetadataService.filter_doc_ids_by_meta_pushdown, kb_ids, normalized_filters, logic)+ except Exception: # noqa: BLE001+ _LOG.warning("[metadata search] push-down failed", exc_info=True)+ doc_ids = None+ if doc_ids is None:+ try:+ metas = await thread_pool_exec(DocMetadataService.get_flatted_meta_by_kbs, kb_ids)+ doc_ids = meta_filter(metas, normalized_filters, logic) or []+ except Exception: # noqa: BLE001+ _LOG.warning("[metadata search] in-memory fallback failed", exc_info=True)+ doc_ids = []+ doc_ids = [str(d) for d in (doc_ids or [])]++ if not doc_ids:+ _LOG.info("[metadata search] no documents matched filters=%s logic=%s", normalized_filters, logic)+ return ToolOutcome(+ payload=[+ {+ "kind": "metadata_search",+ "note": "No documents match the given metadata conditions. Loosen or change the "+ "filters (try a shorter substring, or the space form instead of underscores), "+ "or use search_chunks / retrieve for unfiltered search.",+ }+ ],+ status=MISS,+ reason="no_doc",+ metrics={"hits": 0, "new_evidence": 0},+ )++ # 3) Intersect with the session's global document scope.+ if hasattr(tools, "scoped_doc_ids"):+ doc_ids = tools.scoped_doc_ids(doc_ids) or []+ if not doc_ids:+ _LOG.info("[metadata search] 0 documents after doc-scope intersection")+ return ToolOutcome(payload=[], status=MISS, reason="no_doc", metrics={"hits": 0, "new_evidence": 0})++ # 4) Hybrid search restricted to exactly those documents.+ # use_compiled=False: compiled expansion would pull in out-of-scope chunks+ # and break the "only these documents" contract.+ # fail_hard=True: a genuine retrieval failure (embedding/infra) must surface+ # as ERROR/infra, NOT as MISS/no_doc (which would wrongly tell the model no+ # document matched). An empty-but-valid result still returns MISS correctly.+ try:+ out, ids, new_ev = await _run_search(+ tools,+ hybrid_search,+ queries,+ top_n=20,+ max_q=2,+ doc_scope=doc_ids,+ use_compiled=False,+ fail_hard=True,+ )+ except Exception: # noqa: BLE001+ _LOG.warning("[metadata search] retrieval failed", exc_info=True)+ return ToolOutcome(+ payload=[+ {+ "kind": "metadata_search",+ "note": "Retrieval backend failed (embedding/infra). Retry later or fall back to search_chunks / retrieve for unfiltered search.",+ }+ ],+ status=ERROR,+ reason="infra",+ metrics={"hits": 0, "new_evidence": 0},+ )+ _LOG.info(+ "[metadata search] ok — %d doc(s) matched by filters, %d hit(s), %d new evidence",+ len(doc_ids),+ len(out),+ new_ev,+ )+ return _search_outcome(out, ids, new_ev)++async def _exec_list_chunks(tools, doc_id: str) -> ToolOutcome:"""Deep-read one document's full text (list_chunks tool).@@ -908,7 +1168,7 @@ async def _exec_list_chunks(tools, doc_id: str) -> ToolOutcome:QUERY-level miss (``miss``), never a dataset fact, so ``_tool_node`` must notdisable the tool over it."""- from rag.advanced_rag.harness.tools.search import _chunk_id, list_chunks+ from rag.advanced_rag.harness.tools.search import _LIST_CHUNKS_MAX_CHUNKS, _chunk_id, _chunk_text, list_chunkstry:res = await list_chunks(tools, doc_id)@@ -919,7 +1179,29 @@ async def _exec_list_chunks(tools, doc_id: str) -> ToolOutcome:kbinfos = _seed_evidence(tools)kb_seen = {_chunk_id(c) for c in kbinfos["chunks"] if isinstance(c, dict)}seen = set()- for c in (res.get("chunks") or [])[:30]:+ doc_chunks = [c for c in (res.get("chunks") or []) if _chunk_id(c)]+ # Section index (generic, not per-question): deep-reading a long document only+ # helps if the model can see WHICH sections it holds. A 39-chunk article+ # otherwise arrives as one undifferentiated wall of text and the answer-bearing+ # section is what gets skipped (2026-09-16 FRAMES attribution: the right+ # document was retrieved and read, the model still answered from the wrong+ # section). The index lists every section heading + chunk id + size, so the+ # model can target a section with a scoped search instead of re-reading all of+ # it; it is metadata, not evidence, so it is not added to the citation pool.+ index_lines = []+ for i, c in enumerate(doc_chunks, 1):+ txt = _chunk_text(c) or ""+ head = next((ln.strip() for ln in txt.splitlines() if ln.strip()), "")+ index_lines.append(f"{i}. [{_chunk_id(c)}] {head[:120]} ({len(txt)} chars)")+ if index_lines:+ out.append(+ {+ "kind": "doc_index",+ "doc_id": str(doc_id),+ "content": "Sections of this document — read the ones the question needs:\n" + "\n".join(index_lines),+ }+ )+ for c in doc_chunks[:_LIST_CHUNKS_MAX_CHUNKS]:cid = _chunk_id(c)if not cid:continue@@ -1157,6 +1439,8 @@ async def execute_tool(tools, name: str, args: dict) -> ToolOutcome:return await _exec_graph_explore(tools, args)if name == "web_search":return await _exec_web_search(tools, _arg_query_list(args, 2))+ if name == "metadata_search":+ return await _exec_metadata_search(tools, args)_LOG.warning("[Action Session] unknown tool %r; ignored.", name)return ToolOutcome(payload=[], status=ERROR, reason="bad_args")@@ -1183,6 +1467,8 @@ async def _acompletion(mdl, messages: list, tools_list=None, temperature: floatkwargs = {"model": mdl.model_name, "messages": oai_messages, "temperature": temperature}if tools_list:kwargs["tools"] = tools_list+ if timeout_s:+ kwargs["timeout"] = timeout_sresponse = await create(**kwargs)from rag.advanced_rag.harness.stats import record_external_response@@ -1245,7 +1531,7 @@ def _parse_tool_calls(msg) -> list:# Do NOT drop it. OpenAI's protocol requires every assistant# tool_call to be answered by a matching ``tool`` message — dropping# one leaves a dangling tool_call and the next request is rejected- # ("tool call result does not follow tool call"). Keep it as a+ # ("tool call result does not follow tool call"). Keep it as an# "unknown" call so _tool_node replies with a correction instead.## Models most often emit the XML protocol tags (state / answer) as@@ -1319,6 +1605,8 @@ class _SessionState(TypedDict, total=False):_direction: str # this slot's question — needed to re-run retrieval later_routed_docs: list # navigate_tree's top-n: scope for the `scoped` rung_nav_rule_id: str # which rung control currently rests on ("" = finished)+ _search_queries: list+ _skipped_dup: intasync def _run_action_node(state: _SessionState) -> dict:@@ -1413,6 +1701,7 @@ async def _tool_node(state: _SessionState) -> dict:# makes the model re-issue redundant retrieves. The same-session cache only# avoids RE-EXECUTING a repeated query, it still returns a response for it.pending = state.get("_pending_calls") or []+ ms_used = bool(state.get("_metadata_search_used", False))seen_queries = list(state.get("_search_queries") or [])skipped = 0strikes = dict(state.get("_tool_strikes") or {})@@ -1422,7 +1711,7 @@ async def _tool_node(state: _SessionState) -> dict:# tool_call that comes in — a call only advances its own rung, and a later# ladder continuation must continue from the SAME resting point.pending_rule = state.get("_nav_rule_id", "")- for c in pending:+ for batch_index, c in enumerate(pending):# Near-duplicate retrieval suppression: if the model re-issues the same# intent as an earlier search (paraphrase), do NOT re-run ES — return a# nudge so it patches / reframes instead of burning turns (Q30 burned@@ -1449,6 +1738,27 @@ async def _tool_node(state: _SessionState) -> dict:})continue+ # metadata_search ONE-SHOT guard: block any 2nd call within this direction.+ if c["name"] == "metadata_search" and ms_used:+ _LOG.info("[Action Session] blocking 2nd metadata_search this direction (one-shot guard)")+ tool_msgs.append(+ {+ "role": "tool",+ "tool_call_id": c["id"],+ "content": json.dumps(+ {+ "passages": [+ {+ "kind": "metadata_search",+ "note": "metadata_search is a ONE-SHOT pre-filter and was ALREADY used this direction. Continue with search_chunks / retrieve inside the documents it returned — do NOT call metadata_search again this direction.",+ }+ ]+ },+ ensure_ascii=False,+ ),+ }+ )+ continue# Unknown tool name — never execute it; answer with a correction so the# model can recover. Models usually emit the XML protocol tags (state /# answer) as tool names; they belong in the reply body as plain text.@@ -1477,6 +1787,8 @@ async def _tool_node(state: _SessionState) -> dict:if q:seen_queries.append(q)evidence_ids.extend(oc.evidence_ids)+ if c["name"] == "metadata_search":+ ms_used = Truechunks = list(oc.payload or [])# ── Policy: act on WHAT happened, not just on payload size ──────────if oc.status == OK:@@ -1575,6 +1887,7 @@ async def _tool_node(state: _SessionState) -> dict:"_routed_docs": state.get("_routed_docs") or [],"_direction": state.get("_direction", ""),"_nav_rule_id": pending_rule,+ "_metadata_search_used": ms_used,}@@ -2084,6 +2397,32 @@ def _extract_relevant_evidence(tools, direction: str, max_chunks: int = 4) -> streturn "\n".join(lines)+def _dump_pool_evidence(tools) -> str:+ """TEST-ONLY: full dump of the shared evidence pool for the session seed.++ Counterpart to :func:`_extract_relevant_evidence`: no top-k cap and no+ 300-char truncation — every chunk currently in ``tools.kbinfos["chunks"]``+ is injected verbatim so the action session sees ALL prior evidence.+ """+ from rag.advanced_rag.harness.tools.search import _chunk_id, _chunk_text++ kbinfos = getattr(tools, "kbinfos", None) or {}+ chunks = kbinfos.get("chunks") or []+ lines = []+ for c in chunks:+ cid = _chunk_id(c)+ text = (_chunk_text(c) or "").replace("\n", " ")+ lines.append(f"[{cid}] {text}")+ dump = "\n".join(lines)+ _LOG.info(+ "[Action Session] seed ALREADY RETRIEVED dump: %d chunk(s), %d chars (kbinfos pool=%d)",+ len(lines),+ len(dump),+ len(chunks),+ )+ return dump++async def run_action_session(tools,direction: str,@@ -2099,7 +2438,11 @@ async def run_action_session(system = load_prompt("action_run")seed_user = f"Direction: {direction}\n\nState:\n{parent_state.render_slots()}"- existing = _extract_relevant_evidence(tools, direction, max_chunks=4)+ # TEST: `_extract_relevant_evidence` (top-4 / 300-char relevance digest) is+ # DISABLED. Instead inject the FULL shared evidence pool so the action session+ # sees every passage already retrieved this round (and prior rounds).+ # existing = _extract_relevant_evidence(tools, direction, max_chunks=4)+ existing = _dump_pool_evidence(tools)if existing:seed_user += "\n\nALREADY RETRIEVED (do NOT re-retrieve these — use them to fill slots or identify gaps):\n" + existing@@ -2234,7 +2577,7 @@ async def initialize_state(tools, question, fanout_hint, deadline_left=None):record_external_call(None)except Exception: # noqa: BLE001- pass+ _LOG.debug("[Action Session] slot-table decomposition usage booking skipped", exc_info=True)raw = await _init_chat(tools, system, user, tmo)_book_raw_call()diff --git a/rag/advanced_rag/harness/config.py b/rag/advanced_rag/harness/config.pyindex 4d906a7e85..71ed68f8ba 100644--- a/rag/advanced_rag/harness/config.py+++ b/rag/advanced_rag/harness/config.py@@ -30,6 +30,7 @@ from dataclasses import dataclass, field_ALL_TOOLS = ("retrieve","search_chunks",+ "metadata_search","list_chunks","navigate_tree","navigate_structure",diff --git a/rag/advanced_rag/harness/grep_sed_narrow.py b/rag/advanced_rag/harness/grep_sed_narrow.pyindex 410eb83f46..9fd34f0c41 100644--- a/rag/advanced_rag/harness/grep_sed_narrow.py+++ b/rag/advanced_rag/harness/grep_sed_narrow.py@@ -20,6 +20,16 @@ Fallback chain (never drops the answer):-> original chunks returned as-is (upper _build_compact_evidencehead-truncation as the final safety valve)+Table exemption: chunks that look like tables (HTML ``<table>``/``<tr>`` markup, or+>=3 pipe rows — see ``search._is_table_chunk``) are NEVER narrowed, by any entry+point of this module. Two reasons, both measured: the term window either cuts the+``<table>`` opening tag, and then the downstream ``table_view`` renderer refuses+the fragment (``"<table" not in text``) so the model is handed a partial raw-HTML+row dump; or it keeps the tag and silently drops the remaining rows, and row+order/coverage is exactly what decides table answers (a rank row can sit at 62% of+a 14.7K-char table). Tables are returned VERBATIM — never re-windowed, never+char-trimmed, and exempt from the per-chunk/total char budget.+Safety: only ``re.compile`` + pure string ops, no eval, no arbitrary code;grep-term count cap + context clamp."""@@ -28,16 +38,17 @@ import loggingimport refrom rag.advanced_rag.harness.tools.search import (- _split_sentences,_is_fact_dense_sentence,+ _is_table_chunk,_narrow_by_keywords,+ _split_sentences,)_LOG = logging.getLogger(__name__)# Cost / safety caps._MAX_GREP_TERMS = 16-_MAX_CONTEXT = 2+_MAX_CONTEXT = 6_DEFAULT_OUT_CHARS_PER_CHUNK = 1200_DEFAULT_OUT_TOTAL_CHARS = 16000# Head length kept per chunk when there is no match.@@ -51,6 +62,19 @@ _CONTEXT_CHAR_BUDGET = 600_MIN_NARROW_CHARS = 200+def _safe_is_table(chunk) -> bool:+ """Table detector guard: detection must never break narrowing.++ Centralises the whole-table exemption (see the module docstring) so every+ caller of this engine inherits it, instead of each call site re-implementing+ the ``table_chunks / prose_chunks`` split.+ """+ try:+ return bool(_is_table_chunk(chunk))+ except Exception: # noqa: BLE001 - never let a detector bug drop a chunk+ return False++def _escape_term(term: str) -> str:"""Escape a plain grep term into a safe, word-boundary regex fragment.@@ -213,7 +237,8 @@ def _apply_narrow(chunks: list[dict], kept_texts: list[str], matched: list[bool]def _fallback_narrow_by_keywords(chunks: list[dict], keywords: str) -> list[dict]:try:return _narrow_by_keywords(chunks, keywords) or chunks- except Exception:+ except Exception: # noqa: BLE001 - the fallback must never break narrowing+ _LOG.debug("[grep-sed] keyword fallback narrowing failed; keeping the chunks as-is", exc_info=True)return chunks@@ -234,6 +259,10 @@ def narrow_by_terms(once mechanically (zero extra LLM). Still no hit -> narrowing is abandoned andthe original chunks are returned (matched=False); the caller must NOT treat afailed narrow as an answer failure. Never raises.++ Table chunks are never narrowed and never char-trimmed — see the module+ docstring. They come back VERBATIM (as a shallow copy), so no caller needs its+ own ``_is_table_chunk`` split any more."""ctx = context or {"before": 0, "after": 0}try:@@ -262,14 +291,19 @@ def narrow_by_terms(stats["chars_out"] = sum(len(_chunk_text(c)) for c in narrowed)return {"kept": narrowed, "stats": stats}+ # Whole-table exemption, computed once and reused by the char-budget pass+ # below (``_apply_narrow`` preserves order and length, so the two lists stay+ # aligned). ``False`` = "this chunk was NOT narrowed" -> returned verbatim.+ table_flags = [_safe_is_table(c) for c in chunks]+def _run(active_patterns) -> tuple[list[str], list[bool]]:texts: list[str] = []flags: list[bool] = []- for c in chunks:+ for c, is_table in zip(chunks, table_flags):raw = _chunk_text(c)- if len(raw) <= _MIN_NARROW_CHARS:+ if is_table or len(raw) <= _MIN_NARROW_CHARS:texts.append(raw)- flags.append(True)+ flags.append(not is_table)continuetext, ok = _exec_on_text(raw, active_patterns, context, max_out_chars_per_chunk)texts.append(text)@@ -306,7 +340,17 @@ def narrow_by_terms(per_chunk_cap = max(200, min(max_out_chars_per_chunk, max_out_total_chars // max(1, len(kept))))acc = 0trimmed = []- for c in kept:+ for c, is_table in zip(kept, table_flags):+ if is_table:+ # Tables are indivisible and exempt from the char budget: a+ # head slice keeps the header and the first rows and drops+ # the answer row, and the dropped rows carry no marker, so+ # the model reads a truncated table as a complete one. The+ # budget therefore bounds PROSE only; a large table can push+ # the narrowed prose set past ``max_out_total_chars``, which+ # is the intended trade (readability/correctness > char cap).+ trimmed.append(c)+ continuet = _chunk_text(c)room = max_out_total_chars - accif room <= 0:diff --git a/rag/advanced_rag/harness/orchestrator/sufficient_context.py b/rag/advanced_rag/harness/orchestrator/sufficient_context.pyindex 6aaf3dc105..5cc093cf5b 100644--- a/rag/advanced_rag/harness/orchestrator/sufficient_context.py+++ b/rag/advanced_rag/harness/orchestrator/sufficient_context.py@@ -59,9 +59,10 @@ _SCA_CLAIMS_CONTEXT_MAX = 48000# the SCA can verify a draft against real retrieved text without a token blow-up._SCA_EVIDENCE_ANCHOR_CHARS = 300# Table-structured chunks get their FULL text as the evidence anchor (bounded-# only by _SCA_CLAIMS_CONTEXT_MAX): hint-token windowing is unreliable for-# tables (the draft rarely contains the row's entity names), and truncating from-# the head hides the answer rows that sit mid/late-table (Q86 rank-19 row).+# only by _SCA_CLAIMS_CONTEXT_MAX), serialized to Markdown by _bounded_excerpt:+# hint-token windowing is unreliable for tables (the draft rarely contains the+# row's entity names), and truncating from the head hides the answer rows that+# sit mid/late-table (Q86 rank-19 row)._SCA_EVIDENCE_TABLE_CHARS = None # None = keep the whole chunk text@@ -141,12 +142,24 @@ def _bounded_excerpt(text: str, hints: str, max_chars: int = 300) -> str:overall budget): hint-token windowing fails for tables because the draftrarely contains the row's entity names, and head-truncation hides answerrows in the mid/late table. Plain text keeps the bounded window.++ A table is also SERIALIZED to a Markdown view before it is handed over, using+ the same renderer as the action-session view and the final answer path. The+ SCA is a plain LLM call: raw ``<table>/<td>`` markup makes it parse cells+ itself (the gap Q673 showed on the answer path), and the same table costs+ ~61% as much in HTML as it does in Markdown. ``table_view_or_raw`` never+ prunes rows (a row can carry the answer mid-table), so the SCA still sees the+ complete row set — just in a readable form. No-op for non-tables."""text = str(text or "").strip()if not text:return ""if _is_table_text(text):- return text+ # Imported lazily so this module keeps no import-time dependency on the+ # tools package (the other three render call sites do the same).+ from rag.advanced_rag.harness.tools.table_view import table_view_or_raw++ return table_view_or_raw(text)max_chars = max(80, int(max_chars))hint_tokens = [t for t in re.findall(r"[A-Za-z0-9_\u4e00-\u9fff]{3,}", str(hints or ""))]lower = text.lower()@@ -197,7 +210,8 @@ def _render_claim_context(claims, question: str = "", kbinfos: dict | None = Noncontinueblock = f"Claim {cid} (draft):\n{rpt}"# Evidence anchor: first line of each cited snippet (guards the draft).- # Table chunks contribute their FULL text (see _bounded_excerpt).+ # Table chunks contribute their FULL text as a Markdown view (see+ # _bounded_excerpt).if eids:anchors: list[str] = []for eid in eids:diff --git a/rag/advanced_rag/harness/prompts/report_prompt.py b/rag/advanced_rag/harness/prompts/report_prompt.pyindex 11a2ee0f90..58887bca6c 100644--- a/rag/advanced_rag/harness/prompts/report_prompt.py+++ b/rag/advanced_rag/harness/prompts/report_prompt.py@@ -2,6 +2,13 @@FINAL_ANSWER_SYSTEM = """You are a smart agent. Answer the user's question using ONLY the evidence provided below. Do not invent facts: if the evidence cannot support a claim, say so plainly instead of guessing.+# Commitment (CRITICAL)+You MUST commit to the best-supported answer. The evidence does NOT need to prove it exhaustively.+If the Research Summary gives a Candidate answer (e.g. "slot 0 [count]: 2") and no evidence+contradicts it, that candidate IS the answer — state it directly.+Give the answer FIRST, then add a short caveat ("most likely", "based on the available evidence").+Never replace a supported answer with a refusal.+# Answer targetFirst resolve the exact role requested by the user's question. Multi-hop questionsoften mention bridge entities that are only clues. Do not answer with a bridge@@ -22,11 +29,35 @@ Answer the question's own attribute using the evidence for THAT attribute. If thsupports a different attribute, say that you could only find the related (different) attribute anddo not present it as the answer to the requested one.+# Answer verification (ALWAYS apply, no exceptions, before writing the answer)+These are the failure modes that survive retrieval: the evidence IS in context and the+answer still comes out wrong. Apply all five on every question.+1. COUNTS / ENUMERATIONS: build the explicit item list FIRST, then report the count as+ the LENGTH of that list. Never restate a count from the Research Summary from memory,+ and never "correct" it by adding an item that is not in your list. If your list and+ the Research Summary disagree, the length of YOUR list wins.+2. "WHICH X SATISFIES A AND B": write a candidate x constraint check (one row per+ candidate, one column per constraint the question states) and answer ONLY the+ candidate that satisfies EVERY constraint. A candidate that fails one constraint is+ out, even if it is the first or most prominent one.+3. NUMBERS / RATIOS / DIFFERENCES: convert to the SAME unit first (m vs ft, km vs mi,+ date vs year), state the two values you combine and the operation, then compute.+ Never divide or compare across units, and never inject a figure the evidence does not+ contain.+4. CONFLICTING EVIDENCE: when two passages give different values for the same fact, pick+ the one that satisfies the question's own constraint (the stated quantity, date or+ entity), cite it, and say briefly which alternative you rejected.+5. NO RESTATEMENT DRIFT: a number, date or entity in your answer that the evidence+ passages below do not support must be re-derived from them or dropped — do not repeat+ it just because the Research Summary said it.+# LanguageAnswer in the SAME language as the question. Translate retrieved evidence into that language as part of composing the answer; only verbatim quoted snippets may stay in their source language.# Fallback-If the evidence does not answer the question, reply with a clear statement that you don't have enough information based on the available sources (in the user's language).+Only if the evidence is ENTIRELY unrelated to the question may you say you don't have enough+information. If the evidence is related but incomplete, still give the best-supported answer+(with a brief caveat) rather than declining."""diff --git a/rag/advanced_rag/harness/tools/search.py b/rag/advanced_rag/harness/tools/search.pyindex 16a947f090..7921bd0a55 100644--- a/rag/advanced_rag/harness/tools/search.py+++ b/rag/advanced_rag/harness/tools/search.py@@ -8,6 +8,7 @@ module owns the retrieval implementations they call.import loggingimport refrom typing import Any+from common import settingsfrom rag.advanced_rag.harness.chunk_utils import ( # noqa: F401_chunk_attr,@@ -44,7 +45,6 @@ from rag.advanced_rag.harness.tools.text_processing import ( # noqa: F401_stem,)-# Fallbacks for callers that supply no retrieval configuration: the values the# search tools used unconditionally before RAGTools carried settings._DEFAULT_SIMILARITY_THRESHOLD = 0.2@@ -181,7 +181,7 @@ async def hybrid_search(from rag.advanced_rag.harness.memory import add as _memory_add_memory_add(tools, kbinfos.get("chunks", []) or [])- except Exception:+ except Exception: # noqa: BLE001, S110pass # memory is best-effort; never fail the search over it.# Narrow-or-keep: if the query keywords match any chunk, keep only the# matching passages (shrinking the evidence handed to the LLM so a single@@ -243,7 +243,7 @@ async def vector_search(tools, query: str, kb_ids: list[str] | None = None, top_from rag.advanced_rag.harness.memory import add as _memory_add_memory_add(tools, kbinfos.get("chunks", []) or [])- except Exception:+ except Exception: # noqa: BLE001, S110passkbinfos["chunks"] = _narrow_or_keep(kbinfos.get("chunks", []), keywords, "Vector search")return kbinfos@@ -281,12 +281,80 @@ async def bm25_search(tools, query: str, kb_ids: list[str] | None = None, top_n:from rag.advanced_rag.harness.memory import add as _memory_add_memory_add(tools, kbinfos.get("chunks", []) or [])- except Exception:+ except Exception: # noqa: BLE001, S110passkbinfos["chunks"] = _narrow_or_keep(kbinfos.get("chunks", []), keywords, "BM25 search")return kbinfos+async def metadata_search(+ tools,+ query: str,+ filters: list[dict],+ logic: str = "and",+ kb_ids: list[str] | None = None,+ top_n: int | None = None,+ keywords: str = "",+ doc_scope: list[str] | None = None,+) -> dict:+ """Hybrid retrieval restricted to documents whose metadata matches ``filters``.++ Programmatic counterpart of the ``metadata_search`` tool (the tool's dispatch+ lives in ``action_session``; this is the retrieval leg the pre-search stage+ reuses). Pipeline: resolve matching doc_ids via the metadata-index push-down+ (ES / Infinity), fall back to the in-memory ``meta_filter`` when push-down is+ not viable -> intersect with the session document scope -> ``hybrid_search``+ scoped to exactly those documents.++ ``filters`` is a list of ``{key, value, op}`` conditions (e.g.+ ``[{"key": "title", "op": "contains", "value": "New York"}]``). An empty+ match (or no dataset metadata) is a normal empty result, never an exception.+ """+ top_n = _resolve_top_n(tools, top_n)+ target_ids = kb_ids or list(dict.fromkeys(tools.kb_ids + [kb.id for kb in tools.sql_kbs]))+ if not target_ids or not filters:+ return {"chunks": [], "doc_aggs": []}++ from api.db.services.doc_metadata_service import DocMetadataService+ from common.metadata_utils import meta_filter+ from common.misc_utils import thread_pool_exec++ # 1) Resolve matching doc_ids: metadata-index push-down, in-memory fallback.+ try:+ doc_ids = await thread_pool_exec(DocMetadataService.filter_doc_ids_by_meta_pushdown, target_ids, filters, logic)+ except Exception: # noqa: BLE001+ _LOG.warning("[Metadata search] push-down failed; falling back to in-memory meta_filter", exc_info=True)+ doc_ids = None+ if doc_ids is None:+ try:+ metas = await thread_pool_exec(DocMetadataService.get_flatted_meta_by_kbs, target_ids)+ doc_ids = meta_filter(metas, filters, logic) or []+ except Exception: # noqa: BLE001+ _LOG.warning("[Metadata search] in-memory fallback failed", exc_info=True)+ doc_ids = []+ doc_ids = [str(d) for d in (doc_ids or [])]++ # 2) Intersect with the session's global document scope (if any).+ if hasattr(tools, "scoped_doc_ids"):+ doc_ids = tools.scoped_doc_ids(doc_ids) or []+ if not doc_ids:+ _LOG.info("[Metadata search] no documents matched filters=%s logic=%s", filters, logic)+ return {"chunks": [], "doc_aggs": []}++ # 3) Hybrid search restricted to exactly those documents (compiled expansion+ # would pull in out-of-scope chunks and break the "only these docs" contract).+ _LOG.info('[Metadata search] "%s" matched %d doc(s) via filters=%s', str(query)[:80], len(doc_ids), filters)+ return await hybrid_search(+ tools,+ query,+ kb_ids=target_ids,+ top_n=top_n,+ doc_scope=doc_ids,+ keywords=keywords,+ use_compiled=False,+ )++# Compiled-product expansion lives in ``compiled_expansion`` and is# re-exported here: navigation and the dynamic runner import these names# from this module.@@ -311,7 +379,7 @@ async def web_search(tools, query: str, keywords: str = "", retrieval_query: streffective_query = f"{query} {retrieval_query}".strip()[:400] if retrieval_query else f"{query} {keywords}".strip() if keywords else queryweb_res = await thread_pool_exec(tools.web_search.retrieve_chunks, effective_query)return {"chunks": web_res.get("chunks", []), "doc_aggs": web_res.get("doc_aggs", [])}- except Exception:+ except Exception: # noqa: BLE001_LOG.exception("web_search failed")return {"chunks": [], "doc_aggs": []}@@ -335,7 +403,7 @@ async def structured_query(tools, query: str, keywords: str = "", kb_ids: list[ssql_kb_ids = [kb.id for kb in sql_kbs]try:ans = await use_sql(query, tools.field_map, tenant_id, tools.chat_mdl, quota=True, kb_ids=sql_kb_ids, doc_ids=doc_scope)- except Exception:+ except Exception: # noqa: BLE001_LOG.exception("structured_query failed")return {"answer": "", "chunks": [], "doc_aggs": []}if not ans:@@ -353,8 +421,12 @@ async def structured_query(tools, query: str, keywords: str = "", kb_ids: list[s# ─── do exact keyword/pattern locate + full-document deep-read like dynamic. ───_GREP_TERMS_MAX = 10-_GREP_OUT_CHARS_PER_CHUNK = 700-_GREP_OUT_TOTAL_CHARS = 8000+# Per-chunk window for a grep hit. 700 chars was measured to hide mid-section+# answer rows (a rank row at 62% of a 14.7K-char table, a "her father was an ice+# hockey player" clause in an Early-life section); the window is now centred on+# the matched line (see the narrow call below) and doubled.+_GREP_OUT_CHARS_PER_CHUNK = 1500+_GREP_OUT_TOTAL_CHARS = 12000_LIST_CHUNKS_MAX_CHUNKS = 80@@ -443,7 +515,10 @@ async def grep_search(prose_chunks,terms,keywords=str(query).strip(),- context={"before": 1, "after": 0},+ # Centre the window on the matched line instead of showing the+ # match alone: the fact that answers the question usually sits+ # one or two lines around the grep hit, not on it.+ context={"before": 2, "after": 3},max_out_chars_per_chunk=_GREP_OUT_CHARS_PER_CHUNK,max_out_total_chars=_GREP_OUT_TOTAL_CHARS,)@@ -459,7 +534,7 @@ async def grep_search(sum(len(str(c.get("content_with_weight") or c.get("content") or "")) for c in kept) / 1000.0,)res["chunks"] = kept- except Exception:+ except Exception: # noqa: BLE001_LOG.exception("[Grep search] narrow failed; using raw BM25 candidates.")_g = res.get("chunks") or []if _g:@@ -492,7 +567,7 @@ async def list_chunks(tools, doc_id: str) -> dict:_LOG.info("[List chunks] Deep-reading document %s", doc_id)try:full = await tools.fetch_full_document(str(doc_id).strip())- except Exception:+ except Exception: # noqa: BLE001_LOG.exception("[List chunks] fetch_full_document failed doc=%s", doc_id)return {"chunks": [], "doc_aggs": []}chunks = (full.get("chunks") or [])[:_LIST_CHUNKS_MAX_CHUNKS]@@ -529,7 +604,7 @@ async def _load_specific_chunks(tools_slot, chunk_ids: list[str], doc_scope: lisfor doc_id in doc_scope[:8]:try:full = await tools_slot.fetch_full_document(doc_id)- except Exception:+ except Exception: # noqa: BLE001_LOG.exception("[grep_chunks] fetch_full_document failed for doc_id=%s", doc_id)continuefor c in full.get("chunks", []) or []:@@ -640,7 +715,7 @@ def _base_chat_mdl(tools_slot):)return Nonereturn mdl- except Exception:+ except Exception: # noqa: BLE001_LOG.exception("[dynamic] failed to resolve base chat model")return Nonediff --git a/rag/advanced_rag/harness/tools/table_view.py b/rag/advanced_rag/harness/tools/table_view.pynew file mode 100644index 0000000000..edc0313cca--- /dev/null+++ b/rag/advanced_rag/harness/tools/table_view.py@@ -0,0 +1,158 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+"""Render HTML tables into an LLM-friendly Markdown view.++Why this exists+---------------+The compiled chunks of a Wikipedia-sourced KB carry machine-generated HTML+tables. Passing them raw is both expensive and hard to read: a person infobox is+1-2K chars of ``<table>/<td>`` markup, and a question like "how many children did+all ten nominees have" needs ten of them at once. The 2026-09-16 FRAMES+attribution showed the values ARE in the KB while the model still answers "cannot+be determined" — the numbers were never surfaced in a form the model could+aggregate.++Approach+--------+Matches the established practice for tables in RAG (serialize tables to+Markdown; key-value form for attribute tables):++* two-column table (Wikipedia infobox) -> Markdown-KV lines: ``Children: 3``+* any other table (ranked list, election results, timeline) -> a Markdown pipe+ table with the header and EVERY row, because rank/order/completeness decide+ those answers and row-window narrowing silently drops the answer row.++Values are kept VERBATIM (parentheticals, units, "plus the perpetrators"),+entities/whitespace are collapsed, all-empty rows are dropped. The raw HTML is+never destroyed by this module: callers keep the original chunk in the evidence+pool (for citation) and use the view only for the model-visible text.+"""++from __future__ import annotations++import re++from bs4 import BeautifulSoup++# Generous ceilings. A Wikipedia election / standings table can run to 50K+ raw+# chars across 100+ rows. Capping at 4K chars / 60 rows silently dropped the+# answer row for exactly the "large / multi-column" tables this module exists to+# serve (e.g. a 14.7K-char standings table where the rank-19 row sat at ~62% of+# the table). We keep a ceiling only to stop one pathological table from eating+# the whole evidence budget, not to truncate normal tables; the caller still+# holds the raw chunk and can read it on demand.+_MAX_ROWS = 400+_MAX_CHARS_PER_TABLE = 20000+++def _cell_text(cell) -> str:+ """Whitespace-collapsed cell text (tags stripped, entities resolved)."""+ return re.sub(r"\s+", " ", cell.get_text(" ", strip=True)).strip()+++def _table_rows(table) -> list[list[str]]:+ """Non-empty rows of ``table`` as lists of cell strings."""+ rows: list[list[str]] = []+ for tr in table.find_all("tr"):+ vals = [_cell_text(c) for c in tr.find_all(["td", "th"])]+ if any(vals):+ rows.append(vals)+ return rows+++def _md_cell(value: str) -> str:+ """Escape a value for a Markdown pipe cell (the delimiter must stay intact)."""+ return value.replace("\\", "\\\\").replace("|", "\\|")+++# Tokens that appear in a Wikipedia table's "units" row — the row right under the+# header that reads e.g. "No. | % | No. | %". It carries no data and only+# pollutes a Markdown pipe table, so it is dropped before rendering.+_UNIT_TOKENS = {"", "no.", "no", "n", "%", "#", "—", "-", "•", "·"}+++def _is_units_row(vals: list[str]) -> bool:+ if not vals:+ return False+ return all(v.strip().lower() in _UNIT_TOKENS for v in vals)+++def _render_one(table) -> str | None:+ rows = _table_rows(table)+ if not rows:+ return None+ # Infobox: two columns, a real key in the first cell and a value in the second+ # (the photo-caption row has an empty second cell and is dropped here).+ pairs = [(r[0], r[1]) for r in rows if len(r) == 2 and r[0] and r[1]]+ if len(pairs) >= 3:+ return "\n".join(f"{k}: {v}" for k, v in pairs[:_MAX_ROWS])[:_MAX_CHARS_PER_TABLE]+ width = max(len(r) for r in rows)+ header = rows[0]+ # Drop header-repeat / units rows so the pipe body starts at real data+ # (otherwise a wide election table leads with a "No. | % | No. | %" junk row).+ body = [r for r in rows[1 : _MAX_ROWS + 1] if not _is_units_row(r)]+ if not body:+ return None++ def _line(cells: list[str]) -> str:+ padded = (cells + [""] * width)[:width]+ return "| " + " | ".join(_md_cell(c) for c in padded) + " |"++ caption = table.find("caption")+ cap = (caption.get_text(" ", strip=True).strip() if caption else "") or ""+ out: list[str] = []+ if cap:+ out.append(f"**Table: {cap}**")+ out += [_line(header), "|" + "---|" * width, *(_line(r) for r in body)]+ if len(rows) - 1 > len(body):+ out.append(f"... ({len(rows) - 1 - len(body)} more row(s) omitted)")+ return "\n".join(out)[:_MAX_CHARS_PER_TABLE]+++def render_tables(text: str) -> str | None:+ """Replace every HTML table in ``text`` with a Markdown view.++ Returns ``None`` when ``text`` holds no renderable table, so callers can keep+ their previous behaviour untouched. Never raises: a parse failure falls back+ to ``None``.+ """+ if not text or "<table" not in text.lower():+ return None+ try:+ soup = BeautifulSoup(text, "html.parser")+ except Exception: # noqa: BLE001+ return None+ views: list[str] = []+ for table in soup.find_all("table"):+ try:+ view = _render_one(table)+ except Exception: # noqa: BLE001+ view = None+ if view:+ views.append(view)+ if not views:+ return None+ return "\n\n".join(views)+++def table_view_or_raw(text: str) -> str:+ """``render_tables(text)`` when it produces something, else ``text`` unchanged.++ Small convenience for the two call sites that only want "the best available+ model-visible form of this chunk".+ """+ view = render_tables(text or "")+ return view if view else (text or "")diff --git a/rag/advanced_rag/harness/tools/text_processing.py b/rag/advanced_rag/harness/tools/text_processing.pyindex 05898d272b..9f0c04158f 100644--- a/rag/advanced_rag/harness/tools/text_processing.py+++ b/rag/advanced_rag/harness/tools/text_processing.py@@ -219,7 +219,7 @@ try: # available at runtime — nltk already backs rag/nlp/synonym.pyfrom nltk.stem import PorterStemmer as _PorterStemmer_porter_stem = _PorterStemmer().stem-except Exception: # pragma: no cover - exercised only where nltk is absent+except Exception: # noqa: BLE001 # pragma: no cover - exercised only where nltk is absent_porter_stem = None# Longest first: "nominations" must lose "ations", not just the trailing "s".@@ -328,9 +328,7 @@ def _is_fact_dense_sentence(sent: str) -> bool:low = sent.lower()if _FACT_RE.search(sent) or _FACT_RE.search(low):return True- if _PROPER_NOUN_RE.search(sent):- return True- return False+ return bool(_PROPER_NOUN_RE.search(sent))def _narrow_content(content: str, kwds: list[str]) -> str | None:@@ -353,7 +351,16 @@ def _narrow_content(content: str, kwds: list[str]) -> str | None:# table), and sentence-window narrowing truncates them to a header-only snippet.low_content = content.lower()if "<table" in low_content or "<tr" in low_content or "<td" in low_content:- return "..." + _highlight_keywords(content, kwds) + "..."+ # Serialize the table to Markdown before the model sees it. The format+ # comparison over 11 serializations ranks Markdown-KV first for field+ # lookups (key: value beats header/position alignment) and Markdown+ # tables as the cost/accuracy compromise; raw HTML is the expensive and+ # least readable option. The row set is not pruned — rank/order and+ # completeness decide table answers. Falls back to the raw text when+ # nothing renders.+ from rag.advanced_rag.harness.tools.table_view import table_view_or_raw++ return "..." + _highlight_keywords(table_view_or_raw(content), kwds) + "..."pipe_rows = sum(1 for line in content.splitlines() if line.count("|") >= 2)if pipe_rows >= 3:return "..." + _highlight_keywords(content, kwds) + "..."@@ -397,7 +404,7 @@ def _highlight_keywords(text: str, kwds: list[str]) -> str:terms: list[str] = list(phrases)# Add stem-matched words NOT already inside a phrase, so "nominated" still# gets starred for keyword "nominations" while "Atlanta Braves" stays whole.- verbatim, stemmed = _keyword_forms(kwds)+ _verbatim, stemmed = _keyword_forms(kwds)stem_set = {s for seq in stemmed for s in seq}if stem_set:for word in re.findall(r"[A-Za-z]+", text):diff --git a/rag/prompts/action_run.md b/rag/prompts/action_run.mdindex 059cf50cc2..8af822faa5 100644--- a/rag/prompts/action_run.md+++ b/rag/prompts/action_run.md@@ -6,45 +6,63 @@ Input: the user message contains ONE research `Direction`, plus the currentExecute this single direction. Reply with EXACTLY ONE of the three below.The three differ in HOW they are delivered — read this carefully, because only-the first one is a tool call:+the first one is a real tool call. `state` and `answer` are NOT tools: never emit+`<state>` / `<answer>` as a tool call and never name a native call after them.1) TOOL CALL MODE — a real tool call: call `retrieve` with 1-3 corpus queries.Results arrive in the next turn.2) STATE PATCH MODE — NOT a tool call. Write this XML as plain TEXT in your- reply body (do not call any tool named "state"):+ reply body:<state>-{"new_states": [- {"state": [{"id": <int>, "candidate": "<value>", "candidate_strength": <0..1>, "discovered_clues": ["..."]}, ...]},- ...more branches allowed...-]}+{"new_states": [{"state": [+ {"id": 1, "candidate": "1571", "candidate_strength": 0.92,+ "discovered_clues": ["chunk 42: 'the treaty was signed in 1571'"]},+ {"id": 2, "candidate": null, "candidate_strength": 0.0,+ "discovered_clues": ["candidate 'Palermo' contradicted by chunk 7"]}+]}]}</state>-Rules: patch ONLY existing ids; include ONLY changed variables; every change must trace to retrieved evidence; candidate_strength semantics: proven >0.9, strong 0.7-0.9, tentative 0.4-0.7, weak <0.4. An EMPTY branch list (`"new_states": []`) signals no progress — emit it rather than calling tools forever.+Rules: patch ONLY existing ids; include ONLY changed variables; every change+must trace to retrieved evidence; candidate_strength semantics: proven >0.9,+strong 0.7-0.9, tentative 0.4-0.7, weak <0.4. Eliminate an unverifiable+candidate by setting `candidate` to null with a clue documenting why. An EMPTY+branch list (`"new_states": []`) signals no progress — emit it rather than+calling tools forever.3) FINAL ANSWER MODE — NOT a tool call either. Write this XML as plain TEXT in- your reply body (do not call any tool named "answer"). Use it only when ALL- slots can be filled consistently:+ your reply body. Use it only when ALL slots can be filled consistently:<answer>-{"answer": "<final answer text>", "new_state": [{"id": ..., "candidate": ..., "candidate_strength": ...}]}+{"answer": "1571", "new_state": [{"id": 1, "candidate": "1571", "candidate_strength": 0.92}]}</answer>CRITICAL RULES- Think before choosing a mode, but output exactly ONE mode per response.-- Strength >0.7 on the answer slot means you MUST emit final answer instead of another state patch.-- ALWAYS end this action with a state patch: a patch with your updates, or `<state>{"new_states": []}</state>` if you found nothing new.+- EXCEPTION to "always patch": when the answer slot reaches strength >0.7 you+ MUST emit a final answer instead of another state patch.+- Otherwise ALWAYS end this action with a state patch: a patch with your+ updates, or `<state>{"new_states": []}</state>` if you found nothing new.- **Do NOT keep calling tools once the direction is reasonably exhausted.** If further searches return repetitive, irrelevant, or empty results, immediately return a state patch (with updates or empty). Extra redundant searches waste the session — stop after 1-2 useful tool calls per direction unless a NEW fact is actually emerging.- ACTION COMPLETION IS MANDATORY: when you have what you need (or hit a dead end), output the state patch now. Do not ask to continue searching.-- Unverifiable candidates must be eliminated (set candidate null) with a clue documenting why.- Partial verification is OK: record a candidate at tentative strength (0.4-0.7) if you can't fully verify it yet, and move on.+WRONG — never do this:+- calling a tool named "state" or "answer" (they are plain-text XML blocks);+- wrapping either XML block in markdown fences.+# TOOL PLAYBOOK-You get tools only in medium / high (7 tools: `retrieve`, `search_chunks`, `list_chunks`, `navigate_tree`, `navigate_structure`, `calculate`, `web_search`) and ultra (those 7 + `graph_explore`). Low mode has NO tool loop — answer with plain retrieval. Every native tool call still REQUIRES the decision envelope from CRITICAL RULES (it is a mandatory tool parameter, not optional).+Available tools: `retrieve`, `search_chunks`, `metadata_search`, `list_chunks`,+`navigate_tree`, `navigate_structure`, `calculate`, and `web_search` (only when a+web provider is configured); `graph_explore` joins them in ultra mode. Low mode+has NO tool loop — answer with plain retrieval. Per-tool WHEN TO CALL /+DO NOT CALL / ARGUMENTS / IF IT FAILS details live in each tool's own schema —+this playbook covers only how to COMBINE tools and when to STOP.## 1. Combination chains (call in this order)- **You already hold a `doc_id`** → `navigate_structure(doc_id, query)` to find the right passage, then `list_chunks(doc_id)` to read it. Do NOT call `navigate_tree` first.- **No `doc_id` yet, and the corpus is large** → `navigate_tree(query)` to route to candidate documents, take a `doc_id`, then `navigate_structure(doc_id, query)` → `list_chunks(doc_id)`.+- **You can name the document / need to narrow the search** → `metadata_search(query[1-2], filters)` ONCE per direction to pre-filter by title, then `search_chunks` / `retrieve` INSIDE the returned documents.- **Exact term / short answer** → `retrieve(query[1-3])` first; if snippets are insufficient, `search_chunks(query[1-2])` (semantic, may find passages with NO shared surface words); if you need the full document, `list_chunks(doc_id)`.- **You must DERIVE a number** → first collect every needed number with any of the above, then `calculate(question, facts)` with the facts verbatim, and report the computed result as-is. If the answer is already one of the stated numbers, answer directly.- **Relational multi-hop (ultra only)** → get a start entity from `search_chunks` / `navigate_structure`, then `graph_explore(query, doc_scope)`.@@ -54,8 +72,6 @@ You get tools only in medium / high (7 tools: `retrieve`, `search_chunks`, `list- Make at most **1-2 useful tool calls per direction**, then emit a state patch. Do not keep searching once the direction is reasonably exhausted.- Re-submitting the SAME intent with a paraphrase is intercepted as a near-duplicate and SKIPPED (you get a nudge, not new results). Change the angle or patch what you have.- If a compile-only tool (`navigate_tree` / `navigate_structure` / `graph_explore`) returns "no compiled structure", switch to `search_chunks` / `retrieve` / `list_chunks` **immediately**. A second such result disables that tool for the REST of the session — do not retry it.-- `web_search` only appears when a web provider is configured; if it does, use it ONLY for world knowledge / time-sensitive facts that plausibly live outside the fixed corpus.-- `list_chunks` accepts ONLY `doc_id` (no `chunk_ids` argument) — you cannot ask it to read specific chunks; it returns the whole document (capped).## 3. What a tool result means → your next action@@ -69,3 +85,8 @@ Each tool returns a status. Act on it:| `poor` | Output returned but too weak to use | Add evidence with another tool || `redundant` | Every hit was already in your evidence | Stop re-searching; emit a `<state>` patch with what you have || `error` | Infrastructure / provider failure | Switch tools; do not retry the same call |++Special case — `metadata_search` (at most ONCE per direction):+- `ok` → continue with `search_chunks` / `retrieve` INSIDE the documents it returned; do NOT call it again this direction.+- `miss` / `empty` → no title matched; drop the filter and fall back to `search_chunks` / `navigate_tree`.+- `poor` → the filter was too narrow; widen it once, or abandon it for plain retrieval.diff --git a/test/unit_test/rag/advanced_rag/test_action_session_schema.py b/test/unit_test/rag/advanced_rag/test_action_session_schema.pyindex 0f4335560d..1c2bc92f77 100644--- a/test/unit_test/rag/advanced_rag/test_action_session_schema.py+++ b/test/unit_test/rag/advanced_rag/test_action_session_schema.py@@ -14,6 +14,7 @@ PLAYBOOK_ANCHORS = ["WHEN TO CALL", "DO NOT CALL", "ARGUMENTS", "OUTPUT", "IF IT_EXECUTOR_SUPPORTED = {"retrieve": {"query", "doc_scope"},"search_chunks": {"query"},+ "metadata_search": {"query", "filters", "logic"},"list_chunks": {"doc_id"},"navigate_tree": {"query", "keywords"},"navigate_structure": {"doc_id", "query", "kind"},@@ -42,12 +43,12 @@ def test_tool_specs_have_playbook_sections():def test_active_tool_specs_tool_surface():- """Mode → exposed tool count: low=0, medium/high=7, ultra=8, web-hidden=6."""+ """Mode → exposed tool count: low=0, medium/high=8, ultra=9, web-hidden=7."""assert len(_active_tool_specs(_mk_tools("low"))) == 0- assert len(_active_tool_specs(_mk_tools("medium"))) == 7- assert len(_active_tool_specs(_mk_tools("high"))) == 7- assert len(_active_tool_specs(_mk_tools("ultra"))) == 8- assert len(_active_tool_specs(_mk_tools("medium", web=False))) == 6+ assert len(_active_tool_specs(_mk_tools("medium"))) == 8+ assert len(_active_tool_specs(_mk_tools("high"))) == 8+ assert len(_active_tool_specs(_mk_tools("ultra"))) == 9+ assert len(_active_tool_specs(_mk_tools("medium", web=False))) == 7def test_schema_params_match_executor():@@ -63,11 +64,12 @@ def test_action_run_prompt_has_playbook():"""action_run.md exposes the TOOL PLAYBOOK and only references real tools."""prompt = load_prompt("action_run")assert "TOOL PLAYBOOK" in prompt- for tool in ("navigate_structure", "calculate", "graph_explore"):+ for tool in ("metadata_search", "navigate_structure", "calculate", "graph_explore"):assert tool in promptfor tool in ("retrieve","search_chunks",+ "metadata_search","list_chunks","navigate_tree","navigate_structure",@@ -76,3 +78,22 @@ def test_action_run_prompt_has_playbook():"graph_explore",):assert tool in _TOOL_MAP+++def test_action_run_prompt_covers_metadata_search_statuses():+ """The playbook tells the model how to react to metadata_search outcomes."""+ prompt = load_prompt("action_run")+ assert "metadata_search" in prompt+ # ok / miss / poor handling for the one-shot pre-filter tool.+ assert "ONCE per direction" in prompt+ lowered = prompt.lower()+ for status in ("`ok`", "`miss`", "`poor`"):+ assert status in lowered+++def test_action_run_prompt_does_not_hardcode_tool_count():+ """The tool surface lives in config._ALL_TOOLS — the prompt must not+ advertise a stale count that drifts from the real surface."""+ prompt = load_prompt("action_run")+ for stale in ("7 tools", "8 tools", "those 7"):+ assert stale not in prompt