Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(agentic_rag): slot-table harness, table rendering + pre-search/metadata channels (#19780)

Caobaining197018a5e6017
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml
index daf7d5004d..ae8b465c37 100644
--- a/.github/workflows/sep-tests.yml
+++ b/.github/workflows/sep-tests.yml
@@ -1018,11 +1018,24 @@ jobs:
- name: Collect ragflow log Infinity
if: ${{ !cancelled() }}
+ # Log collection is diagnostics only: rebuild the artifacts path here
+ # (the directory created by "Prepare function test environment" may be
+ # gone by now, which made `cp` fail with ENOENT) and never fail the job
+ # over a log copy error.
run: |
+ set -uo pipefail
+ RUNNER_WORKSPACE_PREFIX="${RUNNER_WORKSPACE_PREFIX:-${HOME}}"
+ ARTIFACTS_DIR="${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}/${API_PROXY_SCHEME}"
+ LOG_DEST="${ARTIFACTS_DIR}/ragflow-logs-infinity"
+ mkdir -p "${ARTIFACTS_DIR}"
if [ -d docker/ragflow-logs ]; then
- destination="${ARTIFACTS_DIR}/ragflow-logs-infinity"
- sudo cp -r docker/ragflow-logs "${destination}"
- sudo chown -R "$(id -u):$(id -g)" "${destination}"
+ sudo rm -rf "${LOG_DEST}"
+ if sudo cp -a docker/ragflow-logs "${LOG_DEST}"; then
+ sudo chown -R "$(id -u):$(id -g)" "${LOG_DEST}" || true
+ echo "Copied ragflow logs to ${LOG_DEST}"
+ else
+ echo "::warning::Failed to copy ragflow logs to ${LOG_DEST}; continuing"
+ fi
echo "ragflow log" && sudo tail -n 200 docker/ragflow-logs/ragflow_server.log || true
else
echo "No docker/ragflow-logs directory found; skipping log collection"
@@ -1697,11 +1710,23 @@ jobs:
- name: Collect ragflow log Elasticsearch
if: ${{ !cancelled() }}
+ # Keep this step self-sufficient: rebuild the artifacts path instead of
+ # relying on the directory created earlier, and downgrade copy failures
+ # to warnings so a diagnostics step can never fail the job.
run: |
+ set -uo pipefail
+ RUNNER_WORKSPACE_PREFIX="${RUNNER_WORKSPACE_PREFIX:-${HOME}}"
+ ARTIFACTS_DIR="${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}/${API_PROXY_SCHEME}"
+ LOG_DEST="${ARTIFACTS_DIR}/ragflow-logs-es"
+ mkdir -p "${ARTIFACTS_DIR}"
if [ -d docker/ragflow-logs ]; then
- destination="${ARTIFACTS_DIR}/ragflow-logs-es"
- sudo cp -r docker/ragflow-logs "${destination}"
- sudo chown -R "$(id -u):$(id -g)" "${destination}"
+ sudo rm -rf "${LOG_DEST}"
+ if sudo cp -a docker/ragflow-logs "${LOG_DEST}"; then
+ sudo chown -R "$(id -u):$(id -g)" "${LOG_DEST}" || true
+ echo "Copied ragflow logs to ${LOG_DEST}"
+ else
+ echo "::warning::Failed to copy ragflow logs to ${LOG_DEST}; continuing"
+ fi
echo "ragflow log" && sudo tail -n 200 docker/ragflow-logs/ragflow_server.log || true
else
echo "No docker/ragflow-logs directory found; skipping log collection"
diff --git a/rag/advanced_rag/agentic_rag_graph.py b/rag/advanced_rag/agentic_rag_graph.py
index d671467d84..d1be162074 100644
--- a/rag/advanced_rag/agentic_rag_graph.py
+++ b/rag/advanced_rag/agentic_rag_graph.py
@@ -494,7 +494,7 @@ async def _expand_fanouts(tools, question: str, answer_conf: dict) -> list[str]:
_u = usage if isinstance(usage, dict) else getattr(mdl, "last_usage", None)
record_external_call(_u if isinstance(_u, dict) else None)
except Exception: # noqa: BLE001
- pass
+ _LOG.debug("[Planner] external-call usage booking skipped", exc_info=True)
fanouts = _parse_fanouts(str(ans or ""))
if not fanouts:
# The model answered the question instead of decomposing it (no JSON, or
@@ -514,7 +514,7 @@ async def _expand_fanouts(tools, question: str, answer_conf: dict) -> list[str]:
_u2 = getattr(mdl, "last_usage", None)
record_external_call(_u2 if isinstance(_u2, dict) else None)
except Exception: # noqa: BLE001
- pass
+ _LOG.debug("[Planner] retry external-call usage booking skipped", exc_info=True)
fanouts = _parse_fanouts(str(ans2 or ""))
except Exception: # noqa: BLE001
_LOG.warning("[Planner] strict fan-out retry failed", exc_info=True)
@@ -527,6 +527,118 @@ async def _expand_fanouts(tools, question: str, answer_conf: dict) -> list[str]:
return [question] if question else []
+# Pre-search metadata channel: the LLM extracts the short NAMED ENTITIES each
+# sub-question targets, so the metadata (title) filter can pre-select the document
+# subset before retrieval. Entities use SPACES, never underscores.
+_METADATA_ENTITY_PROMPT = (
+ "For each numbered search sub-question below, extract up to 3 short named entities "
+ "it targets — the kind of name a document index stores as a title (an article/report "
+ "name, an organisation, a place, an event, a person). Use the exact surface form with "
+ "SPACES, never underscores. If a sub-question has no clear title-like entity, use an "
+ "empty string for it. Keep each entity under 10 words; never copy the whole "
+ "sub-question, never answer it. "
+ 'Respond with JSON only: {"entities": [["<e1>", "<e2>", ...], ...]} — a list of '
+ "entity-arrays in the SAME ORDER as the sub-questions, JSON only, no prose."
+)
+
+# A usable metadata entity filter is a SHORT name — a long string is a copied
+# sub-question or an answer, which would make the title filter match nothing.
+_METADATA_ENTITY_MAX_WORDS = 10
+_METADATA_ENTITY_MAX_CHARS = 80
+_METADATA_MAX_ENTITIES = 3
+
+
+def _entity_looks_usable(entity: str) -> bool:
+ """Guard: the entity fed to the metadata title filter must be a short name."""
+ s = (entity or "").strip()
+ if not s:
+ return False
+ low = s.lower()
+ if any(mark in low for mark in _FANOUT_ANSWER_MARKS):
+ return False
+ return not (len(s) > _METADATA_ENTITY_MAX_CHARS or len(s.split()) > _METADATA_ENTITY_MAX_WORDS)
+
+
+def _parse_fanout_entities(text: str, fanouts: list[str]) -> list[list[str]]:
+ """Extract up to 3 short entities per sub-question from a model reply.
+
+ Order-aligned with ``fanouts``; each element is a (deduped, guarded) entity
+ list. Accepts ``{"entities": [["e1","e2"], ...]}`` (preferred), a flat
+ ``{"entities": ["e1","e2",...]}`` (treated as one group for every sub-question),
+ or ``{"entities": [{"sub_question": ..., "entities": [...]}]}`` matched by text.
+ Entities failing :func:`_entity_looks_usable` are dropped.
+ """
+ out: list[list[str]] = [[] for _ in fanouts]
+ data = _extract_json_object(text)
+ if not isinstance(data, dict):
+ return out
+ items = data.get("entities")
+ if not isinstance(items, list) or not items:
+ return out
+ by_text: dict[str, list[str]] = {}
+ positional: list[list[str]] = []
+ for it in items:
+ if isinstance(it, dict):
+ ents = [str(e).strip() for e in (it.get("entities") or it.get("titles") or [])]
+ sq = str(it.get("sub_question") or it.get("fanout") or "").strip()
+ if sq:
+ by_text[sq] = ents
+ positional.append(ents)
+ elif isinstance(it, str):
+ positional.append([str(it).strip()])
+ elif isinstance(it, list):
+ positional.append([str(e).strip() for e in it])
+ for i, fq in enumerate(fanouts):
+ raw = by_text.get(fq, []) if by_text else (positional[i] if i < len(positional) else [])
+ seen: set[str] = set()
+ cleaned: list[str] = []
+ for e in raw:
+ e = (e or "").strip()
+ if e and e.lower() not in seen and _entity_looks_usable(e):
+ seen.add(e.lower())
+ cleaned.append(e)
+ if len(cleaned) >= _METADATA_MAX_ENTITIES:
+ break
+ out[i] = cleaned
+ return out
+
+
+async def _extract_fanout_entities(tools, fanouts: list[str], answer_conf: dict) -> dict[str, list[str]]:
+ """Map each sub-question to 1-3 short entities for metadata (title) filtering.
+
+ ONE chat call for the whole batch. Returns ``{fanout: [entities]}`` containing
+ only entries with at least one usable entity; any failure yields ``{}`` so the
+ metadata channel is simply skipped for this round — it never blocks pre-search.
+ """
+ queries = [str(f).strip() for f in (fanouts or []) if isinstance(f, str) and str(f).strip()]
+ if not queries:
+ return {}
+ try:
+ from rag.advanced_rag.harness.tools.search import _base_chat_mdl
+
+ mdl = _base_chat_mdl(tools)
+ except Exception: # noqa: BLE001
+ _LOG.warning("[Prefetch] could not resolve base chat model for entity extraction", exc_info=True)
+ return {}
+ if mdl is None:
+ return {}
+ listed = "\n".join(f"{i + 1}. {q}" for i, q in enumerate(queries))
+ try:
+ ans, _ = await mdl.async_chat(
+ _METADATA_ENTITY_PROMPT,
+ [{"role": "user", "content": f"Sub-questions:\n{listed}"}],
+ dict(answer_conf or {}),
+ )
+ entities = _parse_fanout_entities(str(ans or ""), queries)
+ except Exception: # noqa: BLE001
+ _LOG.warning("[Prefetch] entity extraction failed; skipping metadata channel", exc_info=True)
+ return {}
+ mapping = {q: es for q, es in zip(queries, entities) if es}
+ if mapping:
+ _LOG.info("[Prefetch] metadata entities: %s", mapping)
+ return mapping
+
+
# Storage ceiling of the snippet pool across ALL rounds. Storage and REVIEW
# are decoupled: the SCA only reads a ranked view (``_SCA_VIEW_CAP``), so the
# pool may accumulate freely while prompts stay bounded. The former single
@@ -558,17 +670,33 @@ _MAX_SLOT_DEPTH = 3
# the research tree, not the gap-promotion rounds), so the depth test alone
# cannot stop the table from growing one slot per round — this bound can.
_MAX_SLOTS_TOTAL = 8
+_PRESEARCH_MAX_POOL = 18 # pre-search (first prefetch) hard ceiling on TOTAL chunks admitted
+# Channel A (BM25 + narrow) keeps this many chunks per fan-out query. It is the
+# retrieval-window size, NOT the pool share: the pool is divided equally between
+# the channels that have chunks to offer, so A's quota no longer decides how much
+# of the pool it gets (measured 2026-09-16: 3 fan-outs x 5 = 15 chunks used to
+# take 15 of the 18 pre-search slots, starving the metadata channel).
+_CHANNEL_A_QUERY_QUOTA = 5
+# Channel B (semantic hybrid, narrow bypass) per-query quota. It was switched off
+# outright in pre-search by 2b2ae7a; re-enabled 2026-09-16. Two things changed
+# since: the pool share is now split EQUALLY between the channels that actually
+# have chunks, so a third channel no longer steals A's/C's slots, and the
+# zero-score attribution showed the surviving failures are passages whose surface
+# words do not overlap the query at all (a father's occupation, a section buried
+# in a long document) — exactly what only the semantic channel can reach.
+_PRESEARCH_HYBRID_ENABLED = True
+_CHANNEL_B_QUERY_QUOTA = 4
_DRILL_RESERVE = 12 # slots kept free after the FIRST prefetch so the research
# executor can top up evidence
_SCA_VIEW_CAP = 60 # chunks shown to the Sufficient Context Agent per review (24 -> 60: 24 of 225 hid the answer-bearing table chunk from the SCA)
-async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: int | None = None) -> int:
+async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: int | None = None, answer_conf: dict | None = None, use_metadata: bool = False) -> int:
"""Programmatic fan-out retrieval: fetch all queries and fill ``kbinfos``.
Phase 2 ("The RAG Agent searches ... all the query fanouts at once")
for the SNIPPET pool the Sufficient Context Agent reviews (Phase 3 reads
- actual retrieved text). Two collector channels run per query:
+ actual retrieved text). Up to three collector channels run per query:
* Channel A (BM25 + ``narrow_by_terms``): exact-name recall, rewritten to a
short match window.
@@ -576,6 +704,9 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: in
text does NOT share surface words with the query. The 2026-08-27 hybrid
experiment proved these blocks are valuable but are ALL filtered out by the keyword narrow step while
crowding candidates out of top_n — so they now bypass it entirely.
+ * Channel C (metadata title pre-filter, only when ``use_metadata``): the LLM
+ distils each sub-question to a short title/entity, documents are pre-selected
+ by their ``title`` metadata, and hybrid retrieval runs INSIDE that subset.
Called after planning and again after every Query-Rewriter pass. Returns
how many NEW chunks were added; failures are swallowed per-query.
@@ -589,6 +720,7 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: in
_query_to_terms,
bm25_search,
hybrid_search,
+ metadata_search,
)
except Exception: # noqa: BLE001
_LOG.warning("[rag_agent] could not import fan-out search helpers", exc_info=True)
@@ -634,7 +766,16 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: in
_LOG.warning("[Prefetch] unexpected queries payload of type %s; skipping fan-out search", type(fanouts).__name__)
return 0
- async def _search_one(fq: str) -> tuple[list, list]:
+ # Channel C setup: one batched LLM call maps each sub-question to 1-3 short
+ # entities used to pre-filter documents by their ``title`` metadata (OR'ed).
+ # Best-effort: an empty mapping simply disables the channel for this round.
+ entities_by_query: dict[str, list[str]] = {}
+ if use_metadata:
+ entities_by_query = await _extract_fanout_entities(tools, fanouts, answer_conf or {})
+ if not entities_by_query:
+ _LOG.info("[Prefetch] no usable metadata entities; metadata channel disabled this round")
+
+ async def _search_one(fq: str) -> tuple[list, list, list]:
# Each coroutine ONLY retrieves and returns chunk lists; it does NOT
# touch ``kbinfos``. All mutation happens in the main coroutine below,
# so concurrent fan-out searches cannot race on the shared list.
@@ -650,7 +791,7 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: in
terms = _query_to_terms(fq)
keyed = [t for t in terms if len(t) >= 3 and t.lower() not in _FANOUT_STOPWORDS and not t.isdigit() or (len(t) >= 4 and t.isdigit())]
try:
- res = await bm25_search(tools, fq, kb_ids=kb_ids, top_n=60, keywords=" ".join(keyed or terms))
+ res = await bm25_search(tools, fq, kb_ids=kb_ids, top_n=48, keywords=" ".join(keyed or terms))
candidates = res.get("chunks", []) or []
except Exception: # noqa: BLE001
_LOG.warning("[rag_agent] BM25 search failed for %r", fq, exc_info=True)
@@ -663,31 +804,63 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: in
candidates,
keyed or terms,
fallback_terms=None,
- context={"before": 0, "after": 1},
+ # Match-centred window: the answer-bearing row of a long
+ # section is rarely its first line, and the old (0,1) window
+ # showed the match plus one line only.
+ context={"before": 2, "after": 4},
keywords=fq,
- max_out_chars_per_chunk=1200,
+ max_out_chars_per_chunk=1500,
max_out_total_chars=16000,
)
- kept_a = (narrowed.get("kept", []) or [])[: max(1, top_n)]
+ kept_a = (narrowed.get("kept", []) or [])[: max(1, _CHANNEL_A_QUERY_QUOTA)]
except Exception: # noqa: BLE001
_LOG.warning("[rag_agent] narrowing failed for %r; using raw BM25 head", fq, exc_info=True)
- kept_a = candidates[: max(1, top_n)]
+ kept_a = candidates[: max(1, _CHANNEL_A_QUERY_QUOTA)]
- # Channel B: semantic collector, narrow BYPASS. Keep only hits that
- # Channel A did not already surface (dedup happens again at merge).
+ # Channel B: semantic collector (hybrid, narrow bypass). Controlled by
+ # ``_PRESEARCH_HYBRID_ENABLED`` — when off, kept_b stays empty and
+ # Channel C's dedup (seen_ab = seen_ids_a | kept_b) still works unchanged.
kept_b: list = []
seen_ids_a = {_chunk_id(c) for c in kept_a}
- try:
- hres = await hybrid_search(tools, fq, kb_ids=kb_ids, top_n=30)
- for c in hres.get("chunks", []) or []:
- if _chunk_id(c) in seen_ids_a:
- continue
- kept_b.append(c)
- if len(kept_b) >= 4: # modest semantic quota per query
- break
- except Exception: # noqa: BLE001
- _LOG.warning("[rag_agent] hybrid channel failed for %r; skipping", fq, exc_info=True)
- return kept_a, kept_b
+ if _PRESEARCH_HYBRID_ENABLED:
+ try:
+ hres = await hybrid_search(tools, fq, kb_ids=kb_ids, top_n=30)
+ for c in hres.get("chunks", []) or []:
+ if _chunk_id(c) in seen_ids_a:
+ continue
+ kept_b.append(c)
+ if len(kept_b) >= _CHANNEL_B_QUERY_QUOTA: # modest semantic quota per query
+ break
+ except Exception: # noqa: BLE001
+ _LOG.warning("[rag_agent] hybrid channel failed for %r; skipping", fq, exc_info=True)
+
+ # Channel C: metadata title pre-filter (only when usable entities were
+ # extracted for this sub-question). Documents are pre-selected by their
+ # ``title`` metadata matching ANY of the entities (OR), then hybrid
+ # retrieval runs INSIDE that subset; keep only hits A/B did not already
+ # surface (dedup happens again at merge).
+ kept_c: list = []
+ entities = entities_by_query.get(fq)
+ if entities:
+ seen_ab = seen_ids_a | {_chunk_id(c) for c in kept_b}
+ try:
+ mres = await metadata_search(
+ tools,
+ fq,
+ [{"key": "title", "op": "contains", "value": e} for e in entities],
+ logic="or",
+ kb_ids=kb_ids,
+ top_n=top_n,
+ )
+ for c in mres.get("chunks", []) or []:
+ if _chunk_id(c) in seen_ab:
+ continue
+ kept_c.append(c)
+ if len(kept_c) >= 4: # modest metadata quota per query
+ break
+ except Exception: # noqa: BLE001
+ _LOG.warning("[rag_agent] metadata channel failed for %r (entities=%r); skipping", fq, entities, exc_info=True)
+ return kept_a, kept_b, kept_c
async def _collect_evidence() -> list:
"""Channel 0: compiled evidence rows, folded into pool-shaped entries.
@@ -773,11 +946,77 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: in
raw_added += 1
return stop
+ def _channel(pair, idx: int) -> list:
+ if isinstance(pair, Exception) or not isinstance(pair, tuple):
+ return []
+ return pair[idx] if idx < len(pair) else []
+
+ # ── Equal pool share for every channel that actually has chunks ────────
+ # Admitting the collectors strictly in order let channel A (up to
+ # ``_CHANNEL_A_QUERY_QUOTA`` chunks per fan-out query) consume the whole
+ # pre-search pool before the metadata channel was reached: measured
+ # 2026-09-16, 3 fan-outs -> A took 15 of the 18 slots and the metadata
+ # channel was truncated to 3 of the 8 chunks it offered; with 4 fan-outs it
+ # admitted nothing at all while its entity-extraction LLM call had already
+ # been paid for. Every channel that has something to offer now gets an equal
+ # share of the remaining room, so a claim-less dataset (FRAMES has no
+ # ``entity_type_kwd`` rows, so the evidence channel is inactive) with channel
+ # B disabled splits A/C 50/50 instead of A taking everything.
+ counts: dict = {"evidence": 0, "A": 0, "B": 0, "C": 0}
+ offered: dict = {name: sum(len(_channel(p, i)) for p in results) for i, name in ((0, "A"), (1, "B"), (2, "C"))}
+ active: list = (["evidence"] if evidence_chunks else []) + [n for n in ("A", "B", "C") if offered[n]]
+ share = max(1, -(-room // len(active))) if (active and room > 0) else 0
+ _LOG.info(
+ "[Prefetch] pool shares: room=%d active=%s share=%d (offered evidence=%d A=%d B=%d C=%d)",
+ room,
+ active or "-",
+ share,
+ len(evidence_chunks or []),
+ offered["A"],
+ offered["B"],
+ offered["C"],
+ )
+
+ def _log_admission() -> None:
+ _LOG.info(
+ "[Prefetch] admission by channel: evidence=%d A=%d B=%d C=%d (offered A=%d B=%d C=%d) total=%d room=%d",
+ counts["evidence"],
+ counts["A"],
+ counts["B"],
+ counts["C"],
+ offered["A"],
+ offered["B"],
+ offered["C"],
+ added,
+ room,
+ )
+
+ def _run_channel(name: str, idx: int, cap: int) -> bool:
+ # ``_admit`` consumes a whole per-query batch at once, so slice each
+ # batch to the channel's remaining share — otherwise one batch can
+ # overshoot the cap by up to (batch size - 1) and re-create the very
+ # imbalance the share is meant to remove.
+ before = added
+ stop = False
+ for pair in results:
+ remaining = cap - (added - before)
+ if remaining <= 0:
+ break
+ if _admit(_channel(pair, idx)[:remaining]):
+ stop = True
+ break
+ counts[name] += added - before
+ return stop
+
# Evidence rows first: they are answer material, so they must not be
- # crowded out by the chunk channels.
+ # crowded out by the chunk channels. Bounded by its equal share like the
+ # chunk channels are.
+ ev_before = added
if evidence_chunks:
- evidence_chunks = evidence_chunks[:_EVIDENCE_POOL_QUOTA]
+ evidence_chunks = evidence_chunks[: min(_EVIDENCE_POOL_QUOTA, share)]
if _admit(evidence_chunks):
+ counts["evidence"] = added - ev_before
+ _log_admission()
return added
# Directional top-up (external gathering): an evidence row carries a
# verbatim quote but not its surrounding passage, so pull exactly the
@@ -793,23 +1032,24 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: in
from rag.advanced_rag.harness.tools.navigation import _load_chunks_for_ids
fetched = await _load_chunks_for_ids(tools, wanted[:_EVIDENCE_TOP_UP])
- if fetched and _admit(fetched):
+ room_for_evidence = max(0, share - (added - ev_before))
+ if fetched and _admit(fetched[:room_for_evidence]):
+ counts["evidence"] = added - ev_before
+ _log_admission()
return added
except Exception: # noqa: BLE001
_LOG.warning("[rag_agent] evidence top-up failed; continuing", exc_info=True)
- # Channel A first (exact matches earn their slots), then semantic extras.
- for pair in results:
- if isinstance(pair, Exception):
- continue
- kept_a, kept_b = pair if isinstance(pair, tuple) else ([], [])
- if _admit(kept_a):
- break
- for pair in results:
- if isinstance(pair, Exception):
- continue
- _, kept_b = pair if isinstance(pair, tuple) else ([], [])
- if _admit(kept_b):
- break
+ counts["evidence"] = added - ev_before
+ # Channel order still decides who is served first in the only remaining
+ # unequal case: the ceil() rounding of ``share`` can make the shares sum to
+ # slightly more than ``room``, and the global room guard then trims whichever
+ # channel is served last. Each channel is otherwise capped at exactly its
+ # own share, so the pool split stays even and a channel with nothing to offer
+ # costs nobody else a slot.
+ _run_channel("A", 0, share)
+ _run_channel("B", 1, share)
+ _run_channel("C", 2, share)
+ _log_admission()
if room == 0:
_LOG.info("[Prefetch] snippet pool FULL (%d chunks); nothing new admitted", max_total)
@@ -859,8 +1099,53 @@ async def _compose_answer_from_evidence(state: AgenticState, tools, token_queue:
)
from rag.advanced_rag.agentic_rag import _EVIDENCE_BUDGET_TOKENS
- _CITE_CHUNK_CAP = 6
- cite_chunks = ranked[:_CITE_CHUNK_CAP] or all_chunks
+ _CITE_CHUNK_CAP = 12
+ # The answer model's evidence pool is assembled from three prioritised
+ # groups, deduped by chunk id, then trimmed to the cap (``kb_prompt`` still
+ # enforces the token budget on top of this):
+ # 1. slot-cited chunks — the passages that actually produced each slot's
+ # draft/candidate (``state["slot_evidence"]``). This is evidence the
+ # research already committed to, so it belongs in the pool by default
+ # instead of being re-selected by similarity from scratch.
+ # 2. table chunks — aggregation answers ("how many times / how many
+ # children / which year") are tallied from rows, so the table has to
+ # reach the answer model; a pure top-k-by-similarity slice dropped the
+ # deciding election-result table (FRAMES Q673/Q483).
+ # 3. top-similarity chunks — fallback citation reference.
+ # The previous pool was a hard top-6-by-similarity: too few once several
+ # slots each cite distinct evidence, and it starved groups 1 and 2.
+ from rag.advanced_rag.harness.tools.search import _chunk_id, _is_table_chunk
+
+ by_chunk_id: dict[str, dict] = {}
+ for _c in all_chunks:
+ _cid = _chunk_id(_c)
+ if _cid and _cid not in by_chunk_id:
+ by_chunk_id[_cid] = _c
+ _slot_evidence = state.get("slot_evidence") or {}
+ _slot_order: list[str] = []
+ for _meta in _slot_evidence.values():
+ for _eid in _meta.get("evidence_ids") or []:
+ _eid = str(_eid)
+ if _eid not in _slot_order:
+ _slot_order.append(_eid)
+ slot_cite = [by_chunk_id[eid] for eid in _slot_order if eid in by_chunk_id]
+ _TABLE_CITE_CAP = 4
+ table_cite = [c for c in ranked if _is_table_chunk(c)][:_TABLE_CITE_CAP]
+
+ cite_chunks: list = []
+ _seen_cite: set[str] = set()
+ for _group in (slot_cite, table_cite, ranked):
+ for _c in _group:
+ _key = _chunk_id(_c) or str(id(_c))
+ if _key in _seen_cite:
+ continue
+ _seen_cite.add(_key)
+ cite_chunks.append(_c)
+ if len(cite_chunks) >= _CITE_CHUNK_CAP:
+ break
+ if len(cite_chunks) >= _CITE_CHUNK_CAP:
+ break
+ cite_chunks = cite_chunks or all_chunks
# DESIGN ENHANCEMENT (Go parity: RunResponse.SlotCitations): expose the
# slot evidence ids and the citation-pool chunk ids so the rag tool's
# post-processing can rewrite unresolvable [ID:Slot N] markers into real
@@ -1093,8 +1378,20 @@ def build_agentic_graph(
tools.kbinfos = dict(getattr(tools, "kbinfos", None) or state.get("kbinfos") or {"chunks": [], "doc_aggs": []})
t = min(_PREFETCH_TIMEOUT_S, max(10.0, _remaining_s(state) - _MIN_ROUND_HEADROOM_S))
# First-round prefetch leaves drill slots free (see _DRILL_RESERVE).
+ # This is the pre-search stage right after fan-out: besides BM25 + hybrid,
+ # run the metadata (title) channel so sub-questions whose target document
+ # is identifiable by title reach it directly.
+ # The ceiling stays fixed; fairness between the channels is enforced
+ # inside _fanout_search by sharing the pool equally (see "pool shares"
+ # there), instead of letting channel A's per-query quota eat it all.
added = await _bounded(
- _fanout_search(tools, queries, capacity=_MAX_SNIPPET_POOL - _DRILL_RESERVE),
+ _fanout_search(
+ tools,
+ queries,
+ capacity=_PRESEARCH_MAX_POOL,
+ answer_conf=answer_conf,
+ use_metadata=True,
+ ),
t,
"programmatic prefetch",
)
@@ -1484,6 +1781,25 @@ def build_agentic_graph(
return g.compile()
+_NEGATIVE_CANDIDATES = {"0", "0.0", "zero", "none", "no", "null", "n/a", "na", "无", "没有", "沒有"}
+
+
+def _looks_like_negative_candidate(cand) -> bool:
+ """True when a slot candidate is a bare zero/negative (e.g. 0, none, no).
+
+ Such values are frequently produced from "no record was retrieved" rather
+ than from a positive source. The draft renderer flags them when they carry no
+ evidence_ids, so the SCA / finalize treat them as absence-based (unverified)
+ instead of as a positively-evidenced value.
+ """
+ s = str(cand or "").strip().lower()
+ if not s:
+ return False
+ if set(s) <= set("0."): # bare zeros, e.g. "0", "0.0"
+ return True
+ return s in _NEGATIVE_CANDIDATES
+
+
def _render_slot_draft(slot_table, collected_answer: str | None = None, slot_evidence: dict | None = None) -> str:
"""Render a slot table into a fact-preserving draft for the SCA.
@@ -1528,6 +1844,11 @@ def _render_slot_draft(slot_table, collected_answer: str | None = None, slot_evi
details.append(f"terminal={terminal}")
if evidence_ids:
details.append(f"evidence_ids={evidence_ids}")
+ elif _looks_like_negative_candidate(cand):
+ # Absence-based: a bare 0/none that no retrieved passage supports.
+ # Flag it so the SCA / finalize treat it as unverified rather than a
+ # positively-evidenced value (guards "no record found ⇒ 0").
+ details.append("absence-based — unverified, no positive evidence_ids")
suffix = " [" + ", ".join(details) + "]" if details else ""
lines.append(f"- slot {vid} [{vtype}]: {cand} (strength={strength}){suffix}" + (f" — {tail}" if tail else ""))
else:
diff --git a/rag/advanced_rag/harness/action_session.py b/rag/advanced_rag/harness/action_session.py
index 5a1215265c..ed967c0fd9 100644
--- a/rag/advanced_rag/harness/action_session.py
+++ b/rag/advanced_rag/harness/action_session.py
@@ -19,6 +19,7 @@
from __future__ import annotations
import asyncio
+import copy
import json
import logging
import re
@@ -38,7 +39,7 @@ _LOG = logging.getLogger(__name__)
_INIT_TIMEOUT_S = 45.0
_ACTION_TIMEOUT_S = 75.0
_SNIPPETS_PER_QUERY = 4
-_MAX_TOOL_RESPONSE_CHARS = 12000
+_MAX_TOOL_RESPONSE_CHARS = 100000
# Dataset-level empty results (reason="no_structure") a compiled-structure tool
# must accumulate before it is disabled for the rest of the session. Kept above
# 1: a single empty can be SCOPED — graph_explore over a doc_scope that has no
@@ -406,6 +407,66 @@ _GRAPH_EXPLORE_TOOL_SPEC = {
},
}
+_METADATA_SEARCH_TOOL_SPEC = {
+ "type": "function",
+ "function": {
+ "name": "metadata_search",
+ "description": (
+ "WHEN TO CALL: PRE-FILTER the document set by title BEFORE retrieving chunks — the question names a document by title or recognizable name; or you need a named document subset; or the corpus is large and a title filter would sharpen recall. Prefer 'contains' with a distinctive substring. "
+ "CALL AT MOST ONCE PER DIRECTION: then use search_chunks / retrieve inside those documents. "
+ "DO NOT CALL: nothing names a document/subset; you already hold a doc_id (use list_chunks); counting or enumerating. "
+ "ARGUMENTS: query — 1-2 strings. filters — [{key, value, op}]; key is only 'title'; op — see enum; logic 'and'|'or'. For the string ops the value MUST be ONE keyword, never a list — one call per keyword; 'in' takes a list; 'empty' takes no value. Example: [{key: 'title', op: 'contains', value: 'New York'}]. Titles use spaces, not underscores. "
+ "OUTPUT: ranked chunks from ONLY the matching documents. ok = new evidence; redundant = already seen; miss = nothing matched. "
+ "IF IT FAILS: 'no documents match' — shorten the substring, or drop the filter and use search_chunks. Do NOT retry the same filter."
+ ),
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "query": {
+ "type": "array",
+ "items": {"type": "string"},
+ "minItems": 1,
+ "maxItems": 2,
+ },
+ "filters": {
+ "type": "array",
+ "minItems": 1,
+ "items": {
+ "type": "object",
+ "properties": {
+ # Only `title` is exposed for now. Extending to other
+ # metadata fields is a matter of adding to this enum —
+ # the executor below is already field-agnostic.
+ "key": {
+ "type": "string",
+ "enum": ["title"],
+ "description": "the document title",
+ },
+ "value": {
+ "type": ["string", "array", "null"],
+ "description": (
+ "The keyword/value to match. For contains/=/start with/end with/not "
+ "contains this MUST be a SINGLE string keyword (e.g. 'Bowling'), never "
+ "a list — to match several keywords, call metadata_search once per "
+ "keyword. 'in'/'not in' MAY be a list of strings; 'empty'/'not empty' "
+ "take no value."
+ ),
+ },
+ "op": {
+ "type": "string",
+ "enum": ["=", "contains", "not contains", "start with", "end with", "in", "empty", "not empty"],
+ },
+ },
+ "required": ["key", "op"],
+ },
+ },
+ "logic": {"type": "string", "enum": ["and", "or"], "default": "and"},
+ },
+ "required": ["query", "filters"],
+ },
+ },
+}
+
_TOOL_MAP = {
"retrieve": _RETRIEVE_TOOL_SPEC,
"search_chunks": _SEARCH_CHUNKS_TOOL_SPEC,
@@ -415,6 +476,7 @@ _TOOL_MAP = {
"calculate": _CALCULATE_TOOL_SPEC,
"graph_explore": _GRAPH_EXPLORE_TOOL_SPEC,
"web_search": _WEB_SEARCH_TOOL_SPEC,
+ "metadata_search": _METADATA_SEARCH_TOOL_SPEC,
}
@@ -447,7 +509,12 @@ def _active_tool_specs(tools) -> list:
disabled = getattr(tools, "_disabled_tools", None) or set()
if disabled:
names -= set(disabled)
- return [spec for name, spec in _TOOL_MAP.items() if name in names]
+ specs = []
+ for name, spec in _TOOL_MAP.items():
+ if name not in names:
+ continue
+ specs.append(copy.deepcopy(spec))
+ return specs
def _disable_tool(tools, name: str) -> None:
@@ -676,16 +743,23 @@ def _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=True) ->
return False
seen.add(cid)
ids.append(cid)
- # Table chunks pass through UN-truncated: the 1200-char cap hides answer
+ # Table chunks pass through UN-truncated: the 5000-char cap hides answer
# rows in the mid/late table (Q86: rank-19 row at char 5181 of a 8274-char
# standings table was cut, so the session model guessed the athlete). The
# pool (kbinfos) already stores the full chunk; only the model-facing
# session output was truncated here.
_ct = _chunk_text(c) or ""
if _is_table_chunk(c):
- entry = {"id": str(cid), "content": _ct}
+ # Table chunks are shown to the model as a Markdown view (key-value for
+ # infoboxes, a full-row pipe table for ranked/result tables) instead of
+ # raw <table> markup: same rows, a fraction of the tokens, and the format
+ # comparison over 11 serializations ranks Markdown-KV/Markdown above raw
+ # HTML. The shared pool keeps the RAW chunk for citation and re-reads.
+ from rag.advanced_rag.harness.tools.table_view import table_view_or_raw
+
+ entry = {"id": str(cid), "content": table_view_or_raw(_ct)}
else:
- entry = {"id": str(cid), "content": _ct[:1200]}
+ entry = {"id": str(cid), "content": _ct[:5000]}
if include_doc_id:
entry["doc_id"] = _doc_id(c)
out.append(entry)
@@ -770,9 +844,16 @@ async def _run_search(tools, search_fn, queries: list, top_n: int, max_q: int, *
only differences are the search backend (grep vs hybrid), ``top_n``, the
per-call query cap — all passed in. ``kw`` forwards backend kwargs such as
``use_compiled`` (search_chunks' compiled-structure expansion).
+
+ ``fail_hard`` (default False) controls exception handling: most tools want a
+ soft per-query skip so a single backend blip does not kill the whole turn;
+ ``metadata_search`` passes True so a real retrieval failure surfaces as
+ ``ERROR/infra`` instead of being silently downgraded to ``MISS/no_doc``
+ (which would mislead the model into thinking no document matched).
"""
from rag.advanced_rag.harness.tools.search import _chunk_id
+ fail_hard = kw.pop("fail_hard", False)
kb_ids = _kb_ids(tools)
out, ids = [], []
seen = set()
@@ -794,7 +875,9 @@ async def _run_search(tools, search_fn, queries: list, top_n: int, max_q: int, *
try:
res = await search_fn(tools, fq, kb_ids=kb_ids, top_n=top_n, **kw)
cands = res.get("chunks", []) or []
- except Exception: # noqa: BLE001
+ except Exception:
+ if fail_hard:
+ raise
_LOG.warning("[Action Session] %s failed for %r", getattr(search_fn, "__name__", "search"), fq, exc_info=True)
continue
for c in cands[:_SNIPPETS_PER_QUERY]:
@@ -901,6 +984,183 @@ async def _exec_web_search(tools, queries: list) -> ToolOutcome:
return _search_outcome(out, ids, new_ev)
+def _normalize_metadata_value(value, op):
+ """Coerce a model-supplied filter value to the single value the executor
+ expects. String ops (contains, =, start with, end with, not contains) take
+ ONE keyword; if a list sneaks in, collapse it to the first non-empty element
+ (one metadata_search call = one keyword) and log it. 'in'/'not in' keep
+ their list (the value set), and empty/not empty take no value."""
+ if op in ("in", "not in"):
+ return value
+ if isinstance(value, list):
+ flat = [str(v) for v in value if v not in (None, "")]
+ if not flat:
+ return None
+ chosen = flat[0]
+ if len(flat) > 1:
+ _LOG.info(
+ "[metadata search] value list collapsed to single keyword %r (ignored: %s); to match all, call metadata_search once per keyword",
+ chosen,
+ flat[1:],
+ )
+ return chosen
+ return value
+
+
+async def _exec_metadata_search(tools, args: dict) -> ToolOutcome:
+ """Metadata-filtered hybrid retrieval (metadata_search tool).
+
+ Pipeline: validate the requested metadata keys against the dataset's real
+ fields -> resolve matching doc_ids via ES/Infinity push-down (in-memory
+ ``meta_filter`` fallback) -> hybrid_search scoped to exactly those
+ documents.
+
+ The executor is field-agnostic: which keys are reachable is decided by the
+ ``key`` enum in :data:`_METADATA_SEARCH_TOOL_SPEC` (currently ``title``
+ only), so exposing another metadata field later needs no change here.
+ """
+ from api.db.services.doc_metadata_service import DocMetadataService
+ from common.metadata_utils import meta_filter
+ from common.misc_utils import thread_pool_exec
+ from rag.advanced_rag.harness.tools.search import hybrid_search
+
+ queries = _arg_query_list(args, 2)
+ filters = args.get("filters") or []
+ logic = str(args.get("logic") or "and")
+
+ if not queries:
+ _LOG.info("[metadata search] skipped — no query provided")
+ return ToolOutcome(payload=[], status=ERROR, reason="bad_args", metrics={"hits": 0, "new_evidence": 0})
+ if not filters:
+ _LOG.info("[metadata search] skipped — no metadata filters supplied (need ≥1 {key, op, value})")
+ return ToolOutcome(
+ payload=[
+ {
+ "kind": "metadata_search",
+ "note": "No metadata conditions given. metadata_search needs at least one {key, value, op} filter — otherwise use search_chunks / retrieve.",
+ }
+ ],
+ status=MISS,
+ reason="bad_args",
+ metrics={"hits": 0, "new_evidence": 0},
+ )
+
+ kb_ids = _kb_ids(tools)
+ if not kb_ids:
+ return ToolOutcome(payload=[], status=ERROR, reason="infra", metrics={"hits": 0, "new_evidence": 0})
+
+ # 1) Key validation against the dataset's REAL metadata fields. A dataset
+ # without the requested key (e.g. no title) must degrade to a hint, not
+ # to an empty retrieval the model would retry forever.
+ try:
+ known_keys = await thread_pool_exec(DocMetadataService.get_metadata_keys_by_kbs, kb_ids)
+ except Exception: # noqa: BLE001
+ _LOG.warning("[metadata search] get_metadata_keys_by_kbs failed", exc_info=True)
+ known_keys = []
+ known_set = {str(k) for k in (known_keys or [])}
+ bad = [str(f.get("key")) for f in filters if str(f.get("key")) not in known_set]
+ if bad:
+ _LOG.info(
+ "[metadata search] bad key(s) %s — not in dataset metadata (available: %s)",
+ bad,
+ sorted(known_set)[:30],
+ )
+ return ToolOutcome(
+ payload=[
+ {
+ "kind": "metadata_search",
+ "note": f"Metadata key(s) {bad} do not exist in this dataset. Available: {sorted(known_set)[:30] or 'NONE — this dataset has no metadata, use search_chunks / retrieve'}.",
+ }
+ ],
+ status=MISS,
+ reason="bad_args",
+ metrics={"hits": 0, "new_evidence": 0},
+ )
+
+ # Normalize filter values BEFORE they reach meta_filter / ES push-down, so a
+ # model that mistakenly emits a list where one keyword is expected can never
+ # silently produce a no_doc TypeError again. 'in'/'not in' keep their list
+ # semantics; string ops collapse a list to its first keyword (one call = one
+ # keyword) and log it so the model can re-call per keyword if needed.
+ normalized_filters = [{**f, "value": _normalize_metadata_value(f.get("value"), f.get("op"))} for f in filters]
+
+ # 2) Resolve matching doc_ids: ES/Infinity push-down, in-memory fallback.
+ try:
+ doc_ids = await thread_pool_exec(DocMetadataService.filter_doc_ids_by_meta_pushdown, kb_ids, normalized_filters, logic)
+ except Exception: # noqa: BLE001
+ _LOG.warning("[metadata search] push-down failed", exc_info=True)
+ doc_ids = None
+ if doc_ids is None:
+ try:
+ metas = await thread_pool_exec(DocMetadataService.get_flatted_meta_by_kbs, kb_ids)
+ doc_ids = meta_filter(metas, normalized_filters, logic) or []
+ except Exception: # noqa: BLE001
+ _LOG.warning("[metadata search] in-memory fallback failed", exc_info=True)
+ doc_ids = []
+ doc_ids = [str(d) for d in (doc_ids or [])]
+
+ if not doc_ids:
+ _LOG.info("[metadata search] no documents matched filters=%s logic=%s", normalized_filters, logic)
+ return ToolOutcome(
+ payload=[
+ {
+ "kind": "metadata_search",
+ "note": "No documents match the given metadata conditions. Loosen or change the "
+ "filters (try a shorter substring, or the space form instead of underscores), "
+ "or use search_chunks / retrieve for unfiltered search.",
+ }
+ ],
+ status=MISS,
+ reason="no_doc",
+ metrics={"hits": 0, "new_evidence": 0},
+ )
+
+ # 3) Intersect with the session's global document scope.
+ if hasattr(tools, "scoped_doc_ids"):
+ doc_ids = tools.scoped_doc_ids(doc_ids) or []
+ if not doc_ids:
+ _LOG.info("[metadata search] 0 documents after doc-scope intersection")
+ return ToolOutcome(payload=[], status=MISS, reason="no_doc", metrics={"hits": 0, "new_evidence": 0})
+
+ # 4) Hybrid search restricted to exactly those documents.
+ # use_compiled=False: compiled expansion would pull in out-of-scope chunks
+ # and break the "only these documents" contract.
+ # fail_hard=True: a genuine retrieval failure (embedding/infra) must surface
+ # as ERROR/infra, NOT as MISS/no_doc (which would wrongly tell the model no
+ # document matched). An empty-but-valid result still returns MISS correctly.
+ try:
+ out, ids, new_ev = await _run_search(
+ tools,
+ hybrid_search,
+ queries,
+ top_n=20,
+ max_q=2,
+ doc_scope=doc_ids,
+ use_compiled=False,
+ fail_hard=True,
+ )
+ except Exception: # noqa: BLE001
+ _LOG.warning("[metadata search] retrieval failed", exc_info=True)
+ return ToolOutcome(
+ payload=[
+ {
+ "kind": "metadata_search",
+ "note": "Retrieval backend failed (embedding/infra). Retry later or fall back to search_chunks / retrieve for unfiltered search.",
+ }
+ ],
+ status=ERROR,
+ reason="infra",
+ metrics={"hits": 0, "new_evidence": 0},
+ )
+ _LOG.info(
+ "[metadata search] ok — %d doc(s) matched by filters, %d hit(s), %d new evidence",
+ len(doc_ids),
+ len(out),
+ new_ev,
+ )
+ return _search_outcome(out, ids, new_ev)
+
+
async def _exec_list_chunks(tools, doc_id: str) -> ToolOutcome:
"""Deep-read one document's full text (list_chunks tool).
@@ -908,7 +1168,7 @@ async def _exec_list_chunks(tools, doc_id: str) -> ToolOutcome:
QUERY-level miss (``miss``), never a dataset fact, so ``_tool_node`` must not
disable the tool over it.
"""
- from rag.advanced_rag.harness.tools.search import _chunk_id, list_chunks
+ from rag.advanced_rag.harness.tools.search import _LIST_CHUNKS_MAX_CHUNKS, _chunk_id, _chunk_text, list_chunks
try:
res = await list_chunks(tools, doc_id)
@@ -919,7 +1179,29 @@ async def _exec_list_chunks(tools, doc_id: str) -> ToolOutcome:
kbinfos = _seed_evidence(tools)
kb_seen = {_chunk_id(c) for c in kbinfos["chunks"] if isinstance(c, dict)}
seen = set()
- for c in (res.get("chunks") or [])[:30]:
+ doc_chunks = [c for c in (res.get("chunks") or []) if _chunk_id(c)]
+ # Section index (generic, not per-question): deep-reading a long document only
+ # helps if the model can see WHICH sections it holds. A 39-chunk article
+ # otherwise arrives as one undifferentiated wall of text and the answer-bearing
+ # section is what gets skipped (2026-09-16 FRAMES attribution: the right
+ # document was retrieved and read, the model still answered from the wrong
+ # section). The index lists every section heading + chunk id + size, so the
+ # model can target a section with a scoped search instead of re-reading all of
+ # it; it is metadata, not evidence, so it is not added to the citation pool.
+ index_lines = []
+ for i, c in enumerate(doc_chunks, 1):
+ txt = _chunk_text(c) or ""
+ head = next((ln.strip() for ln in txt.splitlines() if ln.strip()), "")
+ index_lines.append(f"{i}. [{_chunk_id(c)}] {head[:120]} ({len(txt)} chars)")
+ if index_lines:
+ out.append(
+ {
+ "kind": "doc_index",
+ "doc_id": str(doc_id),
+ "content": "Sections of this document — read the ones the question needs:\n" + "\n".join(index_lines),
+ }
+ )
+ for c in doc_chunks[:_LIST_CHUNKS_MAX_CHUNKS]:
cid = _chunk_id(c)
if not cid:
continue
@@ -1157,6 +1439,8 @@ async def execute_tool(tools, name: str, args: dict) -> ToolOutcome:
return await _exec_graph_explore(tools, args)
if name == "web_search":
return await _exec_web_search(tools, _arg_query_list(args, 2))
+ if name == "metadata_search":
+ return await _exec_metadata_search(tools, args)
_LOG.warning("[Action Session] unknown tool %r; ignored.", name)
return ToolOutcome(payload=[], status=ERROR, reason="bad_args")
@@ -1183,6 +1467,8 @@ async def _acompletion(mdl, messages: list, tools_list=None, temperature: float
kwargs = {"model": mdl.model_name, "messages": oai_messages, "temperature": temperature}
if tools_list:
kwargs["tools"] = tools_list
+ if timeout_s:
+ kwargs["timeout"] = timeout_s
response = await create(**kwargs)
from rag.advanced_rag.harness.stats import record_external_response
@@ -1245,7 +1531,7 @@ def _parse_tool_calls(msg) -> list:
# Do NOT drop it. OpenAI's protocol requires every assistant
# tool_call to be answered by a matching ``tool`` message — dropping
# one leaves a dangling tool_call and the next request is rejected
- # ("tool call result does not follow tool call"). Keep it as a
+ # ("tool call result does not follow tool call"). Keep it as an
# "unknown" call so _tool_node replies with a correction instead.
#
# Models most often emit the XML protocol tags (state / answer) as
@@ -1319,6 +1605,8 @@ class _SessionState(TypedDict, total=False):
_direction: str # this slot's question — needed to re-run retrieval later
_routed_docs: list # navigate_tree's top-n: scope for the `scoped` rung
_nav_rule_id: str # which rung control currently rests on ("" = finished)
+ _search_queries: list
+ _skipped_dup: int
async def _run_action_node(state: _SessionState) -> dict:
@@ -1413,6 +1701,7 @@ async def _tool_node(state: _SessionState) -> dict:
# makes the model re-issue redundant retrieves. The same-session cache only
# avoids RE-EXECUTING a repeated query, it still returns a response for it.
pending = state.get("_pending_calls") or []
+ ms_used = bool(state.get("_metadata_search_used", False))
seen_queries = list(state.get("_search_queries") or [])
skipped = 0
strikes = dict(state.get("_tool_strikes") or {})
@@ -1422,7 +1711,7 @@ async def _tool_node(state: _SessionState) -> dict:
# tool_call that comes in — a call only advances its own rung, and a later
# ladder continuation must continue from the SAME resting point.
pending_rule = state.get("_nav_rule_id", "")
- for c in pending:
+ for batch_index, c in enumerate(pending):
# Near-duplicate retrieval suppression: if the model re-issues the same
# intent as an earlier search (paraphrase), do NOT re-run ES — return a
# nudge so it patches / reframes instead of burning turns (Q30 burned
@@ -1449,6 +1738,27 @@ async def _tool_node(state: _SessionState) -> dict:
}
)
continue
+ # metadata_search ONE-SHOT guard: block any 2nd call within this direction.
+ if c["name"] == "metadata_search" and ms_used:
+ _LOG.info("[Action Session] blocking 2nd metadata_search this direction (one-shot guard)")
+ tool_msgs.append(
+ {
+ "role": "tool",
+ "tool_call_id": c["id"],
+ "content": json.dumps(
+ {
+ "passages": [
+ {
+ "kind": "metadata_search",
+ "note": "metadata_search is a ONE-SHOT pre-filter and was ALREADY used this direction. Continue with search_chunks / retrieve inside the documents it returned — do NOT call metadata_search again this direction.",
+ }
+ ]
+ },
+ ensure_ascii=False,
+ ),
+ }
+ )
+ continue
# Unknown tool name — never execute it; answer with a correction so the
# model can recover. Models usually emit the XML protocol tags (state /
# answer) as tool names; they belong in the reply body as plain text.
@@ -1477,6 +1787,8 @@ async def _tool_node(state: _SessionState) -> dict:
if q:
seen_queries.append(q)
evidence_ids.extend(oc.evidence_ids)
+ if c["name"] == "metadata_search":
+ ms_used = True
chunks = list(oc.payload or [])
# ── Policy: act on WHAT happened, not just on payload size ──────────
if oc.status == OK:
@@ -1575,6 +1887,7 @@ async def _tool_node(state: _SessionState) -> dict:
"_routed_docs": state.get("_routed_docs") or [],
"_direction": state.get("_direction", ""),
"_nav_rule_id": pending_rule,
+ "_metadata_search_used": ms_used,
}
@@ -2084,6 +2397,32 @@ def _extract_relevant_evidence(tools, direction: str, max_chunks: int = 4) -> st
return "\n".join(lines)
+def _dump_pool_evidence(tools) -> str:
+ """TEST-ONLY: full dump of the shared evidence pool for the session seed.
+
+ Counterpart to :func:`_extract_relevant_evidence`: no top-k cap and no
+ 300-char truncation — every chunk currently in ``tools.kbinfos["chunks"]``
+ is injected verbatim so the action session sees ALL prior evidence.
+ """
+ from rag.advanced_rag.harness.tools.search import _chunk_id, _chunk_text
+
+ kbinfos = getattr(tools, "kbinfos", None) or {}
+ chunks = kbinfos.get("chunks") or []
+ lines = []
+ for c in chunks:
+ cid = _chunk_id(c)
+ text = (_chunk_text(c) or "").replace("\n", " ")
+ lines.append(f"[{cid}] {text}")
+ dump = "\n".join(lines)
+ _LOG.info(
+ "[Action Session] seed ALREADY RETRIEVED dump: %d chunk(s), %d chars (kbinfos pool=%d)",
+ len(lines),
+ len(dump),
+ len(chunks),
+ )
+ return dump
+
+
async def run_action_session(
tools,
direction: str,
@@ -2099,7 +2438,11 @@ async def run_action_session(
system = load_prompt("action_run")
seed_user = f"Direction: {direction}\n\nState:\n{parent_state.render_slots()}"
- existing = _extract_relevant_evidence(tools, direction, max_chunks=4)
+ # TEST: `_extract_relevant_evidence` (top-4 / 300-char relevance digest) is
+ # DISABLED. Instead inject the FULL shared evidence pool so the action session
+ # sees every passage already retrieved this round (and prior rounds).
+ # existing = _extract_relevant_evidence(tools, direction, max_chunks=4)
+ existing = _dump_pool_evidence(tools)
if existing:
seed_user += "\n\nALREADY RETRIEVED (do NOT re-retrieve these — use them to fill slots or identify gaps):\n" + existing
@@ -2234,7 +2577,7 @@ async def initialize_state(tools, question, fanout_hint, deadline_left=None):
record_external_call(None)
except Exception: # noqa: BLE001
- pass
+ _LOG.debug("[Action Session] slot-table decomposition usage booking skipped", exc_info=True)
raw = await _init_chat(tools, system, user, tmo)
_book_raw_call()
diff --git a/rag/advanced_rag/harness/config.py b/rag/advanced_rag/harness/config.py
index 4d906a7e85..71ed68f8ba 100644
--- a/rag/advanced_rag/harness/config.py
+++ b/rag/advanced_rag/harness/config.py
@@ -30,6 +30,7 @@ from dataclasses import dataclass, field
_ALL_TOOLS = (
"retrieve",
"search_chunks",
+ "metadata_search",
"list_chunks",
"navigate_tree",
"navigate_structure",
diff --git a/rag/advanced_rag/harness/grep_sed_narrow.py b/rag/advanced_rag/harness/grep_sed_narrow.py
index 410eb83f46..9fd34f0c41 100644
--- a/rag/advanced_rag/harness/grep_sed_narrow.py
+++ b/rag/advanced_rag/harness/grep_sed_narrow.py
@@ -20,6 +20,16 @@ Fallback chain (never drops the answer):
-> original chunks returned as-is (upper _build_compact_evidence
head-truncation as the final safety valve)
+Table exemption: chunks that look like tables (HTML ``<table>``/``<tr>`` markup, or
+>=3 pipe rows — see ``search._is_table_chunk``) are NEVER narrowed, by any entry
+point of this module. Two reasons, both measured: the term window either cuts the
+``<table>`` opening tag, and then the downstream ``table_view`` renderer refuses
+the fragment (``"<table" not in text``) so the model is handed a partial raw-HTML
+row dump; or it keeps the tag and silently drops the remaining rows, and row
+order/coverage is exactly what decides table answers (a rank row can sit at 62% of
+a 14.7K-char table). Tables are returned VERBATIM — never re-windowed, never
+char-trimmed, and exempt from the per-chunk/total char budget.
+
Safety: only ``re.compile`` + pure string ops, no eval, no arbitrary code;
grep-term count cap + context clamp.
"""
@@ -28,16 +38,17 @@ import logging
import re
from rag.advanced_rag.harness.tools.search import (
- _split_sentences,
_is_fact_dense_sentence,
+ _is_table_chunk,
_narrow_by_keywords,
+ _split_sentences,
)
_LOG = logging.getLogger(__name__)
# Cost / safety caps.
_MAX_GREP_TERMS = 16
-_MAX_CONTEXT = 2
+_MAX_CONTEXT = 6
_DEFAULT_OUT_CHARS_PER_CHUNK = 1200
_DEFAULT_OUT_TOTAL_CHARS = 16000
# Head length kept per chunk when there is no match.
@@ -51,6 +62,19 @@ _CONTEXT_CHAR_BUDGET = 600
_MIN_NARROW_CHARS = 200
+def _safe_is_table(chunk) -> bool:
+ """Table detector guard: detection must never break narrowing.
+
+ Centralises the whole-table exemption (see the module docstring) so every
+ caller of this engine inherits it, instead of each call site re-implementing
+ the ``table_chunks / prose_chunks`` split.
+ """
+ try:
+ return bool(_is_table_chunk(chunk))
+ except Exception: # noqa: BLE001 - never let a detector bug drop a chunk
+ return False
+
+
def _escape_term(term: str) -> str:
"""Escape a plain grep term into a safe, word-boundary regex fragment.
@@ -213,7 +237,8 @@ def _apply_narrow(chunks: list[dict], kept_texts: list[str], matched: list[bool]
def _fallback_narrow_by_keywords(chunks: list[dict], keywords: str) -> list[dict]:
try:
return _narrow_by_keywords(chunks, keywords) or chunks
- except Exception:
+ except Exception: # noqa: BLE001 - the fallback must never break narrowing
+ _LOG.debug("[grep-sed] keyword fallback narrowing failed; keeping the chunks as-is", exc_info=True)
return chunks
@@ -234,6 +259,10 @@ def narrow_by_terms(
once mechanically (zero extra LLM). Still no hit -> narrowing is abandoned and
the original chunks are returned (matched=False); the caller must NOT treat a
failed narrow as an answer failure. Never raises.
+
+ Table chunks are never narrowed and never char-trimmed — see the module
+ docstring. They come back VERBATIM (as a shallow copy), so no caller needs its
+ own ``_is_table_chunk`` split any more.
"""
ctx = context or {"before": 0, "after": 0}
try:
@@ -262,14 +291,19 @@ def narrow_by_terms(
stats["chars_out"] = sum(len(_chunk_text(c)) for c in narrowed)
return {"kept": narrowed, "stats": stats}
+ # Whole-table exemption, computed once and reused by the char-budget pass
+ # below (``_apply_narrow`` preserves order and length, so the two lists stay
+ # aligned). ``False`` = "this chunk was NOT narrowed" -> returned verbatim.
+ table_flags = [_safe_is_table(c) for c in chunks]
+
def _run(active_patterns) -> tuple[list[str], list[bool]]:
texts: list[str] = []
flags: list[bool] = []
- for c in chunks:
+ for c, is_table in zip(chunks, table_flags):
raw = _chunk_text(c)
- if len(raw) <= _MIN_NARROW_CHARS:
+ if is_table or len(raw) <= _MIN_NARROW_CHARS:
texts.append(raw)
- flags.append(True)
+ flags.append(not is_table)
continue
text, ok = _exec_on_text(raw, active_patterns, context, max_out_chars_per_chunk)
texts.append(text)
@@ -306,7 +340,17 @@ def narrow_by_terms(
per_chunk_cap = max(200, min(max_out_chars_per_chunk, max_out_total_chars // max(1, len(kept))))
acc = 0
trimmed = []
- for c in kept:
+ for c, is_table in zip(kept, table_flags):
+ if is_table:
+ # Tables are indivisible and exempt from the char budget: a
+ # head slice keeps the header and the first rows and drops
+ # the answer row, and the dropped rows carry no marker, so
+ # the model reads a truncated table as a complete one. The
+ # budget therefore bounds PROSE only; a large table can push
+ # the narrowed prose set past ``max_out_total_chars``, which
+ # is the intended trade (readability/correctness > char cap).
+ trimmed.append(c)
+ continue
t = _chunk_text(c)
room = max_out_total_chars - acc
if room <= 0:
diff --git a/rag/advanced_rag/harness/orchestrator/sufficient_context.py b/rag/advanced_rag/harness/orchestrator/sufficient_context.py
index 6aaf3dc105..5cc093cf5b 100644
--- a/rag/advanced_rag/harness/orchestrator/sufficient_context.py
+++ b/rag/advanced_rag/harness/orchestrator/sufficient_context.py
@@ -59,9 +59,10 @@ _SCA_CLAIMS_CONTEXT_MAX = 48000
# the SCA can verify a draft against real retrieved text without a token blow-up.
_SCA_EVIDENCE_ANCHOR_CHARS = 300
# Table-structured chunks get their FULL text as the evidence anchor (bounded
-# only by _SCA_CLAIMS_CONTEXT_MAX): hint-token windowing is unreliable for
-# tables (the draft rarely contains the row's entity names), and truncating from
-# the head hides the answer rows that sit mid/late-table (Q86 rank-19 row).
+# only by _SCA_CLAIMS_CONTEXT_MAX), serialized to Markdown by _bounded_excerpt:
+# hint-token windowing is unreliable for tables (the draft rarely contains the
+# row's entity names), and truncating from the head hides the answer rows that
+# sit mid/late-table (Q86 rank-19 row).
_SCA_EVIDENCE_TABLE_CHARS = None # None = keep the whole chunk text
@@ -141,12 +142,24 @@ def _bounded_excerpt(text: str, hints: str, max_chars: int = 300) -> str:
overall budget): hint-token windowing fails for tables because the draft
rarely contains the row's entity names, and head-truncation hides answer
rows in the mid/late table. Plain text keeps the bounded window.
+
+ A table is also SERIALIZED to a Markdown view before it is handed over, using
+ the same renderer as the action-session view and the final answer path. The
+ SCA is a plain LLM call: raw ``<table>/<td>`` markup makes it parse cells
+ itself (the gap Q673 showed on the answer path), and the same table costs
+ ~61% as much in HTML as it does in Markdown. ``table_view_or_raw`` never
+ prunes rows (a row can carry the answer mid-table), so the SCA still sees the
+ complete row set — just in a readable form. No-op for non-tables.
"""
text = str(text or "").strip()
if not text:
return ""
if _is_table_text(text):
- return text
+ # Imported lazily so this module keeps no import-time dependency on the
+ # tools package (the other three render call sites do the same).
+ from rag.advanced_rag.harness.tools.table_view import table_view_or_raw
+
+ return table_view_or_raw(text)
max_chars = max(80, int(max_chars))
hint_tokens = [t for t in re.findall(r"[A-Za-z0-9_\u4e00-\u9fff]{3,}", str(hints or ""))]
lower = text.lower()
@@ -197,7 +210,8 @@ def _render_claim_context(claims, question: str = "", kbinfos: dict | None = Non
continue
block = f"Claim {cid} (draft):\n{rpt}"
# Evidence anchor: first line of each cited snippet (guards the draft).
- # Table chunks contribute their FULL text (see _bounded_excerpt).
+ # Table chunks contribute their FULL text as a Markdown view (see
+ # _bounded_excerpt).
if eids:
anchors: list[str] = []
for eid in eids:
diff --git a/rag/advanced_rag/harness/prompts/report_prompt.py b/rag/advanced_rag/harness/prompts/report_prompt.py
index 11a2ee0f90..58887bca6c 100644
--- a/rag/advanced_rag/harness/prompts/report_prompt.py
+++ b/rag/advanced_rag/harness/prompts/report_prompt.py
@@ -2,6 +2,13 @@
FINAL_ANSWER_SYSTEM = """You are a smart agent. Answer the user's question using ONLY the evidence provided below. Do not invent facts: if the evidence cannot support a claim, say so plainly instead of guessing.
+# Commitment (CRITICAL)
+You MUST commit to the best-supported answer. The evidence does NOT need to prove it exhaustively.
+If the Research Summary gives a Candidate answer (e.g. "slot 0 [count]: 2") and no evidence
+contradicts it, that candidate IS the answer — state it directly.
+Give the answer FIRST, then add a short caveat ("most likely", "based on the available evidence").
+Never replace a supported answer with a refusal.
+
# Answer target
First resolve the exact role requested by the user's question. Multi-hop questions
often mention bridge entities that are only clues. Do not answer with a bridge
@@ -22,11 +29,35 @@ Answer the question's own attribute using the evidence for THAT attribute. If th
supports a different attribute, say that you could only find the related (different) attribute and
do not present it as the answer to the requested one.
+# Answer verification (ALWAYS apply, no exceptions, before writing the answer)
+These are the failure modes that survive retrieval: the evidence IS in context and the
+answer still comes out wrong. Apply all five on every question.
+1. COUNTS / ENUMERATIONS: build the explicit item list FIRST, then report the count as
+ the LENGTH of that list. Never restate a count from the Research Summary from memory,
+ and never "correct" it by adding an item that is not in your list. If your list and
+ the Research Summary disagree, the length of YOUR list wins.
+2. "WHICH X SATISFIES A AND B": write a candidate x constraint check (one row per
+ candidate, one column per constraint the question states) and answer ONLY the
+ candidate that satisfies EVERY constraint. A candidate that fails one constraint is
+ out, even if it is the first or most prominent one.
+3. NUMBERS / RATIOS / DIFFERENCES: convert to the SAME unit first (m vs ft, km vs mi,
+ date vs year), state the two values you combine and the operation, then compute.
+ Never divide or compare across units, and never inject a figure the evidence does not
+ contain.
+4. CONFLICTING EVIDENCE: when two passages give different values for the same fact, pick
+ the one that satisfies the question's own constraint (the stated quantity, date or
+ entity), cite it, and say briefly which alternative you rejected.
+5. NO RESTATEMENT DRIFT: a number, date or entity in your answer that the evidence
+ passages below do not support must be re-derived from them or dropped — do not repeat
+ it just because the Research Summary said it.
+
# Language
Answer in the SAME language as the question. Translate retrieved evidence into that language as part of composing the answer; only verbatim quoted snippets may stay in their source language.
# Fallback
-If the evidence does not answer the question, reply with a clear statement that you don't have enough information based on the available sources (in the user's language).
+Only if the evidence is ENTIRELY unrelated to the question may you say you don't have enough
+information. If the evidence is related but incomplete, still give the best-supported answer
+(with a brief caveat) rather than declining.
"""
diff --git a/rag/advanced_rag/harness/tools/search.py b/rag/advanced_rag/harness/tools/search.py
index 16a947f090..7921bd0a55 100644
--- a/rag/advanced_rag/harness/tools/search.py
+++ b/rag/advanced_rag/harness/tools/search.py
@@ -8,6 +8,7 @@ module owns the retrieval implementations they call.
import logging
import re
from typing import Any
+
from common import settings
from rag.advanced_rag.harness.chunk_utils import ( # noqa: F401
_chunk_attr,
@@ -44,7 +45,6 @@ from rag.advanced_rag.harness.tools.text_processing import ( # noqa: F401
_stem,
)
-
# Fallbacks for callers that supply no retrieval configuration: the values the
# search tools used unconditionally before RAGTools carried settings.
_DEFAULT_SIMILARITY_THRESHOLD = 0.2
@@ -181,7 +181,7 @@ async def hybrid_search(
from rag.advanced_rag.harness.memory import add as _memory_add
_memory_add(tools, kbinfos.get("chunks", []) or [])
- except Exception:
+ except Exception: # noqa: BLE001, S110
pass # memory is best-effort; never fail the search over it.
# Narrow-or-keep: if the query keywords match any chunk, keep only the
# matching passages (shrinking the evidence handed to the LLM so a single
@@ -243,7 +243,7 @@ async def vector_search(tools, query: str, kb_ids: list[str] | None = None, top_
from rag.advanced_rag.harness.memory import add as _memory_add
_memory_add(tools, kbinfos.get("chunks", []) or [])
- except Exception:
+ except Exception: # noqa: BLE001, S110
pass
kbinfos["chunks"] = _narrow_or_keep(kbinfos.get("chunks", []), keywords, "Vector search")
return kbinfos
@@ -281,12 +281,80 @@ async def bm25_search(tools, query: str, kb_ids: list[str] | None = None, top_n:
from rag.advanced_rag.harness.memory import add as _memory_add
_memory_add(tools, kbinfos.get("chunks", []) or [])
- except Exception:
+ except Exception: # noqa: BLE001, S110
pass
kbinfos["chunks"] = _narrow_or_keep(kbinfos.get("chunks", []), keywords, "BM25 search")
return kbinfos
+async def metadata_search(
+ tools,
+ query: str,
+ filters: list[dict],
+ logic: str = "and",
+ kb_ids: list[str] | None = None,
+ top_n: int | None = None,
+ keywords: str = "",
+ doc_scope: list[str] | None = None,
+) -> dict:
+ """Hybrid retrieval restricted to documents whose metadata matches ``filters``.
+
+ Programmatic counterpart of the ``metadata_search`` tool (the tool's dispatch
+ lives in ``action_session``; this is the retrieval leg the pre-search stage
+ reuses). Pipeline: resolve matching doc_ids via the metadata-index push-down
+ (ES / Infinity), fall back to the in-memory ``meta_filter`` when push-down is
+ not viable -> intersect with the session document scope -> ``hybrid_search``
+ scoped to exactly those documents.
+
+ ``filters`` is a list of ``{key, value, op}`` conditions (e.g.
+ ``[{"key": "title", "op": "contains", "value": "New York"}]``). An empty
+ match (or no dataset metadata) is a normal empty result, never an exception.
+ """
+ top_n = _resolve_top_n(tools, top_n)
+ target_ids = kb_ids or list(dict.fromkeys(tools.kb_ids + [kb.id for kb in tools.sql_kbs]))
+ if not target_ids or not filters:
+ return {"chunks": [], "doc_aggs": []}
+
+ from api.db.services.doc_metadata_service import DocMetadataService
+ from common.metadata_utils import meta_filter
+ from common.misc_utils import thread_pool_exec
+
+ # 1) Resolve matching doc_ids: metadata-index push-down, in-memory fallback.
+ try:
+ doc_ids = await thread_pool_exec(DocMetadataService.filter_doc_ids_by_meta_pushdown, target_ids, filters, logic)
+ except Exception: # noqa: BLE001
+ _LOG.warning("[Metadata search] push-down failed; falling back to in-memory meta_filter", exc_info=True)
+ doc_ids = None
+ if doc_ids is None:
+ try:
+ metas = await thread_pool_exec(DocMetadataService.get_flatted_meta_by_kbs, target_ids)
+ doc_ids = meta_filter(metas, filters, logic) or []
+ except Exception: # noqa: BLE001
+ _LOG.warning("[Metadata search] in-memory fallback failed", exc_info=True)
+ doc_ids = []
+ doc_ids = [str(d) for d in (doc_ids or [])]
+
+ # 2) Intersect with the session's global document scope (if any).
+ if hasattr(tools, "scoped_doc_ids"):
+ doc_ids = tools.scoped_doc_ids(doc_ids) or []
+ if not doc_ids:
+ _LOG.info("[Metadata search] no documents matched filters=%s logic=%s", filters, logic)
+ return {"chunks": [], "doc_aggs": []}
+
+ # 3) Hybrid search restricted to exactly those documents (compiled expansion
+ # would pull in out-of-scope chunks and break the "only these docs" contract).
+ _LOG.info('[Metadata search] "%s" matched %d doc(s) via filters=%s', str(query)[:80], len(doc_ids), filters)
+ return await hybrid_search(
+ tools,
+ query,
+ kb_ids=target_ids,
+ top_n=top_n,
+ doc_scope=doc_ids,
+ keywords=keywords,
+ use_compiled=False,
+ )
+
+
# Compiled-product expansion lives in ``compiled_expansion`` and is
# re-exported here: navigation and the dynamic runner import these names
# from this module.
@@ -311,7 +379,7 @@ async def web_search(tools, query: str, keywords: str = "", retrieval_query: str
effective_query = f"{query} {retrieval_query}".strip()[:400] if retrieval_query else f"{query} {keywords}".strip() if keywords else query
web_res = await thread_pool_exec(tools.web_search.retrieve_chunks, effective_query)
return {"chunks": web_res.get("chunks", []), "doc_aggs": web_res.get("doc_aggs", [])}
- except Exception:
+ except Exception: # noqa: BLE001
_LOG.exception("web_search failed")
return {"chunks": [], "doc_aggs": []}
@@ -335,7 +403,7 @@ async def structured_query(tools, query: str, keywords: str = "", kb_ids: list[s
sql_kb_ids = [kb.id for kb in sql_kbs]
try:
ans = await use_sql(query, tools.field_map, tenant_id, tools.chat_mdl, quota=True, kb_ids=sql_kb_ids, doc_ids=doc_scope)
- except Exception:
+ except Exception: # noqa: BLE001
_LOG.exception("structured_query failed")
return {"answer": "", "chunks": [], "doc_aggs": []}
if not ans:
@@ -353,8 +421,12 @@ async def structured_query(tools, query: str, keywords: str = "", kb_ids: list[s
# ─── do exact keyword/pattern locate + full-document deep-read like dynamic. ───
_GREP_TERMS_MAX = 10
-_GREP_OUT_CHARS_PER_CHUNK = 700
-_GREP_OUT_TOTAL_CHARS = 8000
+# Per-chunk window for a grep hit. 700 chars was measured to hide mid-section
+# answer rows (a rank row at 62% of a 14.7K-char table, a "her father was an ice
+# hockey player" clause in an Early-life section); the window is now centred on
+# the matched line (see the narrow call below) and doubled.
+_GREP_OUT_CHARS_PER_CHUNK = 1500
+_GREP_OUT_TOTAL_CHARS = 12000
_LIST_CHUNKS_MAX_CHUNKS = 80
@@ -443,7 +515,10 @@ async def grep_search(
prose_chunks,
terms,
keywords=str(query).strip(),
- context={"before": 1, "after": 0},
+ # Centre the window on the matched line instead of showing the
+ # match alone: the fact that answers the question usually sits
+ # one or two lines around the grep hit, not on it.
+ context={"before": 2, "after": 3},
max_out_chars_per_chunk=_GREP_OUT_CHARS_PER_CHUNK,
max_out_total_chars=_GREP_OUT_TOTAL_CHARS,
)
@@ -459,7 +534,7 @@ async def grep_search(
sum(len(str(c.get("content_with_weight") or c.get("content") or "")) for c in kept) / 1000.0,
)
res["chunks"] = kept
- except Exception:
+ except Exception: # noqa: BLE001
_LOG.exception("[Grep search] narrow failed; using raw BM25 candidates.")
_g = res.get("chunks") or []
if _g:
@@ -492,7 +567,7 @@ async def list_chunks(tools, doc_id: str) -> dict:
_LOG.info("[List chunks] Deep-reading document %s", doc_id)
try:
full = await tools.fetch_full_document(str(doc_id).strip())
- except Exception:
+ except Exception: # noqa: BLE001
_LOG.exception("[List chunks] fetch_full_document failed doc=%s", doc_id)
return {"chunks": [], "doc_aggs": []}
chunks = (full.get("chunks") or [])[:_LIST_CHUNKS_MAX_CHUNKS]
@@ -529,7 +604,7 @@ async def _load_specific_chunks(tools_slot, chunk_ids: list[str], doc_scope: lis
for doc_id in doc_scope[:8]:
try:
full = await tools_slot.fetch_full_document(doc_id)
- except Exception:
+ except Exception: # noqa: BLE001
_LOG.exception("[grep_chunks] fetch_full_document failed for doc_id=%s", doc_id)
continue
for c in full.get("chunks", []) or []:
@@ -640,7 +715,7 @@ def _base_chat_mdl(tools_slot):
)
return None
return mdl
- except Exception:
+ except Exception: # noqa: BLE001
_LOG.exception("[dynamic] failed to resolve base chat model")
return None
diff --git a/rag/advanced_rag/harness/tools/table_view.py b/rag/advanced_rag/harness/tools/table_view.py
new file mode 100644
index 0000000000..edc0313cca
--- /dev/null
+++ b/rag/advanced_rag/harness/tools/table_view.py
@@ -0,0 +1,158 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Render HTML tables into an LLM-friendly Markdown view.
+
+Why this exists
+---------------
+The compiled chunks of a Wikipedia-sourced KB carry machine-generated HTML
+tables. Passing them raw is both expensive and hard to read: a person infobox is
+1-2K chars of ``<table>/<td>`` markup, and a question like "how many children did
+all ten nominees have" needs ten of them at once. The 2026-09-16 FRAMES
+attribution showed the values ARE in the KB while the model still answers "cannot
+be determined" — the numbers were never surfaced in a form the model could
+aggregate.
+
+Approach
+--------
+Matches the established practice for tables in RAG (serialize tables to
+Markdown; key-value form for attribute tables):
+
+* two-column table (Wikipedia infobox) -> Markdown-KV lines: ``Children: 3``
+* any other table (ranked list, election results, timeline) -> a Markdown pipe
+ table with the header and EVERY row, because rank/order/completeness decide
+ those answers and row-window narrowing silently drops the answer row.
+
+Values are kept VERBATIM (parentheticals, units, "plus the perpetrators"),
+entities/whitespace are collapsed, all-empty rows are dropped. The raw HTML is
+never destroyed by this module: callers keep the original chunk in the evidence
+pool (for citation) and use the view only for the model-visible text.
+"""
+
+from __future__ import annotations
+
+import re
+
+from bs4 import BeautifulSoup
+
+# Generous ceilings. A Wikipedia election / standings table can run to 50K+ raw
+# chars across 100+ rows. Capping at 4K chars / 60 rows silently dropped the
+# answer row for exactly the "large / multi-column" tables this module exists to
+# serve (e.g. a 14.7K-char standings table where the rank-19 row sat at ~62% of
+# the table). We keep a ceiling only to stop one pathological table from eating
+# the whole evidence budget, not to truncate normal tables; the caller still
+# holds the raw chunk and can read it on demand.
+_MAX_ROWS = 400
+_MAX_CHARS_PER_TABLE = 20000
+
+
+def _cell_text(cell) -> str:
+ """Whitespace-collapsed cell text (tags stripped, entities resolved)."""
+ return re.sub(r"\s+", " ", cell.get_text(" ", strip=True)).strip()
+
+
+def _table_rows(table) -> list[list[str]]:
+ """Non-empty rows of ``table`` as lists of cell strings."""
+ rows: list[list[str]] = []
+ for tr in table.find_all("tr"):
+ vals = [_cell_text(c) for c in tr.find_all(["td", "th"])]
+ if any(vals):
+ rows.append(vals)
+ return rows
+
+
+def _md_cell(value: str) -> str:
+ """Escape a value for a Markdown pipe cell (the delimiter must stay intact)."""
+ return value.replace("\\", "\\\\").replace("|", "\\|")
+
+
+# Tokens that appear in a Wikipedia table's "units" row — the row right under the
+# header that reads e.g. "No. | % | No. | %". It carries no data and only
+# pollutes a Markdown pipe table, so it is dropped before rendering.
+_UNIT_TOKENS = {"", "no.", "no", "n", "%", "#", "—", "-", "•", "·"}
+
+
+def _is_units_row(vals: list[str]) -> bool:
+ if not vals:
+ return False
+ return all(v.strip().lower() in _UNIT_TOKENS for v in vals)
+
+
+def _render_one(table) -> str | None:
+ rows = _table_rows(table)
+ if not rows:
+ return None
+ # Infobox: two columns, a real key in the first cell and a value in the second
+ # (the photo-caption row has an empty second cell and is dropped here).
+ pairs = [(r[0], r[1]) for r in rows if len(r) == 2 and r[0] and r[1]]
+ if len(pairs) >= 3:
+ return "\n".join(f"{k}: {v}" for k, v in pairs[:_MAX_ROWS])[:_MAX_CHARS_PER_TABLE]
+ width = max(len(r) for r in rows)
+ header = rows[0]
+ # Drop header-repeat / units rows so the pipe body starts at real data
+ # (otherwise a wide election table leads with a "No. | % | No. | %" junk row).
+ body = [r for r in rows[1 : _MAX_ROWS + 1] if not _is_units_row(r)]
+ if not body:
+ return None
+
+ def _line(cells: list[str]) -> str:
+ padded = (cells + [""] * width)[:width]
+ return "| " + " | ".join(_md_cell(c) for c in padded) + " |"
+
+ caption = table.find("caption")
+ cap = (caption.get_text(" ", strip=True).strip() if caption else "") or ""
+ out: list[str] = []
+ if cap:
+ out.append(f"**Table: {cap}**")
+ out += [_line(header), "|" + "---|" * width, *(_line(r) for r in body)]
+ if len(rows) - 1 > len(body):
+ out.append(f"... ({len(rows) - 1 - len(body)} more row(s) omitted)")
+ return "\n".join(out)[:_MAX_CHARS_PER_TABLE]
+
+
+def render_tables(text: str) -> str | None:
+ """Replace every HTML table in ``text`` with a Markdown view.
+
+ Returns ``None`` when ``text`` holds no renderable table, so callers can keep
+ their previous behaviour untouched. Never raises: a parse failure falls back
+ to ``None``.
+ """
+ if not text or "<table" not in text.lower():
+ return None
+ try:
+ soup = BeautifulSoup(text, "html.parser")
+ except Exception: # noqa: BLE001
+ return None
+ views: list[str] = []
+ for table in soup.find_all("table"):
+ try:
+ view = _render_one(table)
+ except Exception: # noqa: BLE001
+ view = None
+ if view:
+ views.append(view)
+ if not views:
+ return None
+ return "\n\n".join(views)
+
+
+def table_view_or_raw(text: str) -> str:
+ """``render_tables(text)`` when it produces something, else ``text`` unchanged.
+
+ Small convenience for the two call sites that only want "the best available
+ model-visible form of this chunk".
+ """
+ view = render_tables(text or "")
+ return view if view else (text or "")
diff --git a/rag/advanced_rag/harness/tools/text_processing.py b/rag/advanced_rag/harness/tools/text_processing.py
index 05898d272b..9f0c04158f 100644
--- a/rag/advanced_rag/harness/tools/text_processing.py
+++ b/rag/advanced_rag/harness/tools/text_processing.py
@@ -219,7 +219,7 @@ try: # available at runtime — nltk already backs rag/nlp/synonym.py
from nltk.stem import PorterStemmer as _PorterStemmer
_porter_stem = _PorterStemmer().stem
-except Exception: # pragma: no cover - exercised only where nltk is absent
+except Exception: # noqa: BLE001 # pragma: no cover - exercised only where nltk is absent
_porter_stem = None
# Longest first: "nominations" must lose "ations", not just the trailing "s".
@@ -328,9 +328,7 @@ def _is_fact_dense_sentence(sent: str) -> bool:
low = sent.lower()
if _FACT_RE.search(sent) or _FACT_RE.search(low):
return True
- if _PROPER_NOUN_RE.search(sent):
- return True
- return False
+ return bool(_PROPER_NOUN_RE.search(sent))
def _narrow_content(content: str, kwds: list[str]) -> str | None:
@@ -353,7 +351,16 @@ def _narrow_content(content: str, kwds: list[str]) -> str | None:
# table), and sentence-window narrowing truncates them to a header-only snippet.
low_content = content.lower()
if "<table" in low_content or "<tr" in low_content or "<td" in low_content:
- return "..." + _highlight_keywords(content, kwds) + "..."
+ # Serialize the table to Markdown before the model sees it. The format
+ # comparison over 11 serializations ranks Markdown-KV first for field
+ # lookups (key: value beats header/position alignment) and Markdown
+ # tables as the cost/accuracy compromise; raw HTML is the expensive and
+ # least readable option. The row set is not pruned — rank/order and
+ # completeness decide table answers. Falls back to the raw text when
+ # nothing renders.
+ from rag.advanced_rag.harness.tools.table_view import table_view_or_raw
+
+ return "..." + _highlight_keywords(table_view_or_raw(content), kwds) + "..."
pipe_rows = sum(1 for line in content.splitlines() if line.count("|") >= 2)
if pipe_rows >= 3:
return "..." + _highlight_keywords(content, kwds) + "..."
@@ -397,7 +404,7 @@ def _highlight_keywords(text: str, kwds: list[str]) -> str:
terms: list[str] = list(phrases)
# Add stem-matched words NOT already inside a phrase, so "nominated" still
# gets starred for keyword "nominations" while "Atlanta Braves" stays whole.
- verbatim, stemmed = _keyword_forms(kwds)
+ _verbatim, stemmed = _keyword_forms(kwds)
stem_set = {s for seq in stemmed for s in seq}
if stem_set:
for word in re.findall(r"[A-Za-z]+", text):
diff --git a/rag/prompts/action_run.md b/rag/prompts/action_run.md
index 059cf50cc2..8af822faa5 100644
--- a/rag/prompts/action_run.md
+++ b/rag/prompts/action_run.md
@@ -6,45 +6,63 @@ Input: the user message contains ONE research `Direction`, plus the current
Execute this single direction. Reply with EXACTLY ONE of the three below.
The three differ in HOW they are delivered — read this carefully, because only
-the first one is a tool call:
+the first one is a real tool call. `state` and `answer` are NOT tools: never emit
+`<state>` / `<answer>` as a tool call and never name a native call after them.
1) TOOL CALL MODE — a real tool call: call `retrieve` with 1-3 corpus queries.
Results arrive in the next turn.
2) STATE PATCH MODE — NOT a tool call. Write this XML as plain TEXT in your
- reply body (do not call any tool named "state"):
+ reply body:
<state>
-{"new_states": [
- {"state": [{"id": <int>, "candidate": "<value>", "candidate_strength": <0..1>, "discovered_clues": ["..."]}, ...]},
- ...more branches allowed...
-]}
+{"new_states": [{"state": [
+ {"id": 1, "candidate": "1571", "candidate_strength": 0.92,
+ "discovered_clues": ["chunk 42: 'the treaty was signed in 1571'"]},
+ {"id": 2, "candidate": null, "candidate_strength": 0.0,
+ "discovered_clues": ["candidate 'Palermo' contradicted by chunk 7"]}
+]}]}
</state>
-Rules: patch ONLY existing ids; include ONLY changed variables; every change must trace to retrieved evidence; candidate_strength semantics: proven >0.9, strong 0.7-0.9, tentative 0.4-0.7, weak <0.4. An EMPTY branch list (`"new_states": []`) signals no progress — emit it rather than calling tools forever.
+Rules: patch ONLY existing ids; include ONLY changed variables; every change
+must trace to retrieved evidence; candidate_strength semantics: proven >0.9,
+strong 0.7-0.9, tentative 0.4-0.7, weak <0.4. Eliminate an unverifiable
+candidate by setting `candidate` to null with a clue documenting why. An EMPTY
+branch list (`"new_states": []`) signals no progress — emit it rather than
+calling tools forever.
3) FINAL ANSWER MODE — NOT a tool call either. Write this XML as plain TEXT in
- your reply body (do not call any tool named "answer"). Use it only when ALL
- slots can be filled consistently:
+ your reply body. Use it only when ALL slots can be filled consistently:
<answer>
-{"answer": "<final answer text>", "new_state": [{"id": ..., "candidate": ..., "candidate_strength": ...}]}
+{"answer": "1571", "new_state": [{"id": 1, "candidate": "1571", "candidate_strength": 0.92}]}
</answer>
CRITICAL RULES
- Think before choosing a mode, but output exactly ONE mode per response.
-- Strength >0.7 on the answer slot means you MUST emit final answer instead of another state patch.
-- ALWAYS end this action with a state patch: a patch with your updates, or `<state>{"new_states": []}</state>` if you found nothing new.
+- EXCEPTION to "always patch": when the answer slot reaches strength >0.7 you
+ MUST emit a final answer instead of another state patch.
+- Otherwise ALWAYS end this action with a state patch: a patch with your
+ updates, or `<state>{"new_states": []}</state>` if you found nothing new.
- **Do NOT keep calling tools once the direction is reasonably exhausted.** If further searches return repetitive, irrelevant, or empty results, immediately return a state patch (with updates or empty). Extra redundant searches waste the session — stop after 1-2 useful tool calls per direction unless a NEW fact is actually emerging.
- ACTION COMPLETION IS MANDATORY: when you have what you need (or hit a dead end), output the state patch now. Do not ask to continue searching.
-- Unverifiable candidates must be eliminated (set candidate null) with a clue documenting why.
- Partial verification is OK: record a candidate at tentative strength (0.4-0.7) if you can't fully verify it yet, and move on.
+WRONG — never do this:
+- calling a tool named "state" or "answer" (they are plain-text XML blocks);
+- wrapping either XML block in markdown fences.
+
# TOOL PLAYBOOK
-You get tools only in medium / high (7 tools: `retrieve`, `search_chunks`, `list_chunks`, `navigate_tree`, `navigate_structure`, `calculate`, `web_search`) and ultra (those 7 + `graph_explore`). Low mode has NO tool loop — answer with plain retrieval. Every native tool call still REQUIRES the decision envelope from CRITICAL RULES (it is a mandatory tool parameter, not optional).
+Available tools: `retrieve`, `search_chunks`, `metadata_search`, `list_chunks`,
+`navigate_tree`, `navigate_structure`, `calculate`, and `web_search` (only when a
+web provider is configured); `graph_explore` joins them in ultra mode. Low mode
+has NO tool loop — answer with plain retrieval. Per-tool WHEN TO CALL /
+DO NOT CALL / ARGUMENTS / IF IT FAILS details live in each tool's own schema —
+this playbook covers only how to COMBINE tools and when to STOP.
## 1. Combination chains (call in this order)
- **You already hold a `doc_id`** → `navigate_structure(doc_id, query)` to find the right passage, then `list_chunks(doc_id)` to read it. Do NOT call `navigate_tree` first.
- **No `doc_id` yet, and the corpus is large** → `navigate_tree(query)` to route to candidate documents, take a `doc_id`, then `navigate_structure(doc_id, query)` → `list_chunks(doc_id)`.
+- **You can name the document / need to narrow the search** → `metadata_search(query[1-2], filters)` ONCE per direction to pre-filter by title, then `search_chunks` / `retrieve` INSIDE the returned documents.
- **Exact term / short answer** → `retrieve(query[1-3])` first; if snippets are insufficient, `search_chunks(query[1-2])` (semantic, may find passages with NO shared surface words); if you need the full document, `list_chunks(doc_id)`.
- **You must DERIVE a number** → first collect every needed number with any of the above, then `calculate(question, facts)` with the facts verbatim, and report the computed result as-is. If the answer is already one of the stated numbers, answer directly.
- **Relational multi-hop (ultra only)** → get a start entity from `search_chunks` / `navigate_structure`, then `graph_explore(query, doc_scope)`.
@@ -54,8 +72,6 @@ You get tools only in medium / high (7 tools: `retrieve`, `search_chunks`, `list
- Make at most **1-2 useful tool calls per direction**, then emit a state patch. Do not keep searching once the direction is reasonably exhausted.
- Re-submitting the SAME intent with a paraphrase is intercepted as a near-duplicate and SKIPPED (you get a nudge, not new results). Change the angle or patch what you have.
- If a compile-only tool (`navigate_tree` / `navigate_structure` / `graph_explore`) returns "no compiled structure", switch to `search_chunks` / `retrieve` / `list_chunks` **immediately**. A second such result disables that tool for the REST of the session — do not retry it.
-- `web_search` only appears when a web provider is configured; if it does, use it ONLY for world knowledge / time-sensitive facts that plausibly live outside the fixed corpus.
-- `list_chunks` accepts ONLY `doc_id` (no `chunk_ids` argument) — you cannot ask it to read specific chunks; it returns the whole document (capped).
## 3. What a tool result means → your next action
@@ -69,3 +85,8 @@ Each tool returns a status. Act on it:
| `poor` | Output returned but too weak to use | Add evidence with another tool |
| `redundant` | Every hit was already in your evidence | Stop re-searching; emit a `<state>` patch with what you have |
| `error` | Infrastructure / provider failure | Switch tools; do not retry the same call |
+
+Special case — `metadata_search` (at most ONCE per direction):
+- `ok` → continue with `search_chunks` / `retrieve` INSIDE the documents it returned; do NOT call it again this direction.
+- `miss` / `empty` → no title matched; drop the filter and fall back to `search_chunks` / `navigate_tree`.
+- `poor` → the filter was too narrow; widen it once, or abandon it for plain retrieval.
diff --git a/test/unit_test/rag/advanced_rag/test_action_session_schema.py b/test/unit_test/rag/advanced_rag/test_action_session_schema.py
index 0f4335560d..1c2bc92f77 100644
--- a/test/unit_test/rag/advanced_rag/test_action_session_schema.py
+++ b/test/unit_test/rag/advanced_rag/test_action_session_schema.py
@@ -14,6 +14,7 @@ PLAYBOOK_ANCHORS = ["WHEN TO CALL", "DO NOT CALL", "ARGUMENTS", "OUTPUT", "IF IT
_EXECUTOR_SUPPORTED = {
"retrieve": {"query", "doc_scope"},
"search_chunks": {"query"},
+ "metadata_search": {"query", "filters", "logic"},
"list_chunks": {"doc_id"},
"navigate_tree": {"query", "keywords"},
"navigate_structure": {"doc_id", "query", "kind"},
@@ -42,12 +43,12 @@ def test_tool_specs_have_playbook_sections():
def test_active_tool_specs_tool_surface():
- """Mode → exposed tool count: low=0, medium/high=7, ultra=8, web-hidden=6."""
+ """Mode → exposed tool count: low=0, medium/high=8, ultra=9, web-hidden=7."""
assert len(_active_tool_specs(_mk_tools("low"))) == 0
- assert len(_active_tool_specs(_mk_tools("medium"))) == 7
- assert len(_active_tool_specs(_mk_tools("high"))) == 7
- assert len(_active_tool_specs(_mk_tools("ultra"))) == 8
- assert len(_active_tool_specs(_mk_tools("medium", web=False))) == 6
+ assert len(_active_tool_specs(_mk_tools("medium"))) == 8
+ assert len(_active_tool_specs(_mk_tools("high"))) == 8
+ assert len(_active_tool_specs(_mk_tools("ultra"))) == 9
+ assert len(_active_tool_specs(_mk_tools("medium", web=False))) == 7
def test_schema_params_match_executor():
@@ -63,11 +64,12 @@ def test_action_run_prompt_has_playbook():
"""action_run.md exposes the TOOL PLAYBOOK and only references real tools."""
prompt = load_prompt("action_run")
assert "TOOL PLAYBOOK" in prompt
- for tool in ("navigate_structure", "calculate", "graph_explore"):
+ for tool in ("metadata_search", "navigate_structure", "calculate", "graph_explore"):
assert tool in prompt
for tool in (
"retrieve",
"search_chunks",
+ "metadata_search",
"list_chunks",
"navigate_tree",
"navigate_structure",
@@ -76,3 +78,22 @@ def test_action_run_prompt_has_playbook():
"graph_explore",
):
assert tool in _TOOL_MAP
+
+
+def test_action_run_prompt_covers_metadata_search_statuses():
+ """The playbook tells the model how to react to metadata_search outcomes."""
+ prompt = load_prompt("action_run")
+ assert "metadata_search" in prompt
+ # ok / miss / poor handling for the one-shot pre-filter tool.
+ assert "ONCE per direction" in prompt
+ lowered = prompt.lower()
+ for status in ("`ok`", "`miss`", "`poor`"):
+ assert status in lowered
+
+
+def test_action_run_prompt_does_not_hardcode_tool_count():
+ """The tool surface lives in config._ALL_TOOLS — the prompt must not
+ advertise a stale count that drifts from the real surface."""
+ prompt = load_prompt("action_run")
+ for stale in ("7 tools", "8 tools", "those 7"):
+ assert stale not in prompt