RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agentic_rag): table-aware retrieval truncation fixes + action-session loop guards (#19112)
Caobaining1880876f60ff8
diff --git a/api/db/services/llm_service.py b/api/db/services/llm_service.pyindex 0e84b7c72b..6c10e6473f 100644--- a/api/db/services/llm_service.py+++ b/api/db/services/llm_service.py@@ -557,6 +557,18 @@ class LLMBundle(LLM4Tenant):raiseif total_tokens:logging.info("LLMBundle.async_chat_streamly used_tokens: %d", total_tokens)+ # The streaming backend returns the authoritative total through+ # the integer sentinel, but may not populate mdl.last_usage.+ # Preserve that total so CountingChatModel can attribute it to+ # the current phase instead of reporting zero tokens.+ split = getattr(self.mdl, "last_usage", None) or {}+ prompt_tokens = int(split.get("prompt_tokens", 0) or 0)+ completion_tokens = int(split.get("completion_tokens", 0) or 0)+ self.mdl.last_usage = {+ "prompt_tokens": prompt_tokens if prompt_tokens + completion_tokens == total_tokens else 0,+ "completion_tokens": completion_tokens if prompt_tokens + completion_tokens == total_tokens else 0,+ "total_tokens": total_tokens,+ }usage_details = self._report_usage(total_tokens)if generation:generation.update(output={"output": ans}, usage_details=usage_details)@@ -604,6 +616,16 @@ class LLMBundle(LLM4Tenant):raiseif total_tokens:logging.info("LLMBundle.async_chat_streamly_delta used_tokens: %d", total_tokens)+ # Streaming providers may expose only the integer total sentinel;+ # make it available to the phase accounting proxy.+ split = getattr(self.mdl, "last_usage", None) or {}+ prompt_tokens = int(split.get("prompt_tokens", 0) or 0)+ completion_tokens = int(split.get("completion_tokens", 0) or 0)+ self.mdl.last_usage = {+ "prompt_tokens": prompt_tokens if prompt_tokens + completion_tokens == total_tokens else 0,+ "completion_tokens": completion_tokens if prompt_tokens + completion_tokens == total_tokens else 0,+ "total_tokens": total_tokens,+ }usage_details = self._report_usage(total_tokens)if generation:generation.update(output={"output": ans}, usage_details=usage_details)diff --git a/rag/advanced_rag/agentic_rag_graph.py b/rag/advanced_rag/agentic_rag_graph.pyindex b8fc238235..541af606c5 100644--- a/rag/advanced_rag/agentic_rag_graph.py+++ b/rag/advanced_rag/agentic_rag_graph.py@@ -426,7 +426,7 @@ async def _expand_fanouts(tools, question: str, answer_conf: dict) -> list[str]:_MAX_SNIPPET_POOL = 60_DRILL_RESERVE = 12 # slots kept free after the FIRST prefetch so the research# executor can top up evidence-_SCA_VIEW_CAP = 24 # chunks shown to the Sufficient Context Agent per review+_SCA_VIEW_CAP = 60 # chunks shown to the Sufficient Context Agent per review (24 -> 60: 24 of 225 hid the answer-bearing table chunk from the SCA)async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: int | None = None) -> int:@@ -879,6 +879,17 @@ def build_agentic_graph(if time_left < _MIN_ROUND_HEADROOM_S:_LOG.info("[RAGAgent] only %.0fs left of the research budget; skipping further passes.", time_left)return {}+ round_no = int(state.get("search_rounds", 0)) + 1+ pool_before = len(((getattr(tools, "kbinfos", None) or state.get("kbinfos") or {}).get("chunks")) or [])+ unresolved_ids = [str(getattr(v, "id", "")) for v in getattr(state.get("slot_table"), "state", None) or [] if not getattr(v, "candidate", None)]+ _LOG.info(+ "[RAGAgent] ROUND %d start (search_rounds=%d, time_left=%.0fs, pool=%d chunks, unresolved slots=%s)",+ round_no,+ int(state.get("search_rounds", 0)),+ time_left,+ pool_before,+ unresolved_ids or "-",+ )t = max(20.0, min(_PASS_TIMEOUT_S, time_left - 25.0))slot_result = await _bounded(_run_slot_research_pass(tools, question, state, answer_conf, deadline_left=t),@@ -894,6 +905,14 @@ def build_agentic_graph(sum(1 for s in getattr(slot_table, "state", []) if getattr(s, "candidate", None)),len(slot_result.get("unresolved_slots") or []),)+ pool_after = len(((getattr(tools, "kbinfos", None) or state.get("kbinfos") or {}).get("chunks")) or [])+ _LOG.info(+ "[RAGAgent] ROUND %d end (+%d new chunks, pool=%d, unresolved=%d)",+ round_no,+ pool_after - pool_before,+ pool_after,+ len(slot_result.get("unresolved_slots") or []),+ )return slot_result# ── Node: draft (intermediate draft — Phase 3 reviewee) ──@@ -942,6 +961,41 @@ def build_agentic_graph(_LOG.info("[SCA] review view UNCHANGED since last round (%d stored / %d viewed); closing out.", len(chunks), len(view))return {"verdict": {"status": "INSUFFICIENT"}, "sca": {}, "no_progress": True}+ # Ensure the passages the action sessions actually used to fill slots+ # are visible to the SCA. slot_evidence maps slot_id -> evidence_ids+ # (chunk ids as recorded by _chunk_id). Resolve them to the chunks in+ # the pool, append any missing to the view, and pass their positions to+ # sufficient_context_agent (which indexes kbinfos["chunks"] by position).+ from rag.advanced_rag.harness.tools.search import _chunk_id++ slot_evidence = state.get("slot_evidence") or {}+ view_by_id = {_chunk_id(c): i for i, c in enumerate(view) if _chunk_id(c)}+ view_index_by_id = {}+ claims = []+ if draft_text:+ claims.append(("c0", draft_text, []))+ for sid, meta in (slot_evidence or {}).items():+ eids = list(meta.get("evidence_ids") or [])+ if not eids:+ continue+ # resolve chunk-id -> view position, appending missing chunks+ positions = []+ for eid in eids:+ pos = view_index_by_id.get(eid)+ if pos is None:+ match = next((i for i, c in enumerate(chunks) if _chunk_id(c) == eid), None)+ if match is not None:+ if _chunk_id(chunks[match]) not in view_by_id:+ view.append(chunks[match])+ view_by_id[_chunk_id(chunks[match])] = len(view) - 1+ pos = view_by_id[_chunk_id(chunks[match])]+ if pos is not None:+ positions.append(pos)+ view_index_by_id[eid] = pos+ claims.append((str(sid), (meta.get("candidate") or "")[:400] or f"(slot {sid} evidence)", positions))+ if not claims:+ claims = [("c0", draft_text or "(no draft)", list(range(len(view))))]+# The SCA renders claim evidence from ``tools.kbinfos`` by INDEX, so swap# in a view-only copy for the duration of the review — the indexed ids# then point at exactly the selected chunks. Restore afterwards so other@@ -956,7 +1010,7 @@ def build_agentic_graph(sufficient_context_agent(tools,question,- claims=[("c0", draft_text or "(no draft)", list(range(len(view))))],+ claims=claims,),min(_SCA_TIMEOUT_S, max(15.0, _remaining_s(state) - 10.0)),"sufficient-context review",@@ -1134,21 +1188,30 @@ def build_agentic_graph(return g.compile()-def _render_slot_draft(slot_table, collected_answer: str | None = None) -> str:+def _render_slot_draft(slot_table, collected_answer: str | None = None, slot_evidence: dict | None = None) -> str:"""Render a slot table into a fact-preserving draft for the SCA.Each resolved slot becomes a ``<type>: <candidate> [strength]`` line with itsdiscovered-clue tail; unresolved slots are listed explicitly so the SCA cancall them out as gaps. When the tree surfaced a ``collected_answer`` it leadsthe draft (it is the strongest candidate); the slot view stays as structured- context below it.-- Takes the shared ``State`` (slot table) object directly.+ context below it. When ``slot_evidence`` is provided, resolved slots and the+ collected answer carry their evidence ids / terminal type so the SCA can+ verify candidates against the passages that produced them."""slots = list(getattr(slot_table, "state", None) or [])lines = []if collected_answer:- lines.append(f"Candidate answer: {collected_answer}")+ answer_meta = (slot_evidence or {}).get("_answer", {})+ answer_ids = answer_meta.get("evidence_ids") or []+ answer_terminal = answer_meta.get("terminal_type")+ parts = []+ if answer_terminal:+ parts.append(f"terminal={answer_terminal}")+ if answer_ids:+ parts.append(f"evidence_ids={answer_ids}")+ suffix = " [" + ", ".join(parts) + "]" if parts else ""+ lines.append(f"Candidate answer: {collected_answer}{suffix}")lines.append("")if not slots:return "\n".join(lines) if lines else ""@@ -1160,8 +1223,17 @@ def _render_slot_draft(slot_table, collected_answer: str | None = None) -> str:cs = getattr(v, "candidate_strength", None)strength = f"{float(cs):.2f}" if isinstance(cs, (int, float)) else "?"clues = list(getattr(v, "discovered_clues", None) or [])- tail = "; ".join(str(c)[:80] for c in clues[-2:])- lines.append(f"- slot {vid} [{vtype}]: {cand} (strength={strength})" + (f" — {tail}" if tail else ""))+ tail = "; ".join(str(c)[:240] for c in clues[-4:])+ meta = (slot_evidence or {}).get(str(vid), {})+ evidence_ids = meta.get("evidence_ids") or []+ terminal = meta.get("terminal_type")+ details = []+ if terminal:+ details.append(f"terminal={terminal}")+ if evidence_ids:+ details.append(f"evidence_ids={evidence_ids}")+ suffix = " [" + ", ".join(details) + "]" if details else ""+ lines.append(f"- slot {vid} [{vtype}]: {cand} (strength={strength}){suffix}" + (f" — {tail}" if tail else ""))else:qc = list(getattr(v, "question_clues", None) or [])lines.append(f"- slot {vid} [{vtype}]: NOT RESOLVED ({'; '.join(str(c)[:80] for c in qc[:2])})")@@ -1233,16 +1305,23 @@ async def _run_slot_research_pass(tools, question: str, state: AgenticState, ansbase = getattr(tools, "kbinfos", None) or state.get("kbinfos") or {"chunks": [], "doc_aggs": []}tools.kbinfos = dict(base)+ # Shared cache across sessions to avoid duplicate retrievals+ shared_tool_cache = {}+ shared_search_queries = []+async def _one(v):direction = v.question_clues[0] if v.question_clues else questionasync with sem:- return await run_action_session(+ result = await run_action_session(tools=tools,direction=direction,parent_state=slot_table,deadline_left=max(20.0, deadline_left - 10.0),base_summary="",+ shared_tool_cache=shared_tool_cache,+ shared_search_queries=shared_search_queries,)+ return v.id, resultresults = await asyncio.gather(*[_one(v) for v in unresolved[:3]],@@ -1251,13 +1330,26 @@ async def _run_slot_research_pass(tools, question: str, state: AgenticState, anscollected = state.get("collected_answer")ledger = [e for e in _safe_list(state.get("attempted"), "state.attempted") if isinstance(e, dict)]- # Fold each session's new-state branches back into the slot table.- for r in results:- if isinstance(r, Exception):- _LOG.warning("[SlotResearch] action_session raised: %s", r)+ session_evidence: dict[str, dict] = {}+ # Fold each session's new-state branches back into the slot table, keeping+ # the session's evidence IDs and terminal type so the SCA can verify the+ # candidate against the passages that actually produced it.+ for item in results:+ if isinstance(item, Exception):+ _LOG.warning("[SlotResearch] action_session raised: %s", item)continue+ slot_id, r = itemif r.found_answer and not collected:collected = r.found_answer+ ev_ids = list(r.retrieved_evidence_ids or [])+ if ev_ids:+ session_evidence[str(slot_id)] = {+ "evidence_ids": ev_ids,+ "terminal_type": r.terminal_type,+ "terminal_payload": r.terminal_payload or {},+ "candidate": r.found_answer,+ "strength": None,+ }for ns in r.new_states or []:# merge the branch's candidate values into the shared tablemerged = _merge_slot_patch(slot_table, ns)@@ -1266,6 +1358,23 @@ async def _run_slot_research_pass(tools, question: str, state: AgenticState, ansledger.append({"q": (r.found_answer or str(getattr(r, "messages", [])))[:80] if hasattr(r, "found_answer") else "", "new": 1})+ # session_evidence is already keyed by slot_id; normalize into a+ # slot-evidence map the SCA consumer can read. Sessions that produced only+ # a collected answer (no slot patch) land under the "_answer" key.+ slot_evidence: dict[str, dict] = {}+ for sid, rec in session_evidence.items():+ slot_evidence[sid] = {+ "evidence_ids": list(dict.fromkeys(rec.get("evidence_ids") or [])),+ "terminal_type": rec.get("terminal_type"),+ "candidate": rec.get("candidate"),+ "strength": rec.get("strength"),+ }+ if slot_evidence:+ _LOG.info(+ "[SlotResearch] slot evidence bound: %s",+ {k: len(v["evidence_ids"]) for k, v in slot_evidence.items()},+ )+# Expose the updated table and its unresolved slots for the rewriter.out_table = slot_tableunresolved_slots = [@@ -1277,17 +1386,19 @@ async def _run_slot_research_pass(tools, question: str, state: AgenticState, ans}for v in slot_table.unresolved()]- draft = _render_slot_draft(out_table, collected)+ draft = _render_slot_draft(out_table, collected, slot_evidence=slot_evidence)_LOG.info("[SlotResearch] round done — %d slot(s) filled, unresolved=%d, collected_answer=%s",sum(1 for v in getattr(out_table, "state", []) if getattr(v, "candidate", None)),len(unresolved_slots),bool(collected),)+ _LOG.info("[SlotResearch] slot table after round:\n%s", draft)return {"slot_table": out_table,"collected_answer": collected,"unresolved_slots": unresolved_slots,+ "slot_evidence": slot_evidence,"slot_draft": draft,"rag_answer": draft or (state.get("rag_answer") or ""),"kbinfos": tools.kbinfos,diff --git a/rag/advanced_rag/harness/action_session.py b/rag/advanced_rag/harness/action_session.pyindex eaa32138d9..a3123b677e 100644--- a/rag/advanced_rag/harness/action_session.py+++ b/rag/advanced_rag/harness/action_session.py@@ -145,6 +145,8 @@ class Result:new_states: list = field(default_factory=list)found_answer: str | None = Noneretrieved_evidence_ids: list = field(default_factory=list)+ terminal_type: str | None = None+ terminal_payload: dict | None = None# ── Unified tool-result contract ─────────────────────────────────────────@@ -650,14 +652,23 @@ def _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=True) ->Imports the chunk helpers locally to keep ``tools.search`` (and its heavydeepdoc dependency chain) out of module-import time."""- from rag.advanced_rag.harness.tools.search import _chunk_id, _chunk_text, _doc_id+ from rag.advanced_rag.harness.tools.search import _chunk_id, _chunk_text, _doc_id, _is_table_chunkcid = _chunk_id(c)if cid in seen:return Falseseen.add(cid)ids.append(cid)- entry = {"id": str(cid), "content": (_chunk_text(c) or "")[:1200]}+ # Table chunks pass through UN-truncated: the 1200-char cap hides answer+ # rows in the mid/late table (Q86: rank-19 row at char 5181 of a 8274-char+ # standings table was cut, so the session model guessed the athlete). The+ # pool (kbinfos) already stores the full chunk; only the model-facing+ # session output was truncated here.+ _ct = _chunk_text(c) or ""+ if _is_table_chunk(c):+ entry = {"id": str(cid), "content": _ct}+ else:+ entry = {"id": str(cid), "content": _ct[:1200]}if include_doc_id:entry["doc_id"] = _doc_id(c)out.append(entry)@@ -1052,7 +1063,11 @@ async def _acompletion(mdl, messages: list, tools_list=None, temperature: floatkwargs = {"model": mdl.model_name, "messages": oai_messages, "temperature": temperature}if tools_list:kwargs["tools"] = tools_list- return await create(**kwargs)+ response = await create(**kwargs)+ from rag.advanced_rag.harness.stats import record_external_response++ record_external_response(response)+ return response# LiteLLMBase path — construct args via its own public method so provider# prefix / api_key / retries all ride along; then swap OUR tool schema in.import litellm@@ -1067,7 +1082,11 @@ async def _acompletion(mdl, messages: list, tools_list=None, temperature: floatargs["tools"] = tools_listargs["tool_choice"] = "auto"args.setdefault("num_retries", 0)- return await litellm.acompletion(**args, drop_params=True, timeout=timeout_s)+ response = await litellm.acompletion(**args, drop_params=True, timeout=timeout_s)+ from rag.advanced_rag.harness.stats import record_external_response++ record_external_response(response)+ return responseasync def _llm_once_with_tools(tools, mdl, messages: list):@@ -1136,7 +1155,7 @@ def _parse_terminal(content: str, parent_state: State) -> tuple:ns = apply_patch(parent_state, br.get("state", []) if isinstance(br, dict) else [])if ns is not None:branches.append(ns)- return branches, None+ return branches, None, "state", dataif "<answer>" in content:block = extract_tag(content, "answer") or "{}"data = extract_json(block) or {}@@ -1146,8 +1165,8 @@ def _parse_terminal(content: str, parent_state: State) -> tuple:patched = apply_patch(parent_state, data["new_state"])if patched is not None:final_state = patched- return [final_state], answer- return [], None+ return [final_state], answer, "answer", data+ return [], None, None, None# ── LangGraph subgraph state ─────────────────────────────────────────────@@ -1163,6 +1182,8 @@ class _SessionState(TypedDict, total=False):# terminal outputsnew_states: listfound_answer: Any+ terminal_type: str | None+ terminal_payload: dict | Noneretrieved_evidence_ids: list# budgetattempts: int@@ -1226,11 +1247,13 @@ async def _run_action_node(state: _SessionState) -> dict:"attempts": attempts,}- new_states, found_answer = _parse_terminal(content, state["parent_state"])+ new_states, found_answer, terminal_type, terminal_payload = _parse_terminal(content, state["parent_state"])if found_answer is not None or new_states:return {"new_states": new_states,"found_answer": found_answer,+ "terminal_type": terminal_type,+ "terminal_payload": terminal_payload,"_done": True,"attempts": attempts,}@@ -1440,6 +1463,7 @@ async def _finalize_node(state: _SessionState) -> dict:JSON, salvaging whatever the session learned."""parent = state["parent_state"]new_states, found_answer = [], None+ terminal_type, terminal_payload = None, Noneevidence_ids = list(state.get("retrieved_evidence_ids") or [])budget_prompt = (@@ -1453,16 +1477,16 @@ async def _finalize_node(state: _SessionState) -> dict:# (e.g. tool execution failed), else the provider rejects the history.finalize_msgs = _strip_unpaired_tool_calls(list(state["messages"])) + [HumanMessage(content=budget_prompt)]try:- async with asyncio.timeout(max(15.0, min(45.0, state.get("deadline_left") or 45.0))):+ async with asyncio.timeout(max(15.0, min(150.0, state.get("deadline_left") or 150.0))):fresp = await _acompletion(state["mdl"],finalize_msgs,tools_list=None,temperature=0.3,- timeout_s=45.0,+ timeout_s=150.0,)fcontent = fresp.choices[0].message.content or ""- new_states, found_answer = _parse_terminal(fcontent, parent)+ new_states, found_answer, terminal_type, terminal_payload = _parse_terminal(fcontent, parent)if found_answer:_LOG.info("[Action Session] answer salvaged from exhausted session")elif new_states:@@ -1493,7 +1517,14 @@ async def _finalize_node(state: _SessionState) -> dict:new_states = [patched]_LOG.warning("[Action Session] loose-clue patch (narrative)")- return {"new_states": new_states, "found_answer": found_answer, "_done": True, "retrieved_evidence_ids": evidence_ids}+ return {+ "new_states": new_states,+ "found_answer": found_answer,+ "terminal_type": terminal_type,+ "terminal_payload": terminal_payload,+ "_done": True,+ "retrieved_evidence_ids": evidence_ids,+ }def _strip_unpaired_tool_calls(messages: list) -> list:@@ -1547,6 +1578,19 @@ def _route(state: _SessionState) -> str:return "run_action" # nudge retry+def _route_after_tool(state: _SessionState) -> str:+ """Route after executing a tool batch: the turn budget is checked AFTER the+ tool responses are appended (a pending tool_call must always receive its+ matching tool response, but once the budget is spent we must NOT go back to+ run_action — that was the Q86 infinite-loop: the model kept emitting+ tool_calls, _pending_calls stayed non-empty, so the attempts check in+ ``_route`` was never reached and the session burned the whole PASS_TIMEOUT).+ """+ if state.get("attempts", 0) >= _action_max_turns(state):+ return "finalize"+ return "run_action"++def _build_session_graph():g = StateGraph(_SessionState)g.add_node("run_action", _run_action_node)@@ -1558,7 +1602,11 @@ def _build_session_graph():_route,{"tool": "tool", "finalize": "finalize", END: END, "run_action": "run_action"},)- g.add_edge("tool", "run_action")+ g.add_conditional_edges(+ "tool",+ _route_after_tool,+ {"run_action": "run_action", "finalize": "finalize"},+ )g.add_edge("finalize", END)return g.compile()@@ -1888,18 +1936,53 @@ async def run_nav_prefix(tools, direction: str, deadline_left: float, ctx: _NavC_SESSION_GRAPH = _build_session_graph()+def _extract_relevant_evidence(tools, direction: str, max_chunks: int = 4) -> str:+ """Extract chunks from tools.kbinfos relevant to direction, for seed_user injection."""+ from rag.advanced_rag.harness.tools.search import _chunk_id, _chunk_text++ kbinfos = getattr(tools, "kbinfos", None) or {}+ chunks = kbinfos.get("chunks") or []+ if not chunks:+ return ""++ dir_tokens = set(re.findall(r"[a-zA-Z0-9\u4e00-\u9fff]{2,}", (direction or "").lower()))+ if not dir_tokens:+ ranked = chunks[-max_chunks:]+ else:++ def _rel(c):+ text = (_chunk_text(c) or "").lower()+ return sum(1 for t in dir_tokens if t in text)++ ranked = sorted(chunks, key=_rel, reverse=True)++ lines = []+ for c in ranked[:max_chunks]:+ cid = _chunk_id(c)+ text = (_chunk_text(c) or "")[:300].replace("\n", " ")+ lines.append(f"[{cid}] {text}")+ return "\n".join(lines)++async def run_action_session(tools,direction: str,parent_state: State,deadline_left: float | None = None,base_summary: str = "",+ shared_tool_cache: dict | None = None,+ shared_search_queries: list | None = None,) -> Result:"""Bounded graph-edge session pursuing ONE direction."""from rag.prompts.template import load_promptsystem = load_prompt("action_run")seed_user = f"Direction: {direction}\n\nState:\n{parent_state.render_slots()}"++ existing = _extract_relevant_evidence(tools, direction, max_chunks=4)+ if existing:+ seed_user += "\n\nALREADY RETRIEVED (do NOT re-retrieve these — use them to fill slots or identify gaps):\n" + existing+if base_summary:seed_user += f"\n\nPrior round summary:\n{base_summary}"@@ -1951,8 +2034,8 @@ async def run_action_session("deadline_left": max(10.0, budget_left - spent),"_ctx_budget": _MAX_TOOL_RESPONSE_CHARS * 4,"_tool_chars": 0,- "_tool_cache": {},- "_search_queries": [],+ "_tool_cache": shared_tool_cache if shared_tool_cache is not None else {},+ "_search_queries": shared_search_queries if shared_search_queries is not None else [],"_skipped_dup": 0,"_tool_strikes": {},"_tool_outcomes": list(prefix_outcomes),@@ -1970,6 +2053,8 @@ async def run_action_session(new_states=final.get("new_states", []),found_answer=final.get("found_answer"),retrieved_evidence_ids=final.get("retrieved_evidence_ids", []),+ terminal_type=final.get("terminal_type"),+ terminal_payload=final.get("terminal_payload"),)@@ -2035,5 +2120,5 @@ async def initialize_state(tools, question, fanout_hint, deadline_left=None):elif not first_queries:first_queries = [question]root = State(state=slots, depth=0)- _LOG.info("[Action Session:init] %s", root.brief())+ _LOG.info("[Action Session:init] %s\n%s", root.brief(), root.render_slots())return root, first_queriesdiff --git a/rag/advanced_rag/harness/orchestrator/sufficient_context.py b/rag/advanced_rag/harness/orchestrator/sufficient_context.pyindex 70da66565a..5d68bcb258 100644--- a/rag/advanced_rag/harness/orchestrator/sufficient_context.py+++ b/rag/advanced_rag/harness/orchestrator/sufficient_context.py@@ -30,6 +30,7 @@ from __future__ import annotationsimport jsonimport logging+import refrom rag.advanced_rag.harness.stats import in_phasefrom rag.prompts.generator import PROMPT_JINJA_ENV, gen_json@@ -49,10 +50,28 @@ SCA_REVIEW = load_prompt("sca_select")# evidence and query_check enforces grounded facts.# Total cap for the rendered claims context (all per-claim reports + the overall# intermediate draft) keeps the SCA prompt well inside the model window.-_SCA_CLAIMS_CONTEXT_MAX = 9000+# Enlarged 9000 -> 48000 so a table-bearing evidence anchor (full table text)+# plus several claim reports fit; matches _MAX_TOOL_RESPONSE_CHARS * 4 used by+# the action-session context budget. Table chunks were structurally invisible+# at 9000 (Q86: rank row at ~62% of a 14.7K-char table never entered the view).+_SCA_CLAIMS_CONTEXT_MAX = 48000# Max chars of each cited snippet's first line appended as an evidence anchor so# the SCA can verify a draft against real retrieved text without a token blow-up._SCA_EVIDENCE_ANCHOR_CHARS = 300+# Table-structured chunks get their FULL text as the evidence anchor (bounded+# only by _SCA_CLAIMS_CONTEXT_MAX): hint-token windowing is unreliable for+# tables (the draft rarely contains the row's entity names), and truncating from+# the head hides the answer rows that sit mid/late-table (Q86 rank-19 row).+_SCA_EVIDENCE_TABLE_CHARS = None # None = keep the whole chunk text+++def _is_table_text(text: str) -> bool:+ """Corpus-neutral table detector: HTML table markup or >=3 pipe rows."""+ t = str(text or "")+ if "<table" in t.lower() or "<tr" in t.lower():+ return True+ pipe_rows = sum(1 for line in t.splitlines() if line.count("|") >= 2)+ return pipe_rows >= 3def _clamp(value, lo: float = 0.0, hi: float = 1.0) -> float:@@ -94,6 +113,42 @@ def _render_reports(reports: list[tuple[str, str]]) -> str:return "\n".join(f"Claim {cid}: {rpt}" for cid, rpt in reports if rpt)+def _bounded_excerpt(text: str, hints: str, max_chars: int = 300) -> str:+ """Keep a bounded evidence window around a term from the current draft.++ Table-structured text returns its FULL content (bounded only by the caller's+ overall budget): hint-token windowing fails for tables because the draft+ rarely contains the row's entity names, and head-truncation hides answer+ rows in the mid/late table. Plain text keeps the bounded window.+ """+ text = str(text or "").strip()+ if not text:+ return ""+ if _is_table_text(text):+ return text+ max_chars = max(80, int(max_chars))+ hint_tokens = [t for t in re.findall(r"[A-Za-z0-9_\u4e00-\u9fff]{3,}", str(hints or ""))]+ lower = text.lower()+ start = None+ for token in hint_tokens:+ pos = lower.find(token.lower())+ if pos >= 0:+ start = pos+ break+ if start is None:+ if len(text) <= max_chars:+ return text+ tail = max_chars // 2+ return text[: max_chars - tail] + " … " + text[-tail:]+ half = max_chars // 2+ left = max(0, start - half)+ right = min(len(text), left + max_chars)+ left = max(0, right - max_chars)+ prefix = "…" if left else ""+ suffix = "…" if right < len(text) else ""+ return prefix + text[left:right] + suffix++def _render_claim_context(claims, question: str = "", kbinfos: dict | None = None) -> str:"""Render per-claim context: each claim's report PLUS a brief evidence anchor.@@ -122,24 +177,26 @@ def _render_claim_context(claims, question: str = "", kbinfos: dict | None = Nonblock = f"Claim {cid} (draft):\n{rpt}"used += len(block) + 2# Evidence anchor: first line of each cited snippet (guards the draft).+ # Table chunks contribute their FULL text (see _bounded_excerpt), so+ # account the actual joined length, not the 300-char estimate.if eids:anchors: list[str] = []for eid in eids:ck = id2chunk.get(str(eid)) or id2chunk.get(eid)if not ck:continue- txt = str(ck.get("chunk") or "").strip()+ txt = str(ck.get("content_with_weight") or ck.get("content") or ck.get("chunk") or "").strip()if not txt:continue- first = txt.split("\n")[0].strip()- if len(first) > _SCA_EVIDENCE_ANCHOR_CHARS:- first = first[:_SCA_EVIDENCE_ANCHOR_CHARS] + "…"- anchors.append(first)- if len(anchors) >= 2:+ excerpt = _bounded_excerpt(txt, rpt, max_chars=_SCA_EVIDENCE_ANCHOR_CHARS)+ if excerpt:+ anchors.append(excerpt)+ if len(anchors) >= 3:breakif anchors:- block += "\n Evidence: " + " | ".join(anchors)- used += len(anchors) * (_SCA_EVIDENCE_ANCHOR_CHARS + 4)+ anchor_text = " | ".join(anchors)+ block += "\n Evidence: " + anchor_text+ used += len(anchor_text) + 4blocks.append(block)if used >= _SCA_CLAIMS_CONTEXT_MAX:breakdiff --git a/rag/advanced_rag/harness/stats.py b/rag/advanced_rag/harness/stats.pyindex 7e9ec8a923..a777fdcbf2 100644--- a/rag/advanced_rag/harness/stats.py+++ b/rag/advanced_rag/harness/stats.py@@ -355,6 +355,31 @@ def record_round_claims(name: str, count: int) -> None:stats.record_round_claims(name, count)+def record_external_call(usage: dict | None = None) -> None:+ """Record a successful raw model call that bypasses CountingChatModel."""+ stats = _CURRENT_STATS.get()+ if stats is None:+ return+ phase_name = _CURRENT_PHASE.get()+ stats.record_call(phase_name)+ stats.record_usage(phase_name, usage)+++def record_external_response(response) -> None:+ """Record a raw completion response, including provider usage when present."""+ usage = getattr(response, "usage", None)+ if usage is None and isinstance(response, dict):+ usage = response.get("usage")+ if usage is not None and not isinstance(usage, dict):+ if hasattr(usage, "model_dump"):+ usage = usage.model_dump()+ elif hasattr(usage, "dict"):+ usage = usage.dict()+ else:+ usage = {k: getattr(usage, k) for k in ("prompt_tokens", "completion_tokens", "total_tokens") if hasattr(usage, k)}+ record_external_call(usage if isinstance(usage, dict) else None)++def _last_usage(chat_mdl) -> dict | None:mdl = getattr(chat_mdl, "mdl", None)usage = getattr(mdl, "last_usage", None)diff --git a/rag/advanced_rag/harness/tools/search.py b/rag/advanced_rag/harness/tools/search.pyindex e19e90efd1..9965b027c0 100644--- a/rag/advanced_rag/harness/tools/search.py+++ b/rag/advanced_rag/harness/tools/search.py@@ -191,6 +191,16 @@ async def hybrid_search(if use_compiled and kbinfos.get("chunks"):_LOG.info("[Hybrid search] Compiled expansion enabled — enriching with page_index/tree/KG navigation.")await _expand_with_compiled(tools, query, keywords, kbinfos, doc_scope)+ chunks_now = kbinfos.get("chunks") or []+ if chunks_now:+ _doc_stats: dict = {}+ for _c in chunks_now:+ _d = str(_c.get("docnm_kwd") or _c.get("docnm") or _c.get("doc_name") or "?")+ _s = _doc_stats.setdefault(_d, [0, 0])+ _s[0] += 1+ _s[1] += len(str(_c.get("content") or _c.get("content_with_weight") or ""))+ _detail = "; ".join(f"{d}:{n}chunk({sz}chars)" for d, (n, sz) in sorted(_doc_stats.items()))+ _LOG.info(f'[Hybrid search] "{query[:80]}" -> {len(chunks_now)} chunk(s): {_detail}')if cache is not None:cache[cache_key] = kbinfosreturn kbinfos@@ -345,6 +355,22 @@ _GREP_OUT_TOTAL_CHARS = 8000_LIST_CHUNKS_MAX_CHUNKS = 80+def _is_table_chunk(c: dict) -> bool:+ """Corpus-neutral table detector: HTML table markup or >=3 pipe rows.++ Table chunks must NOT be term-narrowed: their answer rows often sit+ mid/late-table (e.g. a rank row at ~62% of a 14.7K-char table), and the+ grep context window truncates them to a header-only snippet, hiding the+ answer from the action-session model (Q86: 2011 Pan Am standings rank 19+ at char 5181 of 14717 was cut by the 700-char narrow).+ """+ t = str(_chunk_text(c) or "")+ if "<table" in t.lower() or "<tr" in t.lower():+ return True+ pipe_rows = sum(1 for line in t.splitlines() if line.count("|") >= 2)+ return pipe_rows >= 3++def _grep_terms_from_query(query: str, max_terms: int = _GREP_TERMS_MAX) -> list[str]:"""Extract compact grep terms from a query: bare alnum words of length>=2,deduped (order-preserving) and capped. Numbers/ids are preserved as-is."""@@ -404,15 +430,24 @@ async def grep_search(if not chunks or not terms:return restry:- out = narrow_by_terms(- chunks,- terms,- keywords=str(query).strip(),- context={"before": 1, "after": 0},- max_out_chars_per_chunk=_GREP_OUT_CHARS_PER_CHUNK,- max_out_total_chars=_GREP_OUT_TOTAL_CHARS,- )- kept = out.get("kept") or []+ # Table chunks pass through UN-narrowed (full text): term-grep windows+ # truncate them to a header-only snippet and hide mid-table answer rows.+ # Prose chunks keep the compact narrow.+ table_chunks = [c for c in chunks if _is_table_chunk(c)]+ prose_chunks = [c for c in chunks if not _is_table_chunk(c)]+ if prose_chunks:+ out = narrow_by_terms(+ prose_chunks,+ terms,+ keywords=str(query).strip(),+ context={"before": 1, "after": 0},+ max_out_chars_per_chunk=_GREP_OUT_CHARS_PER_CHUNK,+ max_out_total_chars=_GREP_OUT_TOTAL_CHARS,+ )+ kept = out.get("kept") or []+ else:+ kept = []+ kept = table_chunks + keptif kept:_LOG.info("[Grep search] narrowed %d->%d chunk(s), %.1fK chars.",@@ -423,6 +458,20 @@ async def grep_search(res["chunks"] = keptexcept Exception:_LOG.exception("[Grep search] narrow failed; using raw BM25 candidates.")+ _g = res.get("chunks") or []+ if _g:+ _gs: dict = {}+ for _c in _g:+ _d = str(_c.get("docnm_kwd") or _c.get("docnm") or "?")+ _e = _gs.setdefault(_d, [0, 0])+ _e[0] += 1+ _e[1] += len(str(_c.get("content") or _c.get("content_with_weight") or ""))+ _LOG.info(+ '[Grep search] "%s" -> %d chunk(s): %s',+ str(query)[:80],+ len(_g),+ "; ".join(f"{d}:{n}chunk({sz}chars)" for d, (n, sz) in sorted(_gs.items())),+ )return resdiff --git a/rag/advanced_rag/harness/tools/text_processing.py b/rag/advanced_rag/harness/tools/text_processing.pyindex cf0e97d12e..05898d272b 100644--- a/rag/advanced_rag/harness/tools/text_processing.py+++ b/rag/advanced_rag/harness/tools/text_processing.py@@ -348,7 +348,14 @@ def _narrow_content(content: str, kwds: list[str]) -> str | None:# "table truncated at -4.58°N, Maseru (-29.3°) missing" bug on FRAMES Q408.# A table row is one data point, not a sentence, so keyword-window narrowing is# wrong here; keep the full table (it is already rank-sorted by the retriever).- if "<table" in content.lower() or "<tr" in content.lower() or "<td" in content.lower():+ # Markdown pipe tables (>=3 rows with >=2 pipes) get the same full-text pass:+ # their answer rows often sit mid-table (e.g. a rank row at ~62% of a 14.7K-char+ # table), and sentence-window narrowing truncates them to a header-only snippet.+ low_content = content.lower()+ if "<table" in low_content or "<tr" in low_content or "<td" in low_content:+ return "..." + _highlight_keywords(content, kwds) + "..."+ pipe_rows = sum(1 for line in content.splitlines() if line.count("|") >= 2)+ if pipe_rows >= 3:return "..." + _highlight_keywords(content, kwds) + "..."sents = _split_sentences(content)