RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Refactor: Make wiki and web searchable. (#17789)
Kevin Hufac40e510323
diff --git a/rag/advanced_rag/harness/agent.py b/rag/advanced_rag/harness/agent.pyindex c12a6ab286..d1e5a3f377 100644--- a/rag/advanced_rag/harness/agent.py+++ b/rag/advanced_rag/harness/agent.py@@ -155,6 +155,7 @@ async def research_agent_loop(compilation_map=compilation_map,context=context,has_routed_scope=bool(getattr(pipeline, "_routed_docs", None)),+ web_enabled=bool(getattr(tools, "has_web", lambda: False)()),)# Clone so binding tools never leaks onto the shared chat model.diff --git a/rag/advanced_rag/harness/tools/exploration.py b/rag/advanced_rag/harness/tools/exploration.pyindex f44856c1e1..3cc75763e8 100644--- a/rag/advanced_rag/harness/tools/exploration.py+++ b/rag/advanced_rag/harness/tools/exploration.py@@ -3,13 +3,15 @@``graph_explore`` lives in :mod:`navigation` (it shares the compiled-structuremachinery) and is re-exported here so the tool registry keeps one import point.-``wiki_query`` retrieves through ``hybrid_search`` and takes the same-``keywords`` the other search tools do — the keywords drive query expansion and-the keyword-sentence narrowing. Parameter names must match the registered-``_search_schema`` (``query`` + ``keywords``), otherwise every LLM tool call-fails with a TypeError.+``wiki_query`` runs a hybrid (BM25 + dense) search over the searchable wiki draft+rows written by ``_wiki_persist_draft`` (``compile_kwd="wiki_page_draft"``) and+returns each page's markdown as a chunk. It takes the same ``keywords`` the other+search tools do — the keywords drive the keyword-sentence narrowing. Parameter+names must match the registered ``_search_schema`` (``query`` + ``keywords``),+otherwise every LLM tool call fails with a TypeError."""+import jsonimport logging# graph_explore is implemented alongside catalog/mindmap navigation because it@@ -18,15 +20,102 @@ from rag.advanced_rag.harness.tools.navigation import graph_explore # noqa: F40_LOG = logging.getLogger(__name__)+# compile_kwd of the searchable wiki draft rows (see _wiki_persist_draft).+_WIKI_DRAFT_COMPILE_KWD = "wiki_page_draft"+_WIKI_QUERY_TOP_N = 12+async def wiki_query(tools, query: str, keywords: str = "") -> dict:- """Query compiled wiki knowledge.+ """Search the compiled wiki.++ Hybrid (BM25 over ``title_tks`` / ``content_ltks`` / ``content_sm_ltks`` ++ dense over ``q_<dim>_vec``) search across each bound KB's ``wiki_page_draft``+ rows. The page markdown is parsed out of each row's ``content_with_weight``+ (which stays the page JSON) and returned as chunks, narrowed by ``keywords``.- This is currently a placeholder. The final implementation should call the- compiled wiki store.+ :returns: ``{"answer": "", "chunks": [...], "doc_aggs": [...]}``"""- _LOG.info(f'[Wiki lookup] Looking up the compiled wiki for "{query}" (keywords: {keywords})')- # TODO: implement actual wiki lookup- from rag.advanced_rag.harness.tools.search import hybrid_search+ from common import settings+ from common.doc_store.doc_store_base import FusionExpr, OrderByExpr+ from common.misc_utils import thread_pool_exec+ from rag.nlp import search as _rag_search+ from rag.advanced_rag.harness.tools.search import _narrow_by_keywords++ _LOG.info(f'[Wiki lookup] Searching the compiled wiki for "{query}" (keywords: {keywords})')++ kbs = getattr(tools, "kbs", []) or []+ text = f"{query} {keywords}".strip()+ if not kbs or not text:+ return {"answer": "", "chunks": [], "doc_aggs": []}++ fields = ["content_with_weight", "docnm_kwd", "title_kwd", "wiki_slug_kwd", "source_doc_ids", "doc_id"]+ qryr = settings.retriever.qryr+ chunks: list[dict] = []++ for kb in kbs:+ kb_id = kb.id+ tenant_id = kb.tenant_id+ index = _rag_search.index_name(tenant_id)+ try:+ # BM25 over the standard tokenized fields, fused with dense when an+ # embedder is available — mirrors the retriever's own hybrid search.+ match_text, _ = qryr.question(text, min_match=0.3)+ exprs = [match_text]+ if getattr(tools, "embed_mdl", None):+ try:+ match_dense = await settings.retriever.get_vector(text, tools.embed_mdl, _WIKI_QUERY_TOP_N, 0.1)+ exprs = [match_text, match_dense, FusionExpr("weighted_sum", _WIKI_QUERY_TOP_N, {"weights": "0.001, 1"})]+ except Exception:+ _LOG.exception("[Wiki lookup] dense expr build failed; BM25 only")+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ fields,+ [],+ {"compile_kwd": [_WIKI_DRAFT_COMPILE_KWD]},+ exprs,+ OrderByExpr(),+ 0,+ _WIKI_QUERY_TOP_N,+ index,+ [kb_id],+ )+ rows = settings.docStoreConn.get_fields(res, fields) or {}+ except Exception:+ _LOG.exception("[Wiki lookup] search failed for kb=%s", kb_id)+ continue++ for cid, row in rows.items():+ try:+ page = json.loads(row.get("content_with_weight") or "{}")+ except Exception:+ page = {}+ if not isinstance(page, dict):+ page = {}+ content = page.get("content_md_rendered") or page.get("content_md") or page.get("content_md_raw") or ""+ if not content:+ continue+ title = row.get("docnm_kwd") or page.get("title") or row.get("title_kwd") or ""+ slug = row.get("wiki_slug_kwd") or page.get("slug") or ""+ chunks.append(+ {+ "chunk_id": cid,+ "content_with_weight": content,+ "docnm_kwd": title,+ "doc_id": slug or row.get("doc_id") or kb_id,+ "wiki_slug_kwd": slug,+ }+ )++ before = len(chunks)+ chunks = _narrow_by_keywords(chunks, keywords)+ _LOG.info("[Wiki lookup] Found %d wiki page(s), kept %d after keyword filtering.", before, len(chunks))++ doc_aggs: list[dict] = []+ seen: set = set()+ for c in chunks:+ did = c.get("doc_id")+ if did and did not in seen:+ seen.add(did)+ doc_aggs.append({"doc_id": did, "doc_name": c.get("docnm_kwd") or ""})- return await hybrid_search(tools, query=query, keywords=keywords)+ return {"answer": "", "chunks": chunks, "doc_aggs": doc_aggs}diff --git a/rag/advanced_rag/harness/tools/gating.py b/rag/advanced_rag/harness/tools/gating.pyindex 3605c1a6ab..7b1f2485c9 100644--- a/rag/advanced_rag/harness/tools/gating.py+++ b/rag/advanced_rag/harness/tools/gating.py@@ -25,12 +25,13 @@ SEARCH_PHASES = {"tools_priority": ["hybrid_search","bm25_search",+ "web_search","graph_explore","inspector_open_context","inspector_request_adjacent",],"max_returned": 4,- "tool_hint": "Prefer retrieval tools to gather detailed information within the located region.",+ "tool_hint": "Prefer retrieval tools to gather detailed information within the located region; use web_search when the knowledge base lacks the answer or the question needs current/external facts.",},"verify": {"goal": "Verify consistency across multiple sources.",@@ -38,11 +39,11 @@ SEARCH_PHASES = {"inspector_open_context","inspector_compare","inspector_grep_within",- "hybrid_search","web_search",+ "hybrid_search",],"max_returned": 4,- "tool_hint": "Prefer inspector tools to compare existing evidence before searching for new content.",+ "tool_hint": "Prefer inspector tools to compare existing evidence; use web_search to corroborate against external sources.",},"cross_domain": {"goal": "Explore cross-domain relationships for discovered entities.",@@ -89,16 +90,20 @@ def get_gated_tools(compilation_map: dict[str, set[str]],context: OrchestratorContext,has_routed_scope: bool = False,+ web_enabled: bool = True,) -> list[dict]:"""Filter, sort, and gate tools by phase priority and context."""phase_config = SEARCH_PHASES.get(phase)if not phase_config:- return _default_defs(available_tools)+ return _default_defs(available_tools, web_enabled)sorted_tools = []for tool_name in phase_config["tools_priority"]:if tool_name not in available_tools:continue+ if tool_name == "web_search" and not web_enabled:+ # No web provider configured — don't bind a tool that no-ops.+ continueif not compilation_available(tool_name, compilation_map):continueif not tool_fits_context(tool_name, context, has_routed_scope):@@ -113,8 +118,8 @@ def get_gated_tools(return defs-def _default_defs(tool_names: list[str]) -> list[dict]:- return [TOOL_REGISTRY[n]["function_schema"] for n in tool_names if n in TOOL_REGISTRY]+def _default_defs(tool_names: list[str], web_enabled: bool = True) -> list[dict]:+ return [TOOL_REGISTRY[n]["function_schema"] for n in tool_names if n in TOOL_REGISTRY and (web_enabled or n != "web_search")]def determine_current_phase(context: OrchestratorContext) -> str:diff --git a/rag/advanced_rag/harness/tools/search.py b/rag/advanced_rag/harness/tools/search.pyindex 4f26e974ab..492a6d6583 100644--- a/rag/advanced_rag/harness/tools/search.py+++ b/rag/advanced_rag/harness/tools/search.py@@ -217,6 +217,7 @@ async def hybrid_search(tools, query: str, kb_ids: list[str] | None = None, top_aggs=True,highlight=False,doc_ids=doc_scope,+ must_not={"exists": "compile_kwd"}, # plain retrieval = document chunks only; compiled products have their own tools)kbinfos = _normalize(kbinfos, tools.tenant_ids)if keywords:@@ -253,6 +254,7 @@ async def vector_search(tools, query: str, kb_ids: list[str] | None = None, top_aggs=False,highlight=False,doc_ids=doc_scope,+ must_not={"exists": "compile_kwd"},)kbinfos = _normalize(kbinfos, tools.tenant_ids)if keywords:@@ -280,6 +282,7 @@ async def bm25_search(tools, query: str, kb_ids: list[str] | None = None, top_n:aggs=False,highlight=False,doc_ids=doc_scope,+ must_not={"exists": "compile_kwd"},)kbinfos = _normalize(kbinfos, tools.tenant_ids)if keywords:diff --git a/rag/advanced_rag/knowlege_compile/wiki.py b/rag/advanced_rag/knowlege_compile/wiki.pyindex d1a37c4c79..fc2437d3fb 100644--- a/rag/advanced_rag/knowlege_compile/wiki.py+++ b/rag/advanced_rag/knowlege_compile/wiki.py@@ -3375,16 +3375,25 @@ async def _wiki_persist_draft(tenant_id: str,kb_id: str,plan_input_hash: str = "",+ embd_mdl=None,) -> None:- """Upsert one non-searchable wiki_page_draft row (resume cache).+ """Upsert one wiki_page_draft row (resume cache + searchable page).``plan_input_hash`` is the PLAN's ``input_hash_kwd`` at the time thisdraft was produced. The next REFINE re-entry compares it against thecurrent PLAN hash to decide whether the cached draft is stillvalid; a mismatch forces a rewrite for that slug.++ When ``embd_mdl`` is provided the row is made searchable: the title/body are+ tokenized (``title_tks`` / ``content_ltks`` / ``content_sm_ltks``) and a+ ``q_<dim>_vec`` page embedding is attached, with ``available_int=1`` so the+ agent's ``wiki_query`` tool can retrieve it. Without an embedder the row stays+ a non-searchable resume cache. ``content_with_weight`` is left as the page+ JSON either way, so ``_wiki_load_refine_resume`` still restores the draft."""from common import settingsfrom rag.nlp import search as _rag_search+ from rag.nlp import rag_tokenizerslug = page.get("slug") or ""if not slug:@@ -3401,8 +3410,37 @@ async def _wiki_persist_draft("source_doc_ids": draft_doc_ids,"input_hash_kwd": plan_input_hash,"content_with_weight": content_with_weight,- "available_int": 0, # non-searchable+ "available_int": 0, # non-searchable unless made searchable below}++ # Make the draft searchable when an embedder is available. content_with_weight+ # is deliberately left untouched (the page JSON) — the tokenized fields drive+ # BM25 and q_<dim>_vec drives dense retrieval.+ if embd_mdl is not None:+ title = str(page.get("title") or slug)+ body = str(page.get("content_md_rendered") or page.get("content_md") or page.get("content_md_raw") or "")+ summary = str(page.get("summary") or "")+ content_ltks = rag_tokenizer.tokenize(body)+ row.update(+ {+ "docnm_kwd": title,+ "title_kwd": title,+ "title_tks": rag_tokenizer.tokenize(title),+ "content_ltks": content_ltks,+ "content_sm_ltks": rag_tokenizer.fine_grained_tokenize(content_ltks),+ }+ )+ try:+ emb_text = (summary or f"{title}\n{body}").strip()[:2048] or title+ vectors, _ = await thread_pool_exec(embd_mdl.encode, [emb_text])+ vec = vectors[0]+ vec_list = vec.tolist() if hasattr(vec, "tolist") else list(vec)+ if vec_list:+ row[f"q_{len(vec_list)}_vec"] = vec_list+ row["available_int"] = 1+ except Exception:+ logging.exception("wiki_refine: draft embedding failed slug=%s; row stays non-searchable", slug)+try:try:await thread_pool_exec(@@ -3750,6 +3788,7 @@ async def wiki_refine_from_plan(tenant_id,kb_id,plan_input_hash=plan_input_hash,+ embd_mdl=embd_mdl,)except Exception:logging.exception("wiki_refine: persist_draft failed for slug=%s", slug)@@ -3820,6 +3859,7 @@ async def wiki_refine_from_plan(tenant_id,kb_id,plan_input_hash=plan_input_hash,+ embd_mdl=embd_mdl,)except Exception:logging.exception("wiki_refine: persist cleaned draft failed for slug=%s", page.get("slug"))diff --git a/rag/llm/chat_model.py b/rag/llm/chat_model.pyindex c19f89a3ec..9a4d972782 100644--- a/rag/llm/chat_model.py+++ b/rag/llm/chat_model.py@@ -1583,6 +1583,53 @@ class GreenPTChat(Base):super().__init__(key, model_name, base_url or "https://api.greenpt.ai/v1", **kwargs)+# MiniMax models sometimes emit their bracket-delimited control/boundary tokens+# into `content` instead of as structured control — e.g. "]<]minimax[>[" — most+# often on tool-calling turns. The token is streamed split across many deltas,+# so it can't be removed per-delta; it must be filtered over a window that spans+# chunk boundaries. This pattern only matches the vendor name when it is wrapped+# in bracket noise on BOTH sides, so ordinary prose that mentions "MiniMax" is+# left untouched. Extend the alternation as further control tokens are observed.+_MINIMAX_CONTROL_TOKEN_RE = re.compile(r"[\[\]<>]+\s*minimax\s*[\[\]<>]+", re.IGNORECASE)+++class _StreamSanitizer:+ """Strip a regex from a token stream even when matches span chunk boundaries.++ A control token is bracket+letter characters, and it can arrive split across+ many deltas, so we hold back the trailing run of token-ish characters (which+ might still be forming a match) and only ``sub`` + emit the part before it.+ Applying ``sub`` to a partial trailing run would fire prematurely and leak the+ unmatched remainder — hence the hold. ``flush()`` sanitizes and returns the+ remainder at end of stream. ``keep`` caps how long a run is buffered so a very+ long separator-less word can't stall the stream forever.+ """++ _TOKENISH = re.compile(r"[\[\]<>A-Za-z]*$")++ def __init__(self, pattern: re.Pattern, keep: int = 64) -> None:+ self._pat = pattern+ self._keep = keep+ self._buf = ""++ def feed(self, text: str) -> str:+ if not text:+ return ""+ self._buf += text+ match = self._TOKENISH.search(self._buf)+ hold_start = match.start() if match else len(self._buf)+ if len(self._buf) - hold_start > self._keep:+ hold_start = len(self._buf) - self._keep+ emit = self._pat.sub("", self._buf[:hold_start])+ self._buf = self._buf[hold_start:]+ return emit++ def flush(self) -> str:+ out = self._pat.sub("", self._buf)+ self._buf = ""+ return out++class LiteLLMBase(ABC):_FACTORY_NAME = ["Tongyi-Qianwen",@@ -1722,6 +1769,18 @@ class LiteLLMBase(ABC):def _need_reasoning_content_back(self) -> bool:return self.provider == SupportedLiteLLMProvider.DeepSeek+ def _content_stream_sanitizer(self) -> "_StreamSanitizer | None":+ """A per-stream filter for providers whose control tokens leak into content."""+ if self.provider == SupportedLiteLLMProvider.MiniMax:+ return _StreamSanitizer(_MINIMAX_CONTROL_TOKEN_RE)+ return None++ def _sanitize_answer(self, text: str) -> str:+ """Strip provider control-token noise from a fully-assembled answer."""+ if text and self.provider == SupportedLiteLLMProvider.MiniMax:+ return _MINIMAX_CONTROL_TOKEN_RE.sub("", text)+ return text+async def async_chat(self, system, history, gen_conf, **kwargs):hist = list(history) if history else []if system:@@ -2021,7 +2080,7 @@ class LiteLLMBase(ABC):ans += message.content or ""if response.choices[0].finish_reason == "length":ans = self._length_stop(ans)- return ans, tk_count+ return self._sanitize_answer(ans), tk_countasync def _exec_tool(tc):name = tc.function.name@@ -2060,7 +2119,7 @@ class LiteLLMBase(ABC):agg_usage["total_tokens"] += int(_fb.get("total_tokens", 0) or token_count)tk_count = agg_usage["total_tokens"]self.last_usage = dict(agg_usage)- return ans, tk_count+ return self._sanitize_answer(ans), tk_countexcept Exception as e:e = await self._exceptions_async(e, attempt)@@ -2115,6 +2174,9 @@ class LiteLLMBase(ABC):answer = ""round_usage = Noneround_estimate = 0+ # Per-round filter for providers (MiniMax) whose control tokens+ # leak into content split across deltas; None for others.+ _sanitizer = self._content_stream_sanitizer()async for resp in response:# Usage-only final chunk may carry no choices — read it first.@@ -2154,7 +2216,12 @@ class LiteLLMBase(ABC):else:reasoning_start = Falseanswer += delta.content- yield delta.content+ if _sanitizer is not None:+ emitted = _sanitizer.feed(delta.content)+ if emitted:+ yield emitted+ else:+ yield delta.contentif not _u["total_tokens"]:round_estimate += num_tokens_from_string(delta.content)@@ -2163,6 +2230,12 @@ class LiteLLMBase(ABC):if finish_reason == "length":yield self._length_stop("")+ # Flush any held-back (sanitized) answer content for this round.+ if _sanitizer is not None:+ tail = _sanitizer.flush()+ if tail:+ yield tail+# Commit this round's tokens to the running aggregate._commit_round(round_usage, round_estimate)diff --git a/rag/nlp/search.py b/rag/nlp/search.pyindex 32940fdff5..50d2462903 100644--- a/rag/nlp/search.py+++ b/rag/nlp/search.py@@ -563,6 +563,7 @@ class Dealer:highlight=False,rank_feature: dict | None = {PAGERANK_FLD: 10},trace_id=None,+ must_not: dict | None = None,):ranks = {"total": 0, "chunks": [], "doc_aggs": {}}if not question:@@ -587,6 +588,8 @@ class Dealer:"similarity": similarity_threshold,"available_int": 1,}+ if isinstance(must_not, dict) and must_not:+ req["must_not"] = must_notlogging.debug(f"[Search] global_offset={global_offset}, rerank_limit={RERANK_LIMIT}, page_size={page_size}, page={page}")if isinstance(tenant_ids, str):