RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Search in navigation tree & store graph content in nav_doc (#18303)
qinling0210554fb1133ac3
diff --git a/api/apps/restful_apis/dataset_api.py b/api/apps/restful_apis/dataset_api.pyindex cd0004b6a7..4cc0f969c8 100644--- a/api/apps/restful_apis/dataset_api.py+++ b/api/apps/restful_apis/dataset_api.py@@ -1028,12 +1028,23 @@ async def list_dataset_nav(tenant_id, dataset_id):"""First level of the dataset navigation tree — the top-level clusters.GET /api/v1/datasets/<dataset_id>/navigation- Success: {"code": 0, "data": {"total": <n>, "items": [{name, description, doc_count, type, has_children}, ...]}}+ GET /api/v1/datasets/<dataset_id>/navigation?keywords=<query>&top_k=<n>+ Success: {"code": 0, "data": {"total": <n>, "items": [{name, description, doc_count, type, has_children, ...}, ...]}}"""+ q = (request.args.get("keywords") or "").strip() or None+ top_k_raw = request.args.get("top_k")+ top_k = None+ if top_k_raw:+ try:+ top_k = max(1, int(top_k_raw))+ except (ValueError, TypeError):+ return get_error_data_result(message="top_k must be a positive integer")try:success, result = await dataset_api_service.list_nav_clusters(dataset_id,tenant_id,+ q=q,+ top_k=top_k,)if success:return get_result(data=result)@@ -1068,7 +1079,7 @@ async def search_dataset_nav(tenant_id, dataset_id):documents; omitted → all documents of the dataset.Success: {"code": 0, "data": {"mode": <mode>, "total": <n>,- "items": [{"doc_id": str, "score": float}, ...]}}+ "items": [{name, description, doc_count, type, has_children, doc_id, score, ...}, ...]}}"""q = (request.args.get("q") or "").strip()if not q:@@ -1092,6 +1103,7 @@ async def search_dataset_nav(tenant_id, dataset_id):doc_scope=doc_scope,)if success:+ result["items"] = await dataset_api_service._enrich_nav_items(dataset_id, tenant_id, result.get("items", []))return get_result(data=result)if isinstance(result, dict):msg = result.get("error", result)diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex 5af1da3300..a5cb969f41 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -3206,6 +3206,32 @@ _NAV_FIELDS = []+def _first_str(val) -> str:+ """Extract the first string from a value that may be str, list, tuple, or set."""+ if isinstance(val, (list, tuple, set)):+ return str(next(iter(val), "") or "")+ return str(val or "")+++def _resolve_embd_mdl(kb):+ """Resolve the embedding model for a knowledge base, or None on failure."""+ from api.db.joint_services.tenant_model_service import get_tenant_default_model_by_type+ from api.db.services.llm_service import LLMBundle+ from common.constants import LLMType++ try:+ if kb.embd_id:+ embd_model_config = resolve_model_config(kb.tenant_id, LLMType.EMBEDDING, kb.embd_id)+ else:+ embd_model_config = get_tenant_default_model_by_type(kb.tenant_id, LLMType.EMBEDDING)+ if embd_model_config is None:+ return None+ return LLMBundle(kb.tenant_id, embd_model_config)+ except Exception:+ logging.exception("Failed to resolve embedding model for kb=%s", kb.id)+ return None++def _nav_item(row: dict) -> dict:"""Shape one nav row into a UI node: name, description, doc count, type."""try:@@ -3276,8 +3302,19 @@ async def _nav_search(dataset_id: str, tenant_id: str, condition: dict, page: inreturn True, {"total": int(total or 0), "items": items}-async def list_nav_clusters(dataset_id: str, tenant_id: str, page: int = 1, page_size: int = 1000):- """First level of the nav tree: the clusters with no parent."""+async def list_nav_clusters(dataset_id: str, tenant_id: str, page: int = 1, page_size: int = 1000, q: str | None = None, top_k: int | None = None):+ """First level of the nav tree: the clusters with no parent.++ When ``q`` is provided, runs a tree-structured search (mode="navigation_tree")+ and returns enriched nav node items in the same ``_nav_item`` shape so the+ frontend tree search can reuse this endpoint.+ """+ if q and q.strip():+ success, result = await search_dataset_layers(dataset_id, tenant_id, q.strip(), "navigation_tree", top_k=top_k or 1000)+ if not success:+ return success, result+ result["items"] = await _enrich_nav_items(dataset_id, tenant_id, result.get("items", []))+ return True, {"total": result.get("total", 0), "items": result["items"]}condition = {"compile_kwd": [_NAV_COMPILE_KWD],"type_kwd": ["nav_cluster"],@@ -3475,13 +3512,12 @@ async def generate_nav()from api.db.services.llm_service import LLMBundlefrom common.constants import LLMType- from rag.advanced_rag.knowlege_compile.dataset_nav import upsert_dataset_nav_doc+ from rag.advanced_rag.knowlege_compile.dataset_nav import (+ build_nav_graph_text,+ upsert_dataset_nav_doc,+ )- if kb.embd_id:- embd_model_config = resolve_model_config(kb.tenant_id, LLMType.EMBEDDING, kb.embd_id)- else:- embd_model_config = get_tenant_default_model_by_type(kb.tenant_id, LLMType.EMBEDDING)- embd_mdl = LLMBundle(kb.tenant_id, embd_model_config)+ embd_mdl = _resolve_embd_mdl(kb)chat_model_config = get_tenant_default_model_by_type(kb.tenant_id, LLMType.CHAT)chat_mdl = LLMBundle(kb.tenant_id, chat_model_config)@@ -3552,53 +3588,15 @@ async def generate_nav(graph = json.loads(row.get("content_with_weight") or "{}")except Exception:continue- entities = graph.get("entities") or []- relations = graph.get("relations") or []- child_names = {r.get("to") for r in relations if isinstance(r, dict)}-# Build both:# - root_summary: first line of root desc (short,# for the display title / description field)# - graph_text: structured text from ALL entities# and relations (for embedding, keyword extraction,# entity extraction, and stored as graph_content).- # entity name -> description- name_desc: dict[str, str] = {}- for ent in entities:- if not isinstance(ent, dict):- continue- nm = (ent.get("name") or "").strip()- if nm:- name_desc[nm] = (ent.get("description") or "").strip()-- # root entity = entity whose name never appears as a- # relation target (not a child of anyone).- root_name = ""- root_summary = ""- for ent in entities:- if isinstance(ent, dict) and ent.get("name") not in child_names:- root_name = (ent.get("name") or "").strip()- root_summary = name_desc.get(root_name, "")- root_summary = root_summary.splitlines()[0].strip() if root_summary else root_name- break-- # Build the full graph text- graph_parts: list[str] = []- if root_name and name_desc.get(root_name):- graph_parts.append(root_name)- graph_parts.append(name_desc[root_name])-- child_names_set = {n for n in name_desc if n in child_names}- if child_names_set:- graph_parts.append("")- for cname in sorted(child_names_set):- cdesc = name_desc.get(cname, "")- line = f"- {cname}"- if cdesc:- line += f": {cdesc.splitlines()[0].strip()}"- graph_parts.append(line)-- graph_text = "\n".join(graph_parts) if graph_parts else ""+ # Shared with the parse-time path (run_tree_templates)+ # so both produce identical, complete nav_doc content.+ root_summary, graph_text = build_nav_graph_text(graph)if root_summary:raptor_summaries[gid] = {@@ -3633,18 +3631,36 @@ async def generate_nav(return False, "No documents found in dataset."# Step 1: delete the entire existing navigation tree so we start clean.+ # ``deleted`` reports the number of *clusters* removed (not nav_doc leaves),+ # since that is the meaningful unit for the navigation tree.deleted = 0pack = _compiled_index_or_none(kb.tenant_id, dataset_id)if pack is not None:index_nm, _ = packtry:- deleted = await thread_pool_exec(+ from common.doc_store.doc_store_base import OrderByExpr++ # Count existing clusters before wiping the tree.+ count_res = await thread_pool_exec(+ settings.docStoreConn.search,+ ["id"],+ [],+ {"compile_kwd": [_NAV_COMPILE_KWD], "type_kwd": ["nav_cluster"]},+ [],+ OrderByExpr(),+ 0,+ 10000,+ index_nm,+ [dataset_id],+ )+ deleted = len(settings.docStoreConn.get_fields(count_res, ["id"]) or {})++ await thread_pool_exec(settings.docStoreConn.delete,{"compile_kwd": [_NAV_COMPILE_KWD]},index_nm,dataset_id,)- deleted = int(deleted or 0)except Exception:logging.exception("generate_nav: failed to clear existing nav for kb=%s", dataset_id)return False, "Failed to clear existing navigation tree."@@ -3831,12 +3847,15 @@ async def _nav_search_result(tenant_id, dataset_id, query, top_k, embd_mdl, seardoc_id = str(doc_ids[0]).strip() if doc_ids else ""if not doc_id:continue- items.append(- {- "doc_id": doc_id,- "score": round(float(r.get("score", 0.0)), 4),- }- )+ item = {+ "doc_id": doc_id,+ "score": round(float(r.get("score", 0.0)), 4),+ }+ # Preserve the full nav node info so callers can enrich without a+ # second ES round-trip. Clusters have doc_id="" but carry name.+ if r.get("name") or r.get("description"):+ item["_nav"] = r+ items.append(item)return items@@ -3897,7 +3916,7 @@ async def _search_layers_all(tenant_id, dataset_id, query, top_k, embd_mdl, kb,return_exceptions=True,)- doc_scores: dict[str, float] = {}+ doc_scores: dict[str, dict] = {}for result in result_lists:if isinstance(result, Exception):continue@@ -3907,11 +3926,13 @@ async def _search_layers_all(tenant_id, dataset_id, query, top_k, embd_mdl, kb,for item in data.get("items", []):doc_id = item.get("doc_id", "")score = float(item.get("score", 0.0))- if doc_id and score > doc_scores.get(doc_id, -1.0):- doc_scores[doc_id] = score+ # Clusters have no doc_id; key by name so they survive dedup.+ key = doc_id or item.get("_nav", {}).get("name", "")+ if key and score > doc_scores.get(key, {}).get("score", -1.0):+ doc_scores[key] = itemitems = sorted(- ({"doc_id": d, "score": round(s, 4)} for d, s in doc_scores.items()),+ doc_scores.values(),key=lambda x: x["score"],reverse=True,)@@ -3921,6 +3942,178 @@ async def _search_layers_all(tenant_id, dataset_id, query, top_k, embd_mdl, kb,return True, {"mode": "all", "total": len(items), "items": items}+async def _enrich_nav_items(dataset_id: str, tenant_id: str, items: list[dict]) -> list[dict]:+ """Enrich ``{doc_id, score}`` items with full nav node info.++ Items that already carry a ``_nav`` payload (from nav_doc/nav_cluster/+ navigation_tree search) are shaped in-process. Items without it (e.g.+ chunk hits) are batch-fetched from ES by ``doc_id``.++ For every matched doc, its parent cluster is also resolved and prepended+ to the result set (cluster score = max child score) so the frontend can+ highlight both the doc and its containing cluster in the tree.+ """+ from common.doc_store.doc_store_base import OrderByExpr++ if not items:+ return items++ _, kb = KnowledgebaseService.get_by_id(dataset_id)+ pack = _compiled_index_or_none(kb.tenant_id, dataset_id) if kb else None+ if pack is None:+ return items+ index_nm, _ = pack++ # Phase 1: shape items that already have _nav; collect doc_ids for chunk-only hits.+ doc_items: list[dict] = []+ missing_doc_ids: list[str] = []+ for it in items:+ nav = it.get("_nav")+ if nav:+ is_cluster = nav.get("type") == "nav_cluster"+ item = {+ "name": nav.get("name") or "",+ "description": nav.get("description") or "",+ "keywords": nav.get("keywords") or [],+ "entities": nav.get("entities") or [],+ "graph_content": nav.get("graph_content") or "",+ "doc_count": nav.get("doc_count") or (0 if is_cluster else 1),+ "type": "cluster" if is_cluster else "doc",+ "doc_id": nav.get("doc_id"),+ "has_children": is_cluster,+ "score": it.get("score", 0.0),+ }+ # Capture parent_kwd for docs so we can fetch the parent cluster.+ if not is_cluster:+ pn = nav.get("parent_kwd") or []+ if isinstance(pn, (list, tuple, set)):+ pn = next(iter(pn), "")+ if pn:+ item["_parent_kwd"] = str(pn)+ doc_items.append(item)+ else:+ doc_id = it.get("doc_id", "")+ if doc_id:+ missing_doc_ids.append(doc_id)+ doc_items.append(it)++ # Phase 2: batch-fetch nav_doc rows for chunk-only hits (need parent_kwd).+ all_doc_ids = missing_doc_ids + [d["doc_id"] for d in doc_items if d.get("type") == "doc" and d.get("doc_id") and not d.get("_parent_kwd")]+ nav_rows_by_doc_id: dict[str, dict] = {}+ if all_doc_ids:+ try:+ res = settings.docStoreConn.search(+ select_fields=_NAV_FIELDS,+ highlight_fields=[],+ condition={"compile_kwd": [_NAV_COMPILE_KWD], "doc_id": all_doc_ids},+ match_expressions=[],+ order_by=OrderByExpr(),+ offset=0,+ limit=len(all_doc_ids),+ index_names=index_nm,+ knowledgebase_ids=[dataset_id],+ )+ field_map = settings.docStoreConn.get_fields(res, _NAV_FIELDS)+ except Exception:+ logging.exception("_enrich_nav_items: docStore search failed for kb=%s", dataset_id)+ field_map = None++ for row in (field_map or {}).values():+ did = row.get("doc_id") or row.get("name") or ""+ if isinstance(did, (list, tuple, set)):+ did = next(iter(did), "")+ if did:+ nav_rows_by_doc_id[str(did)] = row++ # Enrich chunk-only items with nav info + parent_kwd.+ for i, it in enumerate(doc_items):+ if it.get("type"):+ continue+ doc_id = it.get("doc_id", "")+ row = nav_rows_by_doc_id.get(doc_id)+ if row:+ nav_item = _nav_item(row)+ nav_item["score"] = it.get("score", 0.0)+ nav_item["_parent_kwd"] = _first_str(row.get("parent_kwd"))+ doc_items[i] = nav_item++ # Phase 3: get parent_kwd for _nav doc items (search_dataset_nav strips it).+ nav_doc_names = [d["name"] for d in doc_items if d.get("type") == "doc" and not d.get("_parent_kwd") and d.get("name")]+ if nav_doc_names:+ name_field = "name.keyword" if settings.DOC_ENGINE.lower() in {"elasticsearch", "opensearch"} else "name"+ try:+ res = settings.docStoreConn.search(+ select_fields=["name", "parent_kwd"],+ highlight_fields=[],+ condition={"compile_kwd": [_NAV_COMPILE_KWD], "type_kwd": ["nav_doc"], name_field: nav_doc_names},+ match_expressions=[],+ order_by=OrderByExpr(),+ offset=0,+ limit=len(nav_doc_names),+ index_names=index_nm,+ knowledgebase_ids=[dataset_id],+ )+ field_map = settings.docStoreConn.get_fields(res, ["name", "parent_kwd"])+ except Exception:+ field_map = None++ parent_by_name: dict[str, str] = {}+ for row in (field_map or {}).values():+ nm = _first_str(row.get("name"))+ pn = _first_str(row.get("parent_kwd"))+ if nm and pn:+ parent_by_name[nm] = pn++ for it in doc_items:+ if it.get("type") == "doc" and not it.get("_parent_kwd"):+ it["_parent_kwd"] = parent_by_name.get(it.get("name", ""), "")++ # Phase 4: batch-fetch parent cluster rows.+ parent_names = {it["_parent_kwd"] for it in doc_items if it.get("_parent_kwd")}+ cluster_by_name: dict[str, dict] = {}+ if parent_names:+ name_field = "name.keyword" if settings.DOC_ENGINE.lower() in {"elasticsearch", "opensearch"} else "name"+ try:+ res = settings.docStoreConn.search(+ select_fields=_NAV_FIELDS,+ highlight_fields=[],+ condition={"compile_kwd": [_NAV_COMPILE_KWD], "type_kwd": ["nav_cluster"], name_field: list(parent_names)},+ match_expressions=[],+ order_by=OrderByExpr(),+ offset=0,+ limit=len(parent_names),+ index_names=index_nm,+ knowledgebase_ids=[dataset_id],+ )+ field_map = settings.docStoreConn.get_fields(res, _NAV_FIELDS)+ except Exception:+ field_map = None++ for row in (field_map or {}).values():+ nm = _first_str(row.get("name"))+ if nm:+ cluster_by_name[nm] = _nav_item(row)++ # Phase 5: build result — clusters (max child score) then docs.+ cluster_scores: dict[str, float] = {}+ doc_parent: dict[int, str] = {}+ for i, it in enumerate(doc_items):+ pn = it.pop("_parent_kwd", "")+ doc_parent[i] = pn+ if pn and pn in cluster_by_name:+ cluster_scores[pn] = max(cluster_scores.get(pn, 0.0), it.get("score", 0.0))++ clusters = [{**nav_item, "score": round(cluster_scores.get(name, 0.0), 4)} for name, nav_item in cluster_by_name.items()]+ cluster_names = {c["name"] for c in clusters}++ # A matched doc whose parent cluster is also in the result is already+ # represented by that cluster's tree — don't surface it as a second,+ # separate root. This keeps a search that hits both a cluster and its+ # child doc down to a single nav_cluster tree instead of two roots.+ standalone_docs = [it for i, it in enumerate(doc_items) if it.get("type") != "cluster" and doc_parent.get(i) not in cluster_names]+ return clusters + standalone_docs++async def update_wiki_page(dataset_id: str,tenant_id: str,diff --git a/rag/advanced_rag/knowlege_compile/dataset_nav.py b/rag/advanced_rag/knowlege_compile/dataset_nav.pyindex 2ec4d74356..be7009f712 100644--- a/rag/advanced_rag/knowlege_compile/dataset_nav.py+++ b/rag/advanced_rag/knowlege_compile/dataset_nav.py@@ -82,6 +82,15 @@ _NAV_SEARCH_FIELDS = [# Weight of the dense leg in hybrid search (1 - this = BM25 leg weight)._NAV_HYBRID_DENSE_W = 0.5+# Minimum fused relevance score a nav node must reach to be returned by a+# tree search. Below this, the query is treated as "not related" to the+# dataset and search_nav_tree_descent returns an empty result instead of+# falling back to an entire subtree. The dense leg contributes at most+# ``_NAV_HYBRID_DENSE_W`` (cosine ≤ 1) and the text leg at most+# ``1 - _NAV_HYBRID_DENSE_W``, so a score below this threshold means the+# query is neither semantically nor lexically close to any node.+_NAV_TREE_MIN_SCORE = 0.1+# Stop-words skipped when deriving routing tag-words from a summary._NAV_STOP_WORDS = {"the",@@ -769,6 +778,84 @@ async def _llm_create_summary(chat_mdl, doc_summaries: list[str]) -> tuple[str,# ---------------------------------------------------------------------------+def build_nav_graph_text(graph_json: Any) -> tuple[str, str]:+ """Build (title, graph_text) from a RAPTOR graph node's content.++ The ``graph_text`` carries the FULL description of EVERY entity (not just+ the first line, and not only root/child entities). Each entity's+ description already includes its name, so we emit ONLY the description+ (no ``name: `` prefix) to avoid redundant artifacts like ``1819: 1819``.+ The result is complete for both keyword (BM25) and dense (KNN) retrieval.++ This is shared by both the ``generate_nav`` path and the parse-time path+ (``run_tree_templates``) so both produce identical, complete nav_doc+ graph_content.++ Args:+ graph_json: parsed ``content_with_weight`` of a+ ``knowledge_graph_kwd="graph"`` node, i.e. a dict with+ ``entities`` (list of ``{name, description, ...}``) and+ ``relations`` (list of ``{from, to, ...}``).++ Returns:+ ``(title, graph_text)``. ``title`` is the root entity's short+ (first-line) label; ``graph_text`` is the full multi-line content.+ """+ if not isinstance(graph_json, dict):+ return "", ""+ entities = graph_json.get("entities") or []+ relations = graph_json.get("relations") or []++ # child names = relation targets (entities that are "under" someone)+ child_names: set[str] = set()+ for rel in relations:+ if isinstance(rel, dict):+ tgt = (rel.get("to") or "").strip()+ if tgt:+ child_names.add(tgt)++ # entity name -> description+ name_desc: dict[str, str] = {}+ for ent in entities:+ if not isinstance(ent, dict):+ continue+ nm = (ent.get("name") or "").strip()+ if nm:+ name_desc[nm] = (ent.get("description") or "").strip()++ # root entity = entity whose name never appears as a relation target+ root_name = ""+ root_summary = ""+ for ent in entities:+ if isinstance(ent, dict) and ent.get("name") not in child_names:+ root_name = (ent.get("name") or "").strip()+ root_summary = name_desc.get(root_name, "")+ root_summary = root_summary.splitlines()[0].strip() if root_summary else root_name+ break++ # Build the full graph text. Each entity's description already includes+ # its name, so we emit ONLY the description (no redundant "name: " prefix)+ # to avoid artifacts like "1819: 1819".+ graph_parts: list[str] = []+ if root_name:+ root_desc = name_desc.get(root_name, "").strip()+ graph_parts.append(root_desc or root_name)++ emitted = {root_name} if root_name else set()+ if len(name_desc) > len(emitted):+ graph_parts.append("")+ for cname in sorted(name_desc):+ if cname in emitted:+ continue+ cdesc = name_desc.get(cname, "").strip()+ if cdesc:+ graph_parts.append(cdesc)+ emitted.add(cname)++ graph_text = "\n".join(graph_parts)+ return root_summary, graph_text++async def upsert_dataset_nav_doc(tenant_id: str,kb_id: str,@@ -1300,6 +1387,8 @@ async def search_dataset_nav(dense_w = _NAV_HYBRID_DENSE_W if embd_mdl is not None else 0.0# ── Dense leg: KNN over the node-summary vectors ──+ # fused value: [row, score, text_score] — text_score records the lexical+ # contribution so we can reject pure-nearest hits for an unrelated query.if embd_mdl is not None:try:vec = await _embed(embd_mdl, query)@@ -1314,7 +1403,8 @@ async def search_dataset_nav(rk = r.get("name") or r.get("doc_id") or ""if not rk:continue- fused.setdefault(rk, [r, 0.0])[1] += dense_w * _cosine_sim(vec, r.get(vf))+ entry = fused.setdefault(rk, [r, 0.0, 0.0])+ entry[1] += dense_w * _cosine_sim(vec, r.get(vf))except Exception:logging.exception("search_dataset_nav: knn failed for kb=%s", kb_id)@@ -1347,9 +1437,11 @@ async def search_dataset_nav(ts = _nav_text_score(query, r)if ts <= 0:continue- fused.setdefault(rk, [r, 0.0])[1] += text_w * ts+ entry = fused.setdefault(rk, [r, 0.0, 0.0])+ entry[1] += text_w * ts+ entry[2] += ts- rows_with_scores = [(r, s) for r, s in fused.values() if s > 0 and _in_nav_scope(r, allowed_docs)]+ rows_with_scores = [(r, s) for r, s, t in fused.values() if s > 0 and t > 0 and _in_nav_scope(r, allowed_docs)]rows_with_scores.sort(key=lambda item: item[1], reverse=True)if top_k is not None:rows_with_scores = rows_with_scores[:top_k]@@ -1392,6 +1484,7 @@ async def search_dataset_nav("doc_title": payload.get("doc_title") or "","source_type": payload.get("source_type") or "","doc_count": len(doc_ids) if (typ == "nav_cluster" and scoped_cluster) else int(r.get("doc_count_int") or len(doc_ids) or 0),+ "parent_kwd": _as_str_list(r.get("parent_kwd")),"score": float(score or 0.0),})@@ -1606,12 +1699,26 @@ async def search_nav_tree_descent(breakif c.get("type_kwd") == "nav_doc":doc_id = (c.get("doc_id") or "").strip()+ score = c["_score"] if c.get("_score") is not None else parent_scoreif not doc_id or doc_id in seen_docs:continueif allowed_docs and doc_id not in allowed_docs:continue+ # Tree search is driven by a keyword: a nav_doc that only+ # matched by vector similarity (no lexical/keyword hit,+ # i.e. ``_text_score == 0``) is unrelated to the query and+ # must not surface. Vector similarity alone always yields+ # a "nearest" node, so without this gate an unrelated+ # keyword would return the whole nearest subtree.+ if not c.get("_text_score"):+ continue+ # Skip low-relevance leaves: a below-threshold score means+ # the query is unrelated to this node, so it must not+ # surface in the tree-search results.+ if score < _NAV_TREE_MIN_SCORE:+ continueseen_docs.add(doc_id)- collected.append({"doc_id": doc_id, "score": round(c["_score"] or parent_score, 4)})+ collected.append({"doc_id": doc_id, "score": round(score, 4)})else:next_level.append(c)@@ -1624,9 +1731,19 @@ async def search_nav_tree_descent(next_level.sort(key=lambda c: c.get("_score", 0.0), reverse=True)current_level = next_level[:beam_width]- # Fallback: collect doc_ids from terminal cluster nodes.+ # Fallback: collect doc_ids from terminal cluster nodes — but only when+ # the terminal cluster itself is relevant (its score clears the threshold+ # AND it has a lexical hit). Without this guard, an unrelated query+ # (e.g. random gibberish) would always have a nearest cluster and the+ # search would dump the entire subtree's documents back, defeating the+ # purpose of tree search.if not collected and current_level:for node in current_level:+ node_score = node.get("_score", 0.0) or 0.0+ if node_score < _NAV_TREE_MIN_SCORE:+ continue+ if not node.get("_text_score"):+ continuefor did in node.get("doc_ids_kwd") or []:did_str = str(did).strip()if not did_str or did_str in seen_docs:@@ -1634,7 +1751,7 @@ async def search_nav_tree_descent(if allowed_docs and did_str not in allowed_docs:continueseen_docs.add(did_str)- collected.append({"doc_id": did_str, "score": round(node.get("_score", 0.0), 4)})+ collected.append({"doc_id": did_str, "score": round(node_score, 4)})if top_k is not None and len(collected) >= top_k:breakif top_k is not None and len(collected) >= top_k:@@ -1659,7 +1776,7 @@ def _hybrid_fuse(filters. Here we merge both legs by ``name``/``doc_id``."""text_w = 1.0 - dense_w- fused: dict[str, tuple[dict, float]] = {} # key → (row, score)+ fused: dict[str, tuple[dict, float, float]] = {} # key → (row, score, text_score)# KNN leg: raw cosine * dense_w, matching search_dataset_nav's scale so# both search paths score the dense leg identically.@@ -1668,7 +1785,7 @@ def _hybrid_fuse(if not rk:continuekey = f"knn:{rk}"- fused[key] = (r, _cosine_sim(vec, r.get(vf, [])) * dense_w)+ fused[key] = (r, _cosine_sim(vec, r.get(vf, [])) * dense_w, 0.0)# Text leg.for r in text_rows:@@ -1680,20 +1797,28 @@ def _hybrid_fuse(continuekey = f"text:{rk}"if key in fused:- fused[key] = (fused[key][0], fused[key][1] + text_w * ts)+ fused[key] = (fused[key][0], fused[key][1] + text_w * ts, fused[key][2] + ts)else:key_knn = f"knn:{rk}"if key_knn in fused:- fused[key_knn] = (fused[key_knn][0], fused[key_knn][1] + text_w * ts)+ fused[key_knn] = (+ fused[key_knn][0],+ fused[key_knn][1] + text_w * ts,+ fused[key_knn][2] + ts,+ )else:- fused[key] = (r, text_w * ts)+ fused[key] = (r, text_w * ts, ts)- rows_with_scores = [(r, s) for r, s in fused.values() if s > 0]+ rows_with_scores = [(r, s, t) for r, s, t in fused.values() if s > 0]rows_with_scores.sort(key=lambda item: item[1], reverse=True)result = rows_with_scores[:top_k]- for r, s in result:+ for r, s, t in result:r["_score"] = s- return [r for r, _ in result]+ # Lexical contribution: a node matched only by vector similarity (no+ # keyword/entity hit) is recorded as _text_score == 0 so callers can+ # reject pure-nearest hits for an unrelated keyword query.+ r["_text_score"] = t+ return [r for r, _, _ in result]def _as_str_list(value) -> list[str]:diff --git a/rag/flow/compiler/compiler.py b/rag/flow/compiler/compiler.pyindex a8741bdb8f..b57cf433c6 100644--- a/rag/flow/compiler/compiler.py+++ b/rag/flow/compiler/compiler.py@@ -20,7 +20,7 @@ from types import SimpleNamespaceimport xxhash-from agent.component.llm import LLMParam, LLM+from agent.component.llm import LLM, LLMParamfrom api.db.joint_services.tenant_model_service import get_model_config_by_id, get_tenant_default_model_by_type, resolve_model_configfrom api.db.services.document_service import DocumentServicefrom api.db.services.knowledgebase_service import KnowledgebaseService@@ -28,7 +28,6 @@ from api.db.services.llm_service import LLMBundlefrom api.db.services.task_service import has_canceledfrom common.constants import LLMTypefrom common.token_utils import num_tokens_from_string-from rag.nlp import naive_mergefrom rag.advanced_rag.knowlege_compile.runner import (DOC_STRUCTURE_COMPILE_BATCH_CHUNKS,load_active_templates,@@ -37,6 +36,7 @@ from rag.advanced_rag.knowlege_compile.runner import (split_tree_templates,)from rag.flow.base import ProcessBase, ProcessParamBase+from rag.nlp import naive_mergeclass CompilerParam(ProcessParamBase, LLMParam):@@ -471,13 +471,23 @@ class Compiler(ProcessBase, LLM):continuetry:- from rag.advanced_rag.knowlege_compile.dataset_nav import upsert_dataset_nav_doc+ from rag.advanced_rag.knowlege_compile.dataset_nav import (+ build_nav_graph_text,+ upsert_dataset_nav_doc,+ )+ # Carry the FULL entity descriptions into the parse-time+ # nav_doc as graph_content (graph_text from the RAPTOR graph),+ # without changing the nav clustering input. The `title` keeps+ # using tree["title"] -- exactly what upsert_dataset_nav_doc+ # used to derive from `tree` before this change -- so the parse+ # cluster-title logic is unchanged.+ _, nav_graph_text = build_nav_graph_text(after_graph)await upsert_dataset_nav_doc(tenant_id,kb_id,doc_id,- tree,+ {"title": tree.get("title"), "graph_text": nav_graph_text},embd_mdl=embedding_model,chat_mdl=chat_mdl_by_tid[template_id],)@@ -600,13 +610,7 @@ class Compiler(ProcessBase, LLM):should_rechunk = any(_template_requests_rechunk(cfg) for _, cfg in active_templates)target_token_size = self._PARSER_CANDIDATE_TOKEN_SIZE if should_rechunk else self._PARSER_TEXT_CHUNK_TOKEN_SIZE- if output_format == "chunks":- chunks = self._normalize_upstream_chunks(- kwargs,- split_json_text=should_rechunk,- target_token_size=target_token_size,- )- elif output_format in {"markdown", "text", "html"}:+ if output_format == "chunks" or output_format in {"markdown", "text", "html"}:chunks = self._normalize_upstream_chunks(kwargs,split_json_text=should_rechunk,@@ -627,7 +631,7 @@ class Compiler(ProcessBase, LLM):for idx, ck in enumerate(chunks):ck["doc_id"] = doc_id- ck["id"] = xxhash.xxh64(f"{ck['text']}\x00{ck['doc_id']}\x00{idx}".encode("utf-8")).hexdigest()+ ck["id"] = xxhash.xxh64(f"{ck['text']}\x00{ck['doc_id']}\x00{idx}".encode()).hexdigest()if self._canvas._kb_id:e, kb = KnowledgebaseService.get_by_id(self._canvas._kb_id)diff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.pyindex 16b458204a..e9e5fd799b 100644--- a/rag/svr/task_executor_refactor/chunk_post_processor.py+++ b/rag/svr/task_executor_refactor/chunk_post_processor.py@@ -30,22 +30,20 @@ import randomimport refrom datetime import datetimefrom timeit import default_timer as timer-from typing import Dict, List+from api.db.joint_services.tenant_model_service import resolve_model_config+from api.db.services.doc_metadata_service import DocMetadataService+from api.db.services.llm_service import LLMBundle+from common import settingsfrom common.constants import TAG_FLD, LLMTypefrom common.metadata_utils import turn2jsonschema, update_metadata_to-from common import settings+from rag.graphrag.utils import get_llm_cache, get_tags_from_cache, set_llm_cache, set_tags_to_cachefrom rag.nlp import rag_tokenizer+from rag.prompts.generator import content_tagging, gen_metadata, keyword_extraction, question_proposalfrom rag.svr.task_executor_refactor.task_context import TaskContext-from api.db.services.doc_metadata_service import DocMetadataService-from api.db.services.llm_service import LLMBundle-from api.db.joint_services.tenant_model_service import resolve_model_config-from rag.prompts.generator import gen_metadata, keyword_extraction, question_proposal, content_tagging-from rag.graphrag.utils import get_llm_cache, set_llm_cache, get_tags_from_cache, set_tags_to_cache--async def extract_keywords(docs: List[Dict], ctx: TaskContext) -> None:+async def extract_keywords(docs: list[dict], ctx: TaskContext) -> None:"""Extract keywords for chunks.Args:@@ -79,15 +77,15 @@ async def extract_keywords(docs: List[Dict], ctx: TaskContext) -> None:try:await asyncio.gather(*tasks, return_exceptions=False)except Exception as e:- logging.error("Error in doc_keyword_extraction: {}".format(e))+ logging.error(f"Error in doc_keyword_extraction: {e}")for t in tasks:t.cancel()await asyncio.gather(*tasks, return_exceptions=True)raise- ctx.progress_cb(msg="Keywords generation {} chunks completed in {:.2f}s".format(len(docs), timer() - st))+ ctx.progress_cb(msg=f"Keywords generation {len(docs)} chunks completed in {timer() - st:.2f}s")-async def generate_questions(docs: List[Dict], ctx: TaskContext) -> None:+async def generate_questions(docs: list[dict], ctx: TaskContext) -> None:"""Generate questions for chunks.Args:@@ -125,7 +123,7 @@ async def generate_questions(docs: List[Dict], ctx: TaskContext) -> None:t.cancel()await asyncio.gather(*tasks, return_exceptions=True)raise- ctx.progress_cb(msg="Question generation {} chunks completed in {:.2f}s".format(len(docs), timer() - st))+ ctx.progress_cb(msg=f"Question generation {len(docs)} chunks completed in {timer() - st:.2f}s")def build_metadata_config(parser_config: dict) -> list:@@ -167,7 +165,7 @@ def build_metadata_config(parser_config: dict) -> list:return metadata_conf-async def generate_metadata(docs: List[Dict], ctx: TaskContext) -> None:+async def generate_metadata(docs: list[dict], ctx: TaskContext) -> None:"""Generate metadata for chunks.Args:@@ -219,7 +217,7 @@ async def generate_metadata(docs: List[Dict], ctx: TaskContext) -> None:ctx.write_interceptor.intercept("DocMetadataService.update_document_metadata")else:DocMetadataService.update_document_metadata(ctx.doc_id, metadata)- ctx.progress_cb(msg="Metadata generation {} chunks completed in {:.2f}s".format(len(docs), timer() - st))+ ctx.progress_cb(msg=f"Metadata generation {len(docs)} chunks completed in {timer() - st:.2f}s")def apply_built_in_metadata(ctx: TaskContext) -> None:@@ -244,7 +242,7 @@ def apply_built_in_metadata(ctx: TaskContext) -> None:DocMetadataService.update_document_metadata(ctx.doc_id, existing_meta)-async def apply_tags(docs: List[Dict], ctx: TaskContext) -> None:+async def apply_tags(docs: list[dict], ctx: TaskContext) -> None:"""Apply tags to chunks.Args:@@ -307,15 +305,15 @@ async def apply_tags(docs: List[Dict], ctx: TaskContext) -> None:try:await asyncio.gather(*tasks, return_exceptions=False)except Exception as e:- logging.error("Error tagging docs: {}".format(e))+ logging.error(f"Error tagging docs: {e}")for t in tasks:t.cancel()await asyncio.gather(*tasks, return_exceptions=True)raise- ctx.progress_cb(msg="Tagging {} chunks completed in {:.2f}s".format(len(docs), timer() - st))+ ctx.progress_cb(msg=f"Tagging {len(docs)} chunks completed in {timer() - st:.2f}s")-def count_with_key(docs: List[Dict], key: str) -> int:+def count_with_key(docs: list[dict], key: str) -> int:"""Count docs that have a specific key.Args:@@ -348,36 +346,37 @@ def count_with_key(docs: List[Dict], key: str) -> int:# ``_load_chunks_for_doc`` without a circular import.# =====================================================================-import numpy as np # noqa: E402-from typing import Callable, Optional # noqa: E402+from collections.abc import Callable-from common.misc_utils import thread_pool_exec # noqa: E402-from common.token_utils import num_tokens_from_string # noqa: E402-from rag.nlp import search # noqa: E402-from api.db.services.document_service import DocumentService # noqa: E402-from api.db.services.compilation_template_group_service import ( # noqa: E402+import numpy as np++from api.db.services.compilation_template_group_service import (CompilationTemplateGroupService,)-from api.db.services.task_service import ( # noqa: E402+from api.db.services.document_service import DocumentService+from api.db.services.task_service import (abort_doc_chunking_counter,clear_doc_chunking_counter,credit_doc_chunking_task,is_doc_chunking_aborted,)-+from common.misc_utils import thread_pool_exec+from common.token_utils import num_tokens_from_string# ----- tunables ------------------------------------------------------# The structure-compile batching / merge-flush / chain-correction tunables# and the non-tree compilation core moved to# ``rag.advanced_rag.knowlege_compile.runner`` so the ``rag.flow`` Compiler# component can share them. Re-exported here for backwards compatibility.-from rag.advanced_rag.knowlege_compile.runner import ( # noqa: E402+from rag.advanced_rag.knowlege_compile.runner import (DOC_STRUCTURE_COMPILE_BATCH_CHUNKS,DOC_STRUCTURE_MERGE_MAX_DOCS, # noqa: F401STRUCTURE_CHAIN_CORRECTION_TIMEOUT_S, # noqa: F401load_active_templates,run_structure_compile_over_batches,)+from rag.nlp import search+# ----- parser_config helpers -----------------------------------------@@ -467,7 +466,7 @@ def cap_done_progress(progress_cb: Callable) -> Callable:# ----- tree helpers ---------------------------------------------------def raptor_tree_to_graph(tree: Dict) -> Dict:+def raptor_tree_to_graph(tree: dict) -> dict:"""Project a RAPTOR tree dict (from ``Raptor(is_tree=True)``) ontothe ``{entities, relations}`` shape the document-structure graphendpoint already serves for ``page_index``-kind rows."""@@ -507,7 +506,7 @@ def raptor_tree_to_graph(tree: Dict) -> Dict:tree = _collapse_unary(tree) if isinstance(tree, dict) else tree- def _walk(node: dict, parent_id: Optional[str]) -> None:+ def _walk(node: dict, parent_id: str | None) -> None:if not isinstance(node, dict):returntitle = node.get("title") or ""@@ -534,7 +533,7 @@ def raptor_tree_to_graph(tree: Dict) -> Dict:return {"entities": entities, "relations": relations}-async def rewrite_duplicate_tree_names(tree: Dict, chat_mdl) -> None:+async def rewrite_duplicate_tree_names(tree: dict, chat_mdl) -> None:"""Rewrite only duplicate tree titles whose descriptions differ."""from rag.advanced_rag.knowlege_compile._common import knowledge_compile_gen_conffrom rag.prompts.generator import gen_json@@ -625,7 +624,7 @@ async def load_chunks_with_vec(order_by.asc("page_num_int")order_by.asc("top_int")- out: list[tuple[str, "np.ndarray", str]] = []+ out: list[tuple[str, np.ndarray, str]] = []offset = 0PAGE = 500while True:@@ -687,6 +686,7 @@ async def rechunk_doc_by_tree(via ``available_int=0`` and stamped with ``superseded_by_chunk_id``."""from datetime import datetime+from common.misc_utils import get_uuidctx = handler._task_context@@ -899,8 +899,8 @@ async def run_tree_templates(doc. Each pair runs RAPTOR with ``is_tree=True`` via``RaptorService.build_doc_tree`` and persists a single graph rowvia ``_struct_upsert_graph_json``."""- from rag.svr.task_executor_refactor.raptor_service import RaptorServicefrom rag.advanced_rag.knowlege_compile.structure import _struct_upsert_graph_json+ from rag.svr.task_executor_refactor.raptor_service import RaptorServicectx = handler._task_contextprogress_cb = ctx.progress_cb@@ -997,19 +997,30 @@ async def run_tree_templates()continue+ # Persist the per-doc nav_doc right after the graph node, so parsing a+ # file yields a nav_doc with the FULL entity descriptions as+ # graph_content -- without needing to run GENERATE NAVIGATION separately+ # and without changing the nav clustering input. The `title` keeps using+ # tree["title"] (what upsert_dataset_nav_doc used to derive from `tree`+ # before this change), so the parse cluster-title logic is unchanged.+ # Only do this when the graph actually contains entities, otherwise skip+ # (avoid empty graph_content when RAPTOR produced nothing).try:- from rag.advanced_rag.knowlege_compile.dataset_nav import (- upsert_dataset_nav_doc,- )+ if graph.get("entities"):+ from rag.advanced_rag.knowlege_compile.dataset_nav import (+ build_nav_graph_text,+ upsert_dataset_nav_doc,+ )- await upsert_dataset_nav_doc(- ctx.tenant_id,- ctx.kb_id,- doc_id,- tree,- embd_mdl=embedding_model,- chat_mdl=chat_mdl_by_tid[template_id],- )+ _, nav_graph_text = build_nav_graph_text(graph)+ await upsert_dataset_nav_doc(+ ctx.tenant_id,+ ctx.kb_id,+ doc_id,+ {"title": tree.get("title"), "graph_text": nav_graph_text},+ embd_mdl=embedding_model,+ chat_mdl=chat_mdl_by_tid[template_id],+ )except Exception:logging.exception("tree-template %s: dataset_nav upsert failed for doc %s",