Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Search in navigation tree & store graph content in nav_doc (#18303)

qinling0210554fb1133ac3
diff --git a/api/apps/restful_apis/dataset_api.py b/api/apps/restful_apis/dataset_api.py
index cd0004b6a7..4cc0f969c8 100644
--- a/api/apps/restful_apis/dataset_api.py
+++ b/api/apps/restful_apis/dataset_api.py
@@ -1028,12 +1028,23 @@ async def list_dataset_nav(tenant_id, dataset_id):
"""First level of the dataset navigation tree — the top-level clusters.
GET /api/v1/datasets/<dataset_id>/navigation
- Success: {"code": 0, "data": {"total": <n>, "items": [{name, description, doc_count, type, has_children}, ...]}}
+ GET /api/v1/datasets/<dataset_id>/navigation?keywords=<query>&top_k=<n>
+ Success: {"code": 0, "data": {"total": <n>, "items": [{name, description, doc_count, type, has_children, ...}, ...]}}
"""
+ q = (request.args.get("keywords") or "").strip() or None
+ top_k_raw = request.args.get("top_k")
+ top_k = None
+ if top_k_raw:
+ try:
+ top_k = max(1, int(top_k_raw))
+ except (ValueError, TypeError):
+ return get_error_data_result(message="top_k must be a positive integer")
try:
success, result = await dataset_api_service.list_nav_clusters(
dataset_id,
tenant_id,
+ q=q,
+ top_k=top_k,
)
if success:
return get_result(data=result)
@@ -1068,7 +1079,7 @@ async def search_dataset_nav(tenant_id, dataset_id):
documents; omitted → all documents of the dataset.
Success: {"code": 0, "data": {"mode": <mode>, "total": <n>,
- "items": [{"doc_id": str, "score": float}, ...]}}
+ "items": [{name, description, doc_count, type, has_children, doc_id, score, ...}, ...]}}
"""
q = (request.args.get("q") or "").strip()
if not q:
@@ -1092,6 +1103,7 @@ async def search_dataset_nav(tenant_id, dataset_id):
doc_scope=doc_scope,
)
if success:
+ result["items"] = await dataset_api_service._enrich_nav_items(dataset_id, tenant_id, result.get("items", []))
return get_result(data=result)
if isinstance(result, dict):
msg = result.get("error", result)
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index 5af1da3300..a5cb969f41 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -3206,6 +3206,32 @@ _NAV_FIELDS = [
]
+def _first_str(val) -> str:
+ """Extract the first string from a value that may be str, list, tuple, or set."""
+ if isinstance(val, (list, tuple, set)):
+ return str(next(iter(val), "") or "")
+ return str(val or "")
+
+
+def _resolve_embd_mdl(kb):
+ """Resolve the embedding model for a knowledge base, or None on failure."""
+ from api.db.joint_services.tenant_model_service import get_tenant_default_model_by_type
+ from api.db.services.llm_service import LLMBundle
+ from common.constants import LLMType
+
+ try:
+ if kb.embd_id:
+ embd_model_config = resolve_model_config(kb.tenant_id, LLMType.EMBEDDING, kb.embd_id)
+ else:
+ embd_model_config = get_tenant_default_model_by_type(kb.tenant_id, LLMType.EMBEDDING)
+ if embd_model_config is None:
+ return None
+ return LLMBundle(kb.tenant_id, embd_model_config)
+ except Exception:
+ logging.exception("Failed to resolve embedding model for kb=%s", kb.id)
+ return None
+
+
def _nav_item(row: dict) -> dict:
"""Shape one nav row into a UI node: name, description, doc count, type."""
try:
@@ -3276,8 +3302,19 @@ async def _nav_search(dataset_id: str, tenant_id: str, condition: dict, page: in
return True, {"total": int(total or 0), "items": items}
-async def list_nav_clusters(dataset_id: str, tenant_id: str, page: int = 1, page_size: int = 1000):
- """First level of the nav tree: the clusters with no parent."""
+async def list_nav_clusters(dataset_id: str, tenant_id: str, page: int = 1, page_size: int = 1000, q: str | None = None, top_k: int | None = None):
+ """First level of the nav tree: the clusters with no parent.
+
+ When ``q`` is provided, runs a tree-structured search (mode="navigation_tree")
+ and returns enriched nav node items in the same ``_nav_item`` shape so the
+ frontend tree search can reuse this endpoint.
+ """
+ if q and q.strip():
+ success, result = await search_dataset_layers(dataset_id, tenant_id, q.strip(), "navigation_tree", top_k=top_k or 1000)
+ if not success:
+ return success, result
+ result["items"] = await _enrich_nav_items(dataset_id, tenant_id, result.get("items", []))
+ return True, {"total": result.get("total", 0), "items": result["items"]}
condition = {
"compile_kwd": [_NAV_COMPILE_KWD],
"type_kwd": ["nav_cluster"],
@@ -3475,13 +3512,12 @@ async def generate_nav(
)
from api.db.services.llm_service import LLMBundle
from common.constants import LLMType
- from rag.advanced_rag.knowlege_compile.dataset_nav import upsert_dataset_nav_doc
+ from rag.advanced_rag.knowlege_compile.dataset_nav import (
+ build_nav_graph_text,
+ upsert_dataset_nav_doc,
+ )
- if kb.embd_id:
- embd_model_config = resolve_model_config(kb.tenant_id, LLMType.EMBEDDING, kb.embd_id)
- else:
- embd_model_config = get_tenant_default_model_by_type(kb.tenant_id, LLMType.EMBEDDING)
- embd_mdl = LLMBundle(kb.tenant_id, embd_model_config)
+ embd_mdl = _resolve_embd_mdl(kb)
chat_model_config = get_tenant_default_model_by_type(kb.tenant_id, LLMType.CHAT)
chat_mdl = LLMBundle(kb.tenant_id, chat_model_config)
@@ -3552,53 +3588,15 @@ async def generate_nav(
graph = json.loads(row.get("content_with_weight") or "{}")
except Exception:
continue
- entities = graph.get("entities") or []
- relations = graph.get("relations") or []
- child_names = {r.get("to") for r in relations if isinstance(r, dict)}
-
# Build both:
# - root_summary: first line of root desc (short,
# for the display title / description field)
# - graph_text: structured text from ALL entities
# and relations (for embedding, keyword extraction,
# entity extraction, and stored as graph_content).
- # entity name -> description
- name_desc: dict[str, str] = {}
- for ent in entities:
- if not isinstance(ent, dict):
- continue
- nm = (ent.get("name") or "").strip()
- if nm:
- name_desc[nm] = (ent.get("description") or "").strip()
-
- # root entity = entity whose name never appears as a
- # relation target (not a child of anyone).
- root_name = ""
- root_summary = ""
- for ent in entities:
- if isinstance(ent, dict) and ent.get("name") not in child_names:
- root_name = (ent.get("name") or "").strip()
- root_summary = name_desc.get(root_name, "")
- root_summary = root_summary.splitlines()[0].strip() if root_summary else root_name
- break
-
- # Build the full graph text
- graph_parts: list[str] = []
- if root_name and name_desc.get(root_name):
- graph_parts.append(root_name)
- graph_parts.append(name_desc[root_name])
-
- child_names_set = {n for n in name_desc if n in child_names}
- if child_names_set:
- graph_parts.append("")
- for cname in sorted(child_names_set):
- cdesc = name_desc.get(cname, "")
- line = f"- {cname}"
- if cdesc:
- line += f": {cdesc.splitlines()[0].strip()}"
- graph_parts.append(line)
-
- graph_text = "\n".join(graph_parts) if graph_parts else ""
+ # Shared with the parse-time path (run_tree_templates)
+ # so both produce identical, complete nav_doc content.
+ root_summary, graph_text = build_nav_graph_text(graph)
if root_summary:
raptor_summaries[gid] = {
@@ -3633,18 +3631,36 @@ async def generate_nav(
return False, "No documents found in dataset."
# Step 1: delete the entire existing navigation tree so we start clean.
+ # ``deleted`` reports the number of *clusters* removed (not nav_doc leaves),
+ # since that is the meaningful unit for the navigation tree.
deleted = 0
pack = _compiled_index_or_none(kb.tenant_id, dataset_id)
if pack is not None:
index_nm, _ = pack
try:
- deleted = await thread_pool_exec(
+ from common.doc_store.doc_store_base import OrderByExpr
+
+ # Count existing clusters before wiping the tree.
+ count_res = await thread_pool_exec(
+ settings.docStoreConn.search,
+ ["id"],
+ [],
+ {"compile_kwd": [_NAV_COMPILE_KWD], "type_kwd": ["nav_cluster"]},
+ [],
+ OrderByExpr(),
+ 0,
+ 10000,
+ index_nm,
+ [dataset_id],
+ )
+ deleted = len(settings.docStoreConn.get_fields(count_res, ["id"]) or {})
+
+ await thread_pool_exec(
settings.docStoreConn.delete,
{"compile_kwd": [_NAV_COMPILE_KWD]},
index_nm,
dataset_id,
)
- deleted = int(deleted or 0)
except Exception:
logging.exception("generate_nav: failed to clear existing nav for kb=%s", dataset_id)
return False, "Failed to clear existing navigation tree."
@@ -3831,12 +3847,15 @@ async def _nav_search_result(tenant_id, dataset_id, query, top_k, embd_mdl, sear
doc_id = str(doc_ids[0]).strip() if doc_ids else ""
if not doc_id:
continue
- items.append(
- {
- "doc_id": doc_id,
- "score": round(float(r.get("score", 0.0)), 4),
- }
- )
+ item = {
+ "doc_id": doc_id,
+ "score": round(float(r.get("score", 0.0)), 4),
+ }
+ # Preserve the full nav node info so callers can enrich without a
+ # second ES round-trip. Clusters have doc_id="" but carry name.
+ if r.get("name") or r.get("description"):
+ item["_nav"] = r
+ items.append(item)
return items
@@ -3897,7 +3916,7 @@ async def _search_layers_all(tenant_id, dataset_id, query, top_k, embd_mdl, kb,
return_exceptions=True,
)
- doc_scores: dict[str, float] = {}
+ doc_scores: dict[str, dict] = {}
for result in result_lists:
if isinstance(result, Exception):
continue
@@ -3907,11 +3926,13 @@ async def _search_layers_all(tenant_id, dataset_id, query, top_k, embd_mdl, kb,
for item in data.get("items", []):
doc_id = item.get("doc_id", "")
score = float(item.get("score", 0.0))
- if doc_id and score > doc_scores.get(doc_id, -1.0):
- doc_scores[doc_id] = score
+ # Clusters have no doc_id; key by name so they survive dedup.
+ key = doc_id or item.get("_nav", {}).get("name", "")
+ if key and score > doc_scores.get(key, {}).get("score", -1.0):
+ doc_scores[key] = item
items = sorted(
- ({"doc_id": d, "score": round(s, 4)} for d, s in doc_scores.items()),
+ doc_scores.values(),
key=lambda x: x["score"],
reverse=True,
)
@@ -3921,6 +3942,178 @@ async def _search_layers_all(tenant_id, dataset_id, query, top_k, embd_mdl, kb,
return True, {"mode": "all", "total": len(items), "items": items}
+async def _enrich_nav_items(dataset_id: str, tenant_id: str, items: list[dict]) -> list[dict]:
+ """Enrich ``{doc_id, score}`` items with full nav node info.
+
+ Items that already carry a ``_nav`` payload (from nav_doc/nav_cluster/
+ navigation_tree search) are shaped in-process. Items without it (e.g.
+ chunk hits) are batch-fetched from ES by ``doc_id``.
+
+ For every matched doc, its parent cluster is also resolved and prepended
+ to the result set (cluster score = max child score) so the frontend can
+ highlight both the doc and its containing cluster in the tree.
+ """
+ from common.doc_store.doc_store_base import OrderByExpr
+
+ if not items:
+ return items
+
+ _, kb = KnowledgebaseService.get_by_id(dataset_id)
+ pack = _compiled_index_or_none(kb.tenant_id, dataset_id) if kb else None
+ if pack is None:
+ return items
+ index_nm, _ = pack
+
+ # Phase 1: shape items that already have _nav; collect doc_ids for chunk-only hits.
+ doc_items: list[dict] = []
+ missing_doc_ids: list[str] = []
+ for it in items:
+ nav = it.get("_nav")
+ if nav:
+ is_cluster = nav.get("type") == "nav_cluster"
+ item = {
+ "name": nav.get("name") or "",
+ "description": nav.get("description") or "",
+ "keywords": nav.get("keywords") or [],
+ "entities": nav.get("entities") or [],
+ "graph_content": nav.get("graph_content") or "",
+ "doc_count": nav.get("doc_count") or (0 if is_cluster else 1),
+ "type": "cluster" if is_cluster else "doc",
+ "doc_id": nav.get("doc_id"),
+ "has_children": is_cluster,
+ "score": it.get("score", 0.0),
+ }
+ # Capture parent_kwd for docs so we can fetch the parent cluster.
+ if not is_cluster:
+ pn = nav.get("parent_kwd") or []
+ if isinstance(pn, (list, tuple, set)):
+ pn = next(iter(pn), "")
+ if pn:
+ item["_parent_kwd"] = str(pn)
+ doc_items.append(item)
+ else:
+ doc_id = it.get("doc_id", "")
+ if doc_id:
+ missing_doc_ids.append(doc_id)
+ doc_items.append(it)
+
+ # Phase 2: batch-fetch nav_doc rows for chunk-only hits (need parent_kwd).
+ all_doc_ids = missing_doc_ids + [d["doc_id"] for d in doc_items if d.get("type") == "doc" and d.get("doc_id") and not d.get("_parent_kwd")]
+ nav_rows_by_doc_id: dict[str, dict] = {}
+ if all_doc_ids:
+ try:
+ res = settings.docStoreConn.search(
+ select_fields=_NAV_FIELDS,
+ highlight_fields=[],
+ condition={"compile_kwd": [_NAV_COMPILE_KWD], "doc_id": all_doc_ids},
+ match_expressions=[],
+ order_by=OrderByExpr(),
+ offset=0,
+ limit=len(all_doc_ids),
+ index_names=index_nm,
+ knowledgebase_ids=[dataset_id],
+ )
+ field_map = settings.docStoreConn.get_fields(res, _NAV_FIELDS)
+ except Exception:
+ logging.exception("_enrich_nav_items: docStore search failed for kb=%s", dataset_id)
+ field_map = None
+
+ for row in (field_map or {}).values():
+ did = row.get("doc_id") or row.get("name") or ""
+ if isinstance(did, (list, tuple, set)):
+ did = next(iter(did), "")
+ if did:
+ nav_rows_by_doc_id[str(did)] = row
+
+ # Enrich chunk-only items with nav info + parent_kwd.
+ for i, it in enumerate(doc_items):
+ if it.get("type"):
+ continue
+ doc_id = it.get("doc_id", "")
+ row = nav_rows_by_doc_id.get(doc_id)
+ if row:
+ nav_item = _nav_item(row)
+ nav_item["score"] = it.get("score", 0.0)
+ nav_item["_parent_kwd"] = _first_str(row.get("parent_kwd"))
+ doc_items[i] = nav_item
+
+ # Phase 3: get parent_kwd for _nav doc items (search_dataset_nav strips it).
+ nav_doc_names = [d["name"] for d in doc_items if d.get("type") == "doc" and not d.get("_parent_kwd") and d.get("name")]
+ if nav_doc_names:
+ name_field = "name.keyword" if settings.DOC_ENGINE.lower() in {"elasticsearch", "opensearch"} else "name"
+ try:
+ res = settings.docStoreConn.search(
+ select_fields=["name", "parent_kwd"],
+ highlight_fields=[],
+ condition={"compile_kwd": [_NAV_COMPILE_KWD], "type_kwd": ["nav_doc"], name_field: nav_doc_names},
+ match_expressions=[],
+ order_by=OrderByExpr(),
+ offset=0,
+ limit=len(nav_doc_names),
+ index_names=index_nm,
+ knowledgebase_ids=[dataset_id],
+ )
+ field_map = settings.docStoreConn.get_fields(res, ["name", "parent_kwd"])
+ except Exception:
+ field_map = None
+
+ parent_by_name: dict[str, str] = {}
+ for row in (field_map or {}).values():
+ nm = _first_str(row.get("name"))
+ pn = _first_str(row.get("parent_kwd"))
+ if nm and pn:
+ parent_by_name[nm] = pn
+
+ for it in doc_items:
+ if it.get("type") == "doc" and not it.get("_parent_kwd"):
+ it["_parent_kwd"] = parent_by_name.get(it.get("name", ""), "")
+
+ # Phase 4: batch-fetch parent cluster rows.
+ parent_names = {it["_parent_kwd"] for it in doc_items if it.get("_parent_kwd")}
+ cluster_by_name: dict[str, dict] = {}
+ if parent_names:
+ name_field = "name.keyword" if settings.DOC_ENGINE.lower() in {"elasticsearch", "opensearch"} else "name"
+ try:
+ res = settings.docStoreConn.search(
+ select_fields=_NAV_FIELDS,
+ highlight_fields=[],
+ condition={"compile_kwd": [_NAV_COMPILE_KWD], "type_kwd": ["nav_cluster"], name_field: list(parent_names)},
+ match_expressions=[],
+ order_by=OrderByExpr(),
+ offset=0,
+ limit=len(parent_names),
+ index_names=index_nm,
+ knowledgebase_ids=[dataset_id],
+ )
+ field_map = settings.docStoreConn.get_fields(res, _NAV_FIELDS)
+ except Exception:
+ field_map = None
+
+ for row in (field_map or {}).values():
+ nm = _first_str(row.get("name"))
+ if nm:
+ cluster_by_name[nm] = _nav_item(row)
+
+ # Phase 5: build result — clusters (max child score) then docs.
+ cluster_scores: dict[str, float] = {}
+ doc_parent: dict[int, str] = {}
+ for i, it in enumerate(doc_items):
+ pn = it.pop("_parent_kwd", "")
+ doc_parent[i] = pn
+ if pn and pn in cluster_by_name:
+ cluster_scores[pn] = max(cluster_scores.get(pn, 0.0), it.get("score", 0.0))
+
+ clusters = [{**nav_item, "score": round(cluster_scores.get(name, 0.0), 4)} for name, nav_item in cluster_by_name.items()]
+ cluster_names = {c["name"] for c in clusters}
+
+ # A matched doc whose parent cluster is also in the result is already
+ # represented by that cluster's tree — don't surface it as a second,
+ # separate root. This keeps a search that hits both a cluster and its
+ # child doc down to a single nav_cluster tree instead of two roots.
+ standalone_docs = [it for i, it in enumerate(doc_items) if it.get("type") != "cluster" and doc_parent.get(i) not in cluster_names]
+ return clusters + standalone_docs
+
+
async def update_wiki_page(
dataset_id: str,
tenant_id: str,
diff --git a/rag/advanced_rag/knowlege_compile/dataset_nav.py b/rag/advanced_rag/knowlege_compile/dataset_nav.py
index 2ec4d74356..be7009f712 100644
--- a/rag/advanced_rag/knowlege_compile/dataset_nav.py
+++ b/rag/advanced_rag/knowlege_compile/dataset_nav.py
@@ -82,6 +82,15 @@ _NAV_SEARCH_FIELDS = [
# Weight of the dense leg in hybrid search (1 - this = BM25 leg weight).
_NAV_HYBRID_DENSE_W = 0.5
+# Minimum fused relevance score a nav node must reach to be returned by a
+# tree search. Below this, the query is treated as "not related" to the
+# dataset and search_nav_tree_descent returns an empty result instead of
+# falling back to an entire subtree. The dense leg contributes at most
+# ``_NAV_HYBRID_DENSE_W`` (cosine ≤ 1) and the text leg at most
+# ``1 - _NAV_HYBRID_DENSE_W``, so a score below this threshold means the
+# query is neither semantically nor lexically close to any node.
+_NAV_TREE_MIN_SCORE = 0.1
+
# Stop-words skipped when deriving routing tag-words from a summary.
_NAV_STOP_WORDS = {
"the",
@@ -769,6 +778,84 @@ async def _llm_create_summary(chat_mdl, doc_summaries: list[str]) -> tuple[str,
# ---------------------------------------------------------------------------
+def build_nav_graph_text(graph_json: Any) -> tuple[str, str]:
+ """Build (title, graph_text) from a RAPTOR graph node's content.
+
+ The ``graph_text`` carries the FULL description of EVERY entity (not just
+ the first line, and not only root/child entities). Each entity's
+ description already includes its name, so we emit ONLY the description
+ (no ``name: `` prefix) to avoid redundant artifacts like ``1819: 1819``.
+ The result is complete for both keyword (BM25) and dense (KNN) retrieval.
+
+ This is shared by both the ``generate_nav`` path and the parse-time path
+ (``run_tree_templates``) so both produce identical, complete nav_doc
+ graph_content.
+
+ Args:
+ graph_json: parsed ``content_with_weight`` of a
+ ``knowledge_graph_kwd="graph"`` node, i.e. a dict with
+ ``entities`` (list of ``{name, description, ...}``) and
+ ``relations`` (list of ``{from, to, ...}``).
+
+ Returns:
+ ``(title, graph_text)``. ``title`` is the root entity's short
+ (first-line) label; ``graph_text`` is the full multi-line content.
+ """
+ if not isinstance(graph_json, dict):
+ return "", ""
+ entities = graph_json.get("entities") or []
+ relations = graph_json.get("relations") or []
+
+ # child names = relation targets (entities that are "under" someone)
+ child_names: set[str] = set()
+ for rel in relations:
+ if isinstance(rel, dict):
+ tgt = (rel.get("to") or "").strip()
+ if tgt:
+ child_names.add(tgt)
+
+ # entity name -> description
+ name_desc: dict[str, str] = {}
+ for ent in entities:
+ if not isinstance(ent, dict):
+ continue
+ nm = (ent.get("name") or "").strip()
+ if nm:
+ name_desc[nm] = (ent.get("description") or "").strip()
+
+ # root entity = entity whose name never appears as a relation target
+ root_name = ""
+ root_summary = ""
+ for ent in entities:
+ if isinstance(ent, dict) and ent.get("name") not in child_names:
+ root_name = (ent.get("name") or "").strip()
+ root_summary = name_desc.get(root_name, "")
+ root_summary = root_summary.splitlines()[0].strip() if root_summary else root_name
+ break
+
+ # Build the full graph text. Each entity's description already includes
+ # its name, so we emit ONLY the description (no redundant "name: " prefix)
+ # to avoid artifacts like "1819: 1819".
+ graph_parts: list[str] = []
+ if root_name:
+ root_desc = name_desc.get(root_name, "").strip()
+ graph_parts.append(root_desc or root_name)
+
+ emitted = {root_name} if root_name else set()
+ if len(name_desc) > len(emitted):
+ graph_parts.append("")
+ for cname in sorted(name_desc):
+ if cname in emitted:
+ continue
+ cdesc = name_desc.get(cname, "").strip()
+ if cdesc:
+ graph_parts.append(cdesc)
+ emitted.add(cname)
+
+ graph_text = "\n".join(graph_parts)
+ return root_summary, graph_text
+
+
async def upsert_dataset_nav_doc(
tenant_id: str,
kb_id: str,
@@ -1300,6 +1387,8 @@ async def search_dataset_nav(
dense_w = _NAV_HYBRID_DENSE_W if embd_mdl is not None else 0.0
# ── Dense leg: KNN over the node-summary vectors ──
+ # fused value: [row, score, text_score] — text_score records the lexical
+ # contribution so we can reject pure-nearest hits for an unrelated query.
if embd_mdl is not None:
try:
vec = await _embed(embd_mdl, query)
@@ -1314,7 +1403,8 @@ async def search_dataset_nav(
rk = r.get("name") or r.get("doc_id") or ""
if not rk:
continue
- fused.setdefault(rk, [r, 0.0])[1] += dense_w * _cosine_sim(vec, r.get(vf))
+ entry = fused.setdefault(rk, [r, 0.0, 0.0])
+ entry[1] += dense_w * _cosine_sim(vec, r.get(vf))
except Exception:
logging.exception("search_dataset_nav: knn failed for kb=%s", kb_id)
@@ -1347,9 +1437,11 @@ async def search_dataset_nav(
ts = _nav_text_score(query, r)
if ts <= 0:
continue
- fused.setdefault(rk, [r, 0.0])[1] += text_w * ts
+ entry = fused.setdefault(rk, [r, 0.0, 0.0])
+ entry[1] += text_w * ts
+ entry[2] += ts
- rows_with_scores = [(r, s) for r, s in fused.values() if s > 0 and _in_nav_scope(r, allowed_docs)]
+ rows_with_scores = [(r, s) for r, s, t in fused.values() if s > 0 and t > 0 and _in_nav_scope(r, allowed_docs)]
rows_with_scores.sort(key=lambda item: item[1], reverse=True)
if top_k is not None:
rows_with_scores = rows_with_scores[:top_k]
@@ -1392,6 +1484,7 @@ async def search_dataset_nav(
"doc_title": payload.get("doc_title") or "",
"source_type": payload.get("source_type") or "",
"doc_count": len(doc_ids) if (typ == "nav_cluster" and scoped_cluster) else int(r.get("doc_count_int") or len(doc_ids) or 0),
+ "parent_kwd": _as_str_list(r.get("parent_kwd")),
"score": float(score or 0.0),
}
)
@@ -1606,12 +1699,26 @@ async def search_nav_tree_descent(
break
if c.get("type_kwd") == "nav_doc":
doc_id = (c.get("doc_id") or "").strip()
+ score = c["_score"] if c.get("_score") is not None else parent_score
if not doc_id or doc_id in seen_docs:
continue
if allowed_docs and doc_id not in allowed_docs:
continue
+ # Tree search is driven by a keyword: a nav_doc that only
+ # matched by vector similarity (no lexical/keyword hit,
+ # i.e. ``_text_score == 0``) is unrelated to the query and
+ # must not surface. Vector similarity alone always yields
+ # a "nearest" node, so without this gate an unrelated
+ # keyword would return the whole nearest subtree.
+ if not c.get("_text_score"):
+ continue
+ # Skip low-relevance leaves: a below-threshold score means
+ # the query is unrelated to this node, so it must not
+ # surface in the tree-search results.
+ if score < _NAV_TREE_MIN_SCORE:
+ continue
seen_docs.add(doc_id)
- collected.append({"doc_id": doc_id, "score": round(c["_score"] or parent_score, 4)})
+ collected.append({"doc_id": doc_id, "score": round(score, 4)})
else:
next_level.append(c)
@@ -1624,9 +1731,19 @@ async def search_nav_tree_descent(
next_level.sort(key=lambda c: c.get("_score", 0.0), reverse=True)
current_level = next_level[:beam_width]
- # Fallback: collect doc_ids from terminal cluster nodes.
+ # Fallback: collect doc_ids from terminal cluster nodes — but only when
+ # the terminal cluster itself is relevant (its score clears the threshold
+ # AND it has a lexical hit). Without this guard, an unrelated query
+ # (e.g. random gibberish) would always have a nearest cluster and the
+ # search would dump the entire subtree's documents back, defeating the
+ # purpose of tree search.
if not collected and current_level:
for node in current_level:
+ node_score = node.get("_score", 0.0) or 0.0
+ if node_score < _NAV_TREE_MIN_SCORE:
+ continue
+ if not node.get("_text_score"):
+ continue
for did in node.get("doc_ids_kwd") or []:
did_str = str(did).strip()
if not did_str or did_str in seen_docs:
@@ -1634,7 +1751,7 @@ async def search_nav_tree_descent(
if allowed_docs and did_str not in allowed_docs:
continue
seen_docs.add(did_str)
- collected.append({"doc_id": did_str, "score": round(node.get("_score", 0.0), 4)})
+ collected.append({"doc_id": did_str, "score": round(node_score, 4)})
if top_k is not None and len(collected) >= top_k:
break
if top_k is not None and len(collected) >= top_k:
@@ -1659,7 +1776,7 @@ def _hybrid_fuse(
filters. Here we merge both legs by ``name``/``doc_id``.
"""
text_w = 1.0 - dense_w
- fused: dict[str, tuple[dict, float]] = {} # key → (row, score)
+ fused: dict[str, tuple[dict, float, float]] = {} # key → (row, score, text_score)
# KNN leg: raw cosine * dense_w, matching search_dataset_nav's scale so
# both search paths score the dense leg identically.
@@ -1668,7 +1785,7 @@ def _hybrid_fuse(
if not rk:
continue
key = f"knn:{rk}"
- fused[key] = (r, _cosine_sim(vec, r.get(vf, [])) * dense_w)
+ fused[key] = (r, _cosine_sim(vec, r.get(vf, [])) * dense_w, 0.0)
# Text leg.
for r in text_rows:
@@ -1680,20 +1797,28 @@ def _hybrid_fuse(
continue
key = f"text:{rk}"
if key in fused:
- fused[key] = (fused[key][0], fused[key][1] + text_w * ts)
+ fused[key] = (fused[key][0], fused[key][1] + text_w * ts, fused[key][2] + ts)
else:
key_knn = f"knn:{rk}"
if key_knn in fused:
- fused[key_knn] = (fused[key_knn][0], fused[key_knn][1] + text_w * ts)
+ fused[key_knn] = (
+ fused[key_knn][0],
+ fused[key_knn][1] + text_w * ts,
+ fused[key_knn][2] + ts,
+ )
else:
- fused[key] = (r, text_w * ts)
+ fused[key] = (r, text_w * ts, ts)
- rows_with_scores = [(r, s) for r, s in fused.values() if s > 0]
+ rows_with_scores = [(r, s, t) for r, s, t in fused.values() if s > 0]
rows_with_scores.sort(key=lambda item: item[1], reverse=True)
result = rows_with_scores[:top_k]
- for r, s in result:
+ for r, s, t in result:
r["_score"] = s
- return [r for r, _ in result]
+ # Lexical contribution: a node matched only by vector similarity (no
+ # keyword/entity hit) is recorded as _text_score == 0 so callers can
+ # reject pure-nearest hits for an unrelated keyword query.
+ r["_text_score"] = t
+ return [r for r, _, _ in result]
def _as_str_list(value) -> list[str]:
diff --git a/rag/flow/compiler/compiler.py b/rag/flow/compiler/compiler.py
index a8741bdb8f..b57cf433c6 100644
--- a/rag/flow/compiler/compiler.py
+++ b/rag/flow/compiler/compiler.py
@@ -20,7 +20,7 @@ from types import SimpleNamespace
import xxhash
-from agent.component.llm import LLMParam, LLM
+from agent.component.llm import LLM, LLMParam
from api.db.joint_services.tenant_model_service import get_model_config_by_id, get_tenant_default_model_by_type, resolve_model_config
from api.db.services.document_service import DocumentService
from api.db.services.knowledgebase_service import KnowledgebaseService
@@ -28,7 +28,6 @@ from api.db.services.llm_service import LLMBundle
from api.db.services.task_service import has_canceled
from common.constants import LLMType
from common.token_utils import num_tokens_from_string
-from rag.nlp import naive_merge
from rag.advanced_rag.knowlege_compile.runner import (
DOC_STRUCTURE_COMPILE_BATCH_CHUNKS,
load_active_templates,
@@ -37,6 +36,7 @@ from rag.advanced_rag.knowlege_compile.runner import (
split_tree_templates,
)
from rag.flow.base import ProcessBase, ProcessParamBase
+from rag.nlp import naive_merge
class CompilerParam(ProcessParamBase, LLMParam):
@@ -471,13 +471,23 @@ class Compiler(ProcessBase, LLM):
continue
try:
- from rag.advanced_rag.knowlege_compile.dataset_nav import upsert_dataset_nav_doc
+ from rag.advanced_rag.knowlege_compile.dataset_nav import (
+ build_nav_graph_text,
+ upsert_dataset_nav_doc,
+ )
+ # Carry the FULL entity descriptions into the parse-time
+ # nav_doc as graph_content (graph_text from the RAPTOR graph),
+ # without changing the nav clustering input. The `title` keeps
+ # using tree["title"] -- exactly what upsert_dataset_nav_doc
+ # used to derive from `tree` before this change -- so the parse
+ # cluster-title logic is unchanged.
+ _, nav_graph_text = build_nav_graph_text(after_graph)
await upsert_dataset_nav_doc(
tenant_id,
kb_id,
doc_id,
- tree,
+ {"title": tree.get("title"), "graph_text": nav_graph_text},
embd_mdl=embedding_model,
chat_mdl=chat_mdl_by_tid[template_id],
)
@@ -600,13 +610,7 @@ class Compiler(ProcessBase, LLM):
should_rechunk = any(_template_requests_rechunk(cfg) for _, cfg in active_templates)
target_token_size = self._PARSER_CANDIDATE_TOKEN_SIZE if should_rechunk else self._PARSER_TEXT_CHUNK_TOKEN_SIZE
- if output_format == "chunks":
- chunks = self._normalize_upstream_chunks(
- kwargs,
- split_json_text=should_rechunk,
- target_token_size=target_token_size,
- )
- elif output_format in {"markdown", "text", "html"}:
+ if output_format == "chunks" or output_format in {"markdown", "text", "html"}:
chunks = self._normalize_upstream_chunks(
kwargs,
split_json_text=should_rechunk,
@@ -627,7 +631,7 @@ class Compiler(ProcessBase, LLM):
for idx, ck in enumerate(chunks):
ck["doc_id"] = doc_id
- ck["id"] = xxhash.xxh64(f"{ck['text']}\x00{ck['doc_id']}\x00{idx}".encode("utf-8")).hexdigest()
+ ck["id"] = xxhash.xxh64(f"{ck['text']}\x00{ck['doc_id']}\x00{idx}".encode()).hexdigest()
if self._canvas._kb_id:
e, kb = KnowledgebaseService.get_by_id(self._canvas._kb_id)
diff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.py
index 16b458204a..e9e5fd799b 100644
--- a/rag/svr/task_executor_refactor/chunk_post_processor.py
+++ b/rag/svr/task_executor_refactor/chunk_post_processor.py
@@ -30,22 +30,20 @@ import random
import re
from datetime import datetime
from timeit import default_timer as timer
-from typing import Dict, List
+from api.db.joint_services.tenant_model_service import resolve_model_config
+from api.db.services.doc_metadata_service import DocMetadataService
+from api.db.services.llm_service import LLMBundle
+from common import settings
from common.constants import TAG_FLD, LLMType
from common.metadata_utils import turn2jsonschema, update_metadata_to
-from common import settings
+from rag.graphrag.utils import get_llm_cache, get_tags_from_cache, set_llm_cache, set_tags_to_cache
from rag.nlp import rag_tokenizer
+from rag.prompts.generator import content_tagging, gen_metadata, keyword_extraction, question_proposal
from rag.svr.task_executor_refactor.task_context import TaskContext
-from api.db.services.doc_metadata_service import DocMetadataService
-from api.db.services.llm_service import LLMBundle
-from api.db.joint_services.tenant_model_service import resolve_model_config
-from rag.prompts.generator import gen_metadata, keyword_extraction, question_proposal, content_tagging
-from rag.graphrag.utils import get_llm_cache, set_llm_cache, get_tags_from_cache, set_tags_to_cache
-
-async def extract_keywords(docs: List[Dict], ctx: TaskContext) -> None:
+async def extract_keywords(docs: list[dict], ctx: TaskContext) -> None:
"""Extract keywords for chunks.
Args:
@@ -79,15 +77,15 @@ async def extract_keywords(docs: List[Dict], ctx: TaskContext) -> None:
try:
await asyncio.gather(*tasks, return_exceptions=False)
except Exception as e:
- logging.error("Error in doc_keyword_extraction: {}".format(e))
+ logging.error(f"Error in doc_keyword_extraction: {e}")
for t in tasks:
t.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
raise
- ctx.progress_cb(msg="Keywords generation {} chunks completed in {:.2f}s".format(len(docs), timer() - st))
+ ctx.progress_cb(msg=f"Keywords generation {len(docs)} chunks completed in {timer() - st:.2f}s")
-async def generate_questions(docs: List[Dict], ctx: TaskContext) -> None:
+async def generate_questions(docs: list[dict], ctx: TaskContext) -> None:
"""Generate questions for chunks.
Args:
@@ -125,7 +123,7 @@ async def generate_questions(docs: List[Dict], ctx: TaskContext) -> None:
t.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
raise
- ctx.progress_cb(msg="Question generation {} chunks completed in {:.2f}s".format(len(docs), timer() - st))
+ ctx.progress_cb(msg=f"Question generation {len(docs)} chunks completed in {timer() - st:.2f}s")
def build_metadata_config(parser_config: dict) -> list:
@@ -167,7 +165,7 @@ def build_metadata_config(parser_config: dict) -> list:
return metadata_conf
-async def generate_metadata(docs: List[Dict], ctx: TaskContext) -> None:
+async def generate_metadata(docs: list[dict], ctx: TaskContext) -> None:
"""Generate metadata for chunks.
Args:
@@ -219,7 +217,7 @@ async def generate_metadata(docs: List[Dict], ctx: TaskContext) -> None:
ctx.write_interceptor.intercept("DocMetadataService.update_document_metadata")
else:
DocMetadataService.update_document_metadata(ctx.doc_id, metadata)
- ctx.progress_cb(msg="Metadata generation {} chunks completed in {:.2f}s".format(len(docs), timer() - st))
+ ctx.progress_cb(msg=f"Metadata generation {len(docs)} chunks completed in {timer() - st:.2f}s")
def apply_built_in_metadata(ctx: TaskContext) -> None:
@@ -244,7 +242,7 @@ def apply_built_in_metadata(ctx: TaskContext) -> None:
DocMetadataService.update_document_metadata(ctx.doc_id, existing_meta)
-async def apply_tags(docs: List[Dict], ctx: TaskContext) -> None:
+async def apply_tags(docs: list[dict], ctx: TaskContext) -> None:
"""Apply tags to chunks.
Args:
@@ -307,15 +305,15 @@ async def apply_tags(docs: List[Dict], ctx: TaskContext) -> None:
try:
await asyncio.gather(*tasks, return_exceptions=False)
except Exception as e:
- logging.error("Error tagging docs: {}".format(e))
+ logging.error(f"Error tagging docs: {e}")
for t in tasks:
t.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
raise
- ctx.progress_cb(msg="Tagging {} chunks completed in {:.2f}s".format(len(docs), timer() - st))
+ ctx.progress_cb(msg=f"Tagging {len(docs)} chunks completed in {timer() - st:.2f}s")
-def count_with_key(docs: List[Dict], key: str) -> int:
+def count_with_key(docs: list[dict], key: str) -> int:
"""Count docs that have a specific key.
Args:
@@ -348,36 +346,37 @@ def count_with_key(docs: List[Dict], key: str) -> int:
# ``_load_chunks_for_doc`` without a circular import.
# =====================================================================
-import numpy as np # noqa: E402
-from typing import Callable, Optional # noqa: E402
+from collections.abc import Callable
-from common.misc_utils import thread_pool_exec # noqa: E402
-from common.token_utils import num_tokens_from_string # noqa: E402
-from rag.nlp import search # noqa: E402
-from api.db.services.document_service import DocumentService # noqa: E402
-from api.db.services.compilation_template_group_service import ( # noqa: E402
+import numpy as np
+
+from api.db.services.compilation_template_group_service import (
CompilationTemplateGroupService,
)
-from api.db.services.task_service import ( # noqa: E402
+from api.db.services.document_service import DocumentService
+from api.db.services.task_service import (
abort_doc_chunking_counter,
clear_doc_chunking_counter,
credit_doc_chunking_task,
is_doc_chunking_aborted,
)
-
+from common.misc_utils import thread_pool_exec
+from common.token_utils import num_tokens_from_string
# ----- tunables ------------------------------------------------------
# The structure-compile batching / merge-flush / chain-correction tunables
# and the non-tree compilation core moved to
# ``rag.advanced_rag.knowlege_compile.runner`` so the ``rag.flow`` Compiler
# component can share them. Re-exported here for backwards compatibility.
-from rag.advanced_rag.knowlege_compile.runner import ( # noqa: E402
+from rag.advanced_rag.knowlege_compile.runner import (
DOC_STRUCTURE_COMPILE_BATCH_CHUNKS,
DOC_STRUCTURE_MERGE_MAX_DOCS, # noqa: F401
STRUCTURE_CHAIN_CORRECTION_TIMEOUT_S, # noqa: F401
load_active_templates,
run_structure_compile_over_batches,
)
+from rag.nlp import search
+
# ----- parser_config helpers -----------------------------------------
@@ -467,7 +466,7 @@ def cap_done_progress(progress_cb: Callable) -> Callable:
# ----- tree helpers --------------------------------------------------
-def raptor_tree_to_graph(tree: Dict) -> Dict:
+def raptor_tree_to_graph(tree: dict) -> dict:
"""Project a RAPTOR tree dict (from ``Raptor(is_tree=True)``) onto
the ``{entities, relations}`` shape the document-structure graph
endpoint already serves for ``page_index``-kind rows."""
@@ -507,7 +506,7 @@ def raptor_tree_to_graph(tree: Dict) -> Dict:
tree = _collapse_unary(tree) if isinstance(tree, dict) else tree
- def _walk(node: dict, parent_id: Optional[str]) -> None:
+ def _walk(node: dict, parent_id: str | None) -> None:
if not isinstance(node, dict):
return
title = node.get("title") or ""
@@ -534,7 +533,7 @@ def raptor_tree_to_graph(tree: Dict) -> Dict:
return {"entities": entities, "relations": relations}
-async def rewrite_duplicate_tree_names(tree: Dict, chat_mdl) -> None:
+async def rewrite_duplicate_tree_names(tree: dict, chat_mdl) -> None:
"""Rewrite only duplicate tree titles whose descriptions differ."""
from rag.advanced_rag.knowlege_compile._common import knowledge_compile_gen_conf
from rag.prompts.generator import gen_json
@@ -625,7 +624,7 @@ async def load_chunks_with_vec(
order_by.asc("page_num_int")
order_by.asc("top_int")
- out: list[tuple[str, "np.ndarray", str]] = []
+ out: list[tuple[str, np.ndarray, str]] = []
offset = 0
PAGE = 500
while True:
@@ -687,6 +686,7 @@ async def rechunk_doc_by_tree(
via ``available_int=0`` and stamped with ``superseded_by_chunk_id``.
"""
from datetime import datetime
+
from common.misc_utils import get_uuid
ctx = handler._task_context
@@ -899,8 +899,8 @@ async def run_tree_templates(
doc. Each pair runs RAPTOR with ``is_tree=True`` via
``RaptorService.build_doc_tree`` and persists a single graph row
via ``_struct_upsert_graph_json``."""
- from rag.svr.task_executor_refactor.raptor_service import RaptorService
from rag.advanced_rag.knowlege_compile.structure import _struct_upsert_graph_json
+ from rag.svr.task_executor_refactor.raptor_service import RaptorService
ctx = handler._task_context
progress_cb = ctx.progress_cb
@@ -997,19 +997,30 @@ async def run_tree_templates(
)
continue
+ # Persist the per-doc nav_doc right after the graph node, so parsing a
+ # file yields a nav_doc with the FULL entity descriptions as
+ # graph_content -- without needing to run GENERATE NAVIGATION separately
+ # and without changing the nav clustering input. The `title` keeps using
+ # tree["title"] (what upsert_dataset_nav_doc used to derive from `tree`
+ # before this change), so the parse cluster-title logic is unchanged.
+ # Only do this when the graph actually contains entities, otherwise skip
+ # (avoid empty graph_content when RAPTOR produced nothing).
try:
- from rag.advanced_rag.knowlege_compile.dataset_nav import (
- upsert_dataset_nav_doc,
- )
+ if graph.get("entities"):
+ from rag.advanced_rag.knowlege_compile.dataset_nav import (
+ build_nav_graph_text,
+ upsert_dataset_nav_doc,
+ )
- await upsert_dataset_nav_doc(
- ctx.tenant_id,
- ctx.kb_id,
- doc_id,
- tree,
- embd_mdl=embedding_model,
- chat_mdl=chat_mdl_by_tid[template_id],
- )
+ _, nav_graph_text = build_nav_graph_text(graph)
+ await upsert_dataset_nav_doc(
+ ctx.tenant_id,
+ ctx.kb_id,
+ doc_id,
+ {"title": tree.get("title"), "graph_text": nav_graph_text},
+ embd_mdl=embedding_model,
+ chat_mdl=chat_mdl_by_tid[template_id],
+ )
except Exception:
logging.exception(
"tree-template %s: dataset_nav upsert failed for doc %s",