Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Adjust navigation tools for agentic search (#19172)

Yingfenga11c27e01d10
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index 9f2ba094bd..ffc6270486 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -15,6 +15,7 @@
#
import json
import logging
+import math
import os
import re
@@ -3818,6 +3819,81 @@ _LAYERS_HANDLERS: dict[str, str] = {
"all": "_search_layers_all",
}
+# Document router behind the "navigation_tree" mode. Callers always ask for
+# mode="navigation_tree", so swapping strategy needs no change on their side.
+#
+# "nav_doc" (default) — flat hybrid search over the nav_doc rows. No cluster
+# level is consulted, so nothing is pruned, but the comparison still runs
+# on document-level embeddings. Those rows embed ``graph_text`` — the
+# document's full entity/relation listing — not a one-line summary, so
+# this is not the "compressed representation first" case that SCT measures.
+# "chunk_agg" — PageIndex ``semantics`` recipe: hybrid search over the raw
+# chunk index, hits rolled up per document with
+# DocScore = sum(chunk scores) / sqrt(hits + 1). Sound in principle, but
+# chunk-level similarity is easy to clear: a document passes the 0.2 floor
+# on a single marginally related chunk, so the top-k roster fills with
+# weakly related documents, and the wider routing fans out every later
+# stage. Measured 2026-09-02: 12/12 questions routed a full 12 documents,
+# evidence pool 42-67 passages, 166s per question against a 180s budget,
+# with several timeouts yielding no answer at all.
+# "compiled_agg" — page_index entity rows only; "fusion" — chunk + compiled.
+# "tree" — BFS beam descent over the nav cluster tree. This is what main
+# branch does. Kept as the A/B baseline; drop it, and
+# ``search_nav_tree_descent`` with it, once a winner is settled.
+_NAV_TREE_ROUTER = "chunk_agg"
+
+# Cap on how many documents a flat router hands back. The cluster beam descent
+# that main uses returns ~2 by construction (its per-level pruning keeps only
+# the most relevant branches), while the flat nav_doc sweep returns every row
+# above the 0.2 floor — measured 2026-09-02: beam avg 2.0 vs nav_doc avg 6.2.
+# Routing to 3x the documents made the RAGAgent carry 3x the evidence in every
+# round, so each dynamic LLM call grew 14k -> 18.8k tokens and per-question time
+# grew ~25%. Beam is not capped at 2 either (it occasionally routes 6 for
+# multi-hop), so allow a little headroom above its median rather than a hard 2.
+_NAV_DOC_FOCUS_LIMIT = 3
+
+# Candidate pool for "chunk_agg". DocScore only means something when a document
+# can collect several hits, so the pool has to be wide enough for that; across a
+# 2400-document corpus a pool of a few dozen chunks yields almost entirely
+# single-hit documents, which collapses DocScore into plain best-chunk ranking.
+# This is ``rerank_candidates_count`` — the pool the fusion actually draws from.
+# ``page_size`` is deliberately the same value so the whole pool is returned for
+# aggregation, and it must stay <= the pool or ``retrieval`` raises
+# (page * page_size > rerank_candidates_count).
+_NAV_CHUNK_AGG_POOL = 256
+_NAV_CHUNK_AGG_VEC_WEIGHT = 0.3
+
+# Candidate pool of compiled entity rows for the "compiled_agg" / "fusion"
+# routers. ``get_vector`` builds its own MatchDenseExpr and never routes through
+# ``retrieval``, so ``rerank_candidates_count`` does not apply here — but HNSW
+# needs ``num_candidates`` >= the pool or the search degenerates.
+_NAV_COMPILED_POOL = 256
+_NAV_COMPILED_SIMILARITY = 0.1
+# Entity types aggregated separately. A document carries far more facts than
+# titles, so pooling them would let fact hits drown out title hits; splitting
+# keeps the coarser title signal usable. ``entity_type_kwd`` does not exist on
+# compilation rows (it belongs to graphrag), so the type is read from the
+# payload after the fetch.
+_NAV_COMPILED_TITLE_TYPES = ("title",)
+_NAV_COMPILED_FACT_TYPES = ("fact", "conclusion")
+# Weights applied when the fusion router combines the three legs. Only used
+# when documents are hit by several legs.
+_NAV_FUSION_WEIGHTS = {"chunk": 1.0, "title": 0.8, "fact": 0.9}
+
+# Section bridge (stage 3): resolve each hit fact to its owning title. The
+# relation type carrying TOC containment in the page_index template, plus caps
+# on how many names are resolved and how many relation rows may be scanned.
+_NAV_INCLUDE_RELATION = "include"
+_NAV_BRIDGE_NAMES_LIMIT = 64
+_NAV_BRIDGE_FANOUT = 4
+# Compiled products (tree/entity/relation rows) are written with
+# available_int=1, so they fall inside the chunk filter and would otherwise
+# inflate a document's hit count. The project convention is that plain
+# retrieval reads document chunks only and compiled products are served by their
+# own tools, so exclude them here too. Flip to False to A/B letting compiled
+# rows act as extra per-document entries in the aggregation.
+_NAV_CHUNK_AGG_EXCLUDE_COMPILED = True
+
async def search_dataset_layers(
dataset_id: str,
@@ -3835,7 +3911,8 @@ async def search_dataset_layers(
- chunk: raw document chunks (via the main retrieval pipeline)
- nav_doc: navigation tree document leaves
- nav_cluster: navigation tree cluster nodes
- - navigation_tree: tree-structured BFS beam descent
+ - navigation_tree: document routing, strategy chosen by
+ ``_NAV_TREE_ROUTER`` (chunk aggregation by default)
- all: union of all modes, deduplicated by doc_id with best score
doc_scope: Optional set of documents to restrict the search to. None or
empty means all documents of the dataset. Forwarded to every mode:
@@ -3883,7 +3960,7 @@ async def search_dataset_layers(
elif mode == "nav_cluster":
return await _search_layers_nav_clusters(tenant_id, dataset_id, query, top_k, embd_mdl, search_dataset_nav, doc_scope=doc_scope)
elif mode == "navigation_tree":
- return await _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, doc_scope=doc_scope)
+ return await _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)
elif mode == "chunk":
return await _search_layers_chunks(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)
elif mode == "all":
@@ -3920,20 +3997,608 @@ async def _search_layers_nav_clusters(tenant_id, dataset_id, query, top_k, embd_
return True, {"mode": "nav_cluster", "total": len(items), "items": items}
-async def _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, *, doc_scope=None):
- from rag.advanced_rag.knowlege_compile.dataset_nav import search_nav_tree_descent
+def _nav_focus_items(items: list) -> list:
+ """Truncate a flat routing to the most relevant documents, in place.
- items = await search_nav_tree_descent(
- tenant_id,
- dataset_id,
- query,
- embd_mdl,
- top_k=top_k,
- doc_scope=doc_scope,
- )
+ The flat sweep returns every row above the floor; the cluster descent main
+ uses stays focused by its per-level pruning. See ``_NAV_DOC_FOCUS_LIMIT``
+ for the measured reason this matters.
+ """
+ limit = _NAV_DOC_FOCUS_LIMIT
+ if not limit or len(items) <= limit:
+ return items
+ items.sort(key=lambda it: float(it.get("score", 0.0)), reverse=True)
+ del items[limit:]
+ return items
+
+
+def _nav_label_items(items: list) -> list:
+ """Give each routed item the ``_nav`` summary navigation.py reads.
+
+ ``search_dataset_nav`` returns the leaf row flat (``name``/``description`` at
+ the top level), but navigation.py reads the route label from
+ ``item["_nav"]["description"]``, so without this a route comes back
+ unlabelled. The rows already carry the summary, so this costs no extra
+ query — unlike the chunk router, which has to look one up.
+ """
+ for item in items:
+ if not isinstance(item, dict) or item.get("_nav"):
+ continue
+ item["_nav"] = {
+ "doc_id": str(item.get("doc_id") or "").strip(),
+ "description": str(item.get("description") or item.get("name") or "").strip(),
+ }
+ return items
+
+
+async def _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, kb=None, *, doc_scope=None):
+ """Route to documents, dispatched by ``_NAV_TREE_ROUTER``.
+
+ Every strategy returns the same item shape — ``doc_id``, a 0..1 ``score``,
+ and ``_nav`` carrying the document summary — so callers stay agnostic to
+ which one is active.
+ """
+ if _NAV_TREE_ROUTER == "nav_doc":
+ from rag.advanced_rag.knowlege_compile.dataset_nav import search_dataset_nav
+
+ items = await _nav_search_result(
+ tenant_id,
+ dataset_id,
+ query,
+ top_k,
+ embd_mdl,
+ search_dataset_nav,
+ type_kwd="nav_doc",
+ doc_scope=doc_scope,
+ )
+ _nav_focus_items(items)
+ _nav_label_items(items)
+ return True, {"mode": "navigation_tree", "total": len(items), "items": items}
+
+ if _NAV_TREE_ROUTER == "tree":
+ from rag.advanced_rag.knowlege_compile.dataset_nav import search_nav_tree_descent
+
+ items = await search_nav_tree_descent(
+ tenant_id,
+ dataset_id,
+ query,
+ embd_mdl,
+ top_k=top_k,
+ doc_scope=doc_scope,
+ )
+ return True, {"mode": "navigation_tree", "total": len(items), "items": items}
+
+ if _NAV_TREE_ROUTER in ("compiled_agg", "fusion"):
+ ok, payload = await _search_layers_compiled_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)
+ if _NAV_TREE_ROUTER == "compiled_agg":
+ return ok, payload
+ # Fusion keeps going: the compiled leg ranks alongside the chunk leg.
+ return await _search_layers_fusion(tenant_id, dataset_id, query, top_k, embd_mdl, kb, compiled=payload, doc_scope=doc_scope)
+
+ return await _search_layers_chunk_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)
+
+
+async def _search_layers_chunk_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb=None, *, doc_scope=None):
+ """Route to documents by rolling up raw-chunk hybrid hits (PageIndex style).
+
+ Retrieval runs over the chunk index — uncompressed text — and the hits are
+ aggregated per document with PageIndex's DocScore::
+
+ DocScore = sum(chunk scores) / sqrt(hits + 1)
+
+ The numerator rewards a document that matches several chunks; the
+ square-root denominator damps that reward so a long document cannot win on
+ volume alone.
+
+ Ordering uses DocScore, but the reported ``score`` is the document's best
+ chunk similarity: callers threshold on it (``_NAV_MIN_DOC_SCORE``), and
+ keeping a single chunk's similarity preserves the 0..1 scale those
+ thresholds were set against.
+ """
+ from common import settings
+
+ kwargs = {}
+ if doc_scope:
+ kwargs["doc_ids"] = [str(d) for d in doc_scope if str(d).strip()]
+
+ pool = _NAV_CHUNK_AGG_POOL
+ # Term-only and dense scores live on different scales, so the weight (not the
+ # threshold) carries the "is there an embedding model" decision.
+ vector_weight = _NAV_CHUNK_AGG_VEC_WEIGHT if embd_mdl else 0
+ try:
+ ranks = await settings.retriever.retrieval(
+ query,
+ embd_mdl,
+ [tenant_id],
+ [dataset_id],
+ 1,
+ # page_size == the pool, so every fused candidate reaches the
+ # aggregation below.
+ pool,
+ # No similarity floor: ranking is done by DocScore, so the pool must
+ # stay wide enough for a document to collect several hits. A floor
+ # here would pre-empt the aggregation.
+ 0.0,
+ vector_weight,
+ # knn_top_k is left at its default: it caps how many neighbours the
+ # vector leg feeds into fusion, and narrowing it to top_k would
+ # shrink the pool this strategy depends on.
+ rerank_candidates_count=pool,
+ must_not={"exists": "compile_kwd"} if _NAV_CHUNK_AGG_EXCLUDE_COMPILED else None,
+ **kwargs,
+ )
+ except Exception:
+ logging.exception("search_dataset_layers: chunk-agg retrieval failed for kb=%s", dataset_id)
+ return False, {"error": "chunk retrieval failed", "code": RetCode.SERVER_ERROR}
+
+ agg = _nav_aggregate_chunks(ranks.get("chunks", []))
+ if not agg:
+ return True, {"mode": "navigation_tree", "total": 0, "items": []}
+
+ ranked = sorted(agg.items(), key=lambda kv: _nav_doc_score(kv[1]), reverse=True)
+ # Cap by the caller's top_k and by the focus limit. Without the focus cap
+ # this router returned every document above the floor (measured 6-12),
+ # routing the RAGAgent into 3x the evidence and inflating every dynamic LLM
+ # call — the same regression the nav_doc router had, fixed here the same way.
+ limit = _NAV_DOC_FOCUS_LIMIT or (top_k or _NAV_CHUNK_AGG_POOL)
+ if limit > 0:
+ ranked = ranked[:limit]
+
+ summaries = await _nav_doc_summaries(kb, [doc_id for doc_id, _ in ranked])
+ items = [
+ {
+ "doc_id": doc_id,
+ "score": round(entry["best"], 4),
+ "_nav": {"doc_id": doc_id, "description": summaries.get(doc_id, "")},
+ # Aggregation detail, surfaced for tuning the fetch multiplier.
+ "_agg": {"doc_score": round(_nav_doc_score(entry), 4), "hits": entry["hits"]},
+ }
+ for doc_id, entry in ranked
+ ]
return True, {"mode": "navigation_tree", "total": len(items), "items": items}
+async def _search_layers_compiled_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb=None, *, doc_scope=None):
+ """Route to documents through compiled ``page_index`` entity rows.
+
+ Complements the chunk leg rather than replacing it: chunks are verbatim text
+ and win on exact matches (numbers, proper nouns), while ``fact`` /
+ ``conclusion`` rows are atomic propositions that win on concept matches the
+ surface form misses. Each covers the other's failure mode.
+
+ Compiled rows carry no ``available_int``, which is the project's marker for
+ "visible to the generic retriever" (not a deletion flag), so they are absent
+ from the chunk index and have to be read through the store seam directly --
+ the same seam ``dataset_nav`` and wiki use.
+
+ Returns ``ok=False`` only on a store failure; an empty result set is a
+ legitimate outcome (a KB without page_index products) and is reported
+ ``ok=True`` with ``total=0`` so the caller can fall back.
+ """
+ pack = _compiled_index_or_none(kb.tenant_id, kb.id) if kb is not None else None
+ if pack is None:
+ return True, {"mode": "navigation_tree", "total": 0, "items": []}
+ from common.doc_store.doc_store_base import OrderByExpr
+ from rag.nlp.search import MatchTextExpr
+
+ index_nm, _ = pack
+
+ pool = _NAV_COMPILED_POOL
+ condition = {
+ "compile_kwd": ["page_index"],
+ "knowledge_graph_kwd": ["entity"],
+ # Exclude the KB-wide merged rows written by the Build button.
+ "scope_kwd": ["doc"],
+ }
+ if doc_scope:
+ condition["doc_id"] = [str(d) for d in doc_scope if str(d).strip()]
+
+ fields = ["content_with_weight", "source_chunk_ids", "doc_id", "name_kwd"]
+ try:
+ # Compiled rows tokenize their description into content_ltks, so a
+ # keyword match is a real fallback when no embedding model is configured
+ # or embedding fails -- the leg degrades instead of dropping out.
+ exprs = []
+ if embd_mdl:
+ try:
+ exprs.append(
+ await settings.retriever.get_vector(
+ query,
+ embd_mdl,
+ top_k=pool,
+ # HNSW ef_search -- must be >= top_k or the ANN search
+ # collapses. Positional passing here would hit
+ # num_candidates with the similarity threshold.
+ num_candidates=pool,
+ similarity=_NAV_COMPILED_SIMILARITY,
+ )
+ )
+ except Exception:
+ logging.exception("dataset_nav: compiled vector build failed for kb=%s", kb.id)
+ if not exprs:
+ exprs.append(MatchTextExpr(["content_ltks", "content_sm_ltks"], query, pool))
+ res = await thread_pool_exec(
+ settings.docStoreConn.search,
+ select_fields=fields,
+ highlight_fields=[],
+ condition=condition,
+ match_expressions=exprs,
+ order_by=OrderByExpr(),
+ offset=0,
+ limit=pool,
+ index_names=index_nm,
+ knowledgebase_ids=[kb.id],
+ )
+ field_map = settings.docStoreConn.get_fields(res, fields)
+ except Exception:
+ logging.exception("dataset_nav: compiled-agg retrieval failed for kb=%s", kb.id)
+ return False, {"error": "compiled retrieval failed", "code": RetCode.SERVER_ERROR}
+
+ buckets = _nav_bucket_compiled_rows(field_map or {})
+ ranked = _nav_rank_compiled_buckets(buckets, top_k)
+ if not ranked:
+ return True, {"mode": "navigation_tree", "total": 0, "items": []}
+
+ # Compile rows can outlive a deleted document -- the doc-existence check in
+ # retrieval() is a fallback, not the primary delete mechanism, and this leg
+ # reads the store directly, so it has to apply the check itself.
+ alive = await _nav_existing_doc_ids([doc_id for doc_id, _ in ranked])
+ ranked = [(d, e) for d, e in ranked if d in alive]
+ if not ranked:
+ return True, {"mode": "navigation_tree", "total": 0, "items": []}
+
+ summaries = await _nav_doc_summaries(kb, [doc_id for doc_id, _ in ranked])
+ # Label each document with the TOC sections that scored for it, so the caller
+ # gets an entry point into the document instead of starting
+ # navigate_structure from scratch. Purely additive: it never prunes.
+ fact_names_by_doc = buckets.get("fact_names") or {}
+ bridge = await _nav_bridge_sections(kb, [n for names in fact_names_by_doc.values() for n in names], doc_scope=doc_scope)
+ items = [
+ {
+ "doc_id": doc_id,
+ "score": round(entry["best"], 4),
+ "_nav": {
+ "doc_id": doc_id,
+ "description": summaries.get(doc_id, ""),
+ "matched_sections": _nav_matched_sections(bridge, fact_names_by_doc.get(doc_id) or []),
+ },
+ # Report the value ranking actually used. Recomputing DocScore here
+ # would mix the title and fact hit counts and disagree with ordering.
+ "_agg": {"fused": round(entry["score"], 4), "hits": entry["hits"], "legs": sorted(entry["legs"])},
+ }
+ for doc_id, entry in ranked
+ ]
+ return True, {"mode": "navigation_tree", "total": len(items), "items": items}
+
+
+async def _search_layers_fusion(tenant_id, dataset_id, query, top_k, embd_mdl, kb=None, *, compiled=None, doc_scope=None):
+ """Combine the chunk leg and the compiled leg into one routing decision.
+
+ The two legs measure different things and are not directly comparable: chunk
+ ``_score`` is Infinity's normalized weighted sum, while the compiled leg is a
+ raw cosine. Each leg is therefore normalized by its own maximum before
+ being combined, so neither leg's scale can dominate the other.
+ """
+ ok, chunk_payload = await _search_layers_chunk_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)
+ if not ok:
+ return ok, chunk_payload
+
+ compiled_items = (compiled or {}).get("items") or []
+ if not compiled_items:
+ # No page_index products (or a raptor KB): the chunk leg alone decides.
+ # Tag the items so callers see one shape whichever path produced them.
+ for item in chunk_payload.get("items") or []:
+ item.setdefault("_agg", {}).setdefault("legs", ["chunk"])
+ return ok, chunk_payload
+
+ legs = [("chunk", chunk_payload.get("items") or [], _NAV_FUSION_WEIGHTS["chunk"]), ("compiled", compiled_items, 1.0)]
+ fused = _nav_fuse_legs(legs, top_k)
+
+ summaries = await _nav_doc_summaries(kb, [doc_id for doc_id, _ in fused])
+ items = [
+ {
+ "doc_id": doc_id,
+ # Reported score stays on the 0..1 scale callers threshold on, so it
+ # is taken from the highest-scoring leg rather than the fused value.
+ "score": round(info["best"], 4),
+ "_nav": {"doc_id": doc_id, "description": summaries.get(doc_id, "")},
+ "_agg": {"fused": round(info["fused"], 4), "legs": sorted(info["legs"])},
+ }
+ for doc_id, info in fused
+ ]
+ return True, {"mode": "navigation_tree", "total": len(items), "items": items}
+
+
+def _nav_fuse_legs(legs: list[tuple[str, list, float]], top_k) -> list[tuple[str, dict]]:
+ """Fuse per-document leg rankings, each normalized by its own maximum."""
+ normalized = []
+ for name, items, weight in legs:
+ peak = max((float(i.get("score") or 0.0) for i in items), default=0.0)
+ if peak <= 0:
+ continue
+ normalized.append((name, [(i.get("doc_id"), float(i.get("score") or 0.0) / peak * weight) for i in items], weight))
+
+ merged: dict[str, dict] = {}
+ for name, entries, _ in normalized:
+ for doc_id, value in entries:
+ if not doc_id:
+ continue
+ info = merged.get(doc_id)
+ if info is None:
+ merged[doc_id] = {"fused": value, "best": value, "legs": {name}}
+ continue
+ info["fused"] += value
+ info["legs"].add(name)
+ if value > info["best"]:
+ info["best"] = value
+
+ ranked = sorted(merged.items(), key=lambda kv: kv[1]["fused"], reverse=True)
+ if top_k is not None and top_k > 0:
+ ranked = ranked[:top_k]
+ return ranked
+
+
+def _nav_bucket_compiled_rows(field_map: dict) -> dict[str, dict]:
+ """Split fetched compiled rows into a title bucket and a fact bucket.
+
+ The row ``type`` lives inside the ``content_with_weight`` payload —
+ ``entity_type_kwd`` is a graphrag field and is not written by the
+ compilation path — so rows are binned after the fetch.
+
+ Fact names are kept per document, because the section bridge has to label a
+ document with the sections that scored for *it* — a flat list would attach
+ one document's sections to another.
+ """
+ title_bucket: dict[str, dict] = {}
+ fact_bucket: dict[str, dict] = {}
+ fact_names: dict[str, list[str]] = {}
+ for row in field_map.values():
+ doc_id = str(row.get("doc_id") or "").strip()
+ if not doc_id:
+ continue
+ try:
+ payload = json.loads(str(row.get("content_with_weight") or "{}"))
+ except (TypeError, ValueError):
+ continue
+ if not isinstance(payload, dict):
+ continue
+ rtype = str(payload.get("type") or "").strip().lower()
+ if rtype in _NAV_COMPILED_TITLE_TYPES:
+ bucket = title_bucket
+ elif rtype in _NAV_COMPILED_FACT_TYPES:
+ bucket = fact_bucket
+ # Kept for the section bridge, which labels a hit fact with its
+ # owning title. The raw fetch order is the store's relevance order,
+ # which is the right priority for choosing which sections to show.
+ name = str(payload.get("name") or "").strip()
+ if name:
+ fact_names.setdefault(doc_id, []).append(name)
+ else:
+ continue
+ score = float(row.get("similarity") or row.get("_score") or 0.0)
+ entry = bucket.get(doc_id)
+ if entry is None:
+ bucket[doc_id] = {"total": score, "best": score, "hits": 1}
+ continue
+ entry["total"] += score
+ entry["hits"] += 1
+ if score > entry["best"]:
+ entry["best"] = score
+ return {"title": title_bucket, "fact": fact_bucket, "fact_names": fact_names}
+
+
+def _nav_rank_compiled_buckets(buckets: dict, top_k) -> list[tuple[str, dict]]:
+ """Fuse the title and fact buckets into one per-document ranking.
+
+ A document hit by both legs is stronger than one hit by either alone, so the
+ scores are summed rather than maxed — but each bucket is ranked by its own
+ DocScore first, which is what keeps facts from swamping titles.
+ """
+ merged: dict[str, dict] = {}
+ for leg, bucket in (("title", buckets.get("title") or {}), ("fact", buckets.get("fact") or {})):
+ weight = _NAV_FUSION_WEIGHTS.get(leg, 1.0)
+ for doc_id, entry in bucket.items():
+ contribution = _nav_doc_score(entry) * weight
+ cur = merged.get(doc_id)
+ if cur is None:
+ merged[doc_id] = {
+ "score": contribution,
+ "best": entry["best"],
+ "total": entry["total"],
+ "hits": entry["hits"],
+ "legs": {leg},
+ }
+ continue
+ cur["score"] += contribution
+ cur["total"] += entry["total"]
+ cur["hits"] += entry["hits"]
+ cur["legs"].add(leg)
+ if entry["best"] > cur["best"]:
+ cur["best"] = entry["best"]
+
+ ranked = sorted(merged.items(), key=lambda kv: kv[1]["score"], reverse=True)
+ if top_k is not None and top_k > 0:
+ ranked = ranked[:top_k]
+ return ranked
+
+
+def _nav_matched_sections(bridge: dict[str, str], fact_names: list[str]) -> list[str]:
+ """Titles owning this document's hit facts, in hit order and de-duplicated."""
+ sections: list[str] = []
+ for name in fact_names:
+ title = bridge.get(str(name).strip().lower())
+ if title and title not in sections:
+ sections.append(title)
+ return sections
+
+
+async def _nav_bridge_sections(kb, fact_names: list[str], doc_scope=None) -> dict[str, str]:
+ """Map each hit fact to the title that includes it.
+
+ Routing then answers *which document* and *where in it to look first*:
+ the caller gets a per-document entry point instead of starting
+ ``navigate_structure`` from scratch.
+
+ The TOC is used here purely as a **label**. It never prunes the candidate
+ set — that is the top-down routing failure mode. The lookup happens after
+ the documents are already chosen.
+
+ Returns ``{fact_name: title_name}``; facts without an owning title are
+ omitted.
+ """
+ if not fact_names or kb is None:
+ return {}
+ pack = _compiled_index_or_none(kb.tenant_id, kb.id)
+ if pack is None:
+ return {}
+ index_nm, _ = pack
+
+ from common.doc_store.doc_store_base import OrderByExpr
+
+ names = [str(n).strip().lower() for n in fact_names if str(n).strip()]
+ # Cap the terms query: a wide IN list is slow and the bridge only needs to
+ # label the top hits, not every candidate.
+ names = list(dict.fromkeys(names))[:_NAV_BRIDGE_NAMES_LIMIT]
+
+ condition = {
+ "compile_kwd": ["page_index"],
+ "knowledge_graph_kwd": ["relation"],
+ "scope_kwd": ["doc"],
+ "to_entity_kwd": names,
+ }
+ if doc_scope:
+ condition["doc_id"] = [str(d) for d in doc_scope if str(d).strip()]
+
+ fields = ["content_with_weight", "from_entity_kwd", "to_entity_kwd", "doc_id"]
+ try:
+ res = await thread_pool_exec(
+ settings.docStoreConn.search,
+ select_fields=fields,
+ highlight_fields=[],
+ condition=condition,
+ match_expressions=[],
+ order_by=OrderByExpr(),
+ offset=0,
+ limit=len(names) * _NAV_BRIDGE_FANOUT,
+ index_names=index_nm,
+ knowledgebase_ids=[kb.id],
+ )
+ field_map = settings.docStoreConn.get_fields(res, fields)
+ except Exception:
+ logging.exception("dataset_nav: section bridge lookup failed for kb=%s", kb.id)
+ return {}
+
+ bridge: dict[str, str] = {}
+ for row in (field_map or {}).values():
+ to_name = str(row.get("to_entity_kwd") or "").strip().lower()
+ title = str(row.get("from_entity_kwd") or "").strip()
+ if not to_name or not title or to_name in bridge:
+ continue
+ try:
+ payload = json.loads(str(row.get("content_with_weight") or "{}"))
+ except (TypeError, ValueError):
+ continue
+ # Only the structural edge counts; semantic relations would mislabel.
+ if isinstance(payload, dict) and str(payload.get("type") or "").strip().lower() == _NAV_INCLUDE_RELATION:
+ bridge[to_name] = title
+ return bridge
+
+
+async def _nav_existing_doc_ids(doc_ids: list[str]) -> set[str]:
+ """Keep only documents that still exist.
+
+ Compiled rows are not necessarily cleaned up when a document is deleted, so
+ this leg has to repeat the existence check that ``retrieval`` applies to
+ chunk results.
+ """
+ if not doc_ids:
+ return set()
+ try:
+ return await settings.retriever._existing_doc_ids(list(doc_ids))
+ except Exception:
+ logging.exception("dataset_nav: doc existence check failed")
+ return set(doc_ids)
+
+
+def _nav_aggregate_chunks(chunks: list) -> dict[str, dict]:
+ """Roll chunk hits up per document: total, best and hit count per doc."""
+ agg: dict[str, dict] = {}
+ for c in chunks:
+ doc_id = str(c.get("doc_id") or "").strip()
+ if not doc_id:
+ continue
+ score = float(c.get("similarity") or c.get("score") or 0.0)
+ entry = agg.get(doc_id)
+ if entry is None:
+ agg[doc_id] = {"total": score, "best": score, "hits": 1}
+ continue
+ entry["total"] += score
+ entry["hits"] += 1
+ if score > entry["best"]:
+ entry["best"] = score
+ return agg
+
+
+def _nav_doc_score(entry: dict) -> float:
+ """PageIndex DocScore: hit scores summed, damped by hit count.
+
+ The sum rewards a document matching several chunks; the square-root
+ denominator damps it so a long document cannot win on volume alone.
+ """
+ return entry["total"] / math.sqrt(entry["hits"] + 1)
+
+
+async def _nav_doc_summaries(kb, doc_ids):
+ """Batch-load the nav_doc ``description`` of ``doc_ids`` in one store query.
+
+ Chunk-level routing finds a document without reading any nav row, but the
+ caller still needs a label for it. The nav_doc row already holds the
+ document's overall summary, so read them in a single terms query rather
+ than making the caller load the document for a label.
+ """
+ if not doc_ids or kb is None:
+ return {}
+ pack = _compiled_index_or_none(kb.tenant_id, kb.id)
+ if pack is None:
+ return {}
+ index_nm, _ = pack
+
+ from common.doc_store.doc_store_base import OrderByExpr
+
+ # Single terms query shared by the whole batch; _nav_search would repeat the
+ # access check and KB load the caller already performed.
+ limit = max(1, min(len(doc_ids), 2000))
+ condition = {
+ "compile_kwd": [_NAV_COMPILE_KWD],
+ "type_kwd": ["nav_doc"],
+ "doc_id": [str(d) for d in doc_ids][:limit],
+ }
+ try:
+ res = await thread_pool_exec(
+ settings.docStoreConn.search,
+ select_fields=_NAV_FIELDS,
+ highlight_fields=[],
+ condition=condition,
+ match_expressions=[],
+ order_by=OrderByExpr(),
+ offset=0,
+ limit=limit,
+ index_names=index_nm,
+ knowledgebase_ids=[kb.id],
+ )
+ field_map = settings.docStoreConn.get_fields(res, _NAV_FIELDS)
+ except Exception:
+ logging.exception("dataset_nav: nav_doc summary lookup failed for kb=%s", kb.id)
+ return {}
+ summaries = {}
+ for row in (field_map or {}).values():
+ item = _nav_item(row)
+ doc_id = str(item.get("doc_id") or "").strip()
+ if doc_id:
+ summaries[doc_id] = str(item.get("description") or "").strip()
+ return summaries
+
+
async def _nav_search_result(tenant_id, dataset_id, query, top_k, embd_mdl, search_fn, **kwargs):
doc_scope = kwargs.pop("doc_scope", None)
results = await search_fn(
@@ -3979,11 +4644,14 @@ async def _search_layers_chunks(tenant_id, dataset_id, query, top_k, embd_mdl, k
tenant_ids = [tenant_id]
kwargs = {}
- if top_k is not None:
- kwargs["knn_top_k"] = top_k
if doc_scope:
kwargs["doc_ids"] = [str(d) for d in doc_scope if str(d).strip()]
+ # Here page_size IS the result set: chunks are returned as evidence, not
+ # aggregated into documents. Keep it at the historical fetch size — raising
+ # it to the chunk_agg pool inflated the evidence pool ~7x (25 -> 191 chunks
+ # per question), which pushed every downstream LLM prompt and blew the
+ # 180s per-question research budget.
fetch_k = max(top_k, 10) * 3 if top_k is not None else 1024
try:
ranks = await settings.retriever.retrieval(
@@ -3994,7 +4662,16 @@ async def _search_layers_chunks(tenant_id, dataset_id, query, top_k, embd_mdl, k
1,
fetch_k,
0.0,
- 0.3,
+ _NAV_CHUNK_AGG_VEC_WEIGHT,
+ # The candidate pool only has to cover what is returned; it used to
+ # sit at the 64 default while page_size grew with top_k, which
+ # raises once page * page_size exceeds it.
+ rerank_candidates_count=max(_NAV_CHUNK_AGG_POOL, fetch_k),
+ # knn_top_k is left at its default: narrowing it to top_k caps how
+ # many neighbours the vector leg feeds into fusion.
+ # Chunk mode reports raw chunks; compiled rows are served by their
+ # own tools, so they must not be attributed to a document here.
+ must_not={"exists": "compile_kwd"} if _NAV_CHUNK_AGG_EXCLUDE_COMPILED else None,
**kwargs,
)
except Exception:
@@ -4022,10 +4699,14 @@ async def _search_layers_all(tenant_id, dataset_id, query, top_k, embd_mdl, kb,
"""Run all modes and return the union of doc_ids, with best score per doc."""
import asyncio as _asyncio
+ # With _NAV_TREE_ROUTER="chunk_agg" the navigation_tree leg already reads the
+ # chunk index, so it overlaps the chunk leg below. The union still dedups by
+ # doc_id and keeps the best score, so the overlap costs one extra retrieval
+ # without affecting the result — mode="all" is for comparison, not production.
result_lists = await _asyncio.gather(
_search_layers_nav_docs(tenant_id, dataset_id, query, top_k, embd_mdl, search_fn, doc_scope=doc_scope),
_search_layers_nav_clusters(tenant_id, dataset_id, query, top_k, embd_mdl, search_fn, doc_scope=doc_scope),
- _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, doc_scope=doc_scope),
+ _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope),
_search_layers_chunks(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope),
return_exceptions=True,
)
diff --git a/rag/advanced_rag/harness/tools/compiled_expansion.py b/rag/advanced_rag/harness/tools/compiled_expansion.py
index f4f1bbd9f3..e252fc8bb7 100644
--- a/rag/advanced_rag/harness/tools/compiled_expansion.py
+++ b/rag/advanced_rag/harness/tools/compiled_expansion.py
@@ -22,6 +22,31 @@ from rag.advanced_rag.harness.tools.navigation import _kg_scopes
_LOG = logging.getLogger(__name__)
+# HNSW ef_search floor. Must be >= the requested top_n or the ANN search is
+# bounded by the candidate list instead of by relevance.
+_VECTOR_NUM_CANDIDATES = 256
+_VECTOR_SIMILARITY = 0.1
+
+
+async def _dense_expr(text: str, embd_mdl, top_n: int):
+ """Build the dense match expression for compiled-row search.
+
+ ``get_vector``'s 4th positional argument is ``num_candidates`` (HNSW
+ ef_search), not the similarity threshold — passing the 0.1 threshold
+ positionally collapses the candidate list to a fraction of one entry and
+ silently destroys recall. Both are passed by name here.
+ """
+ from common import settings
+
+ return await settings.retriever.get_vector(
+ text,
+ embd_mdl,
+ top_k=top_n,
+ num_candidates=max(top_n, _VECTOR_NUM_CANDIDATES),
+ similarity=_VECTOR_SIMILARITY,
+ )
+
+
# ─── Compiled product expansion (zero-LLM, used by hybrid_search with use_compiled=True) ───
@@ -156,7 +181,7 @@ async def _search_compiled_rows(
embd_mdl = getattr(tools, "embed_mdl", None)
if embd_mdl:
try:
- exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_n, 0.1))
+ exprs.append(await _dense_expr(text, embd_mdl, top_n))
except Exception:
_LOG.exception("[Compiled expand] vector build failed; using keyword match")
if not exprs:
@@ -400,7 +425,7 @@ async def _search_synthesis_pages(
embd_mdl = getattr(tools, "embed_mdl", None)
if embd_mdl:
try:
- exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_n, 0.1))
+ exprs.append(await _dense_expr(text, embd_mdl, top_n))
except Exception:
_LOG.exception("[Wiki expand] vector build failed; using keyword match")
if not exprs:
diff --git a/rag/advanced_rag/harness/tools/navigation.py b/rag/advanced_rag/harness/tools/navigation.py
index b6dddf36e9..cc44e3e3bc 100644
--- a/rag/advanced_rag/harness/tools/navigation.py
+++ b/rag/advanced_rag/harness/tools/navigation.py
@@ -659,14 +659,20 @@ _NAV_MIN_DOC_SCORE = 0.2 # drop docs below this score
async def _nav_search_titled(tools, topic: str, keywords: str = "", doc_scope: list[str] | None = None) -> list[tuple[str, str]]:
"""Descend the compiled nav TREE and return ``(doc_id, summary)`` pairs.
- Uses ``mode="navigation_tree"`` (BFS beam descent: root cluster → sub-clusters
- → nav_doc leaves) rather than a flat sweep over every nav_doc row. Routing
- through the hierarchy narrows the field level by level, which is what makes it
- accurate on large datasets where many document summaries look alike in a flat
- comparison.
-
- HYBRID descent (vector + BM25): both the KNN legs and the ``_text_score``
- gates stay armed. A pure-vector descent was tried — it dropped the text legs
+ Uses ``mode="navigation_tree"``, which currently resolves to a FLAT hybrid
+ sweep over every ``nav_doc`` row (``_NAV_TREE_ROUTER = "nav_doc"``) rather
+ than a descent through the cluster levels. The strategy lives server-side, so
+ this function is agnostic to which one is active.
+
+ Flat routing is deliberate. A BFS beam descent narrows the field level by
+ level, and a wrong pick at the first level cannot be undone — the correct
+ branch was already pruned and never reaches the comparison. Nav rows embed
+ ``graph_text`` (the document's full entity/relation listing), so they are
+ discriminating enough to be compared directly without the cluster levels
+ narrowing them for us.
+
+ HYBRID sweep (vector + BM25): both the KNN legs and the ``_text_score``
+ gates stay armed. A pure-vector routing was tried — it dropped the text legs
and the ``_text_score > 0`` gate, on the theory that routing cares about
semantic matches a summary never spells out. It regressed routing badly:
scores collapsed onto the dense leg alone, the descent fell below
@@ -742,9 +748,10 @@ async def dataset_navigation_search(tools, topic: str, keywords: str = "", doc_s
descending the dataset's compiled navigation tree.
Routes through :func:`_nav_search_titled`, which runs
- ``search_dataset_layers`` with ``mode="navigation_tree"``: a hybrid BFS beam
- descent (vector + BM25) from the root clusters down to the ``nav_doc``
- leaves, with the ``_text_score`` gates armed.
+ ``search_dataset_layers`` with ``mode="navigation_tree"``: a hybrid
+ (vector + BM25) sweep over the ``nav_doc`` rows, with the ``_text_score``
+ gates armed. Which router runs is decided server-side by
+ ``_NAV_TREE_ROUTER``, so this function does not depend on the strategy.
Returns the routed ``doc_id`` list (capped at ``_NAV_SEARCH_MAX_DOCS``), or
``[]`` when no question/keywords are given or the search returns nothing.
@@ -963,8 +970,12 @@ async def _navigate_structure_impl(
structure of the requested kind, an empty <doc/> is returned (no error) — fall
back to list_chunks / search_chunks.
- This tool performs NO chat-LLM calls: it only reads the compiled structure
- rows from the doc store.
+ Cost: the structure rows themselves are read from the doc store. When the
+ document's nodes carry their own embeddings (page_index), the whole TOC is
+ additionally handed to the chat model once, so it can pick the relevant
+ sections — see ``_STRUCT_TOC_LLM_SELECT``. That pass is a single call per
+ document, and an empty or failed result falls through to local scoring, so
+ the tool still works without a chat model.
:param query: REQUIRED: the question/topic (used to route to a document when
``doc_id`` is not given).
@@ -1056,6 +1067,33 @@ _STRUCT_VEC_BEAM_RATIO = 0.5 # vector beam: drop nodes below best*this similari
_STRUCT_MAX_NODES = 10 # cap on nodes rendered in the outline.
_STRUCT_MAX_CHUNKS = 4 # cap on chunk snippets returned in the outline.
_STRUCT_DESC_SNIPPET = 180 # cap on a node's description snippet in the outline.
+# Chunks recalled per document when a tree cannot score its own nodes and the
+# chunk index has to choose instead. Wide enough that several nodes can be
+# covered, so the outline is not pinned to a single branch.
+_STRUCT_RECALL_TOP_N = 24
+# Snippets rendered on that path. Higher than _STRUCT_MAX_CHUNKS because there
+# the chunks *are* the answer's material — the outline is only the annotation.
+_STRUCT_MAX_CHUNK_HITS = 8
+# Document-in-LLM selection (PageIndex tree-search / VecTree-RAG Layer 1): the
+# whole TOC is handed to the model at once and it returns every node it thinks
+# is relevant. This is deliberately NOT a top-down descent — the model sees all
+# levels simultaneously, so a wrong pick cannot prune a branch, which is the
+# failure mode beam descent has (SCT measures it as routing accuracy 20.2% vs
+# 39.3% for flat selection).
+# OFF by default. Correct in principle — the model sees every level at once, so
+# nothing is pruned — but it costs one chat call per document, and the drill rung
+# calls navigate_structure once per routed document (8 docs = 8 extra calls per
+# question). Measured 2026-09-02: with it on, per-question LLM calls went 9-17
+# -> 37 and the 180s research budget was exhausted on 15 of 16 questions
+# (baseline: 1 of 28). Turn it on only when the LLM has headroom; prefer
+# restricting it to the top-ranked document first.
+_STRUCT_TOC_LLM_SELECT = False
+# Budget for the serialized TOC. VecTree-RAG quotes 200-500 tokens per document
+# for section summaries; facts are included here too, so this allows more.
+# _ask_nav_select truncates past this via message_fit_in regardless.
+_STRUCT_TOC_MAX_NODES = 120
+_STRUCT_TOC_DESC_SNIPPET = 120
+_STRUCT_TOC_MAX_DEPTH = 6
# Snippet length / per-doc cap for the related chunks appended after a
# search_chunks hit (from the document's compiled structure, not a full read).
_STRUCT_RELATED_SNIPPET_CHARS = 300
@@ -1207,7 +1245,24 @@ async def _read_structures(tools_slot, query: str, doc_ids: list[str], kinds: se
relations = structure.get("relations") or []
except Exception:
_LOG.exception("[navigate_structure] structure load failed for doc=%s", doc_id)
- outline, stats, chunk_paths = await _render_toc_drilldown(query, qvec, entities, relations)
+
+ # Two selection strategies, chosen by whether the tree can score itself.
+ hits: list[tuple[str, float]] = []
+ selected: list[str] = []
+ if entities and _has_distinct_node_vectors(entities):
+ # page_index / per-node embeddings: hand the whole TOC to the model
+ # at once. It sees every level simultaneously, so nothing is pruned
+ # by an early bad pick. Empty result just means "caller's problem",
+ # so fall through to the beam only when the model is unavailable.
+ if _STRUCT_TOC_LLM_SELECT:
+ selected = await _select_toc_nodes(tools_slot, query, entities, relations)
+ else:
+ # RAPTOR blob path: every node shares the blob's one embedding, so
+ # beam descent would select arbitrarily. Let the chunk index choose
+ # and keep the tree for labelling where the chosen chunks sit.
+ hits = await _recall_chunk_ids_in_doc(tools_slot, query, doc_id, _STRUCT_RECALL_TOP_N)
+
+ outline, stats, chunk_paths = await _render_toc_drilldown(query, qvec, entities, relations, chunk_hits=hits or None, selected=selected or None)
out.append(
{
"doc_id": doc_id,
@@ -1288,6 +1343,177 @@ def _cosine(a, b):
return 0.0
+def _vecs_equal(a, b) -> bool:
+ """Whether two node embeddings are the same vector."""
+ try:
+ if a is b:
+ return True
+ la, lb = list(a), list(b)
+ if len(la) != len(lb):
+ return False
+ # Distinct embeddings diverge long before the tail; a prefix compare is
+ # enough and keeps this cheap on 1024-dim vectors.
+ return all(x == y for x, y in zip(la[:16], lb[:16]))
+ except Exception:
+ return False
+
+
+def _has_distinct_node_vectors(entities: list[dict]) -> bool:
+ """Whether nodes carry per-node embeddings rather than one shared vector.
+
+ The RAPTOR/tree main path persists a single graph blob, so every node
+ inherits that blob's one vector (see ``_load_entities_with_vectors``) and
+ beam scoring degenerates: every node gets the same cosine, the top-K is
+ whatever the row order happened to be, and the drill carries no signal.
+ page_index and the pipeline Compiler write one row per entity with its own
+ embedding, so their trees do discriminate.
+ """
+ first = None
+ for e in entities:
+ v = e.get("_vec")
+ if v is None:
+ continue
+ if first is None:
+ first = v
+ elif not _vecs_equal(first, v):
+ return True
+ # Either no node has a vector, or they all share one — neither discriminates.
+ return False
+
+
+async def _recall_chunk_ids_in_doc(tools, query: str, doc_id: str, top_n: int) -> list[tuple[str, float]]:
+ """Hybrid chunk retrieval confined to one document, as ``[(chunk_id, score)]``.
+
+ Used when a compiled tree cannot score its own nodes: the chunk index then
+ decides *which* nodes matter, and the tree is left to label where they sit.
+ """
+ if not query or not doc_id:
+ return []
+ from common import settings
+
+ target_ids = getattr(tools, "kb_ids", None) or []
+ if not target_ids:
+ return []
+ embd_mdl = getattr(tools, "embed_mdl", None)
+ try:
+ kbinfos = await settings.retriever.retrieval(
+ query,
+ embd_mdl,
+ getattr(tools, "tenant_ids", None),
+ target_ids,
+ 1,
+ top_n,
+ 0.0,
+ vector_similarity_weight=0.3 if embd_mdl else 0,
+ doc_ids=[str(doc_id).strip()],
+ highlight=False,
+ # RAPTOR cluster nodes live in the chunk index (rechunk_kwd=tree, no
+ # compile_kwd), so they survive this filter and stay retrievable —
+ # which is the point, since they are the structure.
+ must_not={"exists": "compile_kwd"},
+ )
+ except Exception:
+ _LOG.exception("[navigate_structure] chunk recall failed for doc=%s", doc_id)
+ return []
+ hits: list[tuple[str, float]] = []
+ for c in kbinfos.get("chunks", []):
+ cid = str(c.get("chunk_id") or c.get("id") or "").strip()
+ if cid:
+ hits.append((cid, float(c.get("similarity") or 0.0)))
+ return hits
+
+
+def _build_toc_items(entities: list[dict], relations: list[dict]) -> list[dict]:
+ """Serialize the whole TOC, indented by depth, for one-shot LLM selection.
+
+ Indentation is the only structure the model gets: it reads the list the way a
+ reader reads a table of contents. Every node carries its real name through
+ ``_node`` so the selection can be mapped back without string matching.
+
+ Orphans (nodes no relation attaches to a root) are appended last at depth 0 —
+ dropping them would silently hide content from the model.
+ """
+ by_name, children, parents, roots = _build_toc_tree(entities, relations)
+ if not by_name:
+ return []
+
+ items: list[dict] = []
+ seen: set[str] = set()
+
+ def _walk(name: str, depth: int) -> None:
+ if name in seen or depth > _STRUCT_TOC_MAX_DEPTH:
+ return
+ seen.add(name)
+ e = by_name.get(name)
+ if not e:
+ return
+ desc = str(e.get("description") or "").strip().replace("\n", " ")
+ items.append(
+ {
+ "_node": name,
+ "name": " " * depth + name,
+ "description": _snippet(desc, _STRUCT_TOC_DESC_SNIPPET),
+ }
+ )
+ for child in children.get(name, []):
+ _walk(child, depth + 1)
+
+ for r in roots:
+ _walk(r, 0)
+ for name in by_name:
+ if name not in seen:
+ _walk(name, 0)
+ return items
+
+
+async def _select_toc_nodes(tools, query: str, entities: list[dict], relations: list[dict]) -> list[str]:
+ """Let the model pick TOC nodes in one shot. Returns node names (''-filtered).
+
+ Returns ``[]`` on any failure so the caller can fall back — never raises.
+ """
+ if not entities:
+ return []
+ try:
+ items = _build_toc_items(entities, relations)
+ except Exception:
+ _LOG.exception("[navigate_structure] TOC serialization failed")
+ return []
+ if not items:
+ return []
+ try:
+ picked = await _ask_nav_select(tools, query, items, "sections", _STRUCT_TOC_MAX_NODES)
+ except Exception:
+ # _ask_nav_select catches model failures, but serialization/other paths can
+ # still raise; the caller must be able to fall through to local scoring.
+ _LOG.exception("[navigate_structure] TOC model selection failed")
+ return []
+ names = []
+ for it in picked or []:
+ name = str(it.get("_node") or "").strip()
+ if name:
+ names.append(name)
+ return names
+
+
+def _nodes_covering_chunks(entities: list[dict], chunk_ids: set[str]) -> list[str]:
+ """Names of nodes whose ``source_chunk_ids`` cover any of ``chunk_ids``.
+
+ Used only to give retrieved chunks a structural home. A chunk that no node
+ covers is still returned — the tree annotates, it does not select.
+ """
+ if not chunk_ids:
+ return []
+ names: list[str] = []
+ for e in entities:
+ name = (e.get("name") or "").strip()
+ if not name:
+ continue
+ ids = {str(c) for c in (e.get("source_chunk_ids") or []) if isinstance(c, str) and c}
+ if ids & chunk_ids:
+ names.append(name)
+ return names
+
+
def _node_score(qvec, query_terms, entity: dict) -> float:
"""Score a TOC node: cosine similarity (if the node has an embedding and we
have a query vector), else keyword-overlap. Higher is better."""
@@ -1369,13 +1595,34 @@ def _drill_kept_nodes(query_terms: list[str], qvec, entities: list[dict], relati
return kept_nodes, parents, kept_names, best_overall
-async def _render_toc_drilldown(query: str, qvec, entities: list[dict], relations: list[dict]) -> tuple[str, dict, dict]:
- """Drill down the pre-built TOC hierarchy toward the query and render an outline.
-
- Uses VECTOR BEAM descent when the nodes carry embeddings (``_vec``) and a query
- vector is available: from the roots, at each level keep the top-K nodes by
- cosine similarity to the query and descend to their children. Falls back to
- keyword-overlap when no vectors are available.
+async def _render_toc_drilldown(
+ query: str,
+ qvec,
+ entities: list[dict],
+ relations: list[dict],
+ chunk_hits: list[tuple[str, float]] | None = None,
+ selected: list[str] | None = None,
+) -> tuple[str, dict, dict]:
+ """Render a query-focused outline of one document's compiled structure.
+
+ Three ways to choose what to show, in the order the caller tries them:
+
+ ``selected`` — node names already chosen elsewhere (the whole-TOC LLM pass,
+ see ``_select_toc_nodes``). Every level was visible to the model at once,
+ so nothing is pruned; chunks come from the chosen nodes' ``source_ids``.
+
+ ``chunk_hits`` — ``[(chunk_id, score)]`` from chunk retrieval, used when the
+ tree cannot score its own nodes (RAPTOR blob path). Selection belongs to
+ the chunk index outright: the chunks are returned **as retrieved**, not
+ filtered back through the nodes, and the tree is left to do the only job
+ it is still needed for — labelling where each chunk sits. Round-tripping
+ the selection through the nodes would silently drop every chunk no node
+ happens to cover, which for a clustered tree is most of them.
+
+ neither — VECTOR BEAM descent: from the roots, at each level keep the top-K
+ nodes by cosine similarity to the query and descend to their children,
+ falling back to keyword overlap when no vectors are available. Kept as
+ the fallback when the model pass returns nothing.
Returns lines like:
- Paris Demographics (tree_node): <desc> [chunks: c1,c2]
@@ -1388,11 +1635,48 @@ async def _render_toc_drilldown(query: str, qvec, entities: list[dict], relation
structure context onto retrieved evidence without re-deriving the tree.
"""
query_terms = [t for t in re.findall(r"[A-Za-z0-9_]{2,}", (query or "").lower()) if len(t) >= 2]
- if not query_terms and qvec is None:
+ # Pinned nodes were chosen elsewhere (chunk retrieval), so they stand even
+ # when the query yields no usable terms — bailing out here would discard a
+ # perfectly good selection just because the query is short.
+ if not chunk_hits and not selected and not query_terms and qvec is None:
return _render_outline(entities[:_STRUCT_MAX_NODES], relations[:_STRUCT_MAX_NODES]), _outline_stats(entities), {}
- kept_nodes, parents, kept_names, best = _drill_kept_nodes(query_terms, qvec, entities, relations)
- if not kept_nodes:
+ by_name, _children, parents, _roots = _build_toc_tree(entities, relations)
+
+ def _with_ancestors(names: set[str]) -> set[str]:
+ """Pull in ancestors so a path renders as root -> ... -> node."""
+ for name in list(names):
+ cur = parents.get(name)
+ guard = 0
+ while cur and cur not in names and guard < _STRUCT_MAX_DEPTH:
+ names.add(cur)
+ cur = parents.get(cur)
+ guard += 1
+ return names
+
+ selected_ids: list[str] = []
+ if selected:
+ kept_names = _with_ancestors({n for n in selected if n in by_name})
+ kept_nodes = [by_name[n] for n in kept_names if n in by_name]
+ # The model picked nodes, so the chunks are theirs to supply.
+ # No score: the model chooses, it does not rank (see stats["selector"]).
+ best = 0.0
+ selector = "llm_toc"
+ elif chunk_hits:
+ # Keep the retrieval's own ordering — it is the relevance order.
+ seen: set[str] = set()
+ for cid, _score in chunk_hits:
+ if cid and cid not in seen:
+ seen.add(cid)
+ selected_ids.append(cid)
+ kept_names = _with_ancestors({n for n in _nodes_covering_chunks(entities, set(selected_ids)) if n in by_name})
+ kept_nodes = [by_name[n] for n in kept_names if n in by_name]
+ best = max((float(s) for _c, s in chunk_hits), default=0.0)
+ selector = "chunk_retrieval"
+ else:
+ kept_nodes, parents, kept_names, best = _drill_kept_nodes(query_terms, qvec, entities, relations)
+ selector = "beam"
+ if not kept_nodes and not selected_ids:
return _render_outline(entities[:_STRUCT_MAX_NODES], relations[:_STRUCT_MAX_NODES]), _outline_stats(entities), {}
# Render the drilled nodes (indented by depth via ancestor count).
@@ -1438,13 +1722,23 @@ async def _render_toc_drilldown(query: str, qvec, entities: list[dict], relation
line += f" [chunks: {chunks}]"
lines.append(line)
- # Rank the chunks behind the drilled nodes and add short snippets.
- wanted = _collect_chunk_ids(kept_nodes)
+ # Chunk snippets. When chunk retrieval did the selecting, those hits are
+ # returned as-is: filtering them back through the nodes would drop every
+ # chunk no node happens to cover. Otherwise take the chunks behind the
+ # drilled nodes.
+ wanted = selected_ids or _collect_chunk_ids(kept_nodes)
if wanted:
chunks = await _load_chunks_for_ids(_tools_slot(), wanted)
if chunks:
- ranked = _rank_chunks_by_terms(chunks, [query])
- for c in ranked[:_STRUCT_MAX_CHUNKS]:
+ if selected_ids:
+ # Retrieval order is the relevance order; re-ranking by terms
+ # would discard the score that selected them.
+ ranked = sorted(chunks, key=lambda c: selected_ids.index(_chunk_id(c)))
+ limit = _STRUCT_MAX_CHUNK_HITS
+ else:
+ ranked = _rank_chunks_by_terms(chunks, [query])
+ limit = _STRUCT_MAX_CHUNKS
+ for c in ranked[:limit]:
cid = _chunk_id(c)
text = _chunk_text(c).strip()
lines.append(f"- [chunk {cid}]: {_snippet(text, 300)}")
@@ -1452,6 +1746,9 @@ async def _render_toc_drilldown(query: str, qvec, entities: list[dict], relation
"nodes": len(kept_nodes),
"chunk_ptrs": len(wanted),
"top_score": round(float(best), 4),
+ # Which strategy chose the nodes. top_score is only meaningful for
+ # "beam" and "chunk_retrieval"; llm_toc has no comparable score.
+ "selector": selector,
}
return "\n".join(lines), stats, chunk_paths
diff --git a/test/unit_test/api/apps/services/test_dataset_api_service_nav.py b/test/unit_test/api/apps/services/test_dataset_api_service_nav.py
index 12f4a74f5b..d904c0afe4 100644
--- a/test/unit_test/api/apps/services/test_dataset_api_service_nav.py
+++ b/test/unit_test/api/apps/services/test_dataset_api_service_nav.py
@@ -17,6 +17,7 @@
import importlib.util
import json
+import math
import sys
from enum import IntEnum
from pathlib import Path
@@ -47,13 +48,28 @@ def _stub(monkeypatch, name, **attrs):
return mod
-def _load_nav_module(monkeypatch, *, accessible=True, index_pack=("idx-1", None), total=0, field_map=None):
+class _StubMatchTextExpr:
+ """Stand-in for rag.nlp.search.MatchTextExpr (keyword-match fallback expr)."""
+
+ def __init__(self, fields, text, top_k):
+ self.fields, self.text, self.top_k = fields, text, top_k
+
+
+def _load_nav_module(monkeypatch, *, accessible=True, index_pack=("idx-1", None), total=0, field_map=None, retriever=None):
+ # Stubbed before the import: importing the real rag.nlp pulls in heavy
+ # third-party deps the unit tier does not install.
+ _stub(monkeypatch, "rag.nlp")
+ _stub(monkeypatch, "rag.nlp.search", MatchTextExpr=_StubMatchTextExpr)
+
doc_store = MagicMock()
doc_store.search = MagicMock(return_value={})
doc_store.get_fields = MagicMock(return_value=field_map or {})
doc_store.get_total = MagicMock(return_value=total)
doc_store.delete = MagicMock(return_value=0)
+ if retriever is None:
+ retriever = SimpleNamespace(retrieval=AsyncMock(return_value={"chunks": []}))
+
kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")
knowledgebase_service = SimpleNamespace(
accessible=MagicMock(return_value=accessible),
@@ -73,12 +89,12 @@ def _load_nav_module(monkeypatch, *, accessible=True, index_pack=("idx-1", None)
PAGERANK_FLD="pagerank",
LLMType=SimpleNamespace(),
FileSource=SimpleNamespace(KNOWLEDGEBASE="knowledgebase"),
- RetCode=SimpleNamespace(),
+ RetCode=SimpleNamespace(SERVER_ERROR=500, ARGUMENT_ERROR=400),
StatusEnum=SimpleNamespace(),
TaskStatus=SimpleNamespace(),
ModelTypeBinary=_StubModelTypeBinary,
)
- _stub(monkeypatch, "common.settings", docStoreConn=doc_store, DOC_ENGINE="infinity")
+ _stub(monkeypatch, "common.settings", docStoreConn=doc_store, retriever=retriever, DOC_ENGINE="infinity")
_stub(
monkeypatch,
"api.db.db_models",
@@ -234,3 +250,532 @@ async def test_delete_nav_returns_zero_when_index_missing(monkeypatch):
assert ok is True
assert payload == {"deleted": 0}
doc_store.delete.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# Document routing via compiled page_index rows
+# ---------------------------------------------------------------------------
+
+
+def _compiled_row(doc_id, rtype, similarity, name="n", payload=None):
+ body = payload if payload is not None else {"type": rtype, "name": name, "description": "d"}
+ return {"doc_id": doc_id, "similarity": similarity, "content_with_weight": json.dumps(body)}
+
+
+def test_nav_bucket_compiled_rows_splits_title_and_fact(monkeypatch):
+ module, _, _ = _load_nav_module(monkeypatch)
+ buckets = module._nav_bucket_compiled_rows(
+ {
+ "r1": _compiled_row("doc-a", "title", 0.9),
+ "r2": _compiled_row("doc-a", "fact", 0.5),
+ "r3": _compiled_row("doc-a", "conclusion", 0.4),
+ "r4": _compiled_row("doc-b", "title", 0.7),
+ "r5": _compiled_row("doc-c", "relation", 0.6), # non-entity type — dropped
+ }
+ )
+ assert set(buckets["title"]) == {"doc-a", "doc-b"}
+ assert set(buckets["fact"]) == {"doc-a"}
+ # conclusion shares the fact bucket, so doc-a has two fact-class hits.
+ assert buckets["fact"]["doc-a"]["hits"] == 2
+ assert buckets["fact"]["doc-a"]["best"] == pytest.approx(0.5)
+
+
+def test_nav_bucket_compiled_rows_skips_unparseable_payloads(monkeypatch):
+ module, _, _ = _load_nav_module(monkeypatch)
+ buckets = module._nav_bucket_compiled_rows(
+ {
+ "r1": {"doc_id": "doc-a", "similarity": 0.9, "content_with_weight": "{not json"},
+ "r2": {"doc_id": "", "similarity": 0.9, "content_with_weight": json.dumps({"type": "title"})},
+ "r3": {"doc_id": "doc-b", "similarity": 0.9, "content_with_weight": json.dumps([1, 2])},
+ }
+ )
+ assert buckets == {"title": {}, "fact": {}, "fact_names": {}}
+
+
+def test_nav_bucket_compiled_rows_tracks_fact_names_per_document(monkeypatch):
+ """Fact names must stay per-document or the bridge labels the wrong document."""
+ module, _, _ = _load_nav_module(monkeypatch)
+ buckets = module._nav_bucket_compiled_rows(
+ {
+ "r1": _compiled_row("doc-a", "fact", 0.9, name="A1"),
+ "r2": _compiled_row("doc-a", "fact", 0.8, name="A2"),
+ "r3": _compiled_row("doc-b", "fact", 0.7, name="B1"),
+ }
+ )
+ assert buckets["fact_names"] == {"doc-a": ["A1", "A2"], "doc-b": ["B1"]}
+
+
+def test_nav_matched_sections_dedups_and_keeps_hit_order(monkeypatch):
+ module, _, _ = _load_nav_module(monkeypatch)
+ bridge = {"b1": "Discussion", "a1": "Methods", "a2": "Methods"}
+ out = module._nav_matched_sections(bridge, ["a1", "b1", "a2"])
+ assert out == ["Methods", "Discussion"]
+
+
+def test_nav_matched_sections_omits_unbridged_facts(monkeypatch):
+ module, _, _ = _load_nav_module(monkeypatch)
+ assert module._nav_matched_sections({}, ["a1"]) == []
+
+
+async def test_compiled_agg_attaches_matched_sections(monkeypatch):
+ """Routing must hand back a document-internal entry point, not just a doc_id."""
+ module, _, doc_store = _load_nav_module(
+ monkeypatch,
+ field_map={"r1": _compiled_row("doc-a", "fact", 0.9, name="F1")},
+ )
+ doc_store.get_fields = MagicMock(return_value={"r1": _compiled_row("doc-a", "fact", 0.9, name="F1")})
+ monkeypatch.setattr(module, "_nav_bridge_sections", AsyncMock(return_value={"f1": "Methods"}))
+ module.settings.retriever = SimpleNamespace(
+ get_vector=AsyncMock(return_value=SimpleNamespace()),
+ _existing_doc_ids=AsyncMock(return_value={"doc-a"}),
+ )
+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")
+
+ ok, payload = await module._search_layers_compiled_agg("tenant-1", "kb-1", "q", 5, SimpleNamespace(), kb)
+ assert ok is True
+ assert payload["items"][0]["_nav"]["matched_sections"] == ["Methods"]
+
+
+def test_nav_rank_compiled_buckets_rewards_both_legs(monkeypatch):
+ """A document hit by title AND fact outranks one hit by only one leg."""
+ module, _, _ = _load_nav_module(monkeypatch)
+ buckets = module._nav_bucket_compiled_rows(
+ {
+ "r1": _compiled_row("both", "title", 0.8),
+ "r2": _compiled_row("both", "fact", 0.8),
+ "r3": _compiled_row("titleonly", "title", 0.9),
+ }
+ )
+ ranked = module._nav_rank_compiled_buckets(buckets, 10)
+ assert [d for d, _ in ranked] == ["both", "titleonly"]
+ assert ranked[0][1]["legs"] == {"title", "fact"}
+ # best stays on the 0..1 cosine scale callers threshold on.
+ assert ranked[0][1]["best"] == pytest.approx(0.8)
+
+
+def test_nav_rank_compiled_buckets_does_not_mutate_input(monkeypatch):
+ """Regression: ranking used to attach 'legs' onto the caller's bucket entries."""
+ module, _, _ = _load_nav_module(monkeypatch)
+ buckets = module._nav_bucket_compiled_rows({"r1": _compiled_row("doc-a", "title", 0.8)})
+ before = json.dumps(buckets, sort_keys=True, default=str)
+ module._nav_rank_compiled_buckets(buckets, 10)
+ assert json.dumps(buckets, sort_keys=True, default=str) == before
+
+
+def test_nav_rank_compiled_buckets_respects_top_k(monkeypatch):
+ module, _, _ = _load_nav_module(monkeypatch)
+ buckets = module._nav_bucket_compiled_rows({f"r{i}": _compiled_row(f"doc-{i}", "fact", 0.9 - i * 0.1) for i in range(5)})
+ assert len(module._nav_rank_compiled_buckets(buckets, 2)) == 2
+
+
+async def test_compiled_agg_router_uses_keyword_fallback_without_embedding(monkeypatch):
+ """No embedding model: the leg degrades to a keyword match, it does not drop out."""
+ module, _, doc_store = _load_nav_module(
+ monkeypatch,
+ field_map={"r1": _compiled_row("doc-a", "fact", 0.5)},
+ )
+ doc_store.get_fields = MagicMock(return_value={"r1": _compiled_row("doc-a", "fact", 0.5)})
+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")
+
+ ok, payload = await module._search_layers_compiled_agg("tenant-1", "kb-1", "q", 5, None, kb)
+ assert ok is True
+ assert [i["doc_id"] for i in payload["items"]] == ["doc-a"]
+ # _nav_doc_summaries issues a second search, so read the first one.
+ exprs = doc_store.search.call_args_list[0].kwargs["match_expressions"]
+ # A keyword-match expr, not a dense one.
+ assert len(exprs) == 1
+ assert isinstance(exprs[0], _StubMatchTextExpr)
+ assert exprs[0].fields == ["content_ltks", "content_sm_ltks"]
+
+
+async def test_compiled_agg_router_filters_deleted_documents(monkeypatch):
+ """Compiled rows can outlive deleted docs; the leg must check existence itself."""
+ module, _, doc_store = _load_nav_module(
+ monkeypatch,
+ field_map={"r1": _compiled_row("doc-gone", "fact", 0.9)},
+ )
+ doc_store.get_fields = MagicMock(return_value={"r1": _compiled_row("doc-gone", "fact", 0.9)})
+ retriever = SimpleNamespace(
+ get_vector=AsyncMock(return_value=SimpleNamespace()),
+ _existing_doc_ids=AsyncMock(return_value=set()), # nothing survives
+ )
+ module, _, _ = _load_nav_module(monkeypatch, field_map={"r1": _compiled_row("doc-gone", "fact", 0.9)}, retriever=retriever)
+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")
+
+ ok, payload = await module._search_layers_compiled_agg("tenant-1", "kb-1", "q", 5, SimpleNamespace(), kb)
+ assert ok is True
+ assert payload["total"] == 0
+
+
+async def test_compiled_agg_router_builds_vector_with_named_candidates(monkeypatch):
+ """Regression: positional passing put the similarity threshold into num_candidates."""
+ get_vector = AsyncMock(return_value=SimpleNamespace())
+ module, _, doc_store = _load_nav_module(
+ monkeypatch,
+ field_map={"r1": _compiled_row("doc-a", "fact", 0.9)},
+ retriever=SimpleNamespace(get_vector=get_vector, _existing_doc_ids=AsyncMock(return_value={"doc-a"})),
+ )
+ doc_store.get_fields = MagicMock(return_value={"r1": _compiled_row("doc-a", "fact", 0.9)})
+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")
+
+ await module._search_layers_compiled_agg("tenant-1", "kb-1", "q", 5, SimpleNamespace(), kb)
+ kw = get_vector.call_args.kwargs
+ assert kw["top_k"] == module._NAV_COMPILED_POOL
+ # HNSW ef_search must be >= top_k, and must not receive the similarity value.
+ assert kw["num_candidates"] == module._NAV_COMPILED_POOL
+ assert kw["similarity"] == module._NAV_COMPILED_SIMILARITY
+ # _nav_doc_summaries issues a second search, so read the first one.
+ cond = doc_store.search.call_args_list[0].kwargs["condition"]
+ # Compiled rows carry no available_int, so it must NOT be used as a filter.
+ assert "available_int" not in cond
+ assert cond["scope_kwd"] == ["doc"]
+ assert cond["knowledge_graph_kwd"] == ["entity"]
+
+
+# ---------------------------------------------------------------------------
+# Fusion of the chunk leg and the compiled leg
+# ---------------------------------------------------------------------------
+
+
+def test_nav_fuse_legs_normalizes_each_leg(monkeypatch):
+ """Each leg is normalized by its own peak so raw cosine cannot dominate."""
+ module, _, _ = _load_nav_module(monkeypatch)
+ ranked = module._nav_fuse_legs(
+ [
+ ("chunk", [{"doc_id": "a", "score": 0.9}, {"doc_id": "b", "score": 0.45}], 1.0),
+ ("compiled", [{"doc_id": "c", "score": 0.6}], 1.0),
+ ],
+ 10,
+ )
+ scores = {d: info["fused"] for d, info in ranked}
+ # Peak of each leg becomes 1.0, so a document top-ranked by either leg ties.
+ assert scores["a"] == pytest.approx(1.0)
+ assert scores["c"] == pytest.approx(1.0)
+ assert scores["b"] == pytest.approx(0.5)
+
+
+def test_nav_fuse_legs_prefers_documents_hit_by_both(monkeypatch):
+ module, _, _ = _load_nav_module(monkeypatch)
+ ranked = module._nav_fuse_legs(
+ [
+ ("chunk", [{"doc_id": "both", "score": 0.5}, {"doc_id": "chunkonly", "score": 1.0}], 1.0),
+ ("compiled", [{"doc_id": "both", "score": 1.0}], 1.0),
+ ],
+ 10,
+ )
+ assert [d for d, _ in ranked] == ["both", "chunkonly"]
+ assert ranked[0][1]["legs"] == {"chunk", "compiled"}
+
+
+def test_nav_fuse_legs_survives_empty_leg(monkeypatch):
+ module, _, _ = _load_nav_module(monkeypatch)
+ ranked = module._nav_fuse_legs([("chunk", [], 1.0), ("compiled", [], 1.0)], 10)
+ assert ranked == []
+
+
+async def test_fusion_router_falls_back_to_chunk_leg_without_compiled(monkeypatch):
+ """A raptor KB has no page_index rows; the chunk leg must decide alone."""
+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})
+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))
+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")
+
+ ok, payload = await module._search_layers_fusion("tenant-1", "kb-1", "q", 5, None, kb, compiled={"items": []})
+ assert ok is True
+ assert [i["doc_id"] for i in payload["items"]] == ["doc-a"]
+ # No compiled hits means no section bridge lookup either.
+ assert payload["items"][0]["_agg"]["legs"] == ["chunk"]
+
+
+# ---------------------------------------------------------------------------
+# Document routing via chunk aggregation (PageIndex semantics recipe)
+# ---------------------------------------------------------------------------
+
+
+def test_nav_aggregate_chunks_rolls_up_per_document(monkeypatch):
+ module, _, _ = _load_nav_module(monkeypatch)
+ chunks = [
+ {"doc_id": "doc-a", "similarity": 0.9},
+ {"doc_id": "doc-b", "similarity": 0.8},
+ {"doc_id": "doc-a", "similarity": 0.7},
+ {"doc_id": " ", "similarity": 0.5}, # blank doc_id — dropped
+ {"doc_id": "doc-a", "score": 0.5}, # 'score' fallback key
+ ]
+ agg = module._nav_aggregate_chunks(chunks)
+ assert set(agg) == {"doc-a", "doc-b"}
+ assert agg["doc-a"] == {"total": pytest.approx(2.1), "best": 0.9, "hits": 3}
+ assert agg["doc-b"] == {"total": pytest.approx(0.8), "best": 0.8, "hits": 1}
+
+
+def test_nav_doc_score_damps_hit_count(monkeypatch):
+ module, _, _ = _load_nav_module(monkeypatch)
+ single = module._nav_doc_score({"total": 0.9, "best": 0.9, "hits": 1})
+ assert single == pytest.approx(0.9 / math.sqrt(2))
+ # A long document must not win on volume alone: four weak hits stay below
+ # one strong hit.
+ weak_many = module._nav_doc_score({"total": 1.0, "best": 0.25, "hits": 4})
+ assert weak_many == pytest.approx(1.0 / math.sqrt(5))
+ assert weak_many < single
+
+
+async def test_chunk_agg_router_ranks_by_doc_score_not_hit_count(monkeypatch):
+ """doc-b has one strong hit; doc-a has four weak ones. doc-b must win."""
+ retriever = SimpleNamespace(
+ retrieval=AsyncMock(
+ return_value={
+ "chunks": [
+ {"doc_id": "doc-a", "similarity": 0.20},
+ {"doc_id": "doc-a", "similarity": 0.20},
+ {"doc_id": "doc-a", "similarity": 0.20},
+ {"doc_id": "doc-a", "similarity": 0.20},
+ {"doc_id": "doc-b", "similarity": 0.90},
+ ]
+ }
+ )
+ )
+ module, _, _ = _load_nav_module(monkeypatch, retriever=retriever)
+
+ ok, payload = await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 5, None)
+ assert ok is True
+ assert [item["doc_id"] for item in payload["items"]] == ["doc-b", "doc-a"]
+ # score stays on the 0..1 chunk-similarity scale callers threshold on.
+ assert payload["items"][0]["score"] == pytest.approx(0.9)
+ assert payload["items"][1]["score"] == pytest.approx(0.2)
+ assert payload["items"][0]["_agg"]["hits"] == 1
+ assert payload["items"][1]["_agg"]["hits"] == 4
+
+
+async def test_chunk_agg_router_uses_full_candidate_pool(monkeypatch):
+ """page_size covers the pool so every fused candidate reaches the aggregation."""
+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})
+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))
+
+ await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 12, None)
+ args, kw = retrieval.call_args
+ # page_size is positional arg 6; no similarity floor at 7.
+ assert args[5] == module._NAV_CHUNK_AGG_POOL
+ assert args[6] == 0.0
+ # The pool is rerank_candidates_count — page_size alone would leave the
+ # candidate set at the 64 default, too narrow for DocScore to matter.
+ assert kw["rerank_candidates_count"] == module._NAV_CHUNK_AGG_POOL
+ # page * page_size must never exceed rerank_candidates_count or retrieval raises.
+ assert args[4] * args[5] <= kw["rerank_candidates_count"]
+
+
+async def test_chunk_agg_router_does_not_narrow_knn_candidates(monkeypatch):
+ """knn_top_k caps what the vector leg feeds fusion; narrowing it shrinks the pool."""
+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})
+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))
+
+ await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 12, None)
+ assert "knn_top_k" not in retrieval.call_args.kwargs
+
+
+async def test_chunk_agg_router_excludes_compiled_rows(monkeypatch):
+ """Compiled rows are written with available_int=1, so they need explicit exclusion.
+
+ Without this they land in the chunk candidate set and inflate a document's
+ hit count. Project convention: plain retrieval reads chunks, compiled
+ products are served by their own tools.
+ """
+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})
+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))
+
+ await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 12, None)
+ assert retrieval.call_args.kwargs["must_not"] == {"exists": "compile_kwd"}
+
+
+async def test_chunk_agg_router_can_include_compiled_rows(monkeypatch):
+ """Flipping the flag lets compiled rows A/B as extra per-document entries."""
+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})
+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))
+ monkeypatch.setattr(module, "_NAV_CHUNK_AGG_EXCLUDE_COMPILED", False)
+
+ await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 12, None)
+ assert retrieval.call_args.kwargs["must_not"] is None
+
+
+def test_nav_label_items_wraps_flat_row_for_the_caller(monkeypatch):
+ """navigation.py reads the route label from item["_nav"]["description"].
+
+ search_dataset_nav returns the leaf row flat, so without wrapping the route
+ comes back unlabelled.
+ """
+ module, _, _ = _load_nav_module(monkeypatch)
+ out = module._nav_label_items([{"doc_id": "doc-a", "name": "Leaf A", "description": "Summary A"}])
+ assert out[0]["_nav"] == {"doc_id": "doc-a", "description": "Summary A"}
+
+
+def test_nav_label_items_falls_back_to_name(monkeypatch):
+ module, _, _ = _load_nav_module(monkeypatch)
+ out = module._nav_label_items([{"doc_id": "doc-a", "name": "Leaf A", "description": ""}])
+ assert out[0]["_nav"]["description"] == "Leaf A"
+
+
+def test_nav_label_items_preserves_existing_nav(monkeypatch):
+ """A row that already carries _nav must not be overwritten."""
+ module, _, _ = _load_nav_module(monkeypatch)
+ out = module._nav_label_items([{"doc_id": "doc-a", "description": "flat", "_nav": {"description": "nested"}}])
+ assert out[0]["_nav"]["description"] == "nested"
+
+
+def test_nav_label_items_tolerates_non_dicts(monkeypatch):
+ module, _, _ = _load_nav_module(monkeypatch)
+ assert module._nav_label_items(["junk", None]) == ["junk", None]
+
+
+def test_nav_focus_items_sorts_and_truncates_to_limit(monkeypatch):
+ """Regression: the flat sweep returned every row above the 0.2 floor.
+
+ Main's cluster descent routes ~2 documents; the flat sweep returned 6-12,
+ and routing to 3x the documents made the RAGAgent carry 3x the evidence in
+ every round — each dynamic LLM call grew ~33% and per-question time ~25%.
+ """
+ module, _, _ = _load_nav_module(monkeypatch)
+ monkeypatch.setattr(module, "_NAV_DOC_FOCUS_LIMIT", 3)
+ # Not sorted — must sort by score before truncating.
+ items = [{"doc_id": f"doc-{i}", "score": s} for i, s in [(6, 0.1), (1, 0.9), (3, 0.4), (2, 0.6)]]
+ out = module._nav_focus_items(items)
+ assert [i["doc_id"] for i in out] == ["doc-1", "doc-2", "doc-3"]
+ # The input list is truncated in place.
+ assert [i["doc_id"] for i in items] == ["doc-1", "doc-2", "doc-3"]
+
+
+def test_nav_focus_items_keeps_fewer_than_limit_untouched(monkeypatch):
+ """When few docs are relevant, do not pad — routing stays honest."""
+ module, _, _ = _load_nav_module(monkeypatch)
+ monkeypatch.setattr(module, "_NAV_DOC_FOCUS_LIMIT", 3)
+ items = [{"doc_id": "doc-1", "score": 0.9}]
+ out = module._nav_focus_items(items)
+ assert len(out) == 1
+
+
+def test_nav_focus_items_disabled_when_limit_is_zero(monkeypatch):
+ module, _, _ = _load_nav_module(monkeypatch)
+ monkeypatch.setattr(module, "_NAV_DOC_FOCUS_LIMIT", 0)
+ items = [{"doc_id": f"doc-{i}", "score": s} for i, s in [(1, 0.9), (2, 0.6), (3, 0.4)]]
+ out = module._nav_focus_items(items)
+ assert len(out) == 3
+
+
+async def test_chunk_agg_router_focuses_to_limit(monkeypatch):
+ """chunk_agg must cap the returned documents like nav_doc does.
+
+ Without the focus cap this router returned every document above the floor,
+ routing the RAGAgent into 3x the evidence and inflating every dynamic LLM
+ call — the same regression the nav_doc router had.
+ """
+ retriever = SimpleNamespace(retrieval=AsyncMock(return_value={"chunks": [{"doc_id": f"doc-{i}", "similarity": s} for i, s in [(5, 0.1), (1, 0.9), (3, 0.4), (2, 0.6), (4, 0.2)]]}))
+ module, _, _ = _load_nav_module(monkeypatch, retriever=retriever)
+ monkeypatch.setattr(module, "_NAV_DOC_FOCUS_LIMIT", 3)
+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")
+
+ ok, payload = await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 12, None, kb)
+ assert ok is True
+ # Cap to the focus limit even when top_k is higher.
+ assert payload["total"] <= 3
+ assert len(payload["items"]) <= 3
+
+
+async def test_chunk_agg_router_focus_limit_overrides_top_k(monkeypatch):
+ """Focus wins over the caller's top_k — top_k=12 must not widen routing."""
+ retriever = SimpleNamespace(retrieval=AsyncMock(return_value={"chunks": [{"doc_id": f"doc-{i}", "similarity": 0.9 - i * 0.1} for i in range(8)]}))
+ module, _, _ = _load_nav_module(monkeypatch, retriever=retriever)
+ monkeypatch.setattr(module, "_NAV_DOC_FOCUS_LIMIT", 2)
+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")
+
+ ok, payload = await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 12, None, kb)
+ assert ok is True
+ assert payload["total"] == 2
+
+
+async def test_chunk_mode_page_size_stays_bounded(monkeypatch):
+ """Chunk mode returns chunks AS evidence, so page_size is the result size.
+
+ Regression: it once shared the chunk_agg aggregation pool (256), which
+ inflated the evidence pool ~7x (25 -> 191 chunks/question), bloated every
+ downstream LLM prompt and exhausted the 180s research budget on 15/16
+ questions.
+ """
+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})
+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))
+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")
+
+ await module._search_layers_chunks("tenant-1", "kb-1", "q", 12, None, kb)
+ args, kw = retrieval.call_args
+ # Historical fetch size, not the aggregation pool.
+ assert args[5] == 12 * 3
+ # The candidate pool still has to cover the returned page.
+ assert kw["rerank_candidates_count"] >= args[5]
+
+
+async def test_chunk_agg_router_survives_large_top_k(monkeypatch):
+ """Regression: a top_k that pushes page_size past the default pool used to raise."""
+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})
+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))
+
+ ok, payload = await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 100, None)
+ assert ok is True
+ args, kw = retrieval.call_args
+ assert args[4] * args[5] <= kw["rerank_candidates_count"]
+
+
+async def test_chunk_agg_router_falls_back_to_terms_without_embeddings(monkeypatch):
+ """Without an embedding model the dense leg is dropped, not the router."""
+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})
+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))
+
+ ok, payload = await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 5, None)
+ assert ok is True
+ assert payload["total"] == 1
+ assert retrieval.call_args.args[7] == 0
+
+
+async def test_chunk_agg_router_returns_empty_when_no_chunks(monkeypatch):
+ module, _, _ = _load_nav_module(monkeypatch)
+ ok, payload = await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 5, None)
+ assert ok is True
+ assert payload == {"mode": "navigation_tree", "total": 0, "items": []}
+
+
+async def test_nav_doc_summaries_batch_loads_descriptions(monkeypatch):
+ module, _, doc_store = _load_nav_module(
+ monkeypatch,
+ field_map={
+ "row-1": {
+ "name": "leaf-a",
+ "type_kwd": "nav_doc",
+ "content_with_weight": json.dumps({"description": "Summary A"}),
+ "doc_id": "doc-a",
+ },
+ "row-2": {
+ "name": "leaf-b",
+ "type_kwd": "nav_doc",
+ "content_with_weight": json.dumps({"description": "Summary B"}),
+ "doc_id": "doc-b",
+ },
+ },
+ )
+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")
+ summaries = await module._nav_doc_summaries(kb, ["doc-a", "doc-b"])
+ assert summaries == {"doc-a": "Summary A", "doc-b": "Summary B"}
+ # One query for the whole batch, not one per document.
+ assert doc_store.search.call_count == 1
+ condition = doc_store.search.call_args.kwargs["condition"]
+ assert condition["type_kwd"] == ["nav_doc"]
+ assert condition["doc_id"] == ["doc-a", "doc-b"]