RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Adjust navigation tools for agentic search (#19172)
Yingfenga11c27e01d10
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex 9f2ba094bd..ffc6270486 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -15,6 +15,7 @@#import jsonimport logging+import mathimport osimport re@@ -3818,6 +3819,81 @@ _LAYERS_HANDLERS: dict[str, str] = {"all": "_search_layers_all",}+# Document router behind the "navigation_tree" mode. Callers always ask for+# mode="navigation_tree", so swapping strategy needs no change on their side.+#+# "nav_doc" (default) — flat hybrid search over the nav_doc rows. No cluster+# level is consulted, so nothing is pruned, but the comparison still runs+# on document-level embeddings. Those rows embed ``graph_text`` — the+# document's full entity/relation listing — not a one-line summary, so+# this is not the "compressed representation first" case that SCT measures.+# "chunk_agg" — PageIndex ``semantics`` recipe: hybrid search over the raw+# chunk index, hits rolled up per document with+# DocScore = sum(chunk scores) / sqrt(hits + 1). Sound in principle, but+# chunk-level similarity is easy to clear: a document passes the 0.2 floor+# on a single marginally related chunk, so the top-k roster fills with+# weakly related documents, and the wider routing fans out every later+# stage. Measured 2026-09-02: 12/12 questions routed a full 12 documents,+# evidence pool 42-67 passages, 166s per question against a 180s budget,+# with several timeouts yielding no answer at all.+# "compiled_agg" — page_index entity rows only; "fusion" — chunk + compiled.+# "tree" — BFS beam descent over the nav cluster tree. This is what main+# branch does. Kept as the A/B baseline; drop it, and+# ``search_nav_tree_descent`` with it, once a winner is settled.+_NAV_TREE_ROUTER = "chunk_agg"++# Cap on how many documents a flat router hands back. The cluster beam descent+# that main uses returns ~2 by construction (its per-level pruning keeps only+# the most relevant branches), while the flat nav_doc sweep returns every row+# above the 0.2 floor — measured 2026-09-02: beam avg 2.0 vs nav_doc avg 6.2.+# Routing to 3x the documents made the RAGAgent carry 3x the evidence in every+# round, so each dynamic LLM call grew 14k -> 18.8k tokens and per-question time+# grew ~25%. Beam is not capped at 2 either (it occasionally routes 6 for+# multi-hop), so allow a little headroom above its median rather than a hard 2.+_NAV_DOC_FOCUS_LIMIT = 3++# Candidate pool for "chunk_agg". DocScore only means something when a document+# can collect several hits, so the pool has to be wide enough for that; across a+# 2400-document corpus a pool of a few dozen chunks yields almost entirely+# single-hit documents, which collapses DocScore into plain best-chunk ranking.+# This is ``rerank_candidates_count`` — the pool the fusion actually draws from.+# ``page_size`` is deliberately the same value so the whole pool is returned for+# aggregation, and it must stay <= the pool or ``retrieval`` raises+# (page * page_size > rerank_candidates_count).+_NAV_CHUNK_AGG_POOL = 256+_NAV_CHUNK_AGG_VEC_WEIGHT = 0.3++# Candidate pool of compiled entity rows for the "compiled_agg" / "fusion"+# routers. ``get_vector`` builds its own MatchDenseExpr and never routes through+# ``retrieval``, so ``rerank_candidates_count`` does not apply here — but HNSW+# needs ``num_candidates`` >= the pool or the search degenerates.+_NAV_COMPILED_POOL = 256+_NAV_COMPILED_SIMILARITY = 0.1+# Entity types aggregated separately. A document carries far more facts than+# titles, so pooling them would let fact hits drown out title hits; splitting+# keeps the coarser title signal usable. ``entity_type_kwd`` does not exist on+# compilation rows (it belongs to graphrag), so the type is read from the+# payload after the fetch.+_NAV_COMPILED_TITLE_TYPES = ("title",)+_NAV_COMPILED_FACT_TYPES = ("fact", "conclusion")+# Weights applied when the fusion router combines the three legs. Only used+# when documents are hit by several legs.+_NAV_FUSION_WEIGHTS = {"chunk": 1.0, "title": 0.8, "fact": 0.9}++# Section bridge (stage 3): resolve each hit fact to its owning title. The+# relation type carrying TOC containment in the page_index template, plus caps+# on how many names are resolved and how many relation rows may be scanned.+_NAV_INCLUDE_RELATION = "include"+_NAV_BRIDGE_NAMES_LIMIT = 64+_NAV_BRIDGE_FANOUT = 4+# Compiled products (tree/entity/relation rows) are written with+# available_int=1, so they fall inside the chunk filter and would otherwise+# inflate a document's hit count. The project convention is that plain+# retrieval reads document chunks only and compiled products are served by their+# own tools, so exclude them here too. Flip to False to A/B letting compiled+# rows act as extra per-document entries in the aggregation.+_NAV_CHUNK_AGG_EXCLUDE_COMPILED = True+async def search_dataset_layers(dataset_id: str,@@ -3835,7 +3911,8 @@ async def search_dataset_layers(- chunk: raw document chunks (via the main retrieval pipeline)- nav_doc: navigation tree document leaves- nav_cluster: navigation tree cluster nodes- - navigation_tree: tree-structured BFS beam descent+ - navigation_tree: document routing, strategy chosen by+ ``_NAV_TREE_ROUTER`` (chunk aggregation by default)- all: union of all modes, deduplicated by doc_id with best scoredoc_scope: Optional set of documents to restrict the search to. None orempty means all documents of the dataset. Forwarded to every mode:@@ -3883,7 +3960,7 @@ async def search_dataset_layers(elif mode == "nav_cluster":return await _search_layers_nav_clusters(tenant_id, dataset_id, query, top_k, embd_mdl, search_dataset_nav, doc_scope=doc_scope)elif mode == "navigation_tree":- return await _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, doc_scope=doc_scope)+ return await _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)elif mode == "chunk":return await _search_layers_chunks(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)elif mode == "all":@@ -3920,20 +3997,608 @@ async def _search_layers_nav_clusters(tenant_id, dataset_id, query, top_k, embd_return True, {"mode": "nav_cluster", "total": len(items), "items": items}-async def _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, *, doc_scope=None):- from rag.advanced_rag.knowlege_compile.dataset_nav import search_nav_tree_descent+def _nav_focus_items(items: list) -> list:+ """Truncate a flat routing to the most relevant documents, in place.- items = await search_nav_tree_descent(- tenant_id,- dataset_id,- query,- embd_mdl,- top_k=top_k,- doc_scope=doc_scope,- )+ The flat sweep returns every row above the floor; the cluster descent main+ uses stays focused by its per-level pruning. See ``_NAV_DOC_FOCUS_LIMIT``+ for the measured reason this matters.+ """+ limit = _NAV_DOC_FOCUS_LIMIT+ if not limit or len(items) <= limit:+ return items+ items.sort(key=lambda it: float(it.get("score", 0.0)), reverse=True)+ del items[limit:]+ return items+++def _nav_label_items(items: list) -> list:+ """Give each routed item the ``_nav`` summary navigation.py reads.++ ``search_dataset_nav`` returns the leaf row flat (``name``/``description`` at+ the top level), but navigation.py reads the route label from+ ``item["_nav"]["description"]``, so without this a route comes back+ unlabelled. The rows already carry the summary, so this costs no extra+ query — unlike the chunk router, which has to look one up.+ """+ for item in items:+ if not isinstance(item, dict) or item.get("_nav"):+ continue+ item["_nav"] = {+ "doc_id": str(item.get("doc_id") or "").strip(),+ "description": str(item.get("description") or item.get("name") or "").strip(),+ }+ return items+++async def _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, kb=None, *, doc_scope=None):+ """Route to documents, dispatched by ``_NAV_TREE_ROUTER``.++ Every strategy returns the same item shape — ``doc_id``, a 0..1 ``score``,+ and ``_nav`` carrying the document summary — so callers stay agnostic to+ which one is active.+ """+ if _NAV_TREE_ROUTER == "nav_doc":+ from rag.advanced_rag.knowlege_compile.dataset_nav import search_dataset_nav++ items = await _nav_search_result(+ tenant_id,+ dataset_id,+ query,+ top_k,+ embd_mdl,+ search_dataset_nav,+ type_kwd="nav_doc",+ doc_scope=doc_scope,+ )+ _nav_focus_items(items)+ _nav_label_items(items)+ return True, {"mode": "navigation_tree", "total": len(items), "items": items}++ if _NAV_TREE_ROUTER == "tree":+ from rag.advanced_rag.knowlege_compile.dataset_nav import search_nav_tree_descent++ items = await search_nav_tree_descent(+ tenant_id,+ dataset_id,+ query,+ embd_mdl,+ top_k=top_k,+ doc_scope=doc_scope,+ )+ return True, {"mode": "navigation_tree", "total": len(items), "items": items}++ if _NAV_TREE_ROUTER in ("compiled_agg", "fusion"):+ ok, payload = await _search_layers_compiled_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)+ if _NAV_TREE_ROUTER == "compiled_agg":+ return ok, payload+ # Fusion keeps going: the compiled leg ranks alongside the chunk leg.+ return await _search_layers_fusion(tenant_id, dataset_id, query, top_k, embd_mdl, kb, compiled=payload, doc_scope=doc_scope)++ return await _search_layers_chunk_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)+++async def _search_layers_chunk_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb=None, *, doc_scope=None):+ """Route to documents by rolling up raw-chunk hybrid hits (PageIndex style).++ Retrieval runs over the chunk index — uncompressed text — and the hits are+ aggregated per document with PageIndex's DocScore::++ DocScore = sum(chunk scores) / sqrt(hits + 1)++ The numerator rewards a document that matches several chunks; the+ square-root denominator damps that reward so a long document cannot win on+ volume alone.++ Ordering uses DocScore, but the reported ``score`` is the document's best+ chunk similarity: callers threshold on it (``_NAV_MIN_DOC_SCORE``), and+ keeping a single chunk's similarity preserves the 0..1 scale those+ thresholds were set against.+ """+ from common import settings++ kwargs = {}+ if doc_scope:+ kwargs["doc_ids"] = [str(d) for d in doc_scope if str(d).strip()]++ pool = _NAV_CHUNK_AGG_POOL+ # Term-only and dense scores live on different scales, so the weight (not the+ # threshold) carries the "is there an embedding model" decision.+ vector_weight = _NAV_CHUNK_AGG_VEC_WEIGHT if embd_mdl else 0+ try:+ ranks = await settings.retriever.retrieval(+ query,+ embd_mdl,+ [tenant_id],+ [dataset_id],+ 1,+ # page_size == the pool, so every fused candidate reaches the+ # aggregation below.+ pool,+ # No similarity floor: ranking is done by DocScore, so the pool must+ # stay wide enough for a document to collect several hits. A floor+ # here would pre-empt the aggregation.+ 0.0,+ vector_weight,+ # knn_top_k is left at its default: it caps how many neighbours the+ # vector leg feeds into fusion, and narrowing it to top_k would+ # shrink the pool this strategy depends on.+ rerank_candidates_count=pool,+ must_not={"exists": "compile_kwd"} if _NAV_CHUNK_AGG_EXCLUDE_COMPILED else None,+ **kwargs,+ )+ except Exception:+ logging.exception("search_dataset_layers: chunk-agg retrieval failed for kb=%s", dataset_id)+ return False, {"error": "chunk retrieval failed", "code": RetCode.SERVER_ERROR}++ agg = _nav_aggregate_chunks(ranks.get("chunks", []))+ if not agg:+ return True, {"mode": "navigation_tree", "total": 0, "items": []}++ ranked = sorted(agg.items(), key=lambda kv: _nav_doc_score(kv[1]), reverse=True)+ # Cap by the caller's top_k and by the focus limit. Without the focus cap+ # this router returned every document above the floor (measured 6-12),+ # routing the RAGAgent into 3x the evidence and inflating every dynamic LLM+ # call — the same regression the nav_doc router had, fixed here the same way.+ limit = _NAV_DOC_FOCUS_LIMIT or (top_k or _NAV_CHUNK_AGG_POOL)+ if limit > 0:+ ranked = ranked[:limit]++ summaries = await _nav_doc_summaries(kb, [doc_id for doc_id, _ in ranked])+ items = [+ {+ "doc_id": doc_id,+ "score": round(entry["best"], 4),+ "_nav": {"doc_id": doc_id, "description": summaries.get(doc_id, "")},+ # Aggregation detail, surfaced for tuning the fetch multiplier.+ "_agg": {"doc_score": round(_nav_doc_score(entry), 4), "hits": entry["hits"]},+ }+ for doc_id, entry in ranked+ ]return True, {"mode": "navigation_tree", "total": len(items), "items": items}+async def _search_layers_compiled_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb=None, *, doc_scope=None):+ """Route to documents through compiled ``page_index`` entity rows.++ Complements the chunk leg rather than replacing it: chunks are verbatim text+ and win on exact matches (numbers, proper nouns), while ``fact`` /+ ``conclusion`` rows are atomic propositions that win on concept matches the+ surface form misses. Each covers the other's failure mode.++ Compiled rows carry no ``available_int``, which is the project's marker for+ "visible to the generic retriever" (not a deletion flag), so they are absent+ from the chunk index and have to be read through the store seam directly --+ the same seam ``dataset_nav`` and wiki use.++ Returns ``ok=False`` only on a store failure; an empty result set is a+ legitimate outcome (a KB without page_index products) and is reported+ ``ok=True`` with ``total=0`` so the caller can fall back.+ """+ pack = _compiled_index_or_none(kb.tenant_id, kb.id) if kb is not None else None+ if pack is None:+ return True, {"mode": "navigation_tree", "total": 0, "items": []}+ from common.doc_store.doc_store_base import OrderByExpr+ from rag.nlp.search import MatchTextExpr++ index_nm, _ = pack++ pool = _NAV_COMPILED_POOL+ condition = {+ "compile_kwd": ["page_index"],+ "knowledge_graph_kwd": ["entity"],+ # Exclude the KB-wide merged rows written by the Build button.+ "scope_kwd": ["doc"],+ }+ if doc_scope:+ condition["doc_id"] = [str(d) for d in doc_scope if str(d).strip()]++ fields = ["content_with_weight", "source_chunk_ids", "doc_id", "name_kwd"]+ try:+ # Compiled rows tokenize their description into content_ltks, so a+ # keyword match is a real fallback when no embedding model is configured+ # or embedding fails -- the leg degrades instead of dropping out.+ exprs = []+ if embd_mdl:+ try:+ exprs.append(+ await settings.retriever.get_vector(+ query,+ embd_mdl,+ top_k=pool,+ # HNSW ef_search -- must be >= top_k or the ANN search+ # collapses. Positional passing here would hit+ # num_candidates with the similarity threshold.+ num_candidates=pool,+ similarity=_NAV_COMPILED_SIMILARITY,+ )+ )+ except Exception:+ logging.exception("dataset_nav: compiled vector build failed for kb=%s", kb.id)+ if not exprs:+ exprs.append(MatchTextExpr(["content_ltks", "content_sm_ltks"], query, pool))+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ select_fields=fields,+ highlight_fields=[],+ condition=condition,+ match_expressions=exprs,+ order_by=OrderByExpr(),+ offset=0,+ limit=pool,+ index_names=index_nm,+ knowledgebase_ids=[kb.id],+ )+ field_map = settings.docStoreConn.get_fields(res, fields)+ except Exception:+ logging.exception("dataset_nav: compiled-agg retrieval failed for kb=%s", kb.id)+ return False, {"error": "compiled retrieval failed", "code": RetCode.SERVER_ERROR}++ buckets = _nav_bucket_compiled_rows(field_map or {})+ ranked = _nav_rank_compiled_buckets(buckets, top_k)+ if not ranked:+ return True, {"mode": "navigation_tree", "total": 0, "items": []}++ # Compile rows can outlive a deleted document -- the doc-existence check in+ # retrieval() is a fallback, not the primary delete mechanism, and this leg+ # reads the store directly, so it has to apply the check itself.+ alive = await _nav_existing_doc_ids([doc_id for doc_id, _ in ranked])+ ranked = [(d, e) for d, e in ranked if d in alive]+ if not ranked:+ return True, {"mode": "navigation_tree", "total": 0, "items": []}++ summaries = await _nav_doc_summaries(kb, [doc_id for doc_id, _ in ranked])+ # Label each document with the TOC sections that scored for it, so the caller+ # gets an entry point into the document instead of starting+ # navigate_structure from scratch. Purely additive: it never prunes.+ fact_names_by_doc = buckets.get("fact_names") or {}+ bridge = await _nav_bridge_sections(kb, [n for names in fact_names_by_doc.values() for n in names], doc_scope=doc_scope)+ items = [+ {+ "doc_id": doc_id,+ "score": round(entry["best"], 4),+ "_nav": {+ "doc_id": doc_id,+ "description": summaries.get(doc_id, ""),+ "matched_sections": _nav_matched_sections(bridge, fact_names_by_doc.get(doc_id) or []),+ },+ # Report the value ranking actually used. Recomputing DocScore here+ # would mix the title and fact hit counts and disagree with ordering.+ "_agg": {"fused": round(entry["score"], 4), "hits": entry["hits"], "legs": sorted(entry["legs"])},+ }+ for doc_id, entry in ranked+ ]+ return True, {"mode": "navigation_tree", "total": len(items), "items": items}+++async def _search_layers_fusion(tenant_id, dataset_id, query, top_k, embd_mdl, kb=None, *, compiled=None, doc_scope=None):+ """Combine the chunk leg and the compiled leg into one routing decision.++ The two legs measure different things and are not directly comparable: chunk+ ``_score`` is Infinity's normalized weighted sum, while the compiled leg is a+ raw cosine. Each leg is therefore normalized by its own maximum before+ being combined, so neither leg's scale can dominate the other.+ """+ ok, chunk_payload = await _search_layers_chunk_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)+ if not ok:+ return ok, chunk_payload++ compiled_items = (compiled or {}).get("items") or []+ if not compiled_items:+ # No page_index products (or a raptor KB): the chunk leg alone decides.+ # Tag the items so callers see one shape whichever path produced them.+ for item in chunk_payload.get("items") or []:+ item.setdefault("_agg", {}).setdefault("legs", ["chunk"])+ return ok, chunk_payload++ legs = [("chunk", chunk_payload.get("items") or [], _NAV_FUSION_WEIGHTS["chunk"]), ("compiled", compiled_items, 1.0)]+ fused = _nav_fuse_legs(legs, top_k)++ summaries = await _nav_doc_summaries(kb, [doc_id for doc_id, _ in fused])+ items = [+ {+ "doc_id": doc_id,+ # Reported score stays on the 0..1 scale callers threshold on, so it+ # is taken from the highest-scoring leg rather than the fused value.+ "score": round(info["best"], 4),+ "_nav": {"doc_id": doc_id, "description": summaries.get(doc_id, "")},+ "_agg": {"fused": round(info["fused"], 4), "legs": sorted(info["legs"])},+ }+ for doc_id, info in fused+ ]+ return True, {"mode": "navigation_tree", "total": len(items), "items": items}+++def _nav_fuse_legs(legs: list[tuple[str, list, float]], top_k) -> list[tuple[str, dict]]:+ """Fuse per-document leg rankings, each normalized by its own maximum."""+ normalized = []+ for name, items, weight in legs:+ peak = max((float(i.get("score") or 0.0) for i in items), default=0.0)+ if peak <= 0:+ continue+ normalized.append((name, [(i.get("doc_id"), float(i.get("score") or 0.0) / peak * weight) for i in items], weight))++ merged: dict[str, dict] = {}+ for name, entries, _ in normalized:+ for doc_id, value in entries:+ if not doc_id:+ continue+ info = merged.get(doc_id)+ if info is None:+ merged[doc_id] = {"fused": value, "best": value, "legs": {name}}+ continue+ info["fused"] += value+ info["legs"].add(name)+ if value > info["best"]:+ info["best"] = value++ ranked = sorted(merged.items(), key=lambda kv: kv[1]["fused"], reverse=True)+ if top_k is not None and top_k > 0:+ ranked = ranked[:top_k]+ return ranked+++def _nav_bucket_compiled_rows(field_map: dict) -> dict[str, dict]:+ """Split fetched compiled rows into a title bucket and a fact bucket.++ The row ``type`` lives inside the ``content_with_weight`` payload —+ ``entity_type_kwd`` is a graphrag field and is not written by the+ compilation path — so rows are binned after the fetch.++ Fact names are kept per document, because the section bridge has to label a+ document with the sections that scored for *it* — a flat list would attach+ one document's sections to another.+ """+ title_bucket: dict[str, dict] = {}+ fact_bucket: dict[str, dict] = {}+ fact_names: dict[str, list[str]] = {}+ for row in field_map.values():+ doc_id = str(row.get("doc_id") or "").strip()+ if not doc_id:+ continue+ try:+ payload = json.loads(str(row.get("content_with_weight") or "{}"))+ except (TypeError, ValueError):+ continue+ if not isinstance(payload, dict):+ continue+ rtype = str(payload.get("type") or "").strip().lower()+ if rtype in _NAV_COMPILED_TITLE_TYPES:+ bucket = title_bucket+ elif rtype in _NAV_COMPILED_FACT_TYPES:+ bucket = fact_bucket+ # Kept for the section bridge, which labels a hit fact with its+ # owning title. The raw fetch order is the store's relevance order,+ # which is the right priority for choosing which sections to show.+ name = str(payload.get("name") or "").strip()+ if name:+ fact_names.setdefault(doc_id, []).append(name)+ else:+ continue+ score = float(row.get("similarity") or row.get("_score") or 0.0)+ entry = bucket.get(doc_id)+ if entry is None:+ bucket[doc_id] = {"total": score, "best": score, "hits": 1}+ continue+ entry["total"] += score+ entry["hits"] += 1+ if score > entry["best"]:+ entry["best"] = score+ return {"title": title_bucket, "fact": fact_bucket, "fact_names": fact_names}+++def _nav_rank_compiled_buckets(buckets: dict, top_k) -> list[tuple[str, dict]]:+ """Fuse the title and fact buckets into one per-document ranking.++ A document hit by both legs is stronger than one hit by either alone, so the+ scores are summed rather than maxed — but each bucket is ranked by its own+ DocScore first, which is what keeps facts from swamping titles.+ """+ merged: dict[str, dict] = {}+ for leg, bucket in (("title", buckets.get("title") or {}), ("fact", buckets.get("fact") or {})):+ weight = _NAV_FUSION_WEIGHTS.get(leg, 1.0)+ for doc_id, entry in bucket.items():+ contribution = _nav_doc_score(entry) * weight+ cur = merged.get(doc_id)+ if cur is None:+ merged[doc_id] = {+ "score": contribution,+ "best": entry["best"],+ "total": entry["total"],+ "hits": entry["hits"],+ "legs": {leg},+ }+ continue+ cur["score"] += contribution+ cur["total"] += entry["total"]+ cur["hits"] += entry["hits"]+ cur["legs"].add(leg)+ if entry["best"] > cur["best"]:+ cur["best"] = entry["best"]++ ranked = sorted(merged.items(), key=lambda kv: kv[1]["score"], reverse=True)+ if top_k is not None and top_k > 0:+ ranked = ranked[:top_k]+ return ranked+++def _nav_matched_sections(bridge: dict[str, str], fact_names: list[str]) -> list[str]:+ """Titles owning this document's hit facts, in hit order and de-duplicated."""+ sections: list[str] = []+ for name in fact_names:+ title = bridge.get(str(name).strip().lower())+ if title and title not in sections:+ sections.append(title)+ return sections+++async def _nav_bridge_sections(kb, fact_names: list[str], doc_scope=None) -> dict[str, str]:+ """Map each hit fact to the title that includes it.++ Routing then answers *which document* and *where in it to look first*:+ the caller gets a per-document entry point instead of starting+ ``navigate_structure`` from scratch.++ The TOC is used here purely as a **label**. It never prunes the candidate+ set — that is the top-down routing failure mode. The lookup happens after+ the documents are already chosen.++ Returns ``{fact_name: title_name}``; facts without an owning title are+ omitted.+ """+ if not fact_names or kb is None:+ return {}+ pack = _compiled_index_or_none(kb.tenant_id, kb.id)+ if pack is None:+ return {}+ index_nm, _ = pack++ from common.doc_store.doc_store_base import OrderByExpr++ names = [str(n).strip().lower() for n in fact_names if str(n).strip()]+ # Cap the terms query: a wide IN list is slow and the bridge only needs to+ # label the top hits, not every candidate.+ names = list(dict.fromkeys(names))[:_NAV_BRIDGE_NAMES_LIMIT]++ condition = {+ "compile_kwd": ["page_index"],+ "knowledge_graph_kwd": ["relation"],+ "scope_kwd": ["doc"],+ "to_entity_kwd": names,+ }+ if doc_scope:+ condition["doc_id"] = [str(d) for d in doc_scope if str(d).strip()]++ fields = ["content_with_weight", "from_entity_kwd", "to_entity_kwd", "doc_id"]+ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ select_fields=fields,+ highlight_fields=[],+ condition=condition,+ match_expressions=[],+ order_by=OrderByExpr(),+ offset=0,+ limit=len(names) * _NAV_BRIDGE_FANOUT,+ index_names=index_nm,+ knowledgebase_ids=[kb.id],+ )+ field_map = settings.docStoreConn.get_fields(res, fields)+ except Exception:+ logging.exception("dataset_nav: section bridge lookup failed for kb=%s", kb.id)+ return {}++ bridge: dict[str, str] = {}+ for row in (field_map or {}).values():+ to_name = str(row.get("to_entity_kwd") or "").strip().lower()+ title = str(row.get("from_entity_kwd") or "").strip()+ if not to_name or not title or to_name in bridge:+ continue+ try:+ payload = json.loads(str(row.get("content_with_weight") or "{}"))+ except (TypeError, ValueError):+ continue+ # Only the structural edge counts; semantic relations would mislabel.+ if isinstance(payload, dict) and str(payload.get("type") or "").strip().lower() == _NAV_INCLUDE_RELATION:+ bridge[to_name] = title+ return bridge+++async def _nav_existing_doc_ids(doc_ids: list[str]) -> set[str]:+ """Keep only documents that still exist.++ Compiled rows are not necessarily cleaned up when a document is deleted, so+ this leg has to repeat the existence check that ``retrieval`` applies to+ chunk results.+ """+ if not doc_ids:+ return set()+ try:+ return await settings.retriever._existing_doc_ids(list(doc_ids))+ except Exception:+ logging.exception("dataset_nav: doc existence check failed")+ return set(doc_ids)+++def _nav_aggregate_chunks(chunks: list) -> dict[str, dict]:+ """Roll chunk hits up per document: total, best and hit count per doc."""+ agg: dict[str, dict] = {}+ for c in chunks:+ doc_id = str(c.get("doc_id") or "").strip()+ if not doc_id:+ continue+ score = float(c.get("similarity") or c.get("score") or 0.0)+ entry = agg.get(doc_id)+ if entry is None:+ agg[doc_id] = {"total": score, "best": score, "hits": 1}+ continue+ entry["total"] += score+ entry["hits"] += 1+ if score > entry["best"]:+ entry["best"] = score+ return agg+++def _nav_doc_score(entry: dict) -> float:+ """PageIndex DocScore: hit scores summed, damped by hit count.++ The sum rewards a document matching several chunks; the square-root+ denominator damps it so a long document cannot win on volume alone.+ """+ return entry["total"] / math.sqrt(entry["hits"] + 1)+++async def _nav_doc_summaries(kb, doc_ids):+ """Batch-load the nav_doc ``description`` of ``doc_ids`` in one store query.++ Chunk-level routing finds a document without reading any nav row, but the+ caller still needs a label for it. The nav_doc row already holds the+ document's overall summary, so read them in a single terms query rather+ than making the caller load the document for a label.+ """+ if not doc_ids or kb is None:+ return {}+ pack = _compiled_index_or_none(kb.tenant_id, kb.id)+ if pack is None:+ return {}+ index_nm, _ = pack++ from common.doc_store.doc_store_base import OrderByExpr++ # Single terms query shared by the whole batch; _nav_search would repeat the+ # access check and KB load the caller already performed.+ limit = max(1, min(len(doc_ids), 2000))+ condition = {+ "compile_kwd": [_NAV_COMPILE_KWD],+ "type_kwd": ["nav_doc"],+ "doc_id": [str(d) for d in doc_ids][:limit],+ }+ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ select_fields=_NAV_FIELDS,+ highlight_fields=[],+ condition=condition,+ match_expressions=[],+ order_by=OrderByExpr(),+ offset=0,+ limit=limit,+ index_names=index_nm,+ knowledgebase_ids=[kb.id],+ )+ field_map = settings.docStoreConn.get_fields(res, _NAV_FIELDS)+ except Exception:+ logging.exception("dataset_nav: nav_doc summary lookup failed for kb=%s", kb.id)+ return {}+ summaries = {}+ for row in (field_map or {}).values():+ item = _nav_item(row)+ doc_id = str(item.get("doc_id") or "").strip()+ if doc_id:+ summaries[doc_id] = str(item.get("description") or "").strip()+ return summaries++async def _nav_search_result(tenant_id, dataset_id, query, top_k, embd_mdl, search_fn, **kwargs):doc_scope = kwargs.pop("doc_scope", None)results = await search_fn(@@ -3979,11 +4644,14 @@ async def _search_layers_chunks(tenant_id, dataset_id, query, top_k, embd_mdl, ktenant_ids = [tenant_id]kwargs = {}- if top_k is not None:- kwargs["knn_top_k"] = top_kif doc_scope:kwargs["doc_ids"] = [str(d) for d in doc_scope if str(d).strip()]+ # Here page_size IS the result set: chunks are returned as evidence, not+ # aggregated into documents. Keep it at the historical fetch size — raising+ # it to the chunk_agg pool inflated the evidence pool ~7x (25 -> 191 chunks+ # per question), which pushed every downstream LLM prompt and blew the+ # 180s per-question research budget.fetch_k = max(top_k, 10) * 3 if top_k is not None else 1024try:ranks = await settings.retriever.retrieval(@@ -3994,7 +4662,16 @@ async def _search_layers_chunks(tenant_id, dataset_id, query, top_k, embd_mdl, k1,fetch_k,0.0,- 0.3,+ _NAV_CHUNK_AGG_VEC_WEIGHT,+ # The candidate pool only has to cover what is returned; it used to+ # sit at the 64 default while page_size grew with top_k, which+ # raises once page * page_size exceeds it.+ rerank_candidates_count=max(_NAV_CHUNK_AGG_POOL, fetch_k),+ # knn_top_k is left at its default: narrowing it to top_k caps how+ # many neighbours the vector leg feeds into fusion.+ # Chunk mode reports raw chunks; compiled rows are served by their+ # own tools, so they must not be attributed to a document here.+ must_not={"exists": "compile_kwd"} if _NAV_CHUNK_AGG_EXCLUDE_COMPILED else None,**kwargs,)except Exception:@@ -4022,10 +4699,14 @@ async def _search_layers_all(tenant_id, dataset_id, query, top_k, embd_mdl, kb,"""Run all modes and return the union of doc_ids, with best score per doc."""import asyncio as _asyncio+ # With _NAV_TREE_ROUTER="chunk_agg" the navigation_tree leg already reads the+ # chunk index, so it overlaps the chunk leg below. The union still dedups by+ # doc_id and keeps the best score, so the overlap costs one extra retrieval+ # without affecting the result — mode="all" is for comparison, not production.result_lists = await _asyncio.gather(_search_layers_nav_docs(tenant_id, dataset_id, query, top_k, embd_mdl, search_fn, doc_scope=doc_scope),_search_layers_nav_clusters(tenant_id, dataset_id, query, top_k, embd_mdl, search_fn, doc_scope=doc_scope),- _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, doc_scope=doc_scope),+ _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope),_search_layers_chunks(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope),return_exceptions=True,)diff --git a/rag/advanced_rag/harness/tools/compiled_expansion.py b/rag/advanced_rag/harness/tools/compiled_expansion.pyindex f4f1bbd9f3..e252fc8bb7 100644--- a/rag/advanced_rag/harness/tools/compiled_expansion.py+++ b/rag/advanced_rag/harness/tools/compiled_expansion.py@@ -22,6 +22,31 @@ from rag.advanced_rag.harness.tools.navigation import _kg_scopes_LOG = logging.getLogger(__name__)+# HNSW ef_search floor. Must be >= the requested top_n or the ANN search is+# bounded by the candidate list instead of by relevance.+_VECTOR_NUM_CANDIDATES = 256+_VECTOR_SIMILARITY = 0.1+++async def _dense_expr(text: str, embd_mdl, top_n: int):+ """Build the dense match expression for compiled-row search.++ ``get_vector``'s 4th positional argument is ``num_candidates`` (HNSW+ ef_search), not the similarity threshold — passing the 0.1 threshold+ positionally collapses the candidate list to a fraction of one entry and+ silently destroys recall. Both are passed by name here.+ """+ from common import settings++ return await settings.retriever.get_vector(+ text,+ embd_mdl,+ top_k=top_n,+ num_candidates=max(top_n, _VECTOR_NUM_CANDIDATES),+ similarity=_VECTOR_SIMILARITY,+ )++# ─── Compiled product expansion (zero-LLM, used by hybrid_search with use_compiled=True) ───@@ -156,7 +181,7 @@ async def _search_compiled_rows(embd_mdl = getattr(tools, "embed_mdl", None)if embd_mdl:try:- exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_n, 0.1))+ exprs.append(await _dense_expr(text, embd_mdl, top_n))except Exception:_LOG.exception("[Compiled expand] vector build failed; using keyword match")if not exprs:@@ -400,7 +425,7 @@ async def _search_synthesis_pages(embd_mdl = getattr(tools, "embed_mdl", None)if embd_mdl:try:- exprs.append(await settings.retriever.get_vector(text, embd_mdl, top_n, 0.1))+ exprs.append(await _dense_expr(text, embd_mdl, top_n))except Exception:_LOG.exception("[Wiki expand] vector build failed; using keyword match")if not exprs:diff --git a/rag/advanced_rag/harness/tools/navigation.py b/rag/advanced_rag/harness/tools/navigation.pyindex b6dddf36e9..cc44e3e3bc 100644--- a/rag/advanced_rag/harness/tools/navigation.py+++ b/rag/advanced_rag/harness/tools/navigation.py@@ -659,14 +659,20 @@ _NAV_MIN_DOC_SCORE = 0.2 # drop docs below this scoreasync def _nav_search_titled(tools, topic: str, keywords: str = "", doc_scope: list[str] | None = None) -> list[tuple[str, str]]:"""Descend the compiled nav TREE and return ``(doc_id, summary)`` pairs.- Uses ``mode="navigation_tree"`` (BFS beam descent: root cluster → sub-clusters- → nav_doc leaves) rather than a flat sweep over every nav_doc row. Routing- through the hierarchy narrows the field level by level, which is what makes it- accurate on large datasets where many document summaries look alike in a flat- comparison.-- HYBRID descent (vector + BM25): both the KNN legs and the ``_text_score``- gates stay armed. A pure-vector descent was tried — it dropped the text legs+ Uses ``mode="navigation_tree"``, which currently resolves to a FLAT hybrid+ sweep over every ``nav_doc`` row (``_NAV_TREE_ROUTER = "nav_doc"``) rather+ than a descent through the cluster levels. The strategy lives server-side, so+ this function is agnostic to which one is active.++ Flat routing is deliberate. A BFS beam descent narrows the field level by+ level, and a wrong pick at the first level cannot be undone — the correct+ branch was already pruned and never reaches the comparison. Nav rows embed+ ``graph_text`` (the document's full entity/relation listing), so they are+ discriminating enough to be compared directly without the cluster levels+ narrowing them for us.++ HYBRID sweep (vector + BM25): both the KNN legs and the ``_text_score``+ gates stay armed. A pure-vector routing was tried — it dropped the text legsand the ``_text_score > 0`` gate, on the theory that routing cares aboutsemantic matches a summary never spells out. It regressed routing badly:scores collapsed onto the dense leg alone, the descent fell below@@ -742,9 +748,10 @@ async def dataset_navigation_search(tools, topic: str, keywords: str = "", doc_sdescending the dataset's compiled navigation tree.Routes through :func:`_nav_search_titled`, which runs- ``search_dataset_layers`` with ``mode="navigation_tree"``: a hybrid BFS beam- descent (vector + BM25) from the root clusters down to the ``nav_doc``- leaves, with the ``_text_score`` gates armed.+ ``search_dataset_layers`` with ``mode="navigation_tree"``: a hybrid+ (vector + BM25) sweep over the ``nav_doc`` rows, with the ``_text_score``+ gates armed. Which router runs is decided server-side by+ ``_NAV_TREE_ROUTER``, so this function does not depend on the strategy.Returns the routed ``doc_id`` list (capped at ``_NAV_SEARCH_MAX_DOCS``), or``[]`` when no question/keywords are given or the search returns nothing.@@ -963,8 +970,12 @@ async def _navigate_structure_impl(structure of the requested kind, an empty <doc/> is returned (no error) — fallback to list_chunks / search_chunks.- This tool performs NO chat-LLM calls: it only reads the compiled structure- rows from the doc store.+ Cost: the structure rows themselves are read from the doc store. When the+ document's nodes carry their own embeddings (page_index), the whole TOC is+ additionally handed to the chat model once, so it can pick the relevant+ sections — see ``_STRUCT_TOC_LLM_SELECT``. That pass is a single call per+ document, and an empty or failed result falls through to local scoring, so+ the tool still works without a chat model.:param query: REQUIRED: the question/topic (used to route to a document when``doc_id`` is not given).@@ -1056,6 +1067,33 @@ _STRUCT_VEC_BEAM_RATIO = 0.5 # vector beam: drop nodes below best*this similari_STRUCT_MAX_NODES = 10 # cap on nodes rendered in the outline._STRUCT_MAX_CHUNKS = 4 # cap on chunk snippets returned in the outline._STRUCT_DESC_SNIPPET = 180 # cap on a node's description snippet in the outline.+# Chunks recalled per document when a tree cannot score its own nodes and the+# chunk index has to choose instead. Wide enough that several nodes can be+# covered, so the outline is not pinned to a single branch.+_STRUCT_RECALL_TOP_N = 24+# Snippets rendered on that path. Higher than _STRUCT_MAX_CHUNKS because there+# the chunks *are* the answer's material — the outline is only the annotation.+_STRUCT_MAX_CHUNK_HITS = 8+# Document-in-LLM selection (PageIndex tree-search / VecTree-RAG Layer 1): the+# whole TOC is handed to the model at once and it returns every node it thinks+# is relevant. This is deliberately NOT a top-down descent — the model sees all+# levels simultaneously, so a wrong pick cannot prune a branch, which is the+# failure mode beam descent has (SCT measures it as routing accuracy 20.2% vs+# 39.3% for flat selection).+# OFF by default. Correct in principle — the model sees every level at once, so+# nothing is pruned — but it costs one chat call per document, and the drill rung+# calls navigate_structure once per routed document (8 docs = 8 extra calls per+# question). Measured 2026-09-02: with it on, per-question LLM calls went 9-17+# -> 37 and the 180s research budget was exhausted on 15 of 16 questions+# (baseline: 1 of 28). Turn it on only when the LLM has headroom; prefer+# restricting it to the top-ranked document first.+_STRUCT_TOC_LLM_SELECT = False+# Budget for the serialized TOC. VecTree-RAG quotes 200-500 tokens per document+# for section summaries; facts are included here too, so this allows more.+# _ask_nav_select truncates past this via message_fit_in regardless.+_STRUCT_TOC_MAX_NODES = 120+_STRUCT_TOC_DESC_SNIPPET = 120+_STRUCT_TOC_MAX_DEPTH = 6# Snippet length / per-doc cap for the related chunks appended after a# search_chunks hit (from the document's compiled structure, not a full read)._STRUCT_RELATED_SNIPPET_CHARS = 300@@ -1207,7 +1245,24 @@ async def _read_structures(tools_slot, query: str, doc_ids: list[str], kinds: serelations = structure.get("relations") or []except Exception:_LOG.exception("[navigate_structure] structure load failed for doc=%s", doc_id)- outline, stats, chunk_paths = await _render_toc_drilldown(query, qvec, entities, relations)++ # Two selection strategies, chosen by whether the tree can score itself.+ hits: list[tuple[str, float]] = []+ selected: list[str] = []+ if entities and _has_distinct_node_vectors(entities):+ # page_index / per-node embeddings: hand the whole TOC to the model+ # at once. It sees every level simultaneously, so nothing is pruned+ # by an early bad pick. Empty result just means "caller's problem",+ # so fall through to the beam only when the model is unavailable.+ if _STRUCT_TOC_LLM_SELECT:+ selected = await _select_toc_nodes(tools_slot, query, entities, relations)+ else:+ # RAPTOR blob path: every node shares the blob's one embedding, so+ # beam descent would select arbitrarily. Let the chunk index choose+ # and keep the tree for labelling where the chosen chunks sit.+ hits = await _recall_chunk_ids_in_doc(tools_slot, query, doc_id, _STRUCT_RECALL_TOP_N)++ outline, stats, chunk_paths = await _render_toc_drilldown(query, qvec, entities, relations, chunk_hits=hits or None, selected=selected or None)out.append({"doc_id": doc_id,@@ -1288,6 +1343,177 @@ def _cosine(a, b):return 0.0+def _vecs_equal(a, b) -> bool:+ """Whether two node embeddings are the same vector."""+ try:+ if a is b:+ return True+ la, lb = list(a), list(b)+ if len(la) != len(lb):+ return False+ # Distinct embeddings diverge long before the tail; a prefix compare is+ # enough and keeps this cheap on 1024-dim vectors.+ return all(x == y for x, y in zip(la[:16], lb[:16]))+ except Exception:+ return False+++def _has_distinct_node_vectors(entities: list[dict]) -> bool:+ """Whether nodes carry per-node embeddings rather than one shared vector.++ The RAPTOR/tree main path persists a single graph blob, so every node+ inherits that blob's one vector (see ``_load_entities_with_vectors``) and+ beam scoring degenerates: every node gets the same cosine, the top-K is+ whatever the row order happened to be, and the drill carries no signal.+ page_index and the pipeline Compiler write one row per entity with its own+ embedding, so their trees do discriminate.+ """+ first = None+ for e in entities:+ v = e.get("_vec")+ if v is None:+ continue+ if first is None:+ first = v+ elif not _vecs_equal(first, v):+ return True+ # Either no node has a vector, or they all share one — neither discriminates.+ return False+++async def _recall_chunk_ids_in_doc(tools, query: str, doc_id: str, top_n: int) -> list[tuple[str, float]]:+ """Hybrid chunk retrieval confined to one document, as ``[(chunk_id, score)]``.++ Used when a compiled tree cannot score its own nodes: the chunk index then+ decides *which* nodes matter, and the tree is left to label where they sit.+ """+ if not query or not doc_id:+ return []+ from common import settings++ target_ids = getattr(tools, "kb_ids", None) or []+ if not target_ids:+ return []+ embd_mdl = getattr(tools, "embed_mdl", None)+ try:+ kbinfos = await settings.retriever.retrieval(+ query,+ embd_mdl,+ getattr(tools, "tenant_ids", None),+ target_ids,+ 1,+ top_n,+ 0.0,+ vector_similarity_weight=0.3 if embd_mdl else 0,+ doc_ids=[str(doc_id).strip()],+ highlight=False,+ # RAPTOR cluster nodes live in the chunk index (rechunk_kwd=tree, no+ # compile_kwd), so they survive this filter and stay retrievable —+ # which is the point, since they are the structure.+ must_not={"exists": "compile_kwd"},+ )+ except Exception:+ _LOG.exception("[navigate_structure] chunk recall failed for doc=%s", doc_id)+ return []+ hits: list[tuple[str, float]] = []+ for c in kbinfos.get("chunks", []):+ cid = str(c.get("chunk_id") or c.get("id") or "").strip()+ if cid:+ hits.append((cid, float(c.get("similarity") or 0.0)))+ return hits+++def _build_toc_items(entities: list[dict], relations: list[dict]) -> list[dict]:+ """Serialize the whole TOC, indented by depth, for one-shot LLM selection.++ Indentation is the only structure the model gets: it reads the list the way a+ reader reads a table of contents. Every node carries its real name through+ ``_node`` so the selection can be mapped back without string matching.++ Orphans (nodes no relation attaches to a root) are appended last at depth 0 —+ dropping them would silently hide content from the model.+ """+ by_name, children, parents, roots = _build_toc_tree(entities, relations)+ if not by_name:+ return []++ items: list[dict] = []+ seen: set[str] = set()++ def _walk(name: str, depth: int) -> None:+ if name in seen or depth > _STRUCT_TOC_MAX_DEPTH:+ return+ seen.add(name)+ e = by_name.get(name)+ if not e:+ return+ desc = str(e.get("description") or "").strip().replace("\n", " ")+ items.append(+ {+ "_node": name,+ "name": " " * depth + name,+ "description": _snippet(desc, _STRUCT_TOC_DESC_SNIPPET),+ }+ )+ for child in children.get(name, []):+ _walk(child, depth + 1)++ for r in roots:+ _walk(r, 0)+ for name in by_name:+ if name not in seen:+ _walk(name, 0)+ return items+++async def _select_toc_nodes(tools, query: str, entities: list[dict], relations: list[dict]) -> list[str]:+ """Let the model pick TOC nodes in one shot. Returns node names (''-filtered).++ Returns ``[]`` on any failure so the caller can fall back — never raises.+ """+ if not entities:+ return []+ try:+ items = _build_toc_items(entities, relations)+ except Exception:+ _LOG.exception("[navigate_structure] TOC serialization failed")+ return []+ if not items:+ return []+ try:+ picked = await _ask_nav_select(tools, query, items, "sections", _STRUCT_TOC_MAX_NODES)+ except Exception:+ # _ask_nav_select catches model failures, but serialization/other paths can+ # still raise; the caller must be able to fall through to local scoring.+ _LOG.exception("[navigate_structure] TOC model selection failed")+ return []+ names = []+ for it in picked or []:+ name = str(it.get("_node") or "").strip()+ if name:+ names.append(name)+ return names+++def _nodes_covering_chunks(entities: list[dict], chunk_ids: set[str]) -> list[str]:+ """Names of nodes whose ``source_chunk_ids`` cover any of ``chunk_ids``.++ Used only to give retrieved chunks a structural home. A chunk that no node+ covers is still returned — the tree annotates, it does not select.+ """+ if not chunk_ids:+ return []+ names: list[str] = []+ for e in entities:+ name = (e.get("name") or "").strip()+ if not name:+ continue+ ids = {str(c) for c in (e.get("source_chunk_ids") or []) if isinstance(c, str) and c}+ if ids & chunk_ids:+ names.append(name)+ return names++def _node_score(qvec, query_terms, entity: dict) -> float:"""Score a TOC node: cosine similarity (if the node has an embedding and wehave a query vector), else keyword-overlap. Higher is better."""@@ -1369,13 +1595,34 @@ def _drill_kept_nodes(query_terms: list[str], qvec, entities: list[dict], relatireturn kept_nodes, parents, kept_names, best_overall-async def _render_toc_drilldown(query: str, qvec, entities: list[dict], relations: list[dict]) -> tuple[str, dict, dict]:- """Drill down the pre-built TOC hierarchy toward the query and render an outline.-- Uses VECTOR BEAM descent when the nodes carry embeddings (``_vec``) and a query- vector is available: from the roots, at each level keep the top-K nodes by- cosine similarity to the query and descend to their children. Falls back to- keyword-overlap when no vectors are available.+async def _render_toc_drilldown(+ query: str,+ qvec,+ entities: list[dict],+ relations: list[dict],+ chunk_hits: list[tuple[str, float]] | None = None,+ selected: list[str] | None = None,+) -> tuple[str, dict, dict]:+ """Render a query-focused outline of one document's compiled structure.++ Three ways to choose what to show, in the order the caller tries them:++ ``selected`` — node names already chosen elsewhere (the whole-TOC LLM pass,+ see ``_select_toc_nodes``). Every level was visible to the model at once,+ so nothing is pruned; chunks come from the chosen nodes' ``source_ids``.++ ``chunk_hits`` — ``[(chunk_id, score)]`` from chunk retrieval, used when the+ tree cannot score its own nodes (RAPTOR blob path). Selection belongs to+ the chunk index outright: the chunks are returned **as retrieved**, not+ filtered back through the nodes, and the tree is left to do the only job+ it is still needed for — labelling where each chunk sits. Round-tripping+ the selection through the nodes would silently drop every chunk no node+ happens to cover, which for a clustered tree is most of them.++ neither — VECTOR BEAM descent: from the roots, at each level keep the top-K+ nodes by cosine similarity to the query and descend to their children,+ falling back to keyword overlap when no vectors are available. Kept as+ the fallback when the model pass returns nothing.Returns lines like:- Paris Demographics (tree_node): <desc> [chunks: c1,c2]@@ -1388,11 +1635,48 @@ async def _render_toc_drilldown(query: str, qvec, entities: list[dict], relationstructure context onto retrieved evidence without re-deriving the tree."""query_terms = [t for t in re.findall(r"[A-Za-z0-9_]{2,}", (query or "").lower()) if len(t) >= 2]- if not query_terms and qvec is None:+ # Pinned nodes were chosen elsewhere (chunk retrieval), so they stand even+ # when the query yields no usable terms — bailing out here would discard a+ # perfectly good selection just because the query is short.+ if not chunk_hits and not selected and not query_terms and qvec is None:return _render_outline(entities[:_STRUCT_MAX_NODES], relations[:_STRUCT_MAX_NODES]), _outline_stats(entities), {}- kept_nodes, parents, kept_names, best = _drill_kept_nodes(query_terms, qvec, entities, relations)- if not kept_nodes:+ by_name, _children, parents, _roots = _build_toc_tree(entities, relations)++ def _with_ancestors(names: set[str]) -> set[str]:+ """Pull in ancestors so a path renders as root -> ... -> node."""+ for name in list(names):+ cur = parents.get(name)+ guard = 0+ while cur and cur not in names and guard < _STRUCT_MAX_DEPTH:+ names.add(cur)+ cur = parents.get(cur)+ guard += 1+ return names++ selected_ids: list[str] = []+ if selected:+ kept_names = _with_ancestors({n for n in selected if n in by_name})+ kept_nodes = [by_name[n] for n in kept_names if n in by_name]+ # The model picked nodes, so the chunks are theirs to supply.+ # No score: the model chooses, it does not rank (see stats["selector"]).+ best = 0.0+ selector = "llm_toc"+ elif chunk_hits:+ # Keep the retrieval's own ordering — it is the relevance order.+ seen: set[str] = set()+ for cid, _score in chunk_hits:+ if cid and cid not in seen:+ seen.add(cid)+ selected_ids.append(cid)+ kept_names = _with_ancestors({n for n in _nodes_covering_chunks(entities, set(selected_ids)) if n in by_name})+ kept_nodes = [by_name[n] for n in kept_names if n in by_name]+ best = max((float(s) for _c, s in chunk_hits), default=0.0)+ selector = "chunk_retrieval"+ else:+ kept_nodes, parents, kept_names, best = _drill_kept_nodes(query_terms, qvec, entities, relations)+ selector = "beam"+ if not kept_nodes and not selected_ids:return _render_outline(entities[:_STRUCT_MAX_NODES], relations[:_STRUCT_MAX_NODES]), _outline_stats(entities), {}# Render the drilled nodes (indented by depth via ancestor count).@@ -1438,13 +1722,23 @@ async def _render_toc_drilldown(query: str, qvec, entities: list[dict], relationline += f" [chunks: {chunks}]"lines.append(line)- # Rank the chunks behind the drilled nodes and add short snippets.- wanted = _collect_chunk_ids(kept_nodes)+ # Chunk snippets. When chunk retrieval did the selecting, those hits are+ # returned as-is: filtering them back through the nodes would drop every+ # chunk no node happens to cover. Otherwise take the chunks behind the+ # drilled nodes.+ wanted = selected_ids or _collect_chunk_ids(kept_nodes)if wanted:chunks = await _load_chunks_for_ids(_tools_slot(), wanted)if chunks:- ranked = _rank_chunks_by_terms(chunks, [query])- for c in ranked[:_STRUCT_MAX_CHUNKS]:+ if selected_ids:+ # Retrieval order is the relevance order; re-ranking by terms+ # would discard the score that selected them.+ ranked = sorted(chunks, key=lambda c: selected_ids.index(_chunk_id(c)))+ limit = _STRUCT_MAX_CHUNK_HITS+ else:+ ranked = _rank_chunks_by_terms(chunks, [query])+ limit = _STRUCT_MAX_CHUNKS+ for c in ranked[:limit]:cid = _chunk_id(c)text = _chunk_text(c).strip()lines.append(f"- [chunk {cid}]: {_snippet(text, 300)}")@@ -1452,6 +1746,9 @@ async def _render_toc_drilldown(query: str, qvec, entities: list[dict], relation"nodes": len(kept_nodes),"chunk_ptrs": len(wanted),"top_score": round(float(best), 4),+ # Which strategy chose the nodes. top_score is only meaningful for+ # "beam" and "chunk_retrieval"; llm_toc has no comparable score.+ "selector": selector,}return "\n".join(lines), stats, chunk_pathsdiff --git a/test/unit_test/api/apps/services/test_dataset_api_service_nav.py b/test/unit_test/api/apps/services/test_dataset_api_service_nav.pyindex 12f4a74f5b..d904c0afe4 100644--- a/test/unit_test/api/apps/services/test_dataset_api_service_nav.py+++ b/test/unit_test/api/apps/services/test_dataset_api_service_nav.py@@ -17,6 +17,7 @@import importlib.utilimport json+import mathimport sysfrom enum import IntEnumfrom pathlib import Path@@ -47,13 +48,28 @@ def _stub(monkeypatch, name, **attrs):return mod-def _load_nav_module(monkeypatch, *, accessible=True, index_pack=("idx-1", None), total=0, field_map=None):+class _StubMatchTextExpr:+ """Stand-in for rag.nlp.search.MatchTextExpr (keyword-match fallback expr)."""++ def __init__(self, fields, text, top_k):+ self.fields, self.text, self.top_k = fields, text, top_k+++def _load_nav_module(monkeypatch, *, accessible=True, index_pack=("idx-1", None), total=0, field_map=None, retriever=None):+ # Stubbed before the import: importing the real rag.nlp pulls in heavy+ # third-party deps the unit tier does not install.+ _stub(monkeypatch, "rag.nlp")+ _stub(monkeypatch, "rag.nlp.search", MatchTextExpr=_StubMatchTextExpr)+doc_store = MagicMock()doc_store.search = MagicMock(return_value={})doc_store.get_fields = MagicMock(return_value=field_map or {})doc_store.get_total = MagicMock(return_value=total)doc_store.delete = MagicMock(return_value=0)+ if retriever is None:+ retriever = SimpleNamespace(retrieval=AsyncMock(return_value={"chunks": []}))+kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")knowledgebase_service = SimpleNamespace(accessible=MagicMock(return_value=accessible),@@ -73,12 +89,12 @@ def _load_nav_module(monkeypatch, *, accessible=True, index_pack=("idx-1", None)PAGERANK_FLD="pagerank",LLMType=SimpleNamespace(),FileSource=SimpleNamespace(KNOWLEDGEBASE="knowledgebase"),- RetCode=SimpleNamespace(),+ RetCode=SimpleNamespace(SERVER_ERROR=500, ARGUMENT_ERROR=400),StatusEnum=SimpleNamespace(),TaskStatus=SimpleNamespace(),ModelTypeBinary=_StubModelTypeBinary,)- _stub(monkeypatch, "common.settings", docStoreConn=doc_store, DOC_ENGINE="infinity")+ _stub(monkeypatch, "common.settings", docStoreConn=doc_store, retriever=retriever, DOC_ENGINE="infinity")_stub(monkeypatch,"api.db.db_models",@@ -234,3 +250,532 @@ async def test_delete_nav_returns_zero_when_index_missing(monkeypatch):assert ok is Trueassert payload == {"deleted": 0}doc_store.delete.assert_not_called()+++# ---------------------------------------------------------------------------+# Document routing via compiled page_index rows+# ---------------------------------------------------------------------------+++def _compiled_row(doc_id, rtype, similarity, name="n", payload=None):+ body = payload if payload is not None else {"type": rtype, "name": name, "description": "d"}+ return {"doc_id": doc_id, "similarity": similarity, "content_with_weight": json.dumps(body)}+++def test_nav_bucket_compiled_rows_splits_title_and_fact(monkeypatch):+ module, _, _ = _load_nav_module(monkeypatch)+ buckets = module._nav_bucket_compiled_rows(+ {+ "r1": _compiled_row("doc-a", "title", 0.9),+ "r2": _compiled_row("doc-a", "fact", 0.5),+ "r3": _compiled_row("doc-a", "conclusion", 0.4),+ "r4": _compiled_row("doc-b", "title", 0.7),+ "r5": _compiled_row("doc-c", "relation", 0.6), # non-entity type — dropped+ }+ )+ assert set(buckets["title"]) == {"doc-a", "doc-b"}+ assert set(buckets["fact"]) == {"doc-a"}+ # conclusion shares the fact bucket, so doc-a has two fact-class hits.+ assert buckets["fact"]["doc-a"]["hits"] == 2+ assert buckets["fact"]["doc-a"]["best"] == pytest.approx(0.5)+++def test_nav_bucket_compiled_rows_skips_unparseable_payloads(monkeypatch):+ module, _, _ = _load_nav_module(monkeypatch)+ buckets = module._nav_bucket_compiled_rows(+ {+ "r1": {"doc_id": "doc-a", "similarity": 0.9, "content_with_weight": "{not json"},+ "r2": {"doc_id": "", "similarity": 0.9, "content_with_weight": json.dumps({"type": "title"})},+ "r3": {"doc_id": "doc-b", "similarity": 0.9, "content_with_weight": json.dumps([1, 2])},+ }+ )+ assert buckets == {"title": {}, "fact": {}, "fact_names": {}}+++def test_nav_bucket_compiled_rows_tracks_fact_names_per_document(monkeypatch):+ """Fact names must stay per-document or the bridge labels the wrong document."""+ module, _, _ = _load_nav_module(monkeypatch)+ buckets = module._nav_bucket_compiled_rows(+ {+ "r1": _compiled_row("doc-a", "fact", 0.9, name="A1"),+ "r2": _compiled_row("doc-a", "fact", 0.8, name="A2"),+ "r3": _compiled_row("doc-b", "fact", 0.7, name="B1"),+ }+ )+ assert buckets["fact_names"] == {"doc-a": ["A1", "A2"], "doc-b": ["B1"]}+++def test_nav_matched_sections_dedups_and_keeps_hit_order(monkeypatch):+ module, _, _ = _load_nav_module(monkeypatch)+ bridge = {"b1": "Discussion", "a1": "Methods", "a2": "Methods"}+ out = module._nav_matched_sections(bridge, ["a1", "b1", "a2"])+ assert out == ["Methods", "Discussion"]+++def test_nav_matched_sections_omits_unbridged_facts(monkeypatch):+ module, _, _ = _load_nav_module(monkeypatch)+ assert module._nav_matched_sections({}, ["a1"]) == []+++async def test_compiled_agg_attaches_matched_sections(monkeypatch):+ """Routing must hand back a document-internal entry point, not just a doc_id."""+ module, _, doc_store = _load_nav_module(+ monkeypatch,+ field_map={"r1": _compiled_row("doc-a", "fact", 0.9, name="F1")},+ )+ doc_store.get_fields = MagicMock(return_value={"r1": _compiled_row("doc-a", "fact", 0.9, name="F1")})+ monkeypatch.setattr(module, "_nav_bridge_sections", AsyncMock(return_value={"f1": "Methods"}))+ module.settings.retriever = SimpleNamespace(+ get_vector=AsyncMock(return_value=SimpleNamespace()),+ _existing_doc_ids=AsyncMock(return_value={"doc-a"}),+ )+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")++ ok, payload = await module._search_layers_compiled_agg("tenant-1", "kb-1", "q", 5, SimpleNamespace(), kb)+ assert ok is True+ assert payload["items"][0]["_nav"]["matched_sections"] == ["Methods"]+++def test_nav_rank_compiled_buckets_rewards_both_legs(monkeypatch):+ """A document hit by title AND fact outranks one hit by only one leg."""+ module, _, _ = _load_nav_module(monkeypatch)+ buckets = module._nav_bucket_compiled_rows(+ {+ "r1": _compiled_row("both", "title", 0.8),+ "r2": _compiled_row("both", "fact", 0.8),+ "r3": _compiled_row("titleonly", "title", 0.9),+ }+ )+ ranked = module._nav_rank_compiled_buckets(buckets, 10)+ assert [d for d, _ in ranked] == ["both", "titleonly"]+ assert ranked[0][1]["legs"] == {"title", "fact"}+ # best stays on the 0..1 cosine scale callers threshold on.+ assert ranked[0][1]["best"] == pytest.approx(0.8)+++def test_nav_rank_compiled_buckets_does_not_mutate_input(monkeypatch):+ """Regression: ranking used to attach 'legs' onto the caller's bucket entries."""+ module, _, _ = _load_nav_module(monkeypatch)+ buckets = module._nav_bucket_compiled_rows({"r1": _compiled_row("doc-a", "title", 0.8)})+ before = json.dumps(buckets, sort_keys=True, default=str)+ module._nav_rank_compiled_buckets(buckets, 10)+ assert json.dumps(buckets, sort_keys=True, default=str) == before+++def test_nav_rank_compiled_buckets_respects_top_k(monkeypatch):+ module, _, _ = _load_nav_module(monkeypatch)+ buckets = module._nav_bucket_compiled_rows({f"r{i}": _compiled_row(f"doc-{i}", "fact", 0.9 - i * 0.1) for i in range(5)})+ assert len(module._nav_rank_compiled_buckets(buckets, 2)) == 2+++async def test_compiled_agg_router_uses_keyword_fallback_without_embedding(monkeypatch):+ """No embedding model: the leg degrades to a keyword match, it does not drop out."""+ module, _, doc_store = _load_nav_module(+ monkeypatch,+ field_map={"r1": _compiled_row("doc-a", "fact", 0.5)},+ )+ doc_store.get_fields = MagicMock(return_value={"r1": _compiled_row("doc-a", "fact", 0.5)})+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")++ ok, payload = await module._search_layers_compiled_agg("tenant-1", "kb-1", "q", 5, None, kb)+ assert ok is True+ assert [i["doc_id"] for i in payload["items"]] == ["doc-a"]+ # _nav_doc_summaries issues a second search, so read the first one.+ exprs = doc_store.search.call_args_list[0].kwargs["match_expressions"]+ # A keyword-match expr, not a dense one.+ assert len(exprs) == 1+ assert isinstance(exprs[0], _StubMatchTextExpr)+ assert exprs[0].fields == ["content_ltks", "content_sm_ltks"]+++async def test_compiled_agg_router_filters_deleted_documents(monkeypatch):+ """Compiled rows can outlive deleted docs; the leg must check existence itself."""+ module, _, doc_store = _load_nav_module(+ monkeypatch,+ field_map={"r1": _compiled_row("doc-gone", "fact", 0.9)},+ )+ doc_store.get_fields = MagicMock(return_value={"r1": _compiled_row("doc-gone", "fact", 0.9)})+ retriever = SimpleNamespace(+ get_vector=AsyncMock(return_value=SimpleNamespace()),+ _existing_doc_ids=AsyncMock(return_value=set()), # nothing survives+ )+ module, _, _ = _load_nav_module(monkeypatch, field_map={"r1": _compiled_row("doc-gone", "fact", 0.9)}, retriever=retriever)+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")++ ok, payload = await module._search_layers_compiled_agg("tenant-1", "kb-1", "q", 5, SimpleNamespace(), kb)+ assert ok is True+ assert payload["total"] == 0+++async def test_compiled_agg_router_builds_vector_with_named_candidates(monkeypatch):+ """Regression: positional passing put the similarity threshold into num_candidates."""+ get_vector = AsyncMock(return_value=SimpleNamespace())+ module, _, doc_store = _load_nav_module(+ monkeypatch,+ field_map={"r1": _compiled_row("doc-a", "fact", 0.9)},+ retriever=SimpleNamespace(get_vector=get_vector, _existing_doc_ids=AsyncMock(return_value={"doc-a"})),+ )+ doc_store.get_fields = MagicMock(return_value={"r1": _compiled_row("doc-a", "fact", 0.9)})+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")++ await module._search_layers_compiled_agg("tenant-1", "kb-1", "q", 5, SimpleNamespace(), kb)+ kw = get_vector.call_args.kwargs+ assert kw["top_k"] == module._NAV_COMPILED_POOL+ # HNSW ef_search must be >= top_k, and must not receive the similarity value.+ assert kw["num_candidates"] == module._NAV_COMPILED_POOL+ assert kw["similarity"] == module._NAV_COMPILED_SIMILARITY+ # _nav_doc_summaries issues a second search, so read the first one.+ cond = doc_store.search.call_args_list[0].kwargs["condition"]+ # Compiled rows carry no available_int, so it must NOT be used as a filter.+ assert "available_int" not in cond+ assert cond["scope_kwd"] == ["doc"]+ assert cond["knowledge_graph_kwd"] == ["entity"]+++# ---------------------------------------------------------------------------+# Fusion of the chunk leg and the compiled leg+# ---------------------------------------------------------------------------+++def test_nav_fuse_legs_normalizes_each_leg(monkeypatch):+ """Each leg is normalized by its own peak so raw cosine cannot dominate."""+ module, _, _ = _load_nav_module(monkeypatch)+ ranked = module._nav_fuse_legs(+ [+ ("chunk", [{"doc_id": "a", "score": 0.9}, {"doc_id": "b", "score": 0.45}], 1.0),+ ("compiled", [{"doc_id": "c", "score": 0.6}], 1.0),+ ],+ 10,+ )+ scores = {d: info["fused"] for d, info in ranked}+ # Peak of each leg becomes 1.0, so a document top-ranked by either leg ties.+ assert scores["a"] == pytest.approx(1.0)+ assert scores["c"] == pytest.approx(1.0)+ assert scores["b"] == pytest.approx(0.5)+++def test_nav_fuse_legs_prefers_documents_hit_by_both(monkeypatch):+ module, _, _ = _load_nav_module(monkeypatch)+ ranked = module._nav_fuse_legs(+ [+ ("chunk", [{"doc_id": "both", "score": 0.5}, {"doc_id": "chunkonly", "score": 1.0}], 1.0),+ ("compiled", [{"doc_id": "both", "score": 1.0}], 1.0),+ ],+ 10,+ )+ assert [d for d, _ in ranked] == ["both", "chunkonly"]+ assert ranked[0][1]["legs"] == {"chunk", "compiled"}+++def test_nav_fuse_legs_survives_empty_leg(monkeypatch):+ module, _, _ = _load_nav_module(monkeypatch)+ ranked = module._nav_fuse_legs([("chunk", [], 1.0), ("compiled", [], 1.0)], 10)+ assert ranked == []+++async def test_fusion_router_falls_back_to_chunk_leg_without_compiled(monkeypatch):+ """A raptor KB has no page_index rows; the chunk leg must decide alone."""+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")++ ok, payload = await module._search_layers_fusion("tenant-1", "kb-1", "q", 5, None, kb, compiled={"items": []})+ assert ok is True+ assert [i["doc_id"] for i in payload["items"]] == ["doc-a"]+ # No compiled hits means no section bridge lookup either.+ assert payload["items"][0]["_agg"]["legs"] == ["chunk"]+++# ---------------------------------------------------------------------------+# Document routing via chunk aggregation (PageIndex semantics recipe)+# ---------------------------------------------------------------------------+++def test_nav_aggregate_chunks_rolls_up_per_document(monkeypatch):+ module, _, _ = _load_nav_module(monkeypatch)+ chunks = [+ {"doc_id": "doc-a", "similarity": 0.9},+ {"doc_id": "doc-b", "similarity": 0.8},+ {"doc_id": "doc-a", "similarity": 0.7},+ {"doc_id": " ", "similarity": 0.5}, # blank doc_id — dropped+ {"doc_id": "doc-a", "score": 0.5}, # 'score' fallback key+ ]+ agg = module._nav_aggregate_chunks(chunks)+ assert set(agg) == {"doc-a", "doc-b"}+ assert agg["doc-a"] == {"total": pytest.approx(2.1), "best": 0.9, "hits": 3}+ assert agg["doc-b"] == {"total": pytest.approx(0.8), "best": 0.8, "hits": 1}+++def test_nav_doc_score_damps_hit_count(monkeypatch):+ module, _, _ = _load_nav_module(monkeypatch)+ single = module._nav_doc_score({"total": 0.9, "best": 0.9, "hits": 1})+ assert single == pytest.approx(0.9 / math.sqrt(2))+ # A long document must not win on volume alone: four weak hits stay below+ # one strong hit.+ weak_many = module._nav_doc_score({"total": 1.0, "best": 0.25, "hits": 4})+ assert weak_many == pytest.approx(1.0 / math.sqrt(5))+ assert weak_many < single+++async def test_chunk_agg_router_ranks_by_doc_score_not_hit_count(monkeypatch):+ """doc-b has one strong hit; doc-a has four weak ones. doc-b must win."""+ retriever = SimpleNamespace(+ retrieval=AsyncMock(+ return_value={+ "chunks": [+ {"doc_id": "doc-a", "similarity": 0.20},+ {"doc_id": "doc-a", "similarity": 0.20},+ {"doc_id": "doc-a", "similarity": 0.20},+ {"doc_id": "doc-a", "similarity": 0.20},+ {"doc_id": "doc-b", "similarity": 0.90},+ ]+ }+ )+ )+ module, _, _ = _load_nav_module(monkeypatch, retriever=retriever)++ ok, payload = await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 5, None)+ assert ok is True+ assert [item["doc_id"] for item in payload["items"]] == ["doc-b", "doc-a"]+ # score stays on the 0..1 chunk-similarity scale callers threshold on.+ assert payload["items"][0]["score"] == pytest.approx(0.9)+ assert payload["items"][1]["score"] == pytest.approx(0.2)+ assert payload["items"][0]["_agg"]["hits"] == 1+ assert payload["items"][1]["_agg"]["hits"] == 4+++async def test_chunk_agg_router_uses_full_candidate_pool(monkeypatch):+ """page_size covers the pool so every fused candidate reaches the aggregation."""+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))++ await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 12, None)+ args, kw = retrieval.call_args+ # page_size is positional arg 6; no similarity floor at 7.+ assert args[5] == module._NAV_CHUNK_AGG_POOL+ assert args[6] == 0.0+ # The pool is rerank_candidates_count — page_size alone would leave the+ # candidate set at the 64 default, too narrow for DocScore to matter.+ assert kw["rerank_candidates_count"] == module._NAV_CHUNK_AGG_POOL+ # page * page_size must never exceed rerank_candidates_count or retrieval raises.+ assert args[4] * args[5] <= kw["rerank_candidates_count"]+++async def test_chunk_agg_router_does_not_narrow_knn_candidates(monkeypatch):+ """knn_top_k caps what the vector leg feeds fusion; narrowing it shrinks the pool."""+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))++ await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 12, None)+ assert "knn_top_k" not in retrieval.call_args.kwargs+++async def test_chunk_agg_router_excludes_compiled_rows(monkeypatch):+ """Compiled rows are written with available_int=1, so they need explicit exclusion.++ Without this they land in the chunk candidate set and inflate a document's+ hit count. Project convention: plain retrieval reads chunks, compiled+ products are served by their own tools.+ """+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))++ await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 12, None)+ assert retrieval.call_args.kwargs["must_not"] == {"exists": "compile_kwd"}+++async def test_chunk_agg_router_can_include_compiled_rows(monkeypatch):+ """Flipping the flag lets compiled rows A/B as extra per-document entries."""+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))+ monkeypatch.setattr(module, "_NAV_CHUNK_AGG_EXCLUDE_COMPILED", False)++ await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 12, None)+ assert retrieval.call_args.kwargs["must_not"] is None+++def test_nav_label_items_wraps_flat_row_for_the_caller(monkeypatch):+ """navigation.py reads the route label from item["_nav"]["description"].++ search_dataset_nav returns the leaf row flat, so without wrapping the route+ comes back unlabelled.+ """+ module, _, _ = _load_nav_module(monkeypatch)+ out = module._nav_label_items([{"doc_id": "doc-a", "name": "Leaf A", "description": "Summary A"}])+ assert out[0]["_nav"] == {"doc_id": "doc-a", "description": "Summary A"}+++def test_nav_label_items_falls_back_to_name(monkeypatch):+ module, _, _ = _load_nav_module(monkeypatch)+ out = module._nav_label_items([{"doc_id": "doc-a", "name": "Leaf A", "description": ""}])+ assert out[0]["_nav"]["description"] == "Leaf A"+++def test_nav_label_items_preserves_existing_nav(monkeypatch):+ """A row that already carries _nav must not be overwritten."""+ module, _, _ = _load_nav_module(monkeypatch)+ out = module._nav_label_items([{"doc_id": "doc-a", "description": "flat", "_nav": {"description": "nested"}}])+ assert out[0]["_nav"]["description"] == "nested"+++def test_nav_label_items_tolerates_non_dicts(monkeypatch):+ module, _, _ = _load_nav_module(monkeypatch)+ assert module._nav_label_items(["junk", None]) == ["junk", None]+++def test_nav_focus_items_sorts_and_truncates_to_limit(monkeypatch):+ """Regression: the flat sweep returned every row above the 0.2 floor.++ Main's cluster descent routes ~2 documents; the flat sweep returned 6-12,+ and routing to 3x the documents made the RAGAgent carry 3x the evidence in+ every round — each dynamic LLM call grew ~33% and per-question time ~25%.+ """+ module, _, _ = _load_nav_module(monkeypatch)+ monkeypatch.setattr(module, "_NAV_DOC_FOCUS_LIMIT", 3)+ # Not sorted — must sort by score before truncating.+ items = [{"doc_id": f"doc-{i}", "score": s} for i, s in [(6, 0.1), (1, 0.9), (3, 0.4), (2, 0.6)]]+ out = module._nav_focus_items(items)+ assert [i["doc_id"] for i in out] == ["doc-1", "doc-2", "doc-3"]+ # The input list is truncated in place.+ assert [i["doc_id"] for i in items] == ["doc-1", "doc-2", "doc-3"]+++def test_nav_focus_items_keeps_fewer_than_limit_untouched(monkeypatch):+ """When few docs are relevant, do not pad — routing stays honest."""+ module, _, _ = _load_nav_module(monkeypatch)+ monkeypatch.setattr(module, "_NAV_DOC_FOCUS_LIMIT", 3)+ items = [{"doc_id": "doc-1", "score": 0.9}]+ out = module._nav_focus_items(items)+ assert len(out) == 1+++def test_nav_focus_items_disabled_when_limit_is_zero(monkeypatch):+ module, _, _ = _load_nav_module(monkeypatch)+ monkeypatch.setattr(module, "_NAV_DOC_FOCUS_LIMIT", 0)+ items = [{"doc_id": f"doc-{i}", "score": s} for i, s in [(1, 0.9), (2, 0.6), (3, 0.4)]]+ out = module._nav_focus_items(items)+ assert len(out) == 3+++async def test_chunk_agg_router_focuses_to_limit(monkeypatch):+ """chunk_agg must cap the returned documents like nav_doc does.++ Without the focus cap this router returned every document above the floor,+ routing the RAGAgent into 3x the evidence and inflating every dynamic LLM+ call — the same regression the nav_doc router had.+ """+ retriever = SimpleNamespace(retrieval=AsyncMock(return_value={"chunks": [{"doc_id": f"doc-{i}", "similarity": s} for i, s in [(5, 0.1), (1, 0.9), (3, 0.4), (2, 0.6), (4, 0.2)]]}))+ module, _, _ = _load_nav_module(monkeypatch, retriever=retriever)+ monkeypatch.setattr(module, "_NAV_DOC_FOCUS_LIMIT", 3)+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")++ ok, payload = await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 12, None, kb)+ assert ok is True+ # Cap to the focus limit even when top_k is higher.+ assert payload["total"] <= 3+ assert len(payload["items"]) <= 3+++async def test_chunk_agg_router_focus_limit_overrides_top_k(monkeypatch):+ """Focus wins over the caller's top_k — top_k=12 must not widen routing."""+ retriever = SimpleNamespace(retrieval=AsyncMock(return_value={"chunks": [{"doc_id": f"doc-{i}", "similarity": 0.9 - i * 0.1} for i in range(8)]}))+ module, _, _ = _load_nav_module(monkeypatch, retriever=retriever)+ monkeypatch.setattr(module, "_NAV_DOC_FOCUS_LIMIT", 2)+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")++ ok, payload = await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 12, None, kb)+ assert ok is True+ assert payload["total"] == 2+++async def test_chunk_mode_page_size_stays_bounded(monkeypatch):+ """Chunk mode returns chunks AS evidence, so page_size is the result size.++ Regression: it once shared the chunk_agg aggregation pool (256), which+ inflated the evidence pool ~7x (25 -> 191 chunks/question), bloated every+ downstream LLM prompt and exhausted the 180s research budget on 15/16+ questions.+ """+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")++ await module._search_layers_chunks("tenant-1", "kb-1", "q", 12, None, kb)+ args, kw = retrieval.call_args+ # Historical fetch size, not the aggregation pool.+ assert args[5] == 12 * 3+ # The candidate pool still has to cover the returned page.+ assert kw["rerank_candidates_count"] >= args[5]+++async def test_chunk_agg_router_survives_large_top_k(monkeypatch):+ """Regression: a top_k that pushes page_size past the default pool used to raise."""+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))++ ok, payload = await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 100, None)+ assert ok is True+ args, kw = retrieval.call_args+ assert args[4] * args[5] <= kw["rerank_candidates_count"]+++async def test_chunk_agg_router_falls_back_to_terms_without_embeddings(monkeypatch):+ """Without an embedding model the dense leg is dropped, not the router."""+ retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})+ module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))++ ok, payload = await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 5, None)+ assert ok is True+ assert payload["total"] == 1+ assert retrieval.call_args.args[7] == 0+++async def test_chunk_agg_router_returns_empty_when_no_chunks(monkeypatch):+ module, _, _ = _load_nav_module(monkeypatch)+ ok, payload = await module._search_layers_chunk_agg("tenant-1", "kb-1", "q", 5, None)+ assert ok is True+ assert payload == {"mode": "navigation_tree", "total": 0, "items": []}+++async def test_nav_doc_summaries_batch_loads_descriptions(monkeypatch):+ module, _, doc_store = _load_nav_module(+ monkeypatch,+ field_map={+ "row-1": {+ "name": "leaf-a",+ "type_kwd": "nav_doc",+ "content_with_weight": json.dumps({"description": "Summary A"}),+ "doc_id": "doc-a",+ },+ "row-2": {+ "name": "leaf-b",+ "type_kwd": "nav_doc",+ "content_with_weight": json.dumps({"description": "Summary B"}),+ "doc_id": "doc-b",+ },+ },+ )+ kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")+ summaries = await module._nav_doc_summaries(kb, ["doc-a", "doc-b"])+ assert summaries == {"doc-a": "Summary A", "doc-b": "Summary B"}+ # One query for the whole batch, not one per document.+ assert doc_store.search.call_count == 1+ condition = doc_store.search.call_args.kwargs["condition"]+ assert condition["type_kwd"] == ["nav_doc"]+ assert condition["doc_id"] == ["doc-a", "doc-b"]