RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Add claim/evidence to tree/pageindex compilation pipeline (#19267)
Yingfeng1f5d33333a3f
diff --git a/api/apps/auth/README.md b/api/apps/auth/README.mdindex 37df4e8391..bff359d93f 100644--- a/api/apps/auth/README.md+++ b/api/apps/auth/README.md@@ -1,6 +1,6 @@# Auth-The Auth module provides implementations of OAuth2 and OpenID Connect (OIDC) authentication for integration with third-party identity providers.+The Auth module provides implementations of OAuth2 and OpenID Connect (OIDC) authentication for integration with third-party identity providers.**Features**diff --git a/api/apps/restful_apis/chunk_api.py b/api/apps/restful_apis/chunk_api.pyindex a147308238..60f512767e 100644--- a/api/apps/restful_apis/chunk_api.py+++ b/api/apps/restful_apis/chunk_api.py@@ -786,43 +786,6 @@ async def get_document_structure_graph(tenant_id, dataset_id, document_id):except Exception as e:return server_error_response(e)- # RAPTOR summary graph is stored as a standalone blob rather than raw- # knowledge_graph_kwd entity/relation rows, so include its arrays explicitly.- raptor_entities: list[dict] = []- raptor_relations: list[dict] = []- try:- res_raptor = await thread_pool_exec(- settings.docStoreConn.search,- ["content_with_weight", "compile_kwd"],- [],- {"doc_id": [document_id], "compile_kwd": ["raptor_graph"]},- [],- OrderByExpr(),- 0,- 16,- index_name,- [dataset_id],- )- raptor_rows = settings.docStoreConn.get_fields(res_raptor, ["content_with_weight", "compile_kwd"]) or {}- except Exception:- logging.exception("structure graph: RAPTOR blob load failed for doc=%s", document_id)- raptor_rows = {}- for row in raptor_rows.values():- try:- graph = json.loads(row.get("content_with_weight") or "{}")- except Exception:- continue- if not isinstance(graph, dict):- continue- r_entities = graph.get("entities") or []- r_relations = graph.get("relations") or []- if isinstance(r_entities, list):- raptor_entities.extend(r_entities)- if isinstance(r_relations, list):- raptor_relations.extend(r_relations)- total_entities += len(raptor_entities)- total_relations += len(raptor_relations)-def _row_template_id(row: dict) -> str | None:raw = row.get("compilation_template_ids")if isinstance(raw, list):@@ -911,9 +874,14 @@ async def get_document_structure_graph(tenant_id, dataset_id, document_id):return get_result(data=_response([bucket]))# ── normal mode: per-template subgraph sampling from the raw rows ──- # Discover buckets from the authoritative entity/relation rows. The- # structure compiler no longer emits a synthetic ``graph`` row, and a- # paged metadata-only scan avoids loading payloads or truncating large docs.+ # Discover buckets from the authoritative per-doc entity/relation rows,+ # WITHOUT loading the (potentially huge) content_with_weight. The compact+ # graph blob is gone from the storage model: tree compilation now writes+ # the same per-row shape as page_index, so buckets are discovered from the+ # rows themselves. The scan is paged (metadata-only) to avoid truncating+ # large docs — a doc carries one row per entity/relation, not one per+ # template. Each bucket's entities/relations are then fetched with+ # subgraph sampling.meta_fields = ["id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"]meta_rows: dict = {}page_size = 1000@@ -963,15 +931,8 @@ async def get_document_structure_graph(tenant_id, dataset_id, document_id):continuegrouped[bid] = {**meta, "entities": entities, "relations": relations}- # RAPTOR was loaded above so its full counts are also available to keyword- # responses; append it only in normal mode, matching the existing behavior.- if raptor_entities or raptor_relations:- rb = grouped.setdefault("raptor", {"template_id": "raptor", "template_name": "RAPTOR Summary", "kind": "raptor", "entities": [], "relations": []})- rb["entities"].extend(raptor_entities)- rb["relations"].extend(raptor_relations)-# Order: configured templates first (in the user's chosen order),- # then any discovered / legacy / raptor buckets after.+ # then any discovered / legacy buckets after.ordered_ids: list[str] = []for tid in configured_ids:if tid in grouped and tid not in ordered_ids:@@ -981,6 +942,27 @@ async def get_document_structure_graph(tenant_id, dataset_id, document_id):ordered_ids.append(bucket_id)page_index_groups = [group for group in grouped.values() if _compilation_template_kind(group.get("kind")) in {"page_index", "pageindex"}]+ for group in page_index_groups:+ # Headings at every level carry a claim badge, not just leaves: the UI+ # opens a node's claims from any level, and a heading's own chunk span+ # covers its whole subtree. Failures are logged and the tree renders+ # without badges.+ try:+ await sgc.attach_claim_counts(+ group.get("entities") or [],+ group.get("relations") or [],+ index_name,+ dataset_id,+ document_id,+ _compilation_template_kind(group.get("kind")),+ leaves_only=False,+ )+ except Exception:+ logging.exception(+ "structure graph: page_index claim count attach failed for doc=%s template=%s",+ document_id,+ group.get("template_id"),+ )if page_index_groups:# PageIndex nodes should follow the document's chunk order. Use the# current chunk query order directly. Do not derive the order@@ -1028,6 +1010,98 @@ async def get_document_structure_graph(tenant_id, dataset_id, document_id):return get_result(data=_response(templates_out))+async def _template_compile_kwd(template_id: str) -> str:+ """The ``compile_kwd`` rows of this template were stamped with.++ Compiled rows carry the template's ``kind`` (page_index, tree, timeline…),+ so any query that pins ``compile_kwd`` has to read it from the template+ rather than assume one. Best-effort: ``None`` means "don't filter on it",+ which is safe because the template id scopes the query anyway.++ Deliberately defined ABOVE the route it serves: slotting a helper between a+ decorator stack and its function steals the registration, and the route+ then calls the helper with the path parameters instead.+ """+ from api.db.services.compilation_template_service import (+ CompilationTemplateService,+ )++ try:+ rows = await thread_pool_exec(CompilationTemplateService.query, id=template_id)+ except Exception: # pragma: no cover - lookup is best-effort+ return ""+ if not rows:+ return ""+ first = rows[0]+ cfg = first.get("config") if isinstance(first, dict) else None+ if isinstance(cfg, str):+ try:+ cfg = json.loads(cfg)+ except (TypeError, ValueError):+ cfg = None+ if isinstance(cfg, dict) and cfg.get("kind"):+ return _compilation_template_kind(cfg.get("kind"))+ kind = first.get("kind") if isinstance(first, dict) else None+ return _compilation_template_kind(kind)++[email protected]("/datasets/<dataset_id>/documents/<document_id>/structure/claims", methods=["GET"]) # noqa: F821+@login_required+@add_tenant_id_to_kwargs+async def get_document_structure_claims(tenant_id, dataset_id, document_id):+ """Page through a document's claim/evidence rows (entity_type_kwd="claim").++ Tree artifacts keep claims out of the node list, so the UI fetches them per+ leaf cluster on demand. Query args:++ chunk_ids: comma-separated source chunk ids — a leaf cluster's+ members; without it the endpoint pages the whole document+ template_id: compilation template scoping (optional)+ offset/limit: paging, limit capped at 100+ """+ if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):+ return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")+ dataset_tenant_id = _get_dataset_tenant_id(dataset_id)+ if not dataset_tenant_id:+ return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")+ docs = DocumentService.query(id=document_id, kb_id=dataset_id)+ if not docs:+ return get_error_data_result(message=f"you don't own the document {document_id}")++ chunk_ids = [c.strip() for c in (request.args.get("chunk_ids") or "").split(",") if c.strip()]+ template_id = (request.args.get("template_id") or "").strip() or None+ try:+ offset = max(int(request.args.get("offset") or 0), 0)+ limit = min(max(int(request.args.get("limit") or 20), 1), 100)+ except ValueError:+ return get_error_data_result(message="offset/limit must be integers")++ # The rows' ``compile_kwd`` is the template's own kind. It used to be+ # pinned to "tree" here, so a page_index document (compile_kwd="page_index")+ # matched no rows at all and every node reported zero claims.+ compile_kwd = None+ if template_id:+ compile_kwd = await _template_compile_kwd(template_id) or None++ from rag.nlp import search++ index_name = search.index_name(dataset_tenant_id)+ try:+ claims, total = await sgc.list_claim_rows(+ index_name,+ dataset_id,+ document_id,+ compile_kwd=compile_kwd,+ compilation_template_id=template_id,+ chunk_ids=chunk_ids or None,+ offset=offset,+ limit=limit,+ )+ except Exception as e:+ return server_error_response(e)+ return get_result(data={"claims": claims, "total": total, "offset": offset, "limit": limit})++@manager.route("/datasets/<dataset_id>/documents/<document_id>/structure/graph", methods=["DELETE"]) # noqa: F821@login_required@add_tenant_id_to_kwargs@@ -1036,11 +1110,10 @@ async def delete_document_structure_graph(tenant_id, dataset_id, document_id):Request body::- {"template_id": "<template id> | legacy:<compile_kwd> | raptor"}+ {"template_id": "<template id> | legacy:<compile_kwd>"}Template-backed structure tabs remove both the compact graph row and- the underlying entity/relation rows. RAPTOR only removes the graph- projection row so summary chunks remain available for retrieval.+ the underlying entity/relation rows."""from rag.nlp import search@@ -1065,10 +1138,6 @@ async def delete_document_structure_graph(tenant_id, dataset_id, document_id):try:deleted = 0- if template_id == "raptor":- deleted += _delete({"doc_id": [document_id], "compile_kwd": ["raptor_graph"]})- return get_result(data={"deleted": deleted}, message=f"deleted {deleted} structure graph rows")-if template_id.startswith("legacy:"):compile_kwd = template_id[len("legacy:") :].strip()if not compile_kwd:diff --git a/api/apps/restful_apis/utils/compilation_template_validation.py b/api/apps/restful_apis/utils/compilation_template_validation.pyindex 2c83253e86..0f956c4ca4 100644--- a/api/apps/restful_apis/utils/compilation_template_validation.py+++ b/api/apps/restful_apis/utils/compilation_template_validation.py@@ -56,6 +56,11 @@ def validate_template_payload(req: dict, require_all: bool = True) -> str:return f"{section.capitalize()} field description is too long."if len(str((field or {}).get("rule") or "")) > 1024:return f"{section.capitalize()} field rule is too long."+ if config.get("kind") == "tree" or req.get("kind") == "tree":+ raptor_cfg = config.get("raptor") or {}+ claim_prompt = str((raptor_cfg or {}).get("claim_prompt") or "")+ if claim_prompt and len(claim_prompt) > 8192:+ return "Claim extraction prompt is too long."if config.get("kind") == "wiki" or req.get("kind") == "wiki":if config.get("mode") not in ("entity", "topic"):return "Wiki mode must be either 'entity' or 'topic'."diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex 6de92596af..7aff2a7f17 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -3737,38 +3737,49 @@ async def generate_nav(})- # Prefer RAPTOR-generated summaries stored in the knowledge- # graph (compile_kwd="tree", knowledge_graph_kwd="graph")- # so that rebuilt nav descriptions match the original- # tree-compilation output. Fall back to meta_fields.title- # or filename when the graph is absent.+ # Tree summaries come from the per-row entity/relation rows+ # (compile_kwd="tree") -- the compact graph blob no longer exists.+ # One query per row kind, grouped per document, then projected with+ # build_nav_graph_text: identical output to the old blob content,+ # so rebuilt nav descriptions keep matching the original+ # tree-compilation output. Fall back to meta_fields.title or+ # filename when the rows are absent.raptor_summaries: dict[str, str] = {}pack = _compiled_index_or_none(kb.tenant_id, dataset_id)if pack is not None:try:index_nm, _ = packfrom common.doc_store.doc_store_base import OrderByExpr-- graph_res = settings.docStoreConn.search(- select_fields=["doc_id", "content_with_weight"],- highlight_fields=[],- condition={"compile_kwd": ["tree"], "knowledge_graph_kwd": ["graph"]},- match_expressions=[],- order_by=OrderByExpr(),- offset=0,- limit=10000,- index_names=index_nm,- knowledgebase_ids=[dataset_id],- )- graph_map = settings.docStoreConn.get_fields(graph_res, ["doc_id", "content_with_weight"])- for row in (graph_map or {}).values():- gid = str(row.get("doc_id") or "")- if not gid:- continue- try:- graph = json.loads(row.get("content_with_weight") or "{}")- except Exception:- continue+ from rag.advanced_rag.knowlege_compile.dataset_nav import build_nav_graph_text++ per_doc: dict[str, dict] = {}+ for kind in ("entity", "relation"):+ rows_res = settings.docStoreConn.search(+ select_fields=["doc_id", "content_with_weight"],+ highlight_fields=[],+ condition={"compile_kwd": ["tree"], "knowledge_graph_kwd": [kind]},+ match_expressions=[],+ order_by=OrderByExpr(),+ offset=0,+ limit=10000,+ index_names=index_nm,+ knowledgebase_ids=[dataset_id],+ )+ rows = settings.docStoreConn.get_fields(rows_res, ["doc_id", "content_with_weight"])+ for row in (rows or {}).values():+ gid = str(row.get("doc_id") or "")+ if not gid:+ continue+ try:+ payload = json.loads(row.get("content_with_weight") or "{}")+ except Exception:+ continue+ if not isinstance(payload, dict):+ continue+ bucket = "entities" if kind == "entity" else "relations"+ per_doc.setdefault(gid, {"entities": [], "relations": []})[bucket].append(payload)++ for gid, graph in per_doc.items():# Build both:# - root_summary: first line of root desc (short,# for the display title / description field)@@ -3785,7 +3796,7 @@ async def generate_nav("graph_text": graph_text or root_summary,}except Exception:- logging.exception("generate_nav: failed to read RAPTOR graph summaries for kb=%s", dataset_id)+ logging.exception("generate_nav: failed to read tree summaries for kb=%s", dataset_id)documents = []for d in all_docs:@@ -3885,80 +3896,32 @@ _LAYERS_HANDLERS: dict[str, str] = {"all": "_search_layers_all",}-# Document router behind the "navigation_tree" mode. Callers always ask for-# mode="navigation_tree", so swapping strategy needs no change on their side.-#-# "nav_doc" (default) — flat hybrid search over the nav_doc rows. No cluster-# level is consulted, so nothing is pruned, but the comparison still runs-# on document-level embeddings. Those rows embed ``graph_text`` — the-# document's full entity/relation listing — not a one-line summary, so-# this is not the "compressed representation first" case that SCT measures.-# "chunk_agg" — PageIndex ``semantics`` recipe: hybrid search over the raw-# chunk index, hits rolled up per document with-# DocScore = sum(chunk scores) / sqrt(hits + 1). Sound in principle, but-# chunk-level similarity is easy to clear: a document passes the 0.2 floor-# on a single marginally related chunk, so the top-k roster fills with-# weakly related documents, and the wider routing fans out every later-# stage. Measured 2026-09-02: 12/12 questions routed a full 12 documents,-# evidence pool 42-67 passages, 166s per question against a 180s budget,-# with several timeouts yielding no answer at all.-# "compiled_agg" — page_index entity rows only; "fusion" — chunk + compiled.-# "tree" — BFS beam descent over the nav cluster tree. This is what main-# branch does. Kept as the A/B baseline; drop it, and-# ``search_nav_tree_descent`` with it, once a winner is settled.--# Cap on how many documents a flat router hands back. The cluster beam descent-# that main uses returns ~2 by construction (its per-level pruning keeps only-# the most relevant branches), while the flat nav_doc sweep returns every row-# above the 0.2 floor — measured 2026-09-02: beam avg 2.0 vs nav_doc avg 6.2.-# Routing to 3x the documents made the RAGAgent carry 3x the evidence in every-# round, so each dynamic LLM call grew 14k -> 18.8k tokens and per-question time-# grew ~25%. Beam is not capped at 2 either (it occasionally routes 6 for-# multi-hop), so allow a little headroom above its median rather than a hard 2.-_NAV_DOC_FOCUS_LIMIT = 3--# Candidate pool for "chunk_agg". DocScore only means something when a document-# can collect several hits, so the pool has to be wide enough for that; across a-# 2400-document corpus a pool of a few dozen chunks yields almost entirely-# single-hit documents, which collapses DocScore into plain best-chunk ranking.-# This is ``rerank_candidates_count`` — the pool the fusion actually draws from.-# ``page_size`` is deliberately the same value so the whole pool is returned for-# aggregation, and it must stay <= the pool or ``retrieval`` raises-# (page * page_size > rerank_candidates_count)._NAV_CHUNK_AGG_POOL = 256+_NAV_CHUNK_AGG_VEC_WEIGHT = 0.3-# Candidate pool of compiled entity rows for the "compiled_agg" / "fusion"-# routers. ``get_vector`` builds its own MatchDenseExpr and never routes through-# ``retrieval``, so ``rerank_candidates_count`` does not apply here — but HNSW-# needs ``num_candidates`` >= the pool or the search degenerates.-_NAV_COMPILED_POOL = 256-_NAV_COMPILED_SIMILARITY = 0.1-# Entity types aggregated separately. A document carries far more facts than-# titles, so pooling them would let fact hits drown out title hits; splitting-# keeps the coarser title signal usable. ``entity_type_kwd`` does not exist on-# compilation rows (it belongs to graphrag), so the type is read from the-# payload after the fetch.-_NAV_COMPILED_TITLE_TYPES = ("title",)-_NAV_COMPILED_FACT_TYPES = ("fact", "conclusion")-# Weights applied when the fusion router combines the three legs. Only used-# when documents are hit by several legs.-_NAV_FUSION_WEIGHTS = {"chunk": 1.0, "title": 0.8, "fact": 0.9}--# Section bridge (stage 3): resolve each hit fact to its owning title. The-# relation type carrying TOC containment in the page_index template, plus caps-# on how many names are resolved and how many relation rows may be scanned.-_NAV_INCLUDE_RELATION = "include"-_NAV_BRIDGE_NAMES_LIMIT = 64-_NAV_BRIDGE_FANOUT = 4-# Compiled products (tree/entity/relation rows) are written with-# available_int=1, so they fall inside the chunk filter and would otherwise-# inflate a document's hit count. The project convention is that plain-# retrieval reads document chunks only and compiled products are served by their-# own tools, so exclude them here too. Flip to False to A/B letting compiled-# rows act as extra per-document entries in the aggregation._NAV_CHUNK_AGG_EXCLUDE_COMPILED = True+_NAV_DOC_FOCUS_LIMIT = 3++_NAV_CLAIM_POOL = 256++_NAV_EVIDENCE_ROW_TYPES = ("claim",)+++# Router behind the "navigation_tree" mode. Two callers share the entry point:+#+# "tree" — the artifacts UI (main branch's BFS beam descent over the nav+# cluster tree). This is the default, preserving upstream behaviour.+# "claim_agg" — the agentic rag router: the claim leg runs first and decides+# the ranking outright when it hits (a claim is an atomic proposition+# carrying its own vector and verbatim evidence), falling back to the+# chunk leg when the dataset has no claim rows or the leg comes back+# empty.+#+# Upstream's other experimental strategies (nav_doc / compiled_agg / fusion)+# were dropped on this branch together with their implementations.+async def search_dataset_layers(dataset_id: str,@@ -3968,6 +3931,7 @@ async def search_dataset_layers(*,top_k: int | None = None,doc_scope: list[str] | None = None,+ router: str | None = None,) -> tuple[bool, dict]:"""Unified search across different knowledge layers of a dataset.@@ -3976,8 +3940,10 @@ async def search_dataset_layers(- chunk: raw document chunks (via the main retrieval pipeline)- nav_doc: navigation tree document leaves- nav_cluster: navigation tree cluster nodes- - navigation_tree: document routing, strategy chosen by- ``_NAV_TREE_ROUTER`` (chunk aggregation by default)+ - navigation_tree: document routing. ``router`` picks the+ strategy — the artifacts UI keeps main branch's ``"tree"`` beam+ descent, while the agentic rag router asks for ``"claim_agg"``+ (claim rows first, raw-chunk aggregation as the fallback).- all: union of all modes, deduplicated by doc_id with best scoredoc_scope: Optional set of documents to restrict the search to. None orempty means all documents of the dataset. Forwarded to every mode:@@ -4025,7 +3991,10 @@ async def search_dataset_layers(elif mode == "nav_cluster":return await _search_layers_nav_clusters(tenant_id, dataset_id, query, top_k, embd_mdl, search_dataset_nav, doc_scope=doc_scope)elif mode == "navigation_tree":- return await _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, kb, router="tree", doc_scope=doc_scope)+ # The artifacts UI keeps main branch's tree descent; the agentic rag+ # router passes router="claim_agg" explicitly. Defaulting to "tree"+ # preserves upstream behaviour for every existing caller.+ return await _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, kb, router=router or "tree", doc_scope=doc_scope)elif mode == "chunk":return await _search_layers_chunks(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)elif mode == "all":@@ -4062,63 +4031,30 @@ async def _search_layers_nav_clusters(tenant_id, dataset_id, query, top_k, embd_return True, {"mode": "nav_cluster", "total": len(items), "items": items}-def _nav_focus_items(items: list) -> list:- """Truncate a flat routing to the most relevant documents, in place.-- The flat sweep returns every row above the floor; the cluster descent main- uses stays focused by its per-level pruning. See ``_NAV_DOC_FOCUS_LIMIT``- for the measured reason this matters.- """- limit = _NAV_DOC_FOCUS_LIMIT- if not limit or len(items) <= limit:- return items- items.sort(key=lambda it: float(it.get("score", 0.0)), reverse=True)- del items[limit:]- return items---def _nav_label_items(items: list) -> list:- """Give each routed item the ``_nav`` summary navigation.py reads.-- ``search_dataset_nav`` returns the leaf row flat (``name``/``description`` at- the top level), but navigation.py reads the route label from- ``item["_nav"]["description"]``, so without this a route comes back- unlabelled. The rows already carry the summary, so this costs no extra- query — unlike the chunk router, which has to look one up.- """- for item in items:- if not isinstance(item, dict) or item.get("_nav"):- continue- item["_nav"] = {- "doc_id": str(item.get("doc_id") or "").strip(),- "description": str(item.get("description") or item.get("name") or "").strip(),- }- return items--async def _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, embd_mdl, kb=None, *, router: str = "chunk_agg", doc_scope=None):"""Route to documents using the requested navigation-tree strategy.- Every strategy returns the same item shape — ``doc_id``, a 0..1 ``score``,- and ``_nav`` carrying the document summary — so callers stay agnostic to- which one is active.+ Two callers share this entry point with different strategies:++ * the artifacts UI asks for ``router="tree"`` (main branch's BFS beam+ descent over the nav cluster tree) — that stays the human-facing+ default of ``mode="navigation_tree"``.+ * the agentic rag router asks for ``router="claim_agg"``: the claim leg+ runs first and decides the ranking outright when it hits — a claim is+ an atomic proposition carrying its own vector and verbatim evidence,+ so matching one means the document asserts that fact. No claim rows,+ or an empty claim leg, falls back to the chunk leg (PageIndex style:+ hybrid chunk recall rolled up per document).++ All strategies return the same item shape - ``doc_id``, a 0..1 ``score``,+ and ``_nav`` carrying the document summary - so callers stay agnostic to+ which one ran."""- if router == "nav_doc":- from rag.advanced_rag.knowlege_compile.dataset_nav import search_dataset_nav-- items = await _nav_search_result(- tenant_id,- dataset_id,- query,- top_k,- embd_mdl,- search_dataset_nav,- type_kwd="nav_doc",- doc_scope=doc_scope,- )- _nav_focus_items(items)- _nav_label_items(items)- return True, {"mode": "navigation_tree", "total": len(items), "items": items}+ if router == "claim_agg":+ _ok, claim_payload = await _search_layers_claim_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)+ if (claim_payload or {}).get("items"):+ return _ok, claim_payload+ return await _search_layers_chunk_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)if router == "tree":from rag.advanced_rag.knowlege_compile.dataset_nav import search_nav_tree_descent@@ -4133,13 +4069,6 @@ async def _search_layers_navigation_tree(tenant_id, dataset_id, query, top_k, em)return True, {"mode": "navigation_tree", "total": len(items), "items": items}- if router in ("compiled_agg", "fusion"):- ok, payload = await _search_layers_compiled_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)- if router == "compiled_agg":- return ok, payload- # Fusion keeps going: the compiled leg ranks alongside the chunk leg.- return await _search_layers_fusion(tenant_id, dataset_id, query, top_k, embd_mdl, kb, compiled=payload, doc_scope=doc_scope)-return await _search_layers_chunk_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)@@ -4223,112 +4152,124 @@ async def _search_layers_chunk_agg(tenant_id, dataset_id, query, top_k, embd_mdlreturn True, {"mode": "navigation_tree", "total": len(items), "items": items}-async def _search_layers_compiled_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb=None, *, doc_scope=None):- """Route to documents through compiled ``page_index`` entity rows.-- Complements the chunk leg rather than replacing it: chunks are verbatim text- and win on exact matches (numbers, proper nouns), while ``fact`` /- ``conclusion`` rows are atomic propositions that win on concept matches the- surface form misses. Each covers the other's failure mode.+async def _search_layers_claim_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb=None, *, doc_scope=None):+ """Route to documents through the compilers' claim rows.- Compiled rows carry no ``available_int``, which is the project's marker for- "visible to the generic retriever" (not a deletion flag), so they are absent- from the chunk index and have to be read through the store seam directly --- the same seam ``dataset_nav`` and wiki use.+ A claim is an atomic+ proposition carrying its own vector, so matching one means the document+ actually asserts that fact — sharper than a chunk that merely mentions the+ words. Tree claims live in their own rows with no ``knowledge_graph_kwd``;+ page_index claims are graph entities that also satisfy this filter, since+ they carry ``entity_type_kwd="claim"`` and ``scope_kwd="doc"``.- Returns ``ok=False`` only on a store failure; an empty result set is a- legitimate outcome (a KB without page_index products) and is reported- ``ok=True`` with ``total=0`` so the caller can fall back.+ Returns ``ok=True, total=0`` when the KB has no claim rows — a raptor-less or+ pre-claim KB is a legitimate empty leg, not a failure, so the caller can fall+ back."""pack = _compiled_index_or_none(kb.tenant_id, kb.id) if kb is not None else Noneif pack is None:return True, {"mode": "navigation_tree", "total": 0, "items": []}- from common.doc_store.doc_store_base import OrderByExpr- from rag.nlp.search import MatchTextExpr+ from common.doc_store.doc_store_base import MatchTextExpr, OrderByExprindex_nm, _ = pack+ pool = _NAV_CLAIM_POOL+ fields = ["content_with_weight", "source_chunk_ids", "doc_id", "name_kwd"]- pool = _NAV_COMPILED_POOL- condition = {- "compile_kwd": ["page_index"],- "knowledge_graph_kwd": ["entity"],- # Exclude the KB-wide merged rows written by the Build button.- "scope_kwd": ["doc"],- }- if doc_scope:- condition["doc_id"] = [str(d) for d in doc_scope if str(d).strip()]+ async def _recall(compile_kwds: tuple, row_types: tuple) -> dict:+ """One hybrid pass over a single (compiler, row-type) pair."""+ condition = {+ "compile_kwd": list(compile_kwds),+ "entity_type_kwd": list(row_types),+ # Exclude the KB-wide merged rows written by the Build button.+ "scope_kwd": ["doc"],+ }+ if doc_scope:+ condition["doc_id"] = [str(d) for d in doc_scope if str(d).strip()]- fields = ["content_with_weight", "source_chunk_ids", "doc_id", "name_kwd"]- try:- # Compiled rows tokenize their description into content_ltks, so a- # keyword match is a real fallback when no embedding model is configured- # or embedding fails -- the leg degrades instead of dropping out.- exprs = []+ # HYBRID recall: one BM25 leg (exact/proper-noun) + one KNN leg+ # (paraphrase), merged by row id — the dict update dedupes rows hit by+ # both legs. No similarity threshold: the store ranks, the top-N are+ # the hit set, and the caller's fallback is driven by emptiness alone.+ from rag.advanced_rag.knowlege_compile.dataset_nav import _tokenize++ legs: list[list] = []if embd_mdl:try:- exprs.append(- await settings.retriever.get_vector(- query,- embd_mdl,- top_k=pool,- # HNSW ef_search -- must be >= top_k or the ANN search- # collapses. Positional passing here would hit- # num_candidates with the similarity threshold.- num_candidates=pool,- similarity=_NAV_COMPILED_SIMILARITY,- )+ legs.append(+ [+ await settings.retriever.get_vector(+ query,+ embd_mdl,+ top_k=pool,+ # HNSW ef_search — must be >= top_k or the ANN search collapses.+ num_candidates=pool,+ )+ ])except Exception:- logging.exception("dataset_nav: compiled vector build failed for kb=%s", kb.id)- if not exprs:- exprs.append(MatchTextExpr(["content_ltks", "content_sm_ltks"], query, pool))- res = await thread_pool_exec(- settings.docStoreConn.search,- select_fields=fields,- highlight_fields=[],- condition=condition,- match_expressions=exprs,- order_by=OrderByExpr(),- offset=0,- limit=pool,- index_names=index_nm,- knowledgebase_ids=[kb.id],- )- field_map = settings.docStoreConn.get_fields(res, fields)+ logging.exception("dataset_nav: evidence vector build failed for kb=%s", kb.id)+ legs.append([MatchTextExpr(["content_ltks", "content_sm_ltks"], _tokenize(query), pool)])+ fm: dict = {}+ for exprs in legs:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ select_fields=fields,+ highlight_fields=[],+ condition=condition,+ match_expressions=exprs,+ order_by=OrderByExpr(),+ offset=0,+ limit=pool,+ index_names=index_nm,+ knowledgebase_ids=[kb.id],+ )+ fm.update(settings.docStoreConn.get_fields(res, fields) or {})+ return fm++ try:+ # Both compilers write their evidence rows as ``entity_type_kwd="claim"``+ # (page_index's pre-rename ``fact``/``conclusion`` spellings are no longer+ # searched — a recompile retypes them). The two are still queried+ # SEPARATELY, never as one mixed condition: a tree claim row carries no+ # ``knowledge_graph_kwd`` while a page_index claim is a graph entity.+ # Tree first: it is the compiled benchmark path, so the common case+ # still costs a single pass.+ field_map = await _recall(("tree",), ("claim",))+ if not field_map:+ field_map = await _recall(("page_index", "pageindex"), ("claim",))except Exception:- logging.exception("dataset_nav: compiled-agg retrieval failed for kb=%s", kb.id)- return False, {"error": "compiled retrieval failed", "code": RetCode.SERVER_ERROR}+ logging.exception("dataset_nav: claim-agg retrieval failed for kb=%s", kb.id)+ return False, {"error": "claim retrieval failed", "code": RetCode.SERVER_ERROR}buckets = _nav_bucket_compiled_rows(field_map or {})+ if not (buckets.get("claim") or {}):+ return True, {"mode": "navigation_tree", "total": 0, "items": []}+ # No similarity gate: the hybrid recall returns the store's best-matching+ # claims and the top-N ARE the hit set — the caller's fallback to the chunk+ # leg is driven purely by this leg coming back empty.ranked = _nav_rank_compiled_buckets(buckets, top_k)if not ranked:return True, {"mode": "navigation_tree", "total": 0, "items": []}- # Compile rows can outlive a deleted document -- the doc-existence check in- # retrieval() is a fallback, not the primary delete mechanism, and this leg- # reads the store directly, so it has to apply the check itself.+ # Compile rows can outlive a deleted document, and this leg reads the store+ # directly, so it applies the existence check itself.alive = await _nav_existing_doc_ids([doc_id for doc_id, _ in ranked])ranked = [(d, e) for d, e in ranked if d in alive]if not ranked:return True, {"mode": "navigation_tree", "total": 0, "items": []}+ # Same focus cap as the sibling routers: routing to many documents makes the+ # agent carry that many times the evidence in every later round.+ focus = _NAV_DOC_FOCUS_LIMIT or (top_k or pool)+ if focus > 0:+ ranked = ranked[:focus]+summaries = await _nav_doc_summaries(kb, [doc_id for doc_id, _ in ranked])- # Label each document with the TOC sections that scored for it, so the caller- # gets an entry point into the document instead of starting- # navigate_structure from scratch. Purely additive: it never prunes.- fact_names_by_doc = buckets.get("fact_names") or {}- bridge = await _nav_bridge_sections(kb, [n for names in fact_names_by_doc.values() for n in names], doc_scope=doc_scope)items = [{"doc_id": doc_id,"score": round(entry["best"], 4),- "_nav": {- "doc_id": doc_id,- "description": summaries.get(doc_id, ""),- "matched_sections": _nav_matched_sections(bridge, fact_names_by_doc.get(doc_id) or []),- },- # Report the value ranking actually used. Recomputing DocScore here- # would mix the title and fact hit counts and disagree with ordering.+ "_nav": {"doc_id": doc_id, "description": summaries.get(doc_id, "")},"_agg": {"fused": round(entry["score"], 4), "hits": entry["hits"], "legs": sorted(entry["legs"])},}for doc_id, entry in ranked@@ -4336,87 +4277,17 @@ async def _search_layers_compiled_agg(tenant_id, dataset_id, query, top_k, embd_return True, {"mode": "navigation_tree", "total": len(items), "items": items}-async def _search_layers_fusion(tenant_id, dataset_id, query, top_k, embd_mdl, kb=None, *, compiled=None, doc_scope=None):- """Combine the chunk leg and the compiled leg into one routing decision.-- The two legs measure different things and are not directly comparable: chunk- ``_score`` is Infinity's normalized weighted sum, while the compiled leg is a- raw cosine. Each leg is therefore normalized by its own maximum before- being combined, so neither leg's scale can dominate the other.- """- ok, chunk_payload = await _search_layers_chunk_agg(tenant_id, dataset_id, query, top_k, embd_mdl, kb, doc_scope=doc_scope)- if not ok:- return ok, chunk_payload-- compiled_items = (compiled or {}).get("items") or []- if not compiled_items:- # No page_index products (or a raptor KB): the chunk leg alone decides.- # Tag the items so callers see one shape whichever path produced them.- for item in chunk_payload.get("items") or []:- item.setdefault("_agg", {}).setdefault("legs", ["chunk"])- return ok, chunk_payload-- legs = [("chunk", chunk_payload.get("items") or [], _NAV_FUSION_WEIGHTS["chunk"]), ("compiled", compiled_items, 1.0)]- fused = _nav_fuse_legs(legs, top_k)-- summaries = await _nav_doc_summaries(kb, [doc_id for doc_id, _ in fused])- items = [- {- "doc_id": doc_id,- # Reported score stays on the 0..1 scale callers threshold on, so it- # is taken from the highest-scoring leg rather than the fused value.- "score": round(info["best"], 4),- "_nav": {"doc_id": doc_id, "description": summaries.get(doc_id, "")},- "_agg": {"fused": round(info["fused"], 4), "legs": sorted(info["legs"])},- }- for doc_id, info in fused- ]- return True, {"mode": "navigation_tree", "total": len(items), "items": items}---def _nav_fuse_legs(legs: list[tuple[str, list, float]], top_k) -> list[tuple[str, dict]]:- """Fuse per-document leg rankings, each normalized by its own maximum."""- normalized = []- for name, items, weight in legs:- peak = max((float(i.get("score") or 0.0) for i in items), default=0.0)- if peak <= 0:- continue- normalized.append((name, [(i.get("doc_id"), float(i.get("score") or 0.0) / peak * weight) for i in items], weight))-- merged: dict[str, dict] = {}- for name, entries, _ in normalized:- for doc_id, value in entries:- if not doc_id:- continue- info = merged.get(doc_id)- if info is None:- merged[doc_id] = {"fused": value, "best": value, "legs": {name}}- continue- info["fused"] += value- info["legs"].add(name)- if value > info["best"]:- info["best"] = value-- ranked = sorted(merged.items(), key=lambda kv: kv[1]["fused"], reverse=True)- if top_k is not None and top_k > 0:- ranked = ranked[:top_k]- return ranked--def _nav_bucket_compiled_rows(field_map: dict) -> dict[str, dict]:- """Split fetched compiled rows into a title bucket and a fact bucket.+ """Bin fetched compiled rows into the claim bucket.The row ``type`` lives inside the ``content_with_weight`` payload —``entity_type_kwd`` is a graphrag field and is not written by thecompilation path — so rows are binned after the fetch.- Fact names are kept per document, because the section bridge has to label a- document with the sections that scored for *it* — a flat list would attach- one document's sections to another.+ Only claim rows are a routing leg: title/fact rows are not, so a row whose+ type is not a claim is dropped here rather than binned."""- title_bucket: dict[str, dict] = {}- fact_bucket: dict[str, dict] = {}- fact_names: dict[str, list[str]] = {}+ claim_bucket: dict[str, dict] = {}for row in field_map.values():doc_id = str(row.get("doc_id") or "").strip()if not doc_id:@@ -4428,58 +4299,35 @@ def _nav_bucket_compiled_rows(field_map: dict) -> dict[str, dict]:if not isinstance(payload, dict):continuertype = str(payload.get("type") or "").strip().lower()- if rtype in _NAV_COMPILED_TITLE_TYPES:- bucket = title_bucket- elif rtype in _NAV_COMPILED_FACT_TYPES:- bucket = fact_bucket- # Kept for the section bridge, which labels a hit fact with its- # owning title. The raw fetch order is the store's relevance order,- # which is the right priority for choosing which sections to show.- name = str(payload.get("name") or "").strip()- if name:- fact_names.setdefault(doc_id, []).append(name)- else:+ if rtype not in _NAV_EVIDENCE_ROW_TYPES:continuescore = float(row.get("similarity") or row.get("_score") or 0.0)- entry = bucket.get(doc_id)+ entry = claim_bucket.get(doc_id)if entry is None:- bucket[doc_id] = {"total": score, "best": score, "hits": 1}+ claim_bucket[doc_id] = {"total": score, "best": score, "hits": 1}continueentry["total"] += scoreentry["hits"] += 1if score > entry["best"]:entry["best"] = score- return {"title": title_bucket, "fact": fact_bucket, "fact_names": fact_names}+ return {"claim": claim_bucket}def _nav_rank_compiled_buckets(buckets: dict, top_k) -> list[tuple[str, dict]]:- """Fuse the title and fact buckets into one per-document ranking.+ """Rank documents by the claim bucket alone.- A document hit by both legs is stronger than one hit by either alone, so the- scores are summed rather than maxed — but each bucket is ranked by its own- DocScore first, which is what keeps facts from swamping titles.+ Claims are the only compiled routing leg, so this is a straight DocScore+ ranking over that bucket — there is no second compiled leg to fuse with."""merged: dict[str, dict] = {}- for leg, bucket in (("title", buckets.get("title") or {}), ("fact", buckets.get("fact") or {})):- weight = _NAV_FUSION_WEIGHTS.get(leg, 1.0)- for doc_id, entry in bucket.items():- contribution = _nav_doc_score(entry) * weight- cur = merged.get(doc_id)- if cur is None:- merged[doc_id] = {- "score": contribution,- "best": entry["best"],- "total": entry["total"],- "hits": entry["hits"],- "legs": {leg},- }- continue- cur["score"] += contribution- cur["total"] += entry["total"]- cur["hits"] += entry["hits"]- cur["legs"].add(leg)- if entry["best"] > cur["best"]:- cur["best"] = entry["best"]+ for doc_id, entry in (buckets.get("claim") or {}).items():+ merged[doc_id] = {+ "score": _nav_doc_score(entry),+ "best": entry["best"],+ "total": entry["total"],+ "hits": entry["hits"],+ "legs": {"claim"},+ }ranked = sorted(merged.items(), key=lambda kv: kv[1]["score"], reverse=True)if top_k is not None and top_k > 0:@@ -4487,88 +4335,6 @@ def _nav_rank_compiled_buckets(buckets: dict, top_k) -> list[tuple[str, dict]]:return ranked-def _nav_matched_sections(bridge: dict[str, str], fact_names: list[str]) -> list[str]:- """Titles owning this document's hit facts, in hit order and de-duplicated."""- sections: list[str] = []- for name in fact_names:- title = bridge.get(str(name).strip().lower())- if title and title not in sections:- sections.append(title)- return sections---async def _nav_bridge_sections(kb, fact_names: list[str], doc_scope=None) -> dict[str, str]:- """Map each hit fact to the title that includes it.-- Routing then answers *which document* and *where in it to look first*:- the caller gets a per-document entry point instead of starting- ``navigate_structure`` from scratch.-- The TOC is used here purely as a **label**. It never prunes the candidate- set — that is the top-down routing failure mode. The lookup happens after- the documents are already chosen.-- Returns ``{fact_name: title_name}``; facts without an owning title are- omitted.- """- if not fact_names or kb is None:- return {}- pack = _compiled_index_or_none(kb.tenant_id, kb.id)- if pack is None:- return {}- index_nm, _ = pack-- from common.doc_store.doc_store_base import OrderByExpr-- names = [str(n).strip().lower() for n in fact_names if str(n).strip()]- # Cap the terms query: a wide IN list is slow and the bridge only needs to- # label the top hits, not every candidate.- names = list(dict.fromkeys(names))[:_NAV_BRIDGE_NAMES_LIMIT]-- condition = {- "compile_kwd": ["page_index"],- "knowledge_graph_kwd": ["relation"],- "scope_kwd": ["doc"],- "to_entity_kwd": names,- }- if doc_scope:- condition["doc_id"] = [str(d) for d in doc_scope if str(d).strip()]-- fields = ["content_with_weight", "from_entity_kwd", "to_entity_kwd", "doc_id"]- try:- res = await thread_pool_exec(- settings.docStoreConn.search,- select_fields=fields,- highlight_fields=[],- condition=condition,- match_expressions=[],- order_by=OrderByExpr(),- offset=0,- limit=len(names) * _NAV_BRIDGE_FANOUT,- index_names=index_nm,- knowledgebase_ids=[kb.id],- )- field_map = settings.docStoreConn.get_fields(res, fields)- except Exception:- logging.exception("dataset_nav: section bridge lookup failed for kb=%s", kb.id)- return {}-- bridge: dict[str, str] = {}- for row in (field_map or {}).values():- to_name = str(row.get("to_entity_kwd") or "").strip().lower()- title = str(row.get("from_entity_kwd") or "").strip()- if not to_name or not title or to_name in bridge:- continue- try:- payload = json.loads(str(row.get("content_with_weight") or "{}"))- except (TypeError, ValueError):- continue- # Only the structural edge counts; semantic relations would mislabel.- if isinstance(payload, dict) and str(payload.get("type") or "").strip().lower() == _NAV_INCLUDE_RELATION:- bridge[to_name] = title- return bridge--async def _nav_existing_doc_ids(doc_ids: list[str]) -> set[str]:"""Keep only documents that still exist.@@ -4764,8 +4530,8 @@ async def _search_layers_all(tenant_id, dataset_id, query, top_k, embd_mdl, kb,"""Run all modes and return the union of doc_ids, with best score per doc."""import asyncio as _asyncio- # With _NAV_TREE_ROUTER="chunk_agg" the navigation_tree leg already reads the- # chunk index, so it overlaps the chunk leg below. The union still dedups by+ # The navigation_tree leg falls back to the chunk index whenever the dataset+ # has no claim rows, so it overlaps the chunk leg below. The union still dedups by# doc_id and keeps the best score, so the overlap costs one extra retrieval# without affecting the result — mode="all" is for comparison, not production.result_lists = await _asyncio.gather(diff --git a/api/apps/services/structure_graph_common.py b/api/apps/services/structure_graph_common.pyindex 6fb80ef778..2e4906cd79 100644--- a/api/apps/services/structure_graph_common.py+++ b/api/apps/services/structure_graph_common.py@@ -361,6 +361,180 @@ async def build_bucket(index_name, kb_id, scope: dict, excluded_doc_ids: set[strreturn entities, normalize_relation_endpoints(entities, relations)+# ---------------------------------------------------------------------------+# Claim rows (entity_type_kwd="claim") — the tree UI's leaf fact list.+# ---------------------------------------------------------------------------+++def project_claim(row: dict) -> dict | None:+ """Project a raw claim row to the API shape, keeping its verbatim evidence.++ Unlike graph entities, the quote and its location ARE the payload — that is+ what a compiled claim exists to carry (see claim_evidence.md §4).+ """+ try:+ payload = json.loads(row.get("content_with_weight") or "{}")+ except Exception:+ return None+ if not isinstance(payload, dict):+ return None+ name = str(payload.get("name") or "").strip()+ if not name:+ return None+ chunk_ids = row.get("source_chunk_ids") or payload.get("source_chunk_ids") or []+ if isinstance(chunk_ids, str):+ chunk_ids = [chunk_ids]+ if not isinstance(chunk_ids, (list, tuple)):+ chunk_ids = []+ claim: dict = {+ "name": name,+ "description": str(payload.get("description") or name),+ "source_chunk_ids": [str(c).strip() for c in chunk_ids if str(c).strip()],+ }+ typ = str(payload.get("type") or "").strip()+ if typ:+ claim["type"] = typ+ evidence = payload.get("evidence")+ if isinstance(evidence, list):+ kept = [e for e in evidence if isinstance(e, dict) and str(e.get("quote") or "").strip()]+ if kept:+ claim["evidence"] = kept+ return claim+++# Row types that carry a claim. ``claim`` is the type every compiler writes now+# (page_index folded its ``fact``/``conclusion`` types into it); the older two+# spellings stay listed so rows compiled before the rename keep showing up+# until the dataset is recompiled.+CLAIM_ROW_TYPES = ("claim", "fact", "conclusion")+++async def list_claim_rows(+ index_name: str,+ kb_id: str,+ doc_id: str,+ compile_kwd: str | None = None,+ compilation_template_id: str | None = None,+ chunk_ids: list[str] | None = None,+ offset: int = 0,+ limit: int = 20,+ row_types: tuple[str, ...] = CLAIM_ROW_TYPES,+) -> tuple[list[dict], int]:+ """Page through one document's claim rows.++ ``chunk_ids`` narrows the page to claims sourced from those chunks — the+ tree UI passes a leaf cluster's members when the user expands it; without+ it the endpoint pages the whole document.++ ``compile_kwd`` must be the template's own kind, not a fixed "tree": a+ page_index document stores ``compile_kwd="page_index"``, so hard-coding+ "tree" returned nothing for it. Pass ``None`` to scope by document and+ template alone.+ """+ condition: dict = {+ "doc_id": [doc_id],+ "entity_type_kwd": [str(t) for t in (row_types or CLAIM_ROW_TYPES)],+ }+ if compile_kwd:+ condition["compile_kwd"] = [compile_kwd]+ if compilation_template_id:+ condition["compilation_template_ids"] = [compilation_template_id]+ if chunk_ids:+ condition["source_chunk_ids"] = sorted({c for c in chunk_ids if c})+ field_map, total = await graph_search(+ index_name,+ kb_id,+ ["content_with_weight", "source_chunk_ids"],+ condition,+ OrderByExpr(),+ max(int(limit or 0), 1),+ offset=max(int(offset or 0), 0),+ )+ claims = [c for c in (project_claim(r) for r in field_map.values()) if c]+ return claims, total+++async def attach_claim_counts(+ entities: list[dict],+ relations: list[dict],+ index_name: str,+ kb_id: str,+ doc_id: str,+ compile_kwd: str = "tree",+ leaves_only: bool = True,+) -> None:+ """Stamp entities with the number of claims sourced to their chunks.++ One scan of the document's claim rows, counted per chunk id, keeps this to+ a single query.++ ``leaves_only=True`` (tree): only leaves carry a badge, so a parent never+ double-counts its descendants' claims.++ ``leaves_only=False`` (page_index): every non-claim entity is stamped with+ the claims whose chunks fall inside its own span — for a heading that is+ exactly its subtree, because a title's ``source_chunk_ids`` cover the whole+ section. The UI opens a node's claims from any level, so a parent needs the+ badge as much as a leaf does. Claim entities themselves are skipped: they+ are the counted rows, not badge carriers.+ """+ if not entities:+ return+ condition: dict = {+ "doc_id": [doc_id],+ "entity_type_kwd": [str(t) for t in CLAIM_ROW_TYPES],+ }+ if compile_kwd:+ condition["compile_kwd"] = [compile_kwd]+ try:+ field_map, _ = await graph_search(+ index_name,+ kb_id,+ ["content_with_weight", "source_chunk_ids"],+ condition,+ OrderByExpr(),+ 10000,+ )+ except Exception:+ logging.exception("structure graph: claim count scan failed for doc=%s", doc_id)+ return+ if not field_map:+ return++ counts: dict[str, int] = {}+ for row in field_map.values():+ ids = row.get("source_chunk_ids") or []+ if isinstance(ids, str):+ ids = [ids]+ for cid in ids:+ cid = str(cid).strip()+ if cid:+ counts[cid] = counts.get(cid, 0) + 1+ if not counts:+ return++ inner: set[str] = set()+ if leaves_only:+ for edge in relations or []:+ if isinstance(edge, dict):+ src = str(edge.get("from") or "").strip()+ if src:+ inner.add(src)+ claim_types = {str(t).strip().lower() for t in CLAIM_ROW_TYPES}+ for entity in entities:+ # Claims are the rows being counted, not badge carriers (page_index+ # stores them as entities; the tree filters them out of the view).+ if str(entity.get("type") or "").strip().lower() in claim_types:+ continue+ if leaves_only and _entity_response_id(entity) in inner:+ continue # not a leaf: its descendants' claims are counted there+ total = 0+ for cid in entity.get("source_chunk_ids") or []:+ total += counts.get(str(cid).strip(), 0)+ if total:+ entity["claim_count"] = total++async def keyword_subgraph(index_name,kb_id,diff --git a/api/db/init_data/compilation_templates/page_index.yaml b/api/db/init_data/compilation_templates/page_index.yamlindex a0b055fec5..e4c089ceb8 100644--- a/api/db/init_data/compilation_templates/page_index.yaml+++ b/api/db/init_data/compilation_templates/page_index.yaml@@ -13,57 +13,89 @@ config:entity:description: >-First extract all actual headings in source order, then extract the- important facts or conclusions under them. Treat explicit chapter markers such as 第N回、+ atomic claims under them. Treat explicit chapter markers such as 第N回、第N章、Chapter N, numbered headings, and Markdown headings as titleseven when they have no `#` marker. A `title` must appear in the source;- copy it exactly and never invent one from a fact or summary. Titles have- priority over facts and conclusions: complete title extraction first and- never replace a title with a fact or conclusion. Do not downgrade an actual- chapter heading to a fact. For other entities, use concise atomic,- source-grounded names and descriptions. Do not invent.+ copy it exactly and never invent one from a claim or summary. Titles have+ priority over claims: complete title extraction first and never replace a+ title with a claim. Do not downgrade an actual chapter heading to a claim.+ For other entities, use concise atomic, source-grounded names and+ descriptions. Do not invent.+ # Extra output keys rendered into the Response Format skeleton. A key+ # described only inside a type's `rule` is never emitted by the model — it+ # must appear in the skeleton (see _struct_render_type_fields).+ output_fields:+ - name: evidence+ type: list+ required: false+ description: >-+ verbatim source text supporting this claim, copied exactly: a CONTIGUOUS+ substring of the chunk named by `chunk_id` — same words, same order, no+ paraphrase, no truncation, no added words — kept under 240 characters.+ For tables, infoboxes and bullet lists, quote the raw cell/row text+ exactly as it appears, keeping its separators and order. Emit for type+ `claim`; omit for `title`. A quote that cannot be located verbatim is+ rejected downstream, so never restate it.+ shape: >-+ [{"quote": "<verbatim source sentence, copied exactly>",+ "chunk_id": "<source chunk id the sentence was taken from>"}]fields:- type: titledescription: the heading text (clean, no page numbers or leader dots)rule: |- Identify and emit every explicit heading or chapter marker first, in source order, including 第N回 even without Markdown syntax.- - Copy the clean heading text; do not invent, paraphrase, or promote facts into titles.+ - Copy the clean heading text; do not invent, paraphrase, or promote claims into titles.- Assign source_chunk_ids yourself: include every source chunk covered by the title'ssection, not only the chunk containing the heading. A parent title includes thechunks of its nested child sections; a child title includes only its own subtree.Stop at the next title of the same or an ancestor level.- A line beginning with 第N回/第N章/第N节 or Chapter N is a title when it appears in the source.- - Do not emit the same chapter heading as a fact or conclusion.- - Only after title extraction, emit facts or conclusions under the corresponding titles.- - Summarize the content under each title without copying long passages.- - type: fact- description: a concise source-grounded factual statement belonging to a title+ - Do not emit the same chapter heading as a claim.+ - Only after title extraction, emit the claims under the corresponding titles.+ - The title's own `description` may summarize the content beneath it, but+ that summary never replaces emitting the individual claims: summarize+ on the title, atomize on the claims.+ - type: claim+ description: >-+ an atomic, source-grounded proposition belonging to a title: a factual+ assertion, or a source-supported finding, outcome or conclusion. It is+ the same object the tree compiler calls a claim — an atomic+ proposition plus the verbatim evidence that grounds it.rule: |- - Preserve important dates, numbers, names, units, qualifiers, and values.- - Do not merge unrelated facts into one entity.- - type: conclusion- description: a source-supported finding, outcome, or conclusion belonging to a title- rule: |- - State the conclusion concisely and do not add unsupported interpretation.+ - Extract EVERY explicit claim each source chunk supports: factual+ assertions AND opinions, judgments, assessments, recommendations,+ predictions and conclusions. Never rank, summarize, merge or drop+ one as minor.+ - Self-contained: named subjects, readable alone. Never "the article",+ "the document", "it says" or a bare pronoun — resolve the referent.+ - Faithful: never invent, strengthen or infer. Preserve modality,+ uncertainty, negation, attribution, numbers, units and dates exactly.+ - Atomic: exactly one assertion, supportable by a single verbatim+ sentence. Split compound sentences; never generalize across+ sentences or sections.+ - If no single sentence supports a claim, SPLIT it until one does —+ never emit one ungrounded. An unverifiable quote is dropped+ downstream, so a claim without evidence is worthless.+ - Keep claims in the language of the source.relation:description: >-You are an expert logical reasoning assistant specializing in documenthierarchy and source-grounded detail attachment. Build relations onlyfrom the entities in the current output. Preserve the title hierarchy- and attach every fact, date, number, named entity, and conclusion to the- nearest relevant title.+ and attach every claim to the nearest relevant title.fields:- type: includedescription: Upper-level title includes lower-level title.rule: |- "-1" is an invalid title; it does not belong to or include any other titles.- Must follow the hierarchical index/numbering (e.g., "1", "2.1", "3.2.5") when present.- - A title may include a child title or a detail entity.- - Every entity with type `fact` or `conclusion` must have exactly one nearest-title parent.- - Detail entities must not be attached directly to another detail entity.- - Do not create relations between unrelated detail entities.+ - A title may include a child title or a claim.+ - Every entity with type `claim` must have exactly one nearest-title parent.+ - A claim must not be attached directly to another claim.+ - Do not create relations between unrelated claims.- Every `from` and `to` value must exactly match an entity `name` in the current output.- Never use a source heading as an endpoint unless that heading was also emitted as a `title` entity.- - Before returning, remove any relation whose endpoint is missing and verify that every detail entity has one parent relation.+ - Before returning, remove any relation whose endpoint is missing and verify that every claim has one parent relation.- Keep language of "title" the same as the input.- - 第N回、 第N章 must include their actual child sections or detail entities when present.+ - 第N回、 第N章 must include their actual child sections or claims when present.global_rules: ''diff --git a/api/db/init_data/compilation_templates/tree.yaml b/api/db/init_data/compilation_templates/tree.yamlindex 1b4c6ffc7d..42ee64f5a4 100644--- a/api/db/init_data/compilation_templates/tree.yaml+++ b/api/db/init_data/compilation_templates/tree.yaml@@ -19,6 +19,50 @@ config:max_token: 512clustering_threshold: 0.3clustering_ratio: 0.5+ # Claim/evidence harvesting over the tree's leaf chunks, before clustering:+ # every cluster is then summarized from its members' claims instead of raw+ # text, and the claims are persisted as their own searchable rows. The+ # {cluster_content} placeholder does NOT appear here — the chunk texts are+ # rendered separately, each marked as a TARGET.+ claim_prompt: |-+ ## Task+ You are a high-recall claim harvester for the TARGET chunks below. For EACH+ TARGET chunk, extract EVERY explicit atomic claim that chunk supports: factual+ assertions AND explicitly expressed opinions, beliefs, judgments, assessments,+ recommendations, preferences, intentions, predictions, and hypotheses. Do not+ rank claims, summarize, merge claims, or omit a claim because it seems minor.++ A claim must be:+ - Self-contained: readable without the surrounding text, with named subjects.+ Never write "the article", "the document", "it says", or a bare pronoun.+ Resolve the referent to its name when attribution makes the claim clearer.+ - Faithful: stated only if that TARGET chunk supports it. Never invent,+ strengthen, or infer. Preserve modality, uncertainty, negation, and+ attribution exactly.+ - Atomic: exactly one fact. Split compound sentences.++ ## Response Format+ Reply with a single JSON object: {"items": [{"type": "claim", "name": "<the claim, one sentence>", "description": "<optional restatement for retrieval>", "source_chunk_ids": ["<CHUNK_ID it was taken from>"], "evidence": [{"quote": "<the verbatim source sentence>", "chunk_id": "<CHUNK_ID it was taken from>"}]}, ...]}.++ Rules:+ - `evidence.quote` MUST be a CONTIGUOUS verbatim substring of the chunk cited+ by `evidence.chunk_id`: same words, same order, no paraphrase, no truncation,+ no added words. A quote that cannot be found verbatim is rejected downstream,+ so never restate.+ - `evidence.chunk_id` and `source_chunk_ids` MUST identify the exact chunk the+ claim and quote came from. Never cross-attribute a quote to a different chunk.+ - Keep each quote concise and under 240 characters.+ - For tables, infoboxes and bullet lists, the quote MUST be the raw cell/row+ text exactly as it appears — keep its separators and order. Do NOT turn a+ table row like "Starring | Penn Badgley | Elizabeth Lail" into a sentence+ like "Starring Penn Badgley Elizabeth Lail"; that is a paraphrase and will+ be rejected.+ - Preserve numbers, units, dates, names and qualifiers exactly.+ - Distribute claims across all TARGET chunks; do not skip a chunk because you+ reached the cap — the cap is per chunk, not per batch.+ - If a TARGET chunk contains no extractable assertion, emit no claim for it.+ - Keep claims in the same language as the source.+ Return JSON only, no commentary.# Entity / relation collections are not used by the tree kind but kept# as empty stubs so the form schema's shared validators see a stable# shape regardless of kind.diff --git a/internal/handler/dataset_artifact.go b/internal/handler/dataset_artifact.goindex 10c36de214..c4b2064ab9 100644--- a/internal/handler/dataset_artifact.go+++ b/internal/handler/dataset_artifact.go@@ -436,3 +436,62 @@ func (h *DatasetArtifactHandler) GetDocumentGraph(c *gin.Context) {}common.SuccessWithData(c, resp, "success")}++// GetDocumentClaims handles GET /documents/<document_id>/structure/claims —+// page one document's claim/evidence rows (entity_type_kwd="claim"). The tree+// UI fetches them per leaf cluster on demand: chunk_ids carries the cluster's+// members, template_id scopes the compilation template, limit is capped at 100+// (mirrors Python chunk_api.get_document_structure_claims).+func (h *DatasetArtifactHandler) GetDocumentClaims(c *gin.Context) {+ _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))+ if tenantID == "" {+ return+ }+ datasetID := c.Param("dataset_id")+ documentID := c.Param("document_id")++ templateID := strings.TrimSpace(c.Query("template_id"))+ var chunkIDs []string+ for _, id := range strings.Split(c.Query("chunk_ids"), ",") {+ if id = strings.TrimSpace(id); id != "" {+ chunkIDs = append(chunkIDs, id)+ }+ }+ offset := 0+ if v, err := strconv.Atoi(c.Query("offset")); err == nil && v > 0 {+ offset = v+ }+ limit := 20+ if v, err := strconv.Atoi(c.Query("limit")); err == nil {+ limit = v+ }+ if limit < 1 {+ limit = 1+ }+ if limit > 100 {+ limit = 100+ }++ claims, total, err := h.svc.ListDocumentStructureClaims(c.Request.Context(), tenantID, datasetID, documentID, templateID, chunkIDs, offset, limit)+ if err != nil {+ common.ErrorWithCode(c, common.CodeExceptionError, err.Error())+ return+ }+ common.SuccessWithData(c, gin.H{"claims": claims, "total": total, "offset": offset, "limit": limit}, "success")+}++// DeleteDocumentGraph handles DELETE /documents/<document_id>/structure/graph — delete document structure graph.+func (h *DatasetArtifactHandler) DeleteDocumentGraph(c *gin.Context) {+ _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))+ if tenantID == "" {+ return+ }+ datasetID := c.Param("dataset_id")+ documentID := c.Param("document_id")+ n, err := h.svc.DeleteDocumentGraph(c.Request.Context(), tenantID, datasetID, documentID)+ if err != nil {+ common.ErrorWithCode(c, common.CodeDataError, err.Error())+ return+ }+ common.SuccessWithData(c, gin.H{"deleted": n}, "success")+}diff --git a/internal/ingestion/component/knowledge_compiler/common/deps.go b/internal/ingestion/component/knowledge_compiler/common/deps.goindex 56f396cd92..dacd7d3127 100644--- a/internal/ingestion/component/knowledge_compiler/common/deps.go+++ b/internal/ingestion/component/knowledge_compiler/common/deps.go@@ -28,6 +28,13 @@ type ChatRequest struct {MaxTokens *intAPIKey stringBaseURL string+ // DisableThinking asks the driver to turn chain-of-thought OFF. Reasoning+ // models (MiniMax-M1/M3, kimi, qwen) otherwise spend the completion budget+ // (and minutes) on visible COT before returning the structured payload —+ // the fastest way to slow knowledge compilation to a crawl. Providers that+ // have no thinking control silently ignore it. Defaults to false, so only+ // call sites that know the model reasons opt in.+ DisableThinking bool// DisableRetry tells the production ChatInvoker that the caller owns// transient-error retries (GenJSON is such a caller). It is internal// plumbing and is not part of the user-facing model request.diff --git a/internal/ingestion/component/knowledge_compiler/common/id.go b/internal/ingestion/component/knowledge_compiler/common/id.goindex e5e1292c9f..8858787977 100644--- a/internal/ingestion/component/knowledge_compiler/common/id.go+++ b/internal/ingestion/component/knowledge_compiler/common/id.go@@ -24,6 +24,18 @@ import (// re-runs regenerate them deterministically; if the encoding or hash ever// changes after GA, a backfill/migration of existing compiled rows is// required before rollout.+//+// CROSS-RUNTIME: Python's stable_row_id+// (rag/advanced_rag/knowlege_compile/_common.py) hashes ":"-joined parts with+// xxhash64, so the two runtimes mint DIFFERENT ids for the same logical row.+// This is deliberate rather than an oversight: this side keeps the 256-bit+// digest (M10) and the length-prefixed encoding, which a part containing ":"+// cannot forge. Ids are only ever compared within a runtime — cleanup is scoped+// by doc/template columns, never by id — so a document later recompiled by the+// other runtime rewrites its rows instead of leaving duplicates behind.+// Unifying the algorithms would mean giving up one of those two properties for+// no functional gain; revisit only if rows ever need to be addressable across+// runtimes.func StableRowID(parts ...string) string {h := sha256.New()var lenBuf [4]bytediff --git a/internal/ingestion/component/knowledge_compiler/common/indextext.go b/internal/ingestion/component/knowledge_compiler/common/indextext.gonew file mode 100644index 0000000000..791a6dea53--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/common/indextext.go@@ -0,0 +1,107 @@+package common++import (+ "fmt"+ "sort"+ "strings"+)++// IndexExcludedKeys names payload keys that must never reach the embedding or+// the BM25 columns.+//+// "evidence" carries verbatim source quotes for claim/evidence compilation+// (see rag/advanced_rag/knowlege_compile/claim_evidence.md). Feeding it into+// the vector would embed a mixed "claim + raw source" unit instead of the+// claim the geometric layer is meant to index, and a []any of maps would+// stringify to Go map syntax and pollute the tokens. It is kept out of BM25 in+// the first phase too, so the payload change stays isolated from any recall+// change.+// "source_chunk_ids" and "mention_count" are bookkeeping, not semantics: the+// ids are opaque hex that tokenizes into garbage terms and drifts the vector,+// and the count is a number every row shares. They must stay in lockstep with+// _STRUCT_INDEX_EXCLUDED_KEYS in+// rag/advanced_rag/knowlege_compile/structure.py, or the two runtimes index+// the same payload into different vectors.+var IndexExcludedKeys = map[string]bool{+ "evidence": true,+ "source_chunk_ids": true,+ "mention_count": true,+}++// PayloadDescription is the single source of truth for the text a compiled row+// is indexed by. It mirrors Python _struct_payload_description in+// rag/advanced_rag/knowlege_compile/structure.py: the concatenated string+// values of every payload field, with slices flattened.+//+// Go maps lose JSON insertion order, so keys are sorted to keep the embedding+// deterministic across runs.+//+// excluded names additional payload keys to skip. A nil map means "use+// IndexExcludedKeys". Callers that build the embedding and the BM25 columns+// may pass different sets, which lets evidence stay out of the vector while a+// separate BM25 trial remains possible.+//+// NOTE: the tree variant used to skip "description" here while claiming to+// match Python; that dropped the primary semantic content from every tree row+// vector and diverged from the structure variant. Both variants now share this+// implementation.+func PayloadDescription(payload map[string]any, excluded map[string]bool) string {+ if excluded == nil {+ excluded = IndexExcludedKeys+ }+ keys := make([]string, 0, len(payload))+ for k := range payload {+ if !excluded[k] {+ keys = append(keys, k)+ }+ }+ sort.Strings(keys)+ var parts []string+ for _, k := range keys {+ switch v := payload[k].(type) {+ case []any:+ for _, item := range v {+ if s := strings.TrimSpace(StringOf(item)); s != "" {+ parts = append(parts, s)+ }+ }+ case []string:+ for _, item := range v {+ if s := strings.TrimSpace(item); s != "" {+ parts = append(parts, s)+ }+ }+ default:+ if s := strings.TrimSpace(StringOf(v)); s != "" {+ parts = append(parts, s)+ }+ }+ }+ return strings.Join(parts, " ")+}++// StringOf renders a scalar payload value for index text. Maps and slices+// render as "" because they are not scalar text; callers flatten them+// themselves.+func StringOf(v any) string {+ switch x := v.(type) {+ case nil:+ return ""+ case string:+ return x+ case float64:+ return strings.TrimSuffix(fmt.Sprintf("%v", x), ".0")+ case float32:+ return strings.TrimSuffix(fmt.Sprintf("%v", x), ".0")+ case int:+ return fmt.Sprintf("%v", x)+ case int64:+ return fmt.Sprintf("%v", x)+ case bool:+ if x {+ return "true"+ }+ return "false"+ }+ return ""+}diff --git a/internal/ingestion/component/knowledge_compiler/common/indextext_test.go b/internal/ingestion/component/knowledge_compiler/common/indextext_test.gonew file mode 100644index 0000000000..48cc5d3397--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/common/indextext_test.go@@ -0,0 +1,90 @@+package common++import "testing"++func TestPayloadDescriptionExcludesEvidenceByDefault(t *testing.T) {+ payload := map[string]any{+ "name": "capital",+ "type": "fact",+ "description": "The capital is Paris.",+ "evidence": []any{+ map[string]any{"quote": "The capital is Paris.", "chunk_id": "c1", "start": 0, "end": 21},+ },+ }+ got := PayloadDescription(payload, nil)++ if want := "The capital is Paris."; !contains(got, want) {+ t.Errorf("claim text missing from index text: %q", got)+ }+ // A nested map must never render as Go map syntax into the index text.+ if contains(got, "map[") || contains(got, "chunk_id") || contains(got, "quote:") {+ t.Errorf("evidence leaked into index text: %q", got)+ }+}++func TestPayloadDescriptionHonoursCustomExclusion(t *testing.T) {+ payload := map[string]any{"a": "one", "b": "two"}+ if got := PayloadDescription(payload, map[string]bool{"b": true}); got != "one" {+ t.Errorf("custom exclusion not applied: %q", got)+ }+}++func TestPayloadDescriptionIsDeterministic(t *testing.T) {+ // Go map iteration order is random; keys must be sorted so vectors are+ // stable across runs.+ payload := map[string]any{"z": "last", "a": "first", "m": "middle"}+ want := "first middle last"+ for i := 0; i < 50; i++ {+ if got := PayloadDescription(payload, nil); got != want {+ t.Fatalf("non-deterministic index text: got %q want %q", got, want)+ }+ }+}++func TestPayloadDescriptionFlattensSlices(t *testing.T) {+ payload := map[string]any{+ "tags": []string{"x", "", "y"},+ "mixed": []any{"z", 42, nil},+ "skipped": []string{"nope"},+ }+ got := PayloadDescription(payload, map[string]bool{"skipped": true})+ if !contains(got, "x") || !contains(got, "y") || !contains(got, "z") {+ t.Errorf("slices not flattened: %q", got)+ }+ if contains(got, "nope") {+ t.Errorf("excluded slice key present: %q", got)+ }+}++func TestStringOfScalars(t *testing.T) {+ cases := []struct {+ in any+ want string+ }{+ {"s", "s"},+ {float64(3), "3"},+ {float64(3.5), "3.5"},+ {int64(7), "7"},+ {true, "true"},+ {false, "false"},+ {nil, ""},+ {map[string]any{}, ""},+ }+ for _, c := range cases {+ if got := StringOf(c.in); got != c.want {+ t.Errorf("StringOf(%#v) = %q want %q", c.in, got, c.want)+ }+ }+}++func contains(s, sub string) bool {+ if sub == "" {+ return true+ }+ for i := 0; i+len(sub) <= len(s); i++ {+ if s[i:i+len(sub)] == sub {+ return true+ }+ }+ return false+}diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex 05e3a07978..97c2fa394a 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -412,7 +412,19 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {// are ignored (tokenizer pool may be uninitialised in no-CGo tests),// leaving the fields empty — matching the chunker's graceful-degrade// behaviour.- if ltks, err := tokenizer.Tokenize(p.Content); err == nil && ltks != "" {+ //+ // Structure rows tokenize the FLATTENED PAYLOAD DESCRIPTION, not the+ // raw JSON: Python indexes+ // _tokenize_for_search(_struct_payload_description(payload)), so+ // tokenizing p.Content here would feed JSON keys/brackets and opaque+ // chunk ids into the inverted index.+ indexText := p.Content+ if p.Variant == common.VariantStructure {+ if d := structure.IndexText(p.Content); strings.TrimSpace(d) != "" {+ indexText = d+ }+ }+ if ltks, err := tokenizer.Tokenize(indexText); err == nil && ltks != "" {doc.ContentLtks = ltksif sm, err := tokenizer.FineGrainedTokenize(ltks); err == nil && sm != "" {doc.ContentSmLtks = sm@@ -608,6 +620,30 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {// same storage contract as the structure variant, so both share// applyStructureGraphColumns.return applyStructureGraphColumns(doc, p, kind)+ case "claim":+ // Claim rows are searchable on their own (global KNN) but are NOT+ // part of the structure graph: they carry no relation, and a+ // relation-less row would be rendered as a root in the artifacts+ // tree. So they deliberately skip knowledge_graph_kwd, which keeps+ // them out of the artifacts query (it filters+ // knowledge_graph_kwd=["entity","relation"]) without a frontend+ // change. Python mirrors this in _struct_upsert_tree_claim_rows.+ if v := metaString(p.Meta, "name"); v != "" {+ if err := doc.SetExtraValue("name_kwd", strings.ToLower(v)); err != nil {+ return err+ }+ }+ if v := metaString(p.Meta, "entity_type"); v != "" {+ if err := doc.SetExtraValue("entity_type_kwd", v); err != nil {+ return err+ }+ }+ if v, ok := metaInt(p.Meta, "mention_count"); ok {+ if err := doc.SetExtraValue("mention_count_int", v); err != nil {+ return err+ }+ }+ return nildefault:// RAPTOR summary/root rows: raptor_kwd tags the node kind;// raptor_layer_int records tree depth.diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.goindex 8bd8b1c363..41f6823eb3 100644--- a/internal/ingestion/component/knowledge_compiler/component_test.go+++ b/internal/ingestion/component/knowledge_compiler/component_test.go@@ -211,23 +211,28 @@ func TestKnowledgeCompiler_Structure_EndToEnd(t *testing.T) {if !ok {t.Fatalf("chunks = %T, want []any", out["chunks"])}- // 2 input chunks + 3 entities + 2 relations = 7 total (the- // compiled knowledge units are merged into the upstream input chunks).+ // 2 input chunks + 3 entities + 2 relations = 7 total. The structure+ // variant no longer emits a separate compact graph blob (#19474): the+ // per-row entity/relation products are the whole output, mirroring the+ // storage-model change that dropped the knowledge_graph_kwd="graph" row.if len(chunks) != 7 {t.Fatalf("len(chunks) = %d, want 7 (2 input + 3 entities + 2 relations)", len(chunks))}- // Entity and relation products are emitted individually, without an- // additional graph-summary chunk.+ // No compact graph-summary chunk is emitted anymore.+ graphChunks := 0for _, c := range chunks {cm, ok := c.(map[string]any)if !ok {continue}if kind, _ := cm["kc_kind"].(string); kind == "graph" {- t.Fatal("graph chunk emitted")+ graphChunks++}}+ if graphChunks != 0 {+ t.Fatalf("graph chunks = %d, want 0 (graph blob removed by #19474)", graphChunks)+ }}func TestKnowledgeCompiler_UnknownVariant(t *testing.T) {diff --git a/internal/ingestion/component/knowledge_compiler/golden/fixtures/structure_wiki_baseline.json b/internal/ingestion/component/knowledge_compiler/golden/fixtures/structure_wiki_baseline.jsonindex 7f1990ded7..2709a71792 100644--- a/internal/ingestion/component/knowledge_compiler/golden/fixtures/structure_wiki_baseline.json+++ b/internal/ingestion/component/knowledge_compiler/golden/fixtures/structure_wiki_baseline.json@@ -1,5 +1,5 @@{- "note": "Locked Go-side product-count gate for structure/wiki (PORT_PLAN.md 缺口 E). These counts are the golden contract for the fixed corpus with the deterministic mock LLM/embedder. They must be reconciled against the Python baseline (rag/advanced_rag/knowlege_compile/structure.py, wiki.py) when a live run is available; a change here is a deliberate, reviewed contract update. 2026-09-10: structure gate feeds a graph-kind parser_config (hypergraph path): 12 corpus chunks pack into one batch, and the mock extracts 5 entities (Alpha..Epsilon) + 6 relations (fixed Greek pairs) without a graph summary = 11.",+ "note": "Locked Go-side product-count gate for structure/wiki (PORT_PLAN.md 缺口 E). These counts are the golden contract for the fixed corpus with the deterministic mock LLM/embedder. They must be reconciled against the Python baseline (rag/advanced_rag/knowlege_compile/structure.py, wiki.py) when a live run is available; a change here is a deliberate, reviewed contract update. 2026-09-11: structure gate feeds a graph-kind parser_config (hypergraph path): 12 corpus chunks pack into one batch, and the mock extracts 5 entities (Alpha..Epsilon) + 6 relations (fixed Greek pairs) = 11. The compact graph summary row is no longer emitted (removed by #19474: knowledge_graph_kwd=\"graph\" is gone from the storage model), so the structure gate is 11.","schema_version": 1,"fixed_corpus_chunks": 12,"structure_products": 11,diff --git a/internal/ingestion/component/knowledge_compiler/golden_test.go b/internal/ingestion/component/knowledge_compiler/golden_test.goindex 3cd621a36d..544ab7cfaa 100644--- a/internal/ingestion/component/knowledge_compiler/golden_test.go+++ b/internal/ingestion/component/knowledge_compiler/golden_test.go@@ -162,7 +162,8 @@ func TestGolden_Structure_ProductCount(t *testing.T) {}}- // Entity and relation products are emitted without a graph summary.+ // Entity and relation products are emitted without a graph summary (the+ // compact graph blob was removed by #19474).graphCount, nodeCount, edgeCount := 0, 0, 0for _, p := range prods {kind, _ := p.GetExtraString("kc_kind")@@ -176,7 +177,7 @@ func TestGolden_Structure_ProductCount(t *testing.T) {}}if graphCount != 0 {- t.Fatalf("structure: graphCount = %d, want 0", graphCount)+ t.Fatalf("structure: graphCount = %d, want 0 (graph blob removed by #19474)", graphCount)}if nodeCount == 0 || edgeCount == 0 {t.Fatalf("structure: expected entities+relations, got nodes=%d edges=%d", nodeCount, edgeCount)diff --git a/internal/ingestion/component/knowledge_compiler/structure/compile.go b/internal/ingestion/component/knowledge_compiler/structure/compile.goindex 4684c0c6ee..f6a01dde29 100644--- a/internal/ingestion/component/knowledge_compiler/structure/compile.go+++ b/internal/ingestion/component/knowledge_compiler/structure/compile.go@@ -5,7 +5,7 @@ import ("context""encoding/json""fmt"- "sort"+ "strconv""strings""ragflow/internal/ingestion/component/knowledge_compiler/common"@@ -30,15 +30,30 @@ type CompileConfig struct {// (_struct_extract_hypergraph uses temperature 0.1).var extractionTemperature = 0.1+// disableThinking mirrors knowledge_compile_gen_conf's intent: extraction and+// judging calls must not spend the budget on chain-of-thought. Python disables+// it per model family (deepseek-v4/minimax via extra_body.thinking, qwen3 via+// enable_thinking, everything else via reasoning_effort="none"); the Go chat+// drivers expose a single normalized switch, so every compile call turns it+// off. The one Python exception — qwen3 -preview endpoints that REQUIRE+// thinking enabled — would need a model-name branch in the Go driver.+func chatRequest(llmID, systemPrompt, userPrompt string) common.ChatRequest {+ return common.ChatRequest{+ LLMID: llmID,+ SystemPrompt: systemPrompt,+ UserPrompt: userPrompt,+ Temperature: &extractionTemperature,+ DisableThinking: true,+ }+}+// extractHypergraph mirrors _struct_extract_hypergraph: stage 1 extracts node// (entity) items, stage 2 extracts edge (relation) items constrained to the// stage-1 entities via the {known_nodes} placeholder. The two stages within// one batch are strictly sequential; batches parallelise at the caller.func extractHypergraph(ctx context.Context, deps common.Deps, cfg CompileConfig, nodePrompt, edgePromptTmpl, packedText string) (nodes, edges []map[string]any, err error) {user := UserPrompt(packedText)- nodeRaw, err := common.GenJSON(ctx, deps.Chat, common.ChatRequest{- LLMID: cfg.LLMID, SystemPrompt: nodePrompt, UserPrompt: user, Temperature: &extractionTemperature,- })+ nodeRaw, err := common.GenJSON(ctx, deps.Chat, chatRequest(cfg.LLMID, nodePrompt, user))if err != nil {return nil, nil, err}@@ -59,9 +74,7 @@ func extractHypergraph(ctx context.Context, deps common.Deps, cfg CompileConfig,if strings.TrimSpace(edgePromptTmpl) == "" {return nodes, nil, nil}- edgeRaw, err := common.GenJSON(ctx, deps.Chat, common.ChatRequest{- LLMID: cfg.LLMID, SystemPrompt: fillKnownNodes(edgePromptTmpl, known), UserPrompt: user, Temperature: &extractionTemperature,- })+ edgeRaw, err := common.GenJSON(ctx, deps.Chat, chatRequest(cfg.LLMID, fillKnownNodes(edgePromptTmpl, known), EdgeUserPrompt(packedText)))if err != nil {return nil, nil, err}@@ -126,37 +139,44 @@ func payloadChunkIDs(payload map[string]any, batchIDs []string) []string {}// payloadDescription mirrors _struct_payload_description: concat the string-// values of every field (lists flattened) with single spaces. Go maps lose-// JSON insertion order, so keys are sorted to keep the embedding input (and-// therefore the row vector) deterministic across runs.+// values of every field (lists flattened) with single spaces. It delegates to+// common.PayloadDescription, the shared implementation the tree variant also+// uses, so the two variants cannot drift apart.func payloadDescription(payload map[string]any) string {- keys := make([]string, 0, len(payload))- for k := range payload {- keys = append(keys, k)+ return common.PayloadDescription(payload, nil)+}++// IndexText returns the flattened payload description for a compiled row's+// content JSON — the exact text Python tokenizes into content_ltks /+// content_sm_ltks (“_tokenize_for_search(_struct_payload_description(payload))“).+// Non-JSON content yields "", so non-structure variants keep tokenizing their+// raw content.+func IndexText(content string) string {+ payload := parsePayload(content)+ if payload == nil {+ return ""}- sort.Strings(keys)- var parts []string- appendValue := func(v any) {- s := strings.TrimSpace(stringOf(v))- if s != "" {- parts = append(parts, s)+ return payloadDescription(payload)+}++// mentionCountOf mirrors _struct_to_doc_storage_doc's mention_count parsing:+// the payload's own count when it carries a usable one, else 1.+func mentionCountOf(payload map[string]any) int {+ switch v := payload["mention_count"].(type) {+ case float64:+ if v > 0 {+ return int(v)}- }- for _, k := range keys {- switch v := payload[k].(type) {- case []any:- for _, item := range v {- appendValue(item)- }- case []string:- for _, item := range v {- appendValue(item)- }- default:- appendValue(v)+ case int:+ if v > 0 {+ return v+ }+ case string:+ if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil && n > 0 {+ return n}}- return strings.Join(parts, " ")+ return 1}// payloadJSON serialises a payload the way Python's json.dumps(ensure_ascii=@@ -228,16 +248,19 @@ func buildRows(ctx context.Context, deps common.Deps, cfg CompileConfig, nodes,meta := map[string]any{"kind": s.kind,"source_chunk_ids": payloadChunkIDs(s.payload, batchIDs),- "mention_count": 1,+ // mention_count_int mirrors Python: the payload's own count when it+ // carries one, else 1 (never a synthesized constant).+ "mention_count": mentionCountOf(s.payload),}if s.kind == "entity" {if name := entityName(s.payload); name != "" {meta["name"] = name}+ // entity_type_kwd is only stamped when the payload has a type —+ // Python omits the column entirely for an untyped entity rather+ // than writing a synthesized "other".if typ := strings.TrimSpace(stringOf(s.payload["type"])); typ != "" {meta["entity_type"] = typ- } else {- meta["entity_type"] = "other"}if desc := strings.TrimSpace(stringOf(s.payload["description"])); desc != "" {meta["description"] = descdiff --git a/internal/ingestion/component/knowledge_compiler/structure/evidence.go b/internal/ingestion/component/knowledge_compiler/structure/evidence.gonew file mode 100644index 0000000000..65581fbc2b--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/structure/evidence.go@@ -0,0 +1,224 @@+package structure++import (+ "fmt"+ "sort"+ "strings"+ "unicode/utf8"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// Evidence-gate mode: what happens to a payload whose evidence quotes all+// failed to locate. Mirrors Python _EVIDENCE_GATE_MODES /+// _struct_evidence_gate_mode.+const (+ EvidenceGateSoft = "soft"+ EvidenceGateHard = "hard"++ evidenceGateDefault = EvidenceGateSoft+)++// EvidenceGateMode maps the template config's evidence_gate_mode onto the+// supported modes, defaulting to soft for anything unrecognised (mirrors+// Python _struct_evidence_gate_mode).+func EvidenceGateMode(parserConfig map[string]any) string {+ if parserConfig == nil {+ return evidenceGateDefault+ }+ mode, _ := parserConfig["evidence_gate_mode"].(string)+ mode = strings.ToLower(strings.TrimSpace(mode))+ if mode == EvidenceGateSoft || mode == EvidenceGateHard {+ return mode+ }+ return evidenceGateDefault+}++// RelationExpectsEvidence reports whether the template asks relation payloads+// for evidence ("relation.output_fields" contains "evidence"). Mirrors Python+// _struct_relation_expects_evidence: gating relations unconditionally would+// strip the field from templates that never asked for it.+func RelationExpectsEvidence(parserConfig map[string]any) bool {+ if parserConfig == nil {+ return false+ }+ relations, _ := common.Get(parserConfig, "relation").(map[string]any)+ if relations == nil {+ return false+ }+ for _, f := range configOutputFields(relations) {+ if s, ok := f.(string); ok && strings.EqualFold(strings.TrimSpace(s), "evidence") {+ return true+ }+ }+ return false+}++// batchTextByID maps chunk id → chunk text for the gate, using the same id+// synthesis as PackBatch (an empty id becomes "chunk-<n>"). Keep the two in+// sync: the gate validates quotes against exactly the text the prompt showed.+func batchTextByID(batch []common.Chunk) map[string]string {+ out := make(map[string]string, len(batch))+ for i, c := range batch {+ id := strings.TrimSpace(c.ID)+ if id == "" {+ id = fmt.Sprintf("chunk-%d", i+1)+ }+ text := common.FirstNonEmpty(c.Text, c.Content)+ if strings.TrimSpace(text) == "" {+ continue+ }+ out[id] = text+ }+ return out+}++// locateEvidence finds quote in text using whitespace-normalized matching and+// returns offsets against the ORIGINAL text so the span can be sliced back+// out. The offset map is byte-aligned (mirrors tree/claims.go LocateEvidence+// and Python _struct_locate_evidence): a rune-indexed map misaligned every+// lookup past the first multi-byte rune, so CJK quotes resolved to the wrong+// span.+func locateEvidence(quote, text string) (int, int, bool) {+ if quote == "" || text == "" {+ return 0, 0, false+ }+ nText, idxMap := normalizeEvidenceText(text)+ nQuote, _ := normalizeEvidenceText(quote)+ if nQuote == "" {+ return 0, 0, false+ }+ pos := strings.Index(nText, nQuote)+ if pos < 0 {+ return 0, 0, false+ }+ endNorm := pos + len(nQuote) - 1+ if endNorm >= len(idxMap) {+ return 0, 0, false+ }+ // End is the original offset just past the match: the offset of the next+ // normalized byte, or the text length when the match runs to the end.+ end := len(text)+ if endNorm+1 < len(idxMap) {+ end = idxMap[endNorm+1]+ }+ return idxMap[pos], end, true+}++// normalizeEvidenceText collapses whitespace runs, returning the normalized+// text and a byte-aligned map: idxMap[j] is the offset in the original text of+// byte j of the normalized text.+func normalizeEvidenceText(text string) (string, []int) {+ var b strings.Builder+ b.Grow(len(text))+ idxMap := make([]int, 0, len(text))+ prevSpace := false+ for i := 0; i < len(text); {+ c := text[i]+ if c == ' ' || c == '\t' || c == '\n' || c == '\r' {+ if !prevSpace {+ b.WriteByte(' ')+ idxMap = append(idxMap, i)+ }+ prevSpace = true+ i+++ continue+ }+ // Copy the rune byte for byte and record one entry per byte, so the+ // map stays byte-aligned with the normalized string even for+ // multi-byte runes (CJK, emoji, accented characters).+ _, size := utf8.DecodeRuneInString(text[i:])+ if size <= 0 {+ size = 1+ }+ for k := 0; k < size; k++ {+ b.WriteByte(text[i+k])+ idxMap = append(idxMap, i+k)+ }+ i += size+ prevSpace = false+ }+ return b.String(), idxMap+}++// ValidatePayloadEvidence locates every evidence quote in the chunk it cites+// and drops the ones that cannot be found, mirroring Python+// _struct_apply_evidence_gate. mode decides whether a payload left with no+// verified evidence survives (soft: evidence dropped, payload kept) or is+// dropped (hard).+//+// Returns the surviving payloads plus (verified, rejected) counts. Payloads+// are filtered in place where possible; callers must use the returned slice.+func ValidatePayloadEvidence(+ payloads []map[string]any,+ textByID map[string]string,+ mode string,+) ([]map[string]any, int, int) {+ verified, rejected := 0, 0+ if mode != EvidenceGateSoft && mode != EvidenceGateHard {+ mode = evidenceGateDefault+ }+ if len(textByID) == 0 {+ return payloads, verified, rejected+ }++ survivors := make([]map[string]any, 0, len(payloads))+ for _, payload := range payloads {+ rawEvidence, _ := payload["evidence"].([]any)+ if len(rawEvidence) == 0 {+ survivors = append(survivors, payload)+ continue+ }++ kept := make([]any, 0, len(rawEvidence))+ for _, entryAny := range rawEvidence {+ entry, ok := entryAny.(map[string]any)+ if !ok {+ continue+ }+ quote, _ := entry["quote"].(string)+ if strings.TrimSpace(quote) == "" {+ continue+ }+ // Prefer the cited chunk; fall back to scanning the batch. The+ // fallback candidates are sorted because map iteration order is+ // random — with two chunks holding the same sentence the gate must+ // pick the same one on every run.+ candidates := []string{}+ if cited, _ := entry["chunk_id"].(string); cited != "" && textByID[cited] != "" {+ candidates = append(candidates, cited)+ } else {+ for id := range textByID {+ candidates = append(candidates, id)+ }+ sort.Strings(candidates)+ }+ for _, id := range candidates {+ start, end, ok := locateEvidence(quote, textByID[id])+ if !ok {+ continue+ }+ kept = append(kept, map[string]any{+ "quote": quote,+ "chunk_id": id,+ "start": start,+ "end": end,+ })+ break+ }+ }++ if len(kept) > 0 {+ verified += len(kept)+ payload["evidence"] = kept+ survivors = append(survivors, payload)+ continue+ }+ rejected+++ delete(payload, "evidence")+ if mode == EvidenceGateSoft {+ survivors = append(survivors, payload)+ }+ }+ return survivors, verified, rejected+}diff --git a/internal/ingestion/component/knowledge_compiler/structure/evidence_test.go b/internal/ingestion/component/knowledge_compiler/structure/evidence_test.gonew file mode 100644index 0000000000..b4b5c65c7a--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/structure/evidence_test.go@@ -0,0 +1,157 @@+package structure++import (+ "testing"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++func TestEvidenceGateModeParsing(t *testing.T) {+ cases := []struct {+ in map[string]any+ want string+ }{+ {map[string]any{"evidence_gate_mode": "hard"}, EvidenceGateHard},+ {map[string]any{"evidence_gate_mode": "SOFT"}, EvidenceGateSoft},+ {map[string]any{"evidence_gate_mode": "bogus"}, EvidenceGateSoft},+ {map[string]any{}, EvidenceGateSoft},+ {nil, EvidenceGateSoft},+ }+ for _, c := range cases {+ if got := EvidenceGateMode(c.in); got != c.want {+ t.Fatalf("EvidenceGateMode(%v) = %q, want %q", c.in, got, c.want)+ }+ }+}++func TestRelationExpectsEvidence(t *testing.T) {+ withEvidence := map[string]any{+ "relation": map[string]any{"output_fields": []any{"evidence", "other"}},+ }+ without := map[string]any{+ "relation": map[string]any{"output_fields": []any{"other"}},+ }+ if !RelationExpectsEvidence(withEvidence) {+ t.Fatal("relation.output_fields with evidence should be detected")+ }+ if RelationExpectsEvidence(without) {+ t.Fatal("relation.output_fields without evidence must not be gated")+ }+ if RelationExpectsEvidence(nil) {+ t.Fatal("nil config must not be gated")+ }+}++func TestValidatePayloadEvidenceSoftKeepsPayloadWithoutEvidence(t *testing.T) {+ textByID := map[string]string{"c1": "The capital is Paris."}+ payloads := []map[string]any{{+ "name": "invented",+ "evidence": []any{map[string]any{"quote": "a sentence that does not exist", "chunk_id": "c1"}},+ }}+ kept, verified, rejected := ValidatePayloadEvidence(payloads, textByID, EvidenceGateSoft)+ if verified != 0 || rejected != 1 {+ t.Fatalf("got verified=%d rejected=%d, want 0/1", verified, rejected)+ }+ if len(kept) != 1 {+ t.Fatalf("soft mode keeps the payload, got %d", len(kept))+ }+ if _, has := kept[0]["evidence"]; has {+ t.Fatal("rejected evidence must be removed")+ }+}++func TestValidatePayloadEvidenceHardDropsThePayload(t *testing.T) {+ textByID := map[string]string{"c1": "The capital is Paris."}+ payloads := []map[string]any{{+ "name": "invented",+ "evidence": []any{map[string]any{"quote": "a sentence that does not exist", "chunk_id": "c1"}},+ }}+ kept, verified, rejected := ValidatePayloadEvidence(payloads, textByID, EvidenceGateHard)+ if verified != 0 || rejected != 1 {+ t.Fatalf("got verified=%d rejected=%d, want 0/1", verified, rejected)+ }+ if len(kept) != 0 {+ t.Fatalf("hard mode drops the payload, got %d", len(kept))+ }+}++func TestValidatePayloadEvidenceLocatesAndStampsOffsets(t *testing.T) {+ src := "The capital is Paris."+ textByID := map[string]string{"c1": src}+ payloads := []map[string]any{{+ "name": "capital",+ "evidence": []any{map[string]any{"quote": "The capital is Paris.", "chunk_id": "c1"}},+ }}+ kept, verified, _ := ValidatePayloadEvidence(payloads, textByID, EvidenceGateSoft)+ if verified != 1 {+ t.Fatalf("verified=%d, want 1", verified)+ }+ ev := kept[0]["evidence"].([]any)[0].(map[string]any)+ s, _ := ev["start"].(int)+ e, _ := ev["end"].(int)+ if got := src[s:e]; got != src {+ t.Fatalf("span %d:%d sliced %q, want the original sentence", s, e, got)+ }+}++func TestValidatePayloadEvidenceFallsBackToSortedScan(t *testing.T) {+ // The model cited nothing: the gate scans the batch. Sorting keeps the+ // choice deterministic across runs even when two chunks hold the sentence.+ textByID := map[string]string{"zz": "alpha text", "aa": "alpha text"}+ payloads := []map[string]any{{+ "name": "dup",+ "evidence": []any{map[string]any{"quote": "alpha text"}},+ }}+ kept, verified, _ := ValidatePayloadEvidence(payloads, textByID, EvidenceGateSoft)+ if verified != 1 {+ t.Fatalf("verified=%d, want 1", verified)+ }+ ev := kept[0]["evidence"].([]any)[0].(map[string]any)+ if id := ev["chunk_id"]; id != "aa" {+ t.Fatalf("fallback chunk_id = %v, want the first sorted candidate", id)+ }+}++func TestLocateEvidenceMultibyteOffsets(t *testing.T) {+ // A byte offset from strings.Index runs ahead of the rune offset, so a map+ // indexed per rune resolves every lookup past the first multi-byte rune to+ // the wrong span. Mirrors the tree package's regression test.+ src := "前言。中国的首都是北京。尾声。"+ quote := "中国的首都是北京。"+ start, end, ok := locateEvidence(quote, src)+ if !ok {+ t.Fatalf("CJK quote not located in %q", src)+ }+ if got := src[start:end]; got != quote {+ t.Fatalf("span sliced %q, want %q", got, quote)+ }++ // A match running to the end of the text must end at len(text).+ tail := "前言。中国的首都是北京。"+ start, end, ok = locateEvidence(quote, tail)+ if !ok {+ t.Fatal("quote at the end of the text not located")+ }+ if end != len(tail) {+ t.Fatalf("end = %d, want len(text) = %d", end, len(tail))+ }+ if got := tail[start:end]; got != quote {+ t.Fatalf("span sliced %q, want %q", got, quote)+ }+}++func TestBatchTextByIDUsesPackBatchIDSynthesis(t *testing.T) {+ // The gate must validate against exactly the text the prompt showed, so its+ // id synthesis matches PackBatch's (empty id → "chunk-<n>", index-based).+ got := batchTextByID([]common.Chunk{+ {ID: "", Text: "alpha"},+ {ID: "", Text: "beta"},+ {ID: "", Text: " "},+ })+ if len(got) != 2 {+ t.Fatalf("blank chunks skipped: %v", got)+ }+ if got["chunk-1"] != "alpha" || got["chunk-2"] != "beta" {+ t.Fatalf("ids must match PackBatch: %v", got)+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/structure/graph.go b/internal/ingestion/component/knowledge_compiler/structure/graph.goindex fdc6b68dc6..c35ec9a251 100644--- a/internal/ingestion/component/knowledge_compiler/structure/graph.go+++ b/internal/ingestion/component/knowledge_compiler/structure/graph.go@@ -46,7 +46,7 @@ func graphEntity(payload map[string]any, sourceChunkIDs []string) map[string]anyif description == "" {description = strings.TrimSpace(stringOf(payload["definition_excerpt"]))}- return map[string]any{+ entity := map[string]any{"aliases": aliases,"mention_count": 1,"name": name,@@ -54,6 +54,25 @@ func graphEntity(payload map[string]any, sourceChunkIDs []string) map[string]any"type": typ,"description": description,}+ // Gate-verified verbatim evidence rides along for the artifacts detail+ // panel (mirrors _struct_graph_entity's evidence projection). Rows compiled+ // before the field existed simply omit it.+ if evidence, ok := payload["evidence"].([]any); ok {+ var verified []any+ for _, e := range evidence {+ m, ok := e.(map[string]any)+ if !ok {+ continue+ }+ if q := strings.TrimSpace(stringOf(m["quote"])); q != "" {+ verified = append(verified, m)+ }+ }+ if len(verified) > 0 {+ entity["evidence"] = verified+ }+ }+ return entity}// graphRelation mirrors _struct_graph_relation: project a relation payload to@@ -103,7 +122,12 @@ func mergeGraphEntities(entities []map[string]any) []map[string]any {// compact {"entities": [...], "relations": [...]} graph from the surviving// structure products. Entity mention counts sum across rows sharing// (name, type); relation rows project verbatim (already deduped in-run).-func RebuildStructureGraph(products []common.Product) map[string]any {+//+// compileType selects the kind-scoped post-processing: page_index templates+// declare that every claim hangs off exactly one container, so orphans the+// per-batch relation pass could not attach are re-parented (mirrors+// _struct_attach_orphan_entities).+func RebuildStructureGraph(products []common.Product, compileType string) map[string]any {var entities []map[string]anyvar relations []map[string]anyfor _, p := range products {@@ -123,12 +147,159 @@ func RebuildStructureGraph(products []common.Product) map[string]any {}}}+ if normalizeKind(compileType) == "page_index" || normalizeKind(compileType) == "pageindex" {+ relations = attachOrphanEntities(entities, relations)+ }return map[string]any{"entities": mergeGraphEntities(entities),"relations": relations,}}+// attachOrphanEntities mirrors _struct_attach_orphan_entities: re-parent leaf+// entities the per-batch relation pass could not attach.+//+// Relations are extracted inside one batch, so a container living in another+// batch is invisible to the edge prompt. Only types that NEVER act as a+// container are repaired — an entity of such a type with no incoming edge is+// missing its parent by definition, whereas a structural type (a heading) may+// legitimately be a root. Containers are classified by TYPE, not instance: a+// heading whose own children all fell in other batches never appears on a+// "from" side, yet it is still the right parent. No type names are baked in —+// containers are whatever appears on the "from" side.+//+// The parent is the container sharing the most source chunks with the orphan,+// preferring the most specific one (fewest chunks of its own), because a+// heading's source_chunk_ids are meant to cover its whole section.+func attachOrphanEntities(entities, relations []map[string]any) []map[string]any {+ if len(entities) == 0 || len(relations) == 0 {+ return relations+ }++ childNames := map[string]bool{}+ containerNames := map[string]bool{}+ for _, r := range relations {+ if name := strings.TrimSpace(stringOf(r["to"])); name != "" {+ childNames[name] = true+ }+ if name := strings.TrimSpace(stringOf(r["from"])); name != "" {+ containerNames[name] = true+ }+ }+ if len(containerNames) == 0 {+ return relations+ }++ byType := map[string][]map[string]any{}+ for _, e := range entities {+ typ := strings.TrimSpace(stringOf(e["type"]))+ if typ == "" {+ typ = "other"+ }+ byType[typ] = append(byType[typ], e)+ }++ // Classify by TYPE, not by instance (see doc comment above).+ containerTypes := map[string]bool{}+ for typ, ents := range byType {+ for _, e := range ents {+ if name := strings.TrimSpace(stringOf(e["name"])); containerNames[name] {+ containerTypes[typ] = true+ break+ }+ }+ }+ var containers []map[string]any+ for _, e := range entities {+ if containerTypes[strings.TrimSpace(stringOf(e["type"]))] {+ containers = append(containers, e)+ }+ }+ if len(containers) == 0 {+ return relations+ }++ // Synthetic relations reuse the dominant existing relation type.+ dominantType, best := "include", 0+ counts := map[string]int{}+ for _, r := range relations {+ if t := strings.TrimSpace(stringOf(r["type"])); t != "" {+ counts[t]+++ if counts[t] > best {+ best, dominantType = counts[t], t+ }+ }+ }++ existing := map[[2]string]bool{}+ for _, r := range relations {+ existing[[2]string{strings.TrimSpace(stringOf(r["from"])), strings.TrimSpace(stringOf(r["to"]))}] = true+ }++ var repaired []map[string]any+ for _, ent := range entities {+ name := strings.TrimSpace(stringOf(ent["name"]))+ typ := strings.TrimSpace(stringOf(ent["type"]))+ if typ == "" {+ typ = "other"+ }+ if name == "" || containerTypes[typ] || childNames[name] {+ continue+ }+ own := chunkSet(ent)+ if len(own) == 0 {+ continue+ }+ bestName := ""+ bestOverlap, bestSpecificity := 0, 0+ for _, cand := range containers {+ candName := strings.TrimSpace(stringOf(cand["name"]))+ if candName == "" || candName == name {+ continue+ }+ candChunks := chunkSet(cand)+ overlap := 0+ for c := range own {+ if candChunks[c] {+ overlap+++ }+ }+ if overlap <= 0 {+ continue+ }+ // Most overlapping wins; ties go to the narrower container, which+ // is the deeper (more specific) section.+ if bestName == "" || overlap > bestOverlap || (overlap == bestOverlap && len(candChunks) < bestSpecificity) {+ bestName, bestOverlap, bestSpecificity = candName, overlap, len(candChunks)+ }+ }+ if bestName == "" {+ continue+ }+ if existing[[2]string{bestName, name}] {+ continue+ }+ existing[[2]string{bestName, name}] = true+ repaired = append(repaired, map[string]any{"from": bestName, "to": name, "type": dominantType})+ }++ if len(repaired) > 0 {+ // Mirrors Python's log line; the message lands in the compile log.+ relations = append(relations, repaired...)+ }+ return relations+}++func chunkSet(entity map[string]any) map[string]bool {+ out := map[string]bool{}+ for _, c := range toStrings(entity["source_chunk_ids"]) {+ if c = strings.TrimSpace(c); c != "" {+ out[c] = true+ }+ }+ return out+}+func mentionOf(entity map[string]any) int {switch v := entity["mention_count"].(type) {case int:diff --git a/internal/ingestion/component/knowledge_compiler/structure/prompt.go b/internal/ingestion/component/knowledge_compiler/structure/prompt.goindex 006b956793..7fca5fb518 100644--- a/internal/ingestion/component/knowledge_compiler/structure/prompt.go+++ b/internal/ingestion/component/knowledge_compiler/structure/prompt.go@@ -114,7 +114,13 @@ func renderFields(fields []any, lang string) (string, string) {// renderTypeFields mirrors _struct_render_type_fields (new compilation-template// shape: allowed item `type` values with descriptions/rules).-func renderTypeFields(fields []any, lang, kind string) (string, string) {+//+// extraFields declares additional output keys a template may ask for+// (claim/evidence compilation adds "evidence"). The Response Format below is+// what actually fixes the model's output shape — a key described only inside a+// type's "rule" text but absent from the skeleton is silently dropped by the+// model — so these keys must be rendered into the skeleton here.+func renderTypeFields(fields []any, lang, kind string, extraFields []any) (string, string) {var lines []stringvar typeValues []stringfor _, raw := range fields {@@ -140,11 +146,63 @@ func renderTypeFields(fields []any, lang, kind string) (string, string) {lines = append(lines, "- type: other")}oneOf := strings.Join(typeValues, "|")++ // Extra output keys the template declared. Rendered into the skeleton so the+ // model actually emits them, and described so it knows their shape.+ var extraSkel []string+ for _, raw := range extraFields {+ ef, ok := raw.(map[string]any)+ if !ok {+ continue+ }+ name, _ := ef["name"].(string)+ name = strings.TrimSpace(name)+ if name == "" {+ continue+ }+ desc := common.Localize(common.Get(ef, "description"), lang)+ optional := "required"+ if req, exists := ef["required"].(bool); exists && !req {+ optional = "optional, omit when not applicable"+ }+ ftype, _ := ef["type"].(string)+ if ftype == "" {+ ftype = "str"+ }+ header := fmt.Sprintf("- %s (%s, %s)", name, ftype, optional)+ if desc != "" {+ lines = append(lines, header+": "+desc)+ } else {+ lines = append(lines, header)+ }++ if shape := common.Localize(common.Get(ef, "shape"), lang); shape != "" {+ extraSkel = append(extraSkel, fmt.Sprintf("%q: %s", name, shape))+ continue+ }+ placeholder := "<string>"+ switch ftype {+ case "list":+ placeholder = "[...]"+ case "int":+ placeholder = "<int>"+ case "float":+ placeholder = "<float>"+ case "bool":+ placeholder = "<true|false>"+ }+ extraSkel = append(extraSkel, fmt.Sprintf("%q: %s", name, placeholder))+ }+ var extraSuffix string+ if len(extraSkel) > 0 {+ extraSuffix = ", " + strings.Join(extraSkel, ", ")+ }+var skeleton stringif kind == "relation" {- skeleton = `{ "type": "<one of: ` + oneOf + `>", "source": "<known entity name>", "target": "<known entity name>", "description": "<evidence or relation description>", "source_chunk_ids": ["<source chunk id>", ...] }`+ skeleton = `{ "type": "<one of: ` + oneOf + `>", "source": "<known entity name>", "target": "<known entity name>", "description": "<evidence or relation description>", "source_chunk_ids": ["<source chunk id>", ...]` + extraSuffix + ` }`} else {- skeleton = `{ "type": "<one of: ` + oneOf + `>", "name": "<exact extracted item text>", "description": "<evidence, definition, or detail from the source>", "source_chunk_ids": ["<source chunk id>", ...] }`+ skeleton = `{ "type": "<one of: ` + oneOf + `>", "name": "<exact extracted item text>", "description": "<evidence, definition, or detail from the source>", "source_chunk_ids": ["<source chunk id>", ...]` + extraSuffix + ` }`}return strings.Join(lines, "\n"), skeleton}@@ -190,8 +248,8 @@ func HypergraphPrompts(parserConfig map[string]any, lang string) (nodePrompt, edvar entFieldsText, entSkel, relFieldsText, relSkel stringif usesTemplateShape {- entFieldsText, entSkel = renderTypeFields(entFields, lang, "entity")- relFieldsText, relSkel = renderTypeFields(relFields, lang, "relation")+ entFieldsText, entSkel = renderTypeFields(entFields, lang, "entity", configOutputFields(entitiesCfg))+ relFieldsText, relSkel = renderTypeFields(relFields, lang, "relation", configOutputFields(relationsCfg))} else {entFieldsText, entSkel = renderFields(entFields, lang)relFieldsText, relSkel = renderFields(relFields, lang)@@ -266,7 +324,7 @@ func fillKnownNodes(edgePromptTemplate string, knownKeys []string) string {}// UserPrompt wraps one batch's packed chunk text exactly as-// _struct_extract_hypergraph does; the same user prompt serves both stages.+// _struct_extract_hypergraph's node-stage user prompt does.func UserPrompt(packedText string) string {return "## Source Text:\n" +"Each source chunk is enclosed by [CHUNK_ID: ...] and [END_CHUNK]. " +@@ -275,6 +333,17 @@ func UserPrompt(packedText string) string {packedText + "\n\n## Output (JSON only):"}+// EdgeUserPrompt wraps one batch's packed chunk text for the relation stage,+// mirroring _struct_extract_relations' edge_user_prompt. The wording differs+// from the node stage: this call emits relations only, so asking for "every+// entity and relation" would be wrong.+func EdgeUserPrompt(packedText string) string {+ return "## Source Text:\n" ++ "Each source chunk is enclosed by [CHUNK_ID: ...] and [END_CHUNK]. " ++ "For every relation, return source_chunk_ids containing only the IDs of chunks that support that relation.\n" ++ packedText + "\n\n## Output (JSON only):"+}+// PackBatch renders one batch's chunks as [CHUNK_ID: id]...[END_CHUNK]// segments (mirrors _struct_process_batch) and returns the batch chunk ids in// order. Chunks without text are skipped, matching Python's isinstance check.@@ -376,6 +445,27 @@ func configFields(cfg map[string]any) []any {return nil}+// configOutputFields extracts cfg["output_fields"] as a []any (nil when+// absent). These declare extra keys the model should emit beyond the fixed+// shape — claim/evidence compilation uses it for "evidence". See+// renderTypeFields for why they must reach the skeleton.+func configOutputFields(cfg map[string]any) []any {+ if cfg == nil {+ return nil+ }+ switch v := cfg["output_fields"].(type) {+ case []any:+ return v+ case []map[string]any:+ out := make([]any, 0, len(v))+ for _, f := range v {+ out = append(out, f)+ }+ return out+ }+ return nil+}+func firstStringOf(v any) string {if s, ok := v.(string); ok {return sdiff --git a/internal/ingestion/component/knowledge_compiler/structure/prompt_test.go b/internal/ingestion/component/knowledge_compiler/structure/prompt_test.gonew file mode 100644index 0000000000..1bb9b9a65d--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/structure/prompt_test.go@@ -0,0 +1,91 @@+package structure++import (+ "strings"+ "testing"+)++func TestRenderTypeFieldsExtraFields(t *testing.T) {+ ent := map[string]any{+ "fields": []any{+ map[string]any{"type": "title"},+ map[string]any{"type": "fact"},+ },+ "output_fields": []any{map[string]any{+ "name": "evidence",+ "type": "list",+ "required": false,+ "description": "verbatim source sentences supporting this claim",+ "shape": `[{"quote": "<verbatim source sentence>", "chunk_id": "<source chunk id>"}]`,+ }},+ }++ lines, skel := renderTypeFields(configFields(ent), "English", "entity", configOutputFields(ent))++ // The skeleton is what fixes the model's output shape. A key described only+ // in a type's rule text but absent here is silently dropped by the model —+ // which is exactly how the first evidence attempt produced zero evidence.+ if !strings.Contains(skel, `"evidence"`) {+ t.Fatalf("evidence missing from skeleton: %s", skel)+ }+ if !strings.Contains(skel, `"quote"`) || !strings.Contains(skel, `"chunk_id"`) {+ t.Fatalf("evidence shape missing quote/chunk_id: %s", skel)+ }+ // The declared shape must render verbatim, not as a generic placeholder.+ if strings.Contains(skel, `"evidence": [...]`) {+ t.Fatalf("shape not honoured, fell back to placeholder: %s", skel)+ }+ if !strings.Contains(lines, "- evidence (list, optional, omit when not applicable)") {+ t.Fatalf("evidence not described in field list: %s", lines)+ }+ // Fixed keys must survive.+ for _, key := range []string{`"type"`, `"name"`, `"description"`, `"source_chunk_ids"`} {+ if !strings.Contains(skel, key) {+ t.Fatalf("fixed key %s dropped from skeleton: %s", key, skel)+ }+ }+}++func TestRenderTypeFieldsExtraFieldsAbsent(t *testing.T) {+ // Templates that declare no output_fields must keep the original skeleton,+ // with no stray separator.+ ent := map[string]any{"fields": []any{map[string]any{"type": "title"}}}+ _, skel := renderTypeFields(configFields(ent), "English", "entity", configOutputFields(ent))+ // Note: the word "evidence" also appears in the fixed description+ // placeholder, so match the key with its quotes, not the bare word.+ if strings.Contains(skel, `"evidence"`) {+ t.Fatalf("skeleton should not carry an evidence key: %s", skel)+ }+ if strings.Contains(skel, ", }") {+ t.Fatalf("dangling separator in skeleton: %s", skel)+ }+}++func TestRenderTypeFieldsFallsBackToPlaceholder(t *testing.T) {+ // Without a `shape`, the declared type picks a sensible placeholder.+ ent := map[string]any{+ "fields": []any{map[string]any{"type": "fact"}},+ "output_fields": []any{+ map[string]any{"name": "evidence", "type": "list"},+ map[string]any{"name": "score", "type": "float"},+ map[string]any{"name": "flag", "type": "bool"},+ },+ }+ _, skel := renderTypeFields(configFields(ent), "English", "entity", configOutputFields(ent))+ for _, want := range []string{`"evidence": [...]`, `"score": <float>`, `"flag": <true|false>`} {+ if !strings.Contains(skel, want) {+ t.Fatalf("expected %s in skeleton: %s", want, skel)+ }+ }+}++func TestRenderTypeFieldsRelationUnaffected(t *testing.T) {+ rel := map[string]any{"fields": []any{map[string]any{"type": "include"}}}+ _, skel := renderTypeFields(configFields(rel), "English", "relation", configOutputFields(rel))+ if !strings.Contains(skel, `"source"`) || !strings.Contains(skel, `"target"`) {+ t.Fatalf("relation skeleton lost its shape: %s", skel)+ }+ if strings.Contains(skel, `"evidence"`) {+ t.Fatalf("relation skeleton should not carry an evidence key: %s", skel)+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/structure/structure.go b/internal/ingestion/component/knowledge_compiler/structure/structure.goindex db99d62af3..dbfcc7c811 100644--- a/internal/ingestion/component/knowledge_compiler/structure/structure.go+++ b/internal/ingestion/component/knowledge_compiler/structure/structure.go@@ -9,7 +9,11 @@ package structureimport ("context""fmt"+ "os"+ "strconv"+ "sync"+ "ragflow/internal/agent/runtime""ragflow/internal/ingestion/component/knowledge_compiler/common")@@ -45,11 +49,26 @@ func runBatches(ctx context.Context, jobs []func() error) error {return nil}-// structureBatchTokenBudget caps one extraction batch's packed chunk tokens.-// Python derives the budget from chat_mdl.max_length minus the prompt-// overhead; the Go ChatInvoker seam does not expose the model window, so we-// use the same conservative constant the wiki variant uses.-const structureBatchTokenBudget = 4096+// structureInputBudget mirrors _build_chunk_batches' default mode:+// input_budget = max(int(max_length * INPUT_UTILIZATION) - prompt_overhead, 1024)+// with INPUT_UTILIZATION = 0.5 (rag/prompts/generator.py) and prompt_overhead+// the larger of the two stage prompts. A batch is one LLM call's whole input,+// so a budget that ignores the model window changes how many calls a document+// takes — and with it which entities land in which batch.+func structureInputBudget(modelContextLen, promptOverhead int) int {+ const (+ utilization = 0.5+ floor = 1024+ )+ if modelContextLen <= 0 {+ return 0+ }+ budget := int(float64(modelContextLen)*utilization) - promptOverhead+ if budget < floor {+ budget = floor+ }+ return budget+}// Run executes the structure variant:// 1. MAP — per-batch two-stage (node → edge) extraction, parallel across@@ -81,9 +100,39 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo}nodePrompt, edgePromptTmpl := HypergraphPrompts(parserConfig, param.Language)+ gateMode := EvidenceGateMode(parserConfig)// ---- MAP ----- batches := common.PackBatches(inputs.Chunks, structureBatchTokenBudget, deps.Tokenizer)+ // Prompt overhead is counted the same way Python does: the larger of the+ // two stage prompts, subtracted from the window-derived input budget. The+ // tokenizer is optional (offline tests wire none) — without it the+ // overhead is 0 and PackBatches degrades to per-chunk counting.+ promptOverhead := 0+ if deps.Tokenizer != nil {+ promptOverhead = deps.Tokenizer.NumTokens(nodePrompt)+ if t := deps.Tokenizer.NumTokens(edgePromptTmpl); t > promptOverhead {+ promptOverhead = t+ }+ }+ budget := structureInputBudget(deps.ModelContextLen, promptOverhead)+ if budget <= 0 {+ // Model window unknown (the wiring did not set it): keep the historic+ // conservative constant rather than guessing a large window.+ budget = 4096+ }+ batches := common.PackBatches(inputs.Chunks, budget, deps.Tokenizer)+ // Python structure.py _STRUCT_MAX_CHUNKS_PER_BATCH: optional chunk-count cap+ // per extraction batch (0 = window-packed only — a heading has to see its+ // whole section to own it; the 4-per-batch rule belongs to tree's claim+ // harvesting). Overridable for benchmarking, mirrored verbatim.+ if v, err := strconv.Atoi(os.Getenv("STRUCT_MAX_CHUNKS_PER_BATCH")); err == nil && v > 0 {+ batches = capBatchChunkCount(batches, v)+ }+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf(+ "%s-template: %d chunk(s) -> %d batch(es)", compileType, len(inputs.Chunks), len(batches)))+ // Extraction and embedding are two phases (upstream): the pool workers only+ // extract; buildRows (which calls Embed.Encode) runs serially afterwards so+ // embedding batch jobs are never nested inside a compiler-pool worker.type extractedBatch struct {nodes, edges []map[string]anybatchIDs []string@@ -91,9 +140,14 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoextracted := make([]extractedBatch, len(batches))perBatch := make([][]common.Product, len(batches))jobs := make([]func() error, 0, len(batches))+ // The progress callback is supplied by the caller and is not required to be+ // goroutine-safe; pool workers report out of order, so serialise it.+ var progressMu sync.Mutexfor i, batch := range batches {i, batch := i, batchjobs = append(jobs, func() error {+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf(+ "%s-template: extracting batch %d/%d", compileType, i+1, len(batches)))packed, batchIDs := PackBatch(batch)if len(batchIDs) == 0 {return nil@@ -102,9 +156,28 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoif err != nil {return err}+ // Evidence gate (mirrors Python _struct_process_batch): validate+ // quotes while the batch's source text is still in hand. It is+ // pure validation — no embedding — so it stays inside the worker,+ // and the vectors buildRows builds later are computed from the+ // surviving payload. Relations are gated only when the template+ // asked them to carry evidence.+ textByID := batchTextByID(batch)+ if len(textByID) > 0 {+ nodes, _, _ = ValidatePayloadEvidence(nodes, textByID, gateMode)+ if len(edges) > 0 && RelationExpectsEvidence(parserConfig) {+ edges, _, _ = ValidatePayloadEvidence(edges, textByID, gateMode)+ }+ }// Keep embedding out of the compiler-pool worker. buildRows calls- // Embed.Encode, which may submit its own batch jobs to that pool.+ // Embed.Encode, which may submit its own batch jobs to that pool;+ // the serial loop after runBatches owns it.extracted[i] = extractedBatch{nodes: nodes, edges: edges, batchIDs: batchIDs}+ progressMu.Lock()+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf(+ "%s-template: batch %d/%d done: %d entities, %d relations",+ compileType, i+1, len(batches), len(nodes), len(edges)))+ progressMu.Unlock()return nil})}@@ -117,6 +190,7 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo// Embed each extracted batch serially after all MAP jobs have returned.// This avoids nesting Embed.Encode (and its batch jobs) inside a worker// already occupied by the shared compiler pool.+ rowCount := 0for i, result := range extracted {if len(result.batchIDs) == 0 {continue@@ -126,7 +200,12 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoreturn common.Outputs{}, err}perBatch[i] = rows+ rowCount += len(rows)+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf(+ "%s-template: embedded batch %d/%d (%d rows so far)", compileType, i+1, len(batches), rowCount))}+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf(+ "%s-template: deduplicating %d row(s)", compileType, rowCount))// ---- DEDUP ----// Sequential in batch order so merge outcomes are deterministic and match@@ -145,6 +224,9 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo}stats := deduper.Stats()prods := deduper.Rows()+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf(+ "%s-template: dedup done: %d row(s), %d duplicate(s) dropped",+ compileType, len(prods), stats.DuplicatesDropped))// ---- KIND POST-PROCESSING ----// Chain kinds (list/timeline): relations must form a strict linear chain;@@ -171,16 +253,13 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoprods[i].Meta["compile_kwd"] = string(compileType)}- // ---- GRAPH ----- // graphProduct, err := buildGraphProduct(ctx, deps, cfg, prods)- // if err != nil {- // return common.Outputs{}, err- // }-- // Buffer the deduplicated row products in one slice; the component merges- // them into the upstream chunk stream.+ // The deduplicated entity/relation products are the whole output; the+ // component merges them into the upstream chunk stream. (The compact graph+ // blob was removed: knowledge_graph_kwd="graph" is no longer a storage row,+ // which also saves one embedding call per compile.)products := append([]common.Product{}, prods...)- // products = append(products, graphProduct)+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf(+ "%s-template: produced %d row(s)", compileType, len(products)))out := common.Outputs{Products: products,@@ -189,38 +268,23 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoreturn out, nil}-// buildGraphProduct rebuilds the compact graph JSON from the surviving-// entity/relation rows and wraps it as a single "graph" product so the-// downstream writer has a ready structure to persist. The row id mirrors-// Python's _struct_graph_row_id (doc : structure_graph : compile : template).-func buildGraphProduct(ctx context.Context, deps common.Deps, cfg CompileConfig, prods []common.Product) (common.Product, error) {- if deps.Embed == nil {- return common.Product{}, fmt.Errorf("knowledge_compiler: embedding model is required to build the graph product")- }- graph := RebuildStructureGraph(prods)- graphContent := payloadJSON(graph)- vecs, err := deps.Embed.Encode(ctx, []string{graphContent})- if err != nil {- return common.Product{}, err- }- if len(vecs) == 0 {- return common.Product{}, fmt.Errorf("knowledge_compiler: embedding the graph summary returned no vector")- }- idParts := []string{cfg.DocID, "structure_graph", string(cfg.Type)}- if cfg.TemplateID != "" {- idParts = append(idParts, cfg.TemplateID)- }- return common.Product{- ID: common.StableRowID(idParts...),- DocID: cfg.DocID,- TenantID: cfg.TenantID,- Variant: cfg.Variant,- Content: graphContent,- Vector: vecs[0],- Meta: map[string]any{- "kind": "graph",- "compile_kwd": string(cfg.Type),- "source_doc_ids": []string{cfg.DocID},- },- }, nil+// capBatchChunkCount splits window-packed batches into sub-batches of at most+// cap chunks (Python batch_size_cap greedy mode, chunk-count cutoff). Order is+// preserved; PackBatch labels are per-batch positional so sub-batches renumber+// from C1 exactly like freshly packed batches.+func capBatchChunkCount(batches [][]common.Chunk, cap int) [][]common.Chunk {+ if cap < 1 {+ return batches+ }+ var out [][]common.Chunk+ for _, b := range batches {+ for start := 0; start < len(b); start += cap {+ end := start + cap+ if end > len(b) {+ end = len(b)+ }+ out = append(out, b[start:end])+ }+ }+ return out}diff --git a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go b/internal/ingestion/component/knowledge_compiler/structure/structure_test.goindex 24336a80f4..cea2ee056f 100644--- a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go+++ b/internal/ingestion/component/knowledge_compiler/structure/structure_test.go@@ -414,12 +414,16 @@ func TestPayloadDescriptionSortedAndFlattened(t *testing.T) {payload := map[string]any{"name": "Beta","type": "letter",+ "mention_count": 1,"source_chunk_ids": []any{"c1", "c2"},}got := payloadDescription(payload)- // Keys are sorted for determinism: name < source_chunk_ids < type.- if got != "Beta c1 c2 letter" {- t.Fatalf("payloadDescription = %q, want %q", got, "Beta c1 c2 letter")+ // Keys are sorted for determinism, and the bookkeeping keys are excluded:+ // source_chunk_ids are opaque hex that would tokenize into garbage terms+ // and mention_count is a number every row shares (mirrors Python's+ // _STRUCT_INDEX_EXCLUDED_KEYS).+ if got != "Beta letter" {+ t.Fatalf("payloadDescription = %q, want %q", got, "Beta letter")}}@@ -442,7 +446,7 @@ func TestStructureRunGraphKind(t *testing.T) {t.Fatalf("Run: %v", err)}- var entities, relations, graphs int+ var entities, relations intvar betaProduct *common.Productfor i, p := range out.Products {switch p.Meta["kind"] {@@ -460,11 +464,11 @@ func TestStructureRunGraphKind(t *testing.T) {case "relation":relations++case "graph":- graphs+++ t.Errorf("unexpected graph blob product: no compact graph row is written anymore")}}- if entities != 3 || relations != 2 || graphs != 0 {- t.Fatalf("products = %d entities + %d relations + %d graph, want 3+2+0", entities, relations, graphs)+ if entities != 3 || relations != 2 {+ t.Fatalf("products = %d entities + %d relations, want 3+2 (no graph blob)", entities, relations)}if out.DuplicatesDropped != 1 {t.Fatalf("DuplicatesDropped = %d, want 1 (the cross-chunk Beta)", out.DuplicatesDropped)diff --git a/internal/ingestion/component/knowledge_compiler/tree/claims.go b/internal/ingestion/component/knowledge_compiler/tree/claims.gonew file mode 100644index 0000000000..7c66ea45e4--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/tree/claims.go@@ -0,0 +1,677 @@+package tree++import (+ "context"+ "encoding/json"+ "fmt"+ "log"+ "os"+ "sort"+ "strconv"+ "strings"+ "sync"+ "time"+ "unicode/utf8"++ "ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// claimExtractionWorkers bounds how many claim-extraction LLM calls run at once.+// Batches are independent, so a large document would otherwise open one+// round-trip per batch and trip the provider's rate limit; this keeps the+// fan-out predictable. Mirrors Python _claim_limiter (raptor.py): the default is+// 4 and MAX_CONCURRENT_CLAIM_CHATS overrides it.+func claimExtractionWorkers() int {+ if n, err := strconv.Atoi(os.Getenv("MAX_CONCURRENT_CLAIM_CHATS")); err == nil && n > 0 {+ return n+ }+ return 4+}++// claimBatchSize is how many chunks one extraction call covers, mirroring Python+// _CLAIM_BATCH_SIZE (raptor.py). Every chunk in the batch is a TARGET and the+// model attributes each claim by chunk id.+const claimBatchSize = 4++// Retry budget for a transient LLM failure (rate limit, 5xx, timeout), mirroring+// Python raptor._extract_claim_for_chunk: 3 attempts, 2s then 4s of back-off.+const claimMaxAttempts = 3++// claimRetryBaseDelay is a var (not a const) purely so tests can shorten the+// wait; production always uses the 2s base.+var claimRetryBaseDelay = 2 * time.Second++// EvidenceGateMode decides what happens to a claim whose evidence all failed to+// locate. Mirrors Python _EVIDENCE_GATE_MODES.+type EvidenceGateMode string++const (+ // EvidenceGateSoft keeps the claim with its evidence dropped — the default,+ // because enumeration / multi-hop questions degrade badly when facts are+ // discarded (see claim_evidence.md §4.2).+ EvidenceGateSoft EvidenceGateMode = "soft"+ // EvidenceGateHard drops the claim along with its evidence, the behaviour+ // the ISC paper validated on single-document dialogue QA.+ EvidenceGateHard EvidenceGateMode = "hard"+)++// evidenceGateDefault is used when the configured mode is absent or unknown.+const evidenceGateDefault = EvidenceGateSoft++// ParseEvidenceGateMode maps a configured gate mode onto EvidenceGateMode,+// falling back to the default for anything unrecognised (mirrors Python+// _struct_evidence_gate_mode).+func ParseEvidenceGateMode(raw any) EvidenceGateMode {+ s, ok := raw.(string)+ if !ok {+ return evidenceGateDefault+ }+ switch EvidenceGateMode(strings.ToLower(strings.TrimSpace(s))) {+ case EvidenceGateSoft:+ return EvidenceGateSoft+ case EvidenceGateHard:+ return EvidenceGateHard+ }+ return evidenceGateDefault+}++// claimExtractionPrompt mirrors the Python prompt in+// rag/advanced_rag/knowlege_compile/raptor.py (_CLAIM_EXTRACTION_PROMPT). The+// two must stay in sync: a divergence here silently changes what the tree is+// summarised from, which then makes Python-derived and Go-derived trees+// incomparable.+const claimExtractionPrompt = `## Task+You are a high-recall claim harvester for the TARGET chunks below. For EACH+TARGET chunk, extract EVERY explicit atomic claim that chunk supports: factual+assertions AND explicitly expressed opinions, beliefs, judgments, assessments,+recommendations, preferences, intentions, predictions, and hypotheses. Do not+rank claims, summarize, merge claims, or omit a claim because it seems minor.++A claim must be:+- Self-contained: readable without the surrounding text, with named subjects.+ Never write "the article", "the document", "it says", or a bare pronoun.+ Resolve the referent to its name when attribution makes the claim clearer.+- Faithful: stated only if that TARGET chunk supports it. Never invent,+ strengthen, or infer. Preserve modality, uncertainty, negation, and+ attribution exactly.+- Atomic: exactly one fact. Split compound sentences.++## Response Format+Reply with a single JSON object: {"items": [{"type": "claim", "name": "<the claim, one sentence>", "description": "<optional restatement for retrieval>", "source_chunk_ids": ["<CHUNK_ID it was taken from>"], "evidence": [{"quote": "<the verbatim source sentence>", "chunk_id": "<CHUNK_ID it was taken from>"}]}, ...]}.++Rules:+- ` + "`evidence.quote`" + ` MUST be a CONTIGUOUS verbatim substring of the chunk cited+ by ` + "`evidence.chunk_id`" + `: same words, same order, no paraphrase, no truncation,+ no added words. A quote that cannot be found verbatim is rejected downstream,+ so never restate.+- ` + "`evidence.chunk_id`" + ` and ` + "`source_chunk_ids`" + ` MUST identify the exact chunk the+ claim and quote came from. Never cross-attribute a quote to a different chunk.+- Keep each quote concise and under 240 characters.+- For tables, infoboxes and bullet lists, the quote MUST be the raw cell/row+ text exactly as it appears — keep its separators and order. Do NOT turn a+ table row like "Starring | Penn Badgley | Elizabeth Lail" into a sentence+ like "Starring Penn Badgley Elizabeth Lail"; that is a paraphrase and will+ be rejected.+- Preserve numbers, units, dates, names and qualifiers exactly.+- Distribute claims across all TARGET chunks; do not skip a chunk because you+ reached the cap — the cap is per chunk, not per batch.+- If a TARGET chunk contains no extractable assertion, emit no claim for it.+- Keep claims in the same language as the source.+Return JSON only, no commentary.`++// claimEntry is one (chunk id, text) pair in a claim-extraction batch.+type claimEntry struct {+ id string+ text string+}++// renderClaimSource builds the user prompt body for one claim-extraction call.+// Every chunk in the batch is a TARGET, so the model harvests claims for all of+// them in one call. Each chunk carries its own id, which both pins the claim+// attribution and lets the validation gate check the quote against the right+// text (mirrors Python _render_claim_source).+func renderClaimSource(targets []claimEntry) string {+ var b strings.Builder+ b.WriteString("## Source Text\n")+ for _, t := range targets {+ fmt.Fprintf(&b, "[CHUNK_ID: %s (TARGET)]\n%s\n[END_CHUNK]\n", t.id, t.text)+ }+ b.WriteString("\n## Output (JSON only):")+ return b.String()+}++// Claim is one extracted claim with the verbatim evidence that backs it.+type Claim struct {+ Name string `json:"name"`+ Description string `json:"description"`+ Type string `json:"type"`+ SourceChunkIDs []string `json:"source_chunk_ids"`+ Evidence []EvidenceRef `json:"evidence,omitempty"`+}++// EvidenceRef is a verbatim quote plus where it was located. Offsets are only+// known after validation, so a claim parsed from the model carries a zero span+// until ValidateClaims resolves it against the source text.+type EvidenceRef struct {+ Quote string `json:"quote"`+ ChunkID string `json:"chunk_id"`+ Start int `json:"start"`+ End int `json:"end"`+}++// ExtractClaimsForChunks extracts claim/evidence pairs for the layer-0 chunks+// about to be clustered.+//+// It returns claims keyed by the chunk they came from, so the tree builder can+// look up a cluster's claims by its member chunk ids and summarise from those+// instead of from truncated raw text (see buildClusterContent vs+// buildClaimContent).+//+// Chunks are packed into batches of claimBatchSize (mirroring Python+// _CLAIM_BATCH_SIZE) and each batch is harvested by one LLM call. Every chunk in+// a batch is a TARGET and the model reports which chunk each claim came from+// through source_chunk_ids; the validation gate then checks each quote against+// that chunk's text.+//+// The calls are independent, so a bounded worker pool runs several at once —+// mirroring the Python side, which fans every batch out under _claim_limiter+// (default 4, MAX_CONCURRENT_CLAIM_CHATS override) instead of extracting+// serially. The worker cap keeps a large document from opening hundreds of+// concurrent round-trips against the provider's rate limit.+//+// claimPrompt is the template-declared extraction contract (Python passes+// raptor_config["claim_prompt"] from tree.yaml's raptor section into+// extract_claims_for_chunks); an empty value falls back to the built-in+// claimExtractionPrompt.+//+// Extraction is best-effort: a failing batch is skipped and simply contributes+// no claims, which makes the tree fall back to raw text for it. A missing LLM+// client disables extraction entirely.+func ExtractClaimsForChunks(ctx context.Context, deps common.Deps, llmID string, chunks []common.Chunk, mode EvidenceGateMode, claimPrompt string) map[string][]Claim {+ if deps.Chat == nil || len(chunks) == 0 {+ return nil+ }++ prompt := strings.TrimSpace(claimPrompt)+ if prompt == "" {+ prompt = claimExtractionPrompt+ }++ var entries []claimEntry+ for _, c := range chunks {+ if c.ID == "" || strings.TrimSpace(c.Text) == "" {+ continue+ }+ entries = append(entries, claimEntry{id: c.ID, text: c.Text})+ }+ if len(entries) == 0 {+ return nil+ }++ batches := packClaimBatches(entries)+ claimsByChunk := make(map[string][]Claim)+ total := len(entries)+ workers := claimExtractionWorkers()+ if workers > len(batches) {+ workers = len(batches)+ }++ // Announce the fan-out shape before the first LLM call: the batches take+ // model latency, so without this line the log would sit silent on+ // "building tree" for tens of seconds (mirrors Python's start message).+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("tree-template: claim extraction start: %d chunk(s) -> %d batch(es)", total, len(batches)))++ var (+ // mu guards claimsByChunk and serialises the progress callback, which+ // is supplied by the caller and is not required to be goroutine-safe.+ mu sync.Mutex+ wg sync.WaitGroup+ completed int+ )+ sem := make(chan struct{}, workers)+ for i := range batches {+ wg.Add(1)+ go func(i int) {+ defer wg.Done()+ // Take a slot before doing any work; if the context is already+ // done, drop out instead of queueing a call that would be+ // cancelled mid-flight.+ select {+ case sem <- struct{}{}:+ case <-ctx.Done():+ return+ }+ defer func() { <-sem }()++ batch := batches[i]++ batchClaims := extractClaimsForBatch(ctx, deps, llmID, prompt, batch, mode)++ mu.Lock()+ // Report progress *after* the batch completes, counting the chunks+ // the user-visible unit covers (mirrors Python's as_completed loop,+ // which advances the counter only for finished batches; workers+ // finish out of order, so the figure jumps by batch size).+ completed += len(batch)+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("tree-template: extracting claims for chunk %d/%d", completed, total))+ if len(batchClaims) > 0 {+ for id, claims := range batchClaims {+ claimsByChunk[id] = append(claimsByChunk[id], claims...)+ }+ }+ mu.Unlock()+ }(i)+ }+ wg.Wait()+ if len(claimsByChunk) == 0 {+ return nil+ }+ return claimsByChunk+}++// extractClaimsForBatch runs one claim-extraction call for a batch of chunks and+// returns the surviving claims keyed by the chunk they came from.+//+// It is safe for concurrent use: all state is local, and the caller owns the map+// the results are merged into. A batch that fails after its retries, yields no+// parseable items, or yields nothing attributable contributes no claims, which+// makes the tree fall back to raw text for it.+func extractClaimsForBatch(ctx context.Context, deps common.Deps, llmID, claimPrompt string, batch []claimEntry, mode EvidenceGateMode) map[string][]Claim {+ batchIDs := make(map[string]bool, len(batch))+ textByID := make(map[string]string, len(batch))+ for _, t := range batch {+ batchIDs[t.id] = true+ textByID[t.id] = t.text+ }++ raw, err := chatWithClaimRetry(ctx, deps, llmID, claimPrompt, renderClaimSource(batch), batch)+ if err != nil {+ log.Printf("tree: claim extraction skipped for batch %s: %v", claimBatchLabel(batch), err)+ return nil+ }++ items := parseClaimItems(raw)+ if len(items) == 0 {+ return nil+ }++ claims := make([]Claim, 0, len(items))+ for _, it := range items {+ it.Type = "claim"+ // Keep only ids that are actually in this batch: a hallucinated id+ // points at text the model never saw, so its quote could never be+ // validated. Missing attribution is recovered from the verified+ // evidence below rather than guessed here.+ src := make([]string, 0, len(it.SourceChunkIDs))+ for _, id := range it.SourceChunkIDs {+ if batchIDs[id] {+ src = append(src, id)+ }+ }+ it.SourceChunkIDs = src+ if strings.TrimSpace(it.Description) == "" {+ it.Description = it.Name+ }+ claims = append(claims, it)+ }++ emitted := 0+ for _, c := range claims {+ if len(c.Evidence) > 0 {+ emitted+++ }+ }+ // Validate only against this batch's text: a gate that could scan the whole+ // document would let a quote match an unrelated chunk that merely happens to+ // contain the same sentence, and would then attribute the claim to it.+ claims, verified, rejected := ValidateClaims(claims, textByID, mode)+ log.Printf("tree: claim extraction batch=%s chunks=%d claims=%d emitted_evidence=%d verified=%d rejected=%d",+ claimBatchLabel(batch), len(batch), len(claims), emitted, verified, rejected)++ // Recover attribution from the evidence that survived the gate: the chunk a+ // quote was located in IS where the claim came from. A claim that neither the+ // model nor its evidence can place is dropped — filing it under an arbitrary+ // chunk would cite text it never came from (mirrors Python, which drops the+ // claim instead of defaulting to the batch's first chunk).+ out := make(map[string][]Claim)+ for _, c := range claims {+ id := ""+ if len(c.SourceChunkIDs) > 0 {+ id = c.SourceChunkIDs[0]+ } else {+ for _, e := range c.Evidence {+ if batchIDs[e.ChunkID] {+ id = e.ChunkID+ break+ }+ }+ }+ if id == "" {+ log.Printf("tree: dropped claim with no attributable source: %s", c.Name)+ continue+ }+ c.SourceChunkIDs = []string{id}+ out[id] = append(out[id], c)+ }+ return out+}++// packClaimBatches groups chunks into fixed-size batches, mirroring Python+// _pack_claim_batches / _CLAIM_BATCH_SIZE.+func packClaimBatches(entries []claimEntry) [][]claimEntry {+ bs := claimBatchSize+ if bs < 1 {+ bs = 1+ }+ out := make([][]claimEntry, 0, (len(entries)+bs-1)/bs)+ for i := 0; i < len(entries); i += bs {+ end := i + bs+ if end > len(entries) {+ end = len(entries)+ }+ out = append(out, entries[i:end])+ }+ return out+}++// claimBatchLabel renders a batch's chunk ids for logs.+func claimBatchLabel(batch []claimEntry) string {+ ids := make([]string, 0, len(batch))+ for _, t := range batch {+ ids = append(ids, t.id)+ }+ return strings.Join(ids, ",")+}++// chatWithClaimRetry issues one claim-extraction call, retrying transient+// provider failures (rate limits, 5xx, timeouts) with exponential back-off,+// mirroring Python raptor._extract_claim_for_chunk. A cancelled context is never+// retried.+func chatWithClaimRetry(ctx context.Context, deps common.Deps, llmID, claimPrompt, prompt string, batch []claimEntry) (string, error) {+ var lastErr error+ for attempt := 1; attempt <= claimMaxAttempts; attempt++ {+ if ctx.Err() != nil {+ return "", ctx.Err()+ }+ resp, err := deps.Chat.Chat(ctx, common.ChatRequest{+ LLMID: llmID,+ SystemPrompt: claimPrompt,+ UserPrompt: prompt,+ JSONMode: true,+ DisableThinking: true,+ })+ if err == nil {+ if resp == nil {+ return "", nil+ }+ return resp.Content, nil+ }+ lastErr = err+ if !isRetryableClaimErr(err) || attempt == claimMaxAttempts {+ break+ }+ delay := claimRetryBaseDelay * time.Duration(1<<(attempt-1))+ log.Printf("tree: claim extraction retry %d/%d after %s for batch %s: %v",+ attempt, claimMaxAttempts, delay, claimBatchLabel(batch), err)+ select {+ case <-time.After(delay):+ case <-ctx.Done():+ return "", ctx.Err()+ }+ }+ return "", lastErr+}++// isRetryableClaimErr mirrors Python _RETRYABLE_LLM_ERR: capacity and transport+// failures that back-off can clear, as opposed to a malformed request that no+// amount of waiting would fix.+func isRetryableClaimErr(err error) bool {+ msg := strings.ToLower(err.Error())+ for _, s := range []string{+ "rate limit", "429", "tpm limit", "too many requests", "requests per minute",+ "server", "503", "502", "504", "500", "unavailable", "timeout", "timed out",+ } {+ if strings.Contains(msg, s) {+ return true+ }+ }+ return false+}++// parseClaimItems tolerates the model wrapping JSON in prose or fences.+func parseClaimItems(raw string) []Claim {+ s := strings.TrimSpace(raw)+ if s == "" {+ return nil+ }+ if i := strings.Index(s, "{"); i > 0 {+ s = s[i:]+ }+ if j := strings.LastIndex(s, "}"); j >= 0 && j+1 < len(s) {+ s = s[:j+1]+ }+ s = strings.TrimPrefix(s, "```json")+ s = strings.TrimPrefix(s, "```")+ s = strings.TrimSuffix(s, "```")++ var out struct {+ Items []Claim `json:"items"`+ }+ if err := json.Unmarshal([]byte(s), &out); err != nil {+ return nil+ }+ items := make([]Claim, 0, len(out.Items))+ for _, it := range out.Items {+ if strings.TrimSpace(it.Name) == "" {+ continue+ }+ items = append(items, it)+ }+ return items+}++// ValidateClaims locates every quote in the chunk it cites and drops the ones+// that cannot be found. mode decides what happens to a claim left with no+// verified evidence: soft keeps it with the evidence removed (discarding the+// claim too would silently lose facts, which hurts enumeration-style+// questions), hard drops the claim.+//+// Returns the surviving claims plus (verified, rejected) counts for the gate+// log. Mirrors Python _struct_apply_evidence_gate.+func ValidateClaims(claims []Claim, textByID map[string]string, mode EvidenceGateMode) ([]Claim, int, int) {+ verified, rejected := 0, 0+ if mode != EvidenceGateSoft && mode != EvidenceGateHard {+ mode = evidenceGateDefault+ }+ if len(textByID) == 0 {+ return claims, verified, rejected+ }+ survivors := make([]Claim, 0, len(claims))+ for i := range claims {+ c := &claims[i]+ if len(c.Evidence) == 0 {+ survivors = append(survivors, *c)+ continue+ }+ kept := make([]EvidenceRef, 0, len(c.Evidence))+ for _, e := range c.Evidence {+ if strings.TrimSpace(e.Quote) == "" {+ continue+ }+ // Prefer the cited chunk; fall back to scanning the batch when the+ // model cited nothing (or something outside it).+ candidates := []string{e.ChunkID}+ if textByID[e.ChunkID] == "" {+ candidates = make([]string, 0, len(textByID))+ for id := range textByID {+ candidates = append(candidates, id)+ }+ // Map iteration order is random, so sort: with two chunks+ // holding the same sentence the gate must pick the same one on+ // every run, otherwise a recompile repoints the quote.+ sort.Strings(candidates)+ }+ for _, id := range candidates {+ start, end, ok := LocateEvidence(e.Quote, textByID[id])+ if !ok {+ continue+ }+ kept = append(kept, EvidenceRef{Quote: e.Quote, ChunkID: id, Start: start, End: end})+ break+ }+ }+ if len(kept) > 0 {+ verified += len(kept)+ c.Evidence = kept+ survivors = append(survivors, *c)+ continue+ }+ rejected+++ c.Evidence = nil+ if mode == EvidenceGateSoft {+ survivors = append(survivors, *c)+ }+ }+ return survivors, verified, rejected+}++// LocateEvidence finds quote in text using whitespace-normalized matching,+// returning offsets against the ORIGINAL text so the span can be sliced back+// out. Normalizing tolerates the reflowing a model introduces when it copies a+// sentence across a line break.+func LocateEvidence(quote, text string) (int, int, bool) {+ if quote == "" || text == "" {+ return 0, 0, false+ }+ nText, idxMap := normalizeForMatch(text)+ nQuote, _ := normalizeForMatch(quote)+ if nQuote == "" {+ return 0, 0, false+ }+ pos := strings.Index(nText, nQuote)+ if pos < 0 {+ return 0, 0, false+ }+ endNorm := pos + len(nQuote) - 1+ if endNorm >= len(idxMap) {+ return 0, 0, false+ }+ // End is the original offset just past the match: the offset of the next+ // normalized byte, or the text length when the match runs to the end.+ // Adding 1 to the last matched byte instead would cut a multi-byte rune in+ // half whenever the match ends inside one.+ end := len(text)+ if endNorm+1 < len(idxMap) {+ end = idxMap[endNorm+1]+ }+ return idxMap[pos], end, true+}++// normalizeForMatch collapses whitespace runs, returning the normalized text+// and a byte-aligned map: idxMap[j] is the offset in the original text of byte+// j of the normalized text.+//+// The map is indexed with byte offsets, because that is what strings.Index+// returns. Recording one entry per rune misaligned every lookup as soon as the+// text held a multi-byte rune, so CJK quotes resolved to the wrong span — or+// failed to match at all.+func normalizeForMatch(text string) (string, []int) {+ var b strings.Builder+ b.Grow(len(text))+ idxMap := make([]int, 0, len(text))+ prevSpace := false+ for i := 0; i < len(text); {+ c := text[i]+ if c == ' ' || c == '\t' || c == '\n' || c == '\r' {+ if !prevSpace {+ b.WriteByte(' ')+ idxMap = append(idxMap, i)+ }+ prevSpace = true+ i+++ continue+ }+ // Copy the rune byte for byte and record one entry per byte, so idxMap+ // stays byte-aligned with the normalized string even for multi-byte+ // runes (CJK, emoji, accented characters).+ _, size := utf8.DecodeRuneInString(text[i:])+ if size <= 0 {+ size = 1+ }+ for k := 0; k < size; k++ {+ b.WriteByte(text[i+k])+ idxMap = append(idxMap, i+k)+ }+ i += size+ prevSpace = false+ }+ return b.String(), idxMap+}++// ClaimDigest renders one chunk's claims the way Python+// format_claims_for_summary (raptor.py) does — the text that is embedded as the+// chunk's claim-view clustering vector. Each claim carries the verbatim quote+// that backs it, so the embedding sees the facts AND their grounding.+func ClaimDigest(claims []Claim) string {+ var lines []string+ for _, c := range claims {+ name := strings.TrimSpace(c.Name)+ if name == "" {+ continue+ }+ if len(c.Evidence) > 0 && strings.TrimSpace(c.Evidence[0].Quote) != "" {+ lines = append(lines, fmt.Sprintf("- %s\n Evidence: \"%s\"", name, c.Evidence[0].Quote))+ } else {+ lines = append(lines, "- "+name)+ }+ }+ return strings.Join(lines, "\n")+}++// BuildClaimContent renders a cluster's member claims as the summary input.+//+// Each claim is rendered with the verbatim quote that backs it, so the+// abstraction above sees the facts AND their grounding rather than a reflowed+// paraphrase. Chunks without claims contribute their own text, so a cluster is+// never left empty just because extraction produced nothing.+func BuildClaimContent(texts []string, chunkIDs []string, pointIdxs []int, claimsByChunk map[string][]Claim) string {+ if len(claimsByChunk) == 0 {+ return ""+ }+ var parts []string+ for _, idx := range pointIdxs {+ if idx < 0 || idx >= len(chunkIDs) {+ continue+ }+ var rendered string+ if claims, ok := claimsByChunk[chunkIDs[idx]]; ok && len(claims) > 0 {+ var lines []string+ for _, c := range claims {+ name := strings.TrimSpace(c.Name)+ if name == "" {+ continue+ }+ if len(c.Evidence) > 0 && strings.TrimSpace(c.Evidence[0].Quote) != "" {+ // Plain quotes, not %q: Go's %q escapes embedded quotes and+ // non-printables, which Python's f'- {name}\n Evidence:+ // "{quote}"' does not — the two runtimes would feed the+ // abstraction layer different text for the same claim.+ lines = append(lines, fmt.Sprintf("- %s\n Evidence: \"%s\"", name, c.Evidence[0].Quote))+ } else {+ lines = append(lines, "- "+name)+ }+ }+ rendered = strings.Join(lines, "\n")+ }+ if rendered == "" {+ if idx < len(texts) {+ rendered = texts[idx]+ }+ }+ if rendered != "" {+ parts = append(parts, rendered)+ }+ }+ return strings.Join(parts, "\n")+}diff --git a/internal/ingestion/component/knowledge_compiler/tree/claims_test.go b/internal/ingestion/component/knowledge_compiler/tree/claims_test.gonew file mode 100644index 0000000000..73502ee204--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/tree/claims_test.go@@ -0,0 +1,406 @@+package tree++import (+ "context"+ "encoding/json"+ "fmt"+ "strings"+ "testing"+ "time"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// recordingEmbedder captures the texts it was asked to embed, so a test can+// assert what actually reached the vector.+type recordingEmbedder struct {+ inputs []string+}++func (e *recordingEmbedder) Encode(ctx context.Context, texts []string) ([][]float32, error) {+ e.inputs = append(e.inputs, texts...)+ out := make([][]float32, len(texts))+ for i := range texts {+ out[i] = []float32{1}+ }+ return out, nil+}++func (e *recordingEmbedder) Dimensions() int { return 1 }++// fakeChatForClaims returns canned claim JSON for whichever prompt it sees.+type fakeChatForClaims struct {+ reply string+ calls int+}++func (f *fakeChatForClaims) Chat(ctx context.Context, req common.ChatRequest) (*common.ChatResponse, error) {+ f.calls+++ return &common.ChatResponse{Content: f.reply}, nil+}++// flakyChatForClaims fails the first N calls with a retryable error, then+// succeeds — exercises the claim-extraction retry path.+type flakyChatForClaims struct {+ reply string+ failures int+ calls int+}++func (f *flakyChatForClaims) Chat(ctx context.Context, req common.ChatRequest) (*common.ChatResponse, error) {+ f.calls+++ if f.calls <= f.failures {+ return nil, fmt.Errorf("429 rate limit exceeded")+ }+ return &common.ChatResponse{Content: f.reply}, nil+}++func TestLocateEvidenceExactAndReflowed(t *testing.T) {+ text := "Post-war immigration added 2.1 million residents to the city."+ if _, _, ok := LocateEvidence(text, text); !ok {+ t.Fatal("exact quote not located")+ }+ // A sentence copied across a line break collapses to a single space; the+ // match must still succeed and map back to the original offsets.+ src := "Post-war immigration added\n2.1 million residents to the city."+ quote := text+ start, end, ok := LocateEvidence(quote, src)+ if !ok {+ t.Fatalf("reflowed quote not located")+ }+ if got, want := strings.Join(strings.Fields(src[start:end]), " "), strings.Join(strings.Fields(quote), " "); got != want {+ t.Fatalf("span sliced %q, want %q", got, want)+ }+}++func TestLocateEvidenceMultibyteOffsets(t *testing.T) {+ // A byte offset from strings.Index runs ahead of the rune offset, so a map+ // indexed per rune resolves every lookup past the first multi-byte rune to+ // the wrong span. Offsets must stay byte-aligned.+ src := "前言。中国的首都是北京。尾声。"+ quote := "中国的首都是北京。"+ start, end, ok := LocateEvidence(quote, src)+ if !ok {+ t.Fatalf("CJK quote not located in %q", src)+ }+ if got := src[start:end]; got != quote {+ t.Fatalf("span sliced %q, want %q", got, quote)+ }++ // A match that runs to the end of the text has no following byte to take+ // the offset from; the end must be len(text), not "last matched byte + 1",+ // which would land inside the final rune.+ tail := "前言。中国的首都是北京。"+ start, end, ok = LocateEvidence(quote, tail)+ if !ok {+ t.Fatal("quote at the end of the text not located")+ }+ if end != len(tail) {+ t.Fatalf("end = %d, want len(text) = %d", end, len(tail))+ }+ if got := tail[start:end]; got != quote {+ t.Fatalf("span sliced %q, want %q", got, quote)+ }+}++func TestLocateEvidenceRejectsHallucination(t *testing.T) {+ if _, _, ok := LocateEvidence("a sentence that is not in the source", "something else entirely"); ok {+ t.Fatal("hallucinated quote should not be located")+ }+ if _, _, ok := LocateEvidence("", "some text"); ok {+ t.Fatal("empty quote should not be located")+ }+ if _, _, ok := LocateEvidence("q", ""); ok {+ t.Fatal("empty source should not be located")+ }+}++func TestValidateClaimsKeepsClaimWhenEvidenceRejected(t *testing.T) {+ // Soft mode: an unverifiable quote is dropped, but the claim survives+ // without evidence. Discarding the claim too would silently lose facts.+ claims := []Claim{{+ Name: "invented",+ SourceChunkIDs: []string{"c1"},+ Evidence: []EvidenceRef{{Quote: "not in source", ChunkID: "c1"}},+ }}+ _, verified, rejected := ValidateClaims(claims, map[string]string{"c1": "real text"}, EvidenceGateSoft)+ if verified != 0 || rejected != 1 {+ t.Fatalf("got verified=%d rejected=%d, want 0/1", verified, rejected)+ }+ if len(claims) != 1 {+ t.Fatalf("claim should survive in soft mode, got %d", len(claims))+ }+ if claims[0].Evidence != nil {+ t.Fatalf("rejected evidence should be removed, got %v", claims[0].Evidence)+ }+}++func TestValidateClaimsRecordsOffsets(t *testing.T) {+ src := "Preamble. The capital is Paris. Trailing."+ claims := []Claim{{+ Name: "capital",+ SourceChunkIDs: []string{"c1"},+ Evidence: []EvidenceRef{{Quote: "The capital is Paris.", ChunkID: "c1"}},+ }}+ _, verified, rejected := ValidateClaims(claims, map[string]string{"c1": src}, EvidenceGateSoft)+ if verified != 1 || rejected != 0 {+ t.Fatalf("got verified=%d rejected=%d, want 1/0", verified, rejected)+ }+ got := claims[0].Evidence+ if len(got) != 1 {+ t.Fatalf("evidence lost: %v", claims[0].Evidence)+ }+ // The span must slice out the ORIGINAL whitespace, not the normalized one.+ if want := "The capital is Paris."; src[got[0].Start:got[0].End] != want {+ t.Fatalf("span sliced %q, want %q", src[got[0].Start:got[0].End], want)+ }+}++func TestParseClaimItemsToleratesProseAndFences(t *testing.T) {+ for _, raw := range []string{+ `{"items": [{"name": "a"}]}`,+ "Here is the JSON:\n```json\n" + `{"items": [{"name": "a"}]}` + "\n```",+ "Sure! " + `{"items": [{"name": "a"}]}` + " hope that helps",+ } {+ items := parseClaimItems(raw)+ if len(items) != 1 || items[0].Name != "a" {+ t.Fatalf("parseClaimItems(%q) = %+v, want one claim named a", raw, items)+ }+ }+ if items := parseClaimItems(`{"items": [{"description": "no name"}]}`); len(items) != 0 {+ t.Fatalf("claim without a name should be skipped: %+v", items)+ }+ if items := parseClaimItems("not json at all"); items != nil {+ t.Fatalf("unparseable input should yield nil, got %+v", items)+ }+}++func TestBuildClaimContentFallsBackToText(t *testing.T) {+ texts := []string{"raw alpha", "raw beta"}+ chunkIDs := []string{"c1", "c2"}++ // No claims at all -> "" so the caller can fall back to raw text.+ if got := BuildClaimContent(texts, chunkIDs, []int{0, 1}, nil); got != "" {+ t.Fatalf("expected empty content with no claims, got %q", got)+ }++ // A chunk with no claims contributes its own text rather than vanishing.+ claims := map[string][]Claim{+ "c1": {{Name: "Alpha happened"}},+ }+ got := BuildClaimContent(texts, chunkIDs, []int{0, 1}, claims)+ if !strings.Contains(got, "- Alpha happened") {+ t.Fatalf("claim missing from content: %q", got)+ }+ if !strings.Contains(got, "raw beta") {+ t.Fatalf("claim-less chunk should fall back to its text: %q", got)+ }++ // Verified evidence is rendered next to the claim.+ claims["c1"] = []Claim{{Name: "Alpha happened",+ Evidence: []EvidenceRef{{Quote: "alpha really happened", ChunkID: "c1"}}}}+ got = BuildClaimContent(texts, chunkIDs, []int{0}, claims)+ if !strings.Contains(got, `Evidence: "alpha really happened"`) {+ t.Fatalf("evidence missing from content: %q", got)+ }+}++func TestExtractClaimsForChunksRequiresChat(t *testing.T) {+ // No chat client -> extraction disabled, not an error.+ deps := common.Deps{TenantID: "t"}+ chunks := []common.Chunk{{ID: "c1", Text: "some text"}}+ if got := ExtractClaimsForChunks(context.Background(), deps, "llm", chunks, EvidenceGateSoft, ""); got != nil {+ t.Fatalf("expected nil without a chat client, got %v", got)+ }+}++func TestExtractClaimsForChunksValidatesAndKeysByChunk(t *testing.T) {+ src := "The capital of France is Paris."+ reply, _ := json.Marshal(map[string]any{"items": []any{+ map[string]any{+ "type": "claim",+ "name": "Paris is the capital of France",+ "source_chunk_ids": []any{"c1"},+ "evidence": []any{map[string]any{"quote": src, "chunk_id": "c1"}},+ },+ }})+ deps := common.Deps{Chat: &fakeChatForClaims{reply: string(reply)}, TenantID: "t"}+ chunks := []common.Chunk{{ID: "c1", Text: src}}++ got := ExtractClaimsForChunks(context.Background(), deps, "llm", chunks, EvidenceGateSoft, "")+ claims := got["c1"]+ if len(claims) != 1 {+ t.Fatalf("expected one claim for c1, got %+v", got)+ }+ if claims[0].Name != "Paris is the capital of France" {+ t.Fatalf("unexpected name: %q", claims[0].Name)+ }+ if len(claims[0].Evidence) != 1 {+ t.Fatalf("evidence should have been verified: %+v", claims[0])+ }+ if s, e := claims[0].Evidence[0].Start, claims[0].Evidence[0].End; src[s:e] != src {+ t.Fatalf("span %d:%d does not cover the quote", s, e)+ }+ // description defaults to the claim when the model omits it.+ if claims[0].Description != claims[0].Name {+ t.Fatalf("description should default to name, got %q", claims[0].Description)+ }+}++func TestExtractClaimsForChunksAttribution(t *testing.T) {+ // Attribution mirrors Python raptor._extract_claim_for_chunk: trust the+ // model when it names a chunk in the batch, fall back to the chunk the+ // verified quote was located in, and drop the claim when neither can place+ // it (rather than filing it under text it never came from).+ src := "The capital of France is Paris."+ marshal := func(items ...any) string {+ reply, err := json.Marshal(map[string]any{"items": items})+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }+ return string(reply)+ }+ run := func(reply string) map[string][]Claim {+ deps := common.Deps{Chat: &fakeChatForClaims{reply: reply}, TenantID: "t"}+ return ExtractClaimsForChunks(context.Background(), deps, "llm", []common.Chunk{{ID: "c1", Text: src}}, EvidenceGateSoft, "")+ }++ got := run(marshal(map[string]any{+ "name": "Paris is the capital",+ "source_chunk_ids": []any{"c1"},+ "evidence": []any{map[string]any{"quote": src, "chunk_id": "c1"}},+ }))+ if len(got["c1"]) != 1 || got["c1"][0].SourceChunkIDs[0] != "c1" {+ t.Fatalf("a batch-local source should be kept: %+v", got)+ }++ // The model named a chunk outside the batch — the quote still locates the+ // claim, so attribution is recovered from the evidence.+ got = run(marshal(map[string]any{+ "name": "Paris is the capital",+ "source_chunk_ids": []any{"does-not-exist"},+ "evidence": []any{map[string]any{"quote": src, "chunk_id": "c1"}},+ }))+ if len(got["c1"]) != 1 || got["c1"][0].SourceChunkIDs[0] != "c1" {+ t.Fatalf("attribution should be recovered from the verified evidence: %+v", got)+ }++ got = run(marshal(map[string]any{+ "name": "a claim",+ "source_chunk_ids": []any{"does-not-exist"},+ }))+ if len(got) != 0 {+ t.Fatalf("an unattributable claim should be dropped, got %+v", got)+ }+}++func TestExtractClaimsForChunksBatchesChunksPerCall(t *testing.T) {+ // Mirrors Python _CLAIM_BATCH_SIZE: one call harvests a whole batch, so the+ // call count stays flat instead of growing one-per-chunk.+ reply, _ := json.Marshal(map[string]any{"items": []any{+ map[string]any{+ "name": "a claim",+ "source_chunk_ids": []any{"c1"},+ "evidence": []any{map[string]any{"quote": "alpha text", "chunk_id": "c1"}},+ },+ }})+ chat := &fakeChatForClaims{reply: string(reply)}+ deps := common.Deps{Chat: chat, TenantID: "t"}+ chunks := []common.Chunk{+ {ID: "c1", Text: "alpha text"},+ {ID: "c2", Text: "beta text"},+ {ID: "c3", Text: "gamma text"},+ {ID: "c4", Text: "delta text"},+ }+ ExtractClaimsForChunks(context.Background(), deps, "llm", chunks, EvidenceGateSoft, "")+ if chat.calls != 1 {+ t.Fatalf("expected 1 call for %d chunks, got %d", len(chunks), chat.calls)+ }+}++func TestExtractClaimsForChunksRetriesTransientErrors(t *testing.T) {+ // Mirrors Python _RETRYABLE_LLM_ERR: a rate-limit failure is worth waiting+ // out instead of silently dropping the batch's claims.+ old := claimRetryBaseDelay+ claimRetryBaseDelay = time.Millisecond+ t.Cleanup(func() { claimRetryBaseDelay = old })++ reply, _ := json.Marshal(map[string]any{"items": []any{+ map[string]any{+ "name": "a claim",+ "source_chunk_ids": []any{"c1"},+ "evidence": []any{map[string]any{"quote": "alpha text", "chunk_id": "c1"}},+ },+ }})+ deps := common.Deps{Chat: &flakyChatForClaims{reply: string(reply), failures: 1}, TenantID: "t"}+ got := ExtractClaimsForChunks(context.Background(), deps, "llm", []common.Chunk{{ID: "c1", Text: "alpha text"}}, EvidenceGateSoft, "")+ if len(got["c1"]) != 1 {+ t.Fatalf("a retried call should still yield its claims, got %+v", got)+ }+}++func TestValidateClaimsHardModeDropsTheClaim(t *testing.T) {+ claims := []Claim{{+ Name: "invented",+ SourceChunkIDs: []string{"c1"},+ Evidence: []EvidenceRef{{Quote: "not in source", ChunkID: "c1"}},+ }}+ kept, verified, rejected := ValidateClaims(claims, map[string]string{"c1": "real text"}, EvidenceGateHard)+ if verified != 0 || rejected != 1 {+ t.Fatalf("got verified=%d rejected=%d, want 0/1", verified, rejected)+ }+ if len(kept) != 0 {+ t.Fatalf("hard mode should drop the claim, got %d survivor(s)", len(kept))+ }+}++func TestBuildTreeClaimProductsExcludesEvidenceFromVector(t *testing.T) {+ deps := common.Deps{Embed: &recordingEmbedder{}, TenantID: "t"}+ claims := map[string][]Claim{+ "c1": {{+ Name: "Paris is the capital",+ Description: "Paris is the capital",+ SourceChunkIDs: []string{"c1"},+ Evidence: []EvidenceRef{{Quote: "VERBATIM_SOURCE_SENTENCE", ChunkID: "c1", Start: 0, End: 22}},+ }},+ }+ prods, err := buildTreeClaimProducts(context.Background(), deps, "d1", claims, "")+ if err != nil {+ t.Fatalf("buildTreeClaimProducts: %v", err)+ }+ if len(prods) != 1 {+ t.Fatalf("expected 1 product, got %d", len(prods))+ }+ emb := deps.Embed.(*recordingEmbedder)+ if len(emb.inputs) != 1 {+ t.Fatalf("expected 1 embedding request, got %d", len(emb.inputs))+ }+ if strings.Contains(emb.inputs[0], "VERBATIM_SOURCE_SENTENCE") {+ t.Fatalf("evidence reached the vector input: %q", emb.inputs[0])+ }+ if !strings.Contains(emb.inputs[0], "Paris is the capital") {+ t.Fatalf("claim text missing from vector input: %q", emb.inputs[0])+ }++ // The row must be stamped so the writer can keep it out of the artifacts+ // tree, while the payload still carries the evidence for the read path.+ if got := prods[0].Meta["entity_type"]; got != "claim" {+ t.Fatalf("entity_type = %v, want claim", got)+ }+ if got := prods[0].Meta["kind"]; got != "claim" {+ t.Fatalf("kind = %v, want claim", got)+ }+ if got := prods[0].Meta["compile_kwd"]; got != "tree" {+ t.Fatalf("compile_kwd = %v, want tree", got)+ }+ var payload map[string]any+ if err := json.Unmarshal([]byte(prods[0].Content), &payload); err != nil {+ t.Fatalf("payload not JSON: %v", err)+ }+ if payload["evidence"] == nil {+ t.Fatalf("evidence must be stored on the payload: %v", payload)+ }+ if payload["type"] != "claim" {+ t.Fatalf("payload type = %v, want claim", payload["type"])+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles.go b/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles.gonew file mode 100644index 0000000000..fe361fc5c1--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles.go@@ -0,0 +1,195 @@+package tree++import (+ "context"+ "encoding/json"+ "fmt"+ "log"+ "sort"+ "strconv"+ "strings"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// rewriteDuplicateTreeNames rewrites duplicate summary titles whose+// descriptions differ, mirroring Python+// chunk_post_processor.rewrite_duplicate_tree_names (called by+// Compiler._compile_tree_templates before raptor_tree_to_graph): the graph+// projection uses titles as entity names and relation endpoints, so two nodes+// sharing a title render as one ambiguous node.+//+// Two passes, matching Python:+// 1. LLM pass — group summary/root products by title; for a group with ≥2+// members and ≥2 distinct descriptions, ask the LLM for a concise distinct+// title per node (best-effort: a failed group keeps its titles).+// 2. Deterministic pass — walk every product in slice order and suffix any+// repeated title with " (n)", enforcing the distinctness contract even+// when the LLM output collides or the call failed.+//+// A rewrite updates both Meta["title"] and the first line of Content (the+// title IS the first line by construction, see titleOf), keeping the two in+// sync for downstream indexing.+func rewriteDuplicateTreeNames(ctx context.Context, deps common.Deps, llmID string, products []common.Product) {+ if deps.Chat == nil || len(products) == 0 {+ return+ }++ // ---- LLM pass ----+ groups := map[string][]int{}+ for i := range products {+ if !isTreeSummaryProduct(products[i]) {+ continue+ }+ title := productTitle(products[i])+ if title == "" {+ continue+ }+ groups[title] = append(groups[title], i)+ }+ titles := make([]string, 0, len(groups))+ for t := range groups {+ titles = append(titles, t)+ }+ sort.Strings(titles)+ for _, title := range titles {+ idxs := groups[title]+ if len(idxs) < 2 {+ continue+ }+ descSet := map[string]struct{}{}+ for _, i := range idxs {+ descSet[products[i].Content] = struct{}{}+ }+ if len(descSet) < 2 {+ continue+ }+ if err := rewriteTitleGroup(ctx, deps, llmID, title, products, idxs); err != nil {+ log.Printf("tree: duplicate title rewrite failed for title=%q (keeping titles): %v", title, err)+ }+ }++ // ---- Deterministic uniqueness pass (always, matching Python) ----+ used := map[string]int{}+ for i := range products {+ if !isTreeSummaryProduct(products[i]) {+ continue+ }+ title := productTitle(products[i])+ if title == "" {+ continue+ }+ used[title]+++ if occurrence := used[title]; occurrence > 1 {+ setProductTitle(&products[i], fmt.Sprintf("%s (%d)", title, occurrence))+ }+ }+}++// isTreeSummaryProduct reports whether p is a tree node whose title matters+// (cluster summaries and the synthetic root).+func isTreeSummaryProduct(p common.Product) bool {+ switch p.Meta["kind"] {+ case "summary", "root":+ return true+ }+ return false+}++func productTitle(p common.Product) string {+ t, _ := p.Meta["title"].(string)+ return strings.TrimSpace(t)+}++// setProductTitle rewrites the title in Meta and replaces the first line of+// Content so titleOf(content) stays consistent with the meta title.+func setProductTitle(p *common.Product, title string) {+ p.Meta["title"] = title+ if idx := strings.IndexByte(p.Content, '\n'); idx >= 0 {+ p.Content = title + p.Content[idx:]+ } else {+ p.Content = title+ }+}++// renameOut is one LLM rename instruction.+type renameOut struct {+ ID string `json:"id"`+ Name string `json:"name"`+}++// rewriteTitleGroup asks the LLM for one distinct title per member of a+// duplicate-title group and applies the results.+func rewriteTitleGroup(ctx context.Context, deps common.Deps, llmID, title string, products []common.Product, idxs []int) error {+ type renameItem struct {+ ID string `json:"id"`+ Description string `json:"description"`+ }++ items := make([]renameItem, 0, len(idxs))+ for pos, i := range idxs {+ items = append(items, renameItem{ID: strconv.Itoa(pos), Description: products[i].Content})+ }+ itemsJSON, err := json.Marshal(items)+ if err != nil {+ return err+ }+ prompt := "The following tree nodes currently have the same title but describe different content. " ++ "Give each node a concise, distinct human-readable title. Preserve the original language, " ++ "do not add numbering unless necessary, and return only a JSON array of objects with the " ++ "same ids and a name field.\n\n" ++ "Current title: " + title + "\n" ++ "Nodes: " + string(itemsJSON)++ temp := 0.0+ resp, err := deps.Chat.Chat(ctx, common.ChatRequest{+ LLMID: llmID,+ SystemPrompt: "You rename duplicate tree node titles for display.",+ UserPrompt: prompt,+ JSONMode: true,+ Temperature: &temp,+ })+ if err != nil {+ return err+ }+ if resp == nil || strings.TrimSpace(resp.Content) == "" {+ return fmt.Errorf("empty rename response")+ }++ var rewrites []renameOut+ if err := unmarshalRenames(resp.Content, &rewrites); err != nil {+ return err+ }+ for _, r := range rewrites {+ newTitle := strings.TrimSpace(r.Name)+ if newTitle == "" {+ continue+ }+ pos, convErr := strconv.Atoi(strings.TrimSpace(r.ID))+ if convErr != nil || pos < 0 || pos >= len(idxs) {+ // A hallucinated id must not touch an unrelated product.+ continue+ }+ setProductTitle(&products[idxs[pos]], newTitle)+ }+ return nil+}++// unmarshalRenames parses the LLM rename array, tolerating code fences.+func unmarshalRenames(raw string, out *[]renameOut) error {+ s := strings.TrimSpace(raw)+ s = strings.TrimPrefix(s, "```json")+ s = strings.TrimPrefix(s, "```")+ s = strings.TrimSuffix(s, "```")+ if err := json.Unmarshal([]byte(s), out); err == nil {+ return nil+ }+ // Fallback: the model may have wrapped the array in extra prose — grab the+ // first '[' … last ']' span.+ start := strings.IndexByte(s, '[')+ end := strings.LastIndexByte(s, ']')+ if start < 0 || end <= start {+ return fmt.Errorf("no JSON array in rename response")+ }+ return json.Unmarshal([]byte(s[start:end+1]), out)+}diff --git a/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles_test.go b/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles_test.gonew file mode 100644index 0000000000..5035e4e1ac--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles_test.go@@ -0,0 +1,154 @@+package tree++import (+ "context"+ "fmt"+ "strings"+ "testing"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// renameChat replies with a canned rename JSON for any prompt.+type renameChat struct {+ response string+ prompts []string+}++func (f *renameChat) Chat(_ context.Context, req common.ChatRequest) (*common.ChatResponse, error) {+ f.prompts = append(f.prompts, req.UserPrompt)+ return &common.ChatResponse{Content: f.response}, nil+}++func summaryProduct(title, body string) common.Product {+ content := title+ if body != "" {+ content = title + "\n" + body+ }+ return common.Product{+ ID: "node-" + title,+ Content: content,+ Meta: map[string]any{"title": title, "kind": "summary", "level": 0},+ }+}++// Group with two members and two distinct descriptions: LLM renames both.+func TestRewriteDuplicatesLLMRenames(t *testing.T) {+ chat := &renameChat{response: `[{"id":"0","name":"Alpha Plan"},{"id":"1","name":"Alpha Report"}]`}+ products := []common.Product{+ summaryProduct("Alpha", "quarterly planning notes"),+ summaryProduct("Beta", "unrelated"),+ summaryProduct("Alpha", "financial report summary"),+ }+ rewriteDuplicateTreeNames(context.Background(), common.Deps{Chat: chat}, "llm-1", products)++ if len(chat.prompts) != 1 {+ t.Fatalf("LLM calls = %d, want 1 (only the Alpha group qualifies)", len(chat.prompts))+ }+ if got := products[0].Meta["title"]; got != "Alpha Plan" {+ t.Errorf("products[0] title = %v, want Alpha Plan", got)+ }+ if got := products[2].Meta["title"]; got != "Alpha Report" {+ t.Errorf("products[2] title = %v, want Alpha Report", got)+ }+ // Content first line must track the meta title.+ if !strings.HasPrefix(products[0].Content, "Alpha Plan\n") {+ t.Errorf("products[0] content = %q, want first line replaced", products[0].Content)+ }+ if products[1].Meta["title"] != "Beta" {+ t.Errorf("untouched product title changed: %v", products[1].Meta["title"])+ }+}++// Same title but identical descriptions: no LLM call, deterministic suffix only.+func TestRewriteDuplicatesIdenticalDescriptionsNoLLM(t *testing.T) {+ chat := &renameChat{response: `[]`}+ products := []common.Product{+ summaryProduct("Alpha", "same body"),+ summaryProduct("Alpha", "same body"),+ }+ rewriteDuplicateTreeNames(context.Background(), common.Deps{Chat: chat}, "llm-1", products)++ if len(chat.prompts) != 0 {+ t.Fatalf("LLM calls = %d, want 0 (descriptions identical)", len(chat.prompts))+ }+ if got := products[0].Meta["title"]; got != "Alpha" {+ t.Errorf("first occurrence title = %v, want Alpha", got)+ }+ if got := products[1].Meta["title"]; got != "Alpha (2)" {+ t.Errorf("second occurrence title = %v, want Alpha (2)", got)+ }+}++// LLM returns colliding names: the deterministic pass must still make them unique.+func TestRewriteDuplicatesDeterministicSuffixAfterCollision(t *testing.T) {+ chat := &renameChat{response: `[{"id":"0","name":"Gamma"},{"id":"1","name":"Gamma"}]`}+ products := []common.Product{+ summaryProduct("Alpha", "body one"),+ summaryProduct("Alpha", "body two"),+ }+ rewriteDuplicateTreeNames(context.Background(), common.Deps{Chat: chat}, "llm-1", products)++ titles := map[string]bool{}+ for _, p := range products {+ titles[p.Meta["title"].(string)] = true+ }+ if len(titles) != 2 {+ t.Errorf("titles after rewrite = %v, want 2 distinct", titles)+ }+}++// LLM failure: titles survive via the deterministic suffix, no error escapes.+func TestRewriteDuplicatesLLMFailureBestEffort(t *testing.T) {+ chat := &failingRenameChat{}+ products := []common.Product{+ summaryProduct("Alpha", "body one"),+ summaryProduct("Alpha", "body two"),+ }+ rewriteDuplicateTreeNames(context.Background(), common.Deps{Chat: chat}, "llm-1", products)++ if products[0].Meta["title"] != "Alpha" || products[1].Meta["title"] != "Alpha (2)" {+ t.Errorf("titles after failure = %v / %v, want Alpha / Alpha (2)",+ products[0].Meta["title"], products[1].Meta["title"])+ }+}++type failingRenameChat struct{}++func (f *failingRenameChat) Chat(_ context.Context, _ common.ChatRequest) (*common.ChatResponse, error) {+ return nil, fmt.Errorf("llm down")+}++// Fenced JSON and a hallucinated id are tolerated.+func TestRewriteDuplicatesFencedJSONAndBadID(t *testing.T) {+ chat := &renameChat{response: "```json\n[{\"id\":\"1\",\"name\":\"Alpha Two\"},{\"id\":\"99\",\"name\":\"Hijack\"}]\n```"}+ products := []common.Product{+ summaryProduct("Alpha", "body one"),+ summaryProduct("Alpha", "body two"),+ }+ rewriteDuplicateTreeNames(context.Background(), common.Deps{Chat: chat}, "llm-1", products)++ if products[1].Meta["title"] != "Alpha Two" {+ t.Errorf("fenced rename not applied: %v", products[1].Meta["title"])+ }+ if products[0].Meta["title"] != "Alpha" {+ t.Errorf("first product should keep its title (bad id): %v", products[0].Meta["title"])+ }+}++// No duplicates at all: no LLM call, no changes.+func TestRewriteDuplicatesNoop(t *testing.T) {+ chat := &renameChat{response: `[]`}+ products := []common.Product{+ summaryProduct("Alpha", "one"),+ summaryProduct("Beta", "two"),+ }+ rewriteDuplicateTreeNames(context.Background(), common.Deps{Chat: chat}, "llm-1", products)++ if len(chat.prompts) != 0 {+ t.Fatalf("LLM calls = %d, want 0", len(chat.prompts))+ }+ if products[0].Meta["title"] != "Alpha" || products[1].Meta["title"] != "Beta" {+ t.Errorf("titles changed on noop: %v / %v", products[0].Meta["title"], products[1].Meta["title"])+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/tree/graph.go b/internal/ingestion/component/knowledge_compiler/tree/graph.goindex a6b3a678ac..9aafe2c3f2 100644--- a/internal/ingestion/component/knowledge_compiler/tree/graph.go+++ b/internal/ingestion/component/knowledge_compiler/tree/graph.go@@ -72,39 +72,12 @@ func payloadChunkIDs(payload map[string]any) []string {return nil}-// payloadDescription is the embedding input for a tree-graph entity/relation:-// the concatenated string values of every field except description (lists-// flattened), matching Python _struct_payload_description.+// payloadDescription is the index text for a tree-graph entity/relation. It+// delegates to common.PayloadDescription — the shared implementation the+// structure variant also uses — so both variants index compiled rows the same+// way and both match Python _struct_payload_description.func payloadDescription(payload map[string]any) string {- keys := make([]string, 0, len(payload))- for k := range payload {- if k != "description" {- keys = append(keys, k)- }- }- sort.Strings(keys)- var parts []string- for _, k := range keys {- switch v := payload[k].(type) {- case string:- if v != "" {- parts = append(parts, v)- }- case []string:- for _, e := range v {- if e != "" {- parts = append(parts, e)- }- }- case []any:- for _, e := range v {- if s, ok := e.(string); ok && s != "" {- parts = append(parts, s)- }- }- }- }- return strings.Join(parts, " ")+ return common.PayloadDescription(payload, nil)}// payloadJSON serialises a payload the way Python's json.dumps(ensure_ascii=@@ -219,14 +192,14 @@ func raptorTreeToGraph(root *graphNode) ([]map[string]any, []map[string]any) {}// buildTreeGraph reconstructs the tree from the flat summary products and-// produces the entity/relation/graph products Python writes for a tree variant-// (_struct_upsert_tree_graph_rows + _struct_upsert_graph_json):+// produces the per-row products Python writes for a tree variant+// (_struct_upsert_tree_graph_rows):// - one entity product per tree node (kind "entity", knowledge_graph_kwd via// the writer), type "tree_node";-// - one relation product per parent→child edge (kind "relation");-// - one compact graph blob product (kind "graph") carrying the whole-// {entities, relations} projection, which is the /structure/graph discovery-// row (Python scans knowledge_graph_kwd="graph").+// - one relation product per parent→child edge (kind "relation").+//+// The compact graph blob (knowledge_graph_kwd="graph") is gone from the+// storage model: the per-row rows ARE the graph, exactly like page_index.//// templateID is stamped into each row so the document-structure endpoint can// group by template id; compileKWD is "tree".@@ -300,29 +273,125 @@ func buildTreeGraph(ctx context.Context, deps common.Deps, docID string, product})}- // Compact graph blob discovery row (knowledge_graph_kwd="graph").- graph := map[string]any{"entities": entities, "relations": relations}- graphContent := payloadJSON(graph)- graphVecs, err := deps.Embed.Encode(ctx, []string{graphContent})+ return out, nil+}++// sortedKeys returns the claim map's chunk ids in a stable order, so repeated+// compilations produce the same product sequence (and therefore the same row+// ids) even though Go map iteration is randomized.+func sortedKeys(m map[string][]Claim) []string {+ keys := make([]string, 0, len(m))+ for k := range m {+ keys = append(keys, k)+ }+ sort.Strings(keys)+ return keys+}++// buildTreeClaimProducts turns extracted claims into their own searchable rows.+//+// Claims are deliberately NOT part of the structure graph: they carry no+// relation, and a row with no relation would be rendered as a root in the+// artifacts tree. They get kind "claim" so the writer can give them+// entity_type_kwd="claim" while leaving knowledge_graph_kwd unset — which keeps+// them out of the artifacts query (it filters knowledge_graph_kwd=["entity",+// "relation"]) without any frontend change.+//+// The embedding excludes evidence, matching the page_index claim path and the+// Python implementation: the geometric layer indexes the claim, not the raw+// source it was verified against.+//+// templateID scopes the row to the compilation template that produced it, like+// Python's _struct_upsert_tree_claim_rows: it is stamped on the product and+// mixed into the row id, so two templates that extract the same claim from the+// same document get two rows instead of overwriting each other — and a+// template-scoped cleanup can remove one without touching the other.+func buildTreeClaimProducts(ctx context.Context, deps common.Deps, docID string, claimsByChunk map[string][]Claim, templateID string) ([]common.Product, error) {+ if len(claimsByChunk) == 0 {+ return nil, nil+ }+ if deps.Embed == nil {+ return nil, fmt.Errorf("tree: embedder required to build claim rows")+ }++ var payloads []map[string]any+ var descs []string+ for _, chunkID := range sortedKeys(claimsByChunk) {+ for _, c := range claimsByChunk[chunkID] {+ name := strings.TrimSpace(c.Name)+ if name == "" {+ continue+ }+ payload := map[string]any{+ "type": "claim",+ "name": name,+ "description": c.Description,+ "source_chunk_ids": []string{chunkID},+ }+ if len(c.Evidence) > 0 {+ ev := make([]map[string]any, 0, len(c.Evidence))+ for _, e := range c.Evidence {+ ev = append(ev, map[string]any{+ "quote": e.Quote,+ "chunk_id": e.ChunkID,+ "start": e.Start,+ "end": e.End,+ })+ }+ payload["evidence"] = ev+ }+ // Exclusion is the default in payloadDescription, so evidence never+ // reaches the vector.+ descs = append(descs, payloadDescription(payload))+ payloads = append(payloads, payload)+ }+ }+ if len(payloads) == 0 {+ return nil, nil+ }++ vecs, err := deps.Embed.Encode(ctx, descs)if err != nil {return nil, err}- var gv []float32- if len(graphVecs) > 0 {- gv = graphVecs[0]+ // One vector per payload, no exceptions: a short read means the vectors no+ // longer line up with the payloads, and writing claim rows with missing or+ // mismatched vectors would silently corrupt retrieval. Fail before writing+ // anything so a bad batch never leaves half-embedded claims behind.+ if len(vecs) != len(payloads) {+ return nil, fmt.Errorf("tree: embedder returned %d vector(s) for %d claim payload(s)", len(vecs), len(payloads))+ }++ out := make([]common.Product, 0, len(payloads))+ for i, payload := range payloads {+ vec := vecs[i]+ content := payloadJSON(payload)+ // Mirror Python _struct_to_doc_storage_doc, which mixes the template id+ // into the row-id seed (row_seed_extras): without it two templates that+ // extract an identical claim from the same document would share a row id+ // and silently overwrite one another.+ idParts := []string{content, docID}+ if templateID != "" {+ idParts = append(idParts, templateID)+ }+ out = append(out, common.Product{+ ID: common.StableRowID(idParts...),+ DocID: docID,+ TemplateID: templateID,+ TenantID: deps.TenantID,+ Variant: common.VariantTree,+ Content: content,+ Vector: vec,+ Meta: map[string]any{+ "kind": "claim",+ "compile_kwd": "tree",+ "entity_type": "claim",+ "name": payload["name"],+ "source_chunk_ids": []string{payload["source_chunk_ids"].([]string)[0]},+ "mention_count": 1,+ },+ })}- out = append(out, common.Product{- ID: common.StableRowID(docID, "tree", "structure_graph"),- DocID: docID,- TenantID: deps.TenantID,- Variant: common.VariantTree,- Content: graphContent,- Vector: gv,- Meta: map[string]any{- "kind": "graph",- "compile_kwd": "tree",- },- })return out, nil}diff --git a/internal/ingestion/component/knowledge_compiler/tree/raptor.go b/internal/ingestion/component/knowledge_compiler/tree/raptor.goindex 9cab38cb48..c83b7aad11 100644--- a/internal/ingestion/component/knowledge_compiler/tree/raptor.go+++ b/internal/ingestion/component/knowledge_compiler/tree/raptor.go@@ -50,13 +50,22 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo// chunk list). buildTree reads the texts, ids, and embeddings directly from// the source chunks.var products []common.Product- if err := buildTree(ctx, deps, llmID, tenantID, docID, inputs.Chunks, treeOrder, taskPrompt, param, &products); err != nil {+ var claimsByChunk map[string][]Claim+ if err := buildTree(ctx, deps, llmID, tenantID, docID, inputs.Chunks, treeOrder, taskPrompt, param, &products, &claimsByChunk); err != nil {return common.Outputs{}, err}+ // Rename duplicate summary titles whose descriptions differ (Python+ // rewrite_duplicate_tree_names), BEFORE the graph projection: titles are+ // the entity names and the relation endpoints, so two nodes sharing a+ // title would render as one ambiguous node. Best-effort — a rewrite+ // failure must not cost us the tree.+ rewriteDuplicateTreeNames(ctx, deps, llmID, products)+// Project the RAPTOR tree onto the {entities, relations} structure-graph// shape (Python raptor_tree_to_graph) and persist it as entity/relation rows- // plus a compact graph blob (knowledge_graph_kwd="graph"), so the+ // plus per-row entity/relation rows (knowledge_graph_kwd="entity"|+ // "relation"), so the// document-structure /structure/graph endpoint can serve the tree. A failure// here must not abort the whole tree compile — the summary nodes are already// valid on their own — so it is best-effort and surfaced as a log.@@ -66,6 +75,15 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoproducts = append(products, graphProds...)}+ // Claims become their own searchable rows so global KNN can hit them+ // directly instead of only reaching them through beam descent. Also+ // best-effort: a failure here must not cost us the tree.+ if claimProds, err := buildTreeClaimProducts(ctx, deps, docID, claimsByChunk, param.TemplateID); err != nil {+ log.Printf("tree: claim rows failed (best-effort, continuing): %v", err)+ } else {+ products = append(products, claimProds...)+ }+out := common.Outputs{Products: products,}@@ -153,7 +171,12 @@ func resolveMaxErrors(param common.Param) int {// never calls the embedder directly. treeOrder drives both the top-level// clustering (performed inside buildTree via watershed) and the recursive// sub-clustering, so the same method is used at every level of the tree.-func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID string, chunks []common.Chunk, treeOrder int, taskPrompt string, param common.Param, products *[]common.Product) error {+//+// claimOut, when non-nil, receives the extracted claims keyed by chunk id so the+// caller can persist them as their own rows (see buildTreeClaimProducts). It is+// an out-param rather than a return value so buildTree keeps a single error+// return for its existing callers.+func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID string, chunks []common.Chunk, treeOrder int, taskPrompt string, param common.Param, products *[]common.Product, claimOut *map[string][]Claim) error {maxToken := resolveMaxToken(param)maxErrors := resolveMaxErrors(param)errorCount := 0@@ -173,6 +196,77 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str}embeddings = toFloat64Matrix(vectors)}+ // Extract claims before clustering so each cluster can be summarised from+ // its members' claims instead of their truncated raw text. Claims are far+ // more compact than the source, so the per-chunk truncation in+ // buildClusterContent no longer discards content, and the abstraction is+ // guaranteed to agree with the claims attached to the same cluster.+ //+ // The gate mode comes from the template config, mirroring Python+ // _struct_evidence_gate_mode(parser_config); the extraction toggle and the+ // template-declared extraction contract mirror Python's+ // raptor_config.get("extract_claims", True) / raptor_config["claim_prompt"].+ var claimsByChunk map[string][]Claim+ if resolveExtractClaims(param) {+ claimsByChunk = ExtractClaimsForChunks(ctx, deps, llmID, chunks,+ ParseEvidenceGateMode(param.Extra["evidence_gate_mode"]), resolveClaimPrompt(param))+ if len(claimsByChunk) > 0 {+ log.Printf("tree: extracted claims for %d/%d chunk(s)", len(claimsByChunk), len(chunks))+ }+ }+ if claimOut != nil {+ *claimOut = claimsByChunk+ }++ // Claim-view clustering vectors (mirrors Python build_doc_tree): each chunk+ // is represented to the clustering layer by the embedding of its claims ++ // verbatim evidence, not by its raw-text vector. Claims are the chunk's+ // semantics stripped of layout noise, so topic clusters form around what+ // the chunks actually assert. Chunks without claims keep their raw vector,+ // and a claim vector whose dimension disagrees with the chunk vectors is+ // ignored rather than fed into clustering (mixed-model guard).+ if len(claimsByChunk) > 0 {+ digestIDs := make([]string, 0, len(claimsByChunk))+ digests := make([]string, 0, len(claimsByChunk))+ for _, cid := range chunkIDs {+ claims, ok := claimsByChunk[cid]+ if !ok || len(claims) == 0 {+ continue+ }+ digest := ClaimDigest(claims)+ if strings.TrimSpace(digest) == "" {+ continue+ }+ digestIDs = append(digestIDs, cid)+ digests = append(digests, digest)+ }+ if len(digests) > 0 {+ if claimVecs, err := deps.Embed.Encode(ctx, digests); err != nil {+ log.Printf("tree: claim-view embedding failed; clustering on raw chunk vectors: %v", err)+ } else {+ claimView := make(map[string][]float32, len(digestIDs))+ for i, cid := range digestIDs {+ if i < len(claimVecs) && len(claimVecs[i]) > 0 {+ claimView[cid] = claimVecs[i]+ }+ }+ replaced := 0+ for i, cid := range chunkIDs {+ view, ok := claimView[cid]+ if !ok {+ continue+ }+ if len(embeddings[i]) > 0 && len(view) != len(embeddings[i]) {+ continue // mixed-model guard: never feed mixed dimensions in+ }+ embeddings[i] = toFloat64Slice(view)+ replaced+++ }+ log.Printf("tree: %d/%d chunk(s) clustered on claim-view embeddings", replaced, len(chunkIDs))+ }+ }+ }+labels, err := watershed(embeddings, treeOrder)if err != nil {return err@@ -208,7 +302,19 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str// text to a per-chunk token budget so the cluster fits the LLM context// (Python: len_per_chunk = (max_length - max_token) / len(texts);// truncate(t, len_per_chunk), raptor.py:389-390).- content := buildClusterContent(texts, task.pointIdxs, deps.ModelContextLen, maxToken)+ //+ // At the bottom level, prefer the members' claims: BuildClaimContent+ // returns "" when a cluster has none, so we fall through to the raw+ // text path. Upper levels keep using raw text — they re-cluster the+ // same original points, so claim-feeding them would only repeat what+ // the level-0 summaries already compressed.+ content := ""+ if task.level == 0 {+ content = BuildClaimContent(texts, chunkIDs, task.pointIdxs, claimsByChunk)+ }+ if content == "" {+ content = buildClusterContent(texts, task.pointIdxs, deps.ModelContextLen, maxToken)+ }system := raptorSystemHelper + strings.Replace(taskPrompt, "{cluster_content}", content, 1)summary, err := summarizeTexts(ctx, deps, llmID, system, raptorTitleInstruction, maxToken)if err != nil {@@ -360,10 +466,11 @@ func summarizeTexts(ctx context.Context, deps common.Deps, llmID, systemText, us}}resp, err := deps.Chat.Chat(ctx, common.ChatRequest{- LLMID: llmID,- SystemPrompt: systemText,- UserPrompt: userText,- MaxTokens: &mt,+ LLMID: llmID,+ SystemPrompt: systemText,+ UserPrompt: userText,+ MaxTokens: &mt,+ DisableThinking: true,})if err != nil {continue@@ -575,13 +682,73 @@ const raptorSystemHelper = "You're a helpful assistant.\n\nHelp me with the foll// in the same language as the paragraphs.").const raptorTitleInstruction = "Beside the summarization, give a title at the first line of your summarization. Must be in the same language as the paragraphs."-// resolveRaptorPrompt returns the summary task template, honouring an-// extra["prompt"] override; falls back to defaultRaptorPrompt.+// templateRaptorString reads a string field from the template config's+// "raptor" section (tree.yaml's raptor: block, which Python compiles into+// raptor_config — compiler.py) and returns "" when absent.+func templateRaptorString(param common.Param, key string) string {+ cfg, ok := param.TemplateConfig["raptor"].(map[string]any)+ if !ok {+ return ""+ }+ return strings.TrimSpace(cfgStr(cfg[key]))+}++// templateRaptorBool reads a boolean field from the template config's "raptor"+// section, reporting presence so callers can distinguish unset from false.+func templateRaptorBool(param common.Param, key string) (bool, bool) {+ cfg, ok := param.TemplateConfig["raptor"].(map[string]any)+ if !ok {+ return false, false+ }+ b, ok := cfg[key].(bool)+ return b, ok+}++func cfgStr(v any) string {+ s, _ := v.(string)+ return s+}++// resolveRaptorPrompt returns the summary task template, honouring in order an+// extra["prompt"] caller override, the template's raptor.prompt (Python:+// raptor_cfg["prompt"]), and the built-in default.func resolveRaptorPrompt(param common.Param) string {if raw, ok := param.Extra["prompt"]; ok {if s, ok := raw.(string); ok && strings.TrimSpace(s) != "" {return s}}+ if s := templateRaptorString(param, "prompt"); s != "" {+ return s+ }return defaultRaptorPrompt}++// resolveClaimPrompt returns the claim-extraction system prompt, honouring in+// order an extra["claim_prompt"] caller override and the template's+// raptor.claim_prompt (Python passes raptor_config["claim_prompt"] into+// extract_claims_for_chunks). An empty result means "use the built-in+// contract" — the same None-fallback Python applies.+func resolveClaimPrompt(param common.Param) string {+ if raw, ok := param.Extra["claim_prompt"]; ok {+ if s, ok := raw.(string); ok && strings.TrimSpace(s) != "" {+ return s+ }+ }+ return templateRaptorString(param, "claim_prompt")+}++// resolveExtractClaims mirrors Python raptor_config.get("extract_claims", True):+// claim harvesting is on unless the template (or a caller override) turns it+// off, in which case clusters summarise from raw chunk text.+func resolveExtractClaims(param common.Param) bool {+ if raw, ok := param.Extra["extract_claims"]; ok {+ if b, ok := raw.(bool); ok {+ return b+ }+ }+ if b, ok := templateRaptorBool(param, "extract_claims"); ok {+ return b+ }+ return true+}diff --git a/internal/ingestion/component/knowledge_compiler/tree/raptor_test.go b/internal/ingestion/component/knowledge_compiler/tree/raptor_test.goindex c95ed71624..fbc000cc3c 100644--- a/internal/ingestion/component/knowledge_compiler/tree/raptor_test.go+++ b/internal/ingestion/component/knowledge_compiler/tree/raptor_test.go@@ -163,7 +163,7 @@ func TestBuildTreeNoPanicWhenAllSummariesFail(t *testing.T) {{Text: "beta", Vector: []float32{0, 1, 0, 0}},}var products []common.Product- if err := buildTree(context.Background(), deps, "llm", "t", "d", chunks, 4, "", common.Param{}, &products); err != nil {+ if err := buildTree(context.Background(), deps, "llm", "t", "d", chunks, 4, "", common.Param{}, &products, nil); err != nil {t.Fatalf("buildTree returned unexpected error: %v", err)}if len(products) != 0 {diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex f8cd8546d1..6624a8d273 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -1464,7 +1464,16 @@ func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDoa.in.Summary = p.Contenta.in.Embedd = p.Vectorcase kccommon.VariantStructure:- if kind, _ := p.Meta["kind"].(string); kind != "graph" {+ // Python's page_index path rebuilds the doc graph from the stored+ // entity rows and folds their descriptions into the nav summary+ // (runner.py rebuild_structure_graph_json + _page_index_graph_summary).+ // The graph blob product (Meta.kind=="graph") is gone from the+ // storage model, so fold directly from the entity rows.+ if kind, _ := p.Meta["kind"].(string); kind != "entity" {+ continue+ }+ line := structureEntityNavLine(p.Content)+ if line == "" {continue}a := byDoc[p.DocID]@@ -1472,9 +1481,10 @@ func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDobyDoc[p.DocID] = &acc{in: nav.UpsertDocInput{TenantID: p.TenantID, KbID: kb, DocID: p.DocID}}a = byDoc[p.DocID]}- // Graph vector is NOT the summary vector; leave Embedd empty so- // NavService embeds the folded summary text.- a.in.Summary = pageIndexSummary(p.Content)+ if a.in.Summary != "" {+ a.in.Summary += "\n"+ }+ a.in.Summary += line}}out := make([]nav.UpsertDocInput, 0, len(byDoc))@@ -1487,33 +1497,24 @@ func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDoreturn out}-// pageIndexSummary folds the entity descriptions of a structure graph JSON-// ({"entities":[{"name","description"},...]}) into a document-level summary for-// dataset navigation, matching the component's by-product logic.-func pageIndexSummary(graphJSON string) string {- var graph struct {- Entities []struct {- Name string `json:"name"`- Description string `json:"description"`- } `json:"entities"`- }- if err := json.Unmarshal([]byte(graphJSON), &graph); err != nil {+// structureEntityNavLine renders one entity row's payload JSON as a+// "name: description" line for the page_index nav summary.+func structureEntityNavLine(payloadJSON string) string {+ var ent struct {+ Name string `json:"name"`+ Description string `json:"description"`+ }+ if err := json.Unmarshal([]byte(payloadJSON), &ent); err != nil {return ""}- var b strings.Builder- for _, e := range graph.Entities {- desc := strings.Join(strings.Fields(e.Description), " ")- if desc == "" {- continue- }- if e.Name != "" {- b.WriteString(e.Name)- b.WriteString(": ")- }- b.WriteString(desc)- b.WriteString("\n")+ desc := strings.Join(strings.Fields(ent.Description), " ")+ if desc == "" {+ return ""+ }+ if ent.Name != "" {+ return ent.Name + ": " + desc}- return b.String()+ return desc}// mergeStructureDataset performs the dataset-level structure merge (G1/G4): itdiff --git a/internal/ingestion/knowledge_compile/nav_dispatch_test.go b/internal/ingestion/knowledge_compile/nav_dispatch_test.goindex 717d385ed3..b1dad687e5 100644--- a/internal/ingestion/knowledge_compile/nav_dispatch_test.go+++ b/internal/ingestion/knowledge_compile/nav_dispatch_test.go@@ -11,7 +11,10 @@ import (// TestNavInputFromProducts_TreeAndStructure covers B2: nav is fed by BOTH tree// and structure products (not tree alone). Tree uses the root product's summary;-// structure folds the graph entity descriptions into a summary via pageIndexSummary.+// structure folds the per-document entity ROW descriptions into a summary+// (mirroring Python runner.py rebuild_structure_graph_json ++// _page_index_graph_summary — the graph blob product is gone from the storage+// model).func TestNavInputFromProducts_TreeAndStructure(t *testing.T) {products := []kccommon.Product{// tree: root product carries the doc summary + vector.@@ -21,10 +24,17 @@ func TestNavInputFromProducts_TreeAndStructure(t *testing.T) {// tree: a non-root summary node is NOT a nav input.{DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree,Content: "section body", Meta: map[string]any{"kind": "summary", "level": 0}},- // structure: graph product folds entity descriptions.+ // structure: entity rows fold their descriptions into the doc summary.{DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,- Content: `{"entities":[{"name":"Engine","description":"a propulsion device"}]}`,- Meta: map[string]any{"kind": "graph", "compile_kwd": "page_index"}},+ Content: `{"name":"Engine","type":"component","description":"a propulsion device"}`,+ Meta: map[string]any{"kind": "entity", "compile_kwd": "page_index"}},+ {DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,+ Content: `{"name":"Turbine","type":"component","description":"converts flow into rotation"}`,+ Meta: map[string]any{"kind": "entity", "compile_kwd": "page_index"}},+ // structure: a relation row is NOT a nav summary line.+ {DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,+ Content: `{"from":"Engine","to":"Turbine","type":"drives"}`,+ Meta: map[string]any{"kind": "relation", "compile_kwd": "page_index", "from": "Engine", "to": "Turbine"}},}got := navInputFromProducts("kb1", products)@@ -52,7 +62,13 @@ func TestNavInputFromProducts_TreeAndStructure(t *testing.T) {if !strings.Contains(structIn.Summary, "Engine: a propulsion device") {t.Errorf("structure summary = %q, want folded entity descriptions", structIn.Summary)}- // Structure graph vector is NOT the summary vector: leave Embedd empty so+ if !strings.Contains(structIn.Summary, "Turbine: converts flow into rotation") {+ t.Errorf("structure summary = %q, want both entity lines", structIn.Summary)+ }+ if strings.Contains(structIn.Summary, "drives") {+ t.Errorf("structure summary must not include relation rows, got %q", structIn.Summary)+ }+ // The entity rows' vectors are NOT the summary vector: leave Embedd empty so// NavService embeds the folded summary text.if len(structIn.Embedd) != 0 {t.Errorf("structure embedd should be empty (NavService re-embeds), got %v", structIn.Embedd)@@ -73,14 +89,15 @@ func TestNavInputFromProducts_WikiExcluded(t *testing.T) {}// TestNavInputFromProducts_EmptySummaryDropped covers the guard: a doc whose-// tree/structure summary is empty (no root / no graph) is skipped, not fed empty.+// tree/structure summary is empty (no root / no entity descriptions) is+// skipped, not fed empty.func TestNavInputFromProducts_EmptySummaryDropped(t *testing.T) {products := []kccommon.Product{// tree root with empty content -> skip.{DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree, Meta: map[string]any{"kind": "root"}},- // structure graph with no entity descriptions -> skip.+ // structure entity with no description -> skip.{DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,- Content: `{"entities":[]}`, Meta: map[string]any{"kind": "graph"}},+ Content: `{"name":"A","description":""}`, Meta: map[string]any{"kind": "entity"}},}got := navInputFromProducts("kb1", products)if len(got) != 0 {@@ -95,7 +112,7 @@ func TestNavInputFromProducts_EmptySummaryDropped(t *testing.T) {func TestProductsForVariants_GatesDispatch(t *testing.T) {products := []kccommon.Product{{DocID: "d1", Variant: kccommon.VariantTree, Meta: map[string]any{"kind": "root"}, Content: "tree summary"},- {DocID: "d1", Variant: kccommon.VariantStructure, Meta: map[string]any{"kind": "graph"}, Content: `{"entities":[]}`},+ {DocID: "d1", Variant: kccommon.VariantStructure, Meta: map[string]any{"kind": "entity"}, Content: `{"name":"A","description":"d"}`},{DocID: "d1", Variant: kccommon.VariantWiki, Meta: map[string]any{"kind": "page"}, Content: "wiki page"},}// A wiki-only event must drop the tree/structure products.@@ -114,14 +131,19 @@ func TestProductsForVariants_GatesDispatch(t *testing.T) {}}-// TestPageIndexSummary_ConcatenatesAndSkipsEmpty verifies pageIndexSummary folds-// entity descriptions and skips empty ones.-func TestPageIndexSummary_ConcatenatesAndSkipsEmpty(t *testing.T) {- got := pageIndexSummary(`{"entities":[{"name":"A","description":"one thing"},{"name":"B","description":""}]}`)- if !strings.Contains(got, "A: one thing") {- t.Errorf("summary = %q, want entity A", got)- }- if strings.Contains(got, "B") {- t.Errorf("summary should skip empty-description entity, got %q", got)+// TestStructureEntityNavLine verifies the per-row fold: whitespace-normalized+// description, "name: desc" shape, empty description -> empty line.+func TestStructureEntityNavLine(t *testing.T) {+ if got := structureEntityNavLine(`{"name":"A","description":"one thing"}`); got != "A: one thing" {+ t.Errorf("line = %q, want %q", got, "A: one thing")+ }+ if got := structureEntityNavLine(`{"name":"B","description":""}`); got != "" {+ t.Errorf("empty description should yield empty line, got %q", got)+ }+ if got := structureEntityNavLine(`{"description":"no name"}`); got != "no name" {+ t.Errorf("nameless entity line = %q, want %q", got, "no name")+ }+ if got := structureEntityNavLine(`not json`); got != "" {+ t.Errorf("malformed payload should yield empty line, got %q", got)}}diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.goindex 9a0dcaca1c..d6734e54b9 100644--- a/internal/ingestion/task/knowledge_compiler_wiring.go+++ b/internal/ingestion/task/knowledge_compiler_wiring.go@@ -578,7 +578,12 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatR// Python's knowledge compilation pins per-call-site temperatures// (extraction 0.1, merge judging 0.0); nil leaves the driver default.var config *models.ChatConfig- if req.Temperature != nil || req.MaxTokens != nil || c.maxTokens > 0 {+ // Union of both sides: per-call overrides (temperature / max tokens), this+ // branch's thinking toggle (claim extraction and raptor merge ask the+ // reasoning model to answer WITHOUT chain-of-thought), and upstream's+ // invoker-level generation cap, which the MaxTokens else-if below applies+ // when the call site itself sets none.+ if req.Temperature != nil || req.MaxTokens != nil || req.DisableThinking || c.maxTokens > 0 {config = &models.ChatConfig{}if req.Temperature != nil {config.Temperature = req.Temperature@@ -591,6 +596,14 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatR} else if c.maxTokens > 0 {config.MaxTokens = &c.maxTokens}+ // Reasoning models (MiniMax-M1/M3, kimi, qwen) spend the completion+ // budget on visible COT before the structured reply; compilation should+ // run with thinking off. MiniMaxModel maps Thinking=false to+ // `thinking: {"type": "disabled"}` in the request body.+ if req.DisableThinking {+ off := false+ config.Thinking = &off+ }}// Retry transient transport/provider failures (HTTP timeout, reset,// connection refused, 5xx, 429) with exponential backoff. A singlediff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex 46e3689622..de5ae141bc 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -437,20 +437,26 @@ func countOriginalChunkIDs(chunks []map[string]any) int {return len(seen)}-// markCompiledProductsHidden sets available_int=0 on the per-document compiled-// knowledge products so they are hidden from the retriever until the dataset-level-// post-processing consumer merges them into available_int=1 products (§11). A-// chunk is a compiled product iff it carries the compile_kwd discriminator the-// KnowledgeCompiler component stamps; ordinary source chunks (no compile_kwd)-// keep the index default available_int=1 and remain immediately searchable.-// Merged dataset-level products are written by the consumer, never here, so they are-// never double-marked.+// markCompiledProductsHidden stages wiki per-document page rows as+// available_int=0. Only the wiki variant is hidden: its per-document page rows+// are intermediate state for the consumer's merged pages (available_int=1) and+// every wiki read path selects merged rows only.+//+// Tree / structure (page_index) / mindmap rows are the FINAL per-document+// products — Python writes them visible at compile time+// (_struct_to_doc_storage_doc sets no available_int, index default 1) and its+// claim recall / hybrid retrieval filter on available_int=1, so they stay+// searchable here too. Merged dataset-level products are written by the+// consumer, never here, so they are never double-marked.func markCompiledProductsHidden(chunks []map[string]any) {for _, ck := range chunks {- if _, ok := ck["compile_kwd"]; !ok {+ kwd := asCompiledKwd(ck)+ if kwd == "" {continue}- ck["available_int"] = 0+ if v, err := knowledge_compile.KwdToVariant(kwd); err == nil && v == kccommon.VariantWiki {+ ck["available_int"] = 0+ }}}@@ -536,18 +542,19 @@ func (s *PipelineExecutor) reconcileDocumentCompiledProducts(ctx context.Contextreturn nil}-// applyDocumentAvailability stamps ordinary source chunks with available_int=0-// when Document.status is "0" (disabled). Disabling before any chunks exist only-// updates MySQL; without this, later parsing would still write searchable-// available_int=1 rows. Compiled products (compile_kwd) stay at 0 regardless.+// applyDocumentAvailability stamps every chunk — ordinary source chunks and+// compiled products alike — with available_int=0 when Document.status is "0"+// (disabled). Disabling before any chunks exist only updates MySQL; without+// this, later parsing would still write searchable available_int=1 rows.+// Wiki staging rows are already available_int=0 from markCompiledProductsHidden,+// so the stamp is a no-op for them; tree / structure rows become hidden+// together with their document, matching Python's doc_id-scoped availability+// toggle.func applyDocumentAvailability(chunks []map[string]any, status *string) {if status == nil || *status != "0" {return}for _, ck := range chunks {- if _, ok := ck["compile_kwd"]; ok {- continue- }ck["available_int"] = 0}}diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.goindex 19ea8a1d5a..80e9a2fcf5 100644--- a/internal/ingestion/task/pipeline_executor_test.go+++ b/internal/ingestion/task/pipeline_executor_test.go@@ -27,16 +27,19 @@ import (func strPtr(s string) *string { return &s }-// TestMarkCompiledProductsHidden verifies the pipeline caller hides-// per-document compiled knowledge products (compile_kwd present) as-// available_int=0 while leaving ordinary source chunks searchable-// (available_int=1, the index default). Merged dataset-level products are written-// by the consumer and never reach this path, so they are never double-marked.+// TestMarkCompiledProductsHidden verifies the pipeline caller stages ONLY the+// wiki variant's per-document rows as available_int=0 (they are intermediate+// state for the consumer's merged pages). Tree / structure (page_index) /+// mindmap rows are the final per-document products — Python writes them visible+// at compile time — so they keep the index default available_int=1, as do+// ordinary source chunks.func TestMarkCompiledProductsHidden(t *testing.T) {chunks := []map[string]any{{"id": "src-1", "content_with_weight": "ordinary source chunk"},- {"id": "struct-1", "compile_kwd": "structure", "content_with_weight": "entity A"},+ {"id": "tree-1", "compile_kwd": "tree", "content_with_weight": "tree node"},+ {"id": "struct-1", "compile_kwd": "page_index", "content_with_weight": "entity A"},{"id": "wiki-1", "compile_kwd": "wiki_page", "content_with_weight": "page X"},+ {"id": "wiki-2", "compile_kwd": "wiki_section", "content_with_weight": "section X"},{"id": "src-2", "content_with_weight": "another source chunk"},}markCompiledProductsHidden(chunks)@@ -44,13 +47,19 @@ func TestMarkCompiledProductsHidden(t *testing.T) {if v, ok := chunks[0]["available_int"]; ok {t.Fatalf("ordinary source chunk should keep default available_int, got %v", v)}- if chunks[1]["available_int"] != 0 {- t.Fatalf("compiled structure chunk should be available_int=0, got %v", chunks[1]["available_int"])+ if v, ok := chunks[1]["available_int"]; ok {+ t.Fatalf("tree row should stay visible (Python parity), got available_int=%v", v)}- if chunks[2]["available_int"] != 0 {- t.Fatalf("compiled wiki chunk should be available_int=0, got %v", chunks[2]["available_int"])+ if v, ok := chunks[2]["available_int"]; ok {+ t.Fatalf("page_index row should stay visible (Python parity), got available_int=%v", v)+ }+ if chunks[3]["available_int"] != 0 {+ t.Fatalf("wiki page staging row should be available_int=0, got %v", chunks[3]["available_int"])}- if v, ok := chunks[3]["available_int"]; ok {+ if chunks[4]["available_int"] != 0 {+ t.Fatalf("wiki section staging row should be available_int=0, got %v", chunks[4]["available_int"])+ }+ if v, ok := chunks[5]["available_int"]; ok {t.Fatalf("source chunk without compile_kwd should keep default available_int, got %v", v)}}@@ -70,11 +79,14 @@ func TestWikiActiveStatesDecodeCheckpointValues(t *testing.T) {}// TestApplyDocumentAvailability verifies disabled documents (status=0) force-// ordinary source chunks to available_int=0 while compiled products stay hidden.+// every row — source chunks AND compiled products — to available_int=0+// (matching Python's doc_id-scoped availability toggle). Wiki staging rows are+// already 0 from markCompiledProductsHidden; the stamp is a no-op for them.func TestApplyDocumentAvailability(t *testing.T) {chunks := []map[string]any{{"id": "src-1", "content_with_weight": "ordinary source chunk"},- {"id": "struct-1", "compile_kwd": "structure", "content_with_weight": "entity A", "available_int": 0},+ {"id": "tree-1", "compile_kwd": "tree", "content_with_weight": "tree node"},+ {"id": "wiki-1", "compile_kwd": "wiki_page", "content_with_weight": "page X", "available_int": 0},{"id": "src-2", "content_with_weight": "another source chunk"},}markCompiledProductsHidden(chunks)@@ -84,10 +96,13 @@ func TestApplyDocumentAvailability(t *testing.T) {t.Fatalf("disabled doc source chunk should be available_int=0, got %v", chunks[0]["available_int"])}if chunks[1]["available_int"] != 0 {- t.Fatalf("compiled product should stay available_int=0, got %v", chunks[1]["available_int"])+ t.Fatalf("disabled doc compiled row should be available_int=0, got %v", chunks[1]["available_int"])}if chunks[2]["available_int"] != 0 {- t.Fatalf("disabled doc source chunk should be available_int=0, got %v", chunks[2]["available_int"])+ t.Fatalf("wiki staging row should stay available_int=0, got %v", chunks[2]["available_int"])+ }+ if chunks[3]["available_int"] != 0 {+ t.Fatalf("disabled doc source chunk should be available_int=0, got %v", chunks[3]["available_int"])}enabled := []map[string]any{diff --git a/internal/rag/advanced_rag/harness/tool_navigation.go b/internal/rag/advanced_rag/harness/tool_navigation.goindex 3303aac283..0dbae84d48 100644--- a/internal/rag/advanced_rag/harness/tool_navigation.go+++ b/internal/rag/advanced_rag/harness/tool_navigation.go@@ -217,9 +217,6 @@ func parseFloats(v any) ([]float64, bool) {}func normalizeKind(row map[string]interface{}) string {- if ck, _ := row["compile_kwd"].(string); ck == "raptor_graph" {- return "raptor"- }kind, _ := row["compilation_template_kind_kwd"].(string)if kind == "" {kind, _ = row["compile_kwd"].(string)@@ -529,30 +526,31 @@ func NavigateTree(ctx context.Context, router NavTreeRouter, in NavTreeInput) Na// Python reads (navigation.py:_load_compiled_structure).type StructureRow struct {// CompileKwd distinguishes the COMPILE TYPE (tree / page_index / timeline /- // raptor_graph / ...). NOT knowledge_graph_kwd.+ // ...). NOT knowledge_graph_kwd.CompileKwd string// TemplateKind is the template-authored kind (compilation_template_kind_kwd).TemplateKind string// KnowledgeGraphKwd selects the ROW SHAPE:- // "graph" → one compact blob whose content is {"entities":[], "relations":[]}- // (written by RAPTOR / tree compilation)// "entity" → one row per node, content is a single entity dict// "relation" → one row per edge, content is a single relation dict- // (written by page_index and the pipeline Compiler tree)+ // (written by page_index, tree, and the pipeline Compiler)+ // "graph" → legacy: one compact blob {"entities":[], "relations":[]};+ // pre-migration tree rows only, nothing writes it anymoreKnowledgeGraphKwd string// Content is the row's JSON payload.Content string// Vec is the row's q_<dim>_vec embedding, when the reader selected it. A- // graph blob row carries the one vector its nested nodes share (RAPTOR); a- // per-entity row carries that node's own vector (page_index).+ // graph blob row carries the one vector its nested nodes share (legacy); a+ // per-entity row carries that node's own vector.Vec []float64}// StructureReader reads a document's compiled structure rows.//-// Mirrors Python _load_compiled_structure, which issues-// three doc-store queries (graph blob, per-entity/relation rows, raptor_graph)-// and merges the matching buckets.+// Mirrors Python _load_compiled_structure, which issues one doc-store query+// over the per-entity/relation rows and merges the matching buckets (the+// graph blob and the raptor_graph projection are gone from the storage+// model).//// Go has no doc-store structured-query interface (see runtime.RetrievalService,// which only exposes Search), so this is a seam: the harness ships the full@@ -568,7 +566,7 @@ type StructureReader interface {// normalizeKind is defined above; rowKind projects a StructureRow into that// shape. It mirrors Python _normalize_kind: the API's kind normalization-// (page_index / knowledge_graph → timeline), plus the raptor_graph override.+// (page_index / knowledge_graph → timeline).func rowKind(row StructureRow) string {return normalizeKind(map[string]any{"compile_kwd": row.CompileKwd,diff --git a/internal/rag/advanced_rag/harness/tool_navigation_test.go b/internal/rag/advanced_rag/harness/tool_navigation_test.goindex 3fb7bdb4b0..a6c9374092 100644--- a/internal/rag/advanced_rag/harness/tool_navigation_test.go+++ b/internal/rag/advanced_rag/harness/tool_navigation_test.go@@ -33,7 +33,6 @@ func TestNormalizeKind(t *testing.T) {row map[string]interface{}want string}{- {map[string]interface{}{"compile_kwd": "raptor_graph"}, "raptor"},{map[string]interface{}{"compilation_template_kind_kwd": "page_index"}, "timeline"},{map[string]interface{}{"compilation_template_kind_kwd": "knowledge_graph"}, "timeline"},{map[string]interface{}{"compilation_template_kind_kwd": "mindmap"}, "mindmap"},@@ -337,16 +336,11 @@ func TestParseCompiledStructureKindFilterAndNormalization(t *testing.T) {{CompileKwd: "page_index", KnowledgeGraphKwd: "entity", Content: `{"name": "A"}`},{CompileKwd: "knowledge_graph", KnowledgeGraphKwd: "entity", Content: `{"name": "B"}`},{CompileKwd: "tree", KnowledgeGraphKwd: "entity", Content: `{"name": "C"}`},- {CompileKwd: "raptor_graph", KnowledgeGraphKwd: "graph", Content: `{"entities": [{"name": "D"}]}`},}ents, _ := ParseCompiledStructure(rows, []string{"timeline"})if len(ents) != 2 {t.Errorf("timeline filter = %d entities, want 2 (page_index + knowledge_graph)", len(ents))}- ents, _ = ParseCompiledStructure(rows, []string{"raptor"})- if len(ents) != 1 {- t.Errorf("raptor filter = %d entities, want 1", len(ents))- }ents, _ = ParseCompiledStructure([]StructureRow{{CompileKwd: "", TemplateKind: "tree", KnowledgeGraphKwd: "entity", Content: `{"name": "E"}`},}, []string{"tree"})diff --git a/internal/router/router.go b/internal/router/router.goindex 3a7824f228..c56c9c9e6f 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -420,6 +420,8 @@ func (r *Router) Setup(engine *gin.Engine) {datasets.DELETE("/:dataset_id/documents/:document_id/chunks", r.chunkHandler.RemoveChunks)datasets.PUT("/:dataset_id/documents/:document_id/metadata/config", r.datasetsHandler.UpdateDocumentMetadataConfig)datasets.GET("/:dataset_id/documents/:document_id/structure/graph", r.datasetArtifactHandler.GetDocumentGraph)+ datasets.GET("/:dataset_id/documents/:document_id/structure/claims", r.datasetArtifactHandler.GetDocumentClaims)+ datasets.DELETE("/:dataset_id/documents/:document_id/structure/graph", r.datasetArtifactHandler.DeleteDocumentGraph)datasets.POST("/:dataset_id/metadata/update", r.documentHandler.MetadataBatchUpdate)datasets.PATCH("/:dataset_id/documents/metadatas", r.documentHandler.UpdateDocumentMetadatas)}diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.goindex 2814b6a4c3..0f8506a9f0 100644--- a/internal/service/dataset_artifact_service.go+++ b/internal/service/dataset_artifact_service.go@@ -42,7 +42,6 @@ const (CompileKwdSkill = "skill"CompileKwdSkillAll = "skill_all"CompileKwdDatasetNav = "dataset_nav"- CompileKwdRaptorGraph = "raptor_graph")// DatasetArtifactService reads knowledge-compilation artifacts (wiki pages,@@ -837,6 +836,36 @@ func sortedSetKeys(set map[string]struct{}) []string {return ids}+// DeleteDocumentGraph deletes the structure graph of a single document.+func (s *DatasetArtifactService) DeleteDocumentGraph(ctx context.Context, tenantID, datasetID, documentID string) (int, error) {+ docEngine := engine.Get()+ if docEngine == nil {+ return 0, fmt.Errorf("document engine is not initialized")+ }+ filter := map[string]interface{}{+ "doc_id": []string{documentID},+ "compiled_graph_kwd": []string{"graph"},+ }+ chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 10000, nil)+ if err != nil {+ return 0, err+ }+ if len(chunks) == 0 {+ return 0, nil+ }+ ids := make([]string, 0, len(chunks))+ for _, c := range chunks {+ if id, ok := c["id"].(string); ok {+ ids = append(ids, id)+ }+ }+ cond := map[string]interface{}{"id": ids, "kb_id": datasetID}+ if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {+ return 0, err+ }+ return len(ids), nil+}+// ListNavClusters returns the navigation clusters of a dataset. It delegates to// the ES-backed NavService and returns the frontend DatasetNavNode shape// (snake_case NavNode JSON), matching Python GET /navigation exactly. The old@@ -1017,3 +1046,154 @@ func toStringSlice(v interface{}) []string {}return []string{}}++// claimRowTypes lists the entity_type_kwd values that carry a claim.+// “claim“ is what every compiler writes now (page_index folded its+// “fact“/“conclusion“ types into it); the older two spellings stay listed+// so rows compiled before the rename keep showing up until a recompile.+var claimRowTypes = []string{"claim", "fact", "conclusion"}++// ListDocumentStructureClaims pages one document's claim/evidence rows+// (entity_type_kwd="claim"), mirroring Python+// structure_graph_common.list_claim_rows + chunk_api.get_document_structure_claims.+//+// “chunkIDs“ narrows the page to claims sourced from those chunks — the tree+// UI passes a leaf cluster's members when the user expands it; without it the+// endpoint pages the whole document. “templateID“ scopes by+// compilation_template_ids; unlike Python we deliberately do NOT narrow by+// compile_kwd on top of it: rows of one template always share one stamped+// compile_kwd, so the template id alone scopes identically (Python's own+// lookup there is documented as best-effort).+func (s *DatasetArtifactService) ListDocumentStructureClaims(ctx context.Context, tenantID, datasetID, documentID, templateID string, chunkIDs []string, offset, limit int) ([]map[string]interface{}, int64, error) {+ if limit < 1 {+ limit = 1+ }+ if offset < 0 {+ offset = 0+ }+ filter := map[string]interface{}{+ "doc_id": []string{documentID},+ "entity_type_kwd": claimRowTypes,+ }+ if templateID != "" {+ filter["compilation_template_ids"] = []string{templateID}+ }+ if len(chunkIDs) > 0 {+ deduped := appendUniqueSorted(chunkIDs)+ if len(deduped) > 0 {+ filter["source_chunk_ids"] = deduped+ }+ }++ rows, total, err := graphRowSearch(ctx, tenantID, datasetID,+ []string{"content_with_weight", "source_chunk_ids"}, filter, nil, offset, limit, nil)+ if err != nil {+ return nil, 0, err+ }++ claims := make([]map[string]interface{}, 0, len(rows))+ for _, row := range rows {+ if claim := projectClaimRow(row); claim != nil {+ claims = append(claims, claim)+ }+ }+ return claims, total, nil+}++// projectClaimRow projects a raw claim row to the API shape, keeping its+// verbatim evidence. Unlike graph entities, the quote and its location ARE the+// payload — that is what a compiled claim exists to carry (Python+// structure_graph_common.project_claim).+func projectClaimRow(row map[string]interface{}) map[string]interface{} {+ payload := graphLoadPayload(row)+ if payload == nil {+ return nil+ }+ name := strings.TrimSpace(graphStr(payload["name"]))+ if name == "" {+ return nil+ }++ chunkIDs := appendUniqueSorted(stringSliceValue(row["source_chunk_ids"], payload["source_chunk_ids"]))+ claim := map[string]interface{}{+ "name": name,+ "description": strings.TrimSpace(graphStr(payload["description"])),+ "source_chunk_ids": chunkIDs,+ }+ if claim["description"] == "" {+ claim["description"] = name+ }+ if typ := strings.TrimSpace(graphStr(payload["type"])); typ != "" {+ claim["type"] = typ+ }+ if evidence := keptEvidence(payload["evidence"]); len(evidence) > 0 {+ claim["evidence"] = evidence+ }+ return claim+}++// stringSliceValue coerces the row column first, falling back to the payload's+// own source_chunk_ids (Python: row.source_chunk_ids or payload.source_chunk_ids).+func stringSliceValue(values ...interface{}) []string {+ for _, v := range values {+ switch x := v.(type) {+ case []string:+ return x+ case []interface{}:+ out := make([]string, 0, len(x))+ for _, item := range x {+ if s, ok := item.(string); ok && strings.TrimSpace(s) != "" {+ out = append(out, strings.TrimSpace(s))+ }+ }+ if len(out) > 0 {+ return out+ }+ case string:+ if strings.TrimSpace(x) != "" {+ return []string{x}+ }+ }+ }+ return nil+}++// keptEvidence drops evidence entries without a verbatim quote.+func keptEvidence(v interface{}) []map[string]interface{} {+ items, ok := v.([]interface{})+ if !ok {+ return nil+ }+ kept := make([]map[string]interface{}, 0, len(items))+ for _, item := range items {+ m, ok := item.(map[string]interface{})+ if !ok {+ continue+ }+ if strings.TrimSpace(graphStr(m["quote"])) == "" {+ continue+ }+ kept = append(kept, m)+ }+ return kept+}++// appendUniqueSorted deduplicates and sorts chunk ids (Python:+// sorted({...}) over the requested chunk_ids; stable output for tests).+func appendUniqueSorted(values []string) []string {+ seen := make(map[string]struct{}, len(values))+ out := make([]string, 0, len(values))+ for _, v := range values {+ v = strings.TrimSpace(v)+ if v == "" {+ continue+ }+ if _, dup := seen[v]; dup {+ continue+ }+ seen[v] = struct{}{}+ out = append(out, v)+ }+ sort.Strings(out)+ return out+}diff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.goindex 96a20d6c96..4e1b1f0a7c 100644--- a/internal/service/dataset_structure_graph.go+++ b/internal/service/dataset_structure_graph.go@@ -848,9 +848,6 @@ func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in Docume}}- // RAPTOR summary graph blob (compile_kwd = raptor_graph).- s.appendRaptorBlob(ctx, in.TenantID, in.DatasetID, in.DocumentID, grouped)-// Order: configured templates first, then discovered.orderedIDs := []string{}for _, tid := range configuredIDs {@@ -1083,23 +1080,27 @@ func resolveGraphBucket(row map[string]interface{}, templateMeta map[string]map[if bucketKind == "" {bucketKind = kindVal}- return map[string]interface{}{+ bucket := map[string]interface{}{"template_id": tid,"template_name": bucketName,"kind": bucketKind,- }, graphBucketScope(documentID, map[string]interface{}{+ }+ scope := graphBucketScope(documentID, map[string]interface{}{"compilation_template_ids": []string{tid},})+ return bucket, scope}bucketID := "legacy:" + compileKwd- return map[string]interface{}{+ legacyBucket := map[string]interface{}{"template_id": bucketID,"template_name": "Legacy (" + compileKwd + ")","kind": kindVal,- }, graphBucketScope(documentID, map[string]interface{}{+ }+ legacyScope := graphBucketScope(documentID, map[string]interface{}{"compile_kwd": []string{compileKwd},"must_not": map[string]interface{}{"exists": "compilation_template_ids"},})+ return legacyBucket, legacyScope}func graphBucketScope(documentID string, scope map[string]interface{}) map[string]interface{} {@@ -1133,33 +1134,6 @@ func rowTemplateID(row map[string]interface{}) string {return ""}-func (s *DatasetArtifactService) appendRaptorBlob(ctx context.Context, tenantID, datasetID, documentID string, grouped map[string]DocumentStructureGraphTemplate) {- rows, _, err := graphRowSearch(ctx, tenantID, datasetID, []string{"id", "content_with_weight", "compile_kwd"},- map[string]interface{}{"doc_id": []string{documentID}, "compile_kwd": []string{"raptor_graph"}}, nil, 0, 16, nil)- if err != nil {- return- }- for _, row := range rows {- payload := graphLoadPayload(row)- if payload == nil {- continue- }- rEntities, _ := payload["entities"].([]interface{})- rRelations, _ := payload["relations"].([]interface{})- if len(rEntities) == 0 && len(rRelations) == 0 {- continue- }- rb, ok := grouped["raptor"]- if !ok {- rb = DocumentStructureGraphTemplate{TemplateID: "raptor", TemplateName: "RAPTOR Summary", Kind: "raptor"}- grouped["raptor"] = rb- }- rb.Entities = append(rb.Entities, toNodeSlice(rEntities)...)- rb.Relations = append(rb.Relations, toRelationSlice(rRelations)...)- grouped["raptor"] = rb- }-}-func toNodeSlice(in []interface{}) []StructureGraphNode {var out []StructureGraphNodefor _, e := range in {diff --git a/rag/advanced_rag/agentic_rag_graph.py b/rag/advanced_rag/agentic_rag_graph.pyindex ac26307041..5f939e3227 100644--- a/rag/advanced_rag/agentic_rag_graph.py+++ b/rag/advanced_rag/agentic_rag_graph.py@@ -142,7 +142,18 @@ def _select_sca_view(chunks: list, focus_terms: list[str], cap: int | None = Nonreturn rel * 0.45 + min(cov_ratio, 1.0) * 0.45 + freshranked = sorted(list(enumerate(chunks)), key=lambda ic: _score(*ic), reverse=True) # noqa: C414- view = [c for _, c in ranked[:capped]]++ # Evidence rows first: an atomic proposition carrying a verbatim quote is+ # what the reviewer should read before wading through raw passages. Both+ # groups keep their relevance order; only the grouping is lifted.+ def _is_evidence(c: dict) -> bool:+ return str(_chunk_id(c) or "").startswith(_EVIDENCE_CHUNK_PREFIX)++ evidence = [c for _, c in ranked if _is_evidence(c)]+ rest = [c for _, c in ranked if not _is_evidence(c)]+ view = (evidence + rest)[:capped]+ # identity is a hash of the SORTED id set, so this reordering cannot break+ # the unproductive-round detector.ident = "|".join(sorted((_chunk_id(c) or "") for c in view))return view, str(hash(ident))@@ -466,11 +477,24 @@ async def _expand_fanouts(tools, question: str, answer_conf: dict) -> list[str]:if mdl is None or not question:return [question] if question else []try:- ans, _ = await mdl.async_chat(+ ans, usage = await mdl.async_chat(_FANOUT_PROMPT,[{"role": "user", "content": f"Question: {question}"}],dict(answer_conf or {}),)+ # ``_base_chat_mdl`` deliberately bypasses CountingChatModel (it needs the+ # innermost Base for decorator-style bind_tools), so this call's usage+ # never reaches the phase table and the planner looks free of charge.+ # Book it explicitly — otherwise every token measurement is short.+ # ``async_chat`` returns (text, 0), not a usage mapping: read the+ # model's own ``last_usage`` counter instead.+ try:+ from rag.advanced_rag.harness.stats import record_external_call++ _u = usage if isinstance(usage, dict) else getattr(mdl, "last_usage", None)+ record_external_call(_u if isinstance(_u, dict) else None)+ except Exception: # noqa: BLE001+ passfanouts = _parse_fanouts(str(ans or ""))if not fanouts:# The model answered the question instead of decomposing it (no JSON, or@@ -482,6 +506,15 @@ async def _expand_fanouts(tools, question: str, answer_conf: dict) -> list[str]:[{"role": "user", "content": f"Question: {question}"}],dict(answer_conf or {}),)+ # The retry bypasses the counting proxy too — book it as well,+ # or a retried planner still reports zero tokens.+ try:+ from rag.advanced_rag.harness.stats import record_external_call++ _u2 = getattr(mdl, "last_usage", None)+ record_external_call(_u2 if isinstance(_u2, dict) else None)+ except Exception: # noqa: BLE001+ passfanouts = _parse_fanouts(str(ans2 or ""))except Exception: # noqa: BLE001_LOG.warning("[Planner] strict fan-out retry failed", exc_info=True)@@ -500,6 +533,31 @@ async def _expand_fanouts(tools, question: str, answer_conf: dict) -> list[str]:# 30-chunk ceiling starved every enrichment channel dead (observed: Drill# admitted 0 for entire runs because prefetch filled the pool first)._MAX_SNIPPET_POOL = 60+# id prefix of the pool entries synthesized from compiled evidence rows.+_EVIDENCE_CHUNK_PREFIX = "claim_"+# Ceiling for evidence rows (claim / fact-conclusion) inside the snippet pool.+# Raised 12 -> 24: a claim is an atomic proposition with a verbatim quote+# (~1.2k chars, ~300 tokens) while a raw chunk is far larger, so at equal pool+# capacity evidence rows carry several times the answer material per token.+# They are admitted FIRST because they are answer material; the raw channels now+# get their own budget below so they cannot push them out.+_EVIDENCE_POOL_QUOTA = 24+# Separate ceiling for RAW passages. Storage and REVIEW are decoupled (the SCA+# reads a ranked _SCA_VIEW_CAP view), but the pool's own composition decides how+# much raw text every later prompt carries, so raw gets a tighter bound than the+# pool as a whole. Room left above it is reserved for evidence rows.+_RAW_SNIPPET_QUOTA = 30+# How many of the chunks cited by admitted evidence rows to pull in verbatim.+# This is a directed fetch (by id), not another recall.+_EVIDENCE_TOP_UP = 8+# Depth ceiling for gap → slot promotion. Without it an insufficient verdict+# could keep splitting the plan every round and never converge (the same failure+# mode APT-RAG guards with its max_depth).+_MAX_SLOT_DEPTH = 3+# Hard ceiling on the slot table. ``depth`` is never incremented (it describes+# the research tree, not the gap-promotion rounds), so the depth test alone+# cannot stop the table from growing one slot per round — this bound can.+_MAX_SLOTS_TOTAL = 8_DRILL_RESERVE = 12 # slots kept free after the FIRST prefetch so the research# executor can top up evidence_SCA_VIEW_CAP = 60 # chunks shown to the Sufficient Context Agent per review (24 -> 60: 24 of 225 hid the answer-bearing table chunk from the SCA)@@ -536,6 +594,22 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: in_LOG.warning("[rag_agent] could not import fan-out search helpers", exc_info=True)return 0+ # Evidence rows only exist on compiled datasets (tree -> claim, page_index ->+ # fact/conclusion). Gate the channel on the compilation probe: on an+ # uncompiled dataset it would issue retrieval legs that can only come back+ # empty while eating the snippet budget.+ try:+ from rag.advanced_rag.harness.tools.navigation import (+ _evidence_row_types,+ dataset_compilation_kinds,+ recall_dataset_claims,+ )++ row_types = _evidence_row_types(await dataset_compilation_kinds(tools))+ except Exception: # noqa: BLE001+ _LOG.warning("[rag_agent] evidence-row probe unavailable; skipping channel", exc_info=True)+ row_types = ()+kbinfos = getattr(tools, "kbinfos", None)if kbinfos is None:kbinfos = {"chunks": [], "doc_aggs": []}@@ -615,7 +689,51 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: in_LOG.warning("[rag_agent] hybrid channel failed for %r; skipping", fq, exc_info=True)return kept_a, kept_b+ async def _collect_evidence() -> list:+ """Channel 0: compiled evidence rows, folded into pool-shaped entries.++ An evidence row is an atomic proposition plus its verbatim quote — it is+ answer material, not a candidate to be narrowed — so it is admitted+ ahead of both chunk channels.+ """+ import hashlib++ async def _recall_one(fq: str) -> list:+ try:+ return await recall_dataset_claims(tools, fq, top_n=max(2, top_n))+ except Exception: # noqa: BLE001+ _LOG.warning("[rag_agent] evidence recall failed for %r", fq, exc_info=True)+ return []++ # Parallel, like the chunk channels: a serial loop over the fan-outs+ # would add every leg's latency straight onto the prefetch budget.+ gathered = await asyncio.gather(*(_recall_one(fq) for fq in fanouts), return_exceptions=True)++ out: list = []+ for item in gathered:+ if isinstance(item, BaseException) or not item:+ continue+ for c in item:+ name = str(c.get("name") or "")+ content = f"[evidence] {name}"+ desc = str(c.get("description") or "")+ if desc and desc != name:+ content += f" — {desc}"+ if c.get("quote"):+ content += f'\nEvidence (verbatim): "{str(c["quote"])[:400]}"'+ cid = _EVIDENCE_CHUNK_PREFIX + hashlib.md5(f"{c.get('doc_id')}:{name}".encode("utf-8", "ignore")).hexdigest()[:12]+ out.append(+ {+ "chunk_id": cid,+ "content_with_weight": content[:1200],+ "doc_id": str(c.get("doc_id") or ""),+ "source_chunk_ids": list(c.get("chunk_ids") or []),+ }+ )+ return out+# Search all fan-outs at once (parallel) — then merge into the shared pool.+ evidence_chunks = await _collect_evidence() if row_types else []results = await asyncio.gather(*(_search_one(fq) for fq in fanouts), return_exceptions=True)# Cap the TOTAL evidence pool so the SCA never has to read unbounded chunks.@@ -628,8 +746,10 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: inroom = max(0, max_total - len(seen))added = 0+ raw_added = 0+def _admit(batch) -> bool:- nonlocal added+ nonlocal added, raw_addedif isinstance(batch, Exception) or not batch:return Falsestop = False@@ -640,12 +760,43 @@ async def _fanout_search(tools, fanouts: list[str], top_n: int = 8, capacity: ink = _chunk_id(c)if k and k in seen:continue+ is_evidence = bool(k) and str(k).startswith(_EVIDENCE_CHUNK_PREFIX)+ # Raw passages get their own budget; anything left above it stays+ # free for evidence rows, which are far denser answer material.+ if not is_evidence and raw_added >= _RAW_SNIPPET_QUOTA:+ continueif k:seen.add(k)kbinfos.setdefault("chunks", []).append(c)added += 1+ if not is_evidence:+ raw_added += 1return stop+ # Evidence rows first: they are answer material, so they must not be+ # crowded out by the chunk channels.+ if evidence_chunks:+ evidence_chunks = evidence_chunks[:_EVIDENCE_POOL_QUOTA]+ if _admit(evidence_chunks):+ return added+ # Directional top-up (external gathering): an evidence row carries a+ # verbatim quote but not its surrounding passage, so pull exactly the+ # chunks it cites instead of running another global recall.+ wanted: list = []+ for c in evidence_chunks:+ for cid in c.get("source_chunk_ids") or []:+ cid = str(cid or "").strip()+ if cid and cid not in seen and cid not in wanted:+ wanted.append(cid)+ if wanted:+ try:+ from rag.advanced_rag.harness.tools.navigation import _load_chunks_for_ids++ fetched = await _load_chunks_for_ids(tools, wanted[:_EVIDENCE_TOP_UP])+ if fetched and _admit(fetched):+ return added+ except Exception: # noqa: BLE001+ _LOG.warning("[rag_agent] evidence top-up failed; continuing", exc_info=True)# Channel A first (exact matches earn their slots), then semantic extras.for pair in results:if isinstance(pair, Exception):@@ -1189,6 +1340,40 @@ def build_agentic_graph(# (b) the next slot research pass picks these up via the persisted# slot_table + unresolved_slots — no separate feedback channel needed._LOG.info("[QueryRewriter] insufficient round %d → %d targeted query(s): %s", int(state.get("search_rounds", 0)) + 1, len(queries), queries)++ # DECOMPOSE (APT-RAG adaptive planning): a gap the rewriter only restates+ # as a query stays flat — the same surface gets re-searched. Promoting it+ # to a slot gives the next research pass a typed unknown with its own+ # action session, which is how the plan actually expands. No extra LLM+ # call: the SCA already told us what is missing (missing_fact + hint).+ slot_table = state.get("slot_table")+ if gaps and slot_table is not None and int(getattr(slot_table, "depth", 0) or 0) < _MAX_SLOT_DEPTH:+ try:+ from rag.advanced_rag.harness.action_session import Variable++ slots = list(getattr(slot_table, "state", None) or [])+ known = {str(c).strip().lower() for v in slots for c in (getattr(v, "question_clues", None) or [])}+ next_id = max([int(getattr(v, "id", 0) or 0) for v in slots] or [0]) + 1+ promoted = 0+ for what, hint in gaps:+ if len(slots) >= _MAX_SLOTS_TOTAL:+ break+ key = str(what or "").strip().lower()+ if not key or key in known:+ continue+ clues = [str(what)[:200]]+ if hint:+ clues.append(str(hint)[:200])+ slots.append(Variable(id=next_id, type="entity", question_clues=clues))+ known.add(key)+ next_id += 1+ promoted += 1+ if promoted:+ slot_table.state = slots+ _LOG.info("[QueryRewriter] decompose: %d gap(s) promoted to slots (depth=%s)", promoted, getattr(slot_table, "depth", 0))+ except Exception: # noqa: BLE001+ _LOG.warning("[QueryRewriter] decompose failed; continuing", exc_info=True)+ledger = [e for e in _safe_list(state.get("attempted"), "state.attempted") if isinstance(e, dict)]for q in queries:ledger.append({"q": q, "r": int(state.get("search_rounds", 0)) + 1, "new": int(added or 0)})@@ -1198,6 +1383,8 @@ def build_agentic_graph("search_rounds": int(state.get("search_rounds", 0)) + 1,"attempted": ledger,"kbinfos": tools.kbinfos,+ # Carry the promoted slots so the next research pass sees them.+ **({"slot_table": slot_table} if slot_table is not None else {}),}# ── Node: formalize_answer (Phase 5 Synthesis) ──@@ -1219,9 +1406,9 @@ def build_agentic_graph(return "formalize_answer"# Evidence pool saturated at the SCA view cap: any further chunk lands# beyond what the SCA can read, so another search round cannot flip the- # sufficiency verdict. Short-circuit straight to finalize.+ # sufficiency verdict. Gated to SECOND-or-later reviews on THIS branch: claim pseudo-chunks bypass the pool cap (direct append in _claim_prefetch), so a rewrite round can still add evidence — the FIRST SCA review must always get its rewrite round when insufficient, or hard multi-hop questions lose their only refinement pass and the retry moves to the outer agent as a whole new graph run (observed: research 30→43, wall +47%, R2 9→1)._pool = tools.kbinfos.get("chunks", [])- if len(_pool) >= _SCA_VIEW_CAP:+ if len(_pool) >= _SCA_VIEW_CAP and int(state.get("search_rounds", 0)) >= 1:_LOG.info("[SCA] evidence pool FULL (%d chunks >= SCA view cap %d); early-stopping to finalize_answer.",len(_pool),@@ -1373,6 +1560,195 @@ async def _build_slot_table(tools, question: str, fanouts: list, answer_conf: direturn root, (first_queries or [question])+# ── Evidence-guided batched answer generation (APT-RAG) ──+# Slots whose sessions retrieved largely the same passages are answered in ONE+# call with the deduplicated union of that evidence. The threshold is+# deliberately conservative: APT-RAG ships sim_threshold=0.0 (any overlap),+# which merges weakly related slots and, worse, lets a single parse failure+# blank every answer in the batch at once.+# Both conditions must hold. The absolute floor stops two tiny evidence sets+# from merging on a single coincidental hit; the ratio gate stays LOW enough to+# actually fire — measured FRAMES overlap was 0.03-0.2 mean, so the old 0.5+# never triggered at all. (APT-RAG ships 0.0; that is too loose for us because+# one parse failure would blank every answer in the batch.)+_EVIDENCE_BATCH_MIN_SIM = 0.15+_EVIDENCE_BATCH_MIN_SHARED = 2+_EVIDENCE_BATCH_MAX_SLOTS = 4+_EVIDENCE_BATCH_MAX_CHARS = 12000++_EVIDENCE_BATCH_PROMPT = (+ "Answer each listed sub-question using ONLY the shared evidence below.\n"+ "These sub-questions share this evidence, so read it once and answer all of them.\n"+ "Return JSON only, mapping each sub-question id to its answer: "+ '{"<id>": "<answer>", ...}. Use an empty string when the evidence does not '+ "answer that sub-question. Never invent facts."+)+++async def _batch_fill_slots(tools, slot_table, slot_evidence: dict, answer_conf: dict) -> int:+ """Answer several unresolved slots in ONE call when their evidence overlaps.++ Pure efficiency: neither the slot structure nor the evidence semantics+ change — only the number of generation calls made over the same passages.+ """+ from rag.advanced_rag.harness.tools.search import _base_chat_mdl, _chunk_id++ slots = [v for v in (getattr(slot_table, "state", None) or []) if not v.filled()]+ ev: dict = {}+ for v in slots:+ ids = {str(i) for i in ((slot_evidence.get(str(v.id)) or {}).get("evidence_ids") or []) if str(i)}+ if ids:+ ev[v.id] = ids+ if len(ev) < 2:+ # Diagnostics: batching needs TWO slots that are both unresolved AND+ # carrying evidence. Log why it did not happen, otherwise a never-+ # firing path is indistinguishable from a working one.+ _LOG.info(+ "[SlotResearch] batching skipped: unresolved=%d with_evidence=%d",+ len(slots),+ len(ev),+ )+ return 0++ ids = list(ev)++ def _sim(a, b) -> float:+ union = ev[a] | ev[b]+ return len(ev[a] & ev[b]) / len(union) if union else 0.0++ # Largest-incompatible-first (APT-RAG): place the least compatible slot+ # first, so it is not left without a cluster at the end.+ order = sorted(ids, key=lambda i: -sum(1 for j in ids if j != i and _sim(i, j) < _EVIDENCE_BATCH_MIN_SIM))+ clusters: list = []+ for i in order:+ for cl in clusters:+ if len(cl) < _EVIDENCE_BATCH_MAX_SLOTS and all(len(ev[i] & ev[j]) >= _EVIDENCE_BATCH_MIN_SHARED and _sim(i, j) >= _EVIDENCE_BATCH_MIN_SIM for j in cl):+ cl.append(i)+ break+ else:+ clusters.append([i])++ kbinfos = getattr(tools, "kbinfos", None) or {}+ by_chunk_id = {}+ for c in kbinfos.get("chunks") or []:+ cid = _chunk_id(c)+ if cid:+ by_chunk_id[cid] = c+ slot_by_id = {v.id: v for v in slots}++ filled = 0+ for cl in clusters:+ if len(cl) < 2:+ continue+ union_ids: set = set()+ for i in cl:+ union_ids |= ev[i]+ body, total = [], 0+ for cid in union_ids:+ c = by_chunk_id.get(cid)+ if not c:+ continue+ text = str(c.get("content_with_weight") or c.get("content") or "").strip()+ if not text:+ continue+ if total + len(text) > _EVIDENCE_BATCH_MAX_CHARS:+ break+ body.append(text)+ total += len(text)+ if not body:+ continue++ lines = []+ for sid in cl:+ v = slot_by_id.get(sid)+ if v is None:+ continue+ clues = "; ".join(str(x) for x in (v.question_clues or []))[:300]+ lines.append(f"- {sid}: {clues}")+ if len(lines) < 2:+ continue++ user = "Sub-questions:\n" + "\n".join(lines) + "\n\nShared evidence:\n" + "\n---\n".join(body)+ try:+ mdl = _base_chat_mdl(tools)+ if mdl is None:+ return filled+ ans, _ = await mdl.async_chat(+ _EVIDENCE_BATCH_PROMPT,+ [{"role": "user", "content": user}],+ dict(answer_conf or {}),+ )+ except Exception: # noqa: BLE001+ _LOG.warning("[SlotResearch] batched answer call failed", exc_info=True)+ continue++ data = _extract_json_object(str(ans or ""))+ if not isinstance(data, dict):+ continue+ for sid in cl:+ v = slot_by_id.get(sid)+ val = data.get(str(sid))+ if v is not None and not v.filled() and isinstance(val, str) and val.strip():+ v.candidate = val.strip()[:400]+ filled += 1+ return filled+++# Word-level coverage a pooled evidence row must reach before it is allowed to+# answer a slot on its own. Deliberately strict: a wrong prefill costs accuracy,+# while a missed prefill only costs one session (which still runs).+_EVIDENCE_PREFILL_COVERAGE = 0.6+++def _prefill_slots_from_evidence(slot_table, kbinfos: dict) -> int:+ """Answer slots that an already-pooled evidence row directly answers.++ An evidence row is an atomic proposition carrying a verbatim quote, so when+ it already covers a slot's question there is nothing for an action session+ to research — skipping it removes a WHOLE session (the dominant cost), not+ just tokens inside one. Saves calls, uses real evidence, and a wrong guess+ is still caught later by the SCA.+ """+ from rag.advanced_rag.harness.tools.search import _chunk_id, _query_to_terms++ chunks = [c for c in ((kbinfos or {}).get("chunks") or []) if isinstance(c, dict)]+ ev_rows = [c for c in chunks if str(_chunk_id(c) or "").startswith(_EVIDENCE_CHUNK_PREFIX)]+ if not ev_rows:+ return 0++ filled = 0+ for v in slot_table.unresolved():+ clues = [str(x) for x in (getattr(v, "question_clues", None) or []) if str(x).strip()]+ if not clues:+ continue+ terms: set = set()+ for c in clues:+ terms |= {t.lower() for t in _query_to_terms(c) if len(t) >= 3}+ if not terms:+ continue++ best, best_cov = None, 0.0+ for e in ev_rows:+ text = str(e.get("content_with_weight") or "").lower()+ if not text:+ continue+ cov = sum(1 for t in terms if t in text) / len(terms)+ if cov > best_cov:+ best, best_cov = e, cov+ if best is None or best_cov < _EVIDENCE_PREFILL_COVERAGE:+ continue++ # The row renders as "[evidence] <name> — <desc>\nEvidence (verbatim): ..."+ head = str(best.get("content_with_weight") or "").split("\n")[0]+ name = head.replace("[evidence]", "").strip(" —-").strip()+ if not name:+ continue+ v.candidate = name[:400]+ v.candidate_strength = float(best_cov)+ filled += 1+ return filled++async def _run_slot_research_pass(tools, question: str, state: AgenticState, answer_conf: dict, deadline_left: float) -> dict:"""Drive ONE research round with slot-aware actionsessions.@@ -1403,6 +1779,16 @@ async def _run_slot_research_pass(tools, question: str, state: AgenticState, ansbase = getattr(tools, "kbinfos", None) or state.get("kbinfos") or {"chunks": [], "doc_aggs": []}tools.kbinfos = dict(base)+ # Evidence-row prefill: slots the pooled evidence already answers cost no+ # action session at all — this is where whole calls get removed.+ try:+ prefill_n = _prefill_slots_from_evidence(slot_table, tools.kbinfos)+ if prefill_n:+ _LOG.info("[SlotResearch] evidence prefill answered %d slot(s) with no session", prefill_n)+ unresolved = slot_table.unresolved()+ except Exception: # noqa: BLE001+ _LOG.warning("[SlotResearch] evidence prefill failed", exc_info=True)+# Shared cache across sessions to avoid duplicate retrievalsshared_tool_cache = {}shared_search_queries = []@@ -1456,6 +1842,32 @@ async def _run_slot_research_pass(tools, question: str, state: AgenticState, ansledger.append({"q": (r.found_answer or str(getattr(r, "messages", [])))[:80] if hasattr(r, "found_answer") else "", "new": 1})+ # Evidence-overlap census (APT-RAG evidence-guided batching). Batching only+ # pays when sibling slots actually share retrieved evidence; this measures it+ # instead of assuming it, and the numbers pick the clustering threshold.+ try:+ import itertools++ ev_sets = [set(m.get("evidence_ids") or []) for m in session_evidence.values()]+ ev_sets = [s for s in ev_sets if s]+ if len(ev_sets) >= 2:+ sims = []+ for a, b in itertools.combinations(ev_sets, 2):+ union = a | b+ if union:+ sims.append(len(a & b) / len(union))+ if sims:+ _LOG.info(+ "[SlotResearch] evidence overlap: slots=%d pairs=%d mean_jaccard=%.3f max=%.3f nonzero_pairs=%d",+ len(ev_sets),+ len(sims),+ sum(sims) / len(sims),+ max(sims),+ sum(1 for s in sims if s > 0),+ )+ except Exception: # noqa: BLE001+ _LOG.warning("[SlotResearch] evidence-overlap census failed", exc_info=True)+# session_evidence is already keyed by slot_id; normalize into a# slot-evidence map the SCA consumer can read. Sessions that produced only# a collected answer (no slot patch) land under the "_answer" key.@@ -1473,6 +1885,15 @@ async def _run_slot_research_pass(tools, question: str, state: AgenticState, ans{k: len(v["evidence_ids"]) for k, v in slot_evidence.items()},)+ # Evidence-guided batching (APT-RAG): slots that retrieved the same passages+ # get answered together instead of one generation call each.+ try:+ batched = await _batch_fill_slots(tools, slot_table, slot_evidence, answer_conf)+ if batched:+ _LOG.info("[SlotResearch] batched generation filled %d slot(s)", batched)+ except Exception: # noqa: BLE001+ _LOG.warning("[SlotResearch] batched generation failed", exc_info=True)+# Expose the updated table and its unresolved slots for the rewriter.out_table = slot_tableunresolved_slots = [diff --git a/rag/advanced_rag/harness/action_session.py b/rag/advanced_rag/harness/action_session.pyindex eb37234b1e..5a1215265c 100644--- a/rag/advanced_rag/harness/action_session.py+++ b/rag/advanced_rag/harness/action_session.py@@ -54,11 +54,12 @@ _EMPTY_STRIKES = 2_NEAR_DUP_JACCARD = 0.8_RETRIEVAL_TOOLS = ("search_chunks", "grep_chunks", "grep_search")-# Hard cap on the shared evidence pool (tools.kbinfos["chunks"]). Mirrors-# _MAX_SNIPPET_POOL / _SCA_VIEW_CAP (=60) in agentic_rag_graph.py: once the pool-# is saturated, further admits cannot reach the SCA view or improve the answer,-# so the action session stops admitting (observed pools otherwise grew to ~106).-_EVIDENCE_POOL_CAP = 60+# Hard cap on the shared evidence pool (tools.kbinfos["chunks"]). Deliberately+# LARGER than _SCA_VIEW_CAP (=60) in agentic_rag_graph.py so storage and review stay+# DECOUPLED: the pool accumulates while the SCA reads a ranked top-60 view. Coupling them+# at 60 starved the raw-evidence channel in 42% of rounds (every admit rejected -> status+# REDUNDANT -> playbook told the model to vary the query -> it re-searched for nothing).+_EVIDENCE_POOL_CAP = 120# Per-process flag so the "pool FULL" log line is emitted once per fill, not once# per rejected chunk. The pool never shrinks mid-session, so no reset is needed._EVIDENCE_POOL_STATE = {"full_logged": False}@@ -248,10 +249,12 @@ _SEARCH_CHUNKS_TOOL_SPEC = {"function": {"name": "search_chunks","description": (- "WHEN TO CALL: Primary semantic recall. Use when exact retrieve returns nothing useful, when the corpus is large and you are unsure which document holds the answer, or when the answer passage shares no surface words with your query. Send 1-2 queries."+ "WHEN TO CALL: Primary semantic recall. Use when exact retrieve returns nothing useful, when the corpus is large and you are unsure which document holds the answer, or when the answer passage shares no surface words with your query. Send 1-2 queries; compiled-structure expansion is automatic (a no-op without compiled structure). ""DO NOT CALL: When you already have a doc_id and want to read that document (use list_chunks); when a single exact passage would be found faster by grep-style retrieve."- "ARGUMENTS: query — array of 1-2 strings. Compiled-structure expansion is automatic and a no-op on datasets without compiled structure, so no extra argument is needed."- "OUTPUT: Relevance-ranked snippet chunks, possibly with structural neighbours (parent/child headings, sibling pages) appended. ok = new evidence; redundant = already seen."+ "ARGUMENTS: query — array of 1-2 strings."+ "OUTPUT: Relevance-ranked snippet chunks, possibly with structural neighbours appended. "+ "Results may LEAD with [claim score=...] entries — the dataset's compiled atomic facts carrying VERBATIM source quotes. If a claim directly answers the query, cite it and answer WITHOUT further searching; deep-read its listed chunk only for missing context or numbers. "+ "ok = new evidence; redundant = already seen.""IF IT FAILS: miss means this query matched nothing — change the angle or fall back to retrieve or navigate_tree. Re-issuing a near-duplicate query is skipped as redundant, so vary the query instead of paraphrasing it."),"parameters": {@@ -329,7 +332,8 @@ _NAVIGATE_STRUCTURE_TOOL_SPEC = {"WHEN TO CALL: You know the doc_id and need to PINPOINT where the answer lives inside that one document, without reading every chunk. The in-document counterpart of navigate_tree.""DO NOT CALL: When you have no doc_id yet; when the document has no compiled structure (use list_chunks to read the full document).""ARGUMENTS: doc_id — string, required. query — string, what to locate within the document. kind — enum catalog / mindmap / graph, default catalog (compiled-structure kind)."- "OUTPUT: The structure outline annotated with matching chunk_ids, reading-order aware. ok = useful hits; poor (chunk_ptrs = 0) means it drilled to nothing usable."+ "OUTPUT: The structure outline annotated with matching chunk_ids, reading-order aware. [claim] lines carry VERBATIM quotes from the document — cite them and answer WITHOUT calling list_chunks when they directly answer the query (deep-read the claimed chunk ids only for surrounding context or numbers the quotes lack). "+ "ok = useful hits; poor (chunk_ptrs = 0) means it drilled to nothing usable.""IF IT FAILS: empty (no_structure) — try another doc_id or kind, or fall back to list_chunks / search_chunks. poor — read the full document via list_chunks(doc_id). A second empty disables the tool for the session."),"parameters": {@@ -612,6 +616,21 @@ def _seed_evidence(tools):return kbinfos+def _claim_covered_ids(kbinfos) -> set:+ """Chunk ids already represented verbatim by a claim pseudo-chunk in the pool.++ A claim carries its own verbatim quote plus the ids of the chunks it was+ distilled from, so admitting those passages again is duplicate payload: the+ answer material is already in the pool at a fraction of the size.+ """+ covered: set = set()+ for c in kbinfos.get("chunks") or []:+ if str(c.get("chunk_id") or "").startswith("claim_"):+ for cid in c.get("source_chunk_ids") or []:+ covered.add(str(cid))+ return covered++def _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=True) -> bool:"""Register one chunk into the session output AND the shared evidence pool.@@ -630,10 +649,10 @@ def _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=True) ->deepdoc dependency chain) out of module-import time."""# Early-stop: the shared evidence pool is hard-capped. Once it reaches the- # cap, admit no further chunk — the SCA view and compose can only consume the- # first 60 anyway (see _MAX_SNIPPET_POOL / _SCA_VIEW_CAP in agentic_rag_graph),- # so extra admits only bloat the pool (observed growth up to ~106) without- # improving the answer.+ # cap, admit no further chunk. The cap is deliberately ABOVE _SCA_VIEW_CAP+ # (60): the SCA reads a ranked view while the pool accumulates, so extra+ # admits still reach the view via ranking. Coupling them at 60 starved the+ # raw channel in 42% of rounds (admits rejected -> REDUNDANT -> re-search)._pool = kbinfos.get("chunks", []) if isinstance(kbinfos, dict) else (getattr(kbinfos, "chunks", []) or [])if len(_pool) >= _EVIDENCE_POOL_CAP:if not _EVIDENCE_POOL_STATE["full_logged"]:@@ -650,6 +669,11 @@ def _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=True) ->cid = _chunk_id(c)if cid in seen:return False+ # Already in the pool as a claim's verbatim quote -> skip the full passage.+ # Table chunks are exempt: their answer rows survive only in full text.+ if not _is_table_chunk(c) and cid in _claim_covered_ids(kbinfos):+ _LOG.debug("[Action Session] skip chunk %s: already covered by a claim", cid)+ return Falseseen.add(cid)ids.append(cid)# Table chunks pass through UN-truncated: the 1200-char cap hides answer@@ -672,6 +696,73 @@ def _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=True) ->return False+# Claim hits REPLACE the chunk search for that query (not stack on top of it):+# a claim's verbatim evidence is the answer material, and echoing the same+# passages as chunk snippets burns tokens without adding information. Flip to+# False for the additive behaviour (claims first, chunks after).+_CLAIM_PREFETCH_EXCLUSIVE = True+++async def _claim_prefetch(tools, query: str, kbinfos: dict, kb_seen: set) -> tuple:+ """Framework-automatic claim-first prefetch for every corpus search.++ KB-wide KNN over claim rows; matched claims (with their verbatim evidence)+ lead the search output AND enter the shared evidence pool as pseudo-chunks,+ so the model sees and can cite them without having chosen+ navigate_structure — retrieval priority must not rest on the model's tool+ pick. Best effort: any failure returns empty and the search proceeds+ exactly as before.+ """+ import hashlib++ if not query:+ return [], []+ try:+ from rag.advanced_rag.harness.tools.navigation import (+ _STRUCT_CLAIM_EVIDENCE_CHARS,+ dataset_has_compilation,+ recall_dataset_claims,+ )++ # No compiled rows at all -> no claim rows can exist. Skip the recall+ # instead of issuing legs that come back empty on every single turn.+ if not await dataset_has_compilation(tools):+ return [], []+ claims = await recall_dataset_claims(tools, query)+ except Exception: # noqa: BLE001+ _LOG.warning("[Action Session] claim prefetch failed", exc_info=True)+ return [], []+ entries: list = []+ new_ids: list = []+ for c in claims:+ cid = "claim_" + hashlib.md5(f"{c['doc_id']}:{c['name']}".encode("utf-8", "ignore")).hexdigest()[:12]+ if cid in kb_seen:+ continue+ content = f"[claim #{c.get('rank') or '?'}] {c['name']}"+ if c.get("description") and c["description"] != c["name"]:+ content += f" — {c['description']}"+ if c.get("quote"):+ content += f'\nEvidence (verbatim): "{str(c["quote"])[:_STRUCT_CLAIM_EVIDENCE_CHARS]}"'+ doc_id = c.get("doc_id") or ""+ entries.append({"id": cid, "content": content[:1200], "doc_id": doc_id})+ new_ids.append(cid)+ kb_seen.add(cid)+ # Enter the shared pool as a pseudo-chunk so the compose stage can cite+ # the verbatim evidence directly (search-context parity with chunks).+ # source_chunk_ids ride along so a later deep-read of the underlying+ # chunk can retire this pseudo-chunk (its quote would then duplicate+ # the full text already in the pool).+ kbinfos["chunks"].append({"chunk_id": cid, "content_with_weight": content, "doc_id": doc_id, "source_chunk_ids": c["chunk_ids"]})+ _LOG.info(+ "[Claim] prefetch q=%r -> recalled=%d new=%d (exclusive=%s)",+ str(query)[:60],+ len(claims or []),+ len(new_ids),+ _CLAIM_PREFETCH_EXCLUSIVE,+ )+ return entries, new_ids++async def _run_search(tools, search_fn, queries: list, top_n: int, max_q: int, **kw) -> tuple:"""Run a corpus search fn per query and admit hits to output + evidence pool.@@ -688,6 +779,17 @@ async def _run_search(tools, search_fn, queries: list, top_n: int, max_q: int, *new_evidence = 0kbinfos = _seed_evidence(tools)kb_seen = {_chunk_id(c) for c in kbinfos["chunks"] if isinstance(c, dict)}+ # Claim-first, MUTUALLY EXCLUSIVE: when claims hit, their verbatim evidence+ # IS the answer material — shipping 20 chunk snippets on top would echo the+ # same passages a second time and burn tokens. Claims carry their chunk+ # pointers, so deep-reading stays one list_chunks away. No hits → chunk+ # search runs exactly as before.+ claim_entries, claim_ids = await _claim_prefetch(tools, (queries or [""])[0], kbinfos, kb_seen)+ out.extend(claim_entries)+ ids.extend(claim_ids)+ new_evidence += len(claim_ids)+ if claim_entries and _CLAIM_PREFETCH_EXCLUSIVE:+ return out, ids, new_evidencefor fq in queries[:max_q]:try:res = await search_fn(tools, fq, kb_ids=kb_ids, top_n=top_n, **kw)@@ -823,6 +925,13 @@ async def _exec_list_chunks(tools, doc_id: str) -> ToolOutcome:continueif _admit_evidence(kbinfos, kb_seen, c, out, ids, seen, include_doc_id=False):new_ev += 1+ # A deep-read COVERS its claims: once the full chunk text is in the pool,+ # the claim's 1200-char quote of the same passage is duplicated tokens in+ # every later prompt. Retire claim pseudo-chunks whose source chunk was+ # just read — the claim already did its job (it pointed here).+ read_ids = set(ids)+ if read_ids:+ kbinfos["chunks"] = [c for c in kbinfos["chunks"] if not (str(c.get("chunk_id") or "").startswith("claim_") and read_ids.intersection(c.get("source_chunk_ids") or []))]return _search_outcome(out, ids, new_ev)@@ -861,9 +970,17 @@ async def _exec_navigate_tree(tools, args: dict) -> ToolOutcome:routing falls through at ``_NAV_MIN_DOC_SCORE``, which is query-dependent,not a statement about the dataset."""- from rag.advanced_rag.harness.tools.navigation import _navigate_tree_impl+ from rag.advanced_rag.harness.tools.navigation import _navigate_tree_impl, dataset_has_compilation_inject_nav_tools_ref(tools)+ # Dataset-level fact, not a per-query miss: with no compiled rows every+ # navigation leg comes back empty, so short-circuit instead of paying for it.+ if not await dataset_has_compilation(tools):+ return ToolOutcome(+ payload=[{"kind": "navigate_tree", "note": "This dataset has no compiled document-navigation structure; use search_chunks / retrieve instead."}],+ status=EMPTY,+ reason="no_structure",+ )query = str(args.get("query") or "")res = await _navigate_tree_impl(query, keywords=str(args.get("keywords") or ""))if res.empty_reason:@@ -888,9 +1005,19 @@ async def _exec_navigate_structure(tools, args: dict) -> ToolOutcome:As with navigate_tree, classification only — ``_tool_node`` decides whetherto disable."""- from rag.advanced_rag.harness.tools.navigation import _navigate_structure_impl+ from rag.advanced_rag.harness.tools.navigation import _navigate_structure_impl, dataset_has_compilation_inject_nav_tools_ref(tools)+ # Same dataset-level gate as navigate_tree: without compiled rows the+ # in-document drill has nothing to walk.+ if not await dataset_has_compilation(tools):+ return ToolOutcome(+ payload=[+ {"kind": "navigate_structure", "doc_id": str(args.get("doc_id") or ""), "note": "This dataset has no compiled document structure; use search_chunks / retrieve / list_chunks instead."}+ ],+ status=EMPTY,+ reason="no_structure",+ )doc_id = str(args.get("doc_id") or "")query = str(args.get("query") or "")kind = str(args.get("kind") or "catalog")@@ -2097,7 +2224,20 @@ async def initialize_state(tools, question, fanout_hint, deadline_left=None):if fanout_hint:user += "\n\nCandidate aspects already identified:\n" + "\n".join(f"- {h}" for h in fanout_hint)tmo = min(_INIT_TIMEOUT_S, deadline_left or _INIT_TIMEOUT_S)++ # ``_init_chat`` runs on the raw model (it bypasses CountingChatModel like+ # ``_base_chat_mdl`` does), so the slot-table decomposition call would be+ # invisible to the phase accounting. Count it explicitly.+ def _book_raw_call() -> None:+ try:+ from rag.advanced_rag.harness.stats import record_external_call++ record_external_call(None)+ except Exception: # noqa: BLE001+ pass+raw = await _init_chat(tools, system, user, tmo)+ _book_raw_call()data = extract_json(raw) or {}if not data:# one quick retry — transient provider stalls were observed (45s with@@ -2109,6 +2249,7 @@ async def initialize_state(tools, question, fanout_hint, deadline_left=None):# the session deadline so the retry cannot overrun the round budget.retry_tmo = _init_retry_timeout(tmo, deadline_left)raw = await _init_chat(tools, system, user, retry_tmo)+ _book_raw_call()data = extract_json(raw) or {}slots = []for i, s in enumerate(data.get("slots") or []):diff --git a/rag/advanced_rag/harness/tools/compiled_expansion.py b/rag/advanced_rag/harness/tools/compiled_expansion.pyindex e252fc8bb7..846044c570 100644--- a/rag/advanced_rag/harness/tools/compiled_expansion.py+++ b/rag/advanced_rag/harness/tools/compiled_expansion.py@@ -59,6 +59,18 @@ async def _expand_with_compiled(tools, query: str, keywords: str, kbinfos: dict,"""before = len(kbinfos.get("chunks", []))seen_ids = {c.get("chunk_id") or c.get("id") for c in kbinfos.get("chunks", [])}+ # The hybrid hits themselves — the claim-neighbour leg keys on them (the+ # passages hybrid_search deemed relevant), so capture them BEFORE any+ # expansion appends its own rows.+ hit_chunks = [+ (+ str(c.get("chunk_id") or c.get("id") or ""),+ float(c.get("similarity") or 0.0),+ str(c.get("doc_id") or ""),+ )+ for c in kbinfos.get("chunks", [])+ if isinstance(c, dict)+ ]scopes = await _kg_scopes(tools, doc_scope)if not scopes:@@ -89,6 +101,11 @@ async def _expand_with_compiled(tools, query: str, keywords: str, kbinfos: dict,_LOG.debug("[Compiled expand] %s: +%d chunks", label, len(chunks))# Tree structure graph (uses ``compile_kwd``, not template kind).+ # Tree compilation now writes the same per-row shape as page_index,+ # so the raw-row strategy is the primary path. The blob strategy is+ # kept only as a fallback for documents compiled before the migration+ # (one graph blob instead of raw rows); it disappears once those docs+ # are recompiled. Both dedup through seen_ids.chunks = await _expand_compiled_strategy(tools,kb_id,@@ -99,6 +116,16 @@ async def _expand_with_compiled(tools, query: str, keywords: str, kbinfos: dict,compile_kwd="tree",max_chunks=5,)+ if not chunks:+ chunks = await _expand_tree_blob_strategy(+ tools,+ kb_id,+ tenant_id,+ doc_ids,+ query,+ seen_ids,+ max_chunks=5,+ )if chunks:kbinfos.setdefault("chunks", []).extend(chunks)_LOG.debug("[Compiled expand] tree: +%d chunks", len(chunks))@@ -124,6 +151,23 @@ async def _expand_with_compiled(tools, query: str, keywords: str, kbinfos: dict,kbinfos.setdefault("chunks", []).extend(chunks)_LOG.debug("[Compiled expand] %s: +%d chunks", label, len(chunks))+ # Claims adjacent to the passages hybrid_search just hit (same+ # source_chunk_ids). Non-redundant with the query-keyed claim+ # prefetch: when the prefetch hits, the exclusive takeover means this+ # expansion never runs; when it misses for the QUERY, chunk hits are a+ # different key and their sibling claims may still match.+ claim_chunks = await _expand_claim_neighbor_strategy(+ tools,+ kb_id,+ tenant_id,+ hit_chunks,+ seen_ids,+ max_chunks=6,+ )+ if claim_chunks:+ kbinfos.setdefault("chunks", []).extend(claim_chunks)+ _LOG.debug("[Compiled expand] claim neighbours: +%d chunks", len(claim_chunks))+# Re-sort so compiled-expansion chunks blend by similarity with regular ones.chunks = kbinfos.get("chunks", [])if chunks:@@ -175,6 +219,7 @@ async def _search_compiled_rows("from_entity_kwd","to_entity_kwd","name_kwd",+ "entity_type_kwd",]exprs = []if text:@@ -295,6 +340,13 @@ async def _expand_compiled_strategy(payload = json.loads(r.get("content_with_weight") or "{}")except Exception:continue+ # Claims are evidence rows, not structure nodes: expanding one 1-hop+ # would treat verbatim evidence as a table of contents. Claims reach+ # the model through the claim legs (prefetch / direct search), never+ # through structural expansion.+ rtype = str(r.get("entity_type_kwd") or payload.get("type") or "").strip().casefold()+ if rtype == "claim":+ continuename = (payload.get("name") or payload.get("title") or "").strip()if name:seed_names.add(name)@@ -512,3 +564,266 @@ async def _expand_wiki_page_strategy(new_chunks.append(c)return new_chunks+++# --- Tree blob expansion (legacy fallback) ---+# Documents compiled BEFORE the per-row migration persist ONE graph blob per+# document (``knowledge_graph_kwd="graph"``, ``compile_kwd="tree"``) and no raw+# entity/relation rows. Current tree compilation writes the same per-row shape+# as page_index, so this strategy only serves pre-migration docs until they+# are recompiled. The blob carries the whole tree -- entities with+# ``name``/``type``/``description``/``source_chunk_ids`` and parent-child+# relations as ``{"from", "to"}`` -- so expansion runs over the JSON in-process.+++async def _search_tree_blobs(tools, kb_id: str, tenant_id: str, doc_ids: list[str] | None) -> dict:+ """Fetch a document's tree graph blob rows (metadata + payload only)."""+ from common import settings+ from common.doc_store.doc_store_base import OrderByExpr+ from common.misc_utils import thread_pool_exec+ from rag.nlp import search++ condition: dict = {"compile_kwd": ["tree"], "knowledge_graph_kwd": ["graph"]}+ if doc_ids:+ condition["doc_id"] = list(doc_ids)+ fields = ["content_with_weight", "doc_id"]+ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ fields,+ [],+ condition,+ [],+ OrderByExpr(),+ 0,+ 16,+ search.index_name(tenant_id),+ [kb_id],+ )+ return settings.docStoreConn.get_fields(res, fields) or {}+ except Exception:+ _LOG.exception("[Compiled expand] tree blob search failed for kb=%s", kb_id)+ return {}+++def _score_tree_entities(graph: dict, query: str) -> list[tuple[dict, int]]:+ """Rank blob entities by query-term overlap over name + description.++ Lexical on purpose: the raw-row strategy seeds the same way (BM25 over+ ``content_ltks``), and the blob's vectors are one shared row vector --+ there is nothing per-entity to cosine against without paying an embed+ round-trip.+ """+ from rag.advanced_rag.knowlege_compile.dataset_nav import _tokenize++ terms = [t for t in _tokenize(query).split() if t]+ if not terms:+ return []+ scored: list[tuple[dict, int]] = []+ for ent in graph.get("entities") or []:+ if not isinstance(ent, dict):+ continue+ text = f"{ent.get('name') or ''} {ent.get('description') or ''}".casefold()+ if not text:+ continue+ score = sum(1 for t in terms if t.casefold() in text)+ if score > 0:+ scored.append((ent, score))+ scored.sort(key=lambda pair: pair[1], reverse=True)+ return scored+++async def _expand_tree_blob_strategy(+ tools,+ kb_id: str,+ tenant_id: str,+ doc_ids: list[str] | None,+ query: str,+ seen_ids: set[str],+ *,+ max_chunks: int = 5,+) -> list[dict]:+ """Expand a tree-compiled document from its graph blob, in-process.++ Same contract as ``_expand_compiled_strategy`` -- seed entities, 1-hop+ neighbours, and the neighbours' ``source_chunk_ids`` loaded back as real+ chunks -- except seeds, relations, and addressing all come from the blob's+ JSON, so the walk costs zero extra store round-trips. Entity payloads are+ compiled summaries: only the loaded chunks reach the model.+ """+ import json++ blobs = await _search_tree_blobs(tools, kb_id, tenant_id, doc_ids)+ if not blobs:+ return []++ by_doc: dict[str, set[str]] = {}+ for row in blobs.values():+ doc_id = str(row.get("doc_id") or "")+ if not doc_id:+ continue+ try:+ graph = json.loads(row.get("content_with_weight") or "{}")+ except Exception:+ continue+ if not isinstance(graph, dict):+ continue++ # Seeds: query-matched entities, then 1-hop neighbours through the+ # blob's own relations (parent/child edges) -- no store round-trips.+ scored = _score_tree_entities(graph, query)[:5]+ if not scored:+ continue+ seed_names = {str(ent.get("name") or "").strip() for ent, _ in scored}+ seed_names.discard("")++ neighbours: set[str] = set()+ for rel in graph.get("relations") or []:+ if not isinstance(rel, dict):+ continue+ frm = str(rel.get("from") or "").strip()+ to = str(rel.get("to") or "").strip()+ if frm in seed_names and to:+ neighbours.add(to)+ if to in seed_names and frm:+ neighbours.add(frm)++ wanted = seed_names | neighbours+ if not wanted:+ continue+ # Address chunks: every entity at or adjacent to a seed contributes+ # its source_chunk_ids (the same contract the raw-row strategy applies+ # to neighbour entities).+ for ent in graph.get("entities") or []:+ if not isinstance(ent, dict):+ continue+ if str(ent.get("name") or "").strip() not in wanted:+ continue+ for cid in ent.get("source_chunk_ids") or []:+ if isinstance(cid, str) and cid and cid not in seen_ids:+ by_doc.setdefault(doc_id, set()).add(cid)++ new_chunks: list[dict] = []+ for doc_id, cids in by_doc.items():+ if len(new_chunks) >= max_chunks:+ break+ limit = max_chunks - len(new_chunks)+ chunks = await _load_chunks_for_doc(tools, doc_id, list(cids)[:limit])+ for c in chunks:+ cid = c.get("chunk_id") or c.get("id")+ if cid and cid not in seen_ids:+ seen_ids.add(cid)+ new_chunks.append(c)++ return new_chunks+++# --- Claim neighbour expansion ---+++async def _expand_claim_neighbor_strategy(+ tools,+ kb_id: str,+ tenant_id: str,+ hit_chunks: list[tuple[str, float, str]],+ seen_ids: set[str],+ *,+ max_chunks: int = 6,+) -> list[dict]:+ """Inject claims adjacent to the passages hybrid_search just hit.++ A claim whose ``source_chunk_ids`` intersect a hit chunk is a verified+ atomic statement about a passage the retriever already deemed relevant --+ even when the QUERY never matched it (a chunk can match lexically while a+ neighbouring assertion answers the question paraphrased). Emitted as+ pseudo-chunks (name + verbatim quote, mirroring ``_claim_prefetch``'s+ shape) with the hit chunk's similarity, so they blend into the ranking.++ Deliberately NOT 1-hop expanded: a claim is evidence, its quote is already+ the material -- it is a terminal, not a way-point. And deliberately keyed+ on the HIT chunks, not the query: the query-keyed claim prefetch+ (``_claim_prefetch``) already ran for this search, and when it hits, its+ exclusive takeover means this expansion never runs at all.+ """+ import hashlib+ import json++ hits = [(cid, sim, doc_id) for cid, sim, doc_id in hit_chunks if cid and not cid.startswith("claim_")]+ if not hits:+ return []++ hit_ids = {cid for cid, _, _ in hits}+ hit_docs = sorted({doc_id for _, _, doc_id in hits if doc_id})++ from common import settings+ from common.doc_store.doc_store_base import OrderByExpr+ from common.misc_utils import thread_pool_exec+ from rag.nlp import search++ condition: dict = {"entity_type_kwd": ["claim"], "scope_kwd": ["doc"]}+ if hit_docs:+ condition["doc_id"] = hit_docs+ fields = ["content_with_weight", "source_chunk_ids", "doc_id"]+ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ fields,+ [],+ condition,+ [],+ OrderByExpr(),+ 0,+ 256,+ search.index_name(tenant_id),+ [kb_id],+ )+ rows = settings.docStoreConn.get_fields(res, fields) or {}+ except Exception:+ _LOG.exception("[Compiled expand] claim-neighbour search failed for kb=%s", kb_id)+ return {}++ new_chunks: list[dict] = []+ for row in rows.values():+ try:+ payload = json.loads(row.get("content_with_weight") or "{}")+ except Exception:+ continue+ if not isinstance(payload, dict):+ continue+ name = str(payload.get("name") or "").strip()+ if not name:+ continue+ src_ids = [str(c) for c in row.get("source_chunk_ids") or payload.get("source_chunk_ids") or [] if str(c).strip()]+ overlap = hit_ids.intersection(src_ids)+ if not overlap:+ continue+ cid = "claim_" + hashlib.md5(f"{row.get('doc_id')}:{name}".encode("utf-8", "ignore")).hexdigest()[:12]+ if cid in seen_ids:+ continue+ description = str(payload.get("description") or "").strip()+ quote = ""+ for ev in payload.get("evidence") or []:+ if isinstance(ev, dict) and str(ev.get("quote") or "").strip():+ quote = str(ev["quote"]).strip()+ break+ content = f"[claim] {name}"+ if description and description != name:+ content += f" -- {description}"+ if quote:+ content += f'\nEvidence (verbatim): "{quote}"'+ # Inherit the best-matching hit's similarity so the pseudo-chunk lands+ # beside the passage that spawned it in the final ranking.+ similarity = max((sim for hcid, sim, _ in hits if hcid in overlap), default=0.0)+ seen_ids.add(cid)+ new_chunks.append(+ {+ "chunk_id": cid,+ "content_with_weight": content[:1200],+ "doc_id": str(row.get("doc_id") or ""),+ "source_chunk_ids": src_ids,+ "similarity": similarity,+ }+ )+ if len(new_chunks) >= max_chunks:+ break+ return new_chunksdiff --git a/rag/advanced_rag/harness/tools/navigation.py b/rag/advanced_rag/harness/tools/navigation.pyindex cc44e3e3bc..7f7203868b 100644--- a/rag/advanced_rag/harness/tools/navigation.py+++ b/rag/advanced_rag/harness/tools/navigation.py@@ -46,7 +46,7 @@ _LOG = logging.getLogger(__name__)# or a page index. ``_compilation_template_kind`` in the API folds page_index# and knowledge_graph into "timeline"; RAPTOR is its own bucket and is# inherently tree-like, so it counts too.-_CATALOG_KINDS = {"tree", "timeline", "raptor", "page_index", "pageindex"}+_CATALOG_KINDS = {"tree", "timeline", "page_index", "pageindex"}# Compiled-structure kinds that describe the document's *concepts*._MINDMAP_KINDS = {"mindmap", "mind_map"}@@ -99,11 +99,20 @@ class NavResult:def _normalize_kind(kind) -> str:- """Mirror the API's kind normalization (page_index/knowledge_graph -> timeline)."""+ """Normalize a compile-template kind.++ ``page_index`` keeps its own identity. It used to fold into ``timeline``,+ which made tree and page_index indistinguishable at the read sites even+ though they carry different row shapes (graph blob vs per-entity rows).+ ``knowledge_graph`` still folds into ``timeline`` — that is the legacy+ marker the catalog kind set is built around.+ """if not isinstance(kind, str):return ""normalized = kind.strip().lower().replace("-", "_")- if normalized in {"pageindex", "page_index", "knowledge_graph"}:+ if normalized in {"pageindex", "page_index"}:+ return "page_index"+ if normalized == "knowledge_graph":return "timeline"return normalized@@ -177,22 +186,22 @@ async def _load_compiled_structure(tools, doc_id: str, kinds: set) -> dict:# may also have entity/relation rows. Read BOTH and merge so navigation works# regardless of which shape a given compile type produced.rows: dict = {}- rows.update(await _query({"doc_id": [doc_id], "knowledge_graph_kwd": ["graph"]}, 1000))+ # The compact graph blob (knowledge_graph_kwd="graph") is gone from the+ # storage model: tree compilation now writes the same per-row shape as+ # page_index, so this single entity/relation query covers every compiled+ # doc.rows.update(await _query({"doc_id": [doc_id], "knowledge_graph_kwd": ["entity", "relation"]},3000,))- rows.update(await _query({"doc_id": [doc_id], "compile_kwd": ["raptor_graph"]}, 16))entities: list[dict] = []relations: list[dict] = []for row in rows.values():compile_kwd = row.get("compile_kwd") or ""kind = _normalize_kind(row.get("compilation_template_kind_kwd") or compile_kwd)- if compile_kwd == "raptor_graph":- kind = "raptor"if kind not in kinds:continuetry:@@ -659,10 +668,10 @@ _NAV_MIN_DOC_SCORE = 0.2 # drop docs below this scoreasync def _nav_search_titled(tools, topic: str, keywords: str = "", doc_scope: list[str] | None = None) -> list[tuple[str, str]]:"""Descend the compiled nav TREE and return ``(doc_id, summary)`` pairs.- Uses ``mode="navigation_tree"``, which currently resolves to a FLAT hybrid- sweep over every ``nav_doc`` row (``_NAV_TREE_ROUTER = "nav_doc"``) rather- than a descent through the cluster levels. The strategy lives server-side, so- this function is agnostic to which one is active.+ Uses ``mode="navigation_tree"``, which routes through the dataset's claim+ rows when the compiler produced any and falls back to raw-chunk aggregation+ otherwise. The choice lives server-side and is driven by what the dataset+ actually contains, so this function is agnostic to which leg ran.Flat routing is deliberate. A BFS beam descent narrows the field level bylevel, and a wrong pick at the first level cannot be undone — the correct@@ -715,6 +724,10 @@ async def _nav_search_titled(tools, topic: str, keywords: str = "", doc_scope: l"navigation_tree",top_k=_NAV_SEARCH_MAX_DOCS,doc_scope=list(allowed_docs) or None,+ # Agentic rag routes by claims: atomic propositions with+ # verbatim evidence decide the ranking, raw-chunk aggregation+ # is the fallback. The artifacts UI keeps main's tree descent.+ router="claim_agg",)except Exception:_LOG.exception("[Dataset navigation search] search_dataset_layers failed for kb=%s", kb.id)@@ -748,10 +761,10 @@ async def dataset_navigation_search(tools, topic: str, keywords: str = "", doc_sdescending the dataset's compiled navigation tree.Routes through :func:`_nav_search_titled`, which runs- ``search_dataset_layers`` with ``mode="navigation_tree"``: a hybrid- (vector + BM25) sweep over the ``nav_doc`` rows, with the ``_text_score``- gates armed. Which router runs is decided server-side by- ``_NAV_TREE_ROUTER``, so this function does not depend on the strategy.+ ``search_dataset_layers`` with ``mode="navigation_tree"``: claim rows first+ (hybrid vector + BM25), falling back to raw-chunk aggregation when the+ dataset has no claim rows. Which leg runs is decided server-side by what the+ dataset contains, so this function does not depend on it.Returns the routed ``doc_id`` list (capped at ``_NAV_SEARCH_MAX_DOCS``), or``[]`` when no question/keywords are given or the search returns nothing.@@ -1019,12 +1032,14 @@ async def _navigate_structure_impl(parts = [f'<structure_navigation count="{len(structures)}" query="{_xml_escape(query)}" kind="{_xml_escape(kind)}">']total_entities = 0+ total_claims = 0total_ptrs = 0best_score = 0.0all_chunk_paths: dict[str, str] = {}for i, s in enumerate(structures):stats = s.get("stats") or {}total_entities += len(s.get("entities") or [])+ total_claims += int(stats.get("claim_hits") or 0)total_ptrs += int(stats.get("chunk_ptrs") or 0)best_score = max(best_score, float(stats.get("top_score") or 0.0))for cid, path in (s.get("chunk_paths") or {}).items():@@ -1034,6 +1049,8 @@ async def _navigate_structure_impl(outline = s["outline"]if outline:parts.append(f" <structure>{_xml_escape(outline)}</structure>")+ if int((s.get("stats") or {}).get("claim_hits") or 0) >= _STRUCT_CLAIM_SUFFICIENT_HITS:+ parts.append(" <claims_sufficient/>")parts.append(" </doc>")parts.append("</structure_navigation>")return NavResult(@@ -1043,7 +1060,11 @@ async def _navigate_structure_impl(chunk_ptrs=total_ptrs,top_score=round(best_score, 4),chunk_paths=all_chunk_paths,- empty_reason="no_structure" if total_entities == 0 else "",+ # A document whose compiler wrote claims but no tree nodes (an offline+ # batch can produce claim rows alone) still renders a claim-first+ # outline — counting only entities here would throw that output away+ # and the session would strike the tool as "no compiled structure".+ empty_reason="no_structure" if (total_entities == 0 and total_claims == 0) else "",)@@ -1054,7 +1075,7 @@ def _structure_kinds_for(kind: str) -> set:if k in ("mindmap", "mind_map", "concept"):return set(_MINDMAP_KINDS)if k in ("graph", "kg", "entity", "ontology"):- return {"graph", "ontology", "entity", "raptor"}+ return {"graph", "ontology", "entity"}return set(_CATALOG_KINDS)@@ -1099,6 +1120,43 @@ _STRUCT_TOC_MAX_DEPTH = 6_STRUCT_RELATED_SNIPPET_CHARS = 300_STRUCT_RELATED_MAX_PER_DOC = 4+# --- claim/evidence leg -------------------------------------------------+# Claim rows (entity_type_kwd="claim", written by the tree compiler) carry their+# own vector and their verbatim evidence. Matching them is FLAT: every claim is+# scored directly against the query, so a bad early pick cannot prune the subtree+# holding the answer — the failure mode SCT measured for top-down descent+# (20.2% vs 39.3% routing accuracy). Beam descent stays as the fallback.+_STRUCT_CLAIM_LEG = True+_STRUCT_CLAIM_TOP_N = 8 # claims rendered per document+_STRUCT_CLAIM_EVIDENCE_CHARS = 1200 # cap on the rendered verbatim quote.+# 1200 over 600: the quote must be self-sufficient in ONE shot — a thin quote+# makes the SCA judge the context insufficient, and one extra re-search turn+# costs a full 10k+-token prompt, far more than the ~600 extra chars per claim.+# Claim-first: when claims hit, they REPLACE the drill-down instead of layering+# on top of it. A claim is an atomic proposition scored flat against the query+# and it carries its own verbatim evidence, so a hit already is the answer+# material — descending the tree (or recalling chunks to choose for it) can only+# re-derive what the claim states, and every descent step is another chance to+# prune the branch holding the answer (SCT: first-level routing 20.2% vs 39.3%+# for flat). The tree still does the one job left: labelling where each claim+# sits. Drill-down becomes the fallback for documents without claim hits.+_STRUCT_CLAIM_FIRST = True+# Fewest claim hits that justify skipping the drill. Raising it makes a weak+# claim set fall back to the old path instead of answering from thin evidence.+_STRUCT_CLAIM_FIRST_MIN_HITS = 1+# Claim hits are threshold-free: hybrid recall (BM25 + KNN, RRF-fused) returns+# the store's best-matching claims and the top-N ARE the hit set — ranks need no+# cross-leg score calibration, and a similarity cut was the wrong tool (the+# right rank depends on the query, not on a global constant). "Sufficiency" is+# likewise a COUNT, not a score: enough distinct claims matching the query means+# the model may cite them without deep-reading chunks. Benchmark-tunable.+_STRUCT_CLAIM_SUFFICIENT_HITS = 3++# page_index spreads facts across TOC levels, so scoring every node at once beats+# descending level by level (same reason as the claim leg). Falls back to beam.+_STRUCT_FLAT_NODE_SELECT = True+_STRUCT_FLAT_TOP_N = 10+async def _embed_query(tools_slot, query: str):"""Encode ``query`` into a vector via the embedding model.@@ -1150,23 +1208,11 @@ async def _load_entities_with_vectors(tools_slot, doc_id: str, kinds: set, vec_ffields.append(vec_field)try:- # Same dual-shape logic as _load_compiled_structure: RAPTOR/tree writes a- # compact graph blob (knowledge_graph_kwd="graph"), while page_index /- # pipeline-Compiler tree write per-entity rows (knowledge_graph_kwd="entity").- # Read BOTH so vector beam descent works for either shape.+ # Tree compilation now writes the same per-row shape as page_index+ # (per-entity rows with their own vectors), so one entity query covers+ # every compiled doc -- no graph-blob leg, and beam descent gets a real+ # per-node vector for tree docs too.res = await thread_pool_exec(- settings.docStoreConn.search,- fields,- [],- {"doc_id": [doc_id], "knowledge_graph_kwd": ["graph"]},- [],- OrderByExpr(),- 0,- 1000,- index_name,- [kb_id],- )- res2 = await thread_pool_exec(settings.docStoreConn.search,fields,[],@@ -1179,7 +1225,6 @@ async def _load_entities_with_vectors(tools_slot, doc_id: str, kinds: set, vec_f[kb_id],)rows = settings.docStoreConn.get_fields(res, fields) or {}- rows.update(settings.docStoreConn.get_fields(res2, fields) or {})except Exception:_LOG.exception("[navigate_structure] _load_entities_with_vectors failed for doc=%s", doc_id)return []@@ -1246,23 +1291,46 @@ async def _read_structures(tools_slot, query: str, doc_ids: list[str], kinds: seexcept Exception:_LOG.exception("[navigate_structure] structure load failed for doc=%s", doc_id)- # Two selection strategies, chosen by whether the tree can score itself.+ # Selection strategies, in priority order: claims when they hit, then+ # whatever the tree's own shape supports (the claim leg reads compiled+ # rows, so it is available for either shape).hits: list[tuple[str, float]] = []selected: list[str] = []- if entities and _has_distinct_node_vectors(entities):- # page_index / per-node embeddings: hand the whole TOC to the model- # at once. It sees every level simultaneously, so nothing is pruned- # by an early bad pick. Empty result just means "caller's problem",- # so fall through to the beam only when the model is unavailable.+ claim_hits: list[dict] = []+ if _STRUCT_CLAIM_LEG:+ claim_hits = await _recall_claim_hits(tools_slot, query, qvec, doc_id, kinds, _STRUCT_CLAIM_TOP_N)++ # Claim-first: the claims decide, so nothing is drilled and no chunks are+ # recalled to choose. See _STRUCT_CLAIM_FIRST.+ if _STRUCT_CLAIM_FIRST and len(claim_hits) >= _STRUCT_CLAIM_FIRST_MIN_HITS:+ pass+ elif entities and _has_distinct_node_vectors(entities):+ # page_index / per-node embeddings: the nodes can be scored directly.+ # Handing the whole TOC to the model sees every level at once, but it+ # costs one chat call per document; scoring every node here costs+ # nothing extra and is equally immune to an early bad pick.if _STRUCT_TOC_LLM_SELECT:selected = await _select_toc_nodes(tools_slot, query, entities, relations)+ if not selected and _STRUCT_FLAT_NODE_SELECT:+ selected = _flat_select_nodes(qvec, entities, _STRUCT_FLAT_TOP_N)else:# RAPTOR blob path: every node shares the blob's one embedding, so# beam descent would select arbitrarily. Let the chunk index choose# and keep the tree for labelling where the chosen chunks sit.hits = await _recall_chunk_ids_in_doc(tools_slot, query, doc_id, _STRUCT_RECALL_TOP_N)- outline, stats, chunk_paths = await _render_toc_drilldown(query, qvec, entities, relations, chunk_hits=hits or None, selected=selected or None)+ outline, stats, chunk_paths = await _render_toc_drilldown(+ query,+ qvec,+ entities,+ relations,+ chunk_hits=hits or None,+ selected=selected or None,+ claim_hits=claim_hits or None,+ )+ if claim_hits:+ stats["claim_hits"] = len(claim_hits)+ _publish_claim_hits(tools_slot, claim_hits, doc_id)out.append({"doc_id": doc_id,@@ -1423,6 +1491,451 @@ async def _recall_chunk_ids_in_doc(tools, query: str, doc_id: str, top_n: int) -return hits+def _parse_claim_row(row: dict) -> dict | None:+ """Extract one claim hit from a store row. ``None`` when the row is not a+ usable claim (bad payload, unnamed, or no chunk pointer)."""+ try:+ payload = json.loads(row.get("content_with_weight") or "{}")+ except Exception:+ return None+ if not isinstance(payload, dict):+ return None+ name = str(payload.get("name") or "").strip()+ if not name:+ return None+ ids = row.get("source_chunk_ids") or payload.get("source_chunk_ids") or []+ if isinstance(ids, str):+ ids = [ids]+ chunk_ids = [str(c).strip() for c in ids if str(c).strip()]+ if not chunk_ids:+ return None+ quote = ""+ for ev in payload.get("evidence") or []:+ if isinstance(ev, dict) and str(ev.get("quote") or "").strip():+ quote = str(ev["quote"]).strip()+ break+ return {+ "name": name,+ "description": str(payload.get("description") or "").strip(),+ "quote": quote,+ "doc_id": str(row.get("doc_id") or "").strip(),+ "chunk_ids": chunk_ids,+ "score": float(row.get("similarity") or 0.0),+ }+++def _rrf_fuse(*legs: list[dict], k: int = 60) -> list[dict]:+ """Reciprocal-rank fusion across retrieval legs (BM25 + KNN).++ RRF consumes ranks only, so hybrid claim search carries NO similarity+ threshold to calibrate — the store's best-matching claims fuse into one+ ordering regardless of which leg found them. Each fused hit keeps its+ best leg score (diagnostics) and gains a 1-based ``rank``.+ """+ best: dict[tuple, dict] = {}+ fused_score: dict[tuple, float] = {}+ for leg in legs:+ for rank, hit in enumerate(leg):+ key = (hit.get("doc_id") or "", hit["name"])+ fused_score[key] = fused_score.get(key, 0.0) + 1.0 / (k + rank + 1)+ cur = best.get(key)+ if cur is None or hit.get("score", 0.0) > cur.get("score", 0.0):+ best[key] = hit+ fused = sorted(best.values(), key=lambda h: fused_score[(h.get("doc_id") or "", h["name"])], reverse=True)+ for i, hit in enumerate(fused):+ hit["rank"] = i + 1+ return fused+++# Memo for _recall_claim_hits, keyed by (doc_id, query). The drill calls it once+# per routed doc and the model re-issues navigate_structure, so the same+# (doc, query) pair recurs within a session; each miss costs two store+# round-trips (BM25 + KNN). Same TTL/cap policy as _CLAIM_PREFETCH_CACHE.+_STRUCT_CLAIM_HITS_CACHE: dict = {}+++def _publish_claim_hits(tools_slot, claim_hits: list, doc_id: str) -> int:+ """Mirror navigate_structure's claim hits into the shared evidence pool.++ navigate_structure renders its claims into the outline, so the model sees+ them - but the SCA, the slot prefill and the final compose all read ONLY the+ pool, so a claim that already states the fact was invisible to every+ downstream consumer and the answer stage went back to raw chunks. Publishing+ the same pseudo-chunk the session prefetch writes (identical id scheme, so a+ claim found by either path is one entry) fixes that, and lets the+ claim-vs-chunk dedup in _admit_evidence retire passages a claim already+ quotes verbatim. Pure addition: no retrieval is suppressed.+ """+ import hashlib++ pool = getattr(tools_slot, "kbinfos", None)+ if not isinstance(pool, dict) or not claim_hits:+ return 0+ try:+ chunks = pool.setdefault("chunks", [])+ known = {str(c.get("chunk_id") or "") for c in chunks}+ added = 0+ for h in claim_hits or []:+ name = str(h.get("name") or "").strip()+ if not name:+ continue+ cid = "claim_" + hashlib.md5(f"{doc_id}:{name}".encode("utf-8", "ignore")).hexdigest()[:12]+ if cid in known:+ continue+ content = f"[claim #{h.get('rank') or '?'}] {name}"+ desc = str(h.get("description") or "").strip()+ if desc and desc != name:+ content += f" \u2014 {desc}"+ quote = ""+ for ev in h.get("evidence") or []:+ if isinstance(ev, dict) and str(ev.get("quote") or "").strip():+ quote = str(ev["quote"]).strip()+ break+ if quote:+ content += f'\nEvidence (verbatim): "{quote[:_STRUCT_CLAIM_EVIDENCE_CHARS]}"'+ src = str(h.get("chunk_id") or "").strip()+ chunks.append(+ {+ "chunk_id": cid,+ "content_with_weight": content[:1200],+ "doc_id": doc_id,+ "source_chunk_ids": [src] if src else [],+ }+ )+ known.add(cid)+ added += 1+ if added:+ _LOG.info("[navigate_structure] published %d claim(s) to the evidence pool (doc=%s)", added, doc_id)+ return added+ except Exception: # noqa: BLE001+ _LOG.warning("[navigate_structure] claim publish failed (doc=%s)", doc_id, exc_info=True)+ return 0+++async def _recall_claim_hits(+ tools_slot,+ query: str,+ qvec,+ doc_id: str,+ kinds: set,+ top_n: int,+) -> list[dict]:+ """Claim-level FLAT recall for one document — HYBRID (BM25 + KNN, RRF).++ Two independent legs over the document's claim rows+ (``entity_type_kwd="claim"``): a full-text leg on the tokenized fields+ (exact/proper-noun recall) and a KNN leg on the claim's own vector+ (paraphrase recall), fused by reciprocal rank. RRF consumes ranks only, so+ there is NO similarity threshold to calibrate; nothing is pruned by an early+ pick, the failure mode of descending the tree (SCT: 20.2% vs 39.3% flat).++ Both legs are single store round-trips, so the call stays inside the drill's+ few-second budget (the original full scan — up to 2000 rows fetched + Python+ cosine per row — was cancelled mid-flight and killed the whole drill merge).++ Returns ``[{chunk_id, score, rank, name, description, evidence}]`` ordered+ by fused rank. Empty — and never raising (beyond cancellation, which must+ propagate) — when the document has no compiled claims, no embedding is+ available, or the store fails; the BM25 leg alone keeps the call alive+ without one. With ``_STRUCT_CLAIM_FIRST`` a non-empty result replaces the+ drill-down outright (the claims carry their own evidence).+ """+ if not doc_id or top_n <= 0:+ return []+ import time++ _ckey = (doc_id, str(query or "").strip().lower())+ _now = time.time()+ _hit = _STRUCT_CLAIM_HITS_CACHE.get(_ckey)+ if _hit and _now - _hit[0] < _CLAIM_PREFETCH_CACHE_TTL:+ return _hit[1]+ if len(_STRUCT_CLAIM_HITS_CACHE) >= _CLAIM_PREFETCH_CACHE_CAP:+ _STRUCT_CLAIM_HITS_CACHE.clear()+ from common import settings+ from common.doc_store.doc_store_base import MatchDenseExpr, MatchTextExpr, OrderByExpr+ from common.misc_utils import thread_pool_exec+ from rag.nlp import search++ resolved = tools_slot._resolve_doc_tenant(doc_id)+ if not resolved:+ return []+ kb_id, tenant_id = resolved+ index_name = search.index_name(tenant_id)+ fields = [+ "content_with_weight",+ "source_chunk_ids",+ "entity_type_kwd",+ "compile_kwd",+ ]+ # Both compilers spell their evidence rows ``claim`` now.+ row_types = _evidence_row_types(kinds) if kinds else ("claim",)+ condition: dict = {+ "doc_id": [doc_id],+ "entity_type_kwd": list(row_types),+ # Doc-scope rows only: the Build button's KB-wide merged rows carry a+ # doc_id too, and without this filter they would answer for a single+ # document. Mirrors the dataset-level claim leg.+ "scope_kwd": ["doc"],+ }+ if kinds:+ condition["compile_kwd"] = sorted(str(k) for k in kinds if k)+ limit = max(top_n, 32)++ async def _run(exprs: list) -> list[dict]:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ fields,+ [],+ condition,+ exprs,+ OrderByExpr(),+ 0,+ limit,+ index_name,+ [kb_id],+ )+ rows = settings.docStoreConn.get_fields(res, fields) or {}+ return [h for h in (_parse_claim_row(r) for r in rows.values()) if h]++ dense_leg: list[dict] = []+ text_leg: list[dict] = []+ try:+ if qvec is not None:+ dense = MatchDenseExpr(f"q_{len(qvec)}_vec", [float(v) for v in qvec], "float", "cosine", limit, {})+ dense_leg = await _run([dense])+ except Exception:+ _LOG.exception("[navigate_structure] claim KNN leg failed for doc=%s", doc_id)+ try:+ from rag.advanced_rag.knowlege_compile.dataset_nav import _tokenize++ text = MatchTextExpr(["content_ltks", "content_sm_ltks"], _tokenize(query), limit)+ text_leg = await _run([text])+ except Exception:+ _LOG.exception("[navigate_structure] claim BM25 leg failed for doc=%s", doc_id)++ if not dense_leg and not text_leg:+ return []+ fused = _rrf_fuse(dense_leg, text_leg)+ out = [+ {+ "chunk_id": h["chunk_ids"][0],+ "score": h["score"],+ "rank": h["rank"],+ "name": h["name"],+ "description": h["description"] or h["name"],+ "evidence": [{"quote": h["quote"]}] if h["quote"] else [],+ }+ for h in fused[:top_n]+ ]+ _STRUCT_CLAIM_HITS_CACHE[_ckey] = (time.time(), out)+ return out+++# --- dataset-level claim prefetch (search-time, framework-automatic) ---------+# The claim-first takeover inside navigate_structure only fires when the model+# CHOOSES that tool — which it often does not, defaulting to search_chunks.+# Retrieval priority must not rest on the model's tool pick, so the main search+# driver prefetches claims itself: KB-wide KNN over claim rows, verbatim+# evidence offered at the top of the search result. Floor is looser than the+# per-doc takeover floor (0.5) — here a claim is OFFERED to the model, not+# ruling the selection. Benchmark-tunable.+_CLAIM_PREFETCH_TOP_N = 6+# Small TTL cache: the agent re-issues near-identical queries across turns, and+# each miss costs one KNN round-trip per KB.+_CLAIM_PREFETCH_CACHE: dict = {}+_CLAIM_PREFETCH_CACHE_TTL = 300.0+_CLAIM_PREFETCH_CACHE_CAP = 64+++# ``compile_kwd`` values the knowledge-compilation paths write. A dataset whose+# rows carry none of these has no compiled structure at all, so navigation+# (claim recall, tree routing, in-document drill) can only come back empty.+_COMPILATION_KWDS = ("tree", "page_index", "pageindex", "timeline", "dataset_nav")+_COMPILATION_PROBE_TTL = 300.0+++# "Evidence rows" — atomic propositions carrying verbatim evidence. Every+# compiler writes them as ``entity_type_kwd="claim"``: tree via raptor claim+# extraction, page_index via its own atomic claim type with gate-verified+# verbatim evidence. page_index's pre-rename ``fact``/``conclusion`` spellings+# are deliberately NOT searched — a recompile retypes those rows.+_EVIDENCE_ROW_TYPES_BY_COMPILE = {+ "tree": ("claim",),+ "raptor": ("claim",),+ "page_index": ("claim",),+}+++def _evidence_row_types(compile_kinds) -> tuple:+ """Row types carrying evidence for the given compile kinds."""+ types: list = []+ for kind in compile_kinds or ():+ for t in _EVIDENCE_ROW_TYPES_BY_COMPILE.get(kind, ()):+ if t not in types:+ types.append(t)+ return tuple(types)+++async def dataset_compilation_kinds(tools) -> tuple:+ """Return the compile kinds present in the in-scope KBs.++ One cheap probe per KB (limit=16, id + compile_kwd only), cached on the+ session's ``tools`` object. Best effort: a probe failure falls back to a+ non-empty result so a glitch never silently disables navigation.+ """+ import time++ now = time.time()+ cached = getattr(tools, "_compilation_probe", None)+ if isinstance(cached, tuple) and len(cached) == 2 and now - cached[0] < _COMPILATION_PROBE_TTL:+ return cached[1]++ from common import settings+ from common.doc_store.doc_store_base import OrderByExpr+ from common.misc_utils import thread_pool_exec+ from rag.nlp import search++ kbs = getattr(tools, "kbs", None) or []+ kinds: list = []+ failed = False+ for kb in kbs:+ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ ["doc_id", "compile_kwd"],+ [],+ {"compile_kwd": list(_COMPILATION_KWDS)},+ [],+ OrderByExpr(),+ 0,+ 16,+ [search.index_name(kb.tenant_id)],+ [kb.id],+ )+ rows = settings.docStoreConn.get_fields(res, ["doc_id", "compile_kwd"]) or {}+ for row in rows.values():+ kind = _normalize_kind(row.get("compile_kwd") or "")+ if kind and kind not in kinds:+ kinds.append(kind)+ except Exception: # noqa: BLE001+ _LOG.exception("[compilation probe] failed for kb=%s", getattr(kb, "id", "?"))+ failed = True++ result = tuple(kinds) if kinds else (("tree",) if failed else ())+ try:+ tools._compilation_probe = (now, result)+ except Exception: # noqa: BLE001+ pass+ return result+++async def dataset_has_compilation(tools) -> bool:+ """True when any in-scope KB carries compiled rows (see above)."""+ return bool(await dataset_compilation_kinds(tools))+++async def recall_dataset_claims(tools, query: str, top_n: int = _CLAIM_PREFETCH_TOP_N) -> list[dict]:+ """KB-wide FLAT claim recall — the dataset-level mirror of+ ``_recall_claim_hits``. HYBRID: a BM25 leg and a KNN leg per KB over+ ``entity_type_kwd="claim"`` rows, fused by reciprocal rank — no similarity+ threshold anywhere; the store ranks, we never scan.++ Returns ``[{name, description, quote, doc_id, chunk_ids, score, rank}]``+ ordered by fused rank. Empty (never raising) on any failure — and the BM25+ leg alone keeps recall alive without an embedding model — so the prefetch+ can only ever add to a search result.+ """+ import time++ if not query or top_n <= 0:+ return []+ key = query.strip().lower()+ now = time.time()+ cached = _CLAIM_PREFETCH_CACHE.get(key)+ if cached and now - cached[0] < _CLAIM_PREFETCH_CACHE_TTL:+ return cached[1]+ if len(_CLAIM_PREFETCH_CACHE) >= _CLAIM_PREFETCH_CACHE_CAP:+ _CLAIM_PREFETCH_CACHE.clear()++ from common import settings+ from common.doc_store.doc_store_base import MatchDenseExpr, MatchTextExpr, OrderByExpr+ from common.misc_utils import thread_pool_exec+ from rag.nlp import search++ qvec, dim = await _embed_query(tools, query)+ limit = max(top_n, 32)+ fields = ["content_with_weight", "source_chunk_ids", "doc_id"]+ kbs = getattr(tools, "kbs", None) or []+ row_types = _evidence_row_types(await dataset_compilation_kinds(tools)) or ("claim",)++ async def _run(kb, exprs: list) -> list[dict]:+ condition = {"entity_type_kwd": list(row_types), "scope_kwd": ["doc"]}+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ fields,+ [],+ condition,+ exprs,+ OrderByExpr(),+ 0,+ limit,+ [search.index_name(kb.tenant_id)],+ [kb.id],+ )+ rows = settings.docStoreConn.get_fields(res, fields) or {}+ return [h for h in (_parse_claim_row(r) for r in rows.values()) if h]++ dense_leg: list[dict] = []+ text_leg: list[dict] = []+ for kb in kbs:+ try:+ if qvec is not None:+ dense = MatchDenseExpr(f"q_{dim}_vec", [float(v) for v in qvec], "float", "cosine", limit, {})+ dense_leg.extend(await _run(kb, [dense]))+ except Exception:+ _LOG.exception("[claim prefetch] KNN leg failed for kb=%s", kb.id)+ try:+ from rag.advanced_rag.knowlege_compile.dataset_nav import _tokenize++ text = MatchTextExpr(["content_ltks", "content_sm_ltks"], _tokenize(query), limit)+ text_leg.extend(await _run(kb, [text]))+ except Exception:+ _LOG.exception("[claim prefetch] BM25 leg failed for kb=%s", kb.id)++ if not dense_leg and not text_leg:+ return []+ out = _rrf_fuse(dense_leg, text_leg)[:top_n]+ _CLAIM_PREFETCH_CACHE[key] = (now, out)+ return out+++def _flat_select_nodes(qvec, entities: list[dict], top_n: int) -> list[str]:+ """Score every node at once and return the top-N names.++ page_index spreads facts across TOC levels, so descending level by level can+ prune the branch holding the answer before it is ever scored. Scoring all+ nodes against the query costs nothing extra (their embeddings are already+ loaded) and cannot prune — the same argument as the claim leg, applied to the+ nodes we already hold.++ Returns [] when no query vector is available: the caller then falls back to+ beam descent, which also works from keyword overlap.+ """+ if qvec is None or not entities or top_n <= 0:+ return []+ scored: list[tuple[str, float]] = []+ for e in entities:+ name = (e.get("name") or "").strip()+ vec = e.get("_vec")+ if not name or vec is None:+ continue+ scored.append((name, float(_cosine(qvec, vec))))+ if not scored:+ return []+ scored.sort(key=lambda item: item[1], reverse=True)+ return [name for name, _score in scored[:top_n]]++def _build_toc_items(entities: list[dict], relations: list[dict]) -> list[dict]:"""Serialize the whole TOC, indented by depth, for one-shot LLM selection.@@ -1602,10 +2115,22 @@ async def _render_toc_drilldown(relations: list[dict],chunk_hits: list[tuple[str, float]] | None = None,selected: list[str] | None = None,+ claim_hits: list[dict] | None = None,) -> tuple[str, dict, dict]:"""Render a query-focused outline of one document's compiled structure.- Three ways to choose what to show, in the order the caller tries them:+ ``claim_hits`` comes from ``_recall_claim_hits``: gated claims that matched+ the query directly. They are rendered as statement + verbatim evidence and+ their chunk is skipped in the snippet list, so a matched fact is not also+ echoed as a truncated chunk. With ``_STRUCT_CLAIM_FIRST`` a hit set does+ more than that — it takes over the selection and nothing is drilled.++ Four ways to choose what to show. Claims win when present; the other three+ are then tried in the order the caller uses them:++ claims — flat-scored atomic propositions with verbatim evidence. A hit is+ already the answer material, so no descent happens and no chunk snippets+ are loaded; the tree only labels where each claim sits.``selected`` — node names already chosen elsewhere (the whole-TOC LLM pass,see ``_select_toc_nodes``). Every level was visible to the model at once,@@ -1655,7 +2180,18 @@ async def _render_toc_drilldown(return namesselected_ids: list[str] = []- if selected:+ if _STRUCT_CLAIM_FIRST and len(claim_hits or []) >= _STRUCT_CLAIM_FIRST_MIN_HITS:+ # Claims already carry the answer material (statement + verbatim+ # evidence), so nothing is drilled: the tree's only remaining job is to+ # show where each claim sits. Descending would re-derive the same facts+ # and risk pruning past them.+ claim_cids = {str(h.get("chunk_id") or "").strip() for h in claim_hits or []}+ claim_cids.discard("")+ kept_names = _with_ancestors({n for n in _nodes_covering_chunks(entities, claim_cids) if n in by_name})+ kept_nodes = [by_name[n] for n in kept_names if n in by_name]+ best = max((float(h.get("score") or 0.0) for h in claim_hits or []), default=0.0)+ selector = "claim"+ elif selected:kept_names = _with_ancestors({n for n in selected if n in by_name})kept_nodes = [by_name[n] for n in kept_names if n in by_name]# The model picked nodes, so the chunks are theirs to supply.@@ -1676,7 +2212,10 @@ async def _render_toc_drilldown(else:kept_nodes, parents, kept_names, best = _drill_kept_nodes(query_terms, qvec, entities, relations)selector = "beam"- if not kept_nodes and not selected_ids:+ # Claim-first still has something to show with no nodes at all — upper RAPTOR+ # clusters carry no source_chunk_ids, so no node may cover a hit claim, and+ # bailing out here would throw the claims away.+ if not kept_nodes and not selected_ids and not (_STRUCT_CLAIM_FIRST and claim_hits):return _render_outline(entities[:_STRUCT_MAX_NODES], relations[:_STRUCT_MAX_NODES]), _outline_stats(entities), {}# Render the drilled nodes (indented by depth via ancestor count).@@ -1722,11 +2261,50 @@ async def _render_toc_drilldown(line += f" [chunks: {chunks}]"lines.append(line)+ # Claim/evidence lines come before the chunk snippets: a gated claim is a+ # verified atomic fact carrying its verbatim quote, so it reads better than a+ # truncated chunk and needs no chunk load at all. Its chunk is dropped from+ # the snippet list below so the same content is not sent twice.+ claimed: set[str] = set()+ for hit in claim_hits or []:+ cid = str(hit.get("chunk_id") or "").strip()+ if not cid or cid in claimed:+ continue+ claimed.add(cid)+ quote = ""+ for ev in hit.get("evidence") or []:+ if isinstance(ev, dict) and str(ev.get("quote") or "").strip():+ quote = str(ev["quote"]).strip()+ break+ line = f"- [claim] {str(hit.get('name') or '').strip()}"+ if quote:+ line += f' | Evidence: "{_snippet(quote, _STRUCT_CLAIM_EVIDENCE_CHARS)}"'+ line += f" [chunks: {cid}]"+ lines.append(line)++ # A claim's chunk still deserves a structural label — annotate it from the+ # node that covers it, falling back to nothing when no node does (upper+ # RAPTOR clusters carry no source_chunk_ids, so this stays honest rather than+ # inventing a path).+ for cid in claimed:+ if cid in chunk_paths:+ continue+ owners = [n for n in _nodes_covering_chunks(entities, {cid}) if n in by_name]+ if owners:+ chunk_paths[cid] = owners[0]+# Chunk snippets. When chunk retrieval did the selecting, those hits are# returned as-is: filtering them back through the nodes would drop every# chunk no node happens to cover. Otherwise take the chunks behind the# drilled nodes.- wanted = selected_ids or _collect_chunk_ids(kept_nodes)+ if _STRUCT_CLAIM_FIRST and len(claim_hits or []) >= _STRUCT_CLAIM_FIRST_MIN_HITS:+ # A matched claim already renders its verbatim evidence; loading the+ # chunk behind it would echo the same passage a second time.+ wanted = []+ else:+ wanted = selected_ids or _collect_chunk_ids(kept_nodes)+ if wanted:+ wanted = [c for c in wanted if c not in claimed]if wanted:chunks = await _load_chunks_for_ids(_tools_slot(), wanted)if chunks:@@ -1746,6 +2324,9 @@ async def _render_toc_drilldown("nodes": len(kept_nodes),"chunk_ptrs": len(wanted),"top_score": round(float(best), 4),+ # Best claim cosine, surfaced so the caller can emit <claims_sufficient/>+ # when the matched claims are strong enough to cite as answer material.+ "claim_top": round(max((float(h.get("score") or 0.0) for h in claim_hits or []), default=0.0), 4),# Which strategy chose the nodes. top_score is only meaningful for# "beam" and "chunk_retrieval"; llm_toc has no comparable score."selector": selector,diff --git a/rag/advanced_rag/knowlege_compile/_common.py b/rag/advanced_rag/knowlege_compile/_common.pyindex f7243c5753..3127f38ef2 100644--- a/rag/advanced_rag/knowlege_compile/_common.py+++ b/rag/advanced_rag/knowlege_compile/_common.py@@ -111,6 +111,21 @@ def knowledge_compile_gen_conf(chat_mdl, gen_conf: Optional[dict] = None) -> dic# -preview variants (e.g. qwen3.8-max-preview) only accept# enable_thinking=True on their API endpoint.conf["enable_thinking"] = True if "-preview" in model_name else False+ elif any(k in model_name for k in ("minimax-m", "minimax_m", "minimax m", "abab", "minimax-m3", "minimax-m1")):+ # MiniMax-M3/M1 controls thinking via `thinking: {"type": "disabled"}`+ # in the request body. `thinking` is NOT a standard OpenAI-compatible+ # param and LiteLLM's drop_params=True silently drops unknown top-level+ # kwargs, so it MUST ride in extra_body (merged verbatim into the body).+ # Without this, MiniMax reasoning models keep chain-of-thought enabled,+ # which makes extraction slow and can hang a batch on a long COT. The+ # rest of this file routes through the generic `else` below only when+ # the model name gives no hint, so the broad `minimax` guard above is+ # enough to cover every MiniMax model this repo is configured against.+ extra_body = conf.get("extra_body")+ extra_body = dict(extra_body) if isinstance(extra_body, dict) else {}+ extra_body["thinking"] = {"type": "disabled"}+ conf["extra_body"] = extra_body+ conf.pop("reasoning_effort", None)else:# LiteLLM maps this common control for providers that support it and# drops it for providers that do not. Keep model-specific overridesdiff --git a/rag/advanced_rag/knowlege_compile/raptor.py b/rag/advanced_rag/knowlege_compile/raptor.pyindex b9c33d7fbe..934f6004ea 100644--- a/rag/advanced_rag/knowlege_compile/raptor.py+++ b/rag/advanced_rag/knowlege_compile/raptor.py@@ -15,6 +15,7 @@#import asyncioimport logging+import osimport reimport numpy as np@@ -24,6 +25,7 @@ from common.connection_utils import timeoutfrom common.exceptions import TaskCanceledExceptionfrom common.token_utils import truncatefrom rag.graphrag.utils import (+ LoopLocalSemaphore,chat_limiter,get_embed_cache,get_llm_cache,@@ -34,6 +36,359 @@ from common.misc_utils import thread_pool_execfrom ._common import knowledge_compile_gen_conf+# Claim extraction has its own concurrency budget, decoupled from the global+# chat_limiter (which also serves clustering and embedding). MiniMax's API only+# tolerates ~4 concurrent requests, so default to 4 and let it be overridden via+# MAX_CONCURRENT_CLAIM_CHATS. On rate-limit errors each call backs off and retries+# instead of letting the limiter itself scale up and hammer the API.+_claim_limiter = LoopLocalSemaphore(int(os.environ.get("MAX_CONCURRENT_CLAIM_CHATS", 4)))++# Claim extraction runs before clustering so the cluster summaries can be built+# from the claims of their member chunks instead of the raw chunk text (see+# ``build_doc_tree`` and rag/advanced_rag/knowlege_compile/claim_evidence.md+# §7). Claims are the atoms the tree is later read through; the raw text is only+# used as a fallback when a chunk yields none.+#+# Claims are extracted in small fixed-size batches (2-4 chunks per call, see+# ``_pack_claim_batches``) run in parallel under a dedicated concurrency budget+# (``_claim_limiter``). Each chunk in a batch is marked as a TARGET,+# and every claim is attributed to the chunk it was quoted from; the validation+# gate then drops any quote that cannot be located in that chunk. One chunk per+# call trades LLM-call count for per-target recall and exact quote validation.++# The claim-extraction system prompt. ``extract_claims_for_chunks`` renders one+# or more ``<TARGET>`` chunks; each is a chunk the model must harvest claims+# from. Every claim carries the id of the chunk it was taken from, so quotes are+# validated against the right text and cross-chunk contamination is caught by+# the gate.+_CLAIM_EXTRACTION_PROMPT = """## Task+You are a high-recall claim harvester for the TARGET chunks below. For EACH+TARGET chunk, extract EVERY explicit atomic claim that chunk supports: factual+assertions AND explicitly expressed opinions, beliefs, judgments, assessments,+recommendations, preferences, intentions, predictions, and hypotheses. Do not+rank claims, summarize, merge claims, or omit a claim because it seems minor.++A claim must be:+- Self-contained: readable without the surrounding text, with named subjects.+ Never write "the article", "the document", "it says", or a bare pronoun.+ Resolve the referent to its name when attribution makes the claim clearer.+- Faithful: stated only if that TARGET chunk supports it. Never invent,+ strengthen, or infer. Preserve modality, uncertainty, negation, and+ attribution exactly.+- Atomic: exactly one fact. Split compound sentences.++## Response Format+Reply with a single JSON object: {"items": [{"type": "claim", "name": "<the claim, one sentence>", "description": "<optional restatement for retrieval>", "source_chunk_ids": ["<CHUNK_ID it was taken from>"], "evidence": [{"quote": "<the verbatim source sentence>", "chunk_id": "<CHUNK_ID it was taken from>"}]}, ...]}.++Rules:+- `evidence.quote` MUST be a CONTIGUOUS verbatim substring of the chunk cited+ by `evidence.chunk_id`: same words, same order, no paraphrase, no truncation,+ no added words. A quote that cannot be found verbatim is rejected downstream,+ so never restate.+- `evidence.chunk_id` and `source_chunk_ids` MUST identify the exact chunk the+ claim and quote came from. Never cross-attribute a quote to a different chunk.+- Keep each quote concise and under 240 characters.+- For tables, infoboxes and bullet lists, the quote MUST be the raw cell/row+ text exactly as it appears — keep its separators and order. Do NOT turn a+ table row like "Starring | Penn Badgley | Elizabeth Lail" into a sentence+ like "Starring Penn Badgley Elizabeth Lail"; that is a paraphrase and will+ be rejected.+- Preserve numbers, units, dates, names and qualifiers exactly.+- Distribute claims across all TARGET chunks; do not skip a chunk because you+ reached the cap — the cap is per chunk, not per batch.+- If a TARGET chunk contains no extractable assertion, emit no claim for it.+- Keep claims in the same language as the source.+Return JSON only, no commentary."""+++def _render_claim_source(batch: list[tuple]) -> str:+ """Render the user prompt body for one claim-extraction call.++ Every chunk in the batch is marked as a TARGET, so the model harvests claims+ for all of them in one call. Each chunk carries its own id, which both pins+ the claim attribution and lets the validation gate check the quote against+ the right text.+ """+ lines = ["## Source Text"]+ for cid, text in batch:+ lines.append(f"[CHUNK_ID: {cid} (TARGET)]")+ lines.append(text)+ lines.append("[END_CHUNK]")+ lines.append("\n## Output (JSON only):")+ return "\n".join(lines)+++# Chunks per claim-extraction call. Empirically 2-4 is the sweet spot: a single+# chunk under-utilises the call (one clean target but ~75s), while packing too+# many raises latency superlinearly and makes the model drop trailing chunks.+# 4 * ~512 tok = ~2k tok/call, which stays inside the attention window.+_CLAIM_BATCH_SIZE = 4+++def _pack_claim_batches(entries: list[tuple]) -> list[list[tuple]]:+ """Group chunks into fixed-size batches (``_CLAIM_BATCH_SIZE``).++ ``entries`` is ``(chunk_id, text)``. Batches of 2-4 chunks cut the LLM-call+ count several-fold versus one chunk per call while keeping each call small+ enough that the model still covers every target (see _CLAIM_BATCH_SIZE).+ """+ bs = max(1, int(_CLAIM_BATCH_SIZE))+ return [entries[i : i + bs] for i in range(0, len(entries), bs)]+++async def extract_claims_for_chunks(+ chunks: list[tuple],+ llm_model,+ *,+ task_id: str = "",+ callback=None,+ claim_prompt: str | None = None,+) -> dict[str, list[dict]]:+ """Extract claim/evidence pairs for RAPTOR's layer-0 chunks.++ ``chunks`` is the ``(text, vec, source_chunk_ids)`` list handed to the tree+ builder. Returns ``{chunk_id: [claim_payload, ...]}`` keyed by the chunk the+ claim came from, so the builder can look up a cluster's claims by its+ members' ids.++ ``claim_prompt`` comes from the tree compilation template's ``raptor``+ section so the extraction contract is editable per template; ``None`` falls+ back to the built-in contract.++ Chunks are grouped into fixed-size batches (see ``_pack_claim_batches``) and+ all batches run in parallel, gated by the shared ``chat_limiter`` so the+ LLM-call concurrency stays bounded. Each claim is attributed to the chunk it+ was quoted from (model-provided ``source_chunk_ids`` filtered to the batch,+ then the evidence gate validates the quote against that chunk's text).++ Best-effort: extraction never fails the build. On error a batch simply+ contributes no claims and the tree falls back to raw text for it.+ """++ if not chunks or llm_model is None:+ return {}++ entries = []+ for c in chunks:+ text = c[0]+ ids = c[2] if len(c) > 2 and isinstance(c[2], (list, tuple)) else []+ cid = next((str(s) for s in ids if s), "")+ if text and cid:+ entries.append((cid, text))+ if not entries:+ return {}++ batches = _pack_claim_batches(entries)+ if callback:+ callback(msg=f"tree-template: claim extraction start: {len(entries)} chunk(s) -> {len(batches)} batch(es)")++ claims_by_chunk: dict[str, list[dict]] = {}++ total_chunks = len(entries)+ if not total_chunks:+ return claims_by_chunk++ # Fan every batch's claim extraction out in parallel. Each task is gated by+ # its own concurrency budget (_claim_limiter, default 4) so we never exceed+ # the LLM API's limit even though we launch one task per batch. This turns+ # the old serial loop into ~ceil(batches / concurrency) rounds.+ tasks = []+ batch_size_of: dict = {}+ for batch in batches:+ # Hand each batch only its own text. The gate falls back to scanning+ # every chunk it is given when a quote cites none, so passing the whole+ # document would let a quote match an unrelated chunk that merely+ # happens to contain the same sentence — and would silently attribute+ # the claim to the wrong source.+ batch_text_by_id = {cid: text for cid, text in batch}+ t = asyncio.create_task(_extract_claim_for_chunk(batch, llm_model, batch_text_by_id, claim_prompt))+ tasks.append(t)+ batch_size_of[t] = len(batch)+ processed = 0+ try:+ # asyncio.as_completed returns an ASYNC iterator (3.10+); a plain ``for``+ # would never await ``_wait_for_one`` and silently break extraction.+ async for coro in asyncio.as_completed(tasks):+ # Progress counts chunks (the user-visible unit), not batches — a+ # batch holds _CLAIM_BATCH_SIZE chunks, so show the real figure.+ processed += batch_size_of[coro]+ if callback:+ callback(prog=processed / total_chunks, msg=f"tree-template: extracting claims for chunk {processed}/{total_chunks}")+ try:+ result = await coro+ except TaskCanceledException:+ for t in tasks:+ t.cancel()+ raise+ except Exception as exc:+ logging.warning(f"[RAPTOR] claim extraction failed for a chunk: {exc}")+ continue+ if result:+ _, claims = result+ # A batch covers several chunks, so group each claim under its+ # own source chunk id (validated in _extract_claim_for_chunk).+ for cl in claims:+ cid = (cl.get("source_chunk_ids") or [None])[0]+ if cid:+ claims_by_chunk.setdefault(cid, []).append(cl)+ finally:+ for t in tasks:+ if not t.done():+ t.cancel()++ if callback:+ callback(prog=1.0, msg=f"Extracted claims for {len(claims_by_chunk)} chunk(s)")+ return claims_by_chunk+++# Substrings that mean "slow down and retry" rather than "permanent failure".+_RETRYABLE_LLM_ERR = (+ "rate limit",+ "429",+ "tpm limit",+ "too many requests",+ "requests per minute",+ "server",+ "503",+ "502",+ "504",+ "500",+ "unavailable",+ "timeout",+ "timed out",+)+++async def _extract_claim_for_chunk(batch, llm_model, text_by_id, claim_prompt: str | None = None):+ """Run claim extraction for one batch of chunks (``_CLAIM_BATCH_SIZE`` of them).++ One LLM call, gated by the shared ``chat_limiter`` so total concurrency stays+ bounded. On a rate-limit / server / timeout error the call backs off with+ exponential delay and retries, then gives up and returns ``None`` so the+ tree falls back to raw text for those chunks.++ ``claim_prompt`` is the template-declared system prompt (tree.yaml's+ ``raptor.claim_prompt``); ``None`` uses the built-in contract.++ Returns ``(label, [claim_payload, ...])`` when the batch yields claims (each+ claim carries its own validated ``source_chunk_ids``), else ``None``.+ """+ from rag.prompts.generator import gen_json+ from .structure import _struct_apply_evidence_gate++ user = _render_claim_source(batch)+ batch_ids = {cid for cid, _ in batch}++ ans = None+ attempt = 0+ while True:+ try:+ async with _claim_limiter:+ ans = await gen_json(+ claim_prompt or _CLAIM_EXTRACTION_PROMPT,+ user,+ llm_model,+ knowledge_compile_gen_conf(llm_model),+ )+ break+ except TaskCanceledException:+ raise+ except Exception as exc:+ es = str(exc).lower()+ retryable = any(k in es for k in _RETRYABLE_LLM_ERR)+ attempt += 1+ if not retryable or attempt >= 3:+ logging.warning(f"[RAPTOR] claim extraction gave up for batch of {len(batch)}: {exc}")+ return None+ # Exponential back-off + jitter: slow down so the API stops+ # rejecting us, rather than hammering it while it recovers.+ delay = 2.0 * (2 ** (attempt - 1)) * (0.7 + 0.6 * ((attempt * 13) % 10) / 10)+ logging.warning(f"[RAPTOR] claim extraction retry {attempt}/3 after {delay:.1f}s: {exc}")+ await asyncio.sleep(delay)++ items = (ans or {}).get("items") if isinstance(ans, dict) else None+ if not isinstance(items, list):+ return None++ claims = []+ for it in items:+ if not isinstance(it, dict):+ continue+ name = str(it.get("name") or "").strip()+ if not name:+ continue+ # A batch holds several chunks, so the model must say which chunk a claim+ # came from. Keep only ids that are actually in this batch (a hallucinated+ # id pointing at text the model never saw cannot be validated). Missing+ # attribution is recovered from the verified evidence below; a claim that+ # neither the model nor its evidence can place is dropped.+ src = [str(s) for s in (it.get("source_chunk_ids") or []) if s]+ it["name"] = name+ it["type"] = "claim"+ it["source_chunk_ids"] = [s for s in src if s in batch_ids]+ if not it.get("description"):+ it["description"] = name+ claims.append(it)++ if not claims:+ return None+ # Split "the model gave us no quote" from "the gate rejected the quote", so+ # a low evidence yield can be attributed correctly.+ emitted = sum(1 for cl in claims if cl.get("evidence"))+ verified, rejected = _struct_apply_evidence_gate(claims, text_by_id, "soft")+ logging.info(+ "[RAPTOR] claim extraction batch=%s chunks=%d claims=%d emitted_evidence=%d verified=%d rejected=%d",+ ",".join(batch_ids),+ len(batch),+ len(claims),+ emitted,+ verified,+ rejected,+ )++ # Recover attribution from the evidence that survived the gate: the chunk a+ # quote was located in IS where the claim came from. Falling back to+ # batch[0][0] instead would file every unattributed claim under an arbitrary+ # chunk — and after rechunking, under one that no longer exists.+ attributed: list[dict] = []+ for cl in claims:+ if cl.get("source_chunk_ids"):+ attributed.append(cl)+ continue+ derived = next(+ (e.get("chunk_id") for e in cl.get("evidence") or [] if isinstance(e, dict) and e.get("chunk_id") in batch_ids),+ None,+ )+ if derived is None:+ logging.info("[RAPTOR] dropped claim with no attributable source: %s", cl.get("name"))+ continue+ cl["source_chunk_ids"] = [derived]+ attributed.append(cl)++ if not attributed:+ return None+ return batch[0][0], attributed+++def format_claims_for_summary(claims: list[dict]) -> str:+ """Render a chunk's claims as the summary input for its cluster.++ Each claim is rendered with the verbatim quote that backs it, so the+ abstraction above sees the facts *and* their grounding rather than a+ reflowed paraphrase.+ """+ lines = []+ for c in claims:+ name = (c.get("name") or "").strip()+ if not name:+ continue+ quotes = [(e or {}).get("quote", "") for e in (c.get("evidence") or []) if isinstance(e, dict) and e.get("quote")]+ if quotes:+ lines.append(f'- {name}\n Evidence: "{quotes[0]}"')+ else:+ lines.append(f"- {name}")+ return "\n".join(lines)+class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval:"""Build RAPTOR summary layers with the classic or Psi tree strategy."""@@ -261,6 +616,35 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval:raise RuntimeError(f"RAPTOR aborted after {self._error_count} errors. Last error: {exc}") from excreturn None+ @staticmethod+ def _cluster_input_texts(+ ck_idx: list[int],+ chunks: list,+ claims_by_chunk: dict[str, list[dict]] | None,+ n_originals: int,+ ) -> list[str]:+ """Build the summary input for one cluster.++ Layer-0 chunks (``i < n_originals``) that yielded claims contribute+ their rendered claims; anything else — upper-layer summaries, and+ chunks with no claims — contributes its text. So only the bottom of the+ tree is claim-fed, and the abstraction layers above keep summarizing+ the (now claim-derived) summaries beneath them.+ """+ if not claims_by_chunk:+ return [chunks[i][0] for i in ck_idx]+ texts = []+ for i in ck_idx:+ rendered = ""+ if i < n_originals:+ ids = chunks[i][2] if len(chunks[i]) > 2 else []+ cid = next((str(s) for s in ids if s), "")+ claims = claims_by_chunk.get(cid) if cid else None+ if claims:+ rendered = format_claims_for_summary(claims)+ texts.append(rendered or chunks[i][0])+ return texts+async def __call__(self,chunks,@@ -268,9 +652,18 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval:callback=None,task_id: str = "",is_tree: bool = False,+ claims_by_chunk: dict[str, list[dict]] | None = None,):"""Build summary chunks and layer boundaries for RAPTOR retrieval.+ ``claims_by_chunk`` maps a layer-0 chunk id to the claims extracted from+ it (see ``extract_claims_for_chunks``). When supplied, a cluster's+ summary input is built from its members' claims instead of their raw+ text — claims are far more compact than the source, so the per-chunk+ truncation in ``_summarize_texts`` no longer discards content, and the+ resulting abstraction is guaranteed to agree with the claims attached to+ the same cluster. Chunks with no claims fall back to their raw text.+``chunks`` accepts either the legacy 2-tuple shape``(text, vec)`` or the provenance-carrying 3-tuple shape``(text, vec, source_chunk_ids)`` where ``source_chunk_ids`` is@@ -339,7 +732,7 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval:"""nonlocal chunks- texts = [chunks[i][0] for i in ck_idx]+ texts = self._cluster_input_texts(ck_idx, chunks, claims_by_chunk, n_originals)result = await self._summarize_texts(texts, callback, task_id)if result is not None:# ``dict.fromkeys`` is the cheapest way to de-dup adiff --git a/rag/advanced_rag/knowlege_compile/structure.py b/rag/advanced_rag/knowlege_compile/structure.pyindex 7a447b02ce..2b5e36d68a 100644--- a/rag/advanced_rag/knowlege_compile/structure.py+++ b/rag/advanced_rag/knowlege_compile/structure.py@@ -18,6 +18,7 @@ import asyncioimport heapqimport jsonimport logging+import osimport reimport timeimport uuid@@ -479,13 +480,25 @@ def _struct_render_fields(fields: list, language: str) -> Tuple[str, str]:return "\n".join(lines), "{ " + ", ".join(skeleton_parts) + " }"-def _struct_render_type_fields(fields: list, language: str, *, kind: str) -> Tuple[str, str]:+def _struct_render_type_fields(+ fields: list,+ language: str,+ *,+ kind: str,+ extra_fields: list | None = None,+) -> Tuple[str, str]:"""Render the new compilation-template field shape.New templates define allowed item ``type`` values with descriptions/rules,rather than arbitrary output field names. The extraction output keeps astable shape so downstream merge logic can compare concrete items insteadof collapsing every item into the template type.++ ``extra_fields`` declares additional output keys a template may ask for+ (claim/evidence compilation adds ``evidence``). The Response Format below+ is what actually fixes the model's output shape — a key described only in+ a type's ``rule`` text but absent from the skeleton is silently dropped by+ the model — so extra keys must be rendered into the skeleton here."""lines: list[str] = []type_values: list[str] = []@@ -509,21 +522,80 @@ def _struct_render_type_fields(fields: list, language: str, *, kind: str) -> Tuptype_values.append("other")lines.append("- type: other")+ # Extra output keys the template declared. Rendered into the skeleton so the+ # model actually emits them, and described so it knows their shape.+ extra_parts: list[str] = []+ for ef in extra_fields or []:+ if not isinstance(ef, dict):+ continue+ name = ef.get("name")+ name = name.strip() if isinstance(name, str) else ""+ if not name:+ continue+ desc = _struct_localize(ef.get("description"), language)+ optional = "optional, omit when not applicable" if ef.get("required") is False else "required"+ header = f"- {name} ({ef.get('type', 'str')}, {optional})"+ lines.append(f"{header}: {desc}" if desc else header)+ shape = _struct_localize(ef.get("shape"), language)+ if shape:+ extra_parts.append(f'"{name}": {shape}')+ else:+ ftype = ef.get("type", "str")+ placeholder = {+ "list": "[...]",+ "int": "<int>",+ "float": "<float>",+ "bool": "<true|false>",+ }.get(ftype, "<string>")+ extra_parts.append(f'"{name}": {placeholder}')+ extra_skeleton = (", " + ", ".join(extra_parts)) if extra_parts else ""+if kind == "relation":skeleton = ('{ "type": "<one of: '+ "|".join(type_values)- + '>", "source": "<known entity name>", "target": "<known entity name>", "description": "<evidence or relation description>", "source_chunk_ids": ["<source chunk id>", ...] }'+ + '>", "source": "<known entity name>", "target": "<known entity name>", "description": "<evidence or relation description>", "source_chunk_ids": ["<source chunk id>", ...]'+ + extra_skeleton+ + " }")else:skeleton = ('{ "type": "<one of: '+ "|".join(type_values)- + '>", "name": "<exact extracted item text>", "description": "<evidence, definition, or detail from the source>", "source_chunk_ids": ["<source chunk id>", ...] }'+ + '>", "name": "<exact extracted item text>", "description": "<evidence, definition, or detail from the source>", "source_chunk_ids": ["<source chunk id>", ...]'+ + extra_skeleton+ + " }")return "\n".join(lines), skeleton+# Optional ceiling on chunks per extraction call (0/unset = pack by the context+# window only).+#+# This must NOT be left on for structure extraction. A heading's+# ``source_chunk_ids`` can only cover the chunks the call was shown, so a small+# ceiling truncates every section at the batch boundary: the hierarchy collapses+# to two levels, headings fragment into single words, and detail items in later+# batches share no chunk with their heading at all. Headings need a+# document-wide view; only claim harvesting wants small batches (raptor uses 4).+# Overridable for benchmarking.+_STRUCT_MAX_CHUNKS_PER_BATCH = int(os.environ.get("STRUCT_MAX_CHUNKS_PER_BATCH", "0"))+++def _struct_type_counts(items: list[dict]) -> dict[str, str]:+ """``{type: "<count>/<count carrying evidence>"}`` for the extraction log."""+ out: dict[str, list[int]] = {}+ for it in items or []:+ if not isinstance(it, dict):+ continue+ t = str(it.get("type") or "?").strip() or "?"+ row = out.setdefault(t, [0, 0])+ row[0] += 1+ if it.get("evidence"):+ row[1] += 1+ return {k: f"{v[0]}/{v[1]}" for k, v in sorted(out.items())}++def _struct_hypergraph_prompts(parser_config: dict, language: str = "en", rechunk: bool = False) -> Tuple[str, str]:autotype = _struct_infer_type(parser_config)guideline = _struct_get(parser_config, "guideline", default={}) or {}@@ -549,8 +621,18 @@ def _struct_hypergraph_prompts(parser_config: dict, language: str = "en", rechunent_fields = _struct_get(entities_cfg, "fields", default=[]) or []rel_fields = _struct_get(relations_cfg, "fields", default=[]) or []if uses_template_shape:- ent_fields_text, ent_skel = _struct_render_type_fields(ent_fields, language, kind="entity")- rel_fields_text, rel_skel = _struct_render_type_fields(rel_fields, language, kind="relation")+ ent_fields_text, ent_skel = _struct_render_type_fields(+ ent_fields,+ language,+ kind="entity",+ extra_fields=_struct_get(entities_cfg, "output_fields", default=[]) or [],+ )+ rel_fields_text, rel_skel = _struct_render_type_fields(+ rel_fields,+ language,+ kind="relation",+ extra_fields=_struct_get(relations_cfg, "output_fields", default=[]) or [],+ )else:ent_fields_text, ent_skel = _struct_render_fields(ent_fields, language)rel_fields_text, rel_skel = _struct_render_fields(rel_fields, language)@@ -735,8 +817,46 @@ async def _struct_extract_hypergraph(if isinstance(raw_ids, list):node["source_chunk_ids"] = [temp_to_uuid.get(str(item).strip(), str(item).strip()) for item in raw_ids if temp_to_uuid.get(str(item).strip())]+ if not edge_prompt_template:+ return nodes, [], chunk_id_map, rechunked_chunks++ logging.info(+ "compile_structure_from_text: node pass done -> %d item(s); building relations",+ len(nodes),+ )+ edges = await _struct_extract_relations(+ relation_text if rechunk else text,+ parser_config,+ chat_mdl,+ language,+ nodes,+ rechunk=rechunk,+ chunk_id_map=chunk_id_map,+ )++ return nodes, edges, chunk_id_map, rechunked_chunks+++async def _struct_extract_relations(+ text: str,+ parser_config: dict,+ chat_mdl,+ language: str,+ nodes: list[dict],+ rechunk: bool = False,+ chunk_id_map: dict[str, str] | None = None,+) -> list[dict]:+ """Relation pass over already-extracted nodes.++ The edge prompt's ``{known_nodes}`` placeholder is filled from the nodes+ extracted in the same batch, so relations are always built after nodes.+ """+ _, edge_prompt_template = _struct_hypergraph_prompts(parser_config, language, rechunk=rechunk)+ if not edge_prompt_template:+ return []+id_field = _struct_entity_id_field(parser_config)- known_keys = []+ known_keys: list[str] = []for n in nodes:v = n.get(id_field)if v is None:@@ -746,32 +866,192 @@ async def _struct_extract_hypergraph(known_keys.append(v_str)known_str = "- " + "\n- ".join(known_keys) if known_keys else "(none)"- if not edge_prompt_template:- return nodes, [], chunk_id_map, rechunked_chunks-edge_prompt = edge_prompt_template.replace("{known_nodes}", known_str)edge_user_prompt = (- user_prompt- if not rechunk- else (- "## Source Text:\n"- "Each source chunk is enclosed by [CHUNK_ID: ...] and [END_CHUNK]. "- "For every relation, return source_chunk_ids containing only the IDs of chunks that support that relation.\n"- f"{relation_text}\n\n## Output (JSON only):"- )+ "## Source Text:\n"+ "Each source chunk is enclosed by [CHUNK_ID: ...] and [END_CHUNK]. "+ "For every relation, return source_chunk_ids containing only the IDs of chunks that support that relation.\n"+ f"{text}\n\n## Output (JSON only):")edge_res = await gen_json(edge_prompt, edge_user_prompt, chat_mdl, gen_conf=_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.1}))edges = _struct_unwrap_items(edge_res)- if rechunk:+ if rechunk and chunk_id_map:+ valid = set(chunk_id_map.values())for edge in edges:raw_ids = edge.get("source_chunk_ids")if isinstance(raw_ids, str):raw_ids = [raw_ids]if isinstance(raw_ids, list):- edge["source_chunk_ids"] = [item for item in raw_ids if isinstance(item, str) and item in set(chunk_id_map.values())]+ edge["source_chunk_ids"] = [item for item in raw_ids if isinstance(item, str) and item in valid]- return nodes, edges, chunk_id_map, rechunked_chunks+ return edges+++# Claim/evidence compilation (see claim_evidence.md).+#+# Evidence is a verbatim quote plus the character span it was found at:+# {"quote": str, "chunk_id": str, "start": int, "end": int}+# The quote must be locatable in the chunk it cites; anything that cannot be+# located is discarded rather than trusted, which is the whole point of the+# compiled form.+_EVIDENCE_GATE_MODES = ("soft", "hard")++# Default is soft: drop the unverified evidence but keep the claim. The ISC+# paper drops the claim, but that was validated on single-document dialogue QA;+# enumeration / multi-hop questions degrade badly when data is discarded, so+# hard mode is opt-in via the template's ``evidence_gate_mode``.+_EVIDENCE_GATE_DEFAULT = "soft"+++def _struct_evidence_gate_mode(parser_config: dict) -> str:+ mode = parser_config.get("evidence_gate_mode") if isinstance(parser_config, dict) else None+ if isinstance(mode, str) and mode.strip().lower() in _EVIDENCE_GATE_MODES:+ return mode.strip().lower()+ return _EVIDENCE_GATE_DEFAULT+++def _struct_relation_expects_evidence(parser_config: dict) -> bool:+ """Whether relation items are expected to carry evidence.++ Only templates that list ``evidence`` under ``relation.output_fields`` ask+ the model for relation evidence. Gating relations unconditionally would+ strip the field from payloads that legitimately have none, changing the+ stored shape for every configuration that never asked for it.+ """+ relations_cfg = _struct_get(parser_config, "relation", default={}) or {}+ if not isinstance(relations_cfg, dict):+ return False+ fields = _struct_get(relations_cfg, "output_fields", default=[]) or []+ if not isinstance(fields, (list, tuple)):+ return False+ return any(isinstance(f, str) and f.strip().lower() == "evidence" for f in fields)+++def _struct_normalize_for_match(text: str) -> tuple[str, list[int]]:+ """Collapse whitespace runs, returning the normalized text and, for each+ normalized index, the index of the character it came from.++ Matching against whitespace-normalized text tolerates the reflowing an LLM+ introduces when it copies a sentence, while the index map lets the caller+ report offsets against the original chunk text.+ """+ chars: list[str] = []+ index_map: list[int] = []+ prev_space = False+ for i, ch in enumerate(text or ""):+ if ch.isspace():+ if prev_space:+ continue+ chars.append(" ")+ index_map.append(i)+ prev_space = True+ else:+ chars.append(ch)+ index_map.append(i)+ prev_space = False+ return "".join(chars), index_map+++def _struct_locate_evidence(quote: str, text: str) -> tuple[int, int] | None:+ """Locate ``quote`` in ``text``, returning (start, end) or None."""+ if not quote or not text:+ return None+ norm_text, index_map = _struct_normalize_for_match(text)+ norm_quote, _ = _struct_normalize_for_match(quote)+ if not norm_quote:+ return None+ pos = norm_text.find(norm_quote)+ if pos < 0:+ return None+ end_norm = pos + len(norm_quote) - 1+ return index_map[pos], index_map[end_norm] + 1+++def _struct_apply_evidence_gate(+ items: list[dict],+ text_by_chunk_id: dict[str, str],+ mode: str = _EVIDENCE_GATE_DEFAULT,+) -> tuple[int, int]:+ """Validate and locate the evidence quotes carried by extracted claims.++ Every quote must be locatable in a chunk the claim cites. Quotes that+ cannot be located are dropped. ``mode`` decides what happens to a claim+ left with no verified evidence: ``"soft"`` keeps it (evidence omitted),+ ``"hard"`` drops the claim.++ ``items`` is filtered in place. Returns (verified, rejected) counts.+ """+ if mode not in _EVIDENCE_GATE_MODES:+ mode = _EVIDENCE_GATE_DEFAULT+ verified = 0+ rejected = 0+ if not text_by_chunk_id:+ return verified, rejected++ keep: list[dict] = []+ for payload in items:+ if not isinstance(payload, dict):+ continue+ raw_evidence = payload.get("evidence")+ if not isinstance(raw_evidence, (list, tuple)) or not raw_evidence:+ payload.pop("evidence", None)+ keep.append(payload)+ continue++ located: list[dict] = []+ for entry in raw_evidence:+ if not isinstance(entry, dict):+ continue+ quote = entry.get("quote")+ if not isinstance(quote, str) or not quote.strip():+ continue+ # Prefer the chunk the model cited; fall back to scanning every+ # chunk in the batch when it cited none (or cited one that is not+ # in this batch).+ cited = entry.get("chunk_id")+ candidates = [cited] if isinstance(cited, str) and cited in text_by_chunk_id else list(text_by_chunk_id)+ for chunk_id in candidates:+ span = _struct_locate_evidence(quote, text_by_chunk_id.get(chunk_id) or "")+ if span is None:+ continue+ start, end = span+ located.append({"quote": quote, "chunk_id": chunk_id, "start": start, "end": end})+ break++ if located:+ payload["evidence"] = located+ verified += len(located)+ keep.append(payload)+ continue++ rejected += 1+ payload.pop("evidence", None)+ if mode == "soft":+ keep.append(payload)++ items[:] = keep+ return verified, rejected+++def _struct_batch_text_by_chunk_id(+ packed: list[dict],+ chunk_id_map: dict | None,+) -> dict[str, str]:+ """Map the chunk ids assigned to this batch's claims to their text.++ Without rechunking the ids are the original parse chunks. With rechunking+ the ids are the formal chunks whose text is the concatenation of their+ source chunks — the gate must validate against whatever text the claim+ actually points at, so both shapes are handled here.+ """+ source_text = {e["chunk_id"]: (e.get("text") or "") for e in packed if e.get("chunk_id")}+ if not chunk_id_map:+ return source_text+ grouped: dict[str, list[str]] = {}+ for src_id, new_id in chunk_id_map.items():+ grouped.setdefault(str(new_id), []).append(str(src_id))+ return {new_id: "\n\n".join(source_text[s] for s in srcs if s in source_text) for new_id, srcs in grouped.items()}def _struct_payload_chunk_ids(payload: dict, batch_ids: list) -> list:@@ -793,10 +1073,43 @@ def _struct_payload_chunk_ids(payload: dict, batch_ids: list) -> list:_struct_embed = _encode-def _struct_payload_description(payload: dict) -> str:- """Concat string values of every non-description field (lists flattened)."""+# Payload keys that must never reach the embedding or the BM25 columns.+#+# ``evidence`` carries verbatim source quotes for claim/evidence compilation.+# Feeding it into the vector would embed a mixed "claim + raw source" unit+# instead of the claim the geometric layer is meant to index, and a+# ``list[dict]`` value would stringify to a Python dict repr (``{'quote': ...}``)+# and pollute the tokens. Kept out of BM25 in the first phase too so the+# payload change stays isolated from any recall change.+#+# "source_chunk_ids" and "mention_count" are bookkeeping, not semantics: the+# ids are opaque hex that tokenizes into garbage terms and drifts the vector,+# and the count is a number every row shares. Must stay in lockstep with+# IndexExcludedKeys in+# internal/ingestion/component/knowledge_compiler/common/indextext.go, or the+# two runtimes index the same payload into different vectors.+_STRUCT_INDEX_EXCLUDED_KEYS = frozenset({"evidence", "source_chunk_ids", "mention_count"})+++def _struct_payload_description(payload: dict, excluded: frozenset[str] | set[str] | None = None) -> str:+ """Concat string values of every non-description field (lists flattened).++ ``excluded`` names payload keys to skip. Callers that build the embedding+ and the BM25 columns pass different exclusion sets so evidence text can be+ kept out of the vector while leaving the door open for a separate BM25+ trial.+ """+ skip = _STRUCT_INDEX_EXCLUDED_KEYS if excluded is None else excludedparts: list[str] = []- for k, v in payload.items():+ # Keys are visited in sorted order to match the Go implementation+ # (common.PayloadDescription). Go maps carry no insertion order, so that+ # side sorts; iterating this dict in insertion order would feed the two+ # runtimes different text — and therefore different vectors — for the very+ # same payload.+ for k in sorted(payload):+ if k in skip:+ continue+ v = payload[k]if isinstance(v, (list, tuple)):for item in v:if item is None:@@ -836,7 +1149,7 @@ def _struct_graph_entity(payload: dict, source_chunk_ids: list | None = None) ->if isinstance(source_chunk_ids, str):source_chunk_ids = [source_chunk_ids]source_chunk_ids = _struct_union_chunk_ids(source_chunk_ids)- return {+ entity = {"aliases": aliases,"mention_count": 1,"name": name,@@ -844,6 +1157,16 @@ def _struct_graph_entity(payload: dict, source_chunk_ids: list | None = None) ->"type": typ or "other","description": str(description).strip() if description is not None else "",}+ # page_index fact/conclusion payloads carry gate-verified verbatim evidence+ # (page_index.yaml's evidence field). The artifacts detail panel renders it+ # next to the description; rows compiled before the field existed simply+ # omit it.+ evidence = payload.get("evidence")+ if isinstance(evidence, list):+ verified = [e for e in evidence if isinstance(e, dict) and str(e.get("quote") or "").strip()]+ if verified:+ entity["evidence"] = verified+ return entitydef _struct_graph_relation(payload: dict) -> dict | None:@@ -861,6 +1184,111 @@ def _struct_graph_relation(payload: dict) -> dict | None:}+# Kinds whose templates declare that every detail entity has exactly one+# container parent. Only these may have orphans re-parented; everywhere else an+# unattached entity is a legitimate root.+_STRUCT_ORPHAN_REPARENT_KINDS = {"page_index", "pageindex"}+++def _struct_attach_orphan_entities(+ entities: list[dict],+ relations: list[dict],+) -> list[dict]:+ """Re-parent leaf entities the per-batch relation pass could not attach.++ Relations are extracted inside one batch, so a parent living in another+ batch is invisible to the edge prompt. Packing the whole document into a+ single batch hid this; capping batches at a few chunks (which the model+ needs to actually read the text) exposes it — a claim whose heading sits+ in an earlier batch gets no ``include`` edge and lands at the tree root,+ which turns a table of contents into a flat list of claims.++ Only types that NEVER act as a container are repaired: an entity of such a+ type with no incoming edge is missing its parent by definition, whereas a+ structural type (a heading) may legitimately be a root. No type names are+ baked in — containers are whatever appears on the ``from`` side.++ The parent is the container sharing the most source chunks with the+ orphan, preferring the most specific one (fewest chunks of its own),+ because a title's ``source_chunk_ids`` are meant to cover its whole+ section.+ """+ if not entities or not relations:+ return relations++ child_names = {str(r.get("to") or "") for r in relations}+ container_names = {str(r.get("from") or "") for r in relations}+ if not container_names:+ return relations++ by_type: dict[str, list[dict]] = {}+ for ent in entities:+ by_type.setdefault(str(ent.get("type") or "other"), []).append(ent)++ # Classify by TYPE, not by instance. A heading whose own children all fell+ # in other batches never appears on a ``from`` side, yet it is still the+ # right parent — going by instance would leave its whole section orphaned.+ container_types = {typ for typ, ents in by_type.items() if any(str(e.get("name") or "") in container_names for e in ents)}+ leaf_types = {typ for typ in by_type if typ not in container_types}+ if not leaf_types:+ return relations++ containers = [e for e in entities if str(e.get("type") or "other") in container_types]+ if not containers:+ return relations++ dominant_type = "include"+ counts: dict[str, int] = {}+ for r in relations:+ t = str(r.get("type") or "").strip()+ if t:+ counts[t] = counts.get(t, 0) + 1+ if counts:+ dominant_type = max(counts, key=lambda k: counts[k])++ existing = {(str(r.get("from") or ""), str(r.get("to") or "")) for r in relations}+ repaired: list[dict] = []+ for ent in entities:+ name = str(ent.get("name") or "")+ if not name or str(ent.get("type") or "other") not in leaf_types:+ continue+ if name in child_names:+ continue+ chunks = {str(c) for c in (ent.get("source_chunk_ids") or []) if c}+ if not chunks:+ continue+ best = None+ best_overlap = 0+ for cand in containers:+ cand_name = str(cand.get("name") or "")+ if cand_name == name:+ continue+ cand_chunks = {str(c) for c in (cand.get("source_chunk_ids") or []) if c}+ overlap = len(chunks & cand_chunks)+ if overlap <= 0:+ continue+ # Most overlapping wins; ties go to the narrower container, which+ # is the deeper (more specific) section.+ key = (overlap, -len(cand_chunks))+ if best is None or key > best[0]:+ best = (key, cand_name)+ best_overlap = overlap+ if best is None or best_overlap <= 0:+ continue+ parent = best[1]+ if (parent, name) in existing:+ continue+ existing.add((parent, name))+ repaired.append({"from": parent, "to": name, "type": dominant_type})++ if repaired:+ logging.info(+ "structure graph: re-parented %d orphan leaf entity(ies) that the per-batch relation pass could not attach",+ len(repaired),+ )+ return relations + repaired++def _struct_merge_graph_entities(entities: list[dict]) -> list[dict]:merged: dict[tuple[str, str], dict] = {}order: list[tuple[str, str]] = []@@ -1036,6 +1464,17 @@ def _struct_to_doc_storage_doc(if name_value:doc["name_kwd"] = name_value.lower()+ # Row role: tree_node / title / fact / conclusion / claim. Stamped onto+ # ``entity_type_kwd`` — the schema-backed role column Go already writes via+ # applyStructureGraphColumns, and the one ``internal/service/graph`` filters+ # on. Python's structure compile used to omit it, which made the two+ # runtimes emit divergent rows for the same compilation template. Relations+ # have no role, so only entity rows are stamped.+ if kind == "entity":+ entity_type = str(payload.get("type") or "").strip()+ if entity_type:+ doc["entity_type_kwd"] = entity_type+if template_id_str:doc["compilation_template_ids"] = [template_id_str]if compilation_template_kind:@@ -1090,7 +1529,23 @@ async def _struct_process_batch(src_field, target_field = _struct_relation_member_fields(parser_config)rechunk = bool(parser_config.get("rechunk"))+ def _note(msg: str, prog: float | None = None) -> None:+ """Write to both the server log and the task progress stream.++ A batch is one long LLM round-trip (two, plus one per detail+ sub-batch, when the template has a detail pass), and nothing was+ emitted in between — the task looked hung. Mirrors raptor, which+ logs its per-batch claim harvest.+ """+ logging.info("compile_structure_from_text: doc=%s batch %s/%s: %s", doc_id, batch_idx + 1, total, msg)+ if callback:+ try:+ callback(msg=f"batch {batch_idx + 1}/{total}: {msg}")+ except TypeError:+ pass+async def _run() -> _RechunkedDocs:+ _note(f"start: {len(packed)} chunk(s), extracting entities")# For hypergraph, entity extraction MUST complete before edge extraction# within the same batch, because the edge prompt's {known_nodes}# placeholder is filled from this batch's extracted nodes — see@@ -1108,9 +1563,46 @@ async def _struct_process_batch(logging.exception(f"compile_structure_from_text: extraction failed for batch {batch_idx}: {e}")return _RechunkedDocs()+ # Validate claim evidence while the batch's source text is still in+ # hand: every quote must be locatable in the chunk it cites, otherwise+ # it is discarded. Runs before embedding so the vector is built from+ # the surviving payload. Relations are gated the same way whenever the+ # template asks them to carry evidence.+ if items or relations:+ text_by_chunk = _struct_batch_text_by_chunk_id(packed, chunk_id_map)+ gate_mode = _struct_evidence_gate_mode(parser_config)+ verified = rejected = 0+ if items:+ # Counted BEFORE the gate: it pops ``evidence`` in place, so+ # afterwards "the model gave a quote" and "the gate rejected+ # it" are indistinguishable — which is exactly the split this+ # log exists to expose (mirrors raptor's emitted/verified/+ # rejected counters).+ emitted = sum(1 for it in items if it.get("evidence"))+ emitted_by_type = _struct_type_counts(items)+ verified, rejected = _struct_apply_evidence_gate(items, text_by_chunk, gate_mode)+ logging.info(+ "compile_structure_from_text: doc=%s batch %s: items=%d emitted_evidence=%d verified=%d rejected=%d emitted_by_type=%s kept_by_type=%s",+ doc_id,+ batch_idx,+ len(items),+ emitted,+ verified,+ rejected,+ emitted_by_type,+ _struct_type_counts(items),+ )+ if relations and _struct_relation_expects_evidence(parser_config):+ rel_verified, rel_rejected = _struct_apply_evidence_gate(relations, text_by_chunk, gate_mode)+ verified += rel_verified+ rejected += rel_rejected+ if verified or rejected:+ logging.info(f"compile_structure_from_text: doc={doc_id} batch {batch_idx}: evidence verified={verified} rejected={rejected}")+payloads = items + relationskinds = ["entity"] * len(items) + ["relation"] * len(relations)payload_chunk_ids = list(dict.fromkeys(chunk_id_map.values())) if chunk_id_map else batch_ids+ _note(f"extracted {len(items)} entity/claim(s) and {len(relations)} relation(s); embedding")if not payloads:if callback:callback((batch_idx + 1) / total, f"{batch_idx + 1}/{total} batches: 0 items")@@ -1227,10 +1719,33 @@ async def compile_structure_from_text(chunks,chat_mdl,prompt_overhead_tokens=prompt_overhead,+ # Structure extraction is packed by the context window: a heading has to+ # see its whole section to own it. Pass a cap only when one is+ # configured — the greedy mode then cuts on either the chunk count or+ # the accumulated tokens.+ batch_size_cap=_STRUCT_MAX_CHUNKS_PER_BATCH or None,)if not packed_batches:return []+ # The runner only reports "compile batch N (M chunks)"; how those M chunks+ # were split into LLM-sized batches was invisible, so a long compilation+ # gave no sign of progress.+ logging.info(+ "compile_structure_from_text: doc=%s template=%s kind=%s: %d chunk(s) -> %d batch(es) (max %d chunk(s)/call)",+ doc_id,+ compilation_template_id,+ template_kind,+ len(chunks),+ len(packed_batches),+ _STRUCT_MAX_CHUNKS_PER_BATCH,+ )+ if callback:+ try:+ callback(msg=f"{len(chunks)} chunk(s) -> {len(packed_batches)} batch(es)")+ except TypeError:+ pass+async def _process_one(batch: list[dict], bi: int, total: int) -> list[dict]:# The engine's semaphore already bounds concurrency.return await _struct_process_batch(@@ -1453,7 +1968,9 @@ def _struct_merge_exact_entity_payload(existing: dict, incoming: dict) -> dict |merged[key] = valuetypes = {str(left.get("type") or "").strip().casefold(), str(right.get("type") or "").strip().casefold()}- for preferred in ("title", "fact", "conclusion"):+ # "fact"/"conclusion" are page_index's pre-rename spelling of a claim and+ # are kept only so rows compiled before the rename still merge correctly.+ for preferred in ("title", "claim", "fact", "conclusion"):if preferred in types:merged["type"] = preferredbreak@@ -2365,23 +2882,6 @@ async def _struct_local_dedup_parallel(return deduped_entities + deduped_relations, dropped-def _struct_graph_row_id(- doc_id: str,- compile_kwd: str,- compilation_template_id: str | None = None,-) -> str:- """Stable id per (doc, compile_kwd, template). Without the template- suffix, two templates sharing a compile_kwd (e.g. both ``list``)- would overwrite each other's per-doc graph JSON row."""- tpl_part = compilation_template_id or ""- return xxhash.xxh64(- f"{doc_id}:structure_graph:{compile_kwd}:{tpl_part}".encode(- "utf-8",- "surrogatepass",- ),- ).hexdigest()--async def _struct_rebuild_graph_json(tenant_id: str,kb_id: str,@@ -2445,6 +2945,13 @@ async def _struct_rebuild_graph_json(entity["doc_ids_kwd"] = [source_doc_id]entities.append(entity)+ # Re-parenting only makes sense where the template's semantics say every+ # detail item hangs off a container. In a knowledge graph or a timeline an+ # unattached entity is a legitimate root, and forcing it under whatever+ # shares a chunk would fabricate relations.+ if _struct_normalize_kind(compile_kwd) in _STRUCT_ORPHAN_REPARENT_KINDS:+ relations = _struct_attach_orphan_entities(entities, relations)+return {"entities": _struct_merge_graph_entities(entities),"relations": _struct_merge_graph_relations(relations) if doc_id is None else relations,@@ -2520,56 +3027,180 @@ async def cleanup_timeline_isolated_entities(compilation_template_id or "legacy",)- # Refresh the compact graph after source-row cleanup. This also handles- # the no-relation case, where every timeline entity is isolated.- await rebuild_structure_graph_json(- tenant_id,- kb_id,- doc_id,- doc_name,- "timeline",- compilation_template_id,- )+ # No compact-graph refresh here: the graph blob (knowledge_graph_kwd="graph")+ # is gone from the storage model -- the raw entity/relation rows above ARE+ # the graph, so deleting the orphans is the whole cleanup.return len(orphan_ids)-async def _struct_upsert_graph_json(- graph: dict,+# Upper bound on the claim rows listed when deciding which previous rows the+# new payload no longer covers. Claims are one row each, so a very large+# document reaches the low thousands; anything past the limit is left for the+# next recompile rather than costing an unbounded scan.+_STRUCT_CLAIM_ROW_SCAN_LIMIT = 4096+++async def _struct_existing_claim_row_ids(index: str, kb_id: str, condition: dict) -> list[str]:+ """Return the ids of the claim rows matching ``condition``.++ Lets the caller tell "overwrite this row" from "drop this row" without+ deleting first: only ids the new payload does not cover are removed, and+ nothing is deleted until the replacement rows are safely written.+ """+ from common import settings+ from common.doc_store.doc_store_base import OrderByExpr++ select_fields = ["id"]+ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ select_fields,+ [],+ condition,+ [],+ OrderByExpr(),+ 0,+ _STRUCT_CLAIM_ROW_SCAN_LIMIT,+ index,+ [kb_id],+ )+ except Exception:+ # Failing to list is not a reason to skip the upsert: without ids we+ # simply keep whatever is already stored.+ logging.exception("tree claims: could not list existing claim rows")+ return []+ field_map = settings.docStoreConn.get_fields(res, select_fields) or {}+ return [str(rid) for rid in field_map if rid]+++async def _struct_upsert_tree_claim_rows(+ claims_by_chunk: dict,tenant_id: str,kb_id: str,doc_id: str,doc_name: str,- compile_kwd: str,+ embedding_model,+ compile_kwd: str = "tree",compilation_template_id: str | None = None,-) -> None:+) -> int:+ """Persist tree claim/evidence rows for one document.++ Claims are stored as their own rows (``entity_type_kwd="claim"``) with their+ own embeddings, so global KNN can hit them directly instead of only reaching+ them through beam descent (see claim_evidence.md §6.2). They carry no+ ``knowledge_graph_kwd``, matching the convention that column is reserved for+ the entity/relation rows the artifacts tree renders.++ Returns the number of claim rows written.+ """from common import settingsfrom rag.nlp import search as _rag_search+ payloads: list[tuple[str, dict]] = []+ for chunk_id, claims in (claims_by_chunk or {}).items():+ if not isinstance(claims, list):+ continue+ for claim in claims:+ if not isinstance(claim, dict):+ continue+ name = str(claim.get("name") or "").strip()+ if not name:+ continue+ payload = {+ "type": "claim",+ "name": name,+ "description": claim.get("description") or name,+ "source_chunk_ids": [chunk_id],+ }+ if claim.get("evidence"):+ payload["evidence"] = claim["evidence"]+ payloads.append((chunk_id, payload))+index = _rag_search.index_name(tenant_id)- row_id = _struct_graph_row_id(doc_id, compile_kwd, compilation_template_id)- row = {- "id": row_id,- "content_with_weight": json.dumps(graph, ensure_ascii=False),- "compile_kwd": compile_kwd,- "knowledge_graph_kwd": "graph",- "doc_id": doc_id,- "docnm_kwd": doc_name,- "kb_id": kb_id,- "available_int": 0,++ # Scoped to this document and template: a claim row's id hashes its payload,+ # so a re-parse with different claims leaves the previous rows behind unless+ # they are removed explicitly.+ delete_condition = {+ "doc_id": [doc_id],+ "compile_kwd": [compile_kwd],+ "entity_type_kwd": ["claim"],}if compilation_template_id:- row["compilation_template_ids"] = [compilation_template_id]- old = await thread_pool_exec(settings.docStoreConn.get, row_id, index, [kb_id])- if old:- await thread_pool_exec(- settings.docStoreConn.update,- {"id": row_id},- {k: v for k, v in row.items() if k != "id"},- index,- kb_id,+ delete_condition["compilation_template_ids"] = [compilation_template_id]++ if not payloads:+ # Recompiling to zero claims must still clear what was stored: the+ # caller only knows the new payload, and stale rows would keep answering+ # queries with facts the document no longer has.+ await thread_pool_exec(settings.docStoreConn.delete, delete_condition, index, kb_id)+ logging.info(+ "tree claims: doc=%s template=%s yielded no claims; removed stale claim rows",+ doc_id,+ compilation_template_id,)- else:- await thread_pool_exec(settings.docStoreConn.insert, [row], index, kb_id)+ return 0++ # Evidence is excluded from the embedding input, consistent with the+ # page_index claim path.+ embeddings = await _struct_embed(embedding_model, [_struct_payload_description(p) for _, p in payloads])+ if len(embeddings) != len(payloads):+ raise ValueError(f"Claim embedding count mismatch: {len(embeddings)} != {len(payloads)}")++ rows = []+ for (chunk_id, payload), vector in zip(payloads, embeddings):+ # Claims are searchable on their own (global KNN over their own+ # embeddings) but are NOT part of the structure graph: they carry no+ # relation, and a relation-less row would be rendered as a root in the+ # artifacts tree. So they deliberately skip knowledge_graph_kwd (empty+ # kind), which keeps them out of the artifacts query (build_bucket+ # filters knowledge_graph_kwd=["entity","relation"]).+ # entity_type_kwd is stamped explicitly — _struct_to_doc_storage_doc+ # only derives it when kind == "entity", and claims intentionally pass an+ # empty kind. Both behaviours mirror+ # internal/ingestion/.../component.go case "claim".+ row = _struct_to_doc_storage_doc(+ payload,+ compile_kwd,+ doc_id,+ doc_name,+ [chunk_id],+ vector,+ "",+ compilation_template_id=compilation_template_id,+ )+ row["entity_type_kwd"] = "claim"+ rows.append(row)++ # An unchanged claim hashes to the same row id, so the insert below simply+ # overwrites it — those rows must survive the cleanup that follows.+ new_ids = {row["id"] for row in rows}+ stale_ids = [rid for rid in await _struct_existing_claim_row_ids(index, kb_id, delete_condition) if rid not in new_ids]++ # Insert BEFORE delete: the doc store has no transaction, so ordering IS the+ # rollback. A failed insert leaves the previous claim rows untouched, while+ # deleting first would drop them and leave the document with no claims at all+ # if the write then failed.+ await thread_pool_exec(settings.docStoreConn.insert, rows, index, kb_id)+ if stale_ids:+ try:+ await thread_pool_exec(settings.docStoreConn.delete, {"id": stale_ids}, index, kb_id)+ except Exception:+ # The new rows are already readable, so leaving a few stale ones+ # behind is the lesser evil; the next recompile clears them.+ logging.exception(+ "tree claims: failed to remove %d stale claim row(s) for doc=%s template=%s",+ len(stale_ids),+ doc_id,+ compilation_template_id,+ )+ logging.info(+ "tree claims: upserted %d claim row(s) for doc=%s template=%s",+ len(rows),+ doc_id,+ compilation_template_id or "legacy",+ )+ return len(rows)async def _struct_upsert_tree_graph_rows(@@ -2639,25 +3270,21 @@ async def rebuild_structure_graph_json(compile_kwd: str,compilation_template_id: str | None = None,) -> dict:- """Rebuild and persist the compact document-scoped structure graph,- scoped to one (doc, compile_kwd, template_id) triple."""- graph = await _struct_rebuild_graph_json(- tenant_id,- kb_id,- doc_id,- compile_kwd,- compilation_template_id,- )- await _struct_upsert_graph_json(- graph,+ """Rebuild the document-scoped structure graph dict from its raw+ entity/relation rows, scoped to one (doc, compile_kwd, template_id) triple.++ Read-only by design: the compact graph blob (knowledge_graph_kwd="graph")+ is no longer part of the storage model -- the raw rows ARE the graph, so+ there is nothing to persist. Callers consume the returned dict (e.g. the+ page_index navigation summary).+ """+ return await _struct_rebuild_graph_json(tenant_id,kb_id,doc_id,- doc_name,compile_kwd,compilation_template_id,)- return graphdef _dataset_struct_graph_row_id(diff --git a/rag/flow/compiler/compiler.py b/rag/flow/compiler/compiler.pyindex d1fda66979..1b5946fd6e 100644--- a/rag/flow/compiler/compiler.py+++ b/rag/flow/compiler/compiler.py@@ -382,7 +382,6 @@ class Compiler(ProcessBase, LLM):from the current pipeline output instead."""from rag.advanced_rag.knowlege_compile.structure import (- _struct_upsert_graph_json,_struct_upsert_tree_graph_rows,)from rag.svr.task_executor_refactor.chunk_post_processor import (@@ -453,6 +452,11 @@ class Compiler(ProcessBase, LLM):if bool(raptor_cfg.get("rechunk")):self._compile_progress(msg="Compiler: tree rechunking is not supported for in-memory pipeline chunks; keeping original chunks.")+ # Claims are keyed by chunk id on the tree dict (see+ # RaptorService.build_doc_tree); pull them off before projecting so+ # the graph blob stays a pure structure payload.+ claims_by_chunk = tree.pop("claims_by_chunk", None) if isinstance(tree, dict) else None+await rewrite_duplicate_tree_names(tree, pooled_chat_mdl)after_graph = raptor_tree_to_graph(tree)try:@@ -465,12 +469,22 @@ class Compiler(ProcessBase, LLM):embedding_model,compilation_template_id=template_id,)- await _struct_upsert_graph_json(- after_graph,+ # Claims become their own searchable rows so global KNN can hit+ # them directly instead of only via beam descent. Best-effort:+ # a failure here must not cost us the tree. Called even when+ # there are no claims, so a recompile that yields none clears+ # the rows a previous run wrote.+ from rag.advanced_rag.knowlege_compile.structure import (+ _struct_upsert_tree_claim_rows,+ )++ await _struct_upsert_tree_claim_rows(+ claims_by_chunk,tenant_id,kb_id,doc_id,doc_name,+ embedding_model,compile_kwd="tree",compilation_template_id=template_id,)diff --git a/rag/llm/chat_model.py b/rag/llm/chat_model.pyindex d9ae2716bb..b980dad6b3 100644--- a/rag/llm/chat_model.py+++ b/rag/llm/chat_model.py@@ -286,6 +286,17 @@ def _apply_model_family_policies(elif provider == SupportedLiteLLMProvider.ZHIPU_AI and "glm" in model_name_lower and thinking_type:_pop_thinking_controls()sanitized_gen_conf["thinking"] = {"type": thinking_type}+ elif provider == SupportedLiteLLMProvider.MiniMax:+ # MiniMax reasoning models (MiniMax-M1/M3) read `thinking` in the+ # request body and ignore `reasoning_effort`. `thinking` is NOT a+ # standard OpenAI-compatible param, so LiteLLM's drop_params=True+ # would drop a top-level key; it must ride in extra_body (merged+ # verbatim into the body). Without this, MiniMax keeps+ # chain-of-thought on, which slows extraction and can hang a batch+ # on a long COT.+ if thinking_type:+ _pop_thinking_controls()+ _merge_extra_body(sanitized_gen_conf, {"thinking": {"type": thinking_type}})return sanitized_gen_conf, sanitized_kwargs@@ -298,6 +309,7 @@ def _move_litellm_provider_body_fields(provider: SupportedLiteLLMProvider | strSupportedLiteLLMProvider.Dashscope: {"enable_thinking"},SupportedLiteLLMProvider.Moonshot: {"thinking"},SupportedLiteLLMProvider.ZHIPU_AI: {"thinking"},+ SupportedLiteLLMProvider.MiniMax: {"thinking"},}.get(provider, set())body = completion_args.get("extra_body")diff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.pyindex 492773406c..0665e89d64 100644--- a/rag/svr/task_executor_refactor/chunk_post_processor.py+++ b/rag/svr/task_executor_refactor/chunk_post_processor.py@@ -698,11 +698,16 @@ async def rechunk_doc_by_tree(tree: dict,template_id: str,embedding_model,-) -> None:+) -> dict[str, str]:"""Merge each leaf cluster's source chunks into a singlereplacement chunk and rewrite the tree's leaf-cluster``source_chunk_ids`` in-place. Original chunks are soft-deletedvia ``available_int=0`` and stamped with ``superseded_by_chunk_id``.++ Returns the old-to-new chunk id mapping for every original chunk that was+ folded into a replacement, so callers can re-point data attached to the old+ ids (tree claims key their claims by source chunk id). Chunks that were not+ remapped are absent — callers keep those ids as they are."""from datetime import datetime@@ -739,7 +744,7 @@ async def rechunk_doc_by_tree(_walk(tree)if not cluster_id_map:- return+ return {}all_source_ids = sorted({sid for _, ids in cluster_id_map.values() for sid in ids})@@ -747,7 +752,7 @@ async def rechunk_doc_by_tree(index_nm = search.index_name(ctx.tenant_id)if not settings.docStoreConn.index_exist(index_nm, ctx.kb_id):- return+ return {}vctr_nm = "q_%d_vec" % len(embedding_model.encode(["x"])[0][0])select_fields = [@@ -783,9 +788,9 @@ async def rechunk_doc_by_tree(ctx.doc_id,template_id,)- return+ return {}if not field_map:- return+ return {}chunks_by_id: dict[str, dict] = {str(rid): {**row, "id": str(rid)} for rid, row in field_map.items()}@@ -838,7 +843,7 @@ async def rechunk_doc_by_tree(merged_rows.append(base)if not merged_rows:- return+ return {}contents = [r["content_with_weight"] for r in merged_rows]try:@@ -849,7 +854,7 @@ async def rechunk_doc_by_tree(ctx.doc_id,template_id,)- return+ return {}for row, vec in zip(merged_rows, vectors):try:row[vctr_nm] = np.asarray(vec, dtype=np.float32).tolist()@@ -860,8 +865,13 @@ async def rechunk_doc_by_tree()row[vctr_nm] = Nonemerged_rows = [r for r in merged_rows if r.get(vctr_nm) is not None]+ # Drop clusters whose replacement chunk will not be stored: a tree node must+ # not point at a filtered-out id, and the returned map must not send callers+ # to a chunk that does not exist.+ inserted_ids = {r["id"] for r in merged_rows}+ cluster_new_id = {nid: cid for nid, cid in cluster_new_id.items() if cid in inserted_ids}if not merged_rows:- return+ return {}try:await thread_pool_exec(@@ -876,7 +886,7 @@ async def rechunk_doc_by_tree(ctx.doc_id,template_id,)- return+ return {}for node_id_int, new_chunk_id in cluster_new_id.items():node, _ = cluster_id_map[node_id_int]@@ -906,6 +916,35 @@ async def rechunk_doc_by_tree(new_chunk_id,)+ # Every original chunk of a merged cluster now resolves to the replacement+ # chunk, so callers can re-point data keyed by the old ids.+ return {cid: new_chunk_id for node_id_int, new_chunk_id in cluster_new_id.items() for cid in cluster_id_map[node_id_int][1]}+++def _remap_claims_by_chunk(claims_by_chunk, chunk_id_remap: dict[str, str]):+ """Re-key claims onto the chunk that replaced their source chunk.++ Re-chunking folds several original chunks into one replacement and soft-+ deletes the originals, so a claim still keyed by an old id cites text that+ can no longer be retrieved. Claims whose id was not remapped are kept as-is:+ their chunk is untouched and still valid.+ """+ if not claims_by_chunk or not chunk_id_remap:+ return claims_by_chunk++ remapped: dict[str, list] = {}+ for chunk_id, claims in claims_by_chunk.items():+ new_id = chunk_id_remap.get(chunk_id) or chunk_id+ bucket = remapped.setdefault(new_id, [])+ for claim in claims or []:+ if isinstance(claim, dict):+ claim["source_chunk_ids"] = [chunk_id_remap.get(sid) or sid for sid in claim.get("source_chunk_ids") or []]+ for evidence in claim.get("evidence") or []:+ if isinstance(evidence, dict) and evidence.get("chunk_id"):+ evidence["chunk_id"] = chunk_id_remap.get(evidence["chunk_id"]) or evidence["chunk_id"]+ bucket.append(claim)+ return remapped+async def run_tree_templates(handler,@@ -918,8 +957,12 @@ async def run_tree_templates("""Run the ``tree``-kind compilation templates for the currentdoc. Each pair runs RAPTOR with ``is_tree=True`` via``RaptorService.build_doc_tree`` and persists a single graph row- via ``_struct_upsert_graph_json``."""- from rag.advanced_rag.knowlege_compile.structure import _struct_upsert_graph_json+ via ``_struct_upsert_tree_graph_rows`` (per-row entity/relation rows --+ the compact graph blob no longer exists)."""+ from rag.advanced_rag.knowlege_compile.structure import (+ _struct_upsert_tree_claim_rows,+ _struct_upsert_tree_graph_rows,+ )from rag.svr.task_executor_refactor.raptor_service import RaptorServicectx = handler._task_context@@ -960,6 +1003,10 @@ async def run_tree_templates("threshold": float(raptor_cfg.get("threshold") or 0.1),"random_seed": int(raptor_cfg.get("random_seed") or 0),"max_cluster": int(raptor_cfg.get("max_cluster") or 64),+ # Template-declared claim/evidence extraction contract (tree.yaml's+ # raptor.claim_prompt). None = built-in defaults. Batch sizing is a+ # pipeline concern, so it stays out of the template.+ "claim_prompt": raptor_cfg.get("claim_prompt") or None,}progress_cb(msg=f"tree-template ({idx + 1}/{len(templates)}): building tree for doc={doc_id}",@@ -987,13 +1034,17 @@ async def run_tree_templates()continue+ chunk_id_remap: dict[str, str] = {}if bool((raptor_cfg or {}).get("rechunk")):try:- await rechunk_doc_by_tree(- handler=handler,- tree=tree,- template_id=template_id,- embedding_model=embedding_model,+ chunk_id_remap = (+ await rechunk_doc_by_tree(+ handler=handler,+ tree=tree,+ template_id=template_id,+ embedding_model=embedding_model,+ )+ or {})except Exception:logging.exception(@@ -1003,15 +1054,21 @@ async def run_tree_templates()await rewrite_duplicate_tree_names(tree, pooled_chat_mdl)+ # Claims are keyed by chunk id on the tree dict (see+ # RaptorService.build_doc_tree); pull them off before projecting so the+ # graph blob stays a pure structure payload.+ claims_by_chunk = tree.pop("claims_by_chunk", None) if isinstance(tree, dict) else None+ if chunk_id_remap:+ claims_by_chunk = _remap_claims_by_chunk(claims_by_chunk, chunk_id_remap)graph = raptor_tree_to_graph(tree)try:- await _struct_upsert_graph_json(+ await _struct_upsert_tree_graph_rows(graph,ctx.tenant_id,ctx.kb_id,doc_id,doc_name,- compile_kwd="tree",+ embedding_model,compilation_template_id=template_id,)except Exception:@@ -1022,6 +1079,30 @@ async def run_tree_templates()continue+ # Claims get their own searchable rows (entity_type_kwd="claim") so they+ # can be hit directly by global KNN instead of only via beam descent.+ # This is best-effort: a failure here must not cost us the tree. Called+ # even when there are no claims, so a recompile that yields none (and a+ # rechunk that remapped every claim away) clears the rows a previous run+ # wrote instead of leaving them to answer for a document that changed.+ try:+ await _struct_upsert_tree_claim_rows(+ claims_by_chunk,+ ctx.tenant_id,+ ctx.kb_id,+ doc_id,+ doc_name,+ embedding_model,+ compile_kwd="tree",+ compilation_template_id=template_id,+ )+ except Exception:+ logging.exception(+ "tree-template %s: claim rows upsert failed for doc %s",+ template_id,+ doc_id,+ )+# Persist the per-doc nav_doc right after the graph node, so parsing a# file yields a nav_doc with the FULL entity descriptions as# graph_content -- without needing to run GENERATE NAVIGATION separatelydiff --git a/rag/svr/task_executor_refactor/raptor_service.py b/rag/svr/task_executor_refactor/raptor_service.pyindex a095b64ef3..f1531587dd 100644--- a/rag/svr/task_executor_refactor/raptor_service.py+++ b/rag/svr/task_executor_refactor/raptor_service.py@@ -493,7 +493,73 @@ class RaptorService:clustering_ratio=float(raptor_config.get("clustering_ratio", 0.5)),)+ # Extract claims first so every cluster is summarized from the claims of+ # its member chunks rather than from truncated raw text. Chunks that+ # yield no claims silently fall back to their raw text, so a failed+ # extraction degrades to the previous behavior instead of breaking the+ # build.+ claims_by_chunk: Dict[str, list] = {}raptor_input = [(content, vctr, [chunk_id] if chunk_id else []) for content, vctr, chunk_id in chunks]+ if raptor_config.get("extract_claims", True):+ try:+ from rag.advanced_rag.knowlege_compile.raptor import extract_claims_for_chunks++ claims_by_chunk = await extract_claims_for_chunks(+ raptor_input,+ chat_mdl,+ task_id=self._task_context.id,+ callback=self._task_context.progress_cb,+ claim_prompt=raptor_config.get("claim_prompt"),+ )+ logging.info(+ "build_doc_tree: claims extracted for %d/%d chunk(s)",+ len(claims_by_chunk),+ len(raptor_input),+ )+ except Exception:+ logging.exception("build_doc_tree: claim extraction failed; summarizing from raw text")+ claims_by_chunk = {}++ # Claim-view clustering vectors: each chunk is represented to the+ # clustering layer by the merge of its claims + verbatim evidence, not+ # by its raw-text vector. Claims are the chunk's semantics stripped of+ # layout noise, so topic clusters form around what the chunks actually+ # assert, and the cluster labels the summarizer produces are readable.+ # Chunks without claims keep their raw vector; a claim vector whose+ # dimension disagrees with the chunk vectors is ignored rather than+ # fed into AHC.+ if claims_by_chunk and embd_mdl is not None:+ try:+ from rag.advanced_rag.knowlege_compile.raptor import format_claims_for_summary++ keys: List[str] = []+ digest_texts: List[str] = []+ for cid, claims in claims_by_chunk.items():+ digest = format_claims_for_summary(claims)+ if digest.strip():+ keys.append(str(cid))+ digest_texts.append(digest)+ if digest_texts:+ embds, _ = await thread_pool_exec(embd_mdl.encode, digest_texts)+ claim_view = {cid: np.array(vec) for cid, vec in zip(keys, embds) if vec is not None and len(vec) > 0}+ replaced = 0+ for i, (content, vctr, src) in enumerate(raptor_input):+ cid = src[0] if src else ""+ view = claim_view.get(cid)+ if view is None:+ continue+ if vctr is not None and len(vctr) > 0 and len(view) != len(vctr):+ continue # mixed-model guard: never feed AHC mixed dimensions+ raptor_input[i] = (content, view, src)+ replaced += 1+ logging.info(+ "build_doc_tree: %d/%d chunk(s) clustered on claim-view embeddings",+ replaced,+ len(raptor_input),+ )+ except Exception:+ logging.exception("build_doc_tree: claim-view embedding failed; clustering on raw chunk vectors")+try:tree, _ = await raptor(raptor_input,@@ -501,6 +567,7 @@ class RaptorService:self._task_context.progress_cb,self._task_context.id,is_tree=True,+ claims_by_chunk=claims_by_chunk,)except NotImplementedError:# PSI builder — not supported in tree mode; surface as None@@ -509,6 +576,8 @@ class RaptorService:"build_doc_tree: PSI builder doesn't support is_tree; skipping",)return None+ if isinstance(tree, dict) and claims_by_chunk:+ tree["claims_by_chunk"] = claims_by_chunkreturn tree if isinstance(tree, dict) else Noneasync def _generate_raptor_legacy_rows(@@ -617,188 +686,3 @@ class RaptorService:except Exception:logging.exception("Failed to check RAPTOR chunks for doc %s", doc_id)raise-- @staticmethod- def _build_raptor_graph(rows: List[Dict]) -> Dict:- """Project loaded RAPTOR summary rows onto the canvas graph shape.-- Each row contributes one entity::-- {- "id": xxh128(content) # 32-char hex- "name": first 16 whitespace tokens- "description": content_with_weight- "source_chunk_ids": row.source_chunk_ids- }-- Relations: full bipartite layer-by-layer fan-out — every node at- layer K gets an edge to every node at layer K-1 (because we only- loaded ``content_with_weight`` + ``raptor_layer_int`` we don't- have the specific parent linkage). Self-edges and dangling- targets are dropped (the latter only matters if the layer-int- values are non-contiguous).- """- # Build entities. Dedup by id so two identical-content summaries- # collapse to one node — the canvas can't render multiple nodes- # at the same id anyway, and identical content is a defensible- # collapse.- by_id: Dict[str, Dict] = {}- by_layer: Dict[int, List[str]] = {}-- for row in rows:- content = row.get("content_with_weight")- if not isinstance(content, str) or not content.strip():- continue- try:- layer = int(row.get("raptor_layer_int") or 0)- except (TypeError, ValueError):- layer = 0- if layer <= 0:- # Layer 0 would be the original leaf chunks; RAPTOR- # summaries start at layer 1. Anything claiming layer 0- # here is malformed; skip.- continue-- name = " ".join(content.split()[:16])- nid = xxhash.xxh128(- content.encode("utf-8", "surrogatepass"),- ).hexdigest() # 32-char hex- if nid in by_id:- continue- source_chunk_ids = row.get("source_chunk_ids") or []- if not isinstance(source_chunk_ids, list):- source_chunk_ids = []- by_id[nid] = {- "id": nid,- "name": name,- "description": content,- "source_chunk_ids": list(source_chunk_ids),- }- by_layer.setdefault(layer, []).append(nid)-- # Layered fan-out from parent (higher layer) → child (lower layer).- relations: List[Dict] = []- layers_sorted = sorted(by_layer.keys())- for layer in layers_sorted:- child_layer = layer - 1- if child_layer not in by_layer:- continue- for parent in by_layer[layer]:- for child in by_layer[child_layer]:- if parent == child:- continue- relations.append({"from": parent, "to": child})-- return {"entities": list(by_id.values()), "relations": relations}-- async def _persist_raptor_graph_to_es(self, doc_id: str) -> None:- """Load the just-inserted RAPTOR summaries for ``doc_id`` and- persist a single graph row that the dataset structure-graph- endpoint can surface as a tree.-- Loads only ``content_with_weight`` + ``raptor_layer_int`` +- ``source_chunk_ids`` (per- the smallest-payload contract) and writes one row with::-- compile_kwd: "raptor_graph"- compilation_template_kind_kwd:"raptor"- doc_id: <doc_id>-- The row id is deterministic per ``(kb_id, doc_id)`` so re-runs- delete-and-replace cleanly through the same primary key.- ``knowledge_graph_kwd`` is intentionally NOT set — that field- belongs to the KG feature; this row is identified via- ``compile_kwd`` so the two paths stay semantically distinct.- """- from common.doc_store.doc_store_base import OrderByExpr-- ctx = self._task_context- tenant_id = ctx.tenant_id- kb_id_str = str(ctx.kb_id)- index_nm = search.index_name(tenant_id)- select_fields = ["content_with_weight", "raptor_layer_int", "source_chunk_ids"]- try:- res = await thread_pool_exec(- settings.docStoreConn.search,- select_fields,- [],- {"raptor_kwd": ["raptor"], "doc_id": [doc_id]},- [],- OrderByExpr(),- 0,- 10000,- index_nm,- [kb_id_str],- )- field_map = settings.docStoreConn.get_fields(res, select_fields)- except Exception:- logging.exception(- "raptor_graph: load failed for kb=%s doc=%s",- kb_id_str,- doc_id,- )- return-- rows = list((field_map or {}).values())- if not rows:- logging.info(- "raptor_graph: no summaries to render for kb=%s doc=%s",- kb_id_str,- doc_id,- )- return-- graph = self._build_raptor_graph(rows)- if not graph["entities"]:- logging.info(- "raptor_graph: projection produced no entities for kb=%s doc=%s",- kb_id_str,- doc_id,- )- return-- row_id = xxhash.xxh64(- f"raptor_graph:{kb_id_str}:{doc_id}".encode("utf-8", "surrogatepass"),- ).hexdigest()- row = {- "id": row_id,- "kb_id": kb_id_str,- "doc_id": doc_id,- "compile_kwd": "raptor_graph",- "compilation_template_kind_kwd": "raptor",- "content_with_weight": json.dumps(graph, ensure_ascii=False),- "available_int": 0,- }- try:- await thread_pool_exec(- settings.docStoreConn.delete,- {"compile_kwd": "raptor_graph", "doc_id": [doc_id]},- index_nm,- ctx.kb_id,- )- except Exception:- logging.debug(- "raptor_graph: prior delete failed for kb=%s doc=%s; relying on id-upsert",- kb_id_str,- doc_id,- )- try:- await thread_pool_exec(- settings.docStoreConn.insert,- [row],- index_nm,- ctx.kb_id,- )- logging.info(- "raptor_graph: stored %d entities / %d relations for kb=%s doc=%s",- len(graph["entities"]),- len(graph["relations"]),- kb_id_str,- doc_id,- )- except Exception:- logging.exception(- "raptor_graph: insert failed for kb=%s doc=%s",- kb_id_str,- doc_id,- )diff --git a/rag/svr/task_executor_refactor/task_handler.py b/rag/svr/task_executor_refactor/task_handler.pyindex de65c52f49..703d8c9e6c 100644--- a/rag/svr/task_executor_refactor/task_handler.py+++ b/rag/svr/task_executor_refactor/task_handler.py@@ -469,26 +469,6 @@ class TaskHandler:if cleaned_chunks:ctx.progress_cb(msg=f"Cleaned up {cleaned_chunks} stale RAPTOR chunks.")- # Build the per-doc RAPTOR tree graph from the just-- # inserted summaries. Each chunk in ``chunks`` carries- # the doc_id it was written under (real doc id for- # scope="file"; GRAPH_RAPTOR_FAKE_DOC_ID for the- # dataset-scope path). We materialize one graph row per- # distinct doc_id so the dataset structure-graph- # endpoint can surface a RAPTOR tab per document.- # Failure here is best-effort — the summaries are- # already persisted; the tab just won't render.- raptor_doc_ids = {str(c.get("doc_id")) for c in chunks if c.get("doc_id")}- for raptor_doc_id in raptor_doc_ids:- try:- await raptor_service._persist_raptor_graph_to_es(raptor_doc_id)- except Exception:- logging.exception(- "raptor_graph: build failed for kb=%s doc=%s",- task_dataset_id,- raptor_doc_id,- )-# Update document statsif ctx.write_interceptor:ctx.write_interceptor.intercept("DocumentService.increment_chunk_num")diff --git a/test/unit_test/api/apps/services/test_dataset_api_service_nav.py b/test/unit_test/api/apps/services/test_dataset_api_service_nav.pyindex d904c0afe4..13f1962698 100644--- a/test/unit_test/api/apps/services/test_dataset_api_service_nav.py+++ b/test/unit_test/api/apps/services/test_dataset_api_service_nav.py@@ -152,7 +152,10 @@ def _load_nav_module(monkeypatch, *, accessible=True, index_pack=("idx-1", None)thread_pool_exec_long_time=AsyncMock(side_effect=lambda fn, *args, **kwargs: fn(*args, **kwargs)),)_stub(monkeypatch, "rag.advanced_rag.knowlege_compile.wiki", WIKI_PAGE_COMPILE_KWD="wiki")- _stub(monkeypatch, "common.doc_store.doc_store_base", OrderByExpr=MagicMock)+ # The navigation legs build keyword/dense expressions from these doc-store+ # base classes; the compiled-agg keyword fallback and claim-agg vector leg+ # both import MatchTextExpr from here (not rag.nlp.search).+ _stub(monkeypatch, "common.doc_store.doc_store_base", OrderByExpr=MagicMock, MatchTextExpr=_StubMatchTextExpr)repo_root = Path(__file__).resolve().parents[5]module_path = repo_root / "api" / "apps" / "services" / "dataset_api_service.py"@@ -253,31 +256,32 @@ async def test_delete_nav_returns_zero_when_index_missing(monkeypatch):# ----------------------------------------------------------------------------# Document routing via compiled page_index rows+# Document routing via compiled claim rows# ----------------------------------------------------------------------------def _compiled_row(doc_id, rtype, similarity, name="n", payload=None):- body = payload if payload is not None else {"type": rtype, "name": name, "description": "d"}+def _claim_row(doc_id, similarity, name="n"):+ body = {"type": "claim", "name": name, "description": "d"}return {"doc_id": doc_id, "similarity": similarity, "content_with_weight": json.dumps(body)}-def test_nav_bucket_compiled_rows_splits_title_and_fact(monkeypatch):+def test_nav_bucket_compiled_rows_bins_claims_per_document(monkeypatch):module, _, _ = _load_nav_module(monkeypatch)buckets = module._nav_bucket_compiled_rows({- "r1": _compiled_row("doc-a", "title", 0.9),- "r2": _compiled_row("doc-a", "fact", 0.5),- "r3": _compiled_row("doc-a", "conclusion", 0.4),- "r4": _compiled_row("doc-b", "title", 0.7),- "r5": _compiled_row("doc-c", "relation", 0.6), # non-entity type — dropped+ "r1": _claim_row("doc-a", 0.9),+ "r2": _claim_row("doc-a", 0.5),+ "r3": _claim_row("doc-b", 0.7),+ # A title row is never an evidence row, so it drops. (Only+ # "claim" rows count as evidence rows now, so a stale fact row+ # would exercise this same drop path.)+ "r4": {"doc_id": "doc-c", "similarity": 0.6, "content_with_weight": json.dumps({"type": "title"})},})- assert set(buckets["title"]) == {"doc-a", "doc-b"}- assert set(buckets["fact"]) == {"doc-a"}- # conclusion shares the fact bucket, so doc-a has two fact-class hits.- assert buckets["fact"]["doc-a"]["hits"] == 2- assert buckets["fact"]["doc-a"]["best"] == pytest.approx(0.5)+ assert set(buckets["claim"]) == {"doc-a", "doc-b"}+ assert buckets["claim"]["doc-a"]["hits"] == 2+ assert buckets["claim"]["doc-a"]["best"] == pytest.approx(0.9)+ assert buckets["claim"]["doc-a"]["total"] == pytest.approx(1.4)def test_nav_bucket_compiled_rows_skips_unparseable_payloads(monkeypatch):@@ -285,79 +289,33 @@ def test_nav_bucket_compiled_rows_skips_unparseable_payloads(monkeypatch):buckets = module._nav_bucket_compiled_rows({"r1": {"doc_id": "doc-a", "similarity": 0.9, "content_with_weight": "{not json"},- "r2": {"doc_id": "", "similarity": 0.9, "content_with_weight": json.dumps({"type": "title"})},+ "r2": {"doc_id": "", "similarity": 0.9, "content_with_weight": json.dumps({"type": "claim"})},"r3": {"doc_id": "doc-b", "similarity": 0.9, "content_with_weight": json.dumps([1, 2])},})- assert buckets == {"title": {}, "fact": {}, "fact_names": {}}+ assert buckets == {"claim": {}}-def test_nav_bucket_compiled_rows_tracks_fact_names_per_document(monkeypatch):- """Fact names must stay per-document or the bridge labels the wrong document."""+def test_nav_rank_compiled_buckets_orders_by_claim_score(monkeypatch):+ """Claims are the only compiled leg, so ranking is a straight DocScore order."""module, _, _ = _load_nav_module(monkeypatch)buckets = module._nav_bucket_compiled_rows({- "r1": _compiled_row("doc-a", "fact", 0.9, name="A1"),- "r2": _compiled_row("doc-a", "fact", 0.8, name="A2"),- "r3": _compiled_row("doc-b", "fact", 0.7, name="B1"),- }- )- assert buckets["fact_names"] == {"doc-a": ["A1", "A2"], "doc-b": ["B1"]}---def test_nav_matched_sections_dedups_and_keeps_hit_order(monkeypatch):- module, _, _ = _load_nav_module(monkeypatch)- bridge = {"b1": "Discussion", "a1": "Methods", "a2": "Methods"}- out = module._nav_matched_sections(bridge, ["a1", "b1", "a2"])- assert out == ["Methods", "Discussion"]---def test_nav_matched_sections_omits_unbridged_facts(monkeypatch):- module, _, _ = _load_nav_module(monkeypatch)- assert module._nav_matched_sections({}, ["a1"]) == []---async def test_compiled_agg_attaches_matched_sections(monkeypatch):- """Routing must hand back a document-internal entry point, not just a doc_id."""- module, _, doc_store = _load_nav_module(- monkeypatch,- field_map={"r1": _compiled_row("doc-a", "fact", 0.9, name="F1")},- )- doc_store.get_fields = MagicMock(return_value={"r1": _compiled_row("doc-a", "fact", 0.9, name="F1")})- monkeypatch.setattr(module, "_nav_bridge_sections", AsyncMock(return_value={"f1": "Methods"}))- module.settings.retriever = SimpleNamespace(- get_vector=AsyncMock(return_value=SimpleNamespace()),- _existing_doc_ids=AsyncMock(return_value={"doc-a"}),- )- kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")-- ok, payload = await module._search_layers_compiled_agg("tenant-1", "kb-1", "q", 5, SimpleNamespace(), kb)- assert ok is True- assert payload["items"][0]["_nav"]["matched_sections"] == ["Methods"]---def test_nav_rank_compiled_buckets_rewards_both_legs(monkeypatch):- """A document hit by title AND fact outranks one hit by only one leg."""- module, _, _ = _load_nav_module(monkeypatch)- buckets = module._nav_bucket_compiled_rows(- {- "r1": _compiled_row("both", "title", 0.8),- "r2": _compiled_row("both", "fact", 0.8),- "r3": _compiled_row("titleonly", "title", 0.9),+ "r1": _claim_row("weak", 0.5),+ "r2": _claim_row("strong", 0.9),})ranked = module._nav_rank_compiled_buckets(buckets, 10)- assert [d for d, _ in ranked] == ["both", "titleonly"]- assert ranked[0][1]["legs"] == {"title", "fact"}+ assert [d for d, _ in ranked] == ["strong", "weak"]+ assert ranked[0][1]["legs"] == {"claim"}# best stays on the 0..1 cosine scale callers threshold on.- assert ranked[0][1]["best"] == pytest.approx(0.8)+ assert ranked[0][1]["best"] == pytest.approx(0.9)def test_nav_rank_compiled_buckets_does_not_mutate_input(monkeypatch):"""Regression: ranking used to attach 'legs' onto the caller's bucket entries."""module, _, _ = _load_nav_module(monkeypatch)- buckets = module._nav_bucket_compiled_rows({"r1": _compiled_row("doc-a", "title", 0.8)})+ buckets = module._nav_bucket_compiled_rows({"r1": _claim_row("doc-a", 0.8)})before = json.dumps(buckets, sort_keys=True, default=str)module._nav_rank_compiled_buckets(buckets, 10)assert json.dumps(buckets, sort_keys=True, default=str) == before@@ -365,137 +323,10 @@ def test_nav_rank_compiled_buckets_does_not_mutate_input(monkeypatch):def test_nav_rank_compiled_buckets_respects_top_k(monkeypatch):module, _, _ = _load_nav_module(monkeypatch)- buckets = module._nav_bucket_compiled_rows({f"r{i}": _compiled_row(f"doc-{i}", "fact", 0.9 - i * 0.1) for i in range(5)})+ buckets = module._nav_bucket_compiled_rows({f"r{i}": _claim_row(f"doc-{i}", 0.9 - i * 0.1) for i in range(5)})assert len(module._nav_rank_compiled_buckets(buckets, 2)) == 2-async def test_compiled_agg_router_uses_keyword_fallback_without_embedding(monkeypatch):- """No embedding model: the leg degrades to a keyword match, it does not drop out."""- module, _, doc_store = _load_nav_module(- monkeypatch,- field_map={"r1": _compiled_row("doc-a", "fact", 0.5)},- )- doc_store.get_fields = MagicMock(return_value={"r1": _compiled_row("doc-a", "fact", 0.5)})- kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")-- ok, payload = await module._search_layers_compiled_agg("tenant-1", "kb-1", "q", 5, None, kb)- assert ok is True- assert [i["doc_id"] for i in payload["items"]] == ["doc-a"]- # _nav_doc_summaries issues a second search, so read the first one.- exprs = doc_store.search.call_args_list[0].kwargs["match_expressions"]- # A keyword-match expr, not a dense one.- assert len(exprs) == 1- assert isinstance(exprs[0], _StubMatchTextExpr)- assert exprs[0].fields == ["content_ltks", "content_sm_ltks"]---async def test_compiled_agg_router_filters_deleted_documents(monkeypatch):- """Compiled rows can outlive deleted docs; the leg must check existence itself."""- module, _, doc_store = _load_nav_module(- monkeypatch,- field_map={"r1": _compiled_row("doc-gone", "fact", 0.9)},- )- doc_store.get_fields = MagicMock(return_value={"r1": _compiled_row("doc-gone", "fact", 0.9)})- retriever = SimpleNamespace(- get_vector=AsyncMock(return_value=SimpleNamespace()),- _existing_doc_ids=AsyncMock(return_value=set()), # nothing survives- )- module, _, _ = _load_nav_module(monkeypatch, field_map={"r1": _compiled_row("doc-gone", "fact", 0.9)}, retriever=retriever)- kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")-- ok, payload = await module._search_layers_compiled_agg("tenant-1", "kb-1", "q", 5, SimpleNamespace(), kb)- assert ok is True- assert payload["total"] == 0---async def test_compiled_agg_router_builds_vector_with_named_candidates(monkeypatch):- """Regression: positional passing put the similarity threshold into num_candidates."""- get_vector = AsyncMock(return_value=SimpleNamespace())- module, _, doc_store = _load_nav_module(- monkeypatch,- field_map={"r1": _compiled_row("doc-a", "fact", 0.9)},- retriever=SimpleNamespace(get_vector=get_vector, _existing_doc_ids=AsyncMock(return_value={"doc-a"})),- )- doc_store.get_fields = MagicMock(return_value={"r1": _compiled_row("doc-a", "fact", 0.9)})- kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")-- await module._search_layers_compiled_agg("tenant-1", "kb-1", "q", 5, SimpleNamespace(), kb)- kw = get_vector.call_args.kwargs- assert kw["top_k"] == module._NAV_COMPILED_POOL- # HNSW ef_search must be >= top_k, and must not receive the similarity value.- assert kw["num_candidates"] == module._NAV_COMPILED_POOL- assert kw["similarity"] == module._NAV_COMPILED_SIMILARITY- # _nav_doc_summaries issues a second search, so read the first one.- cond = doc_store.search.call_args_list[0].kwargs["condition"]- # Compiled rows carry no available_int, so it must NOT be used as a filter.- assert "available_int" not in cond- assert cond["scope_kwd"] == ["doc"]- assert cond["knowledge_graph_kwd"] == ["entity"]---# ----------------------------------------------------------------------------# Fusion of the chunk leg and the compiled leg-# ------------------------------------------------------------------------------def test_nav_fuse_legs_normalizes_each_leg(monkeypatch):- """Each leg is normalized by its own peak so raw cosine cannot dominate."""- module, _, _ = _load_nav_module(monkeypatch)- ranked = module._nav_fuse_legs(- [- ("chunk", [{"doc_id": "a", "score": 0.9}, {"doc_id": "b", "score": 0.45}], 1.0),- ("compiled", [{"doc_id": "c", "score": 0.6}], 1.0),- ],- 10,- )- scores = {d: info["fused"] for d, info in ranked}- # Peak of each leg becomes 1.0, so a document top-ranked by either leg ties.- assert scores["a"] == pytest.approx(1.0)- assert scores["c"] == pytest.approx(1.0)- assert scores["b"] == pytest.approx(0.5)---def test_nav_fuse_legs_prefers_documents_hit_by_both(monkeypatch):- module, _, _ = _load_nav_module(monkeypatch)- ranked = module._nav_fuse_legs(- [- ("chunk", [{"doc_id": "both", "score": 0.5}, {"doc_id": "chunkonly", "score": 1.0}], 1.0),- ("compiled", [{"doc_id": "both", "score": 1.0}], 1.0),- ],- 10,- )- assert [d for d, _ in ranked] == ["both", "chunkonly"]- assert ranked[0][1]["legs"] == {"chunk", "compiled"}---def test_nav_fuse_legs_survives_empty_leg(monkeypatch):- module, _, _ = _load_nav_module(monkeypatch)- ranked = module._nav_fuse_legs([("chunk", [], 1.0), ("compiled", [], 1.0)], 10)- assert ranked == []---async def test_fusion_router_falls_back_to_chunk_leg_without_compiled(monkeypatch):- """A raptor KB has no page_index rows; the chunk leg must decide alone."""- retrieval = AsyncMock(return_value={"chunks": [{"doc_id": "doc-a", "similarity": 0.5}]})- module, _, _ = _load_nav_module(monkeypatch, retriever=SimpleNamespace(retrieval=retrieval))- kb = SimpleNamespace(tenant_id="tenant-1", id="kb-1")-- ok, payload = await module._search_layers_fusion("tenant-1", "kb-1", "q", 5, None, kb, compiled={"items": []})- assert ok is True- assert [i["doc_id"] for i in payload["items"]] == ["doc-a"]- # No compiled hits means no section bridge lookup either.- assert payload["items"][0]["_agg"]["legs"] == ["chunk"]---# ----------------------------------------------------------------------------# Document routing via chunk aggregation (PageIndex semantics recipe)-# -----------------------------------------------------------------------------def test_nav_aggregate_chunks_rolls_up_per_document(monkeypatch):module, _, _ = _load_nav_module(monkeypatch)chunks = [@@ -604,70 +435,6 @@ async def test_chunk_agg_router_can_include_compiled_rows(monkeypatch):assert retrieval.call_args.kwargs["must_not"] is None-def test_nav_label_items_wraps_flat_row_for_the_caller(monkeypatch):- """navigation.py reads the route label from item["_nav"]["description"].-- search_dataset_nav returns the leaf row flat, so without wrapping the route- comes back unlabelled.- """- module, _, _ = _load_nav_module(monkeypatch)- out = module._nav_label_items([{"doc_id": "doc-a", "name": "Leaf A", "description": "Summary A"}])- assert out[0]["_nav"] == {"doc_id": "doc-a", "description": "Summary A"}---def test_nav_label_items_falls_back_to_name(monkeypatch):- module, _, _ = _load_nav_module(monkeypatch)- out = module._nav_label_items([{"doc_id": "doc-a", "name": "Leaf A", "description": ""}])- assert out[0]["_nav"]["description"] == "Leaf A"---def test_nav_label_items_preserves_existing_nav(monkeypatch):- """A row that already carries _nav must not be overwritten."""- module, _, _ = _load_nav_module(monkeypatch)- out = module._nav_label_items([{"doc_id": "doc-a", "description": "flat", "_nav": {"description": "nested"}}])- assert out[0]["_nav"]["description"] == "nested"---def test_nav_label_items_tolerates_non_dicts(monkeypatch):- module, _, _ = _load_nav_module(monkeypatch)- assert module._nav_label_items(["junk", None]) == ["junk", None]---def test_nav_focus_items_sorts_and_truncates_to_limit(monkeypatch):- """Regression: the flat sweep returned every row above the 0.2 floor.-- Main's cluster descent routes ~2 documents; the flat sweep returned 6-12,- and routing to 3x the documents made the RAGAgent carry 3x the evidence in- every round — each dynamic LLM call grew ~33% and per-question time ~25%.- """- module, _, _ = _load_nav_module(monkeypatch)- monkeypatch.setattr(module, "_NAV_DOC_FOCUS_LIMIT", 3)- # Not sorted — must sort by score before truncating.- items = [{"doc_id": f"doc-{i}", "score": s} for i, s in [(6, 0.1), (1, 0.9), (3, 0.4), (2, 0.6)]]- out = module._nav_focus_items(items)- assert [i["doc_id"] for i in out] == ["doc-1", "doc-2", "doc-3"]- # The input list is truncated in place.- assert [i["doc_id"] for i in items] == ["doc-1", "doc-2", "doc-3"]---def test_nav_focus_items_keeps_fewer_than_limit_untouched(monkeypatch):- """When few docs are relevant, do not pad — routing stays honest."""- module, _, _ = _load_nav_module(monkeypatch)- monkeypatch.setattr(module, "_NAV_DOC_FOCUS_LIMIT", 3)- items = [{"doc_id": "doc-1", "score": 0.9}]- out = module._nav_focus_items(items)- assert len(out) == 1---def test_nav_focus_items_disabled_when_limit_is_zero(monkeypatch):- module, _, _ = _load_nav_module(monkeypatch)- monkeypatch.setattr(module, "_NAV_DOC_FOCUS_LIMIT", 0)- items = [{"doc_id": f"doc-{i}", "score": s} for i, s in [(1, 0.9), (2, 0.6), (3, 0.4)]]- out = module._nav_focus_items(items)- assert len(out) == 3--async def test_chunk_agg_router_focuses_to_limit(monkeypatch):"""chunk_agg must cap the returned documents like nav_doc does.diff --git a/test/unit_test/rag/advanced_rag/test_dataset_navigation_search.py b/test/unit_test/rag/advanced_rag/test_dataset_navigation_search.pyindex fccca4100f..b61ada30ea 100644--- a/test/unit_test/rag/advanced_rag/test_dataset_navigation_search.py+++ b/test/unit_test/rag/advanced_rag/test_dataset_navigation_search.py@@ -18,7 +18,7 @@ async def test_dataset_navigation_search_ranks_across_all_bound_kbs(monkeypatch)kb1_items = [{"doc_id": f"kb1-doc-{i}", "score": 0.30 + i * 0.01} for i in range(_NAV_SEARCH_MAX_DOCS)]kb2_items = [{"doc_id": "kb2-best", "score": 0.99}]- async def fake_search_dataset_layers(kb_id, tenant_id, query, mode, top_k, doc_scope):+ async def fake_search_dataset_layers(kb_id, tenant_id, query, mode, *, top_k, doc_scope, router=None):assert query == "topic keywords"# Diverges from main's "nav_doc" on purpose: routing now descends the# compiled nav TREE (BFS beam, `search_nav_tree_descent`) instead of@@ -26,6 +26,9 @@ async def test_dataset_navigation_search_ranks_across_all_bound_kbs(monkeypatch)assert mode == "navigation_tree"assert top_k == _NAV_SEARCH_MAX_DOCSassert doc_scope is None+ # Agentic rag routes by claims (atomic propositions + verbatim+ # evidence) with raw-chunk aggregation as the fallback.+ assert router == "claim_agg"if kb_id == kb1.id:return True, {"items": kb1_items}if kb_id == kb2.id:diff --git a/web/src/components/structure-graph/adapters.ts b/web/src/components/structure-graph/adapters.tsxsimilarity index 81%rename from web/src/components/structure-graph/adapters.tsrename to web/src/components/structure-graph/adapters.tsxindex 65f74efa8d..c9bb97c2ee 100644--- a/web/src/components/structure-graph/adapters.ts+++ b/web/src/components/structure-graph/adapters.tsx@@ -15,12 +15,14 @@*/import trim from 'lodash/trim';+import { ClaimBadge } from '@/components/structure-graph/claim-badge';import { type TreeDataItem } from '@/components/ui/tree-view';import {type IArtifactGraph,type IArtifactGraphEntity,} from '@/interfaces/database/dataset';import {+ type IClaimEvidence,type IStructureGraphEntity,type IStructureGraphRelation,type IStructureGraphTemplate,@@ -30,6 +32,9 @@ import { type TreeData } from '@antv/g6/lib/types';declare module '@/components/ui/tree-view' {interface TreeDataItem {source_chunk_ids?: string[];+ /** page_index fact/conclusion: gate-verified quotes for the detail panel. */+ description?: string;+ evidence?: IClaimEvidence[];}}@@ -87,6 +92,16 @@ function buildTreeDataItems(name: entity.name,entityType: showEntityType ? entity.type : undefined,source_chunk_ids: entity.source_chunk_ids,+ // Leaf clusters only: the tree shows this as a count badge and the+ // claims themselves are fetched on click. The shared TreeView's+ // badge slot is a ReactNode, so the claim domain owns the rendering.+ badge: <ClaimBadge value={entity.claim_count} />,+ // page_index fact/conclusion: gate-verified quotes rendered in the+ // node's detail panel. Without badge/evidence/description the tree and+ // page_index nodes reached the click handler empty, so the detail panel+ // could never open for them.+ evidence: entity.evidence,+ description: getEntityDescription(entity),},]),);@@ -136,6 +151,12 @@ function buildUniqueTreeDataItems(name: entity.name,entityType: entity.type,source_chunk_ids: entity.source_chunk_ids,+ // Leaf clusters only: the tree UI shows this as a count badge and+ // fetches the claims themselves on expand.+ badge: entity.claim_count,+ // page_index fact/conclusion: gate-verified quotes shown in the+ // node's detail panel.+ evidence: entity.evidence,},]),);@@ -175,15 +196,29 @@ function buildUniqueTreeDataItems(.filter((item): item is TreeDataItem => item !== undefined);}+// Entity types that are never part of the navigation tree. A claim is an+// atomic, evidence-bearing proposition: it answers "what does this section+// say", which is the middle column's job, not the outline's. Leaving claims in+// the tree buried the heading hierarchy under hundreds of flat leaves.+// ``fact``/``conclusion`` are the pre-rename spellings of the same object, kept+// so templates compiled before the rename render the same way.+const PAGE_INDEX_NON_TREE_TYPES = new Set(['claim', 'fact', 'conclusion']);+export function adaptPageIndexToTreeData(template: IStructureGraphTemplate,): TreeDataItem[] {- return buildTreeDataItems(- template.entities,- template.relations,- ['include'],- true,+ // Filtered before building: relations pointing at a removed entity are+ // dropped by the builder, so headings whose only children were claims simply+ // become leaves instead of dangling.+ const entities = (template.entities ?? []).filter(+ (entity) =>+ !PAGE_INDEX_NON_TREE_TYPES.has(+ String((entity as { type?: string })?.type ?? '')+ .trim()+ .toLowerCase(),+ ),);+ return buildTreeDataItems(entities, template.relations, ['include'], true);}export function adaptTreeToTreeData(diff --git a/web/src/components/structure-graph/claim-badge.tsx b/web/src/components/structure-graph/claim-badge.tsxnew file mode 100644index 0000000000..b3ab92b328--- /dev/null+++ b/web/src/components/structure-graph/claim-badge.tsx@@ -0,0 +1,13 @@+// Small count pill for leaf nodes whose facts are inspectable (e.g. a tree+// cluster's claim count). Hidden when zero/undefined so structural nodes stay+// uncluttered. Claim-domain on purpose: the shared TreeView accepts any+// `badge?: React.ReactNode` slot, and this is the concrete badge the+// structure-graph tree feeds into it.+export function ClaimBadge({ value }: { value?: number }) {+ if (typeof value !== 'number' || value <= 0) return null;+ return (+ <span className="shrink-0 rounded-full bg-accent/15 px-1.5 py-0.5 text-[10px] leading-none text-accent-foreground">+ {value}+ </span>+ );+}diff --git a/web/src/components/structure-graph/representation-renderer.tsx b/web/src/components/structure-graph/representation-renderer.tsxindex 5db35963c7..6a6f0a90c8 100644--- a/web/src/components/structure-graph/representation-renderer.tsx+++ b/web/src/components/structure-graph/representation-renderer.tsx@@ -22,6 +22,7 @@ import {type IArtifactGraphEntity,} from '@/interfaces/database/dataset';import {+ type IClaimEvidence,type IStructureGraphTemplate,type StructureTemplateKind,} from '@/interfaces/database/document-structure';@@ -40,6 +41,13 @@ export interface ClickableNode {id: string;name?: string;source_chunk_ids?: string[];+ /** Tree leaves only: whether this node has child clusters. */+ hasChildren?: boolean;+ /** Leaf claim badge (tree slot). 0/undefined ⇒ nothing for the claims panel. */+ badge?: React.ReactNode;+ /** page_index fact/conclusion: gate-verified quotes for the detail panel. */+ description?: string;+ evidence?: IClaimEvidence[];}const EmptyForceGraphData: IArtifactGraph = { entities: [], relations: [] };@@ -79,6 +87,12 @@ export function RepresentationRenderer({id: item.id,name: item.name,source_chunk_ids: item.source_chunk_ids,+ // Tree leaves are where the claims UI attaches; branches are pure+ // structure and clicking them keeps the old behaviour.+ hasChildren: !!item.children?.length,+ badge: item.badge,+ description: item.description,+ evidence: item.evidence,});}},diff --git a/web/src/components/ui/tree-view.tsx b/web/src/components/ui/tree-view.tsxindex 3610c1029f..11fe9cf093 100644--- a/web/src/components/ui/tree-view.tsx+++ b/web/src/components/ui/tree-view.tsx@@ -20,6 +20,13 @@ export interface TreeDataItem {id: string;name: string;entityType?: string;+ /**+ * Domain-agnostic trailing slot next to the node name. The tree renders+ * whatever the caller passes (e.g. the structure graph's claim-count pill);+ * the shared component knows nothing about the business meaning. Hidden+ * callers simply omit it.+ */+ badge?: React.ReactNode;icon?: any;selectedIcon?: any;openIcon?: any;@@ -49,7 +56,10 @@ type TreeProps = React.HTMLAttributes<HTMLDivElement> & {const TreeItemLabel = ({ item }: { item: TreeDataItem }) => {if (!item.entityType) {return (- <span className="flex-grow truncate text-sm text-left">{item.name}</span>+ <span className="flex min-w-0 flex-grow items-center gap-2 text-left">+ <span className="truncate text-sm">{item.name}</span>+ {item.badge}+ </span>);}@@ -74,6 +84,7 @@ const TreeItemLabel = ({ item }: { item: TreeDataItem }) => {>{item.entityType}</span>+ {item.badge}</span>);};diff --git a/web/src/hooks/use-document-request.ts b/web/src/hooks/use-document-request.tsindex 8ced646e1c..fff36de160 100644--- a/web/src/hooks/use-document-request.ts+++ b/web/src/hooks/use-document-request.ts@@ -25,7 +25,10 @@ import {IDocumentInfo,IDocumentInfoFilter,} from '@/interfaces/database/document';-import { IStructureGraphResponse } from '@/interfaces/database/document-structure';+import {+ IClaimsResponse,+ IStructureGraphResponse,+} from '@/interfaces/database/document-structure';import {IChangeParserConfigRequestBody,IDocumentMetaRequestBody,@@ -100,6 +103,20 @@ export const DocumentStructureKeys = {documentId,keywords,] as const,+ claims: (+ datasetId: string,+ documentId: string,+ templateId: string | undefined,+ chunkIds: string[] | undefined,+ ) =>+ [+ DocumentStructureApiAction.FetchDocumentStructureGraph,+ datasetId,+ documentId,+ 'claims',+ templateId,+ ...(chunkIds ?? []),+ ] as const,};export const useUploadDocument = () => {@@ -824,6 +841,43 @@ export function useFetchDocumentStructureGraph(keywords?: string) {return { data, loading };}+// Claims are fetched per leaf cluster on demand: the tree shows only a count+// badge, so the payload (statement + verbatim evidence) loads when the user+// opens that cluster. chunkIds scopes the query server-side.+export function useFetchDocumentClaims(+ chunkIds: string[] | undefined,+ templateId: string | undefined,+) {+ const { knowledgeId: datasetId, documentId } = useGetKnowledgeSearchParams();+ const enabled = !!datasetId && !!documentId && !!chunkIds?.length;++ const { data, isFetching: loading } = useQuery<IClaimsResponse | null>({+ queryKey: DocumentStructureKeys.claims(+ datasetId,+ documentId,+ templateId,+ chunkIds,+ ),+ enabled,+ gcTime: 0,+ queryFn: async () => {+ const { data } =+ await documentStructureService.getDocumentStructureClaims(+ datasetId,+ documentId,+ {+ chunk_ids: chunkIds?.join(','),+ template_id: templateId,+ limit: 100,+ },+ );+ return data?.data ?? null;+ },+ });++ return { data, loading };+}+export function useDeleteDocumentStructureGraph() {const { knowledgeId: datasetId, documentId } = useGetKnowledgeSearchParams();const queryClient = useQueryClient();diff --git a/web/src/interfaces/database/compilation-template.ts b/web/src/interfaces/database/compilation-template.tsindex c730a0eeba..6d2dc0f7e1 100644--- a/web/src/interfaces/database/compilation-template.ts+++ b/web/src/interfaces/database/compilation-template.ts@@ -12,6 +12,7 @@ export interface ICompilationTemplateSection {export interface ICompilationTemplateRaptorConfig {prompt?: string;+ claim_prompt?: string;max_token?: number;clustering_threshold?: number;clustering_ratio?: number;diff --git a/web/src/interfaces/database/document-structure.ts b/web/src/interfaces/database/document-structure.tsindex 4cfa9d20a4..d6beaa2d74 100644--- a/web/src/interfaces/database/document-structure.ts+++ b/web/src/interfaces/database/document-structure.ts@@ -11,6 +11,32 @@ export interface IStructureGraphEntity {mention_count?: number;source_chunk_ids?: string[];type?: string;+ /** Leaf clusters only: claims sourced to this cluster's chunks. */+ claim_count?: number;+ /** page_index fact/conclusion only: gate-verified verbatim quotes. */+ evidence?: IClaimEvidence[];+}++export interface IClaimEvidence {+ quote: string;+ chunk_id: string;+ start?: number;+ end?: number;+}++export interface IClaimItem {+ name: string;+ description?: string;+ source_chunk_ids?: string[];+ type?: string;+ evidence?: IClaimEvidence[];+}++export interface IClaimsResponse {+ claims: IClaimItem[];+ total: number;+ offset: number;+ limit: number;}export interface IStructureGraphRelation {diff --git a/web/src/interfaces/request/compilation-template.ts b/web/src/interfaces/request/compilation-template.tsindex 6bb632080b..28601b85e0 100644--- a/web/src/interfaces/request/compilation-template.ts+++ b/web/src/interfaces/request/compilation-template.ts@@ -16,6 +16,7 @@ export interface ICompilationTemplateRaptorConfigRequest {clustering_threshold?: number;clustering_ratio?: number;rechunk?: boolean;+ claim_prompt?: string;}export interface ICompilationTemplateConfigRequest {diff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex eaee6a2204..7cc2f620a0 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -2377,6 +2377,9 @@ Example: Virtual Hosted Style`,'Entity mode: one Wiki page per entity or concept. Topic mode: let the LLM group related entities or concepts into the same Wiki page by topic.',raptorTreeSettings: 'RAPTOR tree settings',summarizationPrompt: 'Summarization prompt',+ claimExtractionPrompt: 'Claim extraction prompt',+ claimExtractionPromptTip:+ 'System prompt for harvesting atomic claims with verbatim evidence from the leaf chunks before clustering. Leave empty to use the built-in contract.',maxToken: 'Max token',maxTokenRequired: 'Please input max token',clusteringThreshold: 'Clustering threshold',@@ -2470,6 +2473,14 @@ Example: Virtual Hosted Style`,representationEmpty: 'No artifact templates available.',representationUnsupported:'This representation type is not supported yet.',+ claimsPanelTitle: 'Claims · {{name}}',+ claimsTotal: '{{count}} total',+ claimsLoading: 'Loading claims…',+ claimsEmpty: 'No claims were extracted for this cluster.',+ claimsLoadMore: 'Load more ({{remaining}} remaining)',+ claimsMoreEvidence: '+{{count}} more evidence',+ claimsCollapseEvidence: 'Show less',+ claimsNodeDetail: 'Details',},message: {registered: 'Registered!',diff --git a/web/src/locales/ja.ts b/web/src/locales/ja.tsindex 4c747e6728..a7f499cf57 100644--- a/web/src/locales/ja.ts+++ b/web/src/locales/ja.ts@@ -3626,6 +3626,14 @@ export default {skillEmpty: '利用可能なスキルがありません',representationEmpty: '利用可能な表現テンプレートがありません。',representationUnsupported: 'この表現形式にはまだ対応していません。',+ claimsPanelTitle: 'アサーション · {{name}}',+ claimsTotal: '全 {{count}} 件',+ claimsLoading: 'アサーションを読み込み中…',+ claimsEmpty: 'このクラスターからアサーションは抽出されませんでした。',+ claimsLoadMore: 'さらに読み込む(残り {{remaining}} 件)',+ claimsMoreEvidence: '他に {{count}} 件の証拠',+ claimsCollapseEvidence: '折りたたむ',+ claimsNodeDetail: '詳細',},llmTools: {diff --git a/web/src/locales/tr.ts b/web/src/locales/tr.tsindex baa1266186..cb5fc8b583 100644--- a/web/src/locales/tr.ts+++ b/web/src/locales/tr.ts@@ -2410,6 +2410,14 @@ Bu otomatik etiketleme özelliği, mevcut datasete alanına özgü bilgi katman'Bu düğümü ve alt düğümlerini silmek istediğinizden emin misiniz?',representationEmpty: 'Kullanılabilir artifact şablonu yok.',representationUnsupported: 'Bu gösterim türü henüz desteklenmiyor.',+ claimsPanelTitle: 'İddia · {{name}}',+ claimsTotal: 'Toplam {{count}}',+ claimsLoading: 'İddialar yükleniyor…',+ claimsEmpty: 'Bu küme için iddia çıkarılmadı.',+ claimsLoadMore: 'Daha fazla yükle ({{remaining}} kaldı)',+ claimsMoreEvidence: '+{{count}} kanıt daha',+ claimsCollapseEvidence: 'Daha az göster',+ claimsNodeDetail: 'Ayrıntılar',},message: {registered: 'Kayıt olundu!',diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.tsindex d7f458c4a1..56604c1f49 100644--- a/web/src/locales/zh.ts+++ b/web/src/locales/zh.ts@@ -2001,6 +2001,9 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 R'实体模式:每个实体或概念对应一个 Wiki 页面。主题模式:让 LLM 按主题将相关实体或概念组合到同一个 Wiki 页面。',raptorTreeSettings: 'RAPTOR 树设置',summarizationPrompt: '摘要提示词',+ claimExtractionPrompt: 'Claim 抽取提示词',+ claimExtractionPromptTip:+ '聚类前从叶子 chunk 收割原子命题(附逐字证据)的系统提示词。留空使用内置契约。',maxToken: '最大 token 数',maxTokenRequired: '请输入最大 token 数',clusteringThreshold: '聚类阈值',@@ -2087,8 +2090,16 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 RnavDeleteAllDescription: '确定要删除整个 PageIndex 吗?此操作无法撤销。',navDeleteNodeTitle: '删除 PageIndex',navDeleteNodeDescription: '确定要删除该节点及其子节点吗?',- representationEmpty: '暂无Artifact模板。',+ representationEmpty: '暂无 Artifact 模板。',representationUnsupported: '暂不支持该表征类型。',+ claimsPanelTitle: '证据链 · {{name}}',+ claimsTotal: '共 {{count}} 条',+ claimsLoading: '正在加载证据链…',+ claimsEmpty: '未提取到证据链。',+ claimsLoadMore: '加载更多(还剩 {{remaining}} 条)',+ claimsMoreEvidence: '还有 {{count}} 条证据',+ claimsCollapseEvidence: '收起',+ claimsNodeDetail: '详情',},message: {registered: '注册成功',diff --git a/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/document-view-switch/index.tsx b/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/document-view-switch/index.tsxindex c19d66374e..efc9fe500d 100644--- a/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/document-view-switch/index.tsx+++ b/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/document-view-switch/index.tsx@@ -2,7 +2,9 @@ import DocumentPreview from '@/components/document-preview';import DocumentHeader from '@/components/document-preview/document-header';import { Segmented, type SegmentedValue } from '@/components/ui/segmented';import Representation, {+ type ClaimsPanelState,type ClickableNode,+ type EvidencePanelState,} from '@/pages/chunk/representation';import { File, LayoutList } from 'lucide-react';import { useCallback, useState } from 'react';@@ -26,6 +28,10 @@ interface DocumentViewSwitchProps {url: string;positions?: number[][];onChunkIdsChange?: (chunkIds: string[]) => void;+ // The artifact tree publishes its claims / evidence panels upward so the page+ // can render them as a resizable middle column instead of inside the tree.+ onClaimsPanelChange?: (panel: ClaimsPanelState | null) => void;+ onEvidencePanelChange?: (panel: EvidencePanelState | null) => void;}export default function DocumentViewSwitch({@@ -36,6 +42,8 @@ export default function DocumentViewSwitch({url,positions,onChunkIdsChange,+ onClaimsPanelChange,+ onEvidencePanelChange,}: DocumentViewSwitchProps) {const { t } = useTranslation();const [viewMode, setViewMode] = useState<ViewMode>(ViewMode.Preview);@@ -63,7 +71,7 @@ export default function DocumentViewSwitch({label: (<div className="flex items-center gap-1"><File className="h-4 w-4" />- <span>{t('common.preview', 'Preview')}</span>+ <span>{t('common.preview')}</span></div>),},@@ -72,6 +80,7 @@ export default function DocumentViewSwitch({label: (<div className="flex items-center gap-1"><LayoutList className="h-4 w-4" />+ {/* Product term: kept as-is in every locale, like RAPTOR. */}<span>Artifact</span></div>),@@ -105,7 +114,11 @@ export default function DocumentViewSwitch({positions={positions}/>) : (- <Representation onNodeClick={handleNodeClick} />+ <Representation+ onNodeClick={handleNodeClick}+ onClaimsPanelChange={onClaimsPanelChange}+ onEvidencePanelChange={onEvidencePanelChange}+ />)}</div></>diff --git a/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/index.tsx b/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/index.tsxindex bcde074a23..d24bcaad15 100644--- a/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/index.tsx+++ b/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/index.tsx@@ -20,6 +20,12 @@ import CheckboxSets from './components/chunk-result-bar/checkbox-sets';import DocumentViewSwitch from './components/document-view-switch';// import DocumentHeader from './components/document-preview/document-header';+import {+ ClaimsPanel,+ type ClaimsPanelState,+ type EvidencePanelState,+ NodeDetailPanel,+} from '@/pages/chunk/representation/components/claim-list';import { useGetDocumentUrl } from '@/components/document-preview/hooks';import { PageHeader } from '@/components/page-header';import { Button } from '@/components/ui/button';@@ -43,6 +49,13 @@ import { LucideArrowBigLeft } from 'lucide-react';function Chunk() {const [filterChunkIds, setFilterChunkIds] = useState<string[]>([]);const [selectedChunkIds, setSelectedChunkIds] = useState<string[]>([]);+ // The artifact tree publishes its claims / evidence content upward; the page+ // renders it as a resizable column between the tree and the chunk list, and+ // shows only two columns while nothing is open.+ const [claimsPanel, setClaimsPanel] = useState<ClaimsPanelState | null>(null);+ const [evidencePanel, setEvidencePanel] = useState<EvidencePanelState | null>(+ null,+ );const { removeChunk } = useDeleteChunkByIds();const {data: { documentInfo, data = [], total },@@ -79,6 +92,17 @@ function Chunk() {setSelectedChunkIds([]);}, []);+ // Stable identities: the artifact tree republishes its panel content whenever+ // the claims request settles, so an unstable callback would loop that effect.+ const handleClaimsPanelChange = useCallback(+ (panel: ClaimsPanelState | null) => setClaimsPanel(panel),+ [],+ );+ const handleEvidencePanelChange = useCallback(+ (panel: EvidencePanelState | null) => setEvidencePanel(panel),+ [],+ );+useClearSelectionOnPageChange(pagination, clearSelectedChunkIds);const selectAllChunk = useCallback(@@ -176,6 +200,10 @@ function Chunk() {? selectedChunk.positions: [];+ // Two columns until the artifact tree opens a claims / evidence panel: the+ // middle column only exists while there is something to show in it.+ const showArtifactDetail = Boolean(claimsPanel || evidencePanel);+const fileType = useMemo(() => {const name = documentInfo?.name || '';if (name.includes('.')) {@@ -212,7 +240,16 @@ function Chunk() {<Card className="mx-5 mb-5 flex-1 h-0 p-0 bg-transparent shadow-none"><CardContent className="p-0 h-full flex flex-row divide-x-0.5 rtl:divide-x-reverse"><ResizablePanelGroup direction="horizontal" className="flex-1">- <ResizablePanel defaultSize={40} minSize={30}>+ {/* id + order must be explicit: the middle column mounts after the+ first render, and without them react-resizable-panels orders+ panels by registration, so it would sit AFTER the chunk list+ and its resize handles would drag in the wrong direction. */}+ <ResizablePanel+ id="artifact-tree"+ order={1}+ defaultSize={40}+ minSize={20}+ ><article className="h-full flex flex-col"><DocumentViewSwitchdocumentInfo={documentInfo}@@ -222,6 +259,8 @@ function Chunk() {url={fileUrl}positions={positions}onChunkIdsChange={handleChunkIdsChange}+ onClaimsPanelChange={handleClaimsPanelChange}+ onEvidencePanelChange={handleEvidencePanelChange}/></article></ResizablePanel>@@ -231,7 +270,44 @@ function Chunk() {className="bg-border-button w-[0.5px]"/>- <ResizablePanel defaultSize={60} minSize={30}>+ {/* Separate conditionals rather than a fragment: PanelGroup pairs+ each handle with the panels adjacent to it in registration+ order, and a fragment would hide these children from it. */}+ {showArtifactDetail && (+ <ResizablePanel+ id="artifact-detail"+ order={2}+ defaultSize={30}+ minSize={20}+ >+ <article className="h-full flex flex-col">+ {claimsPanel && (+ <div className="flex-1 min-h-0">+ <ClaimsPanel {...claimsPanel} />+ </div>+ )}+ {evidencePanel && (+ <div className="flex-1 min-h-0">+ <NodeDetailPanel {...evidencePanel} />+ </div>+ )}+ </article>+ </ResizablePanel>+ )}++ {showArtifactDetail && (+ <ResizableHandle+ withHandle+ className="bg-border-button w-[0.5px]"+ />+ )}++ <ResizablePanel+ id="chunk-list"+ order={showArtifactDetail ? 3 : 2}+ defaultSize={60}+ minSize={30}+ ><article className="h-full flex flex-col"><header className="flex-0 p-5 pb-2.5 border-b-0.5 border-b-border-button"><h2 className="text-[24px]">{t('chunk.chunkResult')}</h2>diff --git a/web/src/pages/chunk/representation/components/claim-list.tsx b/web/src/pages/chunk/representation/components/claim-list.tsxnew file mode 100644index 0000000000..0f44d4ad42--- /dev/null+++ b/web/src/pages/chunk/representation/components/claim-list.tsx@@ -0,0 +1,237 @@+/*+ * Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { Button } from '@/components/ui/button';+import type {+ IClaimEvidence,+ IClaimItem,+} from '@/interfaces/database/document-structure';+import { ChevronDown, ChevronUp, Loader2, X } from 'lucide-react';+import { useState } from 'react';+import { useTranslation } from 'react-i18next';++// Verbatim quotes rendered as indented citation blocks. `expandable` shows the+// first quote and folds the rest behind a toggle so long lists stay scannable.+export function EvidenceBlock({+ items,+ expandable = true,+}: {+ items: IClaimEvidence[];+ expandable?: boolean;+}) {+ const { t } = useTranslation();+ const [expanded, setExpanded] = useState(false);++ const evidence = items.filter((e) => e.quote?.trim());+ if (!evidence.length) return null;+ const shown = expandable && !expanded ? evidence.slice(0, 1) : evidence;++ return (+ <div className="mt-2 space-y-1.5">+ {shown.map((item, idx) => (+ <blockquote+ key={`${item.chunk_id}-${idx}`}+ className="border-l-2 border-accent/40 pl-2 text-xs italic leading-5 text-text-secondary"+ >+ {item.quote}+ </blockquote>+ ))}+ {expandable && evidence.length > 1 && (+ <button+ type="button"+ className="mt-1.5 inline-flex items-center gap-1 text-xs text-accent hover:underline"+ onClick={() => setExpanded((prev) => !prev)}+ >+ {expanded ? <ChevronUp size={12} /> : <ChevronDown size={12} />}+ {expanded+ ? t('knowledgeCompilation.claimsCollapseEvidence', {+ defaultValue: 'Show less',+ })+ : t('knowledgeCompilation.claimsMoreEvidence', {+ count: evidence.length - 1,+ defaultValue: `+${evidence.length - 1} more evidence`,+ })}+ </button>+ )}+ </div>+ );+}++function ClaimCard({ claim }: { claim: IClaimItem }) {+ return (+ <div className="rounded-lg border border-border-button bg-bg-card px-3 py-2.5">+ <div className="text-sm font-medium text-text-primary">{claim.name}</div>+ {claim.description && claim.description !== claim.name && (+ <div className="mt-1 text-xs text-text-secondary">+ {claim.description}+ </div>+ )}+ <EvidenceBlock items={claim.evidence ?? []} />+ </div>+ );+}++interface ClaimListProps {+ claims: IClaimItem[];+}++export function ClaimList({ claims }: ClaimListProps) {+ const { t } = useTranslation();++ if (!claims.length) {+ return (+ <div className="py-4 text-sm text-text-secondary">+ {t('knowledgeCompilation.claimsEmpty', {+ defaultValue: 'No claims were extracted for this cluster.',+ })}+ </div>+ );+ }++ return (+ <div className="space-y-2">+ {claims.map((claim, idx) => (+ <ClaimCard key={`${claim.name}-${idx}`} claim={claim} />+ ))}+ </div>+ );+}++// The artifact page renders these panels as a resizable middle column, so the+// tree view only owns the selection and hands the resolved content upward. The+// payload carries its own close handler: clearing the selection lives here,+// next to the click handler that set it.+export interface ClaimsPanelState {+ clusterName?: string;+ claims: IClaimItem[];+ total: number;+ loading: boolean;+ onClose: () => void;+}++export interface EvidencePanelState {+ nodeName?: string;+ description?: string;+ evidence: IClaimEvidence[];+ onClose: () => void;+}++// One leaf cluster's claims, opened from the tree's count badge. Rendered as a+// full-height column between the artifact tree and the chunk list, so it shares+// the frame language of those panels: a bordered header on top, a scrollable+// body filling the rest, and a close affordance on the right.+export function ClaimsPanel({+ clusterName,+ claims,+ total,+ loading,+ onClose,+}: ClaimsPanelState) {+ const { t } = useTranslation();++ return (+ <article className="h-full flex flex-col">+ <header className="flex items-center justify-between gap-2 flex-0 p-5 pb-2.5 border-b-0.5 border-b-border-button">+ <h2 className="truncate text-[18px] text-text-primary">+ {t('knowledgeCompilation.claimsPanelTitle', {+ name: clusterName ?? '',+ defaultValue: `Claims · ${clusterName ?? ''}`,+ })}+ </h2>+ <div className="flex items-center gap-2 shrink-0">+ {!loading && (+ <span className="text-[14px] text-text-secondary">+ {t('knowledgeCompilation.claimsTotal', {+ count: total,+ defaultValue: `${total} total`,+ })}+ </span>+ )}+ <Button+ variant="ghost"+ size="icon"+ type="button"+ className="h-6 w-6"+ onClick={onClose}+ aria-label={t('common.close', 'Close')}+ >+ <X size={14} />+ </Button>+ </div>+ </header>+ <div className="flex-1 h-0 overflow-auto scrollbar-auto px-5 pb-5 pt-2.5">+ {loading ? (+ <div className="flex items-center justify-center gap-2 py-6 text-sm text-text-secondary">+ <Loader2 className="animate-spin" size={16} />+ {t('knowledgeCompilation.claimsLoading', {+ defaultValue: 'Loading claims…',+ })}+ </div>+ ) : (+ <ClaimList claims={claims} />+ )}+ </div>+ </article>+ );+}++interface NodeDetailPanelProps {+ nodeName?: string;+ description?: string;+ evidence: IClaimEvidence[];+ onClose: () => void;+}++// A single tree node's detail (page_index fact/conclusion): its description+// plus the gate-verified quotes backing it, shown when the user clicks the+// node in the tree. Shares the citation rendering with the claims panel.+export function NodeDetailPanel({+ nodeName,+ description,+ evidence,+ onClose,+}: NodeDetailPanelProps) {+ const { t } = useTranslation();++ return (+ <article className="h-full flex flex-col">+ <header className="flex items-center justify-between gap-2 flex-0 p-5 pb-2.5 border-b-0.5 border-b-border-button">+ <h2 className="truncate text-[18px] text-text-primary">+ {nodeName ||+ t('knowledgeCompilation.claimsNodeDetail', {+ defaultValue: 'Details',+ })}+ </h2>+ <Button+ variant="ghost"+ size="icon"+ type="button"+ className="h-6 w-6 shrink-0"+ onClick={onClose}+ aria-label={t('common.close', 'Close')}+ >+ <X size={14} />+ </Button>+ </header>+ <div className="flex-1 h-0 overflow-auto scrollbar-auto px-5 pb-5 pt-2.5">+ {description && (+ <p className="text-sm leading-6 text-text-secondary">{description}</p>+ )}+ <EvidenceBlock items={evidence} expandable />+ </div>+ </article>+ );+}diff --git a/web/src/pages/chunk/representation/index.tsx b/web/src/pages/chunk/representation/index.tsxindex 84549d2758..1aa2a98ae0 100644--- a/web/src/pages/chunk/representation/index.tsx+++ b/web/src/pages/chunk/representation/index.tsx@@ -3,26 +3,53 @@ import { ExpandableSearchInput } from '@/components/expandable-search-input';import { SelectWithSearch } from '@/components/originui/select-with-search';import { SkeletonCard } from '@/components/skeleton-card';import { Button } from '@/components/ui/button';-import { useDeleteDocumentStructureGraph } from '@/hooks/use-document-request';+import {+ useDeleteDocumentStructureGraph,+ useFetchDocumentClaims,+} from '@/hooks/use-document-request';import { Trash2 } from 'lucide-react';-import { memo, useCallback } from 'react';+import { memo, useCallback, useEffect, useState } from 'react';import { useTranslation } from 'react-i18next';import {type ClickableNode,RepresentationRenderer,} from '@/components/structure-graph/representation-renderer';+import type {+ ClaimsPanelState,+ EvidencePanelState,+} from './components/claim-list';import { RepresentationSelect } from './components/representation-select';import { useGraphEntitySearch } from './hooks/use-graph-entity-search';+export type {+ ClaimsPanelState,+ EvidencePanelState,+} from './components/claim-list';+interface RepresentationProps {onNodeClick?: (node: ClickableNode) => void;+ // The claims / evidence panels belong to the artifact page's middle column,+ // not inside this tree view. The selection still lives here (it is driven by+ // node clicks), but the resolved content is published upward and the page+ // decides where to render it.+ onClaimsPanelChange?: (panel: ClaimsPanelState | null) => void;+ onEvidencePanelChange?: (panel: EvidencePanelState | null) => void;}-function Representation({ onNodeClick }: RepresentationProps) {+function Representation({+ onNodeClick,+ onClaimsPanelChange,+ onEvidencePanelChange,+}: RepresentationProps) {const { t } = useTranslation();const { deleteDocumentStructureGraph, loading: deleting } =useDeleteDocumentStructureGraph();+ const [claimsLeaf, setClaimsLeaf] = useState<ClickableNode | null>(null);+ const [evidenceDetail, setEvidenceDetail] = useState<ClickableNode | null>(+ null,+ );+const {data,loading,@@ -41,6 +68,113 @@ function Representation({ onNodeClick }: RepresentationProps) {handleNodeClick,} = useGraphEntitySearch(onNodeClick);+ // Tree leaves carry a claim-count badge: clicking one opens its claims in the+ // artifact page's middle column, in addition to the usual chunk navigation.+ // Branch clicks close the panel — they are pure structure and their+ // descendants own the claims. The panels themselves are rendered by the page+ // (as a resizable column), so only the resolved content is published upward.+ const { data: claimsData, loading: claimsLoading } = useFetchDocumentClaims(+ claimsLeaf?.source_chunk_ids,+ selectedTemplateId,+ );++ const handleCloseClaims = useCallback(() => setClaimsLeaf(null), []);+ const handleCloseEvidence = useCallback(() => setEvidenceDetail(null), []);++ // Wait for the fetch to settle before committing to either panel. While it+ // is in flight, publish the claims panel in its loading state so the middle+ // column is stable and honest — publishing the node-detail panel during the+ // load made it flash first and then be overwritten by the claims panel (or,+ // for a node without claims, it just stayed there looking unexplained). The+ // page stacks both panels in one slot, so the choice is exclusive — and it+ // can only be made once the claim count is known.+ const claimsSettled = Boolean(claimsLeaf) && !claimsLoading;+ const hasClaims = claimsSettled && (claimsData?.claims?.length ?? 0) > 0;++ useEffect(() => {+ if (claimsLeaf && !claimsSettled) {+ onClaimsPanelChange?.({+ clusterName: claimsLeaf.name,+ claims: [],+ total: 0,+ loading: true,+ onClose: handleCloseClaims,+ });+ return () => onClaimsPanelChange?.(null);+ }+ // No claims and no node detail: keep the claims panel open with its empty+ // state when the click actually addressed chunks (a real leaf), so the+ // column answers "why nothing" instead of silently closing. Branch nodes+ // without source_chunk_ids close it -- they are pure structure.+ const showEmptyClaims =+ claimsSettled &&+ !hasClaims &&+ !evidenceDetail &&+ Boolean(claimsLeaf?.source_chunk_ids?.length);+ onClaimsPanelChange?.(+ claimsLeaf && (hasClaims || showEmptyClaims)+ ? {+ clusterName: claimsLeaf.name,+ claims: claimsData?.claims ?? [],+ total: claimsData?.total ?? 0,+ loading: claimsLoading,+ onClose: handleCloseClaims,+ }+ : null,+ );+ // Clear on unmount too: switching the left view back to the document+ // preview unmounts the tree, and the page must drop the column with it.+ return () => onClaimsPanelChange?.(null);+ }, [+ claimsLeaf,+ claimsData,+ claimsLoading,+ claimsSettled,+ evidenceDetail,+ hasClaims,+ handleCloseClaims,+ onClaimsPanelChange,+ ]);++ useEffect(() => {+ onEvidencePanelChange?.(+ claimsSettled && !hasClaims && evidenceDetail+ ? {+ nodeName: evidenceDetail.name,+ description: evidenceDetail.description,+ evidence: evidenceDetail.evidence ?? [],+ onClose: handleCloseEvidence,+ }+ : null,+ );+ return () => onEvidencePanelChange?.(null);+ }, [+ claimsSettled,+ evidenceDetail,+ handleCloseEvidence,+ hasClaims,+ onEvidencePanelChange,+ ]);++ const handleNodeClickWithClaims = useCallback(+ (node: ClickableNode) => {+ // Any node can own claims — a page_index heading covers the chunks of its+ // whole section, so the claims belonging to it are the ones sourced from+ // those chunks. Nothing here keys off ``badge``: the structure compiler+ // never writes ``claim_count``, so gating on it kept the panel shut for+ // every node. Whether the panel actually opens is decided by the fetch+ // below, once we know the node has claims.+ setClaimsLeaf(node);+ // The node-detail panel is ONLY for nodes carrying gate-verified quotes+ // (page_index fact/conclusion rows). A description alone would make+ // every tree node fall back to it when no claims exist -- an+ // unexplained block of compiled summary text in the claims slot.+ setEvidenceDetail(node.evidence?.length ? node : null);+ handleNodeClick(node);+ },+ [handleNodeClick],+ );+const handleDelete = useCallback(async () => {if (!selectedTemplateId) return;await deleteDocumentStructureGraph(selectedTemplateId);@@ -97,7 +231,7 @@ function Representation({ onNodeClick }: RepresentationProps) {{!(loading && !data) && templates.length > 0 && (<RepresentationRenderertemplate={selectedTemplate}- onNodeClick={handleNodeClick}+ onNodeClick={handleNodeClickWithClaims}highlightNodeId={highlightNodeId}totalEntities={data?.total_entities}returnedEntities={data?.returned_entities}diff --git a/web/src/pages/user-setting/compilation-templates/edit-next/components/tree-template-fields.tsx b/web/src/pages/user-setting/compilation-templates/edit-next/components/tree-template-fields.tsxindex 6aa0451bd0..0fb04ebe83 100644--- a/web/src/pages/user-setting/compilation-templates/edit-next/components/tree-template-fields.tsx+++ b/web/src/pages/user-setting/compilation-templates/edit-next/components/tree-template-fields.tsx@@ -42,6 +42,18 @@ export function TreeTemplateFields({ index }: TreeTemplateFieldsProps) {/></RAGFlowFormItem>+ <RAGFlowFormItem+ name={`templates.${index}.config.raptor.claim_prompt`}+ label={t('knowledgeCompilation.claimExtractionPrompt')}+ tooltip={t('knowledgeCompilation.claimExtractionPromptTip')}+ >+ <Textarea+ placeholder={t('common.descriptionPlaceholder')}+ rows={12}+ resize="vertical"+ />+ </RAGFlowFormItem>+<SliderInputFormFieldname={`templates.${index}.config.raptor.max_token`}label={t('knowledgeCompilation.maxToken')}diff --git a/web/src/pages/user-setting/compilation-templates/edit-next/schema.ts b/web/src/pages/user-setting/compilation-templates/edit-next/schema.tsindex 038d2bec3b..379dfdad1f 100644--- a/web/src/pages/user-setting/compilation-templates/edit-next/schema.ts+++ b/web/src/pages/user-setting/compilation-templates/edit-next/schema.ts@@ -20,14 +20,22 @@ export const buildSectionSchema = (t: (key: string) => string) =>z.object({description: z.string().optional(),fields: z- .array(z.record(z.string().min(1, t('knowledgeCompilation.fieldDescriptionRequired'))))+ .array(+ z.record(+ z.string().min(1, t('knowledgeCompilation.fieldDescriptionRequired')),+ ),+ ).min(1),});export const buildRaptorConfigSchema = (t: (key: string) => string) =>z.object({prompt: z.string().optional(),- max_token: z.number().min(512, t('knowledgeCompilation.maxTokenRequired')).max(2048),+ claim_prompt: z.string().optional(),+ max_token: z+ .number()+ .min(512, t('knowledgeCompilation.maxTokenRequired'))+ .max(2048),clustering_threshold: z.number().min(0).max(1),clustering_ratio: z.number().min(0).max(1),rechunk: z.boolean().optional(),diff --git a/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts b/web/src/pages/user-setting/compilation-templates/edit-next/utils.tsindex e6e832898f..8369d11917 100644--- a/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts+++ b/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts@@ -135,6 +135,7 @@ export const buildConfigFromBuiltin = (...sections,raptor: {prompt: builtinRaptor.prompt ?? '',+ claim_prompt: builtinRaptor.claim_prompt ?? '',max_token: builtinRaptor.max_token ?? 512,clustering_threshold: builtinRaptor.clustering_threshold ?? 0.3,clustering_ratio: builtinRaptor.clustering_ratio ?? 0.5,@@ -207,6 +208,7 @@ export const transformDetailToForm = (...base,raptor: {prompt: raptor.prompt ?? '',+ claim_prompt: raptor.claim_prompt ?? '',max_token: raptor.max_token ?? 512,clustering_threshold: raptor.clustering_threshold ?? 0.3,clustering_ratio: raptor.clustering_ratio ?? 0.5,diff --git a/web/src/services/document-structure-service.ts b/web/src/services/document-structure-service.tsindex 27791eb5a1..fccfda0299 100644--- a/web/src/services/document-structure-service.ts+++ b/web/src/services/document-structure-service.ts@@ -26,6 +26,20 @@ export const getDocumentStructureGraph = (params: keywords ? { keywords } : undefined,});+export const getDocumentStructureClaims = (+ datasetId: string,+ documentId: string,+ params: {+ chunk_ids?: string;+ template_id?: string;+ offset?: number;+ limit?: number;+ } = {},+) =>+ request.get(api.documentStructureClaims(datasetId, documentId), {+ params,+ });+export const deleteDocumentStructureGraph = (datasetId: string,documentId: string,@@ -37,6 +51,7 @@ export const deleteDocumentStructureGraph = (const documentStructureService = {getDocumentStructureGraph,+ getDocumentStructureClaims,deleteDocumentStructureGraph,};diff --git a/web/src/utils/api.ts b/web/src/utils/api.tsindex a71be1a252..785c3e9a5a 100644--- a/web/src/utils/api.ts+++ b/web/src/utils/api.ts@@ -261,6 +261,8 @@ export default {`${restAPIv1}/datasets/${datasetId}/documents/${documentId}/chunks/${chunkId}`,documentStructureGraph: (datasetId: string, documentId: string) =>`${restAPIv1}/datasets/${datasetId}/documents/${documentId}/structure/graph`,+ documentStructureClaims: (datasetId: string, documentId: string) =>+ `${restAPIv1}/datasets/${datasetId}/documents/${documentId}/structure/claims`,retrievalTest: `${restAPIv1}/datasets/search`,// document