RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(agentic_rag): put membership and document metadata into the answer's evidence (#19981)
Caobaining1c6bc073d2776
diff --git a/internal/rag/agentic-rag/agentic_rag.go b/internal/rag/agentic-rag/agentic_rag.goindex 6ff1554906..d7297258a0 100644--- a/internal/rag/agentic-rag/agentic_rag.go+++ b/internal/rag/agentic-rag/agentic_rag.go@@ -1399,6 +1399,13 @@ func composeFinalAnswer(ctx context.Context, deps RAGTools, req runtime.RunRequeresp.CiteChunkIDs = kb.CiteChunkIDs}}()+ // Stamp each pool chunk with its document's declared metadata, so the evidence+ // blocks carry the passage's document attributes (file name, update time, ...)+ // beside its text: a two-hop question — which documents, and what is in them —+ // is then answerable from the evidence alone, instead of from the slot record+ // the answer is told not to quote.+ attachDocMetadata(ctx, deps, kb, logger)+adeps := AnswerDeps{Model: deps.Model,CiteRules: deps.CiteRules,diff --git a/internal/rag/agentic-rag/agentic_rag_graph.go b/internal/rag/agentic-rag/agentic_rag_graph.goindex 40c099c1dd..ecdfa3bcf5 100644--- a/internal/rag/agentic-rag/agentic_rag_graph.go+++ b/internal/rag/agentic-rag/agentic_rag_graph.go@@ -830,6 +830,15 @@ func formalizeAnswerNode(ctx context.Context, deps RAGTools, st *AgenticState, l// everything downstream: the count, the record the answer reads, and the answer itself.RunCoverageResolve(ctx, deps, st, logger)+ // Publish what the table already holds, before the answer reads it. The record lists the+ // members, and a list of names with no passage behind it is what the answer states as facts+ // its evidence does not carry (see publishAnchoredMembers). The enumeration's own write-back+ // runs above and publishes its members; this adds the ones no enumeration reached — every+ // member a metadata selection defined.+ if derived, resolved := publishAnchoredMembers(st.KB, &st.SlotTable); derived > 0 {+ logger.Printf("[Finalize][members] derived=%d resolved=%d", derived, resolved)+ }+// "Partial" is a statement about the EVIDENCE, so it is decided by facts rather// than by a verdict that may not exist: unresolved slots are the table's own// list of what it could not fill, NoProgress means the last round learned@@ -2164,6 +2173,119 @@ func anchoredPoolEntries(kb *runtime.Kbinfos) []map[string]any {return out}+// publishAnchoredMembers publishes the slot table's anchored members into the answer's evidence+// ledger: the citation list the compose puts IN FRONT of the answer (see withCitedChunks) and the+// member→passage table its markers are written from (see CiteAnchoredMembers).+//+// It exists because the enumeration is not the only writer of members. Both publish sites sit in the+// enumeration's write-back (coverage_step.go), so a member set a METADATA selection defined never+// reached the answer's evidence at all: measured (2026-09-21) the record listed 6 documents with+// their update times, the answer stated all 6 — and the evidence it read carried six blocks about+// OTHER documents with zero markers resolved. The data was in the table the whole time; nobody+// published it.+//+// The anchors such a session records are not pool chunk ids. metadata_search runs no retrieval and+// admits no passage (tool_executor.go), so it hands the session DOC ids and the session writes those+// in: measured, 12 anchors, all 32-hex doc ids, none resolvable by ChunkByID. Resolution is+// therefore part of publishing rather than a refinement of it (see resolveMemberAnchor).+//+// derived is what the table claims, resolved what the pool could point at. A member that resolves to+// nothing is NOT published — it stays a name in the record, but gets no block and no marker rather+// than a block that opens somebody else's passage.+func publishAnchoredMembers(kb *runtime.Kbinfos, table *runtime.State) (derived, resolved int) {+ if kb == nil || table == nil {+ return 0, 0+ }+ refs := runtime.AnchoredItemRefs(table)+ if derived = len(refs); derived == 0 {+ return 0, 0+ }+ published := make([]runtime.AnchoredRef, 0, len(refs))+ cited := make([]string, 0, len(refs))+ for _, ref := range refs {+ id, quote := resolveMemberAnchor(kb, ref)+ if id == "" {+ continue+ }+ ref.ChunkID, ref.Quote = id, quote+ published = append(published, ref)+ cited = append(cited, id)+ }+ if resolved = len(cited); resolved == 0 {+ return derived, 0+ }+ // UNION, not replace: NoteCitedChunks replaces, and the enumeration's own list may hold+ // passages this derivation does not — a member whose anchor no longer resolves must not cost+ // the answer a passage that still does.+ kb.NoteCitedChunks(appendMissingIDs(kb.CitedChunks(), cited))+ kb.NoteAnchoredRefs(mergeAnchoredRefs(kb.AnchoredRefs(), published))+ return derived, resolved+}++// resolveMemberAnchor maps one member's recorded anchor to a passage the pool holds, and returns the+// quote that block may be rendered from ("" = render the whole passage).+//+// 1. the anchor is a pool chunk id — the enumeration's own case;+// 2. the anchor is a doc id the pool holds chunks for — a metadata selection's case: the member IS+// the document, so it takes that document's passage, preferring the one that carries the quote.+// Which passage stands for a document is otherwise arbitrary, and an arbitrary passage is one+// the answer's own quotation will not match (see CitedAnchoredMembers);+// 3. neither — no passage, and the caller publishes nothing for it.+//+// The quote is KEPT only when the resolved passage carries it. compactAnchored renders a published+// member's block from the quote INSTEAD of the passage, so a quote the passage does not contain —+// measured: "file_name: …; update_time: 2026-09-21 13:35:46", the metadata line the session was+// shown — would replace the passage with something that is not in it.+func resolveMemberAnchor(kb *runtime.Kbinfos, ref runtime.AnchoredRef) (string, string) {+ anchor := strings.TrimSpace(ref.ChunkID)+ if anchor == "" {+ return "", ""+ }+ if chunk := kb.ChunkByID(anchor); chunk != nil {+ if runtime.QuoteHeldBy(runtime.ChunkTextOf(chunk), ref.Quote) {+ return anchor, ref.Quote+ }+ return anchor, ""+ }+ bestID := ""+ bestScore := 0.0+ for _, chunk := range kb.Chunks {+ if runtime.DocIDOf(chunk) != anchor {+ continue+ }+ id := runtime.ChunkIDOf(chunk)+ if id == "" {+ continue+ }+ if runtime.QuoteHeldBy(runtime.ChunkTextOf(chunk), ref.Quote) {+ return id, ref.Quote+ }+ if score := similarityOrScore(chunk); bestID == "" || score > bestScore {+ bestID, bestScore = id, score+ }+ }+ return bestID, ""+}++// mergeAnchoredRefs unions the refs already recorded with the ones just published, first occurrence+// winning: a member the enumeration already matched to a passage keeps that passage, and this step+// only adds the members it did not have.+func mergeAnchoredRefs(existing, added []runtime.AnchoredRef) []runtime.AnchoredRef {+ out := make([]runtime.AnchoredRef, 0, len(existing)+len(added))+ seen := make(map[string]bool, len(existing)+len(added))+ for _, group := range [][]runtime.AnchoredRef{existing, added} {+ for _, ref := range group {+ name := strings.ToLower(strings.TrimSpace(ref.Name))+ if name == "" || strings.TrimSpace(ref.ChunkID) == "" || seen[name] {+ continue+ }+ seen[name] = true+ out = append(out, ref)+ }+ }+ return out+}+// appendMissingIDs appends the ids not already present, in order: a passage the render left out// (the budget admits only the first few whole chunks) still has to hold a position in the// published list, or nothing can cite it.diff --git a/internal/rag/agentic-rag/agentic_rag_graph_test.go b/internal/rag/agentic-rag/agentic_rag_graph_test.goindex a1cefda348..b11fcec921 100644--- a/internal/rag/agentic-rag/agentic_rag_graph_test.go+++ b/internal/rag/agentic-rag/agentic_rag_graph_test.go@@ -3924,6 +3924,99 @@ func TestCiteChunksPutTheItemsPassagesFirst(t *testing.T) {}}+// TestPublishAnchoredMembersResolvesDocIDs pins the metadata case. A session that selected documents+// by metadata records DOC ids as its members' anchors — metadata_search returns doc ids and admits no+// passage, so not one of them resolves in the pool — and publishing has to resolve them to that+// document's own passage. Without resolution the members the record lists reach the answer as no+// evidence at all: measured (2026-09-21) the answer stated 6 documents, the evidence it read carried+// six blocks about OTHER documents, and zero markers resolved.+func TestPublishAnchoredMembersResolvesDocIDs(t *testing.T) {+ kb := &runtime.Kbinfos{Chunks: []map[string]any{+ {"chunk_id": "docA-1", "doc_id": "docA", "content": "无关段落", "similarity": 0.3},+ {"chunk_id": "docA-2", "doc_id": "docA", "content": "勒索病毒事件按 P1/P2/P3 分级", "similarity": 0.9},+ {"chunk_id": "docB-1", "doc_id": "docB", "content": "端点隔离时限 10 分钟", "similarity": 0.4},+ {"chunk_id": "other", "doc_id": "docC", "content": "别的文档", "similarity": 0.99},+ }}+ table := runtime.NewState([]runtime.Variable{+ anchoredItemsVar(0, "list",+ slots.Item{Value: "勒索病毒事件上报与分级管理规范", ChunkID: "docA",+ Quote: "file_name: a.docx; update_time: 2026-09-21 13:35:42"},+ slots.Item{Value: "端点隔离标准", ChunkID: "docB", Quote: "端点隔离时限 10 分钟"},+ ),+ }, 0, nil)++ derived, resolved := publishAnchoredMembers(kb, &table)+ if derived != 2 || resolved != 2 {+ t.Fatalf("derived/resolved = %d/%d, want 2/2", derived, resolved)+ }+ if got := kb.CitedChunks(); len(got) != 2 || got[0] != "docA-2" || got[1] != "docB-1" {+ t.Fatalf("CitedChunks = %v, want each member on its own document's passage", got)+ }+ refs := kb.AnchoredRefs()+ if len(refs) != 2 || refs[0].ChunkID != "docA-2" || refs[1].ChunkID != "docB-1" {+ t.Fatalf("AnchoredRefs = %+v, want the anchors resolved to pool chunk ids", refs)+ }+ // The first member's quote is the METADATA line the session was shown, which its passage does+ // not contain. Kept, compactAnchored would render that block FROM the metadata line instead of+ // the passage — the block would show what the tool printed, not the document.+ if refs[0].Quote != "" {+ t.Errorf("ref[0].Quote = %q, want it DROPPED (its passage does not carry it)", refs[0].Quote)+ }+ // The second member's quote is in its passage: it stands, and the block renders from it.+ if refs[1].Quote != "端点隔离时限 10 分钟" {+ t.Errorf("ref[1].Quote = %q, want it kept", refs[1].Quote)+ }+}++// TestPublishAnchoredMembersUnionsAndSkipsUnresolvable pins the two guards around the ledger. Its+// writer REPLACES (NoteCitedChunks), so publishing must union: the enumeration's passages are already+// recorded, and a member that no longer resolves must not cost the answer a passage that still does.+// An anchor the pool cannot point at publishes nothing — the member stays a name in the record+// rather than becoming a block that opens somebody else's passage.+func TestPublishAnchoredMembersUnionsAndSkipsUnresolvable(t *testing.T) {+ kb := &runtime.Kbinfos{Chunks: []map[string]any{+ {"chunk_id": "enumerated", "doc_id": "docD", "content": "枚举来的段落", "similarity": 0.5},+ {"chunk_id": "docE-1", "doc_id": "docE", "content": "元数据选出的段落", "similarity": 0.2},+ }}+ kb.NoteCitedChunks([]string{"enumerated"})+ kb.NoteAnchoredRefs([]runtime.AnchoredRef{{Name: "枚举成员", ChunkID: "enumerated", Quote: "枚举来的"}})++ table := runtime.NewState([]runtime.Variable{+ anchoredItemsVar(0, "list",+ slots.Item{Value: "元数据成员", ChunkID: "docE"},+ slots.Item{Value: "无迹可循的成员", ChunkID: "docZ"},+ ),+ }, 0, nil)++ derived, resolved := publishAnchoredMembers(kb, &table)+ if derived != 2 || resolved != 1 {+ t.Fatalf("derived/resolved = %d/%d, want 2/1 (one anchor is in no document the pool holds)", derived, resolved)+ }+ if got := kb.CitedChunks(); len(got) != 2 || got[0] != "enumerated" || got[1] != "docE-1" {+ t.Fatalf("CitedChunks = %v, want the enumeration's passage KEPT and the resolved one added", got)+ }+ if refs := kb.AnchoredRefs(); len(refs) != 2 || refs[0].Name != "枚举成员" || refs[1].Name != "元数据成员" {+ t.Fatalf("AnchoredRefs = %+v, want the enumerated ref kept and the resolved one added", refs)+ }+}++// TestPublishAnchoredMembersIsANoOpWithoutAnchors pins the regression guard: a table holding no+// ANCHORED member publishes nothing, so an ordinary question's evidence selection stays exactly what+// it is today — min(citeChunkCap, pool size).+func TestPublishAnchoredMembersIsANoOpWithoutAnchors(t *testing.T) {+ kb := &runtime.Kbinfos{Chunks: []map[string]any{{"chunk_id": "c1", "content": "x"}}}+ table := runtime.NewState([]runtime.Variable{+ typedMembersVar(0, "list", "于禁"),+ }, 0, nil)++ if derived, resolved := publishAnchoredMembers(kb, &table); derived != 0 || resolved != 0 {+ t.Fatalf("derived/resolved = %d/%d, want 0/0 for a table whose members carry no passage", derived, resolved)+ }+ if got := kb.CitedChunks(); len(got) != 0 {+ t.Fatalf("CitedChunks = %v, want nothing published", got)+ }+}+// TestSlotRecordShowsEachItemWithItsPassage pins the citation half of the record: an item's chunk id// is in the value, and the record used to print names only — so the instruction to cite each member// ("every member you list carries the words behind it") had nothing to cite.@@ -4234,6 +4327,15 @@ func typedAnchoredMembersVar(id int, typ string, lists ...string) runtime.Variabreturn runtime.Variable{ID: id, Type: typ, Candidate: &rendered, Value: &v}}+// anchoredItemsVar builds a slot holding items with the anchors and quotes a session chose, which is+// what typedAnchoredMembersVar's synthetic "c-<name>" anchors cannot express — a metadata selection's+// anchors are doc ids and its quotes are the metadata lines it was shown.+func anchoredItemsVar(id int, typ string, items ...slots.Item) runtime.Variable {+ v := slots.Items(items...)+ rendered := slots.Render(v)+ return runtime.Variable{ID: id, Type: typ, Candidate: &rendered, Value: &v}+}+func typedCountVar(id int, typ string, n int, strength float64) runtime.Variable {v := slots.Number(n)rendered := slots.Render(v)diff --git a/internal/rag/agentic-rag/doc_metadata.go b/internal/rag/agentic-rag/doc_metadata.gonew file mode 100644index 0000000000..31a5c6f21f--- /dev/null+++ b/internal/rag/agentic-rag/doc_metadata.go@@ -0,0 +1,214 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package agentic_rag++import (+ "context"+ "log"+ "sort"+ "strings"++ "ragflow/internal/common"+ "ragflow/internal/entity"+ "ragflow/internal/rag/agentic-rag/runtime"+)++// docMetaValueRunes caps one rendered metadata value. The evidence block is a+// per-passage token budget (evidenceBudgetTokens, graph_compose.go), so a long+// title or a stitched value must not spend it: the block carries the value as a+// pointer, not as reading material.+const docMetaValueRunes = 200++// attachDocMetadata stamps the DECLARED document-level metadata onto the pool's+// chunks, so that the final answer's evidence block carries each passage's+// document attributes (file name, update time, title, ...) beside its text.+//+// WHY the evidence block is the only place this can work. The answer prompt is+// built from two carriers (see answerPromptWithEvidence): the slot record, which+// is declared INTERNAL and explicitly NOT evidence, and the numbered evidence+// blocks, which are the only text the citation contract lets the answer rest on.+// Document attributes previously reached the run only through the slot table+// (metadata_search's result block is context for the DIRECTION sessions and+// produces no passages: runtime/tool_executor.go, EvidenceIDs nil). That left a+// two-hop question — "which documents, and what is in them" — with the+// membership in the record and the content in the evidence, and the record's own+// contract ("never quote these lines") weighing against the answer using it.+// Stamping the attributes onto the passages puts both halves in the same block.+//+// WHY the KB row and not the metadata resolver's declared-field read. The+// declared fields live in parser_config, and the run already holds the KB rows+// (RAGTools.KBs, loaded by the caller so this package stays DB-free). Reading+// them here is the same source the metadata_search catalog uses+// (common.DeclaredMetadataFieldsFromParserConfig covers parser_config.metadata+// AND built_in_metadata) with no extra index scan.+//+// WHY only declared fields. The doc-metadata index also carries extraction and+// annotation noise the dataset never declared — a metadata-extraction run that+// echoed a judge's rationale and verdict into the document's metadata is a real+// observed case — and an unfiltered dump would put it in every evidence block.+// A dataset that declares nothing therefore gets no block-level metadata at all,+// which is exactly the behaviour before this function existed.+//+// Best effort by contract: a missing resolver, unresolvable KB rows or a failed+// metadata read degrades to "no document metadata in the evidence" rather than+// failing the answer. The compose call must not depend on it.+func attachDocMetadata(ctx context.Context, deps RAGTools, kb *runtime.Kbinfos, logger *log.Logger) {+ if kb == nil || deps.MetadataResolver == nil || len(kb.Chunks) == 0 {+ return+ }+ keys := declaredDocMetadataKeys(deps.KBs)+ if len(keys) == 0 {+ return+ }+ datasetIDs := kbIDsOf(deps.KBs)+ if len(datasetIDs) == 0 {+ return+ }+ docIDs := pooledDocIDs(kb.Chunks)+ if len(docIDs) == 0 {+ return+ }+ metaByDoc, err := deps.MetadataResolver.MetadataForDocIDs(ctx, datasetIDs, docIDs)+ if err != nil {+ logger.Printf("[Formalize][docmeta] metadata read degraded: %v", err)+ }+ if len(metaByDoc) == 0 {+ return+ }+ stamped := 0+ for _, chunk := range kb.Chunks {+ meta := metaByDoc[chunkDocID(chunk)]+ if len(meta) == 0 {+ continue+ }+ picked := make(map[string]any, len(keys))+ for _, key := range keys {+ if text, ok := docMetaText(meta[key]); ok {+ picked[key] = text+ }+ }+ if len(picked) == 0 {+ continue+ }+ chunk["document_metadata"] = picked+ stamped+++ }+ logger.Printf("[Formalize][docmeta] declared=%v docs=%d chunks=%d/%d",+ keys, len(metaByDoc), stamped, len(kb.Chunks))+}++// declaredDocMetadataKeys returns the union of the datasets' declared metadata+// fields, deduped and sorted so the request and the rendered block are stable.+// Declaration order is per dataset and a run may span several, so the union is+// ordered here rather than left to map iteration.+func declaredDocMetadataKeys(kbs []*entity.Knowledgebase) []string {+ seen := make(map[string]bool, 8)+ keys := make([]string, 0, 8)+ for _, kb := range kbs {+ if kb == nil {+ continue+ }+ for _, def := range common.DeclaredMetadataFieldsFromParserConfig(kb.ParserConfig) {+ key := strings.TrimSpace(def.Key)+ if key == "" || seen[key] {+ continue+ }+ seen[key] = true+ keys = append(keys, key)+ }+ }+ sort.Strings(keys)+ return keys+}++// kbIDsOf returns the datasets of the loaded KB rows, in the caller's order.+func kbIDsOf(kbs []*entity.Knowledgebase) []string {+ ids := make([]string, 0, len(kbs))+ for _, kb := range kbs {+ if kb == nil || strings.TrimSpace(kb.ID) == "" {+ continue+ }+ ids = append(ids, kb.ID)+ }+ return ids+}++// pooledDocIDs returns the distinct document ids the pool holds, sorted so the+// metadata read is deterministic (and so a whole-corpus lookup is one request).+func pooledDocIDs(chunks []map[string]any) []string {+ seen := make(map[string]bool, len(chunks))+ ids := make([]string, 0, len(chunks))+ for _, chunk := range chunks {+ id := chunkDocID(chunk)+ if id == "" || seen[id] {+ continue+ }+ seen[id] = true+ ids = append(ids, id)+ }+ sort.Strings(ids)+ return ids+}++// chunkDocID reads a chunk's DOCUMENT id.+//+// It deliberately does not fall back to "id": in a retrieval chunk that key is+// the CHUNK id (see service.SourcedChunk, which reads ID from chunk_id/id and+// DocID from doc_id/document_id), so keying the metadata lookup on it would ask+// the metadata index for a document that does not exist and stamp nothing.+func chunkDocID(chunk map[string]any) string {+ for _, key := range []string{"doc_id", "document_id"} {+ if id, ok := chunk[key].(string); ok && strings.TrimSpace(id) != "" {+ return strings.TrimSpace(id)+ }+ }+ return ""+}++// docMetaText renders one metadata value for an evidence block, or ok=false when+// the value has no flat reading there.+//+// Text (and a list of text, which the index writes for multi-valued fields) is+// what a "key: value" line can carry. A structured value — a nested outline, a+// list of objects — is dropped rather than stringified into noise: the block is+// a pointer to the document, not a rendering of its metadata.+func docMetaText(value any) (string, bool) {+ var text string+ switch typed := value.(type) {+ case string:+ text = typed+ case []string:+ text = strings.Join(typed, ", ")+ case []any:+ parts := make([]string, 0, len(typed))+ for _, item := range typed {+ s, ok := item.(string)+ if !ok {+ return "", false+ }+ parts = append(parts, s)+ }+ text = strings.Join(parts, ", ")+ default:+ return "", false+ }+ text = strings.TrimSpace(text)+ if text == "" {+ return "", false+ }+ return truncateRunes(text, docMetaValueRunes), true+}diff --git a/internal/rag/agentic-rag/doc_metadata_test.go b/internal/rag/agentic-rag/doc_metadata_test.gonew file mode 100644index 0000000000..3aac9bd647--- /dev/null+++ b/internal/rag/agentic-rag/doc_metadata_test.go@@ -0,0 +1,211 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package agentic_rag++import (+ "context"+ "strings"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/entity"+ "ragflow/internal/rag/agentic-rag/runtime"+)++// docMetadataResolver is a scripted runtime.MetadataResolver: it records the+// documents it was asked for and answers from a fixed doc_id → fields map.+type docMetadataResolver struct {+ meta map[string]map[string]any+ requested []string+ askedKBs []string+ calls int+}++func (r *docMetadataResolver) FilterDocIDsByMetaPushdown(context.Context, []string, []map[string]any, string) ([]string, bool) {+ return nil, false+}++func (r *docMetadataResolver) GetFlattedMetaByKBs(context.Context, []string) (common.MetaData, error) {+ return common.MetaData{}, nil+}++func (r *docMetadataResolver) MetadataForDocIDs(_ context.Context, kbIDs, docIDs []string) (map[string]map[string]any, error) {+ r.calls+++ r.askedKBs = append([]string(nil), kbIDs...)+ r.requested = append([]string(nil), docIDs...)+ return r.meta, nil+}++// declaringKB is a dataset row whose parser_config declares one extracted field+// and two built-in ones — the shape the metadata config API writes.+func declaringKB() []*entity.Knowledgebase {+ return []*entity.Knowledgebase{{+ ID: "kb1",+ ParserConfig: entity.JSONMap{+ "metadata": map[string]any{+ "enabled": true,+ "metadata": []any{+ map[string]any{"key": "title", "type": "string"},+ },+ "built_in_metadata": []any{+ map[string]any{"key": "update_time", "type": "time"},+ map[string]any{"key": "file_name", "type": "string"},+ },+ },+ },+ }}+}++// composePrompt runs the terminal composition and returns the user prompt the+// model received — the text the evidence blocks are rendered into.+func composePrompt(t *testing.T, deps RAGTools, kb *runtime.Kbinfos) string {+ t.Helper()+ model := &fakeModel{replies: []*runtime.ModelReply{{Content: "answer"}}}+ deps.Model = model+ composeFinalAnswer(context.Background(), deps, runtime.RunRequest{Question: "q"}, kb,+ &RunResponse{}, _LOG, false, false, "q")+ return model.lastUserPrompt()+}++// TestAttachDocMetadataStampsDeclaredFieldsOnly pins both halves of the write:+// the DECLARED fields reach the evidence block, and a field the dataset never+// declared does not. The undeclared half is the observed metadata-extraction+// leak (a judge's rationale landing in a document's metadata) — an unfiltered+// dump would put it in every evidence block.+func TestAttachDocMetadataStampsDeclaredFieldsOnly(t *testing.T) {+ resolver := &docMetadataResolver{meta: map[string]map[string]any{+ "doc1": {+ "update_time": "2026-09-20 13:55:35",+ "file_name": "VecTree-RAG.pdf",+ "title": "VecTree-RAG",+ "rationale": "an undeclared field an extraction run leaked",+ },+ }}+ kb := &runtime.Kbinfos{Chunks: []map[string]any{+ {"chunk_id": "c1", "doc_id": "doc1", "content": "the passage body", "similarity": 0.9},+ }}+ deps := RAGTools{KBs: declaringKB(), MetadataResolver: resolver}++ prompt := composePrompt(t, deps, kb)++ for _, want := range []string{+ "\u251c\u2500\u2500 update_time: 2026-09-20 13:55:35",+ "\u251c\u2500\u2500 file_name: VecTree-RAG.pdf",+ "\u251c\u2500\u2500 title: VecTree-RAG",+ } {+ if !strings.Contains(prompt, want) {+ t.Errorf("evidence block missing %q:\n%s", want, prompt)+ }+ }+ if strings.Contains(prompt, "rationale") {+ t.Errorf("an undeclared field reached the evidence block:\n%s", prompt)+ }+ if resolver.calls != 1 || len(resolver.requested) != 1 || resolver.requested[0] != "doc1" {+ t.Errorf("metadata read = %d call(s) for %v, want one read for [doc1]", resolver.calls, resolver.requested)+ }+ if len(resolver.askedKBs) != 1 || resolver.askedKBs[0] != "kb1" {+ t.Errorf("metadata read datasets = %v, want the declaring dataset", resolver.askedKBs)+ }+}++// TestAttachDocMetadataLeavesEvidenceUnchangedWithoutDeclarations pins the two+// gates that keep this a no-op where it cannot help: no resolver, and no+// declared fields. The second one must also skip the read — a dataset that+// declares nothing has no field to put on a block, so the round-trip would be+// paid for an empty map.+func TestAttachDocMetadataLeavesEvidenceUnchangedWithoutDeclarations(t *testing.T) {+ newKB := func() *runtime.Kbinfos {+ return &runtime.Kbinfos{Chunks: []map[string]any{+ {"chunk_id": "c1", "doc_id": "doc1", "content": "the passage body", "similarity": 0.9},+ }}+ }+ resolver := &docMetadataResolver{meta: map[string]map[string]any{+ "doc1": {"update_time": "2026-09-20 13:55:35"},+ }}++ noResolver := composePrompt(t, RAGTools{KBs: declaringKB()}, newKB())+ noDeclarations := composePrompt(t, RAGTools{MetadataResolver: resolver}, newKB())++ for name, prompt := range map[string]string{"no resolver": noResolver, "no declarations": noDeclarations} {+ if strings.Contains(prompt, "document_metadata") || strings.Contains(prompt, "update_time") {+ t.Errorf("%s: evidence block grew a metadata line:\n%s", name, prompt)+ }+ }+ if resolver.calls != 0 {+ t.Errorf("metadata read = %d call(s), want none without a declared field", resolver.calls)+ }+}++// TestAttachDocMetadataTruncatesLongValues pins the budget guard: a long value+// is rendered as a pointer to the document, not as reading material, because+// every metadata line spends the evidence block's token budget.+func TestAttachDocMetadataTruncatesLongValues(t *testing.T) {+ long := strings.Repeat("x", docMetaValueRunes+50)+ resolver := &docMetadataResolver{meta: map[string]map[string]any{+ "doc1": {"file_name": long},+ }}+ kb := &runtime.Kbinfos{Chunks: []map[string]any{+ {"chunk_id": "c1", "doc_id": "doc1", "content": "the passage body", "similarity": 0.9},+ }}++ prompt := composePrompt(t, RAGTools{KBs: declaringKB(), MetadataResolver: resolver}, kb)++ if !strings.Contains(prompt, strings.Repeat("x", docMetaValueRunes)) {+ t.Errorf("evidence block lost the truncated value:\n%s", prompt)+ }+ if strings.Contains(prompt, strings.Repeat("x", docMetaValueRunes+1)) {+ t.Errorf("a value longer than %d runes reached the evidence block", docMetaValueRunes)+ }+}++// TestAttachDocMetadataKeysOnDocID pins the id semantics: a retrieval chunk's+// "id" is its CHUNK id, so keying the metadata lookup on it would ask the index+// for a document that does not exist. The lookup must use doc_id, and a resolver+// that only knows the chunk id must stamp nothing.+func TestAttachDocMetadataKeysOnDocID(t *testing.T) {+ chunk := func() map[string]any {+ // "id" and "doc_id" disagree on purpose: that is the shape a retrieval+ // result has (service.SourcedChunk reads ID from chunk_id/id).+ return map[string]any{+ "id": "chunk-1", "chunk_id": "chunk-1", "doc_id": "doc-1",+ "content": "the passage body", "similarity": 0.9,+ }+ }++ byDocID := &docMetadataResolver{meta: map[string]map[string]any{+ "doc-1": {"update_time": "2026-09-20 13:55:35"},+ }}+ prompt := composePrompt(t, RAGTools{KBs: declaringKB(), MetadataResolver: byDocID},+ &runtime.Kbinfos{Chunks: []map[string]any{chunk()}})++ if len(byDocID.requested) != 1 || byDocID.requested[0] != "doc-1" {+ t.Fatalf("metadata read requested %v, want [doc-1] (the DOCUMENT id)", byDocID.requested)+ }+ if !strings.Contains(prompt, "update_time: 2026-09-20 13:55:35") {+ t.Errorf("evidence block missing the document's metadata:\n%s", prompt)+ }++ byChunkID := &docMetadataResolver{meta: map[string]map[string]any{+ "chunk-1": {"update_time": "2026-09-20 13:55:35"},+ }}+ other := composePrompt(t, RAGTools{KBs: declaringKB(), MetadataResolver: byChunkID},+ &runtime.Kbinfos{Chunks: []map[string]any{chunk()}})++ if strings.Contains(other, "update_time") {+ t.Errorf("a chunk-id-keyed answer stamped metadata onto the block:\n%s", other)+ }+}diff --git a/internal/rag/agentic-rag/graph_compose.go b/internal/rag/agentic-rag/graph_compose.goindex d0a70f89f9..d1c33c4ffc 100644--- a/internal/rag/agentic-rag/graph_compose.go+++ b/internal/rag/agentic-rag/graph_compose.go@@ -212,7 +212,9 @@ func runDirectFallback(ctx context.Context, deps RAGTools, req runtime.RunRequesconst (// evidenceBudgetTokens is the token ceiling of the evidence block.- evidenceBudgetTokens = 8000+ // The compose path still fits the whole prompt to the model window+ // (AnswerDeps.MaxLength / chat.FitMessages), so this is an upper bound.+ evidenceBudgetTokens = 80000// evidencePoolQuota: claim pseudo-chunk// cap across the whole first prefetch.evidencePoolQuota = 24diff --git a/internal/rag/agentic-rag/runtime/items.go b/internal/rag/agentic-rag/runtime/items.goindex b909c49258..5ae4916c1c 100644--- a/internal/rag/agentic-rag/runtime/items.go+++ b/internal/rag/agentic-rag/runtime/items.go@@ -208,6 +208,15 @@ func spaceLess(s string) string {"“", "", "”", "", "\"", "", "…", "").Replace(s)}+// QuoteHeldBy reports whether a passage carries a quotation. It is the ONE definition of "the quote+// is in this passage", asked by both sides of the ledger: the citation step decides which anchor a+// member's marker may open with it (see CitedAnchoredMembers), and publishing decides with it+// whether a recorded quote may stand as that member's block text — a quote the passage does not+// contain must not be rendered IN THE PASSAGE'S PLACE (see compactAnchored).+func QuoteHeldBy(passage, quote string) bool {+ return quote != "" && strings.Contains(spaceLess(passage), spaceLess(quote))+}+// CitedAnchoredMembers is CiteAnchoredMembers plus the number of LINES it rewrote, so a caller can// report whether the step touched the answer at all: a run whose members all carried markers// already is indistinguishable from one the step never reached otherwise.@@ -249,7 +258,7 @@ func CitedAnchoredMembers(answer string, refs []AnchoredRef, citeIDs []string, t// the anchored passage is another published passage tried, and only when none of// them holds either is the line left uncited.holds := func(cand, q string) bool {- return q != "" && strings.Contains(spaceLess(textOf(cand)), spaceLess(q))+ return QuoteHeldBy(textOf(cand), q)}asked, own := answerQuote(line), strings.TrimSpace(r.Quote)if !holds(id, asked) && !holds(id, own) {