RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agentic_rag): number citations from a per-turn evidence registry (#20677)
Caobaining1c5c75b2c5da0
diff --git a/conf/agentic_rag.yaml b/conf/agentic_rag.yamlindex ffa5be6f80..1867cdf72b 100644--- a/conf/agentic_rag.yaml+++ b/conf/agentic_rag.yaml@@ -41,7 +41,7 @@ templates:### The answer- ONE message of plain prose in the user's language: the answer itself, with the smallest structure that helps (short paragraphs; a list only when the question asks for several things). No meta-narration ("Let me check...", "Here is what I found"), no tool calls in the final message, no restating the question.- - **Provenance is mechanical:** end every line that states a corpus-backed fact with its provenance, exactly as the tool output printed it - `chunk_id: <id>`. The pipeline converts these into the numbered citation markers the interface renders; a factual claim without one reads as unsupported.+ - **Provenance is mechanical:** end every line that states a corpus-backed fact with the handle the tool printed for that passage - `[ID:n]`, where n is that passage's `ref` attribute in the tool output. The pipeline converts these handles into the numbered citation markers the interface renders; a factual claim without one reads as unsupported.- If the corpus genuinely does not contain the answer, say so plainly, name what you searched, and give the closest fact the corpus does support - never a specific entity, date or number recalled from world knowledge alone. When the exact figure is missing but a close one exists, commit to it and state the assumption explicitly (e.g. "based on the 2017 figure; no 2019 figure is present").- **Exact numbers:** use the corpus's full precision; never round intermediates. Dates and ages: count completed units from the date the question anchors (1787-12-12 -> 1873-07-01 = 85), not birth years alone. When comparing two people's ages, anchor each to the event the question names, then subtract. Convert units exactly (ft<->m, lb<->kg).diff --git a/internal/agentic_rag/agentic_rag_test.go b/internal/agentic_rag/agentic_rag_test.goindex 003efc7641..fb76bb962e 100644--- a/internal/agentic_rag/agentic_rag_test.go+++ b/internal/agentic_rag/agentic_rag_test.go@@ -38,7 +38,7 @@ func TestPrompt(t *testing.T) {p := Prompt()for _, want := range []string{"grep_chunks", "search_bm25_chunks", "search_semantic_chunks", "list_chunks",- "todo_write", "think", "run_javascript", "chunk_id:", "Evidence-First",+ "todo_write", "think", "run_javascript", "[ID:", "Evidence-First",} {if !strings.Contains(p, want) {t.Errorf("prompt must mention %q", want)diff --git a/internal/agentic_rag/citation_paradigm_compare_test.go b/internal/agentic_rag/citation_paradigm_compare_test.gonew file mode 100644index 0000000000..3831d2a6e1--- /dev/null+++ b/internal/agentic_rag/citation_paradigm_compare_test.go@@ -0,0 +1,115 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package agentic_rag++import (+ "strings"+ "testing"+)++// TestCitationParadigmCompare pins the behavioural difference between the two+// citation paradigms on the SAME answer shapes, so a later live comparison has a+// deterministic baseline to reason about:+//+// OLD (raw chunk id): the model writes `chunk_id: <id>`; the answer is marked+// by first-appearance position of each resolvable id.+// NEW ([ID:n] handle): the model writes the handle the tool stamped; a handle+// the registry never published is DROPPED, and the answer+// is recompacted to the published order.+func TestCitationParadigmCompare(t *testing.T) {+ // Three passages reached the model, stamped [ID:0..2].+ newRegistry := func() *EvidenceRegistry {+ r := NewEvidenceRegistry()+ r.Stamp("c1")+ r.Stamp("c2")+ r.Stamp("c3")+ return r+ }++ cases := []struct {+ name string+ // oldAnswer names passages by raw id; newAnswer names them by handle.+ oldAnswer string+ newAnswer string+ }{+ {+ name: "two passages, one repeated",+ oldAnswer: "A (chunk_id: c1)\nB (chunk_id: c2)\nC (chunk_id: c1)",+ newAnswer: "A [ID:0]\nB [ID:1]\nC [ID:0]",+ },+ {+ name: "a handle the run never published",+ oldAnswer: "A (chunk_id: c1)\nX (chunk_id: no-such-id)",+ newAnswer: "A [ID:0]\nX [ID:9]",+ },+ {+ name: "out-of-order citation compacts",+ oldAnswer: "A (chunk_id: c3)\nB (chunk_id: c1)",+ newAnswer: "A [ID:2]\nB [ID:0]",+ },+ }++ for _, c := range cases {+ // OLD path: the caller fetches the named ids, keeps the resolvable ones,+ // and marks by position. Simulate the fetch keeping only c1..c3.+ oldIDs := ExtractCitedChunkIDs(c.oldAnswer)+ resolved := make([]string, 0, len(oldIDs))+ for _, id := range oldIDs {+ if id == "c1" || id == "c2" || id == "c3" {+ resolved = append(resolved, id)+ }+ }+ oldMarked := InsertCitationMarkers(c.oldAnswer, resolved)++ // NEW path: handles resolve against the registry; publishable is what a+ // fetch would keep.+ reg := newRegistry()+ candidate := CitedIDsFromMarkers(c.newAnswer, reg)+ publishable := make([]string, 0, len(candidate))+ for _, id := range candidate {+ if id == "c1" || id == "c2" || id == "c3" {+ publishable = append(publishable, id)+ }+ }+ newMarked, newCited := ResolveCitations(c.newAnswer, reg, publishable)++ if newCited == nil {+ t.Fatalf("%s: new path produced no citation", c.name)+ }+ // Both paths must agree on WHICH passages end up cited.+ if len(resolved) != len(newCited) {+ t.Logf("%s:\n OLD cited %v\n NEW cited %v", c.name, resolved, newCited)+ }+ t.Logf("%s:\n OLD: %q -> %q (cited %v)\n NEW: %q -> %q (cited %v)",+ c.name, c.oldAnswer, oldMarked, resolved, c.newAnswer, newMarked, newCited)+ }++ // The decisive difference, pinned as an assertion: a handle the registry+ // never published is removed from the text, so it cannot dangle in the answer+ // while pointing at an empty reference slot.+ reg := newRegistry()+ marked, cited := ResolveCitations("A [ID:0]\nX [ID:9]", reg, []string{"c1"})+ if strings.Contains(marked, "[ID:9]") {+ t.Errorf("an unpublished handle survived: %q", marked)+ }+ if strings.Contains(marked, "X [ID:") {+ t.Errorf("the dropped handle left a stray marker: %q", marked)+ }+ if len(cited) != 1 || cited[0] != "c1" {+ t.Errorf("cited = %v, want [c1]", cited)+ }+}diff --git a/internal/agentic_rag/citations.go b/internal/agentic_rag/citations.goindex 1153c0db71..209fbd4b8a 100644--- a/internal/agentic_rag/citations.go+++ b/internal/agentic_rag/citations.go@@ -17,6 +17,7 @@package agentic_ragimport (+ "fmt""regexp""strconv""strings"@@ -131,3 +132,121 @@ func InsertCitationMarkers(final string, chunkIDs []string) string {}return strings.Join(lines, "\n")}++// The [ID:n] handle path.+//+// The tool results the model reads carry a `ref="n"` attribute on every passage+// (see EvidenceRegistry), and the prompt asks it to end each corpus-backed line+// with that handle: `[ID:n]`. The number names the passage the model was SHOWN,+// so it is checkable against what actually reached the context — unlike a chunk+// id the model could invent. The two helpers below turn those handles into the+// reference list the client resolves.++// citedRefRE matches the `[ID:n]` handle the model writes.+var citedRefRE = regexp.MustCompile(`\[ID:(\d+)\]`)++// commaCitationRE matches a handle group the model compressed into ONE marker+// ("[ID:47, ID:115]") instead of writing one marker per passage.+var commaCitationRE = regexp.MustCompile(`\[\s*ID\s*:\s*[0-9]+(?:\s*,\s*ID\s*:\s*[0-9]+)+\s*\]`)++var handleNumberRE = regexp.MustCompile(`[0-9]+`)++// ExpandCommaCitations splits a merged handle marker into one canonical marker+// per passage, so both this package's scanner and the client's citation pass+// can resolve it.+//+// A grouped marker matches neither `\[ID:(\d+)\]` nor the client's marker+// regex — both stop at the comma — so it survives every citation pass and+// reaches the UI as literal text instead of a citation. A handle naming a+// passage the registry never published (n >= maxIndex) is dropped rather than+// left pointing out of range. maxIndex <= 0 means the caller has no registry+// to bound against, and the text is left untouched.+func ExpandCommaCitations(answer string, maxIndex int) string {+ if answer == "" || maxIndex <= 0 || !strings.Contains(answer, ",") {+ return answer+ }+ return commaCitationRE.ReplaceAllStringFunc(answer, func(m string) string {+ var b strings.Builder+ for _, num := range handleNumberRE.FindAllString(m, -1) {+ n, err := strconv.Atoi(num)+ if err != nil || n < 0 || n >= maxIndex {+ continue+ }+ b.WriteString("[ID:")+ b.WriteString(strconv.Itoa(n))+ b.WriteString("]")+ }+ return b.String()+ })+}++// CitedIDsFromMarkers maps the answer's [ID:n] handles back to chunk ids, in+// first-citation order, deduplicated. A handle naming a passage the registry+// never published is ignored. Returns nil when the answer carries no handle+// (or no registry is available), which is the caller's signal to fall back to+// the raw `chunk_id:` path.+func CitedIDsFromMarkers(answer string, registry *EvidenceRegistry) []string {+ if answer == "" || registry == nil {+ return nil+ }+ ids := registry.IDs()+ if len(ids) == 0 {+ return nil+ }+ seen := make(map[string]struct{})+ var out []string+ for _, m := range citedRefRE.FindAllStringSubmatch(answer, -1) {+ n, err := strconv.Atoi(m[1])+ if err != nil || n < 0 || n >= len(ids) {+ continue+ }+ id := ids[n]+ if _, dup := seen[id]; dup {+ continue+ }+ seen[id] = struct{}{}+ out = append(out, id)+ }+ return out+}++// ResolveCitations rewrites the answer's [ID:n] handles so they index the+// reference list the caller publishes, and returns the cited ids in that order.+//+// publishable is the id list (in order) the caller will ship as+// reference.chunks — already narrowed to the passages that resolved. The first+// passage the answer cites becomes [ID:0], the next new one [ID:1], and so on;+// a handle naming a passage absent from publishable is dropped rather than left+// pointing at a wrong index. This is the mechanical compaction that lets a run+// stamp run-level handles ("[ID:265]") while the answer ships "[ID:0]".+func ResolveCitations(answer string, registry *EvidenceRegistry, publishable []string) (string, []string) {+ if answer == "" || registry == nil || len(publishable) == 0 {+ return answer, nil+ }+ ids := registry.IDs()+ pos := make(map[string]int, len(publishable))+ for i, id := range publishable {+ if _, dup := pos[id]; !dup {+ pos[id] = i+ }+ }+ seen := make(map[int]struct{})+ var cited []string+ out := citedRefRE.ReplaceAllStringFunc(answer, func(m string) string {+ sub := citedRefRE.FindStringSubmatch(m)+ n, err := strconv.Atoi(sub[1])+ if err != nil || n < 0 || n >= len(ids) {+ return ""+ }+ k, ok := pos[ids[n]]+ if !ok {+ return ""+ }+ if _, dup := seen[k]; !dup {+ seen[k] = struct{}{}+ cited = append(cited, ids[n])+ }+ return fmt.Sprintf("[ID:%d]", k)+ })+ return out, cited+}diff --git a/internal/agentic_rag/citations_test.go b/internal/agentic_rag/citations_test.goindex 9cdb66e464..0fdaf29379 100644--- a/internal/agentic_rag/citations_test.go+++ b/internal/agentic_rag/citations_test.go@@ -17,6 +17,7 @@package agentic_ragimport (+ "fmt""strings""testing")@@ -120,3 +121,121 @@ func TestInsertCitationMarkersUnresolvableStaysUnmarked(t *testing.T) {t.Errorf("nil id list rewrote the text: %s", got)}}++// CitedIDsFromMarkers maps [ID:n] handles to chunk ids in first-citation order,+// deduplicated, dropping any handle the registry never published.+func TestCitedIDsFromMarkers(t *testing.T) {+ r := NewEvidenceRegistry()+ r.Stamp("c1") // [ID:0]+ r.Stamp("c2") // [ID:1]+ answer := "fact A [ID:1]\nfact B [ID:0]\nfact C [ID:1]\nbogus [ID:9]"++ got := CitedIDsFromMarkers(answer, r)+ want := []string{"c2", "c1"}+ if len(got) != len(want) {+ t.Fatalf("CitedIDsFromMarkers = %v, want %v", got, want)+ }+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("CitedIDsFromMarkers = %v, want %v (first-citation order, deduped)", got, want)+ }+ }+}++func TestCitedIDsFromMarkersNoSignal(t *testing.T) {+ if got := CitedIDsFromMarkers("plain [ID:0]", nil); got != nil {+ t.Errorf("nil registry returned %v, want nil", got)+ }+ if got := CitedIDsFromMarkers("no markers here", NewEvidenceRegistry()); got != nil {+ t.Errorf("marker-free answer returned %v, want nil", got)+ }+}++// ResolveCitations compacts run-level handles into the answer's own+// first-citation order, and drops a handle naming a passage the caller will not+// publish.+func TestResolveCitationsCompactsAndDrops(t *testing.T) {+ r := NewEvidenceRegistry()+ r.Stamp("c1") // [ID:0]+ r.Stamp("c2") // [ID:1]+ r.Stamp("c3") // [ID:2]++ answer := "A [ID:2]\nB [ID:0]\nC [ID:1]"+ // c2 is not publishable (it did not resolve): its handle must be dropped.+ marked, cited := ResolveCitations(answer, r, []string{"c3", "c1"})+ if marked != "A [ID:0]\nB [ID:1]\nC " {+ t.Fatalf("marked = %q, want the compacted handles", marked)+ }+ if len(cited) != 2 || cited[0] != "c3" || cited[1] != "c1" {+ t.Fatalf("cited = %v, want [c3 c1]", cited)+ }+}++func TestResolveCitationsNoSignal(t *testing.T) {+ answer := "no handles"+ if got, cited := ResolveCitations(answer, nil, []string{"c1"}); got != answer || cited != nil {+ t.Errorf("nil registry rewrote the answer: %q %v", got, cited)+ }+ if got, cited := ResolveCitations(answer, NewEvidenceRegistry(), nil); got != answer || cited != nil {+ t.Errorf("empty publishable rewrote the answer: %q %v", got, cited)+ }+}++// A reasoning model merges several handles into ONE marker ("[ID:47, ID:115]").+// That shape matches neither this package's scanner nor the client's marker+// regex — both stop at the comma — so it survives every citation pass and+// reaches the UI as literal text. Expanding it is what makes it resolvable.+func TestExpandCommaCitations(t *testing.T) {+ if got, want := ExpandCommaCitations("A [ID:47, ID:115] B", 200), "A [ID:47][ID:115] B"; got != want {+ t.Errorf("got %q, want %q", got, want)+ }+ // A handle the registry never published is dropped, not left out of range.+ if got, want := ExpandCommaCitations("A [ID:47, ID:115] B", 100), "A [ID:47] B"; got != want {+ t.Errorf("got %q, want %q", got, want)+ }+ // Three handles, loose spacing around the commas and the colons.+ if got, want := ExpandCommaCitations("[ID: 1,ID:2 , ID:3]", 10), "[ID:1][ID:2][ID:3]"; got != want {+ t.Errorf("got %q, want %q", got, want)+ }+ // A single handle needs no expansion, and marker-free text is untouched.+ for _, keep := range []string{"A [ID:47] B", "no handles, just prose"} {+ if got := ExpandCommaCitations(keep, 200); got != keep {+ t.Errorf("rewrote %q into %q", keep, got)+ }+ }+ // No registry to bound against: leave the text alone.+ unbounded := "A [ID:47, ID:115] B"+ if got := ExpandCommaCitations(unbounded, 0); got != unbounded {+ t.Errorf("unbounded rewrite: %q", got)+ }+}++// The observed failure end to end: the model parked the whole answer inside+// <think> and grouped its handles, so nothing downstream resolved them. After+// expansion the handles compact onto the reference payload the client indexes.+func TestGroupedHandlesResolveAfterExpansion(t *testing.T) {+ r := NewEvidenceRegistry()+ for i := 0; i < 120; i++ {+ r.Stamp(fmt.Sprintf("c%d", i))+ }+ answer := "<think>1948: both went Truman [ID:47, ID:115]\n1972: Nixon [ID:47, ID:83]</think>"++ normalized := ExpandCommaCitations(answer, len(r.IDs()))+ cited := CitedIDsFromMarkers(normalized, r)+ want := []string{"c47", "c115", "c83"}+ if len(cited) != len(want) {+ t.Fatalf("cited = %v, want %v", cited, want)+ }+ for i := range want {+ if cited[i] != want[i] {+ t.Fatalf("cited = %v, want %v", cited, want)+ }+ }++ marked, _ := ResolveCitations(normalized, r, cited)+ // c47 -> [ID:0], c115 -> [ID:1], c83 -> [ID:2], in first-citation order.+ wantMarked := "<think>1948: both went Truman [ID:0][ID:1]\n1972: Nixon [ID:0][ID:2]</think>"+ if marked != wantMarked {+ t.Fatalf("marked = %q, want %q", marked, wantMarked)+ }+}diff --git a/internal/agentic_rag/config_test.go b/internal/agentic_rag/config_test.goindex 3df177a71c..f0aa8a3992 100644--- a/internal/agentic_rag/config_test.go+++ b/internal/agentic_rag/config_test.go@@ -201,7 +201,7 @@ func TestShippedConfigIsASingleDirectAnswerTemplate(t *testing.T) {content := tmpl.Contentfor _, want := range []string{"grep_chunks", "search_bm25_chunks", "search_semantic_chunks",- "list_chunks", "chunk_id:", "Evidence-First",+ "list_chunks", "[ID:", "Evidence-First",} {if !strings.Contains(content, want) {t.Errorf("prompt must mention %q", want)diff --git a/internal/agentic_rag/evidence_registry.go b/internal/agentic_rag/evidence_registry.gonew file mode 100644index 0000000000..b1a0bfc417--- /dev/null+++ b/internal/agentic_rag/evidence_registry.go@@ -0,0 +1,112 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package agentic_rag++import (+ "context"+ "strings"+ "sync"+)++// EvidenceRegistry assigns each passage the model is SHOWN a stable [ID:n]+// handle, in first-seen order.+//+// The model can only cite what it was shown, so the number a citation names has+// to be decided at the moment the passage reaches it, not later in a renderer+// the session never sees. A tool result therefore stamps its passages with the+// handle the model is expected to write back ([ID:n], from the passage's+// ref attribute), and this registry is what those handles resolve against.+//+// A passage that reaches the model twice keeps its first number, so the same+// chunk cited on two lines is one place, cited the same way.+type EvidenceRegistry struct {+ mu sync.Mutex+ order []string+ numOf map[string]int+}++// NewEvidenceRegistry returns an empty registry.+func NewEvidenceRegistry() *EvidenceRegistry {+ return &EvidenceRegistry{numOf: map[string]int{}}+}++// Stamp returns the stable handle for a chunk id, allocating the next one the+// first time the passage is seen. An empty id (or a nil registry) yields -1,+// which callers read as "not citable".+func (r *EvidenceRegistry) Stamp(id string) int {+ if r == nil {+ return -1+ }+ id = strings.TrimSpace(id)+ if id == "" {+ return -1+ }+ r.mu.Lock()+ defer r.mu.Unlock()+ if n, ok := r.numOf[id]; ok {+ return n+ }+ n := len(r.order)+ r.order = append(r.order, id)+ r.numOf[id] = n+ return n+}++// Resolve returns the chunk id a handle names, and whether it is published.+func (r *EvidenceRegistry) Resolve(n int) (string, bool) {+ if r == nil {+ return "", false+ }+ r.mu.Lock()+ defer r.mu.Unlock()+ if n < 0 || n >= len(r.order) {+ return "", false+ }+ return r.order[n], true+}++// IDs returns a snapshot of the registry in first-seen order: IDs[n] is the+// chunk the model was shown as [ID:n].+func (r *EvidenceRegistry) IDs() []string {+ if r == nil {+ return nil+ }+ r.mu.Lock()+ defer r.mu.Unlock()+ return append([]string(nil), r.order...)+}++// evidenceRegistryKey carries the run's registry through the request context,+// the same per-run channel web_search uses: the tools read it when they render,+// so concurrent turns never share numbering.+type evidenceRegistryKey struct{}++// WithEvidenceRegistry returns a copy of ctx carrying the run's evidence+// registry. A nil registry is a no-op.+func WithEvidenceRegistry(ctx context.Context, r *EvidenceRegistry) context.Context {+ if r == nil {+ return ctx+ }+ return context.WithValue(ctx, evidenceRegistryKey{}, r)+}++// evidenceRegistryFrom returns the registry carried by ctx, or nil when the+// caller published none (rendering then stamps no ref attribute).+func evidenceRegistryFrom(ctx context.Context) *EvidenceRegistry {+ r, _ := ctx.Value(evidenceRegistryKey{}).(*EvidenceRegistry)+ return r+}diff --git a/internal/agentic_rag/evidence_registry_test.go b/internal/agentic_rag/evidence_registry_test.gonew file mode 100644index 0000000000..f1b8e2a636--- /dev/null+++ b/internal/agentic_rag/evidence_registry_test.go@@ -0,0 +1,123 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package agentic_rag++import (+ "context"+ "strings"+ "testing"++ "ragflow/internal/agent/runtime"+)++func TestEvidenceRegistryStampsInFirstSeenOrder(t *testing.T) {+ r := NewEvidenceRegistry()+ if got := r.Stamp("c1"); got != 0 {+ t.Fatalf("Stamp(c1) = %d, want 0", got)+ }+ if got := r.Stamp("c2"); got != 1 {+ t.Fatalf("Stamp(c2) = %d, want 1", got)+ }+ // A passage that reaches the model twice keeps its first number.+ if got := r.Stamp("c1"); got != 0 {+ t.Fatalf("Stamp(c1) again = %d, want the stable 0", got)+ }+ if got := r.Stamp(" "); got != -1 {+ t.Fatalf("Stamp(blank) = %d, want -1 (not citable)", got)+ }++ ids := r.IDs()+ if len(ids) != 2 || ids[0] != "c1" || ids[1] != "c2" {+ t.Fatalf("IDs() = %v, want [c1 c2]", ids)+ }+ // The snapshot must be a copy: a caller cannot mutate the registry through it.+ ids[0] = "mutated"+ if r.IDs()[0] != "c1" {+ t.Error("IDs() returned a live slice, not a copy")+ }+}++func TestEvidenceRegistryResolve(t *testing.T) {+ r := NewEvidenceRegistry()+ r.Stamp("c1")+ if id, ok := r.Resolve(0); !ok || id != "c1" {+ t.Fatalf("Resolve(0) = (%q, %v), want (c1, true)", id, ok)+ }+ if _, ok := r.Resolve(1); ok {+ t.Error("Resolve(1) resolved an unpublished handle")+ }+ if _, ok := r.Resolve(-1); ok {+ t.Error("Resolve(-1) resolved")+ }+}++func TestEvidenceRegistryNilSafety(t *testing.T) {+ var r *EvidenceRegistry+ if got := r.Stamp("x"); got != -1 {+ t.Errorf("nil.Stamp = %d, want -1", got)+ }+ if r.IDs() != nil {+ t.Error("nil.IDs() must be nil")+ }+ if _, ok := r.Resolve(0); ok {+ t.Error("nil.Resolve resolved")+ }+}++func TestEvidenceRegistryRidesOnContext(t *testing.T) {+ r := NewEvidenceRegistry()+ ctx := WithEvidenceRegistry(context.Background(), r)+ if got := evidenceRegistryFrom(ctx); got != r {+ t.Fatal("the registry did not travel on the context")+ }+ if evidenceRegistryFrom(context.Background()) != nil {+ t.Error("a bare context must carry no registry")+ }+ // A nil registry is a no-op, not a nil-valued key.+ if WithEvidenceRegistry(context.Background(), nil) != context.Background() {+ t.Error("WithEvidenceRegistry(nil) must return the same context")+ }+}++// The locate payload stamps each served passage with its [ID:n] handle when the+// run carries a registry, and stamps nothing when it does not.+func TestLocateResultsStampRefHandle(t *testing.T) {+ hits := []snippetHit{+ {chunk: runtime.RetrievalChunk{ID: "c1"}},+ {chunk: runtime.RetrievalChunk{ID: "c2"}},+ }+ ctx := WithEvidenceRegistry(context.Background(), NewEvidenceRegistry())+ out := formatLocateResultsXML(ctx, "grep_chunks", "q", hits)+ if !strings.Contains(out, `ref="0"`) || !strings.Contains(out, `ref="1"`) {+ t.Fatalf("served passages not stamped with their handles: %.300q", out)+ }++ plain := formatLocateResultsXML(context.Background(), "grep_chunks", "q", hits)+ if strings.Contains(plain, `ref="`) {+ t.Errorf("a run with no registry must publish no handle: %.300q", plain)+ }+}++// Deep-read passages are citable too.+func TestChunksXMLStampsRefHandle(t *testing.T) {+ chunks := []runtime.RetrievalChunk{{ID: "d1"}, {ID: "d2"}}+ ctx := WithEvidenceRegistry(context.Background(), NewEvidenceRegistry())+ out := formatChunksXML(ctx, "doc-1", chunks, "", "")+ if !strings.Contains(out, `ref="0"`) || !strings.Contains(out, `ref="1"`) {+ t.Fatalf("deep-read passages not stamped: %.300q", out)+ }+}diff --git a/internal/agentic_rag/helper.go b/internal/agentic_rag/helper.goindex ce6f696914..1dac45a6bd 100644--- a/internal/agentic_rag/helper.go+++ b/internal/agentic_rag/helper.go@@ -513,6 +513,9 @@ func formatLocateResultsXML(ctx context.Context, tool, query string, hits []snip// records exactly what went in front of the model (silent unless the logger// runs at debug level).logServedChunks(ctx, tool, query, hits)+ // The passages in front of the model are stamped with their [ID:n] handle as+ // they are served, so a citation names a number the model actually saw.+ reg := evidenceRegistryFrom(ctx)var b strings.Builderb.WriteString(fmt.Sprintf("<search_results count=\"%d\" query=\"%s\">\n",len(hits), xmlEscape(query)))@@ -520,6 +523,7 @@ func formatLocateResultsXML(ctx context.Context, tool, query string, hits []snippreviewHits := 0for i, h := range hits {c := h.chunk+ refAttr := refAttrOf(reg, c.ID)truncatedAttr := ""if h.truncated {truncatedAttr = ` truncated="true"`@@ -531,11 +535,11 @@ func formatLocateResultsXML(ctx context.Context, tool, query string, hits []snippreviewHits++}b.WriteString(fmt.Sprintf(- "<chunk rank=\"%d\" chunk_id=\"%s\" doc_id=\"%s\" page_num=\"%d\" chunk_index=\"%d\" dataset_id=\"%s\" doc_name=\"%s\" score=\"%.3f\" chunk_runes=\"%d\"%s%s>\n",+ "<chunk rank=\"%d\" chunk_id=\"%s\" doc_id=\"%s\" page_num=\"%d\" chunk_index=\"%d\" dataset_id=\"%s\" doc_name=\"%s\" score=\"%.3f\" chunk_runes=\"%d\"%s%s%s>\n",i+1,xmlEscape(c.ID), xmlEscape(c.DocumentID), c.PageNum, c.ChunkIndex,xmlEscape(c.DatasetID), xmlEscape(c.DocumentName), c.Score,- utf8.RuneCountInString(c.Content), truncatedAttr, matchAttr,+ utf8.RuneCountInString(c.Content), refAttr, truncatedAttr, matchAttr,))if h.snippet != "" {b.WriteString(fmt.Sprintf("<match_snippet>%s</match_snippet>\n", xmlEscape(h.snippet)))@@ -618,7 +622,10 @@ func dedupStrings(s []string) []string {// findings that must NOT abort the caller's turn (e.g. anchors that did not// resolve), so the model can see and judge them instead of losing the whole// ReAct turn to an error.-func formatChunksXML(docID string, chunks []runtime.RetrievalChunk, anchorMeta, notice string) string {+func formatChunksXML(ctx context.Context, docID string, chunks []runtime.RetrievalChunk, anchorMeta, notice string) string {+ // Deep-read passages are citable too: stamp the same [ID:n] handle the locate+ // tools use, so a fact read from the full text can be cited.+ reg := evidenceRegistryFrom(ctx)var b strings.Builderb.WriteString(fmt.Sprintf("<chunks doc_id=\"%s\" fetched=\"%d\"%s>\n",xmlEscape(docID), len(chunks), anchorMeta))@@ -627,9 +634,9 @@ func formatChunksXML(docID string, chunks []runtime.RetrievalChunk, anchorMeta,}for _, c := range chunks {b.WriteString(fmt.Sprintf(- "<chunk chunk_id=\"%s\" doc_id=\"%s\" page_num=\"%d\" chunk_index=\"%d\" dataset_id=\"%s\" doc_name=\"%s\">\n",+ "<chunk chunk_id=\"%s\" doc_id=\"%s\" page_num=\"%d\" chunk_index=\"%d\" dataset_id=\"%s\" doc_name=\"%s\"%s>\n",xmlEscape(c.ID), xmlEscape(c.DocumentID), c.PageNum, c.ChunkIndex,- xmlEscape(c.DatasetID), xmlEscape(c.DocumentName)))+ xmlEscape(c.DatasetID), xmlEscape(c.DocumentName), refAttrOf(reg, c.ID)))if c.Content != "" {b.WriteString(fmt.Sprintf("<content>%s</content>\n", xmlEscape(c.Content)))}@@ -638,3 +645,13 @@ func formatChunksXML(docID string, chunks []runtime.RetrievalChunk, anchorMeta,b.WriteString("</chunks>")return b.String()}++// refAttrOf stamps a passage with its [ID:n] handle: ` ref="n"` when the run+// carries a registry, empty when it does not (a renderer without a registry+// publishes no handle, and the answer falls back to naming chunk ids).+func refAttrOf(reg *EvidenceRegistry, chunkID string) string {+ if n := reg.Stamp(chunkID); n >= 0 {+ return fmt.Sprintf(` ref="%d"`, n)+ }+ return ""+}diff --git a/internal/agentic_rag/prompt.go b/internal/agentic_rag/prompt.goindex e2ae063066..7c1fd0c17e 100644--- a/internal/agentic_rag/prompt.go+++ b/internal/agentic_rag/prompt.go@@ -56,7 +56,7 @@ Answer the user's question directly, accurately, and with traceable provenance:### The answer- ONE message of plain prose in the user's language: the answer itself, with the smallest structure that helps (short paragraphs; a list only when the question asks for several things). No meta-narration ("Let me check...", "Here is what I found"), no tool calls in the final message, no restating the question.-- **Provenance is mechanical:** end every line that states a corpus-backed fact with its provenance, exactly as the tool output printed it - ` + "`chunk_id: <id>`" + `. The pipeline converts these into the numbered citation markers the interface renders; a factual claim without one reads as unsupported.+- **Provenance is mechanical:** end every line that states a corpus-backed fact with the handle the tool printed for that passage - ` + "`[ID:n]`" + `, where n is that passage's ` + "`ref`" + ` attribute in the tool output. The pipeline converts these handles into the numbered citation markers the interface renders; a factual claim without one reads as unsupported.- If the corpus genuinely does not contain the answer, say so plainly, name what you searched, and give the closest fact the corpus does support - never a specific entity, date or number recalled from world knowledge alone. When the exact figure is missing but a close one exists, commit to it and state the assumption explicitly (e.g. "based on the 2017 figure; no 2019 figure is present").- **Exact numbers:** use the corpus's full precision; never round intermediates. Dates and ages: count completed units from the date the question anchors (1787-12-12 → 1873-07-01 = 85), not birth years alone. When comparing two people's ages, anchor each to the event the question names, then subtract. Convert units exactly (ft↔m, lb↔kg).diff --git a/internal/agentic_rag/tool_list_chunks.go b/internal/agentic_rag/tool_list_chunks.goindex 5e13766a45..7dbcb7de1d 100644--- a/internal/agentic_rag/tool_list_chunks.go+++ b/internal/agentic_rag/tool_list_chunks.go@@ -246,7 +246,7 @@ func (l *ListChunksTool) invokableRun(ctx context.Context, argumentsInJSON strincommon.WarnCtx(ctx, "agentic_rag: list_chunks unresolved anchors",zap.String("doc_id", docID), zap.Strings("anchors", anchors),zap.Strings("missing", missing))- return formatChunksXML(docID, nil, anchorMeta,+ return formatChunksXML(ctx, docID, nil, anchorMeta,unresolvedNotice(missing, len(missing) == len(anchors))), nil}for i, m := range mark {@@ -290,7 +290,7 @@ func (l *ListChunksTool) invokableRun(ctx context.Context, argumentsInJSON strin// Only the nb=0 path reaches here: the nb>0 path returns early on// unresolved anchors, and its empty case is reported by chunksXMLNoText.- return formatChunksXML(docID, chunks, anchorMeta, ""), nil+ return formatChunksXML(ctx, docID, chunks, anchorMeta, ""), nil}// resolveDocDatasetID returns the knowledge-base id that owns the given document,diff --git a/internal/service/agentic_pipeline.go b/internal/service/agentic_pipeline.goindex a0f14164b0..4554f712d7 100644--- a/internal/service/agentic_pipeline.go+++ b/internal/service/agentic_pipeline.go@@ -92,6 +92,11 @@ func (s *ChatPipelineService) agenticRag(runCtx, cancel := context.WithTimeout(ctx, agenticRunTimeout)defer cancel()+ // The run's evidence registry: every tool result stamps its passages with+ // the [ID:n] handle the model cites, and buildAgenticReference resolves+ // those handles against the same numbering.+ registry := agentic_rag.NewEvidenceRegistry()+ runCtx = agentic_rag.WithEvidenceRegistry(runCtx, registry)emitResult := func(result AsyncChatResult) bool {select {case out <- result:@@ -212,11 +217,20 @@ func (s *ChatPipelineService) agenticRag(answer := finalreference := map[string]interface{}{}if quote {- reference, answer = s.buildAgenticReference(ctx, chat.TenantID, chatDatasetIDs(chat), final)- } else if len(agentic_rag.ExtractCitedChunkIDs(final)) > 0 {+ reference, answer = s.buildAgenticReference(ctx, chat.TenantID, chatDatasetIDs(chat), final, registry)+ } else if len(agentic_rag.CitedIDsFromMarkers(final, registry)) > 0 || len(agentic_rag.ExtractCitedChunkIDs(final)) > 0 {common.InfoCtx(ctx, "agentic citations suppressed by quote=false")}- emitResult(AsyncChatResult{Answer: answer, Reference: reference, Final: true})+ // The agentic answer is the run's whole output — <think> stretch included+ // — with its handles already compacted against `reference`, so the session+ // writer must persist THIS text and not the streamed deltas, which still+ // carry the raw handles the model emitted.+ emitResult(AsyncChatResult{+ Answer: answer,+ Reference: reference,+ Final: true,+ AnswerIsAuthoritative: true,+ })}()return out, nil@@ -352,11 +366,7 @@ func convertMessagesToEino(messages []map[string]interface{}) []*schema.Messagereturn out}-// buildAgenticReference turns the answer's own chunk_id citations into the-// reference payload the naive pipeline ships, and rewrites the answer with-// [ID:N] markers. With no resolvable citations both come back unchanged: an-// empty reference keeps the SSE shape the UI expects, and unmarked chunk_id-// text is the honest state of an answer whose sources could not be loaded.+// chatDatasetIDs reads the conversation's bound dataset ids.func chatDatasetIDs(chat *entity.Chat) []string {if chat == nil {return nil@@ -370,14 +380,44 @@ func chatDatasetIDs(chat *entity.Chat) []string {return ids}-func (s *ChatPipelineService) buildAgenticReference(ctx context.Context, tenantID string, datasetIDs []string, final string) (map[string]interface{}, string) {- cited := agentic_rag.ExtractCitedChunkIDs(final)+// buildAgenticReference turns the answer's citations into the reference payload+// the naive pipeline ships, and rewrites the answer's markers to that payload's+// numbering. The answer cites the run's evidence handles ([ID:n], the numbering+// the model was shown); an answer that named passages by raw chunk_id instead+// (a document-level question has no handle to write) falls back to that path.+// With no resolvable citations both come back unchanged: an empty reference+// keeps the SSE shape the UI expects, and the untouched text is the honest state+// of an answer whose sources could not be loaded.+func (s *ChatPipelineService) buildAgenticReference(ctx context.Context, tenantID string, datasetIDs []string, final string, registry *agentic_rag.EvidenceRegistry) (map[string]interface{}, string) {+ // Normalize the answer's markers BEFORE anything reads them. A reasoning+ // model parks the whole answer inside <think>, so the normalization covers+ // the entire text rather than a post-`</think>` tail: a handle sitting in+ // the thinking stretch is cited exactly like one in the visible prose.+ //+ // Two shapes need repair, and both are invisible to every scanner+ // downstream — this package's `\[ID:(\d+)\]` and the client's marker regex+ // each stop at the first non-digit:+ // - a near-canonical marker ("(ID:47)", "[ID: 47]", "【ID:47】"), and+ // - several handles compressed into one marker ("[ID:47, ID:115]").+ normalized := agentic_rag.ExpandCommaCitations(+ RepairBadCitationFormats(final), len(registry.IDs()))++ // Preferred: the answer's own [ID:n] handles, resolved against the run's+ // evidence registry — the same numbering the model was shown, so a marker+ // can only name a passage that actually reached the context.+ cited := agentic_rag.CitedIDsFromMarkers(normalized, registry)+ byHandles := len(cited) > 0+ if !byHandles {+ // Fallback: an answer that named passages by raw chunk_id. A+ // document-level question has no [ID:n] handle to write.+ cited = agentic_rag.ExtractCitedChunkIDs(normalized)+ }if len(cited) == 0 {- return map[string]interface{}{}, final+ return map[string]interface{}{}, normalized}rows := fetchChunksByIDs(ctx, tenantID, datasetIDs, cited)if len(rows) == 0 {- return map[string]interface{}{}, final+ return map[string]interface{}{}, normalized}// Keep only ids that actually resolved, preserving first-appearance// order: the marker number IS the chunk's position in the payload array.@@ -394,16 +434,24 @@ func (s *ChatPipelineService) buildAgenticReference(ctx context.Context, tenantI}}if len(resolved) == 0 {- return map[string]interface{}{}, final+ return map[string]interface{}{}, normalized}ordered := make([]map[string]interface{}, 0, len(resolved))for _, id := range resolved {ordered = append(ordered, byID[id])}- marked := agentic_rag.InsertCitationMarkers(final, resolved)+ marked := normalized+ if byHandles {+ // Compact the run-level handles ("[ID:265]") into the answer's own+ // first-citation order ("[ID:0]"), the numbering the payload ships.+ marked, _ = agentic_rag.ResolveCitations(normalized, registry, resolved)+ } else {+ marked = agentic_rag.InsertCitationMarkers(normalized, resolved)+ }common.InfoCtx(ctx, "agentic citations built",zap.Int("cited", len(cited)),zap.Int("resolved", len(resolved)),+ zap.Bool("by_handle", byHandles),zap.Int("final_bytes", len(marked)))return map[string]interface{}{"chunks": chunksFormat(ordered),diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.goindex 85031f05a3..dfc54532f3 100644--- a/internal/service/chat_pipeline.go+++ b/internal/service/chat_pipeline.go@@ -100,6 +100,11 @@ type AsyncChatResult struct {// the think-block sentence), forwarded so a client can render steps —// tool, status, counts, sources, duration — instead of paragraphs.ThinkEvent *ThinkEvent `json:"think_event,omitempty"`+ // Internal-only: set when Answer is the run's authoritative full text —+ // <think> stretch included, citation handles already compacted against the+ // shipped reference — so the session writer must persist it instead of the+ // accumulated stream deltas, which still carry the pre-compaction handles.+ AnswerIsAuthoritative bool// Internal-only: accumulated answer for building the decorated final result.accumulatedAnswer string}diff --git a/internal/service/chat_session.go b/internal/service/chat_session.goindex 9089a81ff0..4e083f0f21 100644--- a/internal/service/chat_session.go+++ b/internal/service/chat_session.go@@ -1404,7 +1404,7 @@ func (s *ChatSessionService) ChatCompletions(if session != nil && !failed {// Store with <think>thinking content</think>content := fullAnswer.String()- if content == "" {+ if content == "" || result.AnswerIsAuthoritative {content = result.Answer}s.appendAssistantToSession(session, content, messageID)