RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Bugfix: add youcomsearch agent tool (#21080)
ciel8728fde8340f
diff --git a/internal/agent/component/agent.go b/internal/agent/component/agent.goindex fc25c1917d..7f8ea2196f 100644--- a/internal/agent/component/agent.go+++ b/internal/agent/component/agent.go@@ -179,6 +179,7 @@ func runEinoReActAgent(ctx context.Context, p AgentParam) (*schema.Message, erroif err != nil {return nil, fmt.Errorf("build tools: %w", err)}+ tools = wrapReferenceRecordingTools(tools)input := buildAgentInputMessages(ctx, p)// Eino's MaxStep counts graph nodes, not model calls. One ReAct round// consists of a model decision, a tool node, and the following model@@ -500,7 +501,9 @@ func addToolCallMemory(ctx context.Context, db *gorm.DB, p AgentParam, msg *sche// (e.g. "simple-0", "ck-1"). Used only to decide whether a grounding// response still has answer text after the markers are removed. The ID:// prefix is required so bare brackets like [2026] are not treated as markers.-var citationGroundingMarkerRe = regexp.MustCompile(`\[ID:\s*[^\]]+\]`)+// The id cannot contain "[", matching citationGroundingMarkerCaptureRe: an+// unclosed [ID: must not be stripped together with the text up to a later ].+var citationGroundingMarkerRe = regexp.MustCompile(`\[ID:\s*[^\[\]]+\]`)// citationGroundingHasSubstantiveText reports whether s contains any// non-whitespace text once citation markers are stripped. Marker-only@@ -532,7 +535,7 @@ func applyCitationGrounding(ctx context.Context, db *gorm.DB, p AgentParam, contif strings.TrimSpace(content) == "" {return content, nil}- systemPrompt, _ := prompts.CitationPlusPrompt(chunks)+ systemPrompt, ids := prompts.CitationPlusPrompt(chunks)inv := getDefaultChatInvoker()req := ChatInvokeRequest{Driver: p.Driver,@@ -548,9 +551,26 @@ func applyCitationGrounding(ctx context.Context, db *gorm.DB, p AgentParam, contvar resp *ChatInvokeResponsevar err erroremittedVisible := false+ var leadingWhitespace strings.Builder+ emitting := false+ emitContent := func(text string) error {+ if text == "" {+ return nil+ }+ if !emitting {+ leadingWhitespace.WriteString(text)+ if strings.TrimSpace(leadingWhitespace.String()) == "" {+ return nil+ }+ text = leadingWhitespace.String()+ emitting = true+ }+ emittedVisible = true+ runtime.EmitAgentMessage(ctx, text, "")+ return nil+ }if streamer, ok := inv.(chat.StreamingInvoker); ok {- var leadingWhitespace strings.Builder- emitting := false+ streamFilter := newCitationMarkerStreamFilter(ids)resp, err = streamer.Stream(ctx, db, req, func(delta string, isThink bool) error {if ctx.Err() != nil {return ctx.Err()@@ -558,18 +578,13 @@ func applyCitationGrounding(ctx context.Context, db *gorm.DB, p AgentParam, contif isThink || delta == "" {return nil}- if !emitting {- leadingWhitespace.WriteString(delta)- if strings.TrimSpace(leadingWhitespace.String()) == "" {- return nil- }- delta = leadingWhitespace.String()- emitting = true- }- emittedVisible = true- runtime.EmitAgentMessage(ctx, delta, "")- return nil+ return emitContent(streamFilter.Push(delta))})+ if err == nil {+ if flushErr := emitContent(streamFilter.Flush()); flushErr != nil {+ err = flushErr+ }+ }} else {resp, err = inv.Invoke(ctx, db, req)}@@ -582,20 +597,21 @@ func applyCitationGrounding(ctx context.Context, db *gorm.DB, p AgentParam, contif resp == nil {return content, fmt.Errorf("citation grounding returned no response")}- if strings.TrimSpace(resp.Content) == "" {+ grounded := filterCitationMarkers(resp.Content, ids)+ if strings.TrimSpace(grounded) == "" {return content, nil}// Marker-only grounding must not silently replace the first-pass answer// (#20613). Non-stream (or stream that never emitted) can fall back.// Once invalid deltas have already been streamed, surface an error so the// caller does not append the original answer after the bad markers.- if !citationGroundingHasSubstantiveText(resp.Content) {+ if !citationGroundingHasSubstantiveText(grounded) {if emittedVisible {return content, fmt.Errorf("citation grounding returned markers without answer text")}return content, nil}- return resp.Content, nil+ return grounded, nil}// chunksFromState extracts the recorded retrieval chunks from@@ -1103,7 +1119,7 @@ func (c *AgentComponent) invokeNow(ctx context.Context, db *gorm.DB, inputs map[var groundingStatus stringif p.Cite {- chunks := chunksFromState(ctx)+ chunks := limitCitationSources(chunksFromState(ctx), citationGroundingSourceTokenBudget)if len(chunks) == 0 {groundingStatus = "no_chunks"} else {diff --git a/internal/agent/component/citation_sources.go b/internal/agent/component/citation_sources.gonew file mode 100644index 0000000000..3027b85636--- /dev/null+++ b/internal/agent/component/citation_sources.go@@ -0,0 +1,159 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "regexp"+ "strings"++ "ragflow/internal/agent/component/prompts"+ "ragflow/internal/tokenizer"+)++// citationGroundingSourceTokenBudget caps the sources injected into the+// post-stream citation prompt. A multi-search Agent run can accumulate dozens+// of chunks; without a cap the second LLM call eventually overflows the+// model's context window.+const citationGroundingSourceTokenBudget = 30000++// citationMarkerMaxLength bounds a citation marker's full text length. It is+// far above [ID:<chunk id>] for any real chunk id, so a candidate longer than+// this is not a marker. The stream filter stops holding at this bound and the+// final filter leaves such text literal, keeping the streamed and saved+// content identical.+const citationMarkerMaxLength = 128++// citationGroundingMarkerCaptureRe matches a canonical [ID:<id>] marker and+// captures the id. Whitespace after "ID:" is tolerated and normalized away.+// The id cannot contain "[": an unclosed [ID: must not swallow the text up to+// a later ] (for example a Markdown link that follows it).+var citationGroundingMarkerCaptureRe = regexp.MustCompile(`\[ID:\s*([^\[\]]*)\]`)++// limitCitationSources keeps sources in order until the next one would exceed+// the token budget. Empty sources are skipped. The first valid source is+// always kept so a single oversized chunk still gets a grounding attempt.+func limitCitationSources(sources []prompts.CitationSource, budget int) []prompts.CitationSource {+ if budget <= 0 {+ return sources+ }+ kept := make([]prompts.CitationSource, 0, len(sources))+ used := 0+ for _, source := range sources {+ id := strings.TrimSpace(source.ID)+ if id == "" || strings.TrimSpace(source.Content) == "" {+ continue+ }+ tokens := tokenizer.NumTokensFromString(id) + tokenizer.NumTokensFromString(source.Content)+ if len(kept) > 0 && used+tokens > budget {+ break+ }+ kept = append(kept, prompts.CitationSource{ID: id, Content: source.Content})+ used += tokens+ }+ return kept+}++// filterCitationMarkers drops [ID:<id>] markers whose id is not in+// allowedIDs and normalizes the whitespace of the ones that remain.+func filterCitationMarkers(content string, allowedIDs []string) string {+ allowed := make(map[string]struct{}, len(allowedIDs))+ for _, id := range allowedIDs {+ if id = strings.TrimSpace(id); id != "" {+ allowed[id] = struct{}{}+ }+ }+ return filterCitationMarkersWithSet(content, allowed)+}++func filterCitationMarkersWithSet(content string, allowed map[string]struct{}) string {+ if content == "" {+ return ""+ }+ return citationGroundingMarkerCaptureRe.ReplaceAllStringFunc(content, func(marker string) string {+ // A marker longer than the cap is not a real citation id; keep the text+ // visible instead of silently dropping it.+ if len(marker) > citationMarkerMaxLength {+ return marker+ }+ match := citationGroundingMarkerCaptureRe.FindStringSubmatch(marker)+ if len(match) < 2 {+ return ""+ }+ id := strings.TrimSpace(match[1])+ if _, ok := allowed[id]; !ok {+ return ""+ }+ return "[ID:" + id + "]"+ })+}++// citationMarkerStreamFilter applies the same marker filtering to a streamed+// response. A marker can be split across deltas, so trailing text that could+// still become a marker is buffered until the next delta or Flush.+type citationMarkerStreamFilter struct {+ allowed map[string]struct{}+ pending string+}++func newCitationMarkerStreamFilter(allowedIDs []string) *citationMarkerStreamFilter {+ allowed := make(map[string]struct{}, len(allowedIDs))+ for _, id := range allowedIDs {+ if id = strings.TrimSpace(id); id != "" {+ allowed[id] = struct{}{}+ }+ }+ return &citationMarkerStreamFilter{allowed: allowed}+}++func (f *citationMarkerStreamFilter) Push(delta string) string {+ if delta == "" {+ return ""+ }+ text := f.pending + delta+ f.pending = ""+ if hold := citationMarkerHoldStart(text); hold >= 0 && len(text)-hold <= citationMarkerMaxLength {+ f.pending = text[hold:]+ text = text[:hold]+ }+ return filterCitationMarkersWithSet(text, f.allowed)+}++func (f *citationMarkerStreamFilter) Flush() string {+ text := f.pending+ f.pending = ""+ if text == "" {+ return ""+ }+ // Apply the same filter as the final (non-stream) path: a well-formed+ // marker is filtered or kept, while an unclosed [ID:... stays literal+ // instead of swallowing the text that follows it.+ return filterCitationMarkersWithSet(text, f.allowed)+}++// citationMarkerHoldStart returns the index of the trailing text that must be+// buffered because it may still turn into a citation marker.+func citationMarkerHoldStart(text string) int {+ if idx := strings.LastIndex(text, "[ID:"); idx >= 0 && !strings.Contains(text[idx:], "]") {+ return idx+ }+ for _, prefix := range []string{"[ID:", "[ID", "[I", "["} {+ if strings.HasSuffix(text, prefix) {+ return len(text) - len(prefix)+ }+ }+ return -1+}diff --git a/internal/agent/component/citation_sources_test.go b/internal/agent/component/citation_sources_test.gonew file mode 100644index 0000000000..f8a713762d--- /dev/null+++ b/internal/agent/component/citation_sources_test.go@@ -0,0 +1,235 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "strings"+ "testing"++ "ragflow/internal/agent/component/prompts"+ "ragflow/internal/agent/runtime"+ "ragflow/internal/tokenizer"+)++func TestLimitCitationSources_StopsAtBudget(t *testing.T) {+ sources := []prompts.CitationSource{+ {ID: "a", Content: strings.Repeat("alpha ", 20)},+ {ID: "b", Content: strings.Repeat("beta ", 20)},+ {ID: "c", Content: strings.Repeat("gamma ", 20)},+ }+ first := tokenizer.NumTokensFromString(sources[0].ID) + tokenizer.NumTokensFromString(sources[0].Content)+ second := tokenizer.NumTokensFromString(sources[1].ID) + tokenizer.NumTokensFromString(sources[1].Content)++ kept := limitCitationSources(sources, first+second-1)+ if len(kept) != 1 || kept[0].ID != "a" {+ t.Fatalf("kept = %#v, want only the first source", kept)+ }+ if all := limitCitationSources(sources, first+second+1); len(all) != 2 {+ t.Fatalf("kept = %d, want 2", len(all))+ }+}++func TestLimitCitationSources_KeepsFirstOversizedSource(t *testing.T) {+ sources := []prompts.CitationSource{{ID: "a", Content: strings.Repeat("x", 50000)}}+ if kept := limitCitationSources(sources, 1); len(kept) != 1 {+ t.Fatalf("kept = %d, want the first oversized source", len(kept))+ }+}++func TestLimitCitationSources_SkipsEmpty(t *testing.T) {+ sources := []prompts.CitationSource{+ {ID: "", Content: "no id"},+ {ID: "empty-content", Content: " "},+ {ID: "ok", Content: "body"},+ }+ kept := limitCitationSources(sources, 30000)+ if len(kept) != 1 || kept[0].ID != "ok" {+ t.Fatalf("kept = %#v, want only the valid source", kept)+ }+}++func TestFilterCitationMarkers(t *testing.T) {+ cases := []struct {+ name string+ content string+ allowed []string+ want string+ }{+ {"keeps allowed", "answer [ID:good] more", []string{"good"}, "answer [ID:good] more"},+ {"drops unknown", "a [ID:good] b [ID:bad] c", []string{"good"}, "a [ID:good] b c"},+ {"normalizes whitespace", "[ID: good ]", []string{"good"}, "[ID:good]"},+ {"drops empty id", "[ID:]", []string{"good"}, ""},+ {"drops all when no sources", "[ID:good]", nil, ""},+ {"leaves plain brackets", "see [note]", []string{"good"}, "see [note]"},+ {"leaves an unclosed marker before a markdown link", "answer [ID:oops more [link] end", []string{"good"}, "answer [ID:oops more [link] end"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ if got := filterCitationMarkers(tc.content, tc.allowed); got != tc.want {+ t.Fatalf("filterCitationMarkers(%q) = %q, want %q", tc.content, got, tc.want)+ }+ })+ }+}++func TestCitationMarkerStreamFilter_SplitMarkers(t *testing.T) {+ filter := newCitationMarkerStreamFilter([]string{"abc123"})+ var out strings.Builder+ for _, delta := range []string{"answer [ID:a", "bc", "123] done"} {+ out.WriteString(filter.Push(delta))+ }+ out.WriteString(filter.Flush())+ if got, want := out.String(), "answer [ID:abc123] done"; got != want {+ t.Fatalf("streamed = %q, want %q", got, want)+ }+}++func TestCitationMarkerStreamFilter_DropsSplitInvalidMarker(t *testing.T) {+ filter := newCitationMarkerStreamFilter([]string{"good"})+ var out strings.Builder+ for _, delta := range []string{"keep [ID:ba", "d] text"} {+ out.WriteString(filter.Push(delta))+ }+ out.WriteString(filter.Flush())+ if got, want := out.String(), "keep text"; got != want {+ t.Fatalf("streamed = %q, want %q", got, want)+ }+}++func TestCitationMarkerStreamFilter_FlushRules(t *testing.T) {+ filter := newCitationMarkerStreamFilter([]string{"good"})+ if got := filter.Push("see ["); got != "see " {+ t.Fatalf("Push held the partial bracket, got %q", got)+ }+ if got := filter.Flush(); got != "[" {+ t.Fatalf("Flush = %q, want the literal bracket", got)+ }++ unclosed := newCitationMarkerStreamFilter([]string{"good"})+ if got := unclosed.Push("answer [ID:good"); got != "answer " {+ t.Fatalf("Push held the unclosed marker, got %q", got)+ }+ if got := unclosed.Flush(); got != "[ID:good" {+ t.Fatalf("Flush = %q, want the unclosed marker left literal", got)+ }+}++func TestCitationMarkerStreamFilter_KeepsTextAfterUnclosedMarker(t *testing.T) {+ filter := newCitationMarkerStreamFilter([]string{"good"})+ var out strings.Builder+ for _, delta := range []string{"answer ", "[ID:good", " more text"} {+ out.WriteString(filter.Push(delta))+ }+ out.WriteString(filter.Flush())++ content := "answer [ID:good more text"+ want := filterCitationMarkers(content, []string{"good"})+ if got := out.String(); got != want {+ t.Fatalf("streamed = %q, want %q", got, want)+ }+ if !strings.Contains(out.String(), "more text") {+ t.Fatalf("streamed = %q, want the text after the unclosed marker", out.String())+ }+}++func TestCitationMarkerStreamFilter_BoundsLongCandidate(t *testing.T) {+ filter := newCitationMarkerStreamFilter([]string{"good"})+ long := "[ID:" + strings.Repeat("x", citationMarkerMaxLength)++ got := filter.Push(long) + filter.Flush()+ if got == "" {+ t.Fatal("Push buffered an over-long non-marker instead of emitting it")+ }+ if want := filterCitationMarkers(long, []string{"good"}); got != want {+ t.Fatalf("streamed = %q, want %q", got, want)+ }+}++func TestCitationMarkerStreamFilter_MatchesFinalFilter(t *testing.T) {+ allowed := []string{"good", "a2d067519c9e0db1"}+ cases := []struct {+ name string+ content string+ splits []int+ }{+ {name: "valid marker", content: "answer [ID:good] done", splits: []int{7, 12}},+ {name: "invalid marker", content: "keep [ID:bad] text", splits: []int{9}},+ {name: "unclosed marker", content: "answer [ID:good more text", splits: []int{7, 15}},+ {name: "over-long marker", content: "[ID:" + strings.Repeat("x", citationMarkerMaxLength+10) + "] tail", splits: []int{20, 40}},+ {name: "plain brackets", content: "see [note] and [", splits: []int{5}},+ {name: "unclosed marker before a markdown link", content: "answer [ID:oops more [link] end", splits: []int{7}},+ {name: "no markers", content: "just answer text", splits: []int{4}},+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ want := filterCitationMarkers(tc.content, allowed)+ filter := newCitationMarkerStreamFilter(allowed)+ var out strings.Builder+ prev := 0+ for _, cut := range tc.splits {+ if cut <= prev || cut >= len(tc.content) {+ continue+ }+ out.WriteString(filter.Push(tc.content[prev:cut]))+ prev = cut+ }+ out.WriteString(filter.Push(tc.content[prev:]))+ out.WriteString(filter.Flush())+ if got := out.String(); got != want {+ t.Fatalf("streamed = %q, want %q", got, want)+ }+ })+ }+}++func TestFilterCitationMarkers_LeavesOverlongMarkerLiteral(t *testing.T) {+ id := strings.Repeat("a", citationMarkerMaxLength)+ overlong := "[ID:" + id + "]"+ if got := filterCitationMarkers(overlong, []string{id}); got != overlong {+ t.Fatalf("filterCitationMarkers(overlong) = %q, want it left literal", got)+ }+}++func TestApplyCitationGrounding_FiltersUnknownIDs(t *testing.T) {+ inv := &groundingStreamingInvoker{groundingTestInvoker: groundingTestInvoker{+ responses: []string{"answer [ID:good] [ID:bad]"},+ }}+ prev := getDefaultChatInvoker()+ SetDefaultChatInvoker(inv)+ defer SetDefaultChatInvoker(prev)++ state := runtime.NewCanvasState("run-filter", "task-filter")+ ctx := runtime.WithState(t.Context(), state)+ var streamed string+ ctx = runtime.WithAgentMessageEmitter(ctx, func(content, _ string) {+ streamed += content+ })++ got, err := applyCitationGrounding(ctx, nil, AgentParam{ModelID: "stub", Cite: true}, "original", []prompts.CitationSource{+ {ID: "good", Content: "source body"},+ })+ if err != nil {+ t.Fatalf("applyCitationGrounding: %v", err)+ }+ if want := "answer [ID:good] "; got != want {+ t.Fatalf("returned content = %q, want %q", got, want)+ }+ if streamed != got {+ t.Fatalf("streamed = %q, want the filtered content %q", streamed, got)+ }+}diff --git a/internal/agent/component/grounding_test.go b/internal/agent/component/grounding_test.goindex cf66698ddf..1fa6570dbe 100644--- a/internal/agent/component/grounding_test.go+++ b/internal/agent/component/grounding_test.go@@ -266,6 +266,7 @@ func TestCitationGroundingHasSubstantiveText(t *testing.T) {{"The reimbursement request requires manager approval.", true},{"The reimbursement request requires manager approval. [ID:0]", true},{"The reimbursement request requires manager approval. [ID:simple-0]", true},+ {"[ID:oops more [link] end", true},{"答案[ID:0]", true},{"答案[ID:ck-1]", true},}diff --git a/internal/agent/component/reference_recording_tool.go b/internal/agent/component/reference_recording_tool.gonew file mode 100644index 0000000000..732df33ddf--- /dev/null+++ b/internal/agent/component/reference_recording_tool.go@@ -0,0 +1,68 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "context"++ einotool "github.com/cloudwego/eino/components/tool"++ "ragflow/internal/agent/runtime"+ agenttool "ragflow/internal/agent/tool"+)++// referenceRecordingTool decorates an Agent tool so each invocation records+// the retrieval references it produced into the canvas state. Canvas tool+// nodes already do this in ToolBackedComponent.Invoke; tools called from the+// Agent's ReAct loop go straight to InvokableRun, so without this wrapper+// state.Retrieval stays empty and citation grounding has no sources.+type referenceRecordingTool struct {+ einotool.InvokableTool+ builder agenttool.ReferenceBuilder+}++func (t *referenceRecordingTool) InvokableRun(ctx context.Context, argsJSON string, opts ...einotool.Option) (string, error) {+ out, err := t.InvokableTool.InvokableRun(ctx, argsJSON, opts...)+ if err != nil {+ return out, err+ }+ envelope := parseToolEnvelope(out)+ if _, invalid := envelope["_raw"]; invalid {+ return out, nil+ }+ chunks, docAggs := t.builder.BuildReferences(ctx, envelope)+ if state, stateErr := runtime.GetStateFromContext(ctx); stateErr == nil && state != nil {+ state.SetRetrievalReferences(chunks, docAggs)+ }+ return out, nil+}++// wrapReferenceRecordingTools decorates the tools handed to the ReAct loop.+// buildAgentTools keeps returning the raw tools so metadata paths such as+// AgentComponent.GetInputForm and Reset still see the original methods.+func wrapReferenceRecordingTools(tools []einotool.BaseTool) []einotool.BaseTool {+ wrapped := make([]einotool.BaseTool, len(tools))+ copy(wrapped, tools)+ for i, t := range tools {+ builder, buildsReferences := t.(agenttool.ReferenceBuilder)+ invokable, isInvokable := t.(einotool.InvokableTool)+ if buildsReferences && isInvokable {+ wrapped[i] = &referenceRecordingTool{InvokableTool: invokable, builder: builder}+ }+ }+ return wrapped+}diff --git a/internal/agent/component/reference_recording_tool_test.go b/internal/agent/component/reference_recording_tool_test.gonew file mode 100644index 0000000000..a7a2a923cd--- /dev/null+++ b/internal/agent/component/reference_recording_tool_test.go@@ -0,0 +1,139 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "context"+ "errors"+ "fmt"+ "testing"++ einotool "github.com/cloudwego/eino/components/tool"+ "github.com/cloudwego/eino/schema"++ "ragflow/internal/agent/runtime"+)++type referenceRecordingFakeTool struct {+ out string+ err error+}++func (f *referenceRecordingFakeTool) Info(_ context.Context) (*schema.ToolInfo, error) {+ return &schema.ToolInfo{Name: "fake_search"}, nil+}++func (f *referenceRecordingFakeTool) InvokableRun(_ context.Context, _ string, _ ...einotool.Option) (string, error) {+ return f.out, f.err+}++func (f *referenceRecordingFakeTool) BuildReferences(_ context.Context, envelope map[string]any) ([]map[string]any, []map[string]any) {+ results, _ := envelope["results"].([]any)+ chunks := make([]map[string]any, 0, len(results))+ docAggs := make([]map[string]any, 0, len(results))+ for i, raw := range results {+ id := fmt.Sprintf("chunk-%d", i)+ chunks = append(chunks, map[string]any{"id": id, "chunk_id": id, "document_id": id, "content": raw})+ docAggs = append(docAggs, map[string]any{"doc_id": id, "doc_name": raw})+ }+ return chunks, docAggs+}++func TestReferenceRecordingTool_RecordsAgentToolReferences(t *testing.T) {+ state := runtime.NewCanvasState("run-refs", "task-refs")+ ctx := runtime.WithState(t.Context(), state)+ inner := &referenceRecordingFakeTool{out: `{"results":["first","second"]}`}+ recorder := &referenceRecordingTool{InvokableTool: inner, builder: inner}++ out, err := recorder.InvokableRun(ctx, `{"query":"q"}`)+ if err != nil {+ t.Fatalf("InvokableRun: %v", err)+ }+ if out != inner.out {+ t.Fatalf("output = %q, want the inner tool output %q", out, inner.out)+ }+ chunks := state.GetRetrievalChunks()+ if len(chunks) != 2 {+ t.Fatalf("recorded chunks = %d, want 2", len(chunks))+ }+ if chunks[0]["id"] != "chunk-0" || chunks[1]["id"] != "chunk-1" {+ t.Fatalf("recorded chunk ids = %v / %v", chunks[0]["id"], chunks[1]["id"])+ }+ if aggs := state.GetRetrievalDocAggs(); len(aggs) != 2 {+ t.Fatalf("recorded doc aggs = %d, want 2", len(aggs))+ }+}++func TestReferenceRecordingTool_SkipsErrorsAndNonJSON(t *testing.T) {+ state := runtime.NewCanvasState("run-refs", "task-refs")+ ctx := runtime.WithState(t.Context(), state)++ failing := &referenceRecordingFakeTool{err: errors.New("boom")}+ if _, err := (&referenceRecordingTool{InvokableTool: failing, builder: failing}).InvokableRun(ctx, "{}"); err == nil {+ t.Fatal("expected the inner tool error to propagate")+ }+ plain := &referenceRecordingFakeTool{out: "not json"}+ if _, err := (&referenceRecordingTool{InvokableTool: plain, builder: plain}).InvokableRun(ctx, "{}"); err != nil {+ t.Fatalf("InvokableRun: %v", err)+ }+ if chunks := state.GetRetrievalChunks(); len(chunks) != 0 {+ t.Fatalf("recorded chunks = %d, want 0 for failed/non-JSON results", len(chunks))+ }+}++func TestReferenceRecordingTool_KeepsSameTitleDifferentDocumentIDs(t *testing.T) {+ state := runtime.NewCanvasState("run-refs", "task-refs")+ ctx := runtime.WithState(t.Context(), state)+ inner := &referenceRecordingFakeTool{out: `{"results":["same title","same title"]}`}+ recorder := &referenceRecordingTool{InvokableTool: inner, builder: inner}++ if _, err := recorder.InvokableRun(ctx, `{"query":"q"}`); err != nil {+ t.Fatalf("InvokableRun: %v", err)+ }+ if aggs := state.GetRetrievalDocAggs(); len(aggs) != 2 {+ t.Fatalf("recorded doc aggs = %d, want 2 for distinct document ids", len(aggs))+ }+ reference := state.GetRetrievalReference()+ chunks, _ := reference["chunks"].([]any)+ docAggs, _ := reference["doc_aggs"].([]any)+ if len(chunks) != 2 || len(docAggs) != 2 {+ t.Fatalf("reference = %d chunks / %d doc aggs, want 2/2", len(chunks), len(docAggs))+ }+}++// TestWrapReferenceRecordingTools_WrapsReferenceBuilders asserts the ReAct+// tool list is decorated while buildAgentTools keeps the raw tools.+func TestWrapReferenceRecordingTools_WrapsReferenceBuilders(t *testing.T) {+ raw, err := buildAgentTools(t.Context(), AgentParam{Tools: []string{"youcom"}})+ if err != nil {+ t.Fatalf("buildAgentTools: %v", err)+ }+ if len(raw) != 1 {+ t.Fatalf("raw tools = %d, want 1", len(raw))+ }+ if _, ok := raw[0].(*referenceRecordingTool); ok {+ t.Fatal("buildAgentTools must return the raw tool for metadata callers")+ }++ wrapped := wrapReferenceRecordingTools(raw)+ if len(wrapped) != 1 {+ t.Fatalf("wrapped tools = %d, want 1", len(wrapped))+ }+ if _, ok := wrapped[0].(*referenceRecordingTool); !ok {+ t.Fatalf("tool type = %T, want *referenceRecordingTool", wrapped[0])+ }+}diff --git a/internal/agent/component/tool_component.go b/internal/agent/component/tool_component.goindex 5bfe61196b..b5e94a2f0f 100644--- a/internal/agent/component/tool_component.go+++ b/internal/agent/component/tool_component.go@@ -156,6 +156,7 @@ var toolComponentRegistrations = []struct {{componentName: "WenCai", toolName: "wencai"},{componentName: "Wikipedia", toolName: "wikipedia"},{componentName: "YahooFinance", toolName: "yahoo_finance"},+ {componentName: "YouComSearch", toolName: "youcom"},}func init() {diff --git a/internal/agent/component/tool_component_test.go b/internal/agent/component/tool_component_test.goindex 4e31a15199..33386918e0 100644--- a/internal/agent/component/tool_component_test.go+++ b/internal/agent/component/tool_component_test.go@@ -256,6 +256,13 @@ func TestToolBackedComponentRegisteredFactories(t *testing.T) {outputKey: "report",inputKey: "stock_code",},+ {+ name: "YouComSearch",+ toolName: "YouComSearch",+ params: map[string]any{"api_key": "stored-key", "freshness": "week", "top_n": float64(3), "outputs": map[string]any{"json": map[string]any{}}},+ outputKey: "json",+ inputKey: "query",+ },{name: "TavilyExtract",toolName: "TavilyExtract",@@ -311,7 +318,7 @@ func TestToolBackedComponentWenCaiInvoke(t *testing.T) {}func TestToolBackedComponentRegisteredBuildWorkflow(t *testing.T) {- for _, componentName := range []string{"ArXiv", "BGPT", "DuckDuckGo", "Email", "Google", "GoogleScholar", "KeenableSearch", "PubMed", "QueritContents", "QueritSearch", "Search1APICrawl", "Search1APISearch", "SearXNG", "SofyaSearch", "WenCai", "TavilyExtract", "TavilySearch", "Wikipedia", "YahooFinance"} {+ for _, componentName := range []string{"ArXiv", "BGPT", "DuckDuckGo", "Email", "Google", "GoogleScholar", "KeenableSearch", "PubMed", "QueritContents", "QueritSearch", "Search1APICrawl", "Search1APISearch", "SearXNG", "SofyaSearch", "WenCai", "TavilyExtract", "TavilySearch", "Wikipedia", "YahooFinance", "YouComSearch"} {t.Run(componentName, func(t *testing.T) {c := &canvas.Canvas{Components: map[string]canvas.CanvasComponent{diff --git a/internal/agent/runtime/state.go b/internal/agent/runtime/state.goindex a95e7de0b3..2881c7ab62 100644--- a/internal/agent/runtime/state.go+++ b/internal/agent/runtime/state.go@@ -813,7 +813,23 @@ func copyRetrievalDocAggs(value any) []any {for key := range aggs {keys = append(keys, key)}- sort.Strings(keys)+ // Sort by document name to keep the payload stable across runs. The+ // document id breaks ties between sources that share a title.+ sort.SliceStable(keys, func(i, j int) bool {+ aggI, _ := aggs[keys[i]].(map[string]any)+ aggJ, _ := aggs[keys[j]].(map[string]any)+ nameI, _ := aggI["doc_name"].(string)+ nameJ, _ := aggJ["doc_name"].(string)+ if nameI != nameJ {+ return nameI < nameJ+ }+ idI, _ := aggI["doc_id"].(string)+ idJ, _ := aggJ["doc_id"].(string)+ if idI != idJ {+ return idI < idJ+ }+ return keys[i] < keys[j]+ })out := make([]any, 0, len(keys))for _, key := range keys {out = append(out, aggs[key])@@ -846,8 +862,7 @@ func (s *CanvasState) SetRetrievalChunks(chunks []map[string]any) {}// SetRetrievalReferences records the chunks and document aggregates emitted by-// a canvas search component. It is the lock-safe counterpart of Python-// Graph.add_reference for components that produce externally sourced results.+// a canvas search component.func (s *CanvasState) SetRetrievalReferences(chunks, docAggs []map[string]any) {if s == nil {return@@ -885,16 +900,32 @@ func (s *CanvasState) SetRetrievalReferences(chunks, docAggs []map[string]any) {if docAggValues == nil {docAggValues = make(map[string]any, len(docAggs))}+ seenDocAggKeys := make(map[string]struct{}, len(docAggValues)+len(docAggs))+ for _, item := range docAggValues {+ existing, ok := item.(map[string]any)+ if !ok {+ continue+ }+ if key, ok := retrievalDocAggKey(existing); ok {+ seenDocAggKeys[key] = struct{}{}+ }+ }for _, docAgg := range docAggs {- docName, _ := docAgg["doc_name"].(string)- if docName == "" {+ key, ok := retrievalDocAggKey(docAgg)+ if !ok {continue}- // Match Python Graph.add_reference: retain the first aggregate for- // a document name across the run-level reference set.- if _, exists := docAggValues[docName]; !exists {- docAggValues[docName] = docAgg+ // Retain the first aggregate per document id. Aggregates without an+ // id fall back to the document name, matching the previous behavior.+ // Keying by id keeps two sources with the same title but different+ // URLs from collapsing into one doc_agg. Existing entries are re-keyed+ // from their fields so checkpoints written before this change also+ // dedupe correctly.+ if _, exists := seenDocAggKeys[key]; exists {+ continue}+ seenDocAggKeys[key] = struct{}{}+ docAggValues[key] = docAgg}s.Retrieval["chunks"] = chunkValuess.Retrieval["doc_aggs"] = docAggValues@@ -909,7 +940,24 @@ func retrievalReferenceID(chunk map[string]any) (string, bool) {return id, id != ""}-// GetRetrievalDocAggs returns a shallow snapshot keyed by document name.+// retrievalDocAggKey returns the dedupe key for a document aggregate.+// Documents with an id are keyed by that id so same-title sources from+// different URLs stay distinct. Aggregates without an id fall back to the+// document name. Empty document names are skipped, matching the previous+// behavior.+func retrievalDocAggKey(docAgg map[string]any) (string, bool) {+ docName, _ := docAgg["doc_name"].(string)+ if docName == "" {+ return "", false+ }+ if docID, _ := docAgg["doc_id"].(string); docID != "" {+ return "id:" + docID, true+ }+ return "name:" + docName, true+}++// GetRetrievalDocAggs returns a shallow snapshot keyed by document id when+// present, otherwise by document name.func (s *CanvasState) GetRetrievalDocAggs() map[string]map[string]any {if s == nil {return nil@@ -921,10 +969,19 @@ func (s *CanvasState) GetRetrievalDocAggs() map[string]map[string]any {return nil}out := make(map[string]map[string]any, len(raw))- for name, item := range raw {- if agg, ok := item.(map[string]any); ok {- out[name] = agg+ for _, item := range raw {+ agg, ok := item.(map[string]any)+ if !ok {+ continue+ }+ key, _ := agg["doc_id"].(string)+ if key == "" {+ key, _ = agg["doc_name"].(string)+ }+ if key == "" {+ continue}+ out[key] = agg}return out}diff --git a/internal/agent/runtime/state_test.go b/internal/agent/runtime/state_test.goindex 6227dd34f5..67b3544cbe 100644--- a/internal/agent/runtime/state_test.go+++ b/internal/agent/runtime/state_test.go@@ -422,6 +422,79 @@ func TestCanvasState_SetRetrievalReferencesMergesCalls(t *testing.T) {}}+// TestCanvasState_SetRetrievalReferencesKeepsSameTitleDifferentIDs covers two+// search hits that share a title but point at different URLs. Keying doc_aggs+// by document name would drop the second source; the chunk and its doc_agg must+// both survive so the citation card can resolve the document.+func TestCanvasState_SetRetrievalReferencesKeepsSameTitleDifferentIDs(t *testing.T) {+ t.Parallel()++ state := NewCanvasState("run-1", "task-1")+ state.SetRetrievalReferences(+ []map[string]any{{"id": "chunk-1", "document_id": "doc-1"}},+ []map[string]any{{"doc_name": "Same Title", "doc_id": "doc-1", "url": "https://a.example", "count": 1}},+ )+ state.SetRetrievalReferences(+ []map[string]any{{"id": "chunk-2", "document_id": "doc-2"}},+ []map[string]any{{"doc_name": "Same Title", "doc_id": "doc-2", "url": "https://b.example", "count": 1}},+ )+ // A repeated id keeps the first aggregate, preserving the previous+ // first-writer-wins behavior.+ state.SetRetrievalReferences(nil, []map[string]any{+ {"doc_name": "Same Title", "doc_id": "doc-1", "url": "https://duplicate.example", "count": 9},+ })++ docAggs := state.GetRetrievalDocAggs()+ if len(docAggs) != 2 {+ t.Fatalf("doc_aggs length = %d, want 2", len(docAggs))+ }+ if got := docAggs["doc-1"]["url"]; got != "https://a.example" {+ t.Errorf("doc-1 url = %v, want the first aggregate", got)+ }+ if got := docAggs["doc-2"]["url"]; got != "https://b.example" {+ t.Errorf("doc-2 url = %v, want the second aggregate", got)+ }++ reference := state.GetRetrievalReference()+ list, _ := reference["doc_aggs"].([]any)+ if len(list) != 2 {+ t.Fatalf("reference doc_aggs length = %d, want 2", len(list))+ }+ // Same document name sorts by document id, so doc-1 precedes doc-2.+ for i, want := range []string{"doc-1", "doc-2"} {+ agg, _ := list[i].(map[string]any)+ if got := agg["doc_id"]; got != want {+ t.Errorf("doc_aggs[%d].doc_id = %v, want %s", i, got, want)+ }+ }+}++// TestCanvasState_SetRetrievalReferencesDedupesLegacyCheckpointKeys covers a+// checkpoint restored from the previous storage scheme, where doc_aggs were+// keyed by document name. Merging by the aggregate's fields must not append a+// second copy of the same document.+func TestCanvasState_SetRetrievalReferencesDedupesLegacyCheckpointKeys(t *testing.T) {+ t.Parallel()++ state := NewCanvasState("run-1", "task-1")+ state.Retrieval["doc_aggs"] = map[string]any{+ "Same Title": map[string]any{+ "doc_name": "Same Title",+ "doc_id": "doc-1",+ "url": "https://a.example",+ },+ }+ state.SetRetrievalReferences(nil, []map[string]any{+ {"doc_name": "Same Title", "doc_id": "doc-1", "url": "https://duplicate.example"},+ })++ reference := state.GetRetrievalReference()+ list, _ := reference["doc_aggs"].([]any)+ if len(list) != 1 {+ t.Fatalf("reference doc_aggs length = %d, want 1 after legacy merge", len(list))+ }+}+func TestCanvasState_GetRetrievalReferenceReturnsFrontendPayload(t *testing.T) {t.Parallel()diff --git a/internal/agent/tool/arxiv.go b/internal/agent/tool/arxiv.goindex cbd00ce0bb..cc920bc703 100644--- a/internal/agent/tool/arxiv.go+++ b/internal/agent/tool/arxiv.go@@ -19,16 +19,13 @@ package toolimport ("bytes""context"- "crypto/sha1""encoding/json""encoding/xml""fmt""io"- "math/big""net/http""net/url""regexp"- "strconv""strings""unicode/utf8"@@ -321,10 +318,9 @@ func buildArxivReferences(envelope map[string]any) ([]map[string]any, []map[striif content == "" {continue}- documentID := strconv.FormatInt(arxivHashInt(content, 100000000), 10)- displayID := strconv.FormatInt(arxivHashInt(documentID, 500), 10)title := arxivText(paper["title"])resultURL := arxivText(paper["pdf_url"])+ documentID, displayID := referenceIDs(resultURL, content)chunks = append(chunks, map[string]any{"id": displayID,"chunk_id": documentID,@@ -371,12 +367,6 @@ func arxivText(value any) string {return fmt.Sprint(value)}-func arxivHashInt(value string, modulus int64) int64 {- digest := sha1.Sum([]byte(value))- number := new(big.Int).SetBytes(digest[:])- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()-}-func truncateArxivRunes(value string, limit int) string {if utf8.RuneCountInString(value) <= limit {return valuediff --git a/internal/agent/tool/bgpt.go b/internal/agent/tool/bgpt.goindex c6737b4263..4f1eba80d8 100644--- a/internal/agent/tool/bgpt.go+++ b/internal/agent/tool/bgpt.go@@ -18,13 +18,10 @@ package toolimport ("context"- "crypto/sha1""encoding/json""fmt"- "math/big""net/http""regexp"- "strconv""strings""time""unicode/utf8"@@ -231,8 +228,6 @@ func buildBGPTReferences(envelope map[string]any) ([]map[string]any, []map[strinif content == "" {continue}- documentID := strconv.FormatInt(bgptHashInt(content, 100000000), 10)- displayID := strconv.FormatInt(bgptHashInt(documentID, 500), 10)title := bgptFirstField(paper, "title")if title == "-" {title = "Untitled"@@ -241,6 +236,7 @@ func buildBGPTReferences(envelope map[string]any) ([]map[string]any, []map[strinif resultURL == "-" {resultURL = ""}+ documentID, displayID := referenceIDs(resultURL, content)chunks = append(chunks, map[string]any{"id": displayID,"chunk_id": documentID,@@ -313,12 +309,6 @@ func renderBGPTReferences(chunks []map[string]any, maxTokens int) string {return strings.Join(blocks, "\n")}-func bgptHashInt(value string, modulus int64) int64 {- digest := sha1.Sum([]byte(value))- number := new(big.Int).SetBytes(digest[:])- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()-}-func truncateBGPTRunes(value string, limit int) string {if utf8.RuneCountInString(value) <= limit {return valuediff --git a/internal/agent/tool/duckduckgo.go b/internal/agent/tool/duckduckgo.goindex 191d9a06f4..3923ebdf1f 100644--- a/internal/agent/tool/duckduckgo.go+++ b/internal/agent/tool/duckduckgo.go@@ -18,12 +18,10 @@ package toolimport ("context"- "crypto/sha1""encoding/json""fmt""html""io"- "math/big""net/http""net/url""regexp"@@ -277,10 +275,9 @@ func buildDuckDuckGoReferences(envelope map[string]any) ([]map[string]any, []mapif content == "" {continue}- documentID := strconv.FormatInt(duckduckgoHashInt(content, 100000000), 10)- displayID := strconv.FormatInt(duckduckgoHashInt(documentID, 500), 10)title := duckduckgoText(item["title"])resultURL := duckduckgoText(item["url"])+ documentID, displayID := referenceIDs(resultURL, content)chunks = append(chunks, map[string]any{"id": displayID,"chunk_id": documentID,@@ -327,12 +324,6 @@ func duckduckgoText(value any) string {return fmt.Sprint(value)}-func duckduckgoHashInt(value string, modulus int64) int64 {- digest := sha1.Sum([]byte(value))- number := new(big.Int).SetBytes(digest[:])- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()-}-func truncateDuckDuckGoRunes(value string, limit int) string {if utf8.RuneCountInString(value) <= limit {return valuediff --git a/internal/agent/tool/github.go b/internal/agent/tool/github.goindex 4c6a3c800b..3dd21fc88d 100644--- a/internal/agent/tool/github.go+++ b/internal/agent/tool/github.go@@ -18,13 +18,10 @@ package toolimport ("context"- "crypto/sha1""encoding/json""fmt"- "math/big""net/http""net/url"- "strconv""strings""time""unicode/utf8"@@ -230,10 +227,9 @@ func buildGitHubReferences(envelope map[string]any) ([]map[string]any, []map[strif content == "" {continue}- documentID := strconv.FormatInt(githubHashInt(content, 100000000), 10)title := githubValueString(repository["name"])resultURL := githubValueString(repository["html_url"])- displayID := strconv.FormatInt(githubHashInt(documentID, 500), 10)+ documentID, displayID := referenceIDs(resultURL, content)chunks = append(chunks, map[string]any{"id": displayID,"chunk_id": documentID,@@ -311,12 +307,6 @@ func githubValueString(value any) string {return fmt.Sprint(value)}-func githubHashInt(value string, modulus int64) int64 {- sum := sha1.Sum([]byte(value))- number := new(big.Int).SetBytes(sum[:])- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()-}-func truncateGitHubRunes(value string, limit int) string {if utf8.RuneCountInString(value) <= limit {return valuediff --git a/internal/agent/tool/google.go b/internal/agent/tool/google.goindex be9827c8a5..32be198fc4 100644--- a/internal/agent/tool/google.go+++ b/internal/agent/tool/google.go@@ -18,10 +18,8 @@ package toolimport ("context"- "crypto/sha1""encoding/json""fmt"- "math/big""net/http""net/url""regexp"@@ -271,10 +269,9 @@ func buildGoogleReferences(envelope map[string]any) ([]map[string]any, []map[strif content == "" {continue}- documentID := strconv.FormatInt(googleHashInt(content, 100000000), 10)- displayID := strconv.FormatInt(googleHashInt(documentID, 500), 10)title := strings.TrimSpace(googleText(item["title"]))resultURL := strings.TrimSpace(googleText(item["link"]))+ documentID, displayID := referenceIDs(resultURL, content)chunks = append(chunks, map[string]any{"id": displayID,"chunk_id": documentID,@@ -347,12 +344,6 @@ func googlePromptField(value any) string {return googleNewlinePattern.ReplaceAllString(googleText(value), " ")}-func googleHashInt(value string, modulus int64) int64 {- digest := sha1.Sum([]byte(value))- number := new(big.Int).SetBytes(digest[:])- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()-}-func truncateGoogleRunes(value string, limit int) string {if utf8.RuneCountInString(value) <= limit {return valuediff --git a/internal/agent/tool/google_scholar.go b/internal/agent/tool/google_scholar.goindex 8cc1698fb6..12c7aead85 100644--- a/internal/agent/tool/google_scholar.go+++ b/internal/agent/tool/google_scholar.go@@ -18,10 +18,8 @@ package toolimport ("context"- "crypto/sha1""encoding/json""fmt"- "math/big""net/http""net/url""regexp"@@ -256,10 +254,9 @@ func buildGoogleScholarReferences(envelope map[string]any) ([]map[string]any, []if content == "" {continue}- documentID := strconv.FormatInt(googleScholarHashInt(content, 100000000), 10)- displayID := strconv.FormatInt(googleScholarHashInt(documentID, 500), 10)title := googleScholarText(paper["title"])resultURL := googleScholarText(paper["link"])+ documentID, displayID := referenceIDs(resultURL, content)chunks = append(chunks, map[string]any{"id": displayID,"chunk_id": documentID,@@ -308,12 +305,6 @@ func googleScholarText(value any) string {return fmt.Sprint(value)}-func googleScholarHashInt(value string, modulus int64) int64 {- digest := sha1.Sum([]byte(value))- number := new(big.Int).SetBytes(digest[:])- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()-}-func truncateGoogleScholarRunes(value string, limit int) string {if utf8.RuneCountInString(value) <= limit {return valuediff --git a/internal/agent/tool/keenable.go b/internal/agent/tool/keenable.goindex 41790cb471..441a18278f 100644--- a/internal/agent/tool/keenable.go+++ b/internal/agent/tool/keenable.go@@ -18,13 +18,10 @@ package toolimport ("context"- "crypto/sha1""encoding/json""fmt"- "math/big""net/http"neturl "net/url"- "strconv""strings""unicode/utf8"@@ -361,10 +358,9 @@ func buildKeenableReferences(envelope map[string]any) ([]map[string]any, []map[sif content == "" {continue}- documentID := strconv.FormatInt(keenableHashInt(content, 100000000), 10)title := keenableValueString(result["title"])resultURL := keenableValueString(result["url"])- displayID := strconv.FormatInt(keenableHashInt(documentID, 500), 10)+ documentID, displayID := referenceIDs(resultURL, content)chunks = append(chunks, map[string]any{"id": displayID,"chunk_id": documentID,@@ -419,12 +415,6 @@ func keenableValueString(value any) string {return strings.TrimSpace(fmt.Sprint(value))}-func keenableHashInt(value string, modulus int64) int64 {- sum := sha1.Sum([]byte(value))- number := new(big.Int).SetBytes(sum[:])- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()-}-func truncateKeenableRunes(value string, limit int) string {if utf8.RuneCountInString(value) <= limit {return valuediff --git a/internal/agent/tool/pubmed.go b/internal/agent/tool/pubmed.goindex 612552e9d1..a913cf46fd 100644--- a/internal/agent/tool/pubmed.go+++ b/internal/agent/tool/pubmed.go@@ -18,11 +18,9 @@ package toolimport ("context"- "crypto/sha1""encoding/json""encoding/xml""fmt"- "math/big""net/http""net/url""regexp"@@ -282,10 +280,9 @@ func buildPubMedReferences(envelope map[string]any) ([]map[string]any, []map[strif content == "" {continue}- documentID := strconv.FormatInt(pubmedHashInt(content, 100000000), 10)- displayID := strconv.FormatInt(pubmedHashInt(documentID, 500), 10)title := pubmedText(item["title"])resultURL := pubmedText(item["url"])+ documentID, displayID := referenceIDs(resultURL, content)chunks = append(chunks, map[string]any{"id": displayID,"chunk_id": documentID,@@ -332,12 +329,6 @@ func pubmedText(value any) string {return fmt.Sprint(value)}-func pubmedHashInt(value string, modulus int64) int64 {- digest := sha1.Sum([]byte(value))- number := new(big.Int).SetBytes(digest[:])- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()-}-func truncatePubMedRunes(value string, limit int) string {if utf8.RuneCountInString(value) <= limit {return valuediff --git a/internal/agent/tool/querit.go b/internal/agent/tool/querit.goindex ca651148d4..f520f79e8c 100644--- a/internal/agent/tool/querit.go+++ b/internal/agent/tool/querit.go@@ -18,14 +18,11 @@ package toolimport ("context"- "crypto/sha1""encoding/json""fmt""io"- "math/big""net/http""regexp"- "strconv""strings""time"@@ -489,8 +486,7 @@ func (q *QueritTool) BuildReferences(_ context.Context, response map[string]any)if content == "" {continue}- documentID := strconv.FormatInt(queritHashInt(content, 100000000), 10)- displayID := strconv.FormatInt(queritHashInt(documentID, 500), 10)+ documentID, displayID := referenceIDs(resultURL, content)chunks = append(chunks, map[string]any{"id": displayID,"chunk_id": documentID,@@ -577,12 +573,6 @@ func queritPromptField(value any) string {return queritNewlinePattern.ReplaceAllString(queritText(value), " ")}-func queritHashInt(value string, modulus int64) int64 {- digest := sha1.Sum([]byte(value))- number := new(big.Int).SetBytes(digest[:])- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()-}-func truncateQueritRunes(value string, limit int) string {if limit <= 0 {return ""diff --git a/internal/agent/tool/reference_id.go b/internal/agent/tool/reference_id.gonew file mode 100644index 0000000000..58e99ece03--- /dev/null+++ b/internal/agent/tool/reference_id.go@@ -0,0 +1,18 @@+package tool++import (+ "crypto/sha256"+ "encoding/hex"+ "strings"+)++// referenceIDs derives the citation identifiers for one search result.+// documentID keys the chunk and its docAgg; displayID is the citation marker+// rendered into the prompt. The URL participates in the seed so identical+// content served from different URLs stays distinguishable, and the 64-bit+// halves make collisions negligible at search-result volumes.+func referenceIDs(resultURL, content string) (documentID, displayID string) {+ seed := strings.TrimSpace(resultURL) + "\n" + content+ sum := sha256.Sum256([]byte(seed))+ return hex.EncodeToString(sum[:8]), hex.EncodeToString(sum[8:16])+}diff --git a/internal/agent/tool/reference_id_test.go b/internal/agent/tool/reference_id_test.gonew file mode 100644index 0000000000..12cb278799--- /dev/null+++ b/internal/agent/tool/reference_id_test.go@@ -0,0 +1,73 @@+package tool++import (+ "fmt"+ "testing"+)++func TestReferenceIDsDistinguishURLsForSameContent(t *testing.T) {+ content := "the same snippet text"+ docA, displayA := referenceIDs("https://a.example/page", content)+ docB, displayB := referenceIDs("https://b.example/page", content)+ if docA == docB {+ t.Fatalf("documentID collided across URLs: %q", docA)+ }+ if displayA == displayB {+ t.Fatalf("displayID collided across URLs: %q", displayA)+ }+}++func TestReferenceIDsAreDeterministic(t *testing.T) {+ for _, tc := range []struct {+ name string+ url string+ content string+ }{+ {name: "url and content", url: "https://example.com/x", content: "body"},+ {name: "empty url", url: "", content: "body"},+ {name: "blank url", url: " ", content: "body"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ docA, displayA := referenceIDs(tc.url, tc.content)+ docB, displayB := referenceIDs(tc.url, tc.content)+ if docA != docB || displayA != displayB {+ t.Fatalf("not deterministic: (%q,%q) vs (%q,%q)", docA, displayA, docB, displayB)+ }+ })+ }+}++func TestReferenceIDsAreUniqueAcrossInputs(t *testing.T) {+ seenDoc := make(map[string]string, 500)+ seenDisplay := make(map[string]string, 500)+ for i := 0; i < 500; i++ {+ url := fmt.Sprintf("https://example.com/%d", i)+ content := fmt.Sprintf("snippet %d", i)+ docID, displayID := referenceIDs(url, content)+ if prev, ok := seenDoc[docID]; ok {+ t.Fatalf("documentID collision: %q produced by %q and %q", docID, prev, url)+ }+ if prev, ok := seenDisplay[displayID]; ok {+ t.Fatalf("displayID collision: %q produced by %q and %q", displayID, prev, url)+ }+ seenDoc[docID] = url+ seenDisplay[displayID] = url+ }+}++func TestReferenceIDsFormat(t *testing.T) {+ docID, displayID := referenceIDs("https://example.com/page", "body")+ for name, id := range map[string]string{"documentID": docID, "displayID": displayID} {+ if len(id) != 16 {+ t.Fatalf("%s length = %d, want 16 (%q)", name, len(id), id)+ }+ for _, r := range id {+ if (r < '0' || r > '9') && (r < 'a' || r > 'f') {+ t.Fatalf("%s = %q is not lowercase hex", name, id)+ }+ }+ }+ if docID == displayID {+ t.Fatalf("documentID and displayID must differ, both are %q", docID)+ }+}diff --git a/internal/agent/tool/registry.go b/internal/agent/tool/registry.goindex 0affe02dea..29326f502d 100644--- a/internal/agent/tool/registry.go+++ b/internal/agent/tool/registry.go@@ -75,6 +75,7 @@ var registry = map[string]Factory{"wikipedia": buildWikipediaTool,"wikipedia_search": buildWikipediaTool,"yahoo_finance": buildYahooFinanceTool,+ "youcom": buildYouComTool,}// canvasToolNames maps lower-cased Canvas component names to the canonical@@ -93,6 +94,7 @@ var canvasToolNames = map[string]string{"tavilyextract": "tavily_extract","tavilysearch": "tavily","yahoofinance": "yahoo_finance",+ "youcomsearch": "youcom",}func noConfig(name string, fn func() einotool.BaseTool) Factory {@@ -751,6 +753,37 @@ func buildKeenableTool(params map[string]any) (einotool.BaseTool, error) {return newKeenableTool(nil, nil, apiKey, defaults), nil}+func buildYouComTool(params map[string]any) (einotool.BaseTool, error) {+ defaults := youComDefaults{}+ apiKey := ""+ if value, ok := params["api_key"]; ok {+ var valid bool+ apiKey, valid = value.(string)+ if !valid {+ return nil, fmt.Errorf("agent tool: tool %q requires string node-level param api_key", "youcom")+ }+ }+ if value, ok := params["freshness"]; ok {+ freshness, valid := value.(string)+ if !valid {+ return nil, fmt.Errorf("agent tool: tool %q requires string node-level param freshness", "youcom")+ }+ normalized := strings.ToLower(strings.TrimSpace(freshness))+ if !slices.Contains(youComFreshnessValues, normalized) {+ return nil, fmt.Errorf("agent tool: tool %q has unsupported freshness %v", "youcom", value)+ }+ defaults.Freshness = normalized+ }+ if value, ok := params["top_n"]; ok {+ topN, valid := strictInt(value)+ if !valid || topN <= 0 {+ return nil, fmt.Errorf("agent tool: tool %q requires positive integer node-level param top_n", "youcom")+ }+ defaults.TopN = min(topN, youComMaxResults)+ }+ return newYouComTool(nil, apiKey, defaults), nil+}+func buildSofyaTool(params map[string]any) (einotool.BaseTool, error) {defaults := sofyaDefaults{}apiKey := ""diff --git a/internal/agent/tool/registry_test.go b/internal/agent/tool/registry_test.goindex 01e6cf7312..ecd26b4bfc 100644--- a/internal/agent/tool/registry_test.go+++ b/internal/agent/tool/registry_test.go@@ -56,6 +56,7 @@ func TestBuildByName_CanvasComponentNames(t *testing.T) {{name: "TavilyExtract", wantToolName: "tavily_extract"},{name: "TavilySearch", wantToolName: "tavily_search"},{name: "YahooFinance", wantToolName: "yahoo_finance"},+ {name: "YouComSearch", wantToolName: "youcom_search"},}for _, tc := range tests {@@ -159,6 +160,36 @@ func TestBuildByName_QueritContentsAliases(t *testing.T) {}}+func TestBuildByName_YouComReadsCanvasParams(t *testing.T) {+ // The canvas stores tool params under the lower-cased component name, and+ // the node form also carries query/outputs keys the tool must ignore.+ tools, err := BuildAll([]string{"YouComSearch"}, map[string]map[string]any{+ "youcomsearch": {+ "api_key": "stored-key",+ "freshness": "week",+ "top_n": float64(4),+ "outputs": map[string]any{"json": map[string]any{}},+ },+ })+ if err != nil {+ t.Fatalf("BuildAll: %v", err)+ }+ if len(tools) != 1 {+ t.Fatalf("len(tools) = %d, want 1", len(tools))+ }+ youcomTool, ok := tools[0].(*YouComTool)+ if !ok {+ t.Fatalf("tools[0] = %T, want *YouComTool", tools[0])+ }+ if youcomTool.apiKey != "stored-key" || youcomTool.defaults.Freshness != "week" || youcomTool.defaults.TopN != 4 {+ t.Fatalf("defaults = %#v, apiKey = %q", youcomTool.defaults, youcomTool.apiKey)+ }+ info, infoErr := youcomTool.Info(t.Context())+ if infoErr != nil || info.Name != "youcom_search" {+ t.Fatalf("Info() = %#v, %v", info, infoErr)+ }+}+func TestBuildAll_AllRegisteredTools(t *testing.T) {// Every key in registry.names := []string{@@ -168,7 +199,7 @@ func TestBuildAll_AllRegisteredTools(t *testing.T) {"querit", "querit_contents", "querit_search","retrieval", "search_my_dataset", "search_my_dateset", "search1api", "search1api_crawl", "searxng", "sofya","tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", "wikipedia_search",- "yahoo_finance",+ "yahoo_finance", "youcom",}params := map[string]map[string]any{"execute_sql": {@@ -232,7 +263,7 @@ func TestToolRegistry_SchemasAreComplete(t *testing.T) {"querit", "querit_contents", "querit_search","retrieval", "search_my_dataset", "search_my_dateset", "search1api", "search1api_crawl", "searxng", "sofya","tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", "wikipedia_search",- "yahoo_finance",+ "yahoo_finance", "youcom",}params := map[string]map[string]any{"execute_sql": {diff --git a/internal/agent/tool/search1api.go b/internal/agent/tool/search1api.goindex 1cd9ef49b7..3d7bed3bb9 100644--- a/internal/agent/tool/search1api.go+++ b/internal/agent/tool/search1api.go@@ -18,10 +18,8 @@ package toolimport ("context"- "crypto/sha1""encoding/json""fmt"- "math/big""net/http""slices""strconv"@@ -386,10 +384,9 @@ func buildSearch1APIReferences(envelope map[string]any) ([]map[string]any, []mapif content == "" {continue}- documentID := strconv.FormatInt(search1APIHashInt(content, 100000000), 10)- displayID := strconv.FormatInt(search1APIHashInt(documentID, 500), 10)title := search1APIText(result["title"])resultURL := search1APIText(result["link"])+ documentID, displayID := referenceIDs(resultURL, content)chunks = append(chunks, map[string]any{"id": displayID,"chunk_id": documentID,@@ -441,12 +438,6 @@ func search1APIText(value any) string {return strings.TrimSpace(fmt.Sprint(value))}-func search1APIHashInt(value string, modulus int64) int64 {- digest := sha1.Sum([]byte(value))- number := new(big.Int).SetBytes(digest[:])- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()-}-func truncateSearch1APIRunes(value string, limit int) string {if utf8.RuneCountInString(value) <= limit {return valuediff --git a/internal/agent/tool/searxng.go b/internal/agent/tool/searxng.goindex 3dd4c79b5a..fefd5fb624 100644--- a/internal/agent/tool/searxng.go+++ b/internal/agent/tool/searxng.go@@ -18,7 +18,6 @@ package toolimport ("context"- "crypto/sha1""encoding/json""fmt""net"@@ -287,10 +286,9 @@ func buildSearXNGReferences(envelope map[string]any) ([]map[string]any, []map[stcontinue}- documentID := strconv.Itoa(hashSearXNGString(content, 100000000))- displayID := strconv.Itoa(hashSearXNGString(documentID, 500))title := searxngText(item["title"])resultURL := searxngText(item["url"])+ documentID, displayID := referenceIDs(resultURL, content)chunks = append(chunks, map[string]any{"id": displayID,"chunk_id": documentID,@@ -363,15 +361,6 @@ func searxngPromptField(value any) string {return searxngNewlinePattern.ReplaceAllString(searxngText(value), " ")}-func hashSearXNGString(value string, modulus int) int {- digest := sha1.Sum([]byte(value))- result := 0- for _, part := range digest {- result = (result*256 + int(part)) % modulus- }- return result-}-func searxngJSON(env searxngEnvelope) string {data, err := json.Marshal(env)if err != nil {diff --git a/internal/agent/tool/searxng_test.go b/internal/agent/tool/searxng_test.goindex 43b0755572..a487fac9e0 100644--- a/internal/agent/tool/searxng_test.go+++ b/internal/agent/tool/searxng_test.go@@ -23,7 +23,6 @@ import ("net/http""net/http/httptest""net/url"- "strconv""strings""sync/atomic""testing"@@ -233,12 +232,8 @@ func TestSearXNGComponentContractReferencesAndOutputs(t *testing.T) {t.Fatalf("references = %#v / %#v", chunks, docAggs)}cleaned := "RAGFlow content remains"- documentID := hashSearXNGString(cleaned, 100000000)- referenceID := hashSearXNGString(strconv.Itoa(documentID), 500)- if documentID != 93760153 || referenceID != 491 {- t.Fatalf("hash parity = %d/%d, want Python 93760153/491", documentID, referenceID)- }- if chunks[0]["document_id"] != strconv.Itoa(documentID) || chunks[0]["id"] != strconv.Itoa(referenceID) || chunks[0]["content"] != cleaned {+ documentID, referenceID := referenceIDs("https://ragflow.io", cleaned)+ if chunks[0]["document_id"] != documentID || chunks[0]["id"] != referenceID || chunks[0]["content"] != cleaned {t.Fatalf("reference chunk = %#v", chunks[0])}@@ -249,7 +244,7 @@ func TestSearXNGComponentContractReferencesAndOutputs(t *testing.T) {}formalized := outputs["formalized_content"].(string)for _, want := range []string{- "ID: " + strconv.Itoa(referenceID),+ "ID: " + referenceID,"Title: RAGFlow Docs","URL: https://ragflow.io","Content:\n" + cleaned,diff --git a/internal/agent/tool/sofya.go b/internal/agent/tool/sofya.goindex 9dbf52bcd0..527a0edef4 100644--- a/internal/agent/tool/sofya.go+++ b/internal/agent/tool/sofya.go@@ -18,10 +18,8 @@ package toolimport ("context"- "crypto/sha1""encoding/json""fmt"- "math/big""net/http""strconv""strings"@@ -328,10 +326,9 @@ func buildSofyaReferences(envelope map[string]any) ([]map[string]any, []map[striif content == "" {continue}- documentID := strconv.FormatInt(sofyaHashInt(content, 100000000), 10)- displayID := strconv.FormatInt(sofyaHashInt(documentID, 500), 10)title := sofyaText(result["title"])resultURL := sofyaText(result["url"])+ documentID, displayID := referenceIDs(resultURL, content)chunks = append(chunks, map[string]any{"id": displayID,"chunk_id": documentID,@@ -383,12 +380,6 @@ func sofyaText(value any) string {return strings.TrimSpace(fmt.Sprint(value))}-func sofyaHashInt(value string, modulus int64) int64 {- digest := sha1.Sum([]byte(value))- number := new(big.Int).SetBytes(digest[:])- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()-}-func truncateSofyaRunes(value string, limit int) string {if utf8.RuneCountInString(value) <= limit {return valuediff --git a/internal/agent/tool/tavily.go b/internal/agent/tool/tavily.goindex 57fd2ec8e5..da3d4d7cdd 100644--- a/internal/agent/tool/tavily.go+++ b/internal/agent/tool/tavily.go@@ -18,13 +18,10 @@ package toolimport ("context"- "crypto/sha1""encoding/json""fmt"- "math/big""net/http""regexp"- "strconv""strings""unicode/utf8"@@ -444,10 +441,9 @@ func buildTavilyReferences(envelope map[string]any) ([]map[string]any, []map[strif content == "" {continue}- documentID := strconv.FormatInt(tavilyHashInt(content, 100000000), 10)- displayID := strconv.FormatInt(tavilyHashInt(documentID, 500), 10)title := tavilyText(item["title"])resultURL := tavilyText(item["url"])+ documentID, displayID := referenceIDs(resultURL, content)score := item["score"]chunks = append(chunks, map[string]any{"id": displayID,@@ -507,12 +503,6 @@ func tavilyPromptField(value any) string {return tavilyNewlinePattern.ReplaceAllString(tavilyText(value), " ")}-func tavilyHashInt(value string, modulus int64) int64 {- digest := sha1.Sum([]byte(value))- number := new(big.Int).SetBytes(digest[:])- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()-}-func truncateTavilyRunes(value string, limit int) string {if utf8.RuneCountInString(value) <= limit {return valuediff --git a/internal/agent/tool/wikipedia.go b/internal/agent/tool/wikipedia.goindex e4c0cd76c1..7db31a37b3 100644--- a/internal/agent/tool/wikipedia.go+++ b/internal/agent/tool/wikipedia.go@@ -18,15 +18,12 @@ package toolimport ("context"- "crypto/sha1""encoding/json""fmt"- "math/big""net/http""net/url""regexp""sort"- "strconv""strings""unicode/utf8"@@ -308,10 +305,9 @@ func buildWikipediaReferences(envelope map[string]any) ([]map[string]any, []map[if content == "" {continue}- documentID := strconv.FormatInt(wikipediaHashInt(content, 100000000), 10)- displayID := strconv.FormatInt(wikipediaHashInt(documentID, 500), 10)title := wikipediaText(item["title"])resultURL := wikipediaText(item["url"])+ documentID, displayID := referenceIDs(resultURL, content)chunks = append(chunks, map[string]any{"id": displayID,"chunk_id": documentID,@@ -360,12 +356,6 @@ func wikipediaText(value any) string {return fmt.Sprint(value)}-func wikipediaHashInt(value string, modulus int64) int64 {- digest := sha1.Sum([]byte(value))- number := new(big.Int).SetBytes(digest[:])- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()-}-func truncateWikipediaRunes(value string, limit int) string {if utf8.RuneCountInString(value) <= limit {return valuediff --git a/internal/agent/tool/youcom.go b/internal/agent/tool/youcom.gonew file mode 100644index 0000000000..391a26f003--- /dev/null+++ b/internal/agent/tool/youcom.go@@ -0,0 +1,363 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package tool++import (+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "strconv"+ "strings"+ "unicode/utf8"++ "github.com/cloudwego/eino/components/tool"+ "github.com/cloudwego/eino/schema"++ "ragflow/internal/service/youcom"+ "ragflow/internal/tokenizer"+)++const youComToolName = "youcom_search"++const youComToolDescription = `You.com is a web search API. It returns the extracted passages of each result page, so hits carry usable context without a separate fetch. When searching:+ - Use a focused query of the most important terms (and synonyms).+ - Optionally restrict results by how recently they were published.`++// youComMaxResults caps how many results one search returns.+const youComMaxResults = 20++const youComDefaultTopN = 10++const youComPromptMaxTokens = 200000++// youComMaxResponseBytes caps how much of You.com's response body is read: a+// handful of hits of JSON is a few KiB, so 4 MiB is far above any legitimate+// answer while still bounding what a misbehaving upstream can make us allocate.+const youComMaxResponseBytes = 4 << 20++var youComFreshnessValues = []string{"any", "day", "week", "month", "year"}++// youComArgs is the JSON shape the model sends into InvokableRun. max_results+// is decoded loosely: a missing, blank or non-numeric value means "use the+// node's Top N" rather than failing the run.+type youComArgs struct {+ Query string `json:"query"`+ Freshness string `json:"freshness"`+ MaxResults any `json:"max_results"`+}++// youComDefaults holds the node-level configuration.+type youComDefaults struct {+ Freshness string+ TopN int+}++type youComEnvelope struct {+ Results []map[string]any `json:"results"`+ Error string `json:"_ERROR,omitempty"`+}++// YouComTool is the You.com web search tool. It queries+// https://api.you.com/v1/search (or the keyless endpoint when no API key is+// configured) and returns the merged web + news results. The API key is node+// configuration only and never model-visible.+type YouComTool struct {+ helper *HTTPHelper+ apiKey string+ defaults youComDefaults+}++var _ ToolComponent = (*YouComTool)(nil)+var _ ReferenceBuilder = (*YouComTool)(nil)++// NewYouComTool returns a YouComTool with no API key and the default HTTPHelper.+func NewYouComTool() *YouComTool {+ return newYouComTool(nil, "", youComDefaults{})+}++// NewYouComToolWith returns a YouComTool that uses the provided HTTPHelper and+// API key. Useful for tests that want to inject a custom transport.+func NewYouComToolWith(h *HTTPHelper, apiKey string) *YouComTool {+ return newYouComTool(h, apiKey, youComDefaults{})+}++func newYouComTool(h *HTTPHelper, apiKey string, defaults youComDefaults) *YouComTool {+ if h == nil {+ h = NewHTTPHelper()+ }+ if defaults.Freshness == "" {+ defaults.Freshness = "any"+ }+ if defaults.TopN <= 0 {+ defaults.TopN = youComDefaultTopN+ }+ return &YouComTool{helper: h, apiKey: strings.TrimSpace(apiKey), defaults: defaults}+}++// Info returns the tool's metadata for the chat model.+func (t *YouComTool) Info(_ context.Context) (*schema.ToolInfo, error) {+ return &schema.ToolInfo{+ Name: youComToolName,+ Desc: youComToolDescription,+ ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{+ "query": {+ Type: schema.String,+ Desc: "The search keywords to execute with You.com. The most important words/terms (and synonyms) from the original request.",+ Required: true,+ },+ "freshness": {+ Type: schema.String,+ Desc: `Restrict results by recency: "day", "week", "month", "year", or "any" (default) for no limit.`,+ Enum: youComFreshnessValues,+ Required: false,+ },+ "max_results": {+ Type: schema.Integer,+ Desc: "Number of results to return, 1 to 20. Leave it out to use the Top N set on the node.",+ Required: false,+ },+ }),+ }, nil+}++// InvokableRun performs the You.com search.+//+// Recoverable errors (bad arguments, API or network failures) are returned as a+// JSON envelope with an _ERROR field and a nil Go error, so the ReAct agent+// feeds them back to the model instead of failing the run, matching TavilyTool.+func (t *YouComTool) InvokableRun(ctx context.Context, argsJSON string, _ ...tool.Option) (string, error) {+ var args youComArgs+ if err := json.Unmarshal([]byte(argsJSON), &args); err != nil {+ return youComErrJSON(fmt.Errorf("youcom: parse arguments: %w", err)), nil+ }+ if strings.TrimSpace(args.Query) == "" {+ return youComJSON(youComEnvelope{Results: []map[string]any{}}), nil+ }++ freshness, err := youComEnumArg("freshness", args.Freshness, youComFreshnessValues, t.defaults.Freshness)+ if err != nil {+ return youComErrJSON(err), nil+ }+ maxResults := youComMaxResultsArg(args.MaxResults, t.defaults.TopN)++ hits, err := youcom.Search(ctx, t.fetch, youcom.Request{+ Endpoint: youcom.EndpointFor(t.apiKey),+ APIKey: t.apiKey,+ Query: args.Query,+ Freshness: freshness,+ Count: maxResults,+ })+ if err != nil {+ return youComErrJSON(err), nil+ }+ // `count` applies per response section, so the merged web + news list can+ // exceed it; trim back to what the model (or the node's Top N) asked for.+ if len(hits) > maxResults {+ hits = hits[:maxResults]+ }++ results := make([]map[string]any, 0, len(hits))+ for _, hit := range hits {+ results = append(results, map[string]any{+ "title": hit.Title,+ "url": hit.URL,+ "content": hit.Content,+ })+ }+ return youComJSON(youComEnvelope{Results: results}), nil+}++// fetch performs the HTTP GET for the youcom leaf package, applying the status+// check and response-body cap that the leaf leaves to its caller.+func (t *YouComTool) fetch(ctx context.Context, endpoint string, headers map[string]string) ([]byte, error) {+ resp, err := t.helper.Do(ctx, http.MethodGet, endpoint, "", "", headers)+ if err != nil {+ return nil, err+ }+ defer resp.Body.Close()++ if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {+ return nil, fmt.Errorf("upstream returned %d", resp.StatusCode)+ }++ body, err := io.ReadAll(io.LimitReader(resp.Body, youComMaxResponseBytes+1))+ if err != nil {+ return nil, fmt.Errorf("read response: %w", err)+ }+ if len(body) > youComMaxResponseBytes {+ return nil, fmt.Errorf("response body exceeds %d bytes", youComMaxResponseBytes)+ }+ return body, nil+}++// youComEnumArg normalizes a model-supplied enum value. Blank means the+// default; anything else must be a known value, because it is forwarded to+// You.com as-is.+func youComEnumArg(name, value string, allowed []string, fallback string) (string, error) {+ v := strings.ToLower(strings.TrimSpace(value))+ if v == "" {+ return fallback, nil+ }+ for _, a := range allowed {+ if v == a {+ return v, nil+ }+ }+ return "", fmt.Errorf("youcom: %s %q is not supported, it should be one of %v", name, value, allowed)+}++// youComMaxResultsArg returns the model's result count if it gave a usable one,+// else the node's Top N, clamped to [1, youComMaxResults].+func youComMaxResultsArg(value any, topN int) int {+ n := topN+ switch v := value.(type) {+ case float64:+ n = int(v)+ case string:+ if parsed, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {+ n = parsed+ }+ }+ return min(max(n, 1), youComMaxResults)+}++// ComponentSpec returns the YouComSearch Canvas surface.+func (t *YouComTool) ComponentSpec() ComponentSpec {+ return ComponentSpec{+ Inputs: map[string]string{+ "query": "The search keywords to execute with You.com.",+ "freshness": `Recency limit: "day", "week", "month", "year" or "any".`,+ },+ Outputs: map[string]string{+ "formalized_content": "Rendered You.com references for downstream prompts.",+ "json": "Raw You.com result list.",+ },+ InputForm: map[string]any{+ "query": map[string]any{"name": "Query", "type": "line"},+ "freshness": map[string]any{+ "name": "Freshness",+ "type": "options",+ "value": "any",+ "options": youComFreshnessValues,+ },+ },+ }+}++func (t *YouComTool) BuildReferences(_ context.Context, envelope map[string]any) ([]map[string]any, []map[string]any) {+ return buildYouComReferences(envelope)+}++func (t *YouComTool) BuildComponentOutputs(envelope map[string]any) map[string]any {+ results := envelopeSlice(envelope, "results")+ chunks, _ := buildYouComReferences(envelope)+ return map[string]any{+ "formalized_content": renderYouComReferences(chunks, youComPromptMaxTokens),+ "json": results,+ }+}++// buildYouComReferences turns each usable result into the chunk/docAgg shape+// the citation path expects. A result with no text is dropped rather than+// stored as a blank chunk.+func buildYouComReferences(envelope map[string]any) ([]map[string]any, []map[string]any) {+ results := envelopeSlice(envelope, "results")+ chunks := make([]map[string]any, 0, len(results))+ docAggs := make([]map[string]any, 0, len(results))+ for _, item := range results {+ result, ok := item.(map[string]any)+ if !ok {+ continue+ }+ content := truncateYouComRunes(strings.Join(strings.Fields(youComText(result["content"])), " "), 10000)+ if content == "" {+ continue+ }+ title := youComText(result["title"])+ resultURL := youComText(result["url"])+ documentID, displayID := referenceIDs(resultURL, content)+ chunks = append(chunks, map[string]any{+ "id": displayID,+ "chunk_id": documentID,+ "content": content,+ "doc_id": documentID,+ "document_id": documentID,+ "docnm_kwd": title,+ "document_name": title,+ "similarity": 1,+ "score": 1,+ "url": resultURL,+ })+ docAggs = append(docAggs, map[string]any{+ "doc_name": title,+ "doc_id": documentID,+ "count": 1,+ "url": resultURL,+ })+ }+ return chunks, docAggs+}++func renderYouComReferences(chunks []map[string]any, maxTokens int) string {+ usedTokens := 0+ blocks := make([]string, 0, len(chunks))+ for _, chunk := range chunks {+ content := youComText(chunk["content"])+ usedTokens += tokenizer.NumTokensFromString(content)+ blocks = append(blocks, strings.Join([]string{+ "\nID: " + youComText(chunk["id"]),+ "├── Title: " + strings.Join(strings.Fields(youComText(chunk["document_name"])), " "),+ "├── URL: " + youComText(chunk["url"]),+ "└── Content:\n" + content,+ }, "\n"))+ if maxTokens > 0 && float64(maxTokens)*0.97 < float64(usedTokens) {+ break+ }+ }+ return strings.Join(blocks, "\n")+}++func youComText(value any) string {+ if value == nil {+ return ""+ }+ if text, ok := value.(string); ok {+ return strings.TrimSpace(text)+ }+ return strings.TrimSpace(fmt.Sprint(value))+}++func truncateYouComRunes(value string, limit int) string {+ if utf8.RuneCountInString(value) <= limit {+ return value+ }+ return string([]rune(value)[:limit])+}++func youComJSON(env youComEnvelope) string {+ b, err := json.Marshal(env)+ if err != nil {+ return fmt.Sprintf(`{"_ERROR":"youcom: marshal result: %s"}`, err)+ }+ return string(b)+}++func youComErrJSON(err error) string {+ return youComJSON(youComEnvelope{Error: err.Error()})+}diff --git a/internal/agent/tool/youcom_test.go b/internal/agent/tool/youcom_test.gonew file mode 100644index 0000000000..088076db75--- /dev/null+++ b/internal/agent/tool/youcom_test.go@@ -0,0 +1,258 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package tool++import (+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"++ "ragflow/internal/agent/runtime"+)++// youComTestServer records the last request and answers with the given status+// and body.+func youComTestServer(t *testing.T, status int, respBody string) (*HTTPHelper, *http.Request, *atomic.Int32) {+ t.Helper()+ var gotReq http.Request+ calls := &atomic.Int32{}+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ calls.Add(1)+ gotReq = *r.Clone(r.Context())+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(status)+ _, _ = w.Write([]byte(respBody))+ }))+ t.Cleanup(srv.Close)+ helper := NewHTTPHelper().WithClient(&http.Client{Transport: rewriteHostTransport(srv.URL)})+ return helper, &gotReq, calls+}++func decodeYouComEnvelope(t *testing.T, out string) youComEnvelope {+ t.Helper()+ var env youComEnvelope+ if err := json.Unmarshal([]byte(out), &env); err != nil {+ t.Fatalf("decode envelope %q: %v", out, err)+ }+ return env+}++func TestYouCom_BuildRequest(t *testing.T) {+ t.Parallel()+ helper, req, _ := youComTestServer(t, http.StatusOK, `{"results":{"web":[]}}`)+ tool := newYouComTool(helper, " ydc-test ", youComDefaults{Freshness: "week", TopN: 5})++ out, err := tool.InvokableRun(t.Context(), `{"query":"ragflow go","freshness":"month"}`)+ if err != nil {+ t.Fatalf("InvokableRun: %v", err)+ }+ if env := decodeYouComEnvelope(t, out); env.Error != "" {+ t.Fatalf("unexpected _ERROR %q", env.Error)+ }+ if req.Method != http.MethodGet || req.URL.Path != "/v1/search" {+ t.Errorf("request = %s %s, want GET /v1/search", req.Method, req.URL.Path)+ }+ if got := req.Header.Get("X-API-Key"); got != "ydc-test" {+ t.Errorf("X-API-Key = %q, want trimmed key", got)+ }+ if got := req.Header.Get("User-Agent"); got != "RAGFlow youdotcom-integration/infiniflow-ragflow" {+ t.Errorf("User-Agent = %q", got)+ }+ params := req.URL.Query()+ if params.Get("query") != "ragflow go" {+ t.Errorf("query = %q", params.Get("query"))+ }+ if params.Get("count") != "5" {+ t.Errorf("count = %q, want node Top N", params.Get("count"))+ }+ if params.Get("freshness") != "month" {+ t.Errorf("freshness = %q, want model override", params.Get("freshness"))+ }+}++func TestYouCom_KeylessOmitsAuth(t *testing.T) {+ t.Parallel()+ helper, req, _ := youComTestServer(t, http.StatusOK, `{"results":{"web":[]}}`)+ tool := newYouComTool(helper, "", youComDefaults{})++ if _, err := tool.InvokableRun(t.Context(), `{"query":"q"}`); err != nil {+ t.Fatalf("InvokableRun: %v", err)+ }+ if req.URL.Path != "/v1/agents/search" {+ t.Errorf("path = %q, want keyless endpoint", req.URL.Path)+ }+ // The keyless endpoint rejects an auth header, so none may be sent.+ if got, ok := req.Header["X-Api-Key"]; ok {+ t.Errorf("X-API-Key = %v, want absent", got)+ }+ // The node default "any" must not be forwarded.+ if _, ok := req.URL.Query()["freshness"]; ok {+ t.Errorf("freshness sent for \"any\": %v", req.URL.Query().Get("freshness"))+ }+}++func TestYouCom_MaxResultsArg(t *testing.T) {+ t.Parallel()+ cases := []struct {+ value any+ want int+ }{+ {nil, 7}, // not given: node Top N+ {"", 7}, // blank: node Top N+ {"lots", 7}, // unparseable: node Top N+ {float64(3), 3}, // model's value wins+ {"4", 4}, // numeric string+ {float64(0), 1}, // clamped low+ {float64(99), 20}, // clamped to the max+ }+ for _, tc := range cases {+ if got := youComMaxResultsArg(tc.value, 7); got != tc.want {+ t.Errorf("youComMaxResultsArg(%v, 7) = %d, want %d", tc.value, got, tc.want)+ }+ }+}++func TestYouCom_EmptyQuerySkipsRequest(t *testing.T) {+ t.Parallel()+ helper, _, calls := youComTestServer(t, http.StatusOK, `{"results":{"web":[{"title":"x"}]}}`)+ tool := newYouComTool(helper, "key", youComDefaults{})++ out, err := tool.InvokableRun(t.Context(), `{"query":" "}`)+ if err != nil {+ t.Fatalf("InvokableRun: %v", err)+ }+ env := decodeYouComEnvelope(t, out)+ if env.Error != "" || len(env.Results) != 0 {+ t.Fatalf("envelope = %#v, want empty results", env)+ }+ if calls.Load() != 0 {+ t.Errorf("upstream calls = %d, want 0", calls.Load())+ }+}++func TestYouCom_UpstreamErrorBecomesEnvelope(t *testing.T) {+ t.Parallel()+ helper, _, _ := youComTestServer(t, http.StatusPaymentRequired, ``)+ tool := newYouComTool(helper, "key", youComDefaults{})++ out, err := tool.InvokableRun(t.Context(), `{"query":"q"}`)+ if err != nil {+ t.Fatalf("InvokableRun: %v", err)+ }+ env := decodeYouComEnvelope(t, out)+ if !strings.HasPrefix(env.Error, "youcom: ") {+ t.Fatalf("_ERROR = %q, want a youcom-prefixed error", env.Error)+ }+ // The leaf adds the prefix; the tool's transport errors must not add one again.+ if strings.Contains(env.Error, "youcom: youcom:") {+ t.Fatalf("_ERROR = %q, want a single youcom prefix", env.Error)+ }+}++func TestYouCom_TrimsMergedResultsToMaxResults(t *testing.T) {+ t.Parallel()+ web := make([]map[string]string, 0, 6)+ news := make([]map[string]string, 0, 6)+ for i := 0; i < 6; i++ {+ web = append(web, map[string]string{"url": "https://example.com/w", "title": "w", "description": "d"})+ news = append(news, map[string]string{"url": "https://example.com/n", "title": "n", "description": "d"})+ }+ payload, err := json.Marshal(map[string]any{+ "results": map[string]any{"web": web, "news": news},+ })+ if err != nil {+ t.Fatalf("marshal payload: %v", err)+ }+ helper, _, _ := youComTestServer(t, http.StatusOK, string(payload))+ tool := newYouComTool(helper, "key", youComDefaults{})++ out, err := tool.InvokableRun(t.Context(), `{"query":"q","max_results":3}`)+ if err != nil {+ t.Fatalf("InvokableRun: %v", err)+ }+ env := decodeYouComEnvelope(t, out)+ if len(env.Results) != 3 {+ t.Fatalf("results = %d, want 3 (\"count\" is per section, so the merge must be trimmed)", len(env.Results))+ }+}++func TestYouCom_ComponentOutputsAndReferences(t *testing.T) {+ t.Parallel()+ tool := NewYouComTool()+ envelope := map[string]any{+ "results": []any{+ map[string]any{"title": "RAGFlow", "url": "https://ragflow.io", "content": "RAG engine"},+ map[string]any{"title": "Blank", "url": "https://example.com/blank", "content": " "},+ },+ }++ outputs := tool.BuildComponentOutputs(envelope)+ if _, ok := outputs["json"]; !ok {+ t.Fatalf("outputs missing json: %#v", outputs)+ }+ formalized, _ := outputs["formalized_content"].(string)+ if !strings.Contains(formalized, "https://ragflow.io") {+ t.Fatalf("formalized_content = %q, want the usable hit", formalized)+ }++ chunks, docAggs := tool.BuildReferences(t.Context(), envelope)+ if len(chunks) != 1 || len(docAggs) != 1 {+ t.Fatalf("references = %d chunks / %d docAggs, want 1/1 (blank content dropped)", len(chunks), len(docAggs))+ }+ if chunks[0]["url"] != "https://ragflow.io" {+ t.Errorf("chunk url = %v", chunks[0]["url"])+ }+ form := tool.ComponentSpec().InputForm+ if _, ok := form["query"]; !ok {+ t.Errorf("InputForm missing query: %#v", form)+ }+}++// TestYouCom_ReferencesDistinguishSameContentFromDifferentURLs guards the+// citation path: identical snippets served from different URLs must keep+// distinct ids instead of collapsing into one reference.+func TestYouCom_ReferencesDistinguishSameContentFromDifferentURLs(t *testing.T) {+ t.Parallel()+ tool := NewYouComTool()+ content := "identical snippet text"+ envelope := map[string]any{+ "results": []any{+ map[string]any{"title": "First", "url": "https://first.example/page", "content": content},+ map[string]any{"title": "Second", "url": "https://second.example/page", "content": content},+ },+ }++ chunks, docAggs := tool.BuildReferences(t.Context(), envelope)+ if len(chunks) != 2 || len(docAggs) != 2 {+ t.Fatalf("references = %d chunks / %d docAggs, want 2/2", len(chunks), len(docAggs))+ }+ if chunks[0]["id"] == chunks[1]["id"] {+ t.Fatalf("both chunks share display id %v", chunks[0]["id"])+ }+ if chunks[0]["document_id"] == chunks[1]["document_id"] {+ t.Fatalf("both chunks share document id %v", chunks[0]["document_id"])+ }++ state := runtime.NewCanvasState("run-youcom", "task-youcom")+ state.SetRetrievalReferences(chunks, docAggs)+ if got := state.GetRetrievalChunks(); len(got) != 2 {+ t.Fatalf("retrieval chunks = %d, want 2 (distinct URLs must survive dedupe)", len(got))+ }+}diff --git a/internal/service/web_search_provider.go b/internal/service/web_search_provider.goindex d50bf15ff3..d2ebb87995 100644--- a/internal/service/web_search_provider.go+++ b/internal/service/web_search_provider.go@@ -29,6 +29,7 @@ import ("time""ragflow/internal/agentic_rag"+ "ragflow/internal/service/youcom")// Providers, endpoints and clients are listed alphabetically by provider id@@ -57,15 +58,6 @@ const (queritWebSearchEndpoint = "https://api.querit.ai/v1/search"search1apiWebSearchEndpoint = "https://api.search1api.com/search"serplyWebSearchEndpoint = "https://api.serply.io/v1/search/"- // You.com serves the same response shape from two endpoints. The keyless- // one is rate-limited but needs no credentials; the keyed one lifts those- // limits. The keyless endpoint rejects an X-API-Key header, so the endpoint- // and the headers are always chosen together.- youComWebSearchEndpoint = "https://api.you.com/v1/search"- youComKeylessWebSearchEndpoint = "https://api.you.com/v1/agents/search"- // Identifies RAGFlow to You.com. On the keyless endpoint there is no key to- // attribute traffic to, so this is the only signal available.- youComWebSearchUserAgent = "RAGFlow youdotcom-integration/infiniflow-ragflow"// webSearchResultCount is how many hits every provider is asked for. Six// keeps the web block in a prompt the size of one corpus chunk set — the// point is to give the model something to cite, not to mirror a full SERP.@@ -120,7 +112,7 @@ func resolveWebSearchProvider(promptConfig map[string]interface{}) *webSearchPro// keyOptional marks providers usable with no key. Only You.com: it serves a// dedicated keyless endpoint (a different path, same response shape, rate// limited per source IP, and it rejects an X-API-Key header — see- // youComEndpointFor). A key moves to the keyed endpoint and lifts the limit.+ // youcom.EndpointFor). A key moves to the keyed endpoint and lifts the limit.//// Free tier != keyless: Exa's free 1,000 requests/month still requires a key// on every call, so it does not get this carve-out.@@ -267,7 +259,7 @@ func retrieveWebSearchWithTavily(return retrieveYouComWebSearch(ctx,youComWebSearchHTTPClient,- youComEndpointFor(provider.APIKey),+ youcom.EndpointFor(provider.APIKey),provider.APIKey,question,)@@ -974,43 +966,13 @@ func decodeSerplyWebSearchResults(responseBody []byte) ([]serplyWebSearchResult,return results, nil}-type youComWebSearchResult struct {- URL string `json:"url"`- Title string `json:"title"`- Description string `json:"description"`- Snippets []string `json:"snippets"`-}--type youComWebSearchResponse struct {- Results struct {- Web []youComWebSearchResult `json:"web"`- News []youComWebSearchResult `json:"news"`- } `json:"results"`-}--// youComEndpointFor picks the keyless endpoint when no key is configured. The-// keyless endpoint rejects an X-API-Key header, so callers must never send a-// key to it.-func youComEndpointFor(apiKey string) string {- if strings.TrimSpace(apiKey) == "" {- return youComKeylessWebSearchEndpoint- }- return youComWebSearchEndpoint-}--// youComContent prefers the extracted page passages. News hits carry only a-// description.-func youComContent(result youComWebSearchResult) string {- passages := make([]string, 0, len(result.Snippets))- for _, snippet := range result.Snippets {- if strings.TrimSpace(snippet) != "" {- passages = append(passages, snippet)- }- }- if len(passages) > 0 {- return strings.Join(passages, "\n")+// youComFetcher adapts the service's shared web-search request helper to the+// youcom leaf package's transport seam, so the You.com request/response logic+// lives in one place and this package only owns the HTTP client.+func youComFetcher(client *http.Client) youcom.Fetcher {+ return func(ctx context.Context, endpoint string, headers map[string]string) ([]byte, error) {+ return webSearchRequest(ctx, client, http.MethodGet, endpoint, headers, nil)}- return strings.TrimSpace(result.Description)}func retrieveYouComWebSearch(@@ -1020,45 +982,26 @@ func retrieveYouComWebSearch(apiKey string,query string,) (map[string]interface{}, error) {- queryParams := url.Values{}- queryParams.Set("query", query)- queryParams.Set("count", strconv.Itoa(webSearchResultCount))-- headers := map[string]string{- "Accept": "application/json",- "User-Agent": youComWebSearchUserAgent,- }- // The keyless endpoint rejects an X-API-Key header, so the key rides only- // when one is configured.- if trimmedKey := strings.TrimSpace(apiKey); trimmedKey != "" {- headers["X-API-Key"] = trimmedKey- }- responseBody, err := webSearchRequest(ctx, client, http.MethodGet,- endpoint+"?"+queryParams.Encode(), headers, nil)+ hits, err := youcom.Search(ctx, youComFetcher(client), youcom.Request{+ Endpoint: endpoint,+ APIKey: apiKey,+ Query: query,+ Freshness: "",+ Count: webSearchResultCount,+ })if err != nil {- return nil, fmt.Errorf("youcom: %w", err)- }-- var decoded youComWebSearchResponse- if err := json.Unmarshal(responseBody, &decoded); err != nil {- return nil, fmt.Errorf("youcom: decode response: %w", err)+ return nil, err}- // `count` applies per response section, so web and news together can exceed- // it. Web results lead; the merged list is trimmed back afterwards.- merged := make([]youComWebSearchResult, 0, len(decoded.Results.Web)+len(decoded.Results.News))- merged = append(merged, decoded.Results.Web...)- merged = append(merged, decoded.Results.News...)-- hits := make([]webSearchHit, 0, len(merged))- for _, result := range merged {- hits = append(hits, webSearchHit{- Title: result.Title,- URL: result.URL,- Content: youComContent(result),+ webHits := make([]webSearchHit, 0, len(hits))+ for _, hit := range hits {+ webHits = append(webHits, webSearchHit{+ Title: hit.Title,+ URL: hit.URL,+ Content: hit.Content,})}- return webSearchPayload("youcom", hits), nil+ return webSearchPayload("youcom", webHits), nil}// webSearchFunc adapts the pipeline's configured provider to the agent'sdiff --git a/internal/service/web_search_provider_test.go b/internal/service/web_search_provider_test.goindex 0bd2870ab6..25545b88c9 100644--- a/internal/service/web_search_provider_test.go+++ b/internal/service/web_search_provider_test.go@@ -23,6 +23,8 @@ import ("net/http/httptest""net/url""testing"++ "ragflow/internal/service/youcom")func TestWebSearchFuncGatedByProvider(t *testing.T) {@@ -455,58 +457,6 @@ func TestResolveWebSearchProviderStillRequiresKeysForKeyedProviders(t *testing.T}}-func TestYouComEndpointForPicksKeylessWithoutAKey(t *testing.T) {- cases := []struct {- name string- apiKey string- want string- }{- {name: "no key", apiKey: "", want: youComKeylessWebSearchEndpoint},- {name: "whitespace key", apiKey: " ", want: youComKeylessWebSearchEndpoint},- {name: "key set", apiKey: "ydc-test", want: youComWebSearchEndpoint},- }-- for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {- if got := youComEndpointFor(tc.apiKey); got != tc.want {- t.Fatalf("endpoint = %q, want %q", got, tc.want)- }- })- }-}--func TestYouComContentDiscardsBlankSnippetsAndDescriptions(t *testing.T) {- cases := []struct {- name string- result youComWebSearchResult- want string- }{- {- name: "joins non-blank passages",- result: youComWebSearchResult{Snippets: []string{"a", " ", "b"}, Description: "ignored"},- want: "a\nb",- },- {- name: "falls back to the description",- result: youComWebSearchResult{Snippets: []string{" "}, Description: " desc "},- want: "desc",- },- {- name: "whitespace-only description yields nothing",- result: youComWebSearchResult{Description: " "},- want: "",- },- }-- for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {- if got := youComContent(tc.result); got != tc.want {- t.Fatalf("content = %q, want %q", got, tc.want)- }- })- }-}-func TestRetrieveYouComWebSearchSendsNoAuthHeaderWhenKeyless(t *testing.T) {var gotAuth stringvar gotUserAgent string@@ -535,8 +485,8 @@ func TestRetrieveYouComWebSearchSendsNoAuthHeaderWhenKeyless(t *testing.T) {if gotAuth != "" {t.Fatalf("X-API-Key = %q, want empty", gotAuth)}- if gotUserAgent != youComWebSearchUserAgent {- t.Fatalf("User-Agent = %q, want %q", gotUserAgent, youComWebSearchUserAgent)+ if gotUserAgent != youcom.UserAgent {+ t.Fatalf("User-Agent = %q, want %q", gotUserAgent, youcom.UserAgent)}if gotQuery != "What is RAGFlow?" {t.Fatalf("query = %q, want %q", gotQuery, "What is RAGFlow?")diff --git a/internal/service/youcom/youcom.go b/internal/service/youcom/youcom.gonew file mode 100644index 0000000000..b8747e58be--- /dev/null+++ b/internal/service/youcom/youcom.go@@ -0,0 +1,167 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++// Package youcom implements the You.com web-search request/response contract.+// It is a dependency-light leaf package so that both the chat pipeline+// (internal/service) and the agent tool layer (internal/agent/tool) can depend+// on it without an import cycle — mirroring the internal/service/wikisearch and+// internal/service/nav pattern.+package youcom++import (+ "context"+ "encoding/json"+ "fmt"+ "net/url"+ "strconv"+ "strings"+)++const (+ // WebSearchEndpoint is the keyed endpoint; a key lifts the keyless+ // endpoint's per-IP rate limit.+ WebSearchEndpoint = "https://api.you.com/v1/search"+ // KeylessWebSearchEndpoint is rate-limited but needs no credentials, and it+ // rejects an X-API-Key header, so the endpoint and the headers are always+ // chosen together (see EndpointFor).+ KeylessWebSearchEndpoint = "https://api.you.com/v1/agents/search"+ // UserAgent identifies RAGFlow to You.com. On the keyless endpoint there is+ // no key to attribute traffic to, so this is the only signal available.+ UserAgent = "RAGFlow youdotcom-integration/infiniflow-ragflow"+)++// Hit is one normalized search result: the three fields a model can quote and+// cite.+type Hit struct {+ Title string+ URL string+ Content string+}++// Fetcher performs the HTTP GET for one request and returns the raw response+// body. Callers own the transport concerns (status check, response-body cap),+// so the chat pipeline's webSearchRequest and the agent tool layer's HTTPHelper+// can both drive this package.+type Fetcher func(ctx context.Context, endpoint string, headers map[string]string) ([]byte, error)++// Request carries one search query and its node-level configuration.+type Request struct {+ // Endpoint is the already-resolved search URL (see EndpointFor).+ Endpoint string+ // APIKey is only sent when non-blank, and it selects the keyed endpoint+ // via EndpointFor.+ APIKey string+ Query string+ // Freshness restricts results by recency; "" and "any" send nothing.+ Freshness string+ // Count is the per-section result count requested from You.com.+ Count int+}++// EndpointFor picks the keyless endpoint when no key is configured. The keyless+// endpoint rejects an X-API-Key header, so callers must never send a key to it.+func EndpointFor(apiKey string) string {+ if strings.TrimSpace(apiKey) == "" {+ return KeylessWebSearchEndpoint+ }+ return WebSearchEndpoint+}++// Search issues one You.com search and returns the merged web + news hits.+func Search(ctx context.Context, fetch Fetcher, req Request) ([]Hit, error) {+ queryParams := url.Values{}+ queryParams.Set("query", req.Query)+ queryParams.Set("count", strconv.Itoa(req.Count))+ if freshness := normalizeFreshness(req.Freshness); freshness != "" {+ queryParams.Set("freshness", freshness)+ }++ headers := map[string]string{+ "Accept": "application/json",+ "User-Agent": UserAgent,+ }+ // The keyless endpoint rejects an X-API-Key header, so the key rides only+ // when one is configured.+ if trimmedKey := strings.TrimSpace(req.APIKey); trimmedKey != "" {+ headers["X-API-Key"] = trimmedKey+ }++ responseBody, err := fetch(ctx, req.Endpoint+"?"+queryParams.Encode(), headers)+ if err != nil {+ return nil, fmt.Errorf("youcom: %w", err)+ }++ var decoded webSearchResponse+ if err := json.Unmarshal(responseBody, &decoded); err != nil {+ return nil, fmt.Errorf("youcom: decode response: %w", err)+ }++ // `count` applies per response section, so web and news together can exceed+ // it. Web results lead; the caller trims the merged list if it needs to.+ merged := make([]webSearchResult, 0, len(decoded.Results.Web)+len(decoded.Results.News))+ merged = append(merged, decoded.Results.Web...)+ merged = append(merged, decoded.Results.News...)++ hits := make([]Hit, 0, len(merged))+ for _, result := range merged {+ hits = append(hits, Hit{+ Title: result.Title,+ URL: result.URL,+ Content: content(result),+ })+ }+ return hits, nil+}++// normalizeFreshness lower-cases the requested window and drops the values that+// mean "no limit". Unknown values are forwarded as-is; the tool layer validates+// them against the supported set before calling here.+func normalizeFreshness(value string) string {+ freshness := strings.ToLower(strings.TrimSpace(value))+ if freshness == "" || freshness == "any" {+ return ""+ }+ return freshness+}++// content prefers the extracted page passages. News hits carry only a+// description.+func content(result webSearchResult) string {+ passages := make([]string, 0, len(result.Snippets))+ for _, snippet := range result.Snippets {+ if strings.TrimSpace(snippet) != "" {+ passages = append(passages, snippet)+ }+ }+ if len(passages) > 0 {+ return strings.Join(passages, "\n")+ }+ return strings.TrimSpace(result.Description)+}++type webSearchResult struct {+ URL string `json:"url"`+ Title string `json:"title"`+ Description string `json:"description"`+ Snippets []string `json:"snippets"`+}++type webSearchResponse struct {+ Results struct {+ Web []webSearchResult `json:"web"`+ News []webSearchResult `json:"news"`+ } `json:"results"`+}diff --git a/internal/service/youcom/youcom_test.go b/internal/service/youcom/youcom_test.gonew file mode 100644index 0000000000..67f3af2301--- /dev/null+++ b/internal/service/youcom/youcom_test.go@@ -0,0 +1,175 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package youcom++import (+ "context"+ "errors"+ "net/url"+ "strings"+ "testing"+)++func TestEndpointForPicksKeylessWithoutAKey(t *testing.T) {+ cases := []struct {+ name string+ apiKey string+ want string+ }{+ {name: "no key", apiKey: "", want: KeylessWebSearchEndpoint},+ {name: "whitespace key", apiKey: " ", want: KeylessWebSearchEndpoint},+ {name: "key set", apiKey: "ydc-test", want: WebSearchEndpoint},+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ if got := EndpointFor(tc.apiKey); got != tc.want {+ t.Fatalf("endpoint = %q, want %q", got, tc.want)+ }+ })+ }+}++// captureFetch returns a Fetcher that records the endpoint and headers and+// answers with body.+func captureFetch(body string, endpoint *string, headers *map[string]string) Fetcher {+ return func(_ context.Context, gotEndpoint string, gotHeaders map[string]string) ([]byte, error) {+ *endpoint = gotEndpoint+ *headers = gotHeaders+ return []byte(body), nil+ }+}++func TestSearchMergesSectionsAndSendsConfig(t *testing.T) {+ var endpoint string+ var headers map[string]string+ body := `{"results":{"web":[` ++ `{"url":"https://example.com/a","title":"A","snippets":["first"," ","second"],"description":"ignored"},` ++ `{"url":"https://example.com/b","title":"B","description":"desc fallback"}` ++ `],"news":[{"url":"https://news.example.com/c","title":"C","description":"news desc"}]}}`++ hits, err := Search(t.Context(), captureFetch(body, &endpoint, &headers), Request{+ Endpoint: EndpointFor(""),+ APIKey: "",+ Query: "ragflow",+ Freshness: "week",+ Count: 6,+ })+ if err != nil {+ t.Fatalf("Search: %v", err)+ }++ parsed, err := url.Parse(endpoint)+ if err != nil {+ t.Fatalf("parse endpoint: %v", err)+ }+ if parsed.Query().Get("query") != "ragflow" {+ t.Errorf("query = %q, want ragflow", parsed.Query().Get("query"))+ }+ if parsed.Query().Get("count") != "6" {+ t.Errorf("count = %q, want 6", parsed.Query().Get("count"))+ }+ if parsed.Query().Get("freshness") != "week" {+ t.Errorf("freshness = %q, want week", parsed.Query().Get("freshness"))+ }+ if got := headers["User-Agent"]; got != UserAgent {+ t.Errorf("User-Agent = %q, want %q", got, UserAgent)+ }+ // The keyless endpoint rejects an X-API-Key header, so none may be sent.+ if got, ok := headers["X-API-Key"]; ok {+ t.Errorf("X-API-Key = %q, want absent", got)+ }++ if len(hits) != 3 {+ t.Fatalf("hits = %d, want 3", len(hits))+ }+ if hits[0].Content != "first\nsecond" {+ t.Errorf("web content = %q, want joined non-blank snippets", hits[0].Content)+ }+ if hits[1].Content != "desc fallback" {+ t.Errorf("description fallback = %q", hits[1].Content)+ }+ if hits[2].Content != "news desc" || hits[2].Title != "C" {+ t.Errorf("news hit = %#v", hits[2])+ }+}++func TestSearchSendsKeyAndOmitsFreshnessForAny(t *testing.T) {+ var endpoint string+ var headers map[string]string+ hits, err := Search(t.Context(), captureFetch(`{"results":{"web":[]}}`, &endpoint, &headers), Request{+ Endpoint: EndpointFor("ydc-test"),+ APIKey: " ydc-test ",+ Query: "q",+ Freshness: "any",+ Count: 3,+ })+ if err != nil {+ t.Fatalf("Search: %v", err)+ }+ if len(hits) != 0 {+ t.Fatalf("hits = %d, want 0", len(hits))+ }+ if got := headers["X-API-Key"]; got != "ydc-test" {+ t.Errorf("X-API-Key = %q, want trimmed key", got)+ }+ parsed, _ := url.Parse(endpoint)+ if _, ok := parsed.Query()["freshness"]; ok {+ t.Errorf("freshness sent for \"any\": %v", parsed.Query().Get("freshness"))+ }+}++func TestSearchDiscardsBlankContent(t *testing.T) {+ var endpoint string+ var headers map[string]string+ body := `{"results":{"web":[{"url":"https://example.com/blank","title":"Blank","description":" "}]}}`+ hits, err := Search(t.Context(), captureFetch(body, &endpoint, &headers), Request{Count: 6})+ if err != nil {+ t.Fatalf("Search: %v", err)+ }+ // The leaf keeps the hit (the caller decides whether blank content is+ // usable); a blank description must not be turned into non-empty text.+ if len(hits) != 1 || strings.TrimSpace(hits[0].Content) != "" {+ t.Fatalf("hits = %#v, want one blank-content hit", hits)+ }+}++func TestSearchWrapsFetchError(t *testing.T) {+ fetch := func(context.Context, string, map[string]string) ([]byte, error) {+ return nil, errors.New("status 402")+ }+ _, err := Search(t.Context(), fetch, Request{Query: "q", Count: 6})+ if err == nil {+ t.Fatal("expected an error")+ }+ if !strings.Contains(err.Error(), "youcom: status 402") {+ t.Fatalf("err = %q, want youcom-prefixed fetch error", err.Error())+ }+}++func TestSearchRejectsMalformedBody(t *testing.T) {+ fetch := func(context.Context, string, map[string]string) ([]byte, error) {+ return []byte("not json"), nil+ }+ _, err := Search(t.Context(), fetch, Request{Query: "q", Count: 6})+ if err == nil {+ t.Fatal("expected a decode error")+ }+ if !strings.Contains(err.Error(), "youcom: decode response") {+ t.Fatalf("err = %q, want decode error", err.Error())+ }+}diff --git a/web/src/components/__tests__/floating-chat-widget-markdown.test.tsx b/web/src/components/__tests__/floating-chat-widget-markdown.test.tsxnew file mode 100644index 0000000000..0efd5aaf98--- /dev/null+++ b/web/src/components/__tests__/floating-chat-widget-markdown.test.tsx@@ -0,0 +1,170 @@+import { render, screen } from '@testing-library/react';++import FloatingChatWidgetMarkdown from '../floating-chat-widget-markdown';++jest.mock('react-i18next', () => ({+ useTranslation: () => ({ t: (key: string) => key }),+}));++jest.mock('@/hooks/use-document-request', () => ({+ useFetchDocumentThumbnailsByIds: () => ({+ setDocumentIds: jest.fn(),+ data: {},+ }),+ useGetDocumentUrl: () => () => undefined,+}));++jest.mock('@/components/theme-provider', () => ({+ useIsDarkTheme: () => false,+}));++// Render the hover card eagerly so the resolved chunk content is assertable.+jest.mock('@/components/ui/hover-card', () => ({+ HoverCard: ({ children }: any) => (+ <div data-testid="citation">{children}</div>+ ),+ HoverCardTrigger: ({ children }: any) => (+ <span data-testid="chip">{children}</span>+ ),+ HoverCardContent: ({ children }: any) => (+ <div data-testid="popover">{children}</div>+ ),+}));++jest.mock('@radix-ui/react-hover-card', () => ({+ Portal: ({ children }: any) => <>{children}</>,+}));++jest.mock('@/components/ui/tooltip', () => ({+ Tooltip: ({ children }: any) => <>{children}</>,+ TooltipTrigger: ({ children }: any) => <>{children}</>,+ TooltipContent: ({ children }: any) => <>{children}</>,+}));++jest.mock('@/components/svg-icon', () => ({+ __esModule: true,+ default: () => <span />,+}));++jest.mock('@/components/image', () => ({+ __esModule: true,+ default: () => null,+ AuthenticatedImg: () => null,+}));++jest.mock('react-syntax-highlighter', () => ({+ __esModule: true,+ default: ({ children }: any) => <pre>{children}</pre>,+}));++jest.mock('react-syntax-highlighter/dist/esm/styles/prism', () => ({+ oneDark: {},+ oneLight: {},+}));++// react-markdown is ESM-only; feed the whole content through the same+// custom-typography component that replaces citation markers in production.+jest.mock('react-markdown', () => ({+ __esModule: true,+ default: ({ children, components }: any) => {+ const ReactLib = jest.requireActual('react');+ const CustomTypography = components['custom-typography'];+ return ReactLib.createElement(CustomTypography, null, String(children));+ },+}));++jest.mock('rehype-katex', () => jest.fn());+jest.mock('rehype-raw', () => jest.fn());+jest.mock('unist-util-visit-parents', () => ({ visitParents: jest.fn() }));+jest.mock('@/constants/markdown-remark-plugins', () => ({+ MarkdownRemarkPlugins: [],+}));+jest.mock('@/constants/markdown-rehype-plugins', () => ({+ RehypeSanitizeAssistantMarkdown: jest.fn(),+}));++const buildChunk = (id: string, content: string, documentId: string) => ({+ id,+ chunk_id: id,+ content,+ content_with_weight: content,+ document_id: documentId,+ doc_id: documentId,+ image_id: '',+ img_id: '',+});++const Opaque = buildChunk(+ '0000123456789012',+ 'OPAQUE_CHUNK_CONTENT',+ 'doc-opaque',+);+const Hex = buildChunk('a2d067519c9e0db1', 'HEX_CHUNK_CONTENT', 'doc-hex');++const buildReference = () =>+ ({+ chunks: [Opaque, Hex],+ doc_aggs: {+ 'doc-opaque': {+ doc_id: 'doc-opaque',+ doc_name: 'Opaque source.txt',+ count: 1,+ },+ 'doc-hex': {+ doc_id: 'doc-hex',+ doc_name: 'Hex source.txt',+ count: 1,+ },+ },+ total: 2,+ }) as any;++const popoverTexts = () =>+ screen.getAllByTestId('popover').map((popover) => popover.textContent);++describe('FloatingChatWidgetMarkdown citations', () => {+ it('keeps an all-digit opaque id as an id, not a positional index', () => {+ render(+ <FloatingChatWidgetMarkdown+ content="see [ID:0000123456789012]"+ loading={false}+ reference={buildReference()}+ />,+ );++ expect(screen.queryByText('Reference unavailable')).not.toBeInTheDocument();+ const popovers = popoverTexts();+ expect(popovers).toHaveLength(1);+ expect(popovers[0]).toContain('OPAQUE_CHUNK_CONTENT');+ expect(popovers[0]).toContain('Opaque source.txt');+ });++ it('resolves a normal hex id by id', () => {+ render(+ <FloatingChatWidgetMarkdown+ content="see [ID:a2d067519c9e0db1]"+ loading={false}+ reference={buildReference()}+ />,+ );++ const popovers = popoverTexts();+ expect(popovers).toHaveLength(1);+ expect(popovers[0]).toContain('HEX_CHUNK_CONTENT');+ expect(popovers[0]).toContain('Hex source.txt');+ });++ it('keeps short numeric markers positional', () => {+ render(+ <FloatingChatWidgetMarkdown+ content="see [ID:0]"+ loading={false}+ reference={buildReference()}+ />,+ );++ const popovers = popoverTexts();+ expect(popovers).toHaveLength(1);+ expect(popovers[0]).toContain('OPAQUE_CHUNK_CONTENT');+ });+});diff --git a/web/src/components/floating-chat-widget-markdown.tsx b/web/src/components/floating-chat-widget-markdown.tsxindex c89c954ee1..1093a95065 100644--- a/web/src/components/floating-chat-widget-markdown.tsx+++ b/web/src/components/floating-chat-widget-markdown.tsx@@ -24,15 +24,18 @@ import {} from '@/hooks/use-document-request';import { IReference, IReferenceChunk } from '@/interfaces/database/chat';import {- currentReg,parseCitationIndex,preprocessLaTeX,replaceRetrievingToSection,replaceTextByOldReg,replaceThinkToSection,+ resolveCitationChunk,showImage,} from '@/utils/chat';-import { citationMarkerReg } from '@/utils/citation-utils';+import {+ citationMarkerReg,+ citationWholeMarkerReg,+} from '@/utils/citation-utils';import { getExtension } from '@/utils/document-util';import { supportsSourceLocate } from '@/utils/source-locate';import { getDirAttribute } from '@/utils/text-direction';@@ -62,8 +65,7 @@ import { Button } from './ui/button';import { HoverCard, HoverCardContent, HoverCardTrigger } from './ui/hover-card';import { Tooltip, TooltipContent, TooltipTrigger } from './ui/tooltip';-const getChunkIndex = (match: string) =>- parseCitationIndex(match.replace(/\[|\]/g, ''));+const getChunkKey = (match: string) => parseCitationIndex(match);const FloatingChatWidgetMarkdown = ({reference,@@ -144,13 +146,8 @@ const FloatingChatWidgetMarkdown = ({};const getReferenceInfo = useCallback(- (chunkIndex: number) => {- let chunkItem = reference?.chunks?.[chunkIndex];- if (!chunkItem) {- chunkItem = reference?.chunks?.find(- (x: any) => x?.chunk_id === chunkIndex,- ) as IReferenceChunk;- }+ (chunkKey: number | string) => {+ const chunkItem = resolveCitationChunk(reference?.chunks, chunkKey);if (!chunkItem) return null;const docAggsArray = Array.isArray(reference?.doc_aggs)? reference.doc_aggs@@ -179,8 +176,8 @@ const FloatingChatWidgetMarkdown = ({);const getPopoverContent = useCallback(- (chunkIndex: number) => {- const info = getReferenceInfo(chunkIndex);+ (chunkKey: number | string) => {+ const info = getReferenceInfo(chunkKey);if (!info) {return (@@ -280,9 +277,9 @@ const FloatingChatWidgetMarkdown = ({const renderReference = useCallback((text: string) => {- return reactStringReplace(text, currentReg, (match, i) => {- const chunkIndex = getChunkIndex(match) as number;- const info = getReferenceInfo(chunkIndex);+ return reactStringReplace(text, citationWholeMarkerReg, (match, i) => {+ const chunkKey = getChunkKey(match);+ const info = getReferenceInfo(chunkKey);if (!info) {return (@@ -327,7 +324,7 @@ const FloatingChatWidgetMarkdown = ({collisionPadding={8}className="max-h-[var(--radix-hover-card-content-available-height)]">- {getPopoverContent(chunkIndex)}+ {getPopoverContent(chunkKey)}</HoverCardContent></HoverCardPrimitive.Portal></HoverCard>diff --git a/web/src/components/next-markdown-content/index.tsx b/web/src/components/next-markdown-content/index.tsxindex 827db3ab92..8def4bfa9f 100644--- a/web/src/components/next-markdown-content/index.tsx+++ b/web/src/components/next-markdown-content/index.tsx@@ -52,7 +52,10 @@ import {resolveCitationChunk,unescapeAngleBrackets,} from '@/utils/chat';-import type { CitationKey } from '@/utils/citation-utils';+import {+ citationWholeMarkerReg,+ type CitationKey,+} from '@/utils/citation-utils';import { getDirAttribute } from '@/utils/text-direction';import { useFetchDocumentThumbnailsByIds } from '@/hooks/use-document-request';@@ -74,11 +77,6 @@ import styles from './index.module.less';const getChunkKey = (match: string) => parseCitationIndex(match);-// react-string-replace hands the callback the LAST capture group, not the-// full match — citationMarkerReg groups only the id, so replacing with it-// would strip the brackets. This group wraps the whole marker, letting an-// unresolved marker be returned unchanged.-const WholeMarkerReg = /(\[(?:ID:)?[0-9a-fA-F\u0660-\u0669\u06F0-\u06F9]+\])/g;const isArtifactUrl = (url?: string) =>Boolean(url && url.includes('/api/v1/documents/artifact/'));@@ -404,32 +402,36 @@ function MarkdownContent({// payload ships in retrieval order, which is unrelated to the order// the answer cites passages in.const keyToDisplay = buildCitationDisplayMap(text);- const replacedText = reactStringReplace(text, WholeMarkerReg, (match) => {- const chunkKey = getChunkKey(match);- const displayIndex = keyToDisplay.get(chunkKey) ?? 0;- const chunkItem = resolveCitationChunk(reference?.chunks, chunkKey);- // The reference lands at message_end, after the answer stream, so an- // unresolved marker keeps its chip while loading. Afterwards it- // stays as literal text rather than pointing at the wrong chunk.- if (!chunkItem && !loading) {- return match;- }+ const replacedText = reactStringReplace(+ text,+ citationWholeMarkerReg,+ (match) => {+ const chunkKey = getChunkKey(match);+ const displayIndex = keyToDisplay.get(chunkKey) ?? 0;+ const chunkItem = resolveCitationChunk(reference?.chunks, chunkKey);+ // The reference lands at message_end, after the answer stream, so an+ // unresolved marker keeps its chip while loading. Afterwards it+ // stays as literal text rather than pointing at the wrong chunk.+ if (!chunkItem && !loading) {+ return match;+ }- return (- <HoverCard key={`${String(chunkKey)}-${displayIndex}`}>- <HoverCardTrigger>- <bdi className="text-text-secondary bg-bg-card rounded-2xl px-1 mx-1 text-nowrap inline-block">- [{displayIndex + 1}]- </bdi>- </HoverCardTrigger>- {chunkItem && (- <HoverCardContent className="max-w-3xl">- {renderPopoverContent(chunkKey)}- </HoverCardContent>- )}- </HoverCard>- );- });+ return (+ <HoverCard key={`${String(chunkKey)}-${displayIndex}`}>+ <HoverCardTrigger>+ <bdi className="text-text-secondary bg-bg-card rounded-2xl px-1 mx-1 text-nowrap inline-block">+ [{displayIndex + 1}]+ </bdi>+ </HoverCardTrigger>+ {chunkItem && (+ <HoverCardContent className="max-w-3xl">+ {renderPopoverContent(chunkKey)}+ </HoverCardContent>+ )}+ </HoverCard>+ );+ },+ );return replacedText;},diff --git a/web/src/utils/__tests__/citation-utils.test.ts b/web/src/utils/__tests__/citation-utils.test.tsindex 99bb9737ec..226aad67e3 100644--- a/web/src/utils/__tests__/citation-utils.test.ts+++ b/web/src/utils/__tests__/citation-utils.test.ts@@ -1,6 +1,8 @@import type { IReferenceChunk } from '@/interfaces/database/chat';+import reactStringReplace from 'react-string-replace';import {buildCitationDisplayMap,+ citationWholeMarkerReg,parseCitationIndex,resolveCitationChunk,} from '../citation-utils';@@ -27,11 +29,30 @@ describe('parseCitationIndex', () => {expect(parseCitationIndex('[3]')).toBe(3);});+ it('keeps short explicit numeric ids as positional indexes', () => {+ expect(parseCitationIndex('[ID:0]')).toBe(0);+ expect(parseCitationIndex('[ID:12]')).toBe(12);+ expect(parseCitationIndex('[ID:123456789012345]')).toBe(123456789012345);+ });+it('parses hex chunk-id markers as strings', () => {expect(parseCitationIndex('[ID:a2d067519c9e0db1]')).toBe('a2d067519c9e0db1',);});++ it('preserves all-digit hex ids instead of converting them', () => {+ expect(parseCitationIndex('[ID:0000123456789012]')).toBe(+ '0000123456789012',+ );+ expect(parseCitationIndex('[ID:1234567890123456]')).toBe(+ '1234567890123456',+ );+ });++ it('keeps short leading-zero ids as strings', () => {+ expect(parseCitationIndex('[ID:0123]')).toBe('0123');+ });});describe('resolveCitationChunk', () => {@@ -79,6 +100,23 @@ describe('resolveCitationChunk', () => {});});+describe('citationWholeMarkerReg', () => {+ it('hands react-string-replace the whole marker, not the bare id', () => {+ const seen: string[] = [];+ reactStringReplace(+ 'see [ID:0000123456789012] here',+ citationWholeMarkerReg,+ (match) => {+ seen.push(match);+ return match;+ },+ );++ expect(seen).toEqual(['[ID:0000123456789012]']);+ expect(parseCitationIndex(seen[0])).toBe('0000123456789012');+ });+});+describe('buildCitationDisplayMap', () => {it('numbers unique markers by first occurrence, whatever their key', () => {const map = buildCitationDisplayMap(@@ -88,4 +126,12 @@ describe('buildCitationDisplayMap', () => {expect(map.get('a2d067519c9e0db1')).toBe(1);expect(map.size).toBe(2);});++ it('uses numeric keys for short explicit numeric ids', () => {+ const map = buildCitationDisplayMap('[ID:0] [ID:1] [ID:0]');+ expect([...map.entries()]).toEqual([+ [0, 0],+ [1, 1],+ ]);+ });});diff --git a/web/src/utils/citation-utils.ts b/web/src/utils/citation-utils.tsindex 10e0af20c4..d91391b09a 100644--- a/web/src/utils/citation-utils.ts+++ b/web/src/utils/citation-utils.ts@@ -32,6 +32,17 @@ export const normalizeCitationDigits = (text: string) => {export type CitationKey = number | string;+// Knowledge-base citation markers are positional indices produced by+// strconv.Itoa, so they stay numeric and small. Sixteen-digit hex agent IDs+// exceed Number.MAX_SAFE_INTEGER and must keep their string form; leading-zero+// values are opaque ids for the same reason.+const maxNumericCitationIndexDigits = 15;++const isNumericCitationIndex = (value: string) =>+ /^\d+$/.test(value) &&+ value.length <= maxNumericCitationIndexDigits &&+ (value.length === 1 || value[0] !== '0');+export const parseCitationIndex = (value: string): CitationKey => {const normalized = normalizeCitationDigits(value);const markerMatch = normalized.match(@@ -39,6 +50,9 @@ export const parseCitationIndex = (value: string): CitationKey => {);if (markerMatch) {const captured = markerMatch[1];+ if (/^\s*\[?ID:/i.test(normalized)) {+ return isNumericCitationIndex(captured) ? Number(captured) : captured;+ }return /^\d+$/.test(captured) ? Number(captured) : captured;}if (/^\d+$/.test(normalized)) return Number(normalized);@@ -48,15 +62,23 @@ export const parseCitationIndex = (value: string): CitationKey => {export const citationMarkerReg =/\[(?:ID:)?([0-9a-fA-F\u0660-\u0669\u06F0-\u06F9]+)\]/g;+// react-string-replace hands the callback the last capture group, not the whole+// match. citationMarkerReg groups only the id, so feeding it to+// reactStringReplace would strip the brackets and turn an opaque all-digit id+// into a positional index. This expression captures the whole marker instead;+// the renderer can then pass it to parseCitationIndex unchanged.+export const citationWholeMarkerReg =+ /(\[(?:ID:)?[0-9a-fA-F\u0660-\u0669\u06F0-\u06F9]+\])/g;+// Resolve the chunk a citation marker points at. A numeric key is the// chunk's position in the payload — the plain-chat contract, where the// backend numbers each marker by its index into reference.chunks. A string// key is the chunk id an agent answer embeds in its [ID:<chunkId>] markers;// match it against the chunk's own id, never positionally: the payload// ships in retrieval order, which is unrelated to the order the answer-// cites passages in, so a positional read permutes the sources. A chunk id-// is 16 hex chars — an all-digit id would parse as a position (~0.05% per-// id), accepted and recorded here so it is not rediscovered as a mystery.+// cites passages in, so a positional read permutes the sources. A 16-char+// hex chunk id stays a string even when all its characters are digits;+// parseCitationIndex only treats short decimal markers as positions.export const resolveCitationChunk = (chunks:| IReferenceChunk[]