Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Bugfix: add youcomsearch agent tool (#21080)

ciel8728fde8340f
diff --git a/internal/agent/component/agent.go b/internal/agent/component/agent.go
index fc25c1917d..7f8ea2196f 100644
--- a/internal/agent/component/agent.go
+++ b/internal/agent/component/agent.go
@@ -179,6 +179,7 @@ func runEinoReActAgent(ctx context.Context, p AgentParam) (*schema.Message, erro
if err != nil {
return nil, fmt.Errorf("build tools: %w", err)
}
+ tools = wrapReferenceRecordingTools(tools)
input := buildAgentInputMessages(ctx, p)
// Eino's MaxStep counts graph nodes, not model calls. One ReAct round
// consists of a model decision, a tool node, and the following model
@@ -500,7 +501,9 @@ func addToolCallMemory(ctx context.Context, db *gorm.DB, p AgentParam, msg *sche
// (e.g. "simple-0", "ck-1"). Used only to decide whether a grounding
// response still has answer text after the markers are removed. The ID:
// prefix is required so bare brackets like [2026] are not treated as markers.
-var citationGroundingMarkerRe = regexp.MustCompile(`\[ID:\s*[^\]]+\]`)
+// The id cannot contain "[", matching citationGroundingMarkerCaptureRe: an
+// unclosed [ID: must not be stripped together with the text up to a later ].
+var citationGroundingMarkerRe = regexp.MustCompile(`\[ID:\s*[^\[\]]+\]`)
// citationGroundingHasSubstantiveText reports whether s contains any
// non-whitespace text once citation markers are stripped. Marker-only
@@ -532,7 +535,7 @@ func applyCitationGrounding(ctx context.Context, db *gorm.DB, p AgentParam, cont
if strings.TrimSpace(content) == "" {
return content, nil
}
- systemPrompt, _ := prompts.CitationPlusPrompt(chunks)
+ systemPrompt, ids := prompts.CitationPlusPrompt(chunks)
inv := getDefaultChatInvoker()
req := ChatInvokeRequest{
Driver: p.Driver,
@@ -548,9 +551,26 @@ func applyCitationGrounding(ctx context.Context, db *gorm.DB, p AgentParam, cont
var resp *ChatInvokeResponse
var err error
emittedVisible := false
+ var leadingWhitespace strings.Builder
+ emitting := false
+ emitContent := func(text string) error {
+ if text == "" {
+ return nil
+ }
+ if !emitting {
+ leadingWhitespace.WriteString(text)
+ if strings.TrimSpace(leadingWhitespace.String()) == "" {
+ return nil
+ }
+ text = leadingWhitespace.String()
+ emitting = true
+ }
+ emittedVisible = true
+ runtime.EmitAgentMessage(ctx, text, "")
+ return nil
+ }
if streamer, ok := inv.(chat.StreamingInvoker); ok {
- var leadingWhitespace strings.Builder
- emitting := false
+ streamFilter := newCitationMarkerStreamFilter(ids)
resp, err = streamer.Stream(ctx, db, req, func(delta string, isThink bool) error {
if ctx.Err() != nil {
return ctx.Err()
@@ -558,18 +578,13 @@ func applyCitationGrounding(ctx context.Context, db *gorm.DB, p AgentParam, cont
if isThink || delta == "" {
return nil
}
- if !emitting {
- leadingWhitespace.WriteString(delta)
- if strings.TrimSpace(leadingWhitespace.String()) == "" {
- return nil
- }
- delta = leadingWhitespace.String()
- emitting = true
- }
- emittedVisible = true
- runtime.EmitAgentMessage(ctx, delta, "")
- return nil
+ return emitContent(streamFilter.Push(delta))
})
+ if err == nil {
+ if flushErr := emitContent(streamFilter.Flush()); flushErr != nil {
+ err = flushErr
+ }
+ }
} else {
resp, err = inv.Invoke(ctx, db, req)
}
@@ -582,20 +597,21 @@ func applyCitationGrounding(ctx context.Context, db *gorm.DB, p AgentParam, cont
if resp == nil {
return content, fmt.Errorf("citation grounding returned no response")
}
- if strings.TrimSpace(resp.Content) == "" {
+ grounded := filterCitationMarkers(resp.Content, ids)
+ if strings.TrimSpace(grounded) == "" {
return content, nil
}
// Marker-only grounding must not silently replace the first-pass answer
// (#20613). Non-stream (or stream that never emitted) can fall back.
// Once invalid deltas have already been streamed, surface an error so the
// caller does not append the original answer after the bad markers.
- if !citationGroundingHasSubstantiveText(resp.Content) {
+ if !citationGroundingHasSubstantiveText(grounded) {
if emittedVisible {
return content, fmt.Errorf("citation grounding returned markers without answer text")
}
return content, nil
}
- return resp.Content, nil
+ return grounded, nil
}
// chunksFromState extracts the recorded retrieval chunks from
@@ -1103,7 +1119,7 @@ func (c *AgentComponent) invokeNow(ctx context.Context, db *gorm.DB, inputs map[
var groundingStatus string
if p.Cite {
- chunks := chunksFromState(ctx)
+ chunks := limitCitationSources(chunksFromState(ctx), citationGroundingSourceTokenBudget)
if len(chunks) == 0 {
groundingStatus = "no_chunks"
} else {
diff --git a/internal/agent/component/citation_sources.go b/internal/agent/component/citation_sources.go
new file mode 100644
index 0000000000..3027b85636
--- /dev/null
+++ b/internal/agent/component/citation_sources.go
@@ -0,0 +1,159 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "regexp"
+ "strings"
+
+ "ragflow/internal/agent/component/prompts"
+ "ragflow/internal/tokenizer"
+)
+
+// citationGroundingSourceTokenBudget caps the sources injected into the
+// post-stream citation prompt. A multi-search Agent run can accumulate dozens
+// of chunks; without a cap the second LLM call eventually overflows the
+// model's context window.
+const citationGroundingSourceTokenBudget = 30000
+
+// citationMarkerMaxLength bounds a citation marker's full text length. It is
+// far above [ID:<chunk id>] for any real chunk id, so a candidate longer than
+// this is not a marker. The stream filter stops holding at this bound and the
+// final filter leaves such text literal, keeping the streamed and saved
+// content identical.
+const citationMarkerMaxLength = 128
+
+// citationGroundingMarkerCaptureRe matches a canonical [ID:<id>] marker and
+// captures the id. Whitespace after "ID:" is tolerated and normalized away.
+// The id cannot contain "[": an unclosed [ID: must not swallow the text up to
+// a later ] (for example a Markdown link that follows it).
+var citationGroundingMarkerCaptureRe = regexp.MustCompile(`\[ID:\s*([^\[\]]*)\]`)
+
+// limitCitationSources keeps sources in order until the next one would exceed
+// the token budget. Empty sources are skipped. The first valid source is
+// always kept so a single oversized chunk still gets a grounding attempt.
+func limitCitationSources(sources []prompts.CitationSource, budget int) []prompts.CitationSource {
+ if budget <= 0 {
+ return sources
+ }
+ kept := make([]prompts.CitationSource, 0, len(sources))
+ used := 0
+ for _, source := range sources {
+ id := strings.TrimSpace(source.ID)
+ if id == "" || strings.TrimSpace(source.Content) == "" {
+ continue
+ }
+ tokens := tokenizer.NumTokensFromString(id) + tokenizer.NumTokensFromString(source.Content)
+ if len(kept) > 0 && used+tokens > budget {
+ break
+ }
+ kept = append(kept, prompts.CitationSource{ID: id, Content: source.Content})
+ used += tokens
+ }
+ return kept
+}
+
+// filterCitationMarkers drops [ID:<id>] markers whose id is not in
+// allowedIDs and normalizes the whitespace of the ones that remain.
+func filterCitationMarkers(content string, allowedIDs []string) string {
+ allowed := make(map[string]struct{}, len(allowedIDs))
+ for _, id := range allowedIDs {
+ if id = strings.TrimSpace(id); id != "" {
+ allowed[id] = struct{}{}
+ }
+ }
+ return filterCitationMarkersWithSet(content, allowed)
+}
+
+func filterCitationMarkersWithSet(content string, allowed map[string]struct{}) string {
+ if content == "" {
+ return ""
+ }
+ return citationGroundingMarkerCaptureRe.ReplaceAllStringFunc(content, func(marker string) string {
+ // A marker longer than the cap is not a real citation id; keep the text
+ // visible instead of silently dropping it.
+ if len(marker) > citationMarkerMaxLength {
+ return marker
+ }
+ match := citationGroundingMarkerCaptureRe.FindStringSubmatch(marker)
+ if len(match) < 2 {
+ return ""
+ }
+ id := strings.TrimSpace(match[1])
+ if _, ok := allowed[id]; !ok {
+ return ""
+ }
+ return "[ID:" + id + "]"
+ })
+}
+
+// citationMarkerStreamFilter applies the same marker filtering to a streamed
+// response. A marker can be split across deltas, so trailing text that could
+// still become a marker is buffered until the next delta or Flush.
+type citationMarkerStreamFilter struct {
+ allowed map[string]struct{}
+ pending string
+}
+
+func newCitationMarkerStreamFilter(allowedIDs []string) *citationMarkerStreamFilter {
+ allowed := make(map[string]struct{}, len(allowedIDs))
+ for _, id := range allowedIDs {
+ if id = strings.TrimSpace(id); id != "" {
+ allowed[id] = struct{}{}
+ }
+ }
+ return &citationMarkerStreamFilter{allowed: allowed}
+}
+
+func (f *citationMarkerStreamFilter) Push(delta string) string {
+ if delta == "" {
+ return ""
+ }
+ text := f.pending + delta
+ f.pending = ""
+ if hold := citationMarkerHoldStart(text); hold >= 0 && len(text)-hold <= citationMarkerMaxLength {
+ f.pending = text[hold:]
+ text = text[:hold]
+ }
+ return filterCitationMarkersWithSet(text, f.allowed)
+}
+
+func (f *citationMarkerStreamFilter) Flush() string {
+ text := f.pending
+ f.pending = ""
+ if text == "" {
+ return ""
+ }
+ // Apply the same filter as the final (non-stream) path: a well-formed
+ // marker is filtered or kept, while an unclosed [ID:... stays literal
+ // instead of swallowing the text that follows it.
+ return filterCitationMarkersWithSet(text, f.allowed)
+}
+
+// citationMarkerHoldStart returns the index of the trailing text that must be
+// buffered because it may still turn into a citation marker.
+func citationMarkerHoldStart(text string) int {
+ if idx := strings.LastIndex(text, "[ID:"); idx >= 0 && !strings.Contains(text[idx:], "]") {
+ return idx
+ }
+ for _, prefix := range []string{"[ID:", "[ID", "[I", "["} {
+ if strings.HasSuffix(text, prefix) {
+ return len(text) - len(prefix)
+ }
+ }
+ return -1
+}
diff --git a/internal/agent/component/citation_sources_test.go b/internal/agent/component/citation_sources_test.go
new file mode 100644
index 0000000000..f8a713762d
--- /dev/null
+++ b/internal/agent/component/citation_sources_test.go
@@ -0,0 +1,235 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "strings"
+ "testing"
+
+ "ragflow/internal/agent/component/prompts"
+ "ragflow/internal/agent/runtime"
+ "ragflow/internal/tokenizer"
+)
+
+func TestLimitCitationSources_StopsAtBudget(t *testing.T) {
+ sources := []prompts.CitationSource{
+ {ID: "a", Content: strings.Repeat("alpha ", 20)},
+ {ID: "b", Content: strings.Repeat("beta ", 20)},
+ {ID: "c", Content: strings.Repeat("gamma ", 20)},
+ }
+ first := tokenizer.NumTokensFromString(sources[0].ID) + tokenizer.NumTokensFromString(sources[0].Content)
+ second := tokenizer.NumTokensFromString(sources[1].ID) + tokenizer.NumTokensFromString(sources[1].Content)
+
+ kept := limitCitationSources(sources, first+second-1)
+ if len(kept) != 1 || kept[0].ID != "a" {
+ t.Fatalf("kept = %#v, want only the first source", kept)
+ }
+ if all := limitCitationSources(sources, first+second+1); len(all) != 2 {
+ t.Fatalf("kept = %d, want 2", len(all))
+ }
+}
+
+func TestLimitCitationSources_KeepsFirstOversizedSource(t *testing.T) {
+ sources := []prompts.CitationSource{{ID: "a", Content: strings.Repeat("x", 50000)}}
+ if kept := limitCitationSources(sources, 1); len(kept) != 1 {
+ t.Fatalf("kept = %d, want the first oversized source", len(kept))
+ }
+}
+
+func TestLimitCitationSources_SkipsEmpty(t *testing.T) {
+ sources := []prompts.CitationSource{
+ {ID: "", Content: "no id"},
+ {ID: "empty-content", Content: " "},
+ {ID: "ok", Content: "body"},
+ }
+ kept := limitCitationSources(sources, 30000)
+ if len(kept) != 1 || kept[0].ID != "ok" {
+ t.Fatalf("kept = %#v, want only the valid source", kept)
+ }
+}
+
+func TestFilterCitationMarkers(t *testing.T) {
+ cases := []struct {
+ name string
+ content string
+ allowed []string
+ want string
+ }{
+ {"keeps allowed", "answer [ID:good] more", []string{"good"}, "answer [ID:good] more"},
+ {"drops unknown", "a [ID:good] b [ID:bad] c", []string{"good"}, "a [ID:good] b c"},
+ {"normalizes whitespace", "[ID: good ]", []string{"good"}, "[ID:good]"},
+ {"drops empty id", "[ID:]", []string{"good"}, ""},
+ {"drops all when no sources", "[ID:good]", nil, ""},
+ {"leaves plain brackets", "see [note]", []string{"good"}, "see [note]"},
+ {"leaves an unclosed marker before a markdown link", "answer [ID:oops more [link] end", []string{"good"}, "answer [ID:oops more [link] end"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := filterCitationMarkers(tc.content, tc.allowed); got != tc.want {
+ t.Fatalf("filterCitationMarkers(%q) = %q, want %q", tc.content, got, tc.want)
+ }
+ })
+ }
+}
+
+func TestCitationMarkerStreamFilter_SplitMarkers(t *testing.T) {
+ filter := newCitationMarkerStreamFilter([]string{"abc123"})
+ var out strings.Builder
+ for _, delta := range []string{"answer [ID:a", "bc", "123] done"} {
+ out.WriteString(filter.Push(delta))
+ }
+ out.WriteString(filter.Flush())
+ if got, want := out.String(), "answer [ID:abc123] done"; got != want {
+ t.Fatalf("streamed = %q, want %q", got, want)
+ }
+}
+
+func TestCitationMarkerStreamFilter_DropsSplitInvalidMarker(t *testing.T) {
+ filter := newCitationMarkerStreamFilter([]string{"good"})
+ var out strings.Builder
+ for _, delta := range []string{"keep [ID:ba", "d] text"} {
+ out.WriteString(filter.Push(delta))
+ }
+ out.WriteString(filter.Flush())
+ if got, want := out.String(), "keep text"; got != want {
+ t.Fatalf("streamed = %q, want %q", got, want)
+ }
+}
+
+func TestCitationMarkerStreamFilter_FlushRules(t *testing.T) {
+ filter := newCitationMarkerStreamFilter([]string{"good"})
+ if got := filter.Push("see ["); got != "see " {
+ t.Fatalf("Push held the partial bracket, got %q", got)
+ }
+ if got := filter.Flush(); got != "[" {
+ t.Fatalf("Flush = %q, want the literal bracket", got)
+ }
+
+ unclosed := newCitationMarkerStreamFilter([]string{"good"})
+ if got := unclosed.Push("answer [ID:good"); got != "answer " {
+ t.Fatalf("Push held the unclosed marker, got %q", got)
+ }
+ if got := unclosed.Flush(); got != "[ID:good" {
+ t.Fatalf("Flush = %q, want the unclosed marker left literal", got)
+ }
+}
+
+func TestCitationMarkerStreamFilter_KeepsTextAfterUnclosedMarker(t *testing.T) {
+ filter := newCitationMarkerStreamFilter([]string{"good"})
+ var out strings.Builder
+ for _, delta := range []string{"answer ", "[ID:good", " more text"} {
+ out.WriteString(filter.Push(delta))
+ }
+ out.WriteString(filter.Flush())
+
+ content := "answer [ID:good more text"
+ want := filterCitationMarkers(content, []string{"good"})
+ if got := out.String(); got != want {
+ t.Fatalf("streamed = %q, want %q", got, want)
+ }
+ if !strings.Contains(out.String(), "more text") {
+ t.Fatalf("streamed = %q, want the text after the unclosed marker", out.String())
+ }
+}
+
+func TestCitationMarkerStreamFilter_BoundsLongCandidate(t *testing.T) {
+ filter := newCitationMarkerStreamFilter([]string{"good"})
+ long := "[ID:" + strings.Repeat("x", citationMarkerMaxLength)
+
+ got := filter.Push(long) + filter.Flush()
+ if got == "" {
+ t.Fatal("Push buffered an over-long non-marker instead of emitting it")
+ }
+ if want := filterCitationMarkers(long, []string{"good"}); got != want {
+ t.Fatalf("streamed = %q, want %q", got, want)
+ }
+}
+
+func TestCitationMarkerStreamFilter_MatchesFinalFilter(t *testing.T) {
+ allowed := []string{"good", "a2d067519c9e0db1"}
+ cases := []struct {
+ name string
+ content string
+ splits []int
+ }{
+ {name: "valid marker", content: "answer [ID:good] done", splits: []int{7, 12}},
+ {name: "invalid marker", content: "keep [ID:bad] text", splits: []int{9}},
+ {name: "unclosed marker", content: "answer [ID:good more text", splits: []int{7, 15}},
+ {name: "over-long marker", content: "[ID:" + strings.Repeat("x", citationMarkerMaxLength+10) + "] tail", splits: []int{20, 40}},
+ {name: "plain brackets", content: "see [note] and [", splits: []int{5}},
+ {name: "unclosed marker before a markdown link", content: "answer [ID:oops more [link] end", splits: []int{7}},
+ {name: "no markers", content: "just answer text", splits: []int{4}},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ want := filterCitationMarkers(tc.content, allowed)
+ filter := newCitationMarkerStreamFilter(allowed)
+ var out strings.Builder
+ prev := 0
+ for _, cut := range tc.splits {
+ if cut <= prev || cut >= len(tc.content) {
+ continue
+ }
+ out.WriteString(filter.Push(tc.content[prev:cut]))
+ prev = cut
+ }
+ out.WriteString(filter.Push(tc.content[prev:]))
+ out.WriteString(filter.Flush())
+ if got := out.String(); got != want {
+ t.Fatalf("streamed = %q, want %q", got, want)
+ }
+ })
+ }
+}
+
+func TestFilterCitationMarkers_LeavesOverlongMarkerLiteral(t *testing.T) {
+ id := strings.Repeat("a", citationMarkerMaxLength)
+ overlong := "[ID:" + id + "]"
+ if got := filterCitationMarkers(overlong, []string{id}); got != overlong {
+ t.Fatalf("filterCitationMarkers(overlong) = %q, want it left literal", got)
+ }
+}
+
+func TestApplyCitationGrounding_FiltersUnknownIDs(t *testing.T) {
+ inv := &groundingStreamingInvoker{groundingTestInvoker: groundingTestInvoker{
+ responses: []string{"answer [ID:good] [ID:bad]"},
+ }}
+ prev := getDefaultChatInvoker()
+ SetDefaultChatInvoker(inv)
+ defer SetDefaultChatInvoker(prev)
+
+ state := runtime.NewCanvasState("run-filter", "task-filter")
+ ctx := runtime.WithState(t.Context(), state)
+ var streamed string
+ ctx = runtime.WithAgentMessageEmitter(ctx, func(content, _ string) {
+ streamed += content
+ })
+
+ got, err := applyCitationGrounding(ctx, nil, AgentParam{ModelID: "stub", Cite: true}, "original", []prompts.CitationSource{
+ {ID: "good", Content: "source body"},
+ })
+ if err != nil {
+ t.Fatalf("applyCitationGrounding: %v", err)
+ }
+ if want := "answer [ID:good] "; got != want {
+ t.Fatalf("returned content = %q, want %q", got, want)
+ }
+ if streamed != got {
+ t.Fatalf("streamed = %q, want the filtered content %q", streamed, got)
+ }
+}
diff --git a/internal/agent/component/grounding_test.go b/internal/agent/component/grounding_test.go
index cf66698ddf..1fa6570dbe 100644
--- a/internal/agent/component/grounding_test.go
+++ b/internal/agent/component/grounding_test.go
@@ -266,6 +266,7 @@ func TestCitationGroundingHasSubstantiveText(t *testing.T) {
{"The reimbursement request requires manager approval.", true},
{"The reimbursement request requires manager approval. [ID:0]", true},
{"The reimbursement request requires manager approval. [ID:simple-0]", true},
+ {"[ID:oops more [link] end", true},
{"答案[ID:0]", true},
{"答案[ID:ck-1]", true},
}
diff --git a/internal/agent/component/reference_recording_tool.go b/internal/agent/component/reference_recording_tool.go
new file mode 100644
index 0000000000..732df33ddf
--- /dev/null
+++ b/internal/agent/component/reference_recording_tool.go
@@ -0,0 +1,68 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "context"
+
+ einotool "github.com/cloudwego/eino/components/tool"
+
+ "ragflow/internal/agent/runtime"
+ agenttool "ragflow/internal/agent/tool"
+)
+
+// referenceRecordingTool decorates an Agent tool so each invocation records
+// the retrieval references it produced into the canvas state. Canvas tool
+// nodes already do this in ToolBackedComponent.Invoke; tools called from the
+// Agent's ReAct loop go straight to InvokableRun, so without this wrapper
+// state.Retrieval stays empty and citation grounding has no sources.
+type referenceRecordingTool struct {
+ einotool.InvokableTool
+ builder agenttool.ReferenceBuilder
+}
+
+func (t *referenceRecordingTool) InvokableRun(ctx context.Context, argsJSON string, opts ...einotool.Option) (string, error) {
+ out, err := t.InvokableTool.InvokableRun(ctx, argsJSON, opts...)
+ if err != nil {
+ return out, err
+ }
+ envelope := parseToolEnvelope(out)
+ if _, invalid := envelope["_raw"]; invalid {
+ return out, nil
+ }
+ chunks, docAggs := t.builder.BuildReferences(ctx, envelope)
+ if state, stateErr := runtime.GetStateFromContext(ctx); stateErr == nil && state != nil {
+ state.SetRetrievalReferences(chunks, docAggs)
+ }
+ return out, nil
+}
+
+// wrapReferenceRecordingTools decorates the tools handed to the ReAct loop.
+// buildAgentTools keeps returning the raw tools so metadata paths such as
+// AgentComponent.GetInputForm and Reset still see the original methods.
+func wrapReferenceRecordingTools(tools []einotool.BaseTool) []einotool.BaseTool {
+ wrapped := make([]einotool.BaseTool, len(tools))
+ copy(wrapped, tools)
+ for i, t := range tools {
+ builder, buildsReferences := t.(agenttool.ReferenceBuilder)
+ invokable, isInvokable := t.(einotool.InvokableTool)
+ if buildsReferences && isInvokable {
+ wrapped[i] = &referenceRecordingTool{InvokableTool: invokable, builder: builder}
+ }
+ }
+ return wrapped
+}
diff --git a/internal/agent/component/reference_recording_tool_test.go b/internal/agent/component/reference_recording_tool_test.go
new file mode 100644
index 0000000000..a7a2a923cd
--- /dev/null
+++ b/internal/agent/component/reference_recording_tool_test.go
@@ -0,0 +1,139 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "testing"
+
+ einotool "github.com/cloudwego/eino/components/tool"
+ "github.com/cloudwego/eino/schema"
+
+ "ragflow/internal/agent/runtime"
+)
+
+type referenceRecordingFakeTool struct {
+ out string
+ err error
+}
+
+func (f *referenceRecordingFakeTool) Info(_ context.Context) (*schema.ToolInfo, error) {
+ return &schema.ToolInfo{Name: "fake_search"}, nil
+}
+
+func (f *referenceRecordingFakeTool) InvokableRun(_ context.Context, _ string, _ ...einotool.Option) (string, error) {
+ return f.out, f.err
+}
+
+func (f *referenceRecordingFakeTool) BuildReferences(_ context.Context, envelope map[string]any) ([]map[string]any, []map[string]any) {
+ results, _ := envelope["results"].([]any)
+ chunks := make([]map[string]any, 0, len(results))
+ docAggs := make([]map[string]any, 0, len(results))
+ for i, raw := range results {
+ id := fmt.Sprintf("chunk-%d", i)
+ chunks = append(chunks, map[string]any{"id": id, "chunk_id": id, "document_id": id, "content": raw})
+ docAggs = append(docAggs, map[string]any{"doc_id": id, "doc_name": raw})
+ }
+ return chunks, docAggs
+}
+
+func TestReferenceRecordingTool_RecordsAgentToolReferences(t *testing.T) {
+ state := runtime.NewCanvasState("run-refs", "task-refs")
+ ctx := runtime.WithState(t.Context(), state)
+ inner := &referenceRecordingFakeTool{out: `{"results":["first","second"]}`}
+ recorder := &referenceRecordingTool{InvokableTool: inner, builder: inner}
+
+ out, err := recorder.InvokableRun(ctx, `{"query":"q"}`)
+ if err != nil {
+ t.Fatalf("InvokableRun: %v", err)
+ }
+ if out != inner.out {
+ t.Fatalf("output = %q, want the inner tool output %q", out, inner.out)
+ }
+ chunks := state.GetRetrievalChunks()
+ if len(chunks) != 2 {
+ t.Fatalf("recorded chunks = %d, want 2", len(chunks))
+ }
+ if chunks[0]["id"] != "chunk-0" || chunks[1]["id"] != "chunk-1" {
+ t.Fatalf("recorded chunk ids = %v / %v", chunks[0]["id"], chunks[1]["id"])
+ }
+ if aggs := state.GetRetrievalDocAggs(); len(aggs) != 2 {
+ t.Fatalf("recorded doc aggs = %d, want 2", len(aggs))
+ }
+}
+
+func TestReferenceRecordingTool_SkipsErrorsAndNonJSON(t *testing.T) {
+ state := runtime.NewCanvasState("run-refs", "task-refs")
+ ctx := runtime.WithState(t.Context(), state)
+
+ failing := &referenceRecordingFakeTool{err: errors.New("boom")}
+ if _, err := (&referenceRecordingTool{InvokableTool: failing, builder: failing}).InvokableRun(ctx, "{}"); err == nil {
+ t.Fatal("expected the inner tool error to propagate")
+ }
+ plain := &referenceRecordingFakeTool{out: "not json"}
+ if _, err := (&referenceRecordingTool{InvokableTool: plain, builder: plain}).InvokableRun(ctx, "{}"); err != nil {
+ t.Fatalf("InvokableRun: %v", err)
+ }
+ if chunks := state.GetRetrievalChunks(); len(chunks) != 0 {
+ t.Fatalf("recorded chunks = %d, want 0 for failed/non-JSON results", len(chunks))
+ }
+}
+
+func TestReferenceRecordingTool_KeepsSameTitleDifferentDocumentIDs(t *testing.T) {
+ state := runtime.NewCanvasState("run-refs", "task-refs")
+ ctx := runtime.WithState(t.Context(), state)
+ inner := &referenceRecordingFakeTool{out: `{"results":["same title","same title"]}`}
+ recorder := &referenceRecordingTool{InvokableTool: inner, builder: inner}
+
+ if _, err := recorder.InvokableRun(ctx, `{"query":"q"}`); err != nil {
+ t.Fatalf("InvokableRun: %v", err)
+ }
+ if aggs := state.GetRetrievalDocAggs(); len(aggs) != 2 {
+ t.Fatalf("recorded doc aggs = %d, want 2 for distinct document ids", len(aggs))
+ }
+ reference := state.GetRetrievalReference()
+ chunks, _ := reference["chunks"].([]any)
+ docAggs, _ := reference["doc_aggs"].([]any)
+ if len(chunks) != 2 || len(docAggs) != 2 {
+ t.Fatalf("reference = %d chunks / %d doc aggs, want 2/2", len(chunks), len(docAggs))
+ }
+}
+
+// TestWrapReferenceRecordingTools_WrapsReferenceBuilders asserts the ReAct
+// tool list is decorated while buildAgentTools keeps the raw tools.
+func TestWrapReferenceRecordingTools_WrapsReferenceBuilders(t *testing.T) {
+ raw, err := buildAgentTools(t.Context(), AgentParam{Tools: []string{"youcom"}})
+ if err != nil {
+ t.Fatalf("buildAgentTools: %v", err)
+ }
+ if len(raw) != 1 {
+ t.Fatalf("raw tools = %d, want 1", len(raw))
+ }
+ if _, ok := raw[0].(*referenceRecordingTool); ok {
+ t.Fatal("buildAgentTools must return the raw tool for metadata callers")
+ }
+
+ wrapped := wrapReferenceRecordingTools(raw)
+ if len(wrapped) != 1 {
+ t.Fatalf("wrapped tools = %d, want 1", len(wrapped))
+ }
+ if _, ok := wrapped[0].(*referenceRecordingTool); !ok {
+ t.Fatalf("tool type = %T, want *referenceRecordingTool", wrapped[0])
+ }
+}
diff --git a/internal/agent/component/tool_component.go b/internal/agent/component/tool_component.go
index 5bfe61196b..b5e94a2f0f 100644
--- a/internal/agent/component/tool_component.go
+++ b/internal/agent/component/tool_component.go
@@ -156,6 +156,7 @@ var toolComponentRegistrations = []struct {
{componentName: "WenCai", toolName: "wencai"},
{componentName: "Wikipedia", toolName: "wikipedia"},
{componentName: "YahooFinance", toolName: "yahoo_finance"},
+ {componentName: "YouComSearch", toolName: "youcom"},
}
func init() {
diff --git a/internal/agent/component/tool_component_test.go b/internal/agent/component/tool_component_test.go
index 4e31a15199..33386918e0 100644
--- a/internal/agent/component/tool_component_test.go
+++ b/internal/agent/component/tool_component_test.go
@@ -256,6 +256,13 @@ func TestToolBackedComponentRegisteredFactories(t *testing.T) {
outputKey: "report",
inputKey: "stock_code",
},
+ {
+ name: "YouComSearch",
+ toolName: "YouComSearch",
+ params: map[string]any{"api_key": "stored-key", "freshness": "week", "top_n": float64(3), "outputs": map[string]any{"json": map[string]any{}}},
+ outputKey: "json",
+ inputKey: "query",
+ },
{
name: "TavilyExtract",
toolName: "TavilyExtract",
@@ -311,7 +318,7 @@ func TestToolBackedComponentWenCaiInvoke(t *testing.T) {
}
func TestToolBackedComponentRegisteredBuildWorkflow(t *testing.T) {
- for _, componentName := range []string{"ArXiv", "BGPT", "DuckDuckGo", "Email", "Google", "GoogleScholar", "KeenableSearch", "PubMed", "QueritContents", "QueritSearch", "Search1APICrawl", "Search1APISearch", "SearXNG", "SofyaSearch", "WenCai", "TavilyExtract", "TavilySearch", "Wikipedia", "YahooFinance"} {
+ for _, componentName := range []string{"ArXiv", "BGPT", "DuckDuckGo", "Email", "Google", "GoogleScholar", "KeenableSearch", "PubMed", "QueritContents", "QueritSearch", "Search1APICrawl", "Search1APISearch", "SearXNG", "SofyaSearch", "WenCai", "TavilyExtract", "TavilySearch", "Wikipedia", "YahooFinance", "YouComSearch"} {
t.Run(componentName, func(t *testing.T) {
c := &canvas.Canvas{
Components: map[string]canvas.CanvasComponent{
diff --git a/internal/agent/runtime/state.go b/internal/agent/runtime/state.go
index a95e7de0b3..2881c7ab62 100644
--- a/internal/agent/runtime/state.go
+++ b/internal/agent/runtime/state.go
@@ -813,7 +813,23 @@ func copyRetrievalDocAggs(value any) []any {
for key := range aggs {
keys = append(keys, key)
}
- sort.Strings(keys)
+ // Sort by document name to keep the payload stable across runs. The
+ // document id breaks ties between sources that share a title.
+ sort.SliceStable(keys, func(i, j int) bool {
+ aggI, _ := aggs[keys[i]].(map[string]any)
+ aggJ, _ := aggs[keys[j]].(map[string]any)
+ nameI, _ := aggI["doc_name"].(string)
+ nameJ, _ := aggJ["doc_name"].(string)
+ if nameI != nameJ {
+ return nameI < nameJ
+ }
+ idI, _ := aggI["doc_id"].(string)
+ idJ, _ := aggJ["doc_id"].(string)
+ if idI != idJ {
+ return idI < idJ
+ }
+ return keys[i] < keys[j]
+ })
out := make([]any, 0, len(keys))
for _, key := range keys {
out = append(out, aggs[key])
@@ -846,8 +862,7 @@ func (s *CanvasState) SetRetrievalChunks(chunks []map[string]any) {
}
// SetRetrievalReferences records the chunks and document aggregates emitted by
-// a canvas search component. It is the lock-safe counterpart of Python
-// Graph.add_reference for components that produce externally sourced results.
+// a canvas search component.
func (s *CanvasState) SetRetrievalReferences(chunks, docAggs []map[string]any) {
if s == nil {
return
@@ -885,16 +900,32 @@ func (s *CanvasState) SetRetrievalReferences(chunks, docAggs []map[string]any) {
if docAggValues == nil {
docAggValues = make(map[string]any, len(docAggs))
}
+ seenDocAggKeys := make(map[string]struct{}, len(docAggValues)+len(docAggs))
+ for _, item := range docAggValues {
+ existing, ok := item.(map[string]any)
+ if !ok {
+ continue
+ }
+ if key, ok := retrievalDocAggKey(existing); ok {
+ seenDocAggKeys[key] = struct{}{}
+ }
+ }
for _, docAgg := range docAggs {
- docName, _ := docAgg["doc_name"].(string)
- if docName == "" {
+ key, ok := retrievalDocAggKey(docAgg)
+ if !ok {
continue
}
- // Match Python Graph.add_reference: retain the first aggregate for
- // a document name across the run-level reference set.
- if _, exists := docAggValues[docName]; !exists {
- docAggValues[docName] = docAgg
+ // Retain the first aggregate per document id. Aggregates without an
+ // id fall back to the document name, matching the previous behavior.
+ // Keying by id keeps two sources with the same title but different
+ // URLs from collapsing into one doc_agg. Existing entries are re-keyed
+ // from their fields so checkpoints written before this change also
+ // dedupe correctly.
+ if _, exists := seenDocAggKeys[key]; exists {
+ continue
}
+ seenDocAggKeys[key] = struct{}{}
+ docAggValues[key] = docAgg
}
s.Retrieval["chunks"] = chunkValues
s.Retrieval["doc_aggs"] = docAggValues
@@ -909,7 +940,24 @@ func retrievalReferenceID(chunk map[string]any) (string, bool) {
return id, id != ""
}
-// GetRetrievalDocAggs returns a shallow snapshot keyed by document name.
+// retrievalDocAggKey returns the dedupe key for a document aggregate.
+// Documents with an id are keyed by that id so same-title sources from
+// different URLs stay distinct. Aggregates without an id fall back to the
+// document name. Empty document names are skipped, matching the previous
+// behavior.
+func retrievalDocAggKey(docAgg map[string]any) (string, bool) {
+ docName, _ := docAgg["doc_name"].(string)
+ if docName == "" {
+ return "", false
+ }
+ if docID, _ := docAgg["doc_id"].(string); docID != "" {
+ return "id:" + docID, true
+ }
+ return "name:" + docName, true
+}
+
+// GetRetrievalDocAggs returns a shallow snapshot keyed by document id when
+// present, otherwise by document name.
func (s *CanvasState) GetRetrievalDocAggs() map[string]map[string]any {
if s == nil {
return nil
@@ -921,10 +969,19 @@ func (s *CanvasState) GetRetrievalDocAggs() map[string]map[string]any {
return nil
}
out := make(map[string]map[string]any, len(raw))
- for name, item := range raw {
- if agg, ok := item.(map[string]any); ok {
- out[name] = agg
+ for _, item := range raw {
+ agg, ok := item.(map[string]any)
+ if !ok {
+ continue
+ }
+ key, _ := agg["doc_id"].(string)
+ if key == "" {
+ key, _ = agg["doc_name"].(string)
+ }
+ if key == "" {
+ continue
}
+ out[key] = agg
}
return out
}
diff --git a/internal/agent/runtime/state_test.go b/internal/agent/runtime/state_test.go
index 6227dd34f5..67b3544cbe 100644
--- a/internal/agent/runtime/state_test.go
+++ b/internal/agent/runtime/state_test.go
@@ -422,6 +422,79 @@ func TestCanvasState_SetRetrievalReferencesMergesCalls(t *testing.T) {
}
}
+// TestCanvasState_SetRetrievalReferencesKeepsSameTitleDifferentIDs covers two
+// search hits that share a title but point at different URLs. Keying doc_aggs
+// by document name would drop the second source; the chunk and its doc_agg must
+// both survive so the citation card can resolve the document.
+func TestCanvasState_SetRetrievalReferencesKeepsSameTitleDifferentIDs(t *testing.T) {
+ t.Parallel()
+
+ state := NewCanvasState("run-1", "task-1")
+ state.SetRetrievalReferences(
+ []map[string]any{{"id": "chunk-1", "document_id": "doc-1"}},
+ []map[string]any{{"doc_name": "Same Title", "doc_id": "doc-1", "url": "https://a.example", "count": 1}},
+ )
+ state.SetRetrievalReferences(
+ []map[string]any{{"id": "chunk-2", "document_id": "doc-2"}},
+ []map[string]any{{"doc_name": "Same Title", "doc_id": "doc-2", "url": "https://b.example", "count": 1}},
+ )
+ // A repeated id keeps the first aggregate, preserving the previous
+ // first-writer-wins behavior.
+ state.SetRetrievalReferences(nil, []map[string]any{
+ {"doc_name": "Same Title", "doc_id": "doc-1", "url": "https://duplicate.example", "count": 9},
+ })
+
+ docAggs := state.GetRetrievalDocAggs()
+ if len(docAggs) != 2 {
+ t.Fatalf("doc_aggs length = %d, want 2", len(docAggs))
+ }
+ if got := docAggs["doc-1"]["url"]; got != "https://a.example" {
+ t.Errorf("doc-1 url = %v, want the first aggregate", got)
+ }
+ if got := docAggs["doc-2"]["url"]; got != "https://b.example" {
+ t.Errorf("doc-2 url = %v, want the second aggregate", got)
+ }
+
+ reference := state.GetRetrievalReference()
+ list, _ := reference["doc_aggs"].([]any)
+ if len(list) != 2 {
+ t.Fatalf("reference doc_aggs length = %d, want 2", len(list))
+ }
+ // Same document name sorts by document id, so doc-1 precedes doc-2.
+ for i, want := range []string{"doc-1", "doc-2"} {
+ agg, _ := list[i].(map[string]any)
+ if got := agg["doc_id"]; got != want {
+ t.Errorf("doc_aggs[%d].doc_id = %v, want %s", i, got, want)
+ }
+ }
+}
+
+// TestCanvasState_SetRetrievalReferencesDedupesLegacyCheckpointKeys covers a
+// checkpoint restored from the previous storage scheme, where doc_aggs were
+// keyed by document name. Merging by the aggregate's fields must not append a
+// second copy of the same document.
+func TestCanvasState_SetRetrievalReferencesDedupesLegacyCheckpointKeys(t *testing.T) {
+ t.Parallel()
+
+ state := NewCanvasState("run-1", "task-1")
+ state.Retrieval["doc_aggs"] = map[string]any{
+ "Same Title": map[string]any{
+ "doc_name": "Same Title",
+ "doc_id": "doc-1",
+ "url": "https://a.example",
+ },
+ }
+ state.SetRetrievalReferences(nil, []map[string]any{
+ {"doc_name": "Same Title", "doc_id": "doc-1", "url": "https://duplicate.example"},
+ })
+
+ reference := state.GetRetrievalReference()
+ list, _ := reference["doc_aggs"].([]any)
+ if len(list) != 1 {
+ t.Fatalf("reference doc_aggs length = %d, want 1 after legacy merge", len(list))
+ }
+}
+
func TestCanvasState_GetRetrievalReferenceReturnsFrontendPayload(t *testing.T) {
t.Parallel()
diff --git a/internal/agent/tool/arxiv.go b/internal/agent/tool/arxiv.go
index cbd00ce0bb..cc920bc703 100644
--- a/internal/agent/tool/arxiv.go
+++ b/internal/agent/tool/arxiv.go
@@ -19,16 +19,13 @@ package tool
import (
"bytes"
"context"
- "crypto/sha1"
"encoding/json"
"encoding/xml"
"fmt"
"io"
- "math/big"
"net/http"
"net/url"
"regexp"
- "strconv"
"strings"
"unicode/utf8"
@@ -321,10 +318,9 @@ func buildArxivReferences(envelope map[string]any) ([]map[string]any, []map[stri
if content == "" {
continue
}
- documentID := strconv.FormatInt(arxivHashInt(content, 100000000), 10)
- displayID := strconv.FormatInt(arxivHashInt(documentID, 500), 10)
title := arxivText(paper["title"])
resultURL := arxivText(paper["pdf_url"])
+ documentID, displayID := referenceIDs(resultURL, content)
chunks = append(chunks, map[string]any{
"id": displayID,
"chunk_id": documentID,
@@ -371,12 +367,6 @@ func arxivText(value any) string {
return fmt.Sprint(value)
}
-func arxivHashInt(value string, modulus int64) int64 {
- digest := sha1.Sum([]byte(value))
- number := new(big.Int).SetBytes(digest[:])
- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()
-}
-
func truncateArxivRunes(value string, limit int) string {
if utf8.RuneCountInString(value) <= limit {
return value
diff --git a/internal/agent/tool/bgpt.go b/internal/agent/tool/bgpt.go
index c6737b4263..4f1eba80d8 100644
--- a/internal/agent/tool/bgpt.go
+++ b/internal/agent/tool/bgpt.go
@@ -18,13 +18,10 @@ package tool
import (
"context"
- "crypto/sha1"
"encoding/json"
"fmt"
- "math/big"
"net/http"
"regexp"
- "strconv"
"strings"
"time"
"unicode/utf8"
@@ -231,8 +228,6 @@ func buildBGPTReferences(envelope map[string]any) ([]map[string]any, []map[strin
if content == "" {
continue
}
- documentID := strconv.FormatInt(bgptHashInt(content, 100000000), 10)
- displayID := strconv.FormatInt(bgptHashInt(documentID, 500), 10)
title := bgptFirstField(paper, "title")
if title == "-" {
title = "Untitled"
@@ -241,6 +236,7 @@ func buildBGPTReferences(envelope map[string]any) ([]map[string]any, []map[strin
if resultURL == "-" {
resultURL = ""
}
+ documentID, displayID := referenceIDs(resultURL, content)
chunks = append(chunks, map[string]any{
"id": displayID,
"chunk_id": documentID,
@@ -313,12 +309,6 @@ func renderBGPTReferences(chunks []map[string]any, maxTokens int) string {
return strings.Join(blocks, "\n")
}
-func bgptHashInt(value string, modulus int64) int64 {
- digest := sha1.Sum([]byte(value))
- number := new(big.Int).SetBytes(digest[:])
- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()
-}
-
func truncateBGPTRunes(value string, limit int) string {
if utf8.RuneCountInString(value) <= limit {
return value
diff --git a/internal/agent/tool/duckduckgo.go b/internal/agent/tool/duckduckgo.go
index 191d9a06f4..3923ebdf1f 100644
--- a/internal/agent/tool/duckduckgo.go
+++ b/internal/agent/tool/duckduckgo.go
@@ -18,12 +18,10 @@ package tool
import (
"context"
- "crypto/sha1"
"encoding/json"
"fmt"
"html"
"io"
- "math/big"
"net/http"
"net/url"
"regexp"
@@ -277,10 +275,9 @@ func buildDuckDuckGoReferences(envelope map[string]any) ([]map[string]any, []map
if content == "" {
continue
}
- documentID := strconv.FormatInt(duckduckgoHashInt(content, 100000000), 10)
- displayID := strconv.FormatInt(duckduckgoHashInt(documentID, 500), 10)
title := duckduckgoText(item["title"])
resultURL := duckduckgoText(item["url"])
+ documentID, displayID := referenceIDs(resultURL, content)
chunks = append(chunks, map[string]any{
"id": displayID,
"chunk_id": documentID,
@@ -327,12 +324,6 @@ func duckduckgoText(value any) string {
return fmt.Sprint(value)
}
-func duckduckgoHashInt(value string, modulus int64) int64 {
- digest := sha1.Sum([]byte(value))
- number := new(big.Int).SetBytes(digest[:])
- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()
-}
-
func truncateDuckDuckGoRunes(value string, limit int) string {
if utf8.RuneCountInString(value) <= limit {
return value
diff --git a/internal/agent/tool/github.go b/internal/agent/tool/github.go
index 4c6a3c800b..3dd21fc88d 100644
--- a/internal/agent/tool/github.go
+++ b/internal/agent/tool/github.go
@@ -18,13 +18,10 @@ package tool
import (
"context"
- "crypto/sha1"
"encoding/json"
"fmt"
- "math/big"
"net/http"
"net/url"
- "strconv"
"strings"
"time"
"unicode/utf8"
@@ -230,10 +227,9 @@ func buildGitHubReferences(envelope map[string]any) ([]map[string]any, []map[str
if content == "" {
continue
}
- documentID := strconv.FormatInt(githubHashInt(content, 100000000), 10)
title := githubValueString(repository["name"])
resultURL := githubValueString(repository["html_url"])
- displayID := strconv.FormatInt(githubHashInt(documentID, 500), 10)
+ documentID, displayID := referenceIDs(resultURL, content)
chunks = append(chunks, map[string]any{
"id": displayID,
"chunk_id": documentID,
@@ -311,12 +307,6 @@ func githubValueString(value any) string {
return fmt.Sprint(value)
}
-func githubHashInt(value string, modulus int64) int64 {
- sum := sha1.Sum([]byte(value))
- number := new(big.Int).SetBytes(sum[:])
- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()
-}
-
func truncateGitHubRunes(value string, limit int) string {
if utf8.RuneCountInString(value) <= limit {
return value
diff --git a/internal/agent/tool/google.go b/internal/agent/tool/google.go
index be9827c8a5..32be198fc4 100644
--- a/internal/agent/tool/google.go
+++ b/internal/agent/tool/google.go
@@ -18,10 +18,8 @@ package tool
import (
"context"
- "crypto/sha1"
"encoding/json"
"fmt"
- "math/big"
"net/http"
"net/url"
"regexp"
@@ -271,10 +269,9 @@ func buildGoogleReferences(envelope map[string]any) ([]map[string]any, []map[str
if content == "" {
continue
}
- documentID := strconv.FormatInt(googleHashInt(content, 100000000), 10)
- displayID := strconv.FormatInt(googleHashInt(documentID, 500), 10)
title := strings.TrimSpace(googleText(item["title"]))
resultURL := strings.TrimSpace(googleText(item["link"]))
+ documentID, displayID := referenceIDs(resultURL, content)
chunks = append(chunks, map[string]any{
"id": displayID,
"chunk_id": documentID,
@@ -347,12 +344,6 @@ func googlePromptField(value any) string {
return googleNewlinePattern.ReplaceAllString(googleText(value), " ")
}
-func googleHashInt(value string, modulus int64) int64 {
- digest := sha1.Sum([]byte(value))
- number := new(big.Int).SetBytes(digest[:])
- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()
-}
-
func truncateGoogleRunes(value string, limit int) string {
if utf8.RuneCountInString(value) <= limit {
return value
diff --git a/internal/agent/tool/google_scholar.go b/internal/agent/tool/google_scholar.go
index 8cc1698fb6..12c7aead85 100644
--- a/internal/agent/tool/google_scholar.go
+++ b/internal/agent/tool/google_scholar.go
@@ -18,10 +18,8 @@ package tool
import (
"context"
- "crypto/sha1"
"encoding/json"
"fmt"
- "math/big"
"net/http"
"net/url"
"regexp"
@@ -256,10 +254,9 @@ func buildGoogleScholarReferences(envelope map[string]any) ([]map[string]any, []
if content == "" {
continue
}
- documentID := strconv.FormatInt(googleScholarHashInt(content, 100000000), 10)
- displayID := strconv.FormatInt(googleScholarHashInt(documentID, 500), 10)
title := googleScholarText(paper["title"])
resultURL := googleScholarText(paper["link"])
+ documentID, displayID := referenceIDs(resultURL, content)
chunks = append(chunks, map[string]any{
"id": displayID,
"chunk_id": documentID,
@@ -308,12 +305,6 @@ func googleScholarText(value any) string {
return fmt.Sprint(value)
}
-func googleScholarHashInt(value string, modulus int64) int64 {
- digest := sha1.Sum([]byte(value))
- number := new(big.Int).SetBytes(digest[:])
- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()
-}
-
func truncateGoogleScholarRunes(value string, limit int) string {
if utf8.RuneCountInString(value) <= limit {
return value
diff --git a/internal/agent/tool/keenable.go b/internal/agent/tool/keenable.go
index 41790cb471..441a18278f 100644
--- a/internal/agent/tool/keenable.go
+++ b/internal/agent/tool/keenable.go
@@ -18,13 +18,10 @@ package tool
import (
"context"
- "crypto/sha1"
"encoding/json"
"fmt"
- "math/big"
"net/http"
neturl "net/url"
- "strconv"
"strings"
"unicode/utf8"
@@ -361,10 +358,9 @@ func buildKeenableReferences(envelope map[string]any) ([]map[string]any, []map[s
if content == "" {
continue
}
- documentID := strconv.FormatInt(keenableHashInt(content, 100000000), 10)
title := keenableValueString(result["title"])
resultURL := keenableValueString(result["url"])
- displayID := strconv.FormatInt(keenableHashInt(documentID, 500), 10)
+ documentID, displayID := referenceIDs(resultURL, content)
chunks = append(chunks, map[string]any{
"id": displayID,
"chunk_id": documentID,
@@ -419,12 +415,6 @@ func keenableValueString(value any) string {
return strings.TrimSpace(fmt.Sprint(value))
}
-func keenableHashInt(value string, modulus int64) int64 {
- sum := sha1.Sum([]byte(value))
- number := new(big.Int).SetBytes(sum[:])
- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()
-}
-
func truncateKeenableRunes(value string, limit int) string {
if utf8.RuneCountInString(value) <= limit {
return value
diff --git a/internal/agent/tool/pubmed.go b/internal/agent/tool/pubmed.go
index 612552e9d1..a913cf46fd 100644
--- a/internal/agent/tool/pubmed.go
+++ b/internal/agent/tool/pubmed.go
@@ -18,11 +18,9 @@ package tool
import (
"context"
- "crypto/sha1"
"encoding/json"
"encoding/xml"
"fmt"
- "math/big"
"net/http"
"net/url"
"regexp"
@@ -282,10 +280,9 @@ func buildPubMedReferences(envelope map[string]any) ([]map[string]any, []map[str
if content == "" {
continue
}
- documentID := strconv.FormatInt(pubmedHashInt(content, 100000000), 10)
- displayID := strconv.FormatInt(pubmedHashInt(documentID, 500), 10)
title := pubmedText(item["title"])
resultURL := pubmedText(item["url"])
+ documentID, displayID := referenceIDs(resultURL, content)
chunks = append(chunks, map[string]any{
"id": displayID,
"chunk_id": documentID,
@@ -332,12 +329,6 @@ func pubmedText(value any) string {
return fmt.Sprint(value)
}
-func pubmedHashInt(value string, modulus int64) int64 {
- digest := sha1.Sum([]byte(value))
- number := new(big.Int).SetBytes(digest[:])
- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()
-}
-
func truncatePubMedRunes(value string, limit int) string {
if utf8.RuneCountInString(value) <= limit {
return value
diff --git a/internal/agent/tool/querit.go b/internal/agent/tool/querit.go
index ca651148d4..f520f79e8c 100644
--- a/internal/agent/tool/querit.go
+++ b/internal/agent/tool/querit.go
@@ -18,14 +18,11 @@ package tool
import (
"context"
- "crypto/sha1"
"encoding/json"
"fmt"
"io"
- "math/big"
"net/http"
"regexp"
- "strconv"
"strings"
"time"
@@ -489,8 +486,7 @@ func (q *QueritTool) BuildReferences(_ context.Context, response map[string]any)
if content == "" {
continue
}
- documentID := strconv.FormatInt(queritHashInt(content, 100000000), 10)
- displayID := strconv.FormatInt(queritHashInt(documentID, 500), 10)
+ documentID, displayID := referenceIDs(resultURL, content)
chunks = append(chunks, map[string]any{
"id": displayID,
"chunk_id": documentID,
@@ -577,12 +573,6 @@ func queritPromptField(value any) string {
return queritNewlinePattern.ReplaceAllString(queritText(value), " ")
}
-func queritHashInt(value string, modulus int64) int64 {
- digest := sha1.Sum([]byte(value))
- number := new(big.Int).SetBytes(digest[:])
- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()
-}
-
func truncateQueritRunes(value string, limit int) string {
if limit <= 0 {
return ""
diff --git a/internal/agent/tool/reference_id.go b/internal/agent/tool/reference_id.go
new file mode 100644
index 0000000000..58e99ece03
--- /dev/null
+++ b/internal/agent/tool/reference_id.go
@@ -0,0 +1,18 @@
+package tool
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "strings"
+)
+
+// referenceIDs derives the citation identifiers for one search result.
+// documentID keys the chunk and its docAgg; displayID is the citation marker
+// rendered into the prompt. The URL participates in the seed so identical
+// content served from different URLs stays distinguishable, and the 64-bit
+// halves make collisions negligible at search-result volumes.
+func referenceIDs(resultURL, content string) (documentID, displayID string) {
+ seed := strings.TrimSpace(resultURL) + "\n" + content
+ sum := sha256.Sum256([]byte(seed))
+ return hex.EncodeToString(sum[:8]), hex.EncodeToString(sum[8:16])
+}
diff --git a/internal/agent/tool/reference_id_test.go b/internal/agent/tool/reference_id_test.go
new file mode 100644
index 0000000000..12cb278799
--- /dev/null
+++ b/internal/agent/tool/reference_id_test.go
@@ -0,0 +1,73 @@
+package tool
+
+import (
+ "fmt"
+ "testing"
+)
+
+func TestReferenceIDsDistinguishURLsForSameContent(t *testing.T) {
+ content := "the same snippet text"
+ docA, displayA := referenceIDs("https://a.example/page", content)
+ docB, displayB := referenceIDs("https://b.example/page", content)
+ if docA == docB {
+ t.Fatalf("documentID collided across URLs: %q", docA)
+ }
+ if displayA == displayB {
+ t.Fatalf("displayID collided across URLs: %q", displayA)
+ }
+}
+
+func TestReferenceIDsAreDeterministic(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ url string
+ content string
+ }{
+ {name: "url and content", url: "https://example.com/x", content: "body"},
+ {name: "empty url", url: "", content: "body"},
+ {name: "blank url", url: " ", content: "body"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ docA, displayA := referenceIDs(tc.url, tc.content)
+ docB, displayB := referenceIDs(tc.url, tc.content)
+ if docA != docB || displayA != displayB {
+ t.Fatalf("not deterministic: (%q,%q) vs (%q,%q)", docA, displayA, docB, displayB)
+ }
+ })
+ }
+}
+
+func TestReferenceIDsAreUniqueAcrossInputs(t *testing.T) {
+ seenDoc := make(map[string]string, 500)
+ seenDisplay := make(map[string]string, 500)
+ for i := 0; i < 500; i++ {
+ url := fmt.Sprintf("https://example.com/%d", i)
+ content := fmt.Sprintf("snippet %d", i)
+ docID, displayID := referenceIDs(url, content)
+ if prev, ok := seenDoc[docID]; ok {
+ t.Fatalf("documentID collision: %q produced by %q and %q", docID, prev, url)
+ }
+ if prev, ok := seenDisplay[displayID]; ok {
+ t.Fatalf("displayID collision: %q produced by %q and %q", displayID, prev, url)
+ }
+ seenDoc[docID] = url
+ seenDisplay[displayID] = url
+ }
+}
+
+func TestReferenceIDsFormat(t *testing.T) {
+ docID, displayID := referenceIDs("https://example.com/page", "body")
+ for name, id := range map[string]string{"documentID": docID, "displayID": displayID} {
+ if len(id) != 16 {
+ t.Fatalf("%s length = %d, want 16 (%q)", name, len(id), id)
+ }
+ for _, r := range id {
+ if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
+ t.Fatalf("%s = %q is not lowercase hex", name, id)
+ }
+ }
+ }
+ if docID == displayID {
+ t.Fatalf("documentID and displayID must differ, both are %q", docID)
+ }
+}
diff --git a/internal/agent/tool/registry.go b/internal/agent/tool/registry.go
index 0affe02dea..29326f502d 100644
--- a/internal/agent/tool/registry.go
+++ b/internal/agent/tool/registry.go
@@ -75,6 +75,7 @@ var registry = map[string]Factory{
"wikipedia": buildWikipediaTool,
"wikipedia_search": buildWikipediaTool,
"yahoo_finance": buildYahooFinanceTool,
+ "youcom": buildYouComTool,
}
// canvasToolNames maps lower-cased Canvas component names to the canonical
@@ -93,6 +94,7 @@ var canvasToolNames = map[string]string{
"tavilyextract": "tavily_extract",
"tavilysearch": "tavily",
"yahoofinance": "yahoo_finance",
+ "youcomsearch": "youcom",
}
func noConfig(name string, fn func() einotool.BaseTool) Factory {
@@ -751,6 +753,37 @@ func buildKeenableTool(params map[string]any) (einotool.BaseTool, error) {
return newKeenableTool(nil, nil, apiKey, defaults), nil
}
+func buildYouComTool(params map[string]any) (einotool.BaseTool, error) {
+ defaults := youComDefaults{}
+ apiKey := ""
+ if value, ok := params["api_key"]; ok {
+ var valid bool
+ apiKey, valid = value.(string)
+ if !valid {
+ return nil, fmt.Errorf("agent tool: tool %q requires string node-level param api_key", "youcom")
+ }
+ }
+ if value, ok := params["freshness"]; ok {
+ freshness, valid := value.(string)
+ if !valid {
+ return nil, fmt.Errorf("agent tool: tool %q requires string node-level param freshness", "youcom")
+ }
+ normalized := strings.ToLower(strings.TrimSpace(freshness))
+ if !slices.Contains(youComFreshnessValues, normalized) {
+ return nil, fmt.Errorf("agent tool: tool %q has unsupported freshness %v", "youcom", value)
+ }
+ defaults.Freshness = normalized
+ }
+ if value, ok := params["top_n"]; ok {
+ topN, valid := strictInt(value)
+ if !valid || topN <= 0 {
+ return nil, fmt.Errorf("agent tool: tool %q requires positive integer node-level param top_n", "youcom")
+ }
+ defaults.TopN = min(topN, youComMaxResults)
+ }
+ return newYouComTool(nil, apiKey, defaults), nil
+}
+
func buildSofyaTool(params map[string]any) (einotool.BaseTool, error) {
defaults := sofyaDefaults{}
apiKey := ""
diff --git a/internal/agent/tool/registry_test.go b/internal/agent/tool/registry_test.go
index 01e6cf7312..ecd26b4bfc 100644
--- a/internal/agent/tool/registry_test.go
+++ b/internal/agent/tool/registry_test.go
@@ -56,6 +56,7 @@ func TestBuildByName_CanvasComponentNames(t *testing.T) {
{name: "TavilyExtract", wantToolName: "tavily_extract"},
{name: "TavilySearch", wantToolName: "tavily_search"},
{name: "YahooFinance", wantToolName: "yahoo_finance"},
+ {name: "YouComSearch", wantToolName: "youcom_search"},
}
for _, tc := range tests {
@@ -159,6 +160,36 @@ func TestBuildByName_QueritContentsAliases(t *testing.T) {
}
}
+func TestBuildByName_YouComReadsCanvasParams(t *testing.T) {
+ // The canvas stores tool params under the lower-cased component name, and
+ // the node form also carries query/outputs keys the tool must ignore.
+ tools, err := BuildAll([]string{"YouComSearch"}, map[string]map[string]any{
+ "youcomsearch": {
+ "api_key": "stored-key",
+ "freshness": "week",
+ "top_n": float64(4),
+ "outputs": map[string]any{"json": map[string]any{}},
+ },
+ })
+ if err != nil {
+ t.Fatalf("BuildAll: %v", err)
+ }
+ if len(tools) != 1 {
+ t.Fatalf("len(tools) = %d, want 1", len(tools))
+ }
+ youcomTool, ok := tools[0].(*YouComTool)
+ if !ok {
+ t.Fatalf("tools[0] = %T, want *YouComTool", tools[0])
+ }
+ if youcomTool.apiKey != "stored-key" || youcomTool.defaults.Freshness != "week" || youcomTool.defaults.TopN != 4 {
+ t.Fatalf("defaults = %#v, apiKey = %q", youcomTool.defaults, youcomTool.apiKey)
+ }
+ info, infoErr := youcomTool.Info(t.Context())
+ if infoErr != nil || info.Name != "youcom_search" {
+ t.Fatalf("Info() = %#v, %v", info, infoErr)
+ }
+}
+
func TestBuildAll_AllRegisteredTools(t *testing.T) {
// Every key in registry.
names := []string{
@@ -168,7 +199,7 @@ func TestBuildAll_AllRegisteredTools(t *testing.T) {
"querit", "querit_contents", "querit_search",
"retrieval", "search_my_dataset", "search_my_dateset", "search1api", "search1api_crawl", "searxng", "sofya",
"tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", "wikipedia_search",
- "yahoo_finance",
+ "yahoo_finance", "youcom",
}
params := map[string]map[string]any{
"execute_sql": {
@@ -232,7 +263,7 @@ func TestToolRegistry_SchemasAreComplete(t *testing.T) {
"querit", "querit_contents", "querit_search",
"retrieval", "search_my_dataset", "search_my_dateset", "search1api", "search1api_crawl", "searxng", "sofya",
"tavily", "tavily_extract", "tushare", "web_crawler", "wencai", "wikipedia", "wikipedia_search",
- "yahoo_finance",
+ "yahoo_finance", "youcom",
}
params := map[string]map[string]any{
"execute_sql": {
diff --git a/internal/agent/tool/search1api.go b/internal/agent/tool/search1api.go
index 1cd9ef49b7..3d7bed3bb9 100644
--- a/internal/agent/tool/search1api.go
+++ b/internal/agent/tool/search1api.go
@@ -18,10 +18,8 @@ package tool
import (
"context"
- "crypto/sha1"
"encoding/json"
"fmt"
- "math/big"
"net/http"
"slices"
"strconv"
@@ -386,10 +384,9 @@ func buildSearch1APIReferences(envelope map[string]any) ([]map[string]any, []map
if content == "" {
continue
}
- documentID := strconv.FormatInt(search1APIHashInt(content, 100000000), 10)
- displayID := strconv.FormatInt(search1APIHashInt(documentID, 500), 10)
title := search1APIText(result["title"])
resultURL := search1APIText(result["link"])
+ documentID, displayID := referenceIDs(resultURL, content)
chunks = append(chunks, map[string]any{
"id": displayID,
"chunk_id": documentID,
@@ -441,12 +438,6 @@ func search1APIText(value any) string {
return strings.TrimSpace(fmt.Sprint(value))
}
-func search1APIHashInt(value string, modulus int64) int64 {
- digest := sha1.Sum([]byte(value))
- number := new(big.Int).SetBytes(digest[:])
- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()
-}
-
func truncateSearch1APIRunes(value string, limit int) string {
if utf8.RuneCountInString(value) <= limit {
return value
diff --git a/internal/agent/tool/searxng.go b/internal/agent/tool/searxng.go
index 3dd4c79b5a..fefd5fb624 100644
--- a/internal/agent/tool/searxng.go
+++ b/internal/agent/tool/searxng.go
@@ -18,7 +18,6 @@ package tool
import (
"context"
- "crypto/sha1"
"encoding/json"
"fmt"
"net"
@@ -287,10 +286,9 @@ func buildSearXNGReferences(envelope map[string]any) ([]map[string]any, []map[st
continue
}
- documentID := strconv.Itoa(hashSearXNGString(content, 100000000))
- displayID := strconv.Itoa(hashSearXNGString(documentID, 500))
title := searxngText(item["title"])
resultURL := searxngText(item["url"])
+ documentID, displayID := referenceIDs(resultURL, content)
chunks = append(chunks, map[string]any{
"id": displayID,
"chunk_id": documentID,
@@ -363,15 +361,6 @@ func searxngPromptField(value any) string {
return searxngNewlinePattern.ReplaceAllString(searxngText(value), " ")
}
-func hashSearXNGString(value string, modulus int) int {
- digest := sha1.Sum([]byte(value))
- result := 0
- for _, part := range digest {
- result = (result*256 + int(part)) % modulus
- }
- return result
-}
-
func searxngJSON(env searxngEnvelope) string {
data, err := json.Marshal(env)
if err != nil {
diff --git a/internal/agent/tool/searxng_test.go b/internal/agent/tool/searxng_test.go
index 43b0755572..a487fac9e0 100644
--- a/internal/agent/tool/searxng_test.go
+++ b/internal/agent/tool/searxng_test.go
@@ -23,7 +23,6 @@ import (
"net/http"
"net/http/httptest"
"net/url"
- "strconv"
"strings"
"sync/atomic"
"testing"
@@ -233,12 +232,8 @@ func TestSearXNGComponentContractReferencesAndOutputs(t *testing.T) {
t.Fatalf("references = %#v / %#v", chunks, docAggs)
}
cleaned := "RAGFlow content remains"
- documentID := hashSearXNGString(cleaned, 100000000)
- referenceID := hashSearXNGString(strconv.Itoa(documentID), 500)
- if documentID != 93760153 || referenceID != 491 {
- t.Fatalf("hash parity = %d/%d, want Python 93760153/491", documentID, referenceID)
- }
- if chunks[0]["document_id"] != strconv.Itoa(documentID) || chunks[0]["id"] != strconv.Itoa(referenceID) || chunks[0]["content"] != cleaned {
+ documentID, referenceID := referenceIDs("https://ragflow.io", cleaned)
+ if chunks[0]["document_id"] != documentID || chunks[0]["id"] != referenceID || chunks[0]["content"] != cleaned {
t.Fatalf("reference chunk = %#v", chunks[0])
}
@@ -249,7 +244,7 @@ func TestSearXNGComponentContractReferencesAndOutputs(t *testing.T) {
}
formalized := outputs["formalized_content"].(string)
for _, want := range []string{
- "ID: " + strconv.Itoa(referenceID),
+ "ID: " + referenceID,
"Title: RAGFlow Docs",
"URL: https://ragflow.io",
"Content:\n" + cleaned,
diff --git a/internal/agent/tool/sofya.go b/internal/agent/tool/sofya.go
index 9dbf52bcd0..527a0edef4 100644
--- a/internal/agent/tool/sofya.go
+++ b/internal/agent/tool/sofya.go
@@ -18,10 +18,8 @@ package tool
import (
"context"
- "crypto/sha1"
"encoding/json"
"fmt"
- "math/big"
"net/http"
"strconv"
"strings"
@@ -328,10 +326,9 @@ func buildSofyaReferences(envelope map[string]any) ([]map[string]any, []map[stri
if content == "" {
continue
}
- documentID := strconv.FormatInt(sofyaHashInt(content, 100000000), 10)
- displayID := strconv.FormatInt(sofyaHashInt(documentID, 500), 10)
title := sofyaText(result["title"])
resultURL := sofyaText(result["url"])
+ documentID, displayID := referenceIDs(resultURL, content)
chunks = append(chunks, map[string]any{
"id": displayID,
"chunk_id": documentID,
@@ -383,12 +380,6 @@ func sofyaText(value any) string {
return strings.TrimSpace(fmt.Sprint(value))
}
-func sofyaHashInt(value string, modulus int64) int64 {
- digest := sha1.Sum([]byte(value))
- number := new(big.Int).SetBytes(digest[:])
- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()
-}
-
func truncateSofyaRunes(value string, limit int) string {
if utf8.RuneCountInString(value) <= limit {
return value
diff --git a/internal/agent/tool/tavily.go b/internal/agent/tool/tavily.go
index 57fd2ec8e5..da3d4d7cdd 100644
--- a/internal/agent/tool/tavily.go
+++ b/internal/agent/tool/tavily.go
@@ -18,13 +18,10 @@ package tool
import (
"context"
- "crypto/sha1"
"encoding/json"
"fmt"
- "math/big"
"net/http"
"regexp"
- "strconv"
"strings"
"unicode/utf8"
@@ -444,10 +441,9 @@ func buildTavilyReferences(envelope map[string]any) ([]map[string]any, []map[str
if content == "" {
continue
}
- documentID := strconv.FormatInt(tavilyHashInt(content, 100000000), 10)
- displayID := strconv.FormatInt(tavilyHashInt(documentID, 500), 10)
title := tavilyText(item["title"])
resultURL := tavilyText(item["url"])
+ documentID, displayID := referenceIDs(resultURL, content)
score := item["score"]
chunks = append(chunks, map[string]any{
"id": displayID,
@@ -507,12 +503,6 @@ func tavilyPromptField(value any) string {
return tavilyNewlinePattern.ReplaceAllString(tavilyText(value), " ")
}
-func tavilyHashInt(value string, modulus int64) int64 {
- digest := sha1.Sum([]byte(value))
- number := new(big.Int).SetBytes(digest[:])
- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()
-}
-
func truncateTavilyRunes(value string, limit int) string {
if utf8.RuneCountInString(value) <= limit {
return value
diff --git a/internal/agent/tool/wikipedia.go b/internal/agent/tool/wikipedia.go
index e4c0cd76c1..7db31a37b3 100644
--- a/internal/agent/tool/wikipedia.go
+++ b/internal/agent/tool/wikipedia.go
@@ -18,15 +18,12 @@ package tool
import (
"context"
- "crypto/sha1"
"encoding/json"
"fmt"
- "math/big"
"net/http"
"net/url"
"regexp"
"sort"
- "strconv"
"strings"
"unicode/utf8"
@@ -308,10 +305,9 @@ func buildWikipediaReferences(envelope map[string]any) ([]map[string]any, []map[
if content == "" {
continue
}
- documentID := strconv.FormatInt(wikipediaHashInt(content, 100000000), 10)
- displayID := strconv.FormatInt(wikipediaHashInt(documentID, 500), 10)
title := wikipediaText(item["title"])
resultURL := wikipediaText(item["url"])
+ documentID, displayID := referenceIDs(resultURL, content)
chunks = append(chunks, map[string]any{
"id": displayID,
"chunk_id": documentID,
@@ -360,12 +356,6 @@ func wikipediaText(value any) string {
return fmt.Sprint(value)
}
-func wikipediaHashInt(value string, modulus int64) int64 {
- digest := sha1.Sum([]byte(value))
- number := new(big.Int).SetBytes(digest[:])
- return new(big.Int).Mod(number, big.NewInt(modulus)).Int64()
-}
-
func truncateWikipediaRunes(value string, limit int) string {
if utf8.RuneCountInString(value) <= limit {
return value
diff --git a/internal/agent/tool/youcom.go b/internal/agent/tool/youcom.go
new file mode 100644
index 0000000000..391a26f003
--- /dev/null
+++ b/internal/agent/tool/youcom.go
@@ -0,0 +1,363 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package tool
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strconv"
+ "strings"
+ "unicode/utf8"
+
+ "github.com/cloudwego/eino/components/tool"
+ "github.com/cloudwego/eino/schema"
+
+ "ragflow/internal/service/youcom"
+ "ragflow/internal/tokenizer"
+)
+
+const youComToolName = "youcom_search"
+
+const youComToolDescription = `You.com is a web search API. It returns the extracted passages of each result page, so hits carry usable context without a separate fetch. When searching:
+ - Use a focused query of the most important terms (and synonyms).
+ - Optionally restrict results by how recently they were published.`
+
+// youComMaxResults caps how many results one search returns.
+const youComMaxResults = 20
+
+const youComDefaultTopN = 10
+
+const youComPromptMaxTokens = 200000
+
+// youComMaxResponseBytes caps how much of You.com's response body is read: a
+// handful of hits of JSON is a few KiB, so 4 MiB is far above any legitimate
+// answer while still bounding what a misbehaving upstream can make us allocate.
+const youComMaxResponseBytes = 4 << 20
+
+var youComFreshnessValues = []string{"any", "day", "week", "month", "year"}
+
+// youComArgs is the JSON shape the model sends into InvokableRun. max_results
+// is decoded loosely: a missing, blank or non-numeric value means "use the
+// node's Top N" rather than failing the run.
+type youComArgs struct {
+ Query string `json:"query"`
+ Freshness string `json:"freshness"`
+ MaxResults any `json:"max_results"`
+}
+
+// youComDefaults holds the node-level configuration.
+type youComDefaults struct {
+ Freshness string
+ TopN int
+}
+
+type youComEnvelope struct {
+ Results []map[string]any `json:"results"`
+ Error string `json:"_ERROR,omitempty"`
+}
+
+// YouComTool is the You.com web search tool. It queries
+// https://api.you.com/v1/search (or the keyless endpoint when no API key is
+// configured) and returns the merged web + news results. The API key is node
+// configuration only and never model-visible.
+type YouComTool struct {
+ helper *HTTPHelper
+ apiKey string
+ defaults youComDefaults
+}
+
+var _ ToolComponent = (*YouComTool)(nil)
+var _ ReferenceBuilder = (*YouComTool)(nil)
+
+// NewYouComTool returns a YouComTool with no API key and the default HTTPHelper.
+func NewYouComTool() *YouComTool {
+ return newYouComTool(nil, "", youComDefaults{})
+}
+
+// NewYouComToolWith returns a YouComTool that uses the provided HTTPHelper and
+// API key. Useful for tests that want to inject a custom transport.
+func NewYouComToolWith(h *HTTPHelper, apiKey string) *YouComTool {
+ return newYouComTool(h, apiKey, youComDefaults{})
+}
+
+func newYouComTool(h *HTTPHelper, apiKey string, defaults youComDefaults) *YouComTool {
+ if h == nil {
+ h = NewHTTPHelper()
+ }
+ if defaults.Freshness == "" {
+ defaults.Freshness = "any"
+ }
+ if defaults.TopN <= 0 {
+ defaults.TopN = youComDefaultTopN
+ }
+ return &YouComTool{helper: h, apiKey: strings.TrimSpace(apiKey), defaults: defaults}
+}
+
+// Info returns the tool's metadata for the chat model.
+func (t *YouComTool) Info(_ context.Context) (*schema.ToolInfo, error) {
+ return &schema.ToolInfo{
+ Name: youComToolName,
+ Desc: youComToolDescription,
+ ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{
+ "query": {
+ Type: schema.String,
+ Desc: "The search keywords to execute with You.com. The most important words/terms (and synonyms) from the original request.",
+ Required: true,
+ },
+ "freshness": {
+ Type: schema.String,
+ Desc: `Restrict results by recency: "day", "week", "month", "year", or "any" (default) for no limit.`,
+ Enum: youComFreshnessValues,
+ Required: false,
+ },
+ "max_results": {
+ Type: schema.Integer,
+ Desc: "Number of results to return, 1 to 20. Leave it out to use the Top N set on the node.",
+ Required: false,
+ },
+ }),
+ }, nil
+}
+
+// InvokableRun performs the You.com search.
+//
+// Recoverable errors (bad arguments, API or network failures) are returned as a
+// JSON envelope with an _ERROR field and a nil Go error, so the ReAct agent
+// feeds them back to the model instead of failing the run, matching TavilyTool.
+func (t *YouComTool) InvokableRun(ctx context.Context, argsJSON string, _ ...tool.Option) (string, error) {
+ var args youComArgs
+ if err := json.Unmarshal([]byte(argsJSON), &args); err != nil {
+ return youComErrJSON(fmt.Errorf("youcom: parse arguments: %w", err)), nil
+ }
+ if strings.TrimSpace(args.Query) == "" {
+ return youComJSON(youComEnvelope{Results: []map[string]any{}}), nil
+ }
+
+ freshness, err := youComEnumArg("freshness", args.Freshness, youComFreshnessValues, t.defaults.Freshness)
+ if err != nil {
+ return youComErrJSON(err), nil
+ }
+ maxResults := youComMaxResultsArg(args.MaxResults, t.defaults.TopN)
+
+ hits, err := youcom.Search(ctx, t.fetch, youcom.Request{
+ Endpoint: youcom.EndpointFor(t.apiKey),
+ APIKey: t.apiKey,
+ Query: args.Query,
+ Freshness: freshness,
+ Count: maxResults,
+ })
+ if err != nil {
+ return youComErrJSON(err), nil
+ }
+ // `count` applies per response section, so the merged web + news list can
+ // exceed it; trim back to what the model (or the node's Top N) asked for.
+ if len(hits) > maxResults {
+ hits = hits[:maxResults]
+ }
+
+ results := make([]map[string]any, 0, len(hits))
+ for _, hit := range hits {
+ results = append(results, map[string]any{
+ "title": hit.Title,
+ "url": hit.URL,
+ "content": hit.Content,
+ })
+ }
+ return youComJSON(youComEnvelope{Results: results}), nil
+}
+
+// fetch performs the HTTP GET for the youcom leaf package, applying the status
+// check and response-body cap that the leaf leaves to its caller.
+func (t *YouComTool) fetch(ctx context.Context, endpoint string, headers map[string]string) ([]byte, error) {
+ resp, err := t.helper.Do(ctx, http.MethodGet, endpoint, "", "", headers)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+ return nil, fmt.Errorf("upstream returned %d", resp.StatusCode)
+ }
+
+ body, err := io.ReadAll(io.LimitReader(resp.Body, youComMaxResponseBytes+1))
+ if err != nil {
+ return nil, fmt.Errorf("read response: %w", err)
+ }
+ if len(body) > youComMaxResponseBytes {
+ return nil, fmt.Errorf("response body exceeds %d bytes", youComMaxResponseBytes)
+ }
+ return body, nil
+}
+
+// youComEnumArg normalizes a model-supplied enum value. Blank means the
+// default; anything else must be a known value, because it is forwarded to
+// You.com as-is.
+func youComEnumArg(name, value string, allowed []string, fallback string) (string, error) {
+ v := strings.ToLower(strings.TrimSpace(value))
+ if v == "" {
+ return fallback, nil
+ }
+ for _, a := range allowed {
+ if v == a {
+ return v, nil
+ }
+ }
+ return "", fmt.Errorf("youcom: %s %q is not supported, it should be one of %v", name, value, allowed)
+}
+
+// youComMaxResultsArg returns the model's result count if it gave a usable one,
+// else the node's Top N, clamped to [1, youComMaxResults].
+func youComMaxResultsArg(value any, topN int) int {
+ n := topN
+ switch v := value.(type) {
+ case float64:
+ n = int(v)
+ case string:
+ if parsed, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {
+ n = parsed
+ }
+ }
+ return min(max(n, 1), youComMaxResults)
+}
+
+// ComponentSpec returns the YouComSearch Canvas surface.
+func (t *YouComTool) ComponentSpec() ComponentSpec {
+ return ComponentSpec{
+ Inputs: map[string]string{
+ "query": "The search keywords to execute with You.com.",
+ "freshness": `Recency limit: "day", "week", "month", "year" or "any".`,
+ },
+ Outputs: map[string]string{
+ "formalized_content": "Rendered You.com references for downstream prompts.",
+ "json": "Raw You.com result list.",
+ },
+ InputForm: map[string]any{
+ "query": map[string]any{"name": "Query", "type": "line"},
+ "freshness": map[string]any{
+ "name": "Freshness",
+ "type": "options",
+ "value": "any",
+ "options": youComFreshnessValues,
+ },
+ },
+ }
+}
+
+func (t *YouComTool) BuildReferences(_ context.Context, envelope map[string]any) ([]map[string]any, []map[string]any) {
+ return buildYouComReferences(envelope)
+}
+
+func (t *YouComTool) BuildComponentOutputs(envelope map[string]any) map[string]any {
+ results := envelopeSlice(envelope, "results")
+ chunks, _ := buildYouComReferences(envelope)
+ return map[string]any{
+ "formalized_content": renderYouComReferences(chunks, youComPromptMaxTokens),
+ "json": results,
+ }
+}
+
+// buildYouComReferences turns each usable result into the chunk/docAgg shape
+// the citation path expects. A result with no text is dropped rather than
+// stored as a blank chunk.
+func buildYouComReferences(envelope map[string]any) ([]map[string]any, []map[string]any) {
+ results := envelopeSlice(envelope, "results")
+ chunks := make([]map[string]any, 0, len(results))
+ docAggs := make([]map[string]any, 0, len(results))
+ for _, item := range results {
+ result, ok := item.(map[string]any)
+ if !ok {
+ continue
+ }
+ content := truncateYouComRunes(strings.Join(strings.Fields(youComText(result["content"])), " "), 10000)
+ if content == "" {
+ continue
+ }
+ title := youComText(result["title"])
+ resultURL := youComText(result["url"])
+ documentID, displayID := referenceIDs(resultURL, content)
+ chunks = append(chunks, map[string]any{
+ "id": displayID,
+ "chunk_id": documentID,
+ "content": content,
+ "doc_id": documentID,
+ "document_id": documentID,
+ "docnm_kwd": title,
+ "document_name": title,
+ "similarity": 1,
+ "score": 1,
+ "url": resultURL,
+ })
+ docAggs = append(docAggs, map[string]any{
+ "doc_name": title,
+ "doc_id": documentID,
+ "count": 1,
+ "url": resultURL,
+ })
+ }
+ return chunks, docAggs
+}
+
+func renderYouComReferences(chunks []map[string]any, maxTokens int) string {
+ usedTokens := 0
+ blocks := make([]string, 0, len(chunks))
+ for _, chunk := range chunks {
+ content := youComText(chunk["content"])
+ usedTokens += tokenizer.NumTokensFromString(content)
+ blocks = append(blocks, strings.Join([]string{
+ "\nID: " + youComText(chunk["id"]),
+ "├── Title: " + strings.Join(strings.Fields(youComText(chunk["document_name"])), " "),
+ "├── URL: " + youComText(chunk["url"]),
+ "└── Content:\n" + content,
+ }, "\n"))
+ if maxTokens > 0 && float64(maxTokens)*0.97 < float64(usedTokens) {
+ break
+ }
+ }
+ return strings.Join(blocks, "\n")
+}
+
+func youComText(value any) string {
+ if value == nil {
+ return ""
+ }
+ if text, ok := value.(string); ok {
+ return strings.TrimSpace(text)
+ }
+ return strings.TrimSpace(fmt.Sprint(value))
+}
+
+func truncateYouComRunes(value string, limit int) string {
+ if utf8.RuneCountInString(value) <= limit {
+ return value
+ }
+ return string([]rune(value)[:limit])
+}
+
+func youComJSON(env youComEnvelope) string {
+ b, err := json.Marshal(env)
+ if err != nil {
+ return fmt.Sprintf(`{"_ERROR":"youcom: marshal result: %s"}`, err)
+ }
+ return string(b)
+}
+
+func youComErrJSON(err error) string {
+ return youComJSON(youComEnvelope{Error: err.Error()})
+}
diff --git a/internal/agent/tool/youcom_test.go b/internal/agent/tool/youcom_test.go
new file mode 100644
index 0000000000..088076db75
--- /dev/null
+++ b/internal/agent/tool/youcom_test.go
@@ -0,0 +1,258 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package tool
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+
+ "ragflow/internal/agent/runtime"
+)
+
+// youComTestServer records the last request and answers with the given status
+// and body.
+func youComTestServer(t *testing.T, status int, respBody string) (*HTTPHelper, *http.Request, *atomic.Int32) {
+ t.Helper()
+ var gotReq http.Request
+ calls := &atomic.Int32{}
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ calls.Add(1)
+ gotReq = *r.Clone(r.Context())
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(status)
+ _, _ = w.Write([]byte(respBody))
+ }))
+ t.Cleanup(srv.Close)
+ helper := NewHTTPHelper().WithClient(&http.Client{Transport: rewriteHostTransport(srv.URL)})
+ return helper, &gotReq, calls
+}
+
+func decodeYouComEnvelope(t *testing.T, out string) youComEnvelope {
+ t.Helper()
+ var env youComEnvelope
+ if err := json.Unmarshal([]byte(out), &env); err != nil {
+ t.Fatalf("decode envelope %q: %v", out, err)
+ }
+ return env
+}
+
+func TestYouCom_BuildRequest(t *testing.T) {
+ t.Parallel()
+ helper, req, _ := youComTestServer(t, http.StatusOK, `{"results":{"web":[]}}`)
+ tool := newYouComTool(helper, " ydc-test ", youComDefaults{Freshness: "week", TopN: 5})
+
+ out, err := tool.InvokableRun(t.Context(), `{"query":"ragflow go","freshness":"month"}`)
+ if err != nil {
+ t.Fatalf("InvokableRun: %v", err)
+ }
+ if env := decodeYouComEnvelope(t, out); env.Error != "" {
+ t.Fatalf("unexpected _ERROR %q", env.Error)
+ }
+ if req.Method != http.MethodGet || req.URL.Path != "/v1/search" {
+ t.Errorf("request = %s %s, want GET /v1/search", req.Method, req.URL.Path)
+ }
+ if got := req.Header.Get("X-API-Key"); got != "ydc-test" {
+ t.Errorf("X-API-Key = %q, want trimmed key", got)
+ }
+ if got := req.Header.Get("User-Agent"); got != "RAGFlow youdotcom-integration/infiniflow-ragflow" {
+ t.Errorf("User-Agent = %q", got)
+ }
+ params := req.URL.Query()
+ if params.Get("query") != "ragflow go" {
+ t.Errorf("query = %q", params.Get("query"))
+ }
+ if params.Get("count") != "5" {
+ t.Errorf("count = %q, want node Top N", params.Get("count"))
+ }
+ if params.Get("freshness") != "month" {
+ t.Errorf("freshness = %q, want model override", params.Get("freshness"))
+ }
+}
+
+func TestYouCom_KeylessOmitsAuth(t *testing.T) {
+ t.Parallel()
+ helper, req, _ := youComTestServer(t, http.StatusOK, `{"results":{"web":[]}}`)
+ tool := newYouComTool(helper, "", youComDefaults{})
+
+ if _, err := tool.InvokableRun(t.Context(), `{"query":"q"}`); err != nil {
+ t.Fatalf("InvokableRun: %v", err)
+ }
+ if req.URL.Path != "/v1/agents/search" {
+ t.Errorf("path = %q, want keyless endpoint", req.URL.Path)
+ }
+ // The keyless endpoint rejects an auth header, so none may be sent.
+ if got, ok := req.Header["X-Api-Key"]; ok {
+ t.Errorf("X-API-Key = %v, want absent", got)
+ }
+ // The node default "any" must not be forwarded.
+ if _, ok := req.URL.Query()["freshness"]; ok {
+ t.Errorf("freshness sent for \"any\": %v", req.URL.Query().Get("freshness"))
+ }
+}
+
+func TestYouCom_MaxResultsArg(t *testing.T) {
+ t.Parallel()
+ cases := []struct {
+ value any
+ want int
+ }{
+ {nil, 7}, // not given: node Top N
+ {"", 7}, // blank: node Top N
+ {"lots", 7}, // unparseable: node Top N
+ {float64(3), 3}, // model's value wins
+ {"4", 4}, // numeric string
+ {float64(0), 1}, // clamped low
+ {float64(99), 20}, // clamped to the max
+ }
+ for _, tc := range cases {
+ if got := youComMaxResultsArg(tc.value, 7); got != tc.want {
+ t.Errorf("youComMaxResultsArg(%v, 7) = %d, want %d", tc.value, got, tc.want)
+ }
+ }
+}
+
+func TestYouCom_EmptyQuerySkipsRequest(t *testing.T) {
+ t.Parallel()
+ helper, _, calls := youComTestServer(t, http.StatusOK, `{"results":{"web":[{"title":"x"}]}}`)
+ tool := newYouComTool(helper, "key", youComDefaults{})
+
+ out, err := tool.InvokableRun(t.Context(), `{"query":" "}`)
+ if err != nil {
+ t.Fatalf("InvokableRun: %v", err)
+ }
+ env := decodeYouComEnvelope(t, out)
+ if env.Error != "" || len(env.Results) != 0 {
+ t.Fatalf("envelope = %#v, want empty results", env)
+ }
+ if calls.Load() != 0 {
+ t.Errorf("upstream calls = %d, want 0", calls.Load())
+ }
+}
+
+func TestYouCom_UpstreamErrorBecomesEnvelope(t *testing.T) {
+ t.Parallel()
+ helper, _, _ := youComTestServer(t, http.StatusPaymentRequired, ``)
+ tool := newYouComTool(helper, "key", youComDefaults{})
+
+ out, err := tool.InvokableRun(t.Context(), `{"query":"q"}`)
+ if err != nil {
+ t.Fatalf("InvokableRun: %v", err)
+ }
+ env := decodeYouComEnvelope(t, out)
+ if !strings.HasPrefix(env.Error, "youcom: ") {
+ t.Fatalf("_ERROR = %q, want a youcom-prefixed error", env.Error)
+ }
+ // The leaf adds the prefix; the tool's transport errors must not add one again.
+ if strings.Contains(env.Error, "youcom: youcom:") {
+ t.Fatalf("_ERROR = %q, want a single youcom prefix", env.Error)
+ }
+}
+
+func TestYouCom_TrimsMergedResultsToMaxResults(t *testing.T) {
+ t.Parallel()
+ web := make([]map[string]string, 0, 6)
+ news := make([]map[string]string, 0, 6)
+ for i := 0; i < 6; i++ {
+ web = append(web, map[string]string{"url": "https://example.com/w", "title": "w", "description": "d"})
+ news = append(news, map[string]string{"url": "https://example.com/n", "title": "n", "description": "d"})
+ }
+ payload, err := json.Marshal(map[string]any{
+ "results": map[string]any{"web": web, "news": news},
+ })
+ if err != nil {
+ t.Fatalf("marshal payload: %v", err)
+ }
+ helper, _, _ := youComTestServer(t, http.StatusOK, string(payload))
+ tool := newYouComTool(helper, "key", youComDefaults{})
+
+ out, err := tool.InvokableRun(t.Context(), `{"query":"q","max_results":3}`)
+ if err != nil {
+ t.Fatalf("InvokableRun: %v", err)
+ }
+ env := decodeYouComEnvelope(t, out)
+ if len(env.Results) != 3 {
+ t.Fatalf("results = %d, want 3 (\"count\" is per section, so the merge must be trimmed)", len(env.Results))
+ }
+}
+
+func TestYouCom_ComponentOutputsAndReferences(t *testing.T) {
+ t.Parallel()
+ tool := NewYouComTool()
+ envelope := map[string]any{
+ "results": []any{
+ map[string]any{"title": "RAGFlow", "url": "https://ragflow.io", "content": "RAG engine"},
+ map[string]any{"title": "Blank", "url": "https://example.com/blank", "content": " "},
+ },
+ }
+
+ outputs := tool.BuildComponentOutputs(envelope)
+ if _, ok := outputs["json"]; !ok {
+ t.Fatalf("outputs missing json: %#v", outputs)
+ }
+ formalized, _ := outputs["formalized_content"].(string)
+ if !strings.Contains(formalized, "https://ragflow.io") {
+ t.Fatalf("formalized_content = %q, want the usable hit", formalized)
+ }
+
+ chunks, docAggs := tool.BuildReferences(t.Context(), envelope)
+ if len(chunks) != 1 || len(docAggs) != 1 {
+ t.Fatalf("references = %d chunks / %d docAggs, want 1/1 (blank content dropped)", len(chunks), len(docAggs))
+ }
+ if chunks[0]["url"] != "https://ragflow.io" {
+ t.Errorf("chunk url = %v", chunks[0]["url"])
+ }
+ form := tool.ComponentSpec().InputForm
+ if _, ok := form["query"]; !ok {
+ t.Errorf("InputForm missing query: %#v", form)
+ }
+}
+
+// TestYouCom_ReferencesDistinguishSameContentFromDifferentURLs guards the
+// citation path: identical snippets served from different URLs must keep
+// distinct ids instead of collapsing into one reference.
+func TestYouCom_ReferencesDistinguishSameContentFromDifferentURLs(t *testing.T) {
+ t.Parallel()
+ tool := NewYouComTool()
+ content := "identical snippet text"
+ envelope := map[string]any{
+ "results": []any{
+ map[string]any{"title": "First", "url": "https://first.example/page", "content": content},
+ map[string]any{"title": "Second", "url": "https://second.example/page", "content": content},
+ },
+ }
+
+ chunks, docAggs := tool.BuildReferences(t.Context(), envelope)
+ if len(chunks) != 2 || len(docAggs) != 2 {
+ t.Fatalf("references = %d chunks / %d docAggs, want 2/2", len(chunks), len(docAggs))
+ }
+ if chunks[0]["id"] == chunks[1]["id"] {
+ t.Fatalf("both chunks share display id %v", chunks[0]["id"])
+ }
+ if chunks[0]["document_id"] == chunks[1]["document_id"] {
+ t.Fatalf("both chunks share document id %v", chunks[0]["document_id"])
+ }
+
+ state := runtime.NewCanvasState("run-youcom", "task-youcom")
+ state.SetRetrievalReferences(chunks, docAggs)
+ if got := state.GetRetrievalChunks(); len(got) != 2 {
+ t.Fatalf("retrieval chunks = %d, want 2 (distinct URLs must survive dedupe)", len(got))
+ }
+}
diff --git a/internal/service/web_search_provider.go b/internal/service/web_search_provider.go
index d50bf15ff3..d2ebb87995 100644
--- a/internal/service/web_search_provider.go
+++ b/internal/service/web_search_provider.go
@@ -29,6 +29,7 @@ import (
"time"
"ragflow/internal/agentic_rag"
+ "ragflow/internal/service/youcom"
)
// Providers, endpoints and clients are listed alphabetically by provider id
@@ -57,15 +58,6 @@ const (
queritWebSearchEndpoint = "https://api.querit.ai/v1/search"
search1apiWebSearchEndpoint = "https://api.search1api.com/search"
serplyWebSearchEndpoint = "https://api.serply.io/v1/search/"
- // You.com serves the same response shape from two endpoints. The keyless
- // one is rate-limited but needs no credentials; the keyed one lifts those
- // limits. The keyless endpoint rejects an X-API-Key header, so the endpoint
- // and the headers are always chosen together.
- youComWebSearchEndpoint = "https://api.you.com/v1/search"
- youComKeylessWebSearchEndpoint = "https://api.you.com/v1/agents/search"
- // Identifies RAGFlow to You.com. On the keyless endpoint there is no key to
- // attribute traffic to, so this is the only signal available.
- youComWebSearchUserAgent = "RAGFlow youdotcom-integration/infiniflow-ragflow"
// webSearchResultCount is how many hits every provider is asked for. Six
// keeps the web block in a prompt the size of one corpus chunk set — the
// point is to give the model something to cite, not to mirror a full SERP.
@@ -120,7 +112,7 @@ func resolveWebSearchProvider(promptConfig map[string]interface{}) *webSearchPro
// keyOptional marks providers usable with no key. Only You.com: it serves a
// dedicated keyless endpoint (a different path, same response shape, rate
// limited per source IP, and it rejects an X-API-Key header — see
- // youComEndpointFor). A key moves to the keyed endpoint and lifts the limit.
+ // youcom.EndpointFor). A key moves to the keyed endpoint and lifts the limit.
//
// Free tier != keyless: Exa's free 1,000 requests/month still requires a key
// on every call, so it does not get this carve-out.
@@ -267,7 +259,7 @@ func retrieveWebSearchWithTavily(
return retrieveYouComWebSearch(
ctx,
youComWebSearchHTTPClient,
- youComEndpointFor(provider.APIKey),
+ youcom.EndpointFor(provider.APIKey),
provider.APIKey,
question,
)
@@ -974,43 +966,13 @@ func decodeSerplyWebSearchResults(responseBody []byte) ([]serplyWebSearchResult,
return results, nil
}
-type youComWebSearchResult struct {
- URL string `json:"url"`
- Title string `json:"title"`
- Description string `json:"description"`
- Snippets []string `json:"snippets"`
-}
-
-type youComWebSearchResponse struct {
- Results struct {
- Web []youComWebSearchResult `json:"web"`
- News []youComWebSearchResult `json:"news"`
- } `json:"results"`
-}
-
-// youComEndpointFor picks the keyless endpoint when no key is configured. The
-// keyless endpoint rejects an X-API-Key header, so callers must never send a
-// key to it.
-func youComEndpointFor(apiKey string) string {
- if strings.TrimSpace(apiKey) == "" {
- return youComKeylessWebSearchEndpoint
- }
- return youComWebSearchEndpoint
-}
-
-// youComContent prefers the extracted page passages. News hits carry only a
-// description.
-func youComContent(result youComWebSearchResult) string {
- passages := make([]string, 0, len(result.Snippets))
- for _, snippet := range result.Snippets {
- if strings.TrimSpace(snippet) != "" {
- passages = append(passages, snippet)
- }
- }
- if len(passages) > 0 {
- return strings.Join(passages, "\n")
+// youComFetcher adapts the service's shared web-search request helper to the
+// youcom leaf package's transport seam, so the You.com request/response logic
+// lives in one place and this package only owns the HTTP client.
+func youComFetcher(client *http.Client) youcom.Fetcher {
+ return func(ctx context.Context, endpoint string, headers map[string]string) ([]byte, error) {
+ return webSearchRequest(ctx, client, http.MethodGet, endpoint, headers, nil)
}
- return strings.TrimSpace(result.Description)
}
func retrieveYouComWebSearch(
@@ -1020,45 +982,26 @@ func retrieveYouComWebSearch(
apiKey string,
query string,
) (map[string]interface{}, error) {
- queryParams := url.Values{}
- queryParams.Set("query", query)
- queryParams.Set("count", strconv.Itoa(webSearchResultCount))
-
- headers := map[string]string{
- "Accept": "application/json",
- "User-Agent": youComWebSearchUserAgent,
- }
- // The keyless endpoint rejects an X-API-Key header, so the key rides only
- // when one is configured.
- if trimmedKey := strings.TrimSpace(apiKey); trimmedKey != "" {
- headers["X-API-Key"] = trimmedKey
- }
- responseBody, err := webSearchRequest(ctx, client, http.MethodGet,
- endpoint+"?"+queryParams.Encode(), headers, nil)
+ hits, err := youcom.Search(ctx, youComFetcher(client), youcom.Request{
+ Endpoint: endpoint,
+ APIKey: apiKey,
+ Query: query,
+ Freshness: "",
+ Count: webSearchResultCount,
+ })
if err != nil {
- return nil, fmt.Errorf("youcom: %w", err)
- }
-
- var decoded youComWebSearchResponse
- if err := json.Unmarshal(responseBody, &decoded); err != nil {
- return nil, fmt.Errorf("youcom: decode response: %w", err)
+ return nil, err
}
- // `count` applies per response section, so web and news together can exceed
- // it. Web results lead; the merged list is trimmed back afterwards.
- merged := make([]youComWebSearchResult, 0, len(decoded.Results.Web)+len(decoded.Results.News))
- merged = append(merged, decoded.Results.Web...)
- merged = append(merged, decoded.Results.News...)
-
- hits := make([]webSearchHit, 0, len(merged))
- for _, result := range merged {
- hits = append(hits, webSearchHit{
- Title: result.Title,
- URL: result.URL,
- Content: youComContent(result),
+ webHits := make([]webSearchHit, 0, len(hits))
+ for _, hit := range hits {
+ webHits = append(webHits, webSearchHit{
+ Title: hit.Title,
+ URL: hit.URL,
+ Content: hit.Content,
})
}
- return webSearchPayload("youcom", hits), nil
+ return webSearchPayload("youcom", webHits), nil
}
// webSearchFunc adapts the pipeline's configured provider to the agent's
diff --git a/internal/service/web_search_provider_test.go b/internal/service/web_search_provider_test.go
index 0bd2870ab6..25545b88c9 100644
--- a/internal/service/web_search_provider_test.go
+++ b/internal/service/web_search_provider_test.go
@@ -23,6 +23,8 @@ import (
"net/http/httptest"
"net/url"
"testing"
+
+ "ragflow/internal/service/youcom"
)
func TestWebSearchFuncGatedByProvider(t *testing.T) {
@@ -455,58 +457,6 @@ func TestResolveWebSearchProviderStillRequiresKeysForKeyedProviders(t *testing.T
}
}
-func TestYouComEndpointForPicksKeylessWithoutAKey(t *testing.T) {
- cases := []struct {
- name string
- apiKey string
- want string
- }{
- {name: "no key", apiKey: "", want: youComKeylessWebSearchEndpoint},
- {name: "whitespace key", apiKey: " ", want: youComKeylessWebSearchEndpoint},
- {name: "key set", apiKey: "ydc-test", want: youComWebSearchEndpoint},
- }
-
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- if got := youComEndpointFor(tc.apiKey); got != tc.want {
- t.Fatalf("endpoint = %q, want %q", got, tc.want)
- }
- })
- }
-}
-
-func TestYouComContentDiscardsBlankSnippetsAndDescriptions(t *testing.T) {
- cases := []struct {
- name string
- result youComWebSearchResult
- want string
- }{
- {
- name: "joins non-blank passages",
- result: youComWebSearchResult{Snippets: []string{"a", " ", "b"}, Description: "ignored"},
- want: "a\nb",
- },
- {
- name: "falls back to the description",
- result: youComWebSearchResult{Snippets: []string{" "}, Description: " desc "},
- want: "desc",
- },
- {
- name: "whitespace-only description yields nothing",
- result: youComWebSearchResult{Description: " "},
- want: "",
- },
- }
-
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- if got := youComContent(tc.result); got != tc.want {
- t.Fatalf("content = %q, want %q", got, tc.want)
- }
- })
- }
-}
-
func TestRetrieveYouComWebSearchSendsNoAuthHeaderWhenKeyless(t *testing.T) {
var gotAuth string
var gotUserAgent string
@@ -535,8 +485,8 @@ func TestRetrieveYouComWebSearchSendsNoAuthHeaderWhenKeyless(t *testing.T) {
if gotAuth != "" {
t.Fatalf("X-API-Key = %q, want empty", gotAuth)
}
- if gotUserAgent != youComWebSearchUserAgent {
- t.Fatalf("User-Agent = %q, want %q", gotUserAgent, youComWebSearchUserAgent)
+ if gotUserAgent != youcom.UserAgent {
+ t.Fatalf("User-Agent = %q, want %q", gotUserAgent, youcom.UserAgent)
}
if gotQuery != "What is RAGFlow?" {
t.Fatalf("query = %q, want %q", gotQuery, "What is RAGFlow?")
diff --git a/internal/service/youcom/youcom.go b/internal/service/youcom/youcom.go
new file mode 100644
index 0000000000..b8747e58be
--- /dev/null
+++ b/internal/service/youcom/youcom.go
@@ -0,0 +1,167 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Package youcom implements the You.com web-search request/response contract.
+// It is a dependency-light leaf package so that both the chat pipeline
+// (internal/service) and the agent tool layer (internal/agent/tool) can depend
+// on it without an import cycle — mirroring the internal/service/wikisearch and
+// internal/service/nav pattern.
+package youcom
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/url"
+ "strconv"
+ "strings"
+)
+
+const (
+ // WebSearchEndpoint is the keyed endpoint; a key lifts the keyless
+ // endpoint's per-IP rate limit.
+ WebSearchEndpoint = "https://api.you.com/v1/search"
+ // KeylessWebSearchEndpoint is rate-limited but needs no credentials, and it
+ // rejects an X-API-Key header, so the endpoint and the headers are always
+ // chosen together (see EndpointFor).
+ KeylessWebSearchEndpoint = "https://api.you.com/v1/agents/search"
+ // UserAgent identifies RAGFlow to You.com. On the keyless endpoint there is
+ // no key to attribute traffic to, so this is the only signal available.
+ UserAgent = "RAGFlow youdotcom-integration/infiniflow-ragflow"
+)
+
+// Hit is one normalized search result: the three fields a model can quote and
+// cite.
+type Hit struct {
+ Title string
+ URL string
+ Content string
+}
+
+// Fetcher performs the HTTP GET for one request and returns the raw response
+// body. Callers own the transport concerns (status check, response-body cap),
+// so the chat pipeline's webSearchRequest and the agent tool layer's HTTPHelper
+// can both drive this package.
+type Fetcher func(ctx context.Context, endpoint string, headers map[string]string) ([]byte, error)
+
+// Request carries one search query and its node-level configuration.
+type Request struct {
+ // Endpoint is the already-resolved search URL (see EndpointFor).
+ Endpoint string
+ // APIKey is only sent when non-blank, and it selects the keyed endpoint
+ // via EndpointFor.
+ APIKey string
+ Query string
+ // Freshness restricts results by recency; "" and "any" send nothing.
+ Freshness string
+ // Count is the per-section result count requested from You.com.
+ Count int
+}
+
+// EndpointFor picks the keyless endpoint when no key is configured. The keyless
+// endpoint rejects an X-API-Key header, so callers must never send a key to it.
+func EndpointFor(apiKey string) string {
+ if strings.TrimSpace(apiKey) == "" {
+ return KeylessWebSearchEndpoint
+ }
+ return WebSearchEndpoint
+}
+
+// Search issues one You.com search and returns the merged web + news hits.
+func Search(ctx context.Context, fetch Fetcher, req Request) ([]Hit, error) {
+ queryParams := url.Values{}
+ queryParams.Set("query", req.Query)
+ queryParams.Set("count", strconv.Itoa(req.Count))
+ if freshness := normalizeFreshness(req.Freshness); freshness != "" {
+ queryParams.Set("freshness", freshness)
+ }
+
+ headers := map[string]string{
+ "Accept": "application/json",
+ "User-Agent": UserAgent,
+ }
+ // The keyless endpoint rejects an X-API-Key header, so the key rides only
+ // when one is configured.
+ if trimmedKey := strings.TrimSpace(req.APIKey); trimmedKey != "" {
+ headers["X-API-Key"] = trimmedKey
+ }
+
+ responseBody, err := fetch(ctx, req.Endpoint+"?"+queryParams.Encode(), headers)
+ if err != nil {
+ return nil, fmt.Errorf("youcom: %w", err)
+ }
+
+ var decoded webSearchResponse
+ if err := json.Unmarshal(responseBody, &decoded); err != nil {
+ return nil, fmt.Errorf("youcom: decode response: %w", err)
+ }
+
+ // `count` applies per response section, so web and news together can exceed
+ // it. Web results lead; the caller trims the merged list if it needs to.
+ merged := make([]webSearchResult, 0, len(decoded.Results.Web)+len(decoded.Results.News))
+ merged = append(merged, decoded.Results.Web...)
+ merged = append(merged, decoded.Results.News...)
+
+ hits := make([]Hit, 0, len(merged))
+ for _, result := range merged {
+ hits = append(hits, Hit{
+ Title: result.Title,
+ URL: result.URL,
+ Content: content(result),
+ })
+ }
+ return hits, nil
+}
+
+// normalizeFreshness lower-cases the requested window and drops the values that
+// mean "no limit". Unknown values are forwarded as-is; the tool layer validates
+// them against the supported set before calling here.
+func normalizeFreshness(value string) string {
+ freshness := strings.ToLower(strings.TrimSpace(value))
+ if freshness == "" || freshness == "any" {
+ return ""
+ }
+ return freshness
+}
+
+// content prefers the extracted page passages. News hits carry only a
+// description.
+func content(result webSearchResult) string {
+ passages := make([]string, 0, len(result.Snippets))
+ for _, snippet := range result.Snippets {
+ if strings.TrimSpace(snippet) != "" {
+ passages = append(passages, snippet)
+ }
+ }
+ if len(passages) > 0 {
+ return strings.Join(passages, "\n")
+ }
+ return strings.TrimSpace(result.Description)
+}
+
+type webSearchResult struct {
+ URL string `json:"url"`
+ Title string `json:"title"`
+ Description string `json:"description"`
+ Snippets []string `json:"snippets"`
+}
+
+type webSearchResponse struct {
+ Results struct {
+ Web []webSearchResult `json:"web"`
+ News []webSearchResult `json:"news"`
+ } `json:"results"`
+}
diff --git a/internal/service/youcom/youcom_test.go b/internal/service/youcom/youcom_test.go
new file mode 100644
index 0000000000..67f3af2301
--- /dev/null
+++ b/internal/service/youcom/youcom_test.go
@@ -0,0 +1,175 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package youcom
+
+import (
+ "context"
+ "errors"
+ "net/url"
+ "strings"
+ "testing"
+)
+
+func TestEndpointForPicksKeylessWithoutAKey(t *testing.T) {
+ cases := []struct {
+ name string
+ apiKey string
+ want string
+ }{
+ {name: "no key", apiKey: "", want: KeylessWebSearchEndpoint},
+ {name: "whitespace key", apiKey: " ", want: KeylessWebSearchEndpoint},
+ {name: "key set", apiKey: "ydc-test", want: WebSearchEndpoint},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := EndpointFor(tc.apiKey); got != tc.want {
+ t.Fatalf("endpoint = %q, want %q", got, tc.want)
+ }
+ })
+ }
+}
+
+// captureFetch returns a Fetcher that records the endpoint and headers and
+// answers with body.
+func captureFetch(body string, endpoint *string, headers *map[string]string) Fetcher {
+ return func(_ context.Context, gotEndpoint string, gotHeaders map[string]string) ([]byte, error) {
+ *endpoint = gotEndpoint
+ *headers = gotHeaders
+ return []byte(body), nil
+ }
+}
+
+func TestSearchMergesSectionsAndSendsConfig(t *testing.T) {
+ var endpoint string
+ var headers map[string]string
+ body := `{"results":{"web":[` +
+ `{"url":"https://example.com/a","title":"A","snippets":["first"," ","second"],"description":"ignored"},` +
+ `{"url":"https://example.com/b","title":"B","description":"desc fallback"}` +
+ `],"news":[{"url":"https://news.example.com/c","title":"C","description":"news desc"}]}}`
+
+ hits, err := Search(t.Context(), captureFetch(body, &endpoint, &headers), Request{
+ Endpoint: EndpointFor(""),
+ APIKey: "",
+ Query: "ragflow",
+ Freshness: "week",
+ Count: 6,
+ })
+ if err != nil {
+ t.Fatalf("Search: %v", err)
+ }
+
+ parsed, err := url.Parse(endpoint)
+ if err != nil {
+ t.Fatalf("parse endpoint: %v", err)
+ }
+ if parsed.Query().Get("query") != "ragflow" {
+ t.Errorf("query = %q, want ragflow", parsed.Query().Get("query"))
+ }
+ if parsed.Query().Get("count") != "6" {
+ t.Errorf("count = %q, want 6", parsed.Query().Get("count"))
+ }
+ if parsed.Query().Get("freshness") != "week" {
+ t.Errorf("freshness = %q, want week", parsed.Query().Get("freshness"))
+ }
+ if got := headers["User-Agent"]; got != UserAgent {
+ t.Errorf("User-Agent = %q, want %q", got, UserAgent)
+ }
+ // The keyless endpoint rejects an X-API-Key header, so none may be sent.
+ if got, ok := headers["X-API-Key"]; ok {
+ t.Errorf("X-API-Key = %q, want absent", got)
+ }
+
+ if len(hits) != 3 {
+ t.Fatalf("hits = %d, want 3", len(hits))
+ }
+ if hits[0].Content != "first\nsecond" {
+ t.Errorf("web content = %q, want joined non-blank snippets", hits[0].Content)
+ }
+ if hits[1].Content != "desc fallback" {
+ t.Errorf("description fallback = %q", hits[1].Content)
+ }
+ if hits[2].Content != "news desc" || hits[2].Title != "C" {
+ t.Errorf("news hit = %#v", hits[2])
+ }
+}
+
+func TestSearchSendsKeyAndOmitsFreshnessForAny(t *testing.T) {
+ var endpoint string
+ var headers map[string]string
+ hits, err := Search(t.Context(), captureFetch(`{"results":{"web":[]}}`, &endpoint, &headers), Request{
+ Endpoint: EndpointFor("ydc-test"),
+ APIKey: " ydc-test ",
+ Query: "q",
+ Freshness: "any",
+ Count: 3,
+ })
+ if err != nil {
+ t.Fatalf("Search: %v", err)
+ }
+ if len(hits) != 0 {
+ t.Fatalf("hits = %d, want 0", len(hits))
+ }
+ if got := headers["X-API-Key"]; got != "ydc-test" {
+ t.Errorf("X-API-Key = %q, want trimmed key", got)
+ }
+ parsed, _ := url.Parse(endpoint)
+ if _, ok := parsed.Query()["freshness"]; ok {
+ t.Errorf("freshness sent for \"any\": %v", parsed.Query().Get("freshness"))
+ }
+}
+
+func TestSearchDiscardsBlankContent(t *testing.T) {
+ var endpoint string
+ var headers map[string]string
+ body := `{"results":{"web":[{"url":"https://example.com/blank","title":"Blank","description":" "}]}}`
+ hits, err := Search(t.Context(), captureFetch(body, &endpoint, &headers), Request{Count: 6})
+ if err != nil {
+ t.Fatalf("Search: %v", err)
+ }
+ // The leaf keeps the hit (the caller decides whether blank content is
+ // usable); a blank description must not be turned into non-empty text.
+ if len(hits) != 1 || strings.TrimSpace(hits[0].Content) != "" {
+ t.Fatalf("hits = %#v, want one blank-content hit", hits)
+ }
+}
+
+func TestSearchWrapsFetchError(t *testing.T) {
+ fetch := func(context.Context, string, map[string]string) ([]byte, error) {
+ return nil, errors.New("status 402")
+ }
+ _, err := Search(t.Context(), fetch, Request{Query: "q", Count: 6})
+ if err == nil {
+ t.Fatal("expected an error")
+ }
+ if !strings.Contains(err.Error(), "youcom: status 402") {
+ t.Fatalf("err = %q, want youcom-prefixed fetch error", err.Error())
+ }
+}
+
+func TestSearchRejectsMalformedBody(t *testing.T) {
+ fetch := func(context.Context, string, map[string]string) ([]byte, error) {
+ return []byte("not json"), nil
+ }
+ _, err := Search(t.Context(), fetch, Request{Query: "q", Count: 6})
+ if err == nil {
+ t.Fatal("expected a decode error")
+ }
+ if !strings.Contains(err.Error(), "youcom: decode response") {
+ t.Fatalf("err = %q, want decode error", err.Error())
+ }
+}
diff --git a/web/src/components/__tests__/floating-chat-widget-markdown.test.tsx b/web/src/components/__tests__/floating-chat-widget-markdown.test.tsx
new file mode 100644
index 0000000000..0efd5aaf98
--- /dev/null
+++ b/web/src/components/__tests__/floating-chat-widget-markdown.test.tsx
@@ -0,0 +1,170 @@
+import { render, screen } from '@testing-library/react';
+
+import FloatingChatWidgetMarkdown from '../floating-chat-widget-markdown';
+
+jest.mock('react-i18next', () => ({
+ useTranslation: () => ({ t: (key: string) => key }),
+}));
+
+jest.mock('@/hooks/use-document-request', () => ({
+ useFetchDocumentThumbnailsByIds: () => ({
+ setDocumentIds: jest.fn(),
+ data: {},
+ }),
+ useGetDocumentUrl: () => () => undefined,
+}));
+
+jest.mock('@/components/theme-provider', () => ({
+ useIsDarkTheme: () => false,
+}));
+
+// Render the hover card eagerly so the resolved chunk content is assertable.
+jest.mock('@/components/ui/hover-card', () => ({
+ HoverCard: ({ children }: any) => (
+ <div data-testid="citation">{children}</div>
+ ),
+ HoverCardTrigger: ({ children }: any) => (
+ <span data-testid="chip">{children}</span>
+ ),
+ HoverCardContent: ({ children }: any) => (
+ <div data-testid="popover">{children}</div>
+ ),
+}));
+
+jest.mock('@radix-ui/react-hover-card', () => ({
+ Portal: ({ children }: any) => <>{children}</>,
+}));
+
+jest.mock('@/components/ui/tooltip', () => ({
+ Tooltip: ({ children }: any) => <>{children}</>,
+ TooltipTrigger: ({ children }: any) => <>{children}</>,
+ TooltipContent: ({ children }: any) => <>{children}</>,
+}));
+
+jest.mock('@/components/svg-icon', () => ({
+ __esModule: true,
+ default: () => <span />,
+}));
+
+jest.mock('@/components/image', () => ({
+ __esModule: true,
+ default: () => null,
+ AuthenticatedImg: () => null,
+}));
+
+jest.mock('react-syntax-highlighter', () => ({
+ __esModule: true,
+ default: ({ children }: any) => <pre>{children}</pre>,
+}));
+
+jest.mock('react-syntax-highlighter/dist/esm/styles/prism', () => ({
+ oneDark: {},
+ oneLight: {},
+}));
+
+// react-markdown is ESM-only; feed the whole content through the same
+// custom-typography component that replaces citation markers in production.
+jest.mock('react-markdown', () => ({
+ __esModule: true,
+ default: ({ children, components }: any) => {
+ const ReactLib = jest.requireActual('react');
+ const CustomTypography = components['custom-typography'];
+ return ReactLib.createElement(CustomTypography, null, String(children));
+ },
+}));
+
+jest.mock('rehype-katex', () => jest.fn());
+jest.mock('rehype-raw', () => jest.fn());
+jest.mock('unist-util-visit-parents', () => ({ visitParents: jest.fn() }));
+jest.mock('@/constants/markdown-remark-plugins', () => ({
+ MarkdownRemarkPlugins: [],
+}));
+jest.mock('@/constants/markdown-rehype-plugins', () => ({
+ RehypeSanitizeAssistantMarkdown: jest.fn(),
+}));
+
+const buildChunk = (id: string, content: string, documentId: string) => ({
+ id,
+ chunk_id: id,
+ content,
+ content_with_weight: content,
+ document_id: documentId,
+ doc_id: documentId,
+ image_id: '',
+ img_id: '',
+});
+
+const Opaque = buildChunk(
+ '0000123456789012',
+ 'OPAQUE_CHUNK_CONTENT',
+ 'doc-opaque',
+);
+const Hex = buildChunk('a2d067519c9e0db1', 'HEX_CHUNK_CONTENT', 'doc-hex');
+
+const buildReference = () =>
+ ({
+ chunks: [Opaque, Hex],
+ doc_aggs: {
+ 'doc-opaque': {
+ doc_id: 'doc-opaque',
+ doc_name: 'Opaque source.txt',
+ count: 1,
+ },
+ 'doc-hex': {
+ doc_id: 'doc-hex',
+ doc_name: 'Hex source.txt',
+ count: 1,
+ },
+ },
+ total: 2,
+ }) as any;
+
+const popoverTexts = () =>
+ screen.getAllByTestId('popover').map((popover) => popover.textContent);
+
+describe('FloatingChatWidgetMarkdown citations', () => {
+ it('keeps an all-digit opaque id as an id, not a positional index', () => {
+ render(
+ <FloatingChatWidgetMarkdown
+ content="see [ID:0000123456789012]"
+ loading={false}
+ reference={buildReference()}
+ />,
+ );
+
+ expect(screen.queryByText('Reference unavailable')).not.toBeInTheDocument();
+ const popovers = popoverTexts();
+ expect(popovers).toHaveLength(1);
+ expect(popovers[0]).toContain('OPAQUE_CHUNK_CONTENT');
+ expect(popovers[0]).toContain('Opaque source.txt');
+ });
+
+ it('resolves a normal hex id by id', () => {
+ render(
+ <FloatingChatWidgetMarkdown
+ content="see [ID:a2d067519c9e0db1]"
+ loading={false}
+ reference={buildReference()}
+ />,
+ );
+
+ const popovers = popoverTexts();
+ expect(popovers).toHaveLength(1);
+ expect(popovers[0]).toContain('HEX_CHUNK_CONTENT');
+ expect(popovers[0]).toContain('Hex source.txt');
+ });
+
+ it('keeps short numeric markers positional', () => {
+ render(
+ <FloatingChatWidgetMarkdown
+ content="see [ID:0]"
+ loading={false}
+ reference={buildReference()}
+ />,
+ );
+
+ const popovers = popoverTexts();
+ expect(popovers).toHaveLength(1);
+ expect(popovers[0]).toContain('OPAQUE_CHUNK_CONTENT');
+ });
+});
diff --git a/web/src/components/floating-chat-widget-markdown.tsx b/web/src/components/floating-chat-widget-markdown.tsx
index c89c954ee1..1093a95065 100644
--- a/web/src/components/floating-chat-widget-markdown.tsx
+++ b/web/src/components/floating-chat-widget-markdown.tsx
@@ -24,15 +24,18 @@ import {
} from '@/hooks/use-document-request';
import { IReference, IReferenceChunk } from '@/interfaces/database/chat';
import {
- currentReg,
parseCitationIndex,
preprocessLaTeX,
replaceRetrievingToSection,
replaceTextByOldReg,
replaceThinkToSection,
+ resolveCitationChunk,
showImage,
} from '@/utils/chat';
-import { citationMarkerReg } from '@/utils/citation-utils';
+import {
+ citationMarkerReg,
+ citationWholeMarkerReg,
+} from '@/utils/citation-utils';
import { getExtension } from '@/utils/document-util';
import { supportsSourceLocate } from '@/utils/source-locate';
import { getDirAttribute } from '@/utils/text-direction';
@@ -62,8 +65,7 @@ import { Button } from './ui/button';
import { HoverCard, HoverCardContent, HoverCardTrigger } from './ui/hover-card';
import { Tooltip, TooltipContent, TooltipTrigger } from './ui/tooltip';
-const getChunkIndex = (match: string) =>
- parseCitationIndex(match.replace(/\[|\]/g, ''));
+const getChunkKey = (match: string) => parseCitationIndex(match);
const FloatingChatWidgetMarkdown = ({
reference,
@@ -144,13 +146,8 @@ const FloatingChatWidgetMarkdown = ({
};
const getReferenceInfo = useCallback(
- (chunkIndex: number) => {
- let chunkItem = reference?.chunks?.[chunkIndex];
- if (!chunkItem) {
- chunkItem = reference?.chunks?.find(
- (x: any) => x?.chunk_id === chunkIndex,
- ) as IReferenceChunk;
- }
+ (chunkKey: number | string) => {
+ const chunkItem = resolveCitationChunk(reference?.chunks, chunkKey);
if (!chunkItem) return null;
const docAggsArray = Array.isArray(reference?.doc_aggs)
? reference.doc_aggs
@@ -179,8 +176,8 @@ const FloatingChatWidgetMarkdown = ({
);
const getPopoverContent = useCallback(
- (chunkIndex: number) => {
- const info = getReferenceInfo(chunkIndex);
+ (chunkKey: number | string) => {
+ const info = getReferenceInfo(chunkKey);
if (!info) {
return (
@@ -280,9 +277,9 @@ const FloatingChatWidgetMarkdown = ({
const renderReference = useCallback(
(text: string) => {
- return reactStringReplace(text, currentReg, (match, i) => {
- const chunkIndex = getChunkIndex(match) as number;
- const info = getReferenceInfo(chunkIndex);
+ return reactStringReplace(text, citationWholeMarkerReg, (match, i) => {
+ const chunkKey = getChunkKey(match);
+ const info = getReferenceInfo(chunkKey);
if (!info) {
return (
@@ -327,7 +324,7 @@ const FloatingChatWidgetMarkdown = ({
collisionPadding={8}
className="max-h-[var(--radix-hover-card-content-available-height)]"
>
- {getPopoverContent(chunkIndex)}
+ {getPopoverContent(chunkKey)}
</HoverCardContent>
</HoverCardPrimitive.Portal>
</HoverCard>
diff --git a/web/src/components/next-markdown-content/index.tsx b/web/src/components/next-markdown-content/index.tsx
index 827db3ab92..8def4bfa9f 100644
--- a/web/src/components/next-markdown-content/index.tsx
+++ b/web/src/components/next-markdown-content/index.tsx
@@ -52,7 +52,10 @@ import {
resolveCitationChunk,
unescapeAngleBrackets,
} from '@/utils/chat';
-import type { CitationKey } from '@/utils/citation-utils';
+import {
+ citationWholeMarkerReg,
+ type CitationKey,
+} from '@/utils/citation-utils';
import { getDirAttribute } from '@/utils/text-direction';
import { useFetchDocumentThumbnailsByIds } from '@/hooks/use-document-request';
@@ -74,11 +77,6 @@ import styles from './index.module.less';
const getChunkKey = (match: string) => parseCitationIndex(match);
-// react-string-replace hands the callback the LAST capture group, not the
-// full match — citationMarkerReg groups only the id, so replacing with it
-// would strip the brackets. This group wraps the whole marker, letting an
-// unresolved marker be returned unchanged.
-const WholeMarkerReg = /(\[(?:ID:)?[0-9a-fA-F\u0660-\u0669\u06F0-\u06F9]+\])/g;
const isArtifactUrl = (url?: string) =>
Boolean(url && url.includes('/api/v1/documents/artifact/'));
@@ -404,32 +402,36 @@ function MarkdownContent({
// payload ships in retrieval order, which is unrelated to the order
// the answer cites passages in.
const keyToDisplay = buildCitationDisplayMap(text);
- const replacedText = reactStringReplace(text, WholeMarkerReg, (match) => {
- const chunkKey = getChunkKey(match);
- const displayIndex = keyToDisplay.get(chunkKey) ?? 0;
- const chunkItem = resolveCitationChunk(reference?.chunks, chunkKey);
- // The reference lands at message_end, after the answer stream, so an
- // unresolved marker keeps its chip while loading. Afterwards it
- // stays as literal text rather than pointing at the wrong chunk.
- if (!chunkItem && !loading) {
- return match;
- }
+ const replacedText = reactStringReplace(
+ text,
+ citationWholeMarkerReg,
+ (match) => {
+ const chunkKey = getChunkKey(match);
+ const displayIndex = keyToDisplay.get(chunkKey) ?? 0;
+ const chunkItem = resolveCitationChunk(reference?.chunks, chunkKey);
+ // The reference lands at message_end, after the answer stream, so an
+ // unresolved marker keeps its chip while loading. Afterwards it
+ // stays as literal text rather than pointing at the wrong chunk.
+ if (!chunkItem && !loading) {
+ return match;
+ }
- return (
- <HoverCard key={`${String(chunkKey)}-${displayIndex}`}>
- <HoverCardTrigger>
- <bdi className="text-text-secondary bg-bg-card rounded-2xl px-1 mx-1 text-nowrap inline-block">
- [{displayIndex + 1}]
- </bdi>
- </HoverCardTrigger>
- {chunkItem && (
- <HoverCardContent className="max-w-3xl">
- {renderPopoverContent(chunkKey)}
- </HoverCardContent>
- )}
- </HoverCard>
- );
- });
+ return (
+ <HoverCard key={`${String(chunkKey)}-${displayIndex}`}>
+ <HoverCardTrigger>
+ <bdi className="text-text-secondary bg-bg-card rounded-2xl px-1 mx-1 text-nowrap inline-block">
+ [{displayIndex + 1}]
+ </bdi>
+ </HoverCardTrigger>
+ {chunkItem && (
+ <HoverCardContent className="max-w-3xl">
+ {renderPopoverContent(chunkKey)}
+ </HoverCardContent>
+ )}
+ </HoverCard>
+ );
+ },
+ );
return replacedText;
},
diff --git a/web/src/utils/__tests__/citation-utils.test.ts b/web/src/utils/__tests__/citation-utils.test.ts
index 99bb9737ec..226aad67e3 100644
--- a/web/src/utils/__tests__/citation-utils.test.ts
+++ b/web/src/utils/__tests__/citation-utils.test.ts
@@ -1,6 +1,8 @@
import type { IReferenceChunk } from '@/interfaces/database/chat';
+import reactStringReplace from 'react-string-replace';
import {
buildCitationDisplayMap,
+ citationWholeMarkerReg,
parseCitationIndex,
resolveCitationChunk,
} from '../citation-utils';
@@ -27,11 +29,30 @@ describe('parseCitationIndex', () => {
expect(parseCitationIndex('[3]')).toBe(3);
});
+ it('keeps short explicit numeric ids as positional indexes', () => {
+ expect(parseCitationIndex('[ID:0]')).toBe(0);
+ expect(parseCitationIndex('[ID:12]')).toBe(12);
+ expect(parseCitationIndex('[ID:123456789012345]')).toBe(123456789012345);
+ });
+
it('parses hex chunk-id markers as strings', () => {
expect(parseCitationIndex('[ID:a2d067519c9e0db1]')).toBe(
'a2d067519c9e0db1',
);
});
+
+ it('preserves all-digit hex ids instead of converting them', () => {
+ expect(parseCitationIndex('[ID:0000123456789012]')).toBe(
+ '0000123456789012',
+ );
+ expect(parseCitationIndex('[ID:1234567890123456]')).toBe(
+ '1234567890123456',
+ );
+ });
+
+ it('keeps short leading-zero ids as strings', () => {
+ expect(parseCitationIndex('[ID:0123]')).toBe('0123');
+ });
});
describe('resolveCitationChunk', () => {
@@ -79,6 +100,23 @@ describe('resolveCitationChunk', () => {
});
});
+describe('citationWholeMarkerReg', () => {
+ it('hands react-string-replace the whole marker, not the bare id', () => {
+ const seen: string[] = [];
+ reactStringReplace(
+ 'see [ID:0000123456789012] here',
+ citationWholeMarkerReg,
+ (match) => {
+ seen.push(match);
+ return match;
+ },
+ );
+
+ expect(seen).toEqual(['[ID:0000123456789012]']);
+ expect(parseCitationIndex(seen[0])).toBe('0000123456789012');
+ });
+});
+
describe('buildCitationDisplayMap', () => {
it('numbers unique markers by first occurrence, whatever their key', () => {
const map = buildCitationDisplayMap(
@@ -88,4 +126,12 @@ describe('buildCitationDisplayMap', () => {
expect(map.get('a2d067519c9e0db1')).toBe(1);
expect(map.size).toBe(2);
});
+
+ it('uses numeric keys for short explicit numeric ids', () => {
+ const map = buildCitationDisplayMap('[ID:0] [ID:1] [ID:0]');
+ expect([...map.entries()]).toEqual([
+ [0, 0],
+ [1, 1],
+ ]);
+ });
});
diff --git a/web/src/utils/citation-utils.ts b/web/src/utils/citation-utils.ts
index 10e0af20c4..d91391b09a 100644
--- a/web/src/utils/citation-utils.ts
+++ b/web/src/utils/citation-utils.ts
@@ -32,6 +32,17 @@ export const normalizeCitationDigits = (text: string) => {
export type CitationKey = number | string;
+// Knowledge-base citation markers are positional indices produced by
+// strconv.Itoa, so they stay numeric and small. Sixteen-digit hex agent IDs
+// exceed Number.MAX_SAFE_INTEGER and must keep their string form; leading-zero
+// values are opaque ids for the same reason.
+const maxNumericCitationIndexDigits = 15;
+
+const isNumericCitationIndex = (value: string) =>
+ /^\d+$/.test(value) &&
+ value.length <= maxNumericCitationIndexDigits &&
+ (value.length === 1 || value[0] !== '0');
+
export const parseCitationIndex = (value: string): CitationKey => {
const normalized = normalizeCitationDigits(value);
const markerMatch = normalized.match(
@@ -39,6 +50,9 @@ export const parseCitationIndex = (value: string): CitationKey => {
);
if (markerMatch) {
const captured = markerMatch[1];
+ if (/^\s*\[?ID:/i.test(normalized)) {
+ return isNumericCitationIndex(captured) ? Number(captured) : captured;
+ }
return /^\d+$/.test(captured) ? Number(captured) : captured;
}
if (/^\d+$/.test(normalized)) return Number(normalized);
@@ -48,15 +62,23 @@ export const parseCitationIndex = (value: string): CitationKey => {
export const citationMarkerReg =
/\[(?:ID:)?([0-9a-fA-F\u0660-\u0669\u06F0-\u06F9]+)\]/g;
+// react-string-replace hands the callback the last capture group, not the whole
+// match. citationMarkerReg groups only the id, so feeding it to
+// reactStringReplace would strip the brackets and turn an opaque all-digit id
+// into a positional index. This expression captures the whole marker instead;
+// the renderer can then pass it to parseCitationIndex unchanged.
+export const citationWholeMarkerReg =
+ /(\[(?:ID:)?[0-9a-fA-F\u0660-\u0669\u06F0-\u06F9]+\])/g;
+
// Resolve the chunk a citation marker points at. A numeric key is the
// chunk's position in the payload — the plain-chat contract, where the
// backend numbers each marker by its index into reference.chunks. A string
// key is the chunk id an agent answer embeds in its [ID:<chunkId>] markers;
// match it against the chunk's own id, never positionally: the payload
// ships in retrieval order, which is unrelated to the order the answer
-// cites passages in, so a positional read permutes the sources. A chunk id
-// is 16 hex chars — an all-digit id would parse as a position (~0.05% per
-// id), accepted and recorded here so it is not rediscovered as a mystery.
+// cites passages in, so a positional read permutes the sources. A 16-char
+// hex chunk id stays a string even when all its characters are digits;
+// parseCitationIndex only treats short decimal markers as positions.
export const resolveCitationChunk = (
chunks:
| IReferenceChunk[]