RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agent): cap Wikipedia extracts by runes instead of bytes (#19959)
Marcel Jiron · MarMar Labsb01ad151994b
diff --git a/internal/agent/tool/wikipedia.go b/internal/agent/tool/wikipedia.goindex 20651d0633..e4c0cd76c1 100644--- a/internal/agent/tool/wikipedia.go+++ b/internal/agent/tool/wikipedia.go@@ -264,9 +264,7 @@ func (w *WikipediaTool) InvokableRun(ctx context.Context, argsJSON string, _ ...if content == "" {continue}- if len(content) > 10000 {- content = content[:10000]- }+ content = truncateWikipediaRunes(content, 10000)fullURL := p.FullURLif fullURL == "" {fullURL = fmt.Sprintf("https://%s.wikipedia.org/wiki/%s", lang, url.PathEscape(p.Title))diff --git a/internal/agent/tool/wikipedia_test.go b/internal/agent/tool/wikipedia_test.goindex 0a140e2c62..5bf0ba39fb 100644--- a/internal/agent/tool/wikipedia_test.go+++ b/internal/agent/tool/wikipedia_test.go@@ -23,6 +23,7 @@ import ("net/url""strings""testing"+ "unicode/utf8""ragflow/internal/tokenizer")@@ -279,3 +280,87 @@ func TestWikipedia_BuildByNameIgnoresCanvasParams(t *testing.T) {t.Fatalf("node defaults = %d/%q", wikipedia.topN, wikipedia.lang)}}++// wikipediaExtractServer serves one generator=search page whose extract is the+// given string, pointed at by rewriteHostTransport.+func wikipediaExtractServer(t *testing.T, extract string) *WikipediaTool {+ t.Helper()+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"query":{"pages":{"1":{"index":1,"title":"胃癌","extract":"` ++ extract + `","fullurl":"https://zh.wikipedia.org/wiki/胃癌"}}}}`))+ }))+ t.Cleanup(srv.Close)+ helper := NewHTTPHelper().WithClient(&http.Client{Transport: rewriteHostTransport(srv.URL)})+ return NewWikipediaToolWithParams(helper, 10, "zh")+}++func wikipediaSingleResult(t *testing.T, tool *WikipediaTool) wikipediaEnvelope {+ t.Helper()+ out, err := tool.InvokableRun(t.Context(), `{"query":"胃癌"}`)+ if err != nil {+ t.Fatalf("InvokableRun: %v", err)+ }+ var env wikipediaEnvelope+ if jerr := json.Unmarshal([]byte(out), &env); jerr != nil {+ t.Fatalf("output is not valid JSON: %v", jerr)+ }+ if len(env.Results) != 1 {+ t.Fatalf("Results len = %d, want 1", len(env.Results))+ }+ return env+}++func TestWikipedia_CapsExtractOnRuneBoundary(t *testing.T) {+ t.Parallel()++ t.Run("under the limit the extract survives whole", func(t *testing.T) {+ t.Parallel()+ // 4000 three-byte runes. A 10000 byte cut would land inside a rune,+ // while a 10000 rune cap leaves the extract untouched.+ extract := strings.Repeat("中", 4000)+ env := wikipediaSingleResult(t, wikipediaExtractServer(t, extract))+ content := env.Results[0].Content++ if content != extract {+ t.Errorf("Content was altered below the cap. runes=%d bytes=%d, want the %d rune extract unchanged",+ utf8.RuneCountInString(content), len(content), utf8.RuneCountInString(extract))+ }+ if strings.Contains(content, "\uFFFD") {+ t.Errorf("Content carries U+FFFD. The cap cut a rune in half, leaving runes=%d bytes=%d.",+ utf8.RuneCountInString(content), len(content))+ }+ if !strings.Contains(env.FormalizedContent, extract) {+ t.Error("FormalizedContent does not carry the whole extract")+ }+ })++ t.Run("over the limit the extract is a bounded valid prefix", func(t *testing.T) {+ t.Parallel()+ // 12000 three-byte runes, well past the 10000 rune cap.+ extract := strings.Repeat("中", 12000)+ env := wikipediaSingleResult(t, wikipediaExtractServer(t, extract))+ content := env.Results[0].Content++ if content == "" {+ t.Fatal("Content is empty, want the capped extract")+ }+ if !strings.HasPrefix(extract, content) {+ t.Errorf("Content is not a prefix of the source extract. runes=%d bytes=%d",+ utf8.RuneCountInString(content), len(content))+ }+ if !utf8.ValidString(content) {+ t.Error("Content is not valid UTF-8")+ }+ if strings.Contains(content, "\uFFFD") {+ t.Errorf("Content carries U+FFFD. The cap cut a rune in half, leaving runes=%d bytes=%d.",+ utf8.RuneCountInString(content), len(content))+ }+ if got := utf8.RuneCountInString(content); got != 10000 {+ t.Errorf("Content rune count = %d, want exactly 10000", got)+ }+ if strings.Contains(env.FormalizedContent, "\uFFFD") {+ t.Error("FormalizedContent carries U+FFFD")+ }+ })+}