RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(tool): cap tool HTTP response bodies at 4 MiB in arxiv, querit, duckduckgo and yahoo_finance (#21769)
ciel1f862d697bd1
diff --git a/internal/agent/tool/arxiv.go b/internal/agent/tool/arxiv.goindex b493b30cb1..e86b5434c8 100644--- a/internal/agent/tool/arxiv.go+++ b/internal/agent/tool/arxiv.go@@ -45,6 +45,8 @@ const defaultArxivSortBy = "submittedDate"const arxivPromptMaxTokens = 200000+const arxivMaxResponseBytes = 4 << 20+var arxivDataImagePattern = regexp.MustCompile(`!?\[[a-z]+\]\(data:image/png;base64,[ 0-9A-Za-z/_=+\-]+\)`)var arxivNewlinePattern = regexp.MustCompile(`\n+`)@@ -278,11 +280,15 @@ func (a *ArxivTool) InvokableRun(ctx context.Context, argsJSON string, _ ...toolfmt.Errorf("arxiv: upstream returned %d", resp.StatusCode)}- body, err := io.ReadAll(resp.Body)+ body, err := io.ReadAll(io.LimitReader(resp.Body, arxivMaxResponseBytes+1))if err != nil {return arxivErrJSON(fmt.Errorf("arxiv: read body: %w", err)),fmt.Errorf("arxiv: read body: %w", err)}+ if len(body) > arxivMaxResponseBytes {+ return arxivErrJSON(fmt.Errorf("arxiv: response too large")),+ fmt.Errorf("arxiv: response too large")+ }results, err := parseArxivAtom(body)if err != nil {diff --git a/internal/agent/tool/duckduckgo.go b/internal/agent/tool/duckduckgo.goindex 3923ebdf1f..87e3791a51 100644--- a/internal/agent/tool/duckduckgo.go+++ b/internal/agent/tool/duckduckgo.go@@ -17,6 +17,7 @@package toolimport (+ "bytes""context""encoding/json""fmt"@@ -45,6 +46,8 @@ const duckduckgoChannelNews = "news"const duckduckgoPromptMaxTokens = 200000+const duckduckgoMaxResponseBytes = 4 << 20+var duckduckgoSearchEndpoint = "https://duckduckgo.com/html/"var duckduckgoNewsEndpoint = "https://duckduckgo.com/news.js"var duckduckgoNewsBootstrapEndpoint = "https://duckduckgo.com/"@@ -230,7 +233,17 @@ func (d *DuckDuckGoTool) InvokableRun(ctx context.Context, argsJSON string, _ ..fmt.Errorf("duckduckgo: upstream returned %d", resp.StatusCode)}- results, err := parseDuckDuckGoHTML(resp.Body, topN)+ body, err := io.ReadAll(io.LimitReader(resp.Body, duckduckgoMaxResponseBytes+1))+ if err != nil {+ return duckduckgoErrJSON(fmt.Errorf("duckduckgo: read body: %w", err)),+ fmt.Errorf("duckduckgo: read body: %w", err)+ }+ if len(body) > duckduckgoMaxResponseBytes {+ return duckduckgoErrJSON(fmt.Errorf("duckduckgo: response too large")),+ fmt.Errorf("duckduckgo: response too large")+ }++ results, err := parseDuckDuckGoHTML(bytes.NewReader(body), topN)if err != nil {return duckduckgoErrJSON(fmt.Errorf("duckduckgo: parse html: %w", err)),fmt.Errorf("duckduckgo: parse html: %w", err)@@ -354,8 +367,18 @@ func (d *DuckDuckGoTool) runNewsSearch(ctx context.Context, query string, topN ifmt.Errorf("duckduckgo: upstream returned %d", resp.StatusCode)}+ body, err := io.ReadAll(io.LimitReader(resp.Body, duckduckgoMaxResponseBytes+1))+ if err != nil {+ return duckduckgoErrJSON(fmt.Errorf("duckduckgo: read news response: %w", err)),+ fmt.Errorf("duckduckgo: read news response: %w", err)+ }+ if len(body) > duckduckgoMaxResponseBytes {+ return duckduckgoErrJSON(fmt.Errorf("duckduckgo: news response too large")),+ fmt.Errorf("duckduckgo: news response too large")+ }+var raw duckduckgoNewsResponse- if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil {+ if err := json.Unmarshal(body, &raw); err != nil {return duckduckgoErrJSON(fmt.Errorf("duckduckgo: decode news response: %w", err)),fmt.Errorf("duckduckgo: decode news response: %w", err)}@@ -398,10 +421,13 @@ func (d *DuckDuckGoTool) fetchDuckDuckGoNewsVQD(ctx context.Context, query strinreturn "", fmt.Errorf("duckduckgo: news bootstrap returned %d", resp.StatusCode)}- body, err := io.ReadAll(resp.Body)+ body, err := io.ReadAll(io.LimitReader(resp.Body, duckduckgoMaxResponseBytes+1))if err != nil {return "", fmt.Errorf("duckduckgo: read news bootstrap: %w", err)}+ if len(body) > duckduckgoMaxResponseBytes {+ return "", fmt.Errorf("duckduckgo: news bootstrap response too large")+ }matches := duckduckgoVQDPattern.FindSubmatch(body)if len(matches) != 2 {return "", fmt.Errorf("duckduckgo: news bootstrap missing vqd")diff --git a/internal/agent/tool/querit.go b/internal/agent/tool/querit.goindex f520f79e8c..6898ae11f8 100644--- a/internal/agent/tool/querit.go+++ b/internal/agent/tool/querit.go@@ -34,11 +34,12 @@ import ()const (- queritToolName = "querit_search"- queritToolDescription = "Search the web via the Querit API and return the complete search response."- queritEndpoint = "https://api.querit.ai/v1/search"- queritMaxAttempts = 3- queritPromptMaxTokens = 200000+ queritToolName = "querit_search"+ queritToolDescription = "Search the web via the Querit API and return the complete search response."+ queritEndpoint = "https://api.querit.ai/v1/search"+ queritMaxAttempts = 3+ queritPromptMaxTokens = 200000+ queritMaxResponseBytes = 4 << 20)var (@@ -289,11 +290,14 @@ func doQueritRequest(return nil, fmt.Errorf("upstream returned %d", resp.StatusCode)}- raw, readErr := io.ReadAll(resp.Body)+ raw, readErr := io.ReadAll(io.LimitReader(resp.Body, queritMaxResponseBytes+1))_ = resp.Body.Close()if readErr != nil {return nil, fmt.Errorf("read response: %w", readErr)}+ if len(raw) > queritMaxResponseBytes {+ return nil, fmt.Errorf("response too large")+ }return raw, nil}return nil, fmt.Errorf("request exhausted retries")diff --git a/internal/agent/tool/yahoo_finance.go b/internal/agent/tool/yahoo_finance.goindex c550bfa2a0..ee2c737ea9 100644--- a/internal/agent/tool/yahoo_finance.go+++ b/internal/agent/tool/yahoo_finance.go@@ -37,6 +37,8 @@ const yahooFinanceToolDescription = "The Yahoo Finance service provides access tconst yahooFinanceStockCodeDescription = "The stock code or company name."+const yahooFinanceMaxResponseBytes = 4 << 20+const (yahooFinanceInfoDescription = "Fetch stock information."yahooFinanceHistoryDescription = "Fetch historical market data."@@ -435,7 +437,14 @@ func (y *YahooFinanceTool) fetchYahooFinanceJSONWithHeaders(ctx context.Context,if resp.StatusCode < 200 || resp.StatusCode >= 300 {return fmt.Errorf("yahoo_finance: upstream returned %d", resp.StatusCode)}- if err := json.NewDecoder(resp.Body).Decode(target); err != nil {+ body, err := io.ReadAll(io.LimitReader(resp.Body, yahooFinanceMaxResponseBytes+1))+ if err != nil {+ return fmt.Errorf("yahoo_finance: read response: %w", err)+ }+ if len(body) > yahooFinanceMaxResponseBytes {+ return fmt.Errorf("yahoo_finance: response too large")+ }+ if err := json.Unmarshal(body, target); err != nil {return fmt.Errorf("yahoo_finance: decode response: %w", err)}return nil@@ -466,10 +475,13 @@ func (y *YahooFinanceTool) fetchYahooFinanceCrumb(ctx context.Context) (string,if resp.StatusCode < 200 || resp.StatusCode >= 300 {return "", "", fmt.Errorf("yahoo_finance: crumb endpoint returned %d", resp.StatusCode)}- body, err := io.ReadAll(resp.Body)+ body, err := io.ReadAll(io.LimitReader(resp.Body, yahooFinanceMaxResponseBytes+1))if err != nil {return "", "", fmt.Errorf("yahoo_finance: read crumb: %w", err)}+ if len(body) > yahooFinanceMaxResponseBytes {+ return "", "", fmt.Errorf("yahoo_finance: crumb response too large")+ }crumb := strings.TrimSpace(string(body))if crumb == "" {return "", "", fmt.Errorf("yahoo_finance: empty crumb")