Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(tool): cap tool HTTP response bodies at 4 MiB in arxiv, querit, duckduckgo and yahoo_finance (#21769)

ciel1f862d697bd1
diff --git a/internal/agent/tool/arxiv.go b/internal/agent/tool/arxiv.go
index b493b30cb1..e86b5434c8 100644
--- a/internal/agent/tool/arxiv.go
+++ b/internal/agent/tool/arxiv.go
@@ -45,6 +45,8 @@ const defaultArxivSortBy = "submittedDate"
const arxivPromptMaxTokens = 200000
+const arxivMaxResponseBytes = 4 << 20
+
var arxivDataImagePattern = regexp.MustCompile(`!?\[[a-z]+\]\(data:image/png;base64,[ 0-9A-Za-z/_=+\-]+\)`)
var arxivNewlinePattern = regexp.MustCompile(`\n+`)
@@ -278,11 +280,15 @@ func (a *ArxivTool) InvokableRun(ctx context.Context, argsJSON string, _ ...tool
fmt.Errorf("arxiv: upstream returned %d", resp.StatusCode)
}
- body, err := io.ReadAll(resp.Body)
+ body, err := io.ReadAll(io.LimitReader(resp.Body, arxivMaxResponseBytes+1))
if err != nil {
return arxivErrJSON(fmt.Errorf("arxiv: read body: %w", err)),
fmt.Errorf("arxiv: read body: %w", err)
}
+ if len(body) > arxivMaxResponseBytes {
+ return arxivErrJSON(fmt.Errorf("arxiv: response too large")),
+ fmt.Errorf("arxiv: response too large")
+ }
results, err := parseArxivAtom(body)
if err != nil {
diff --git a/internal/agent/tool/duckduckgo.go b/internal/agent/tool/duckduckgo.go
index 3923ebdf1f..87e3791a51 100644
--- a/internal/agent/tool/duckduckgo.go
+++ b/internal/agent/tool/duckduckgo.go
@@ -17,6 +17,7 @@
package tool
import (
+ "bytes"
"context"
"encoding/json"
"fmt"
@@ -45,6 +46,8 @@ const duckduckgoChannelNews = "news"
const duckduckgoPromptMaxTokens = 200000
+const duckduckgoMaxResponseBytes = 4 << 20
+
var duckduckgoSearchEndpoint = "https://duckduckgo.com/html/"
var duckduckgoNewsEndpoint = "https://duckduckgo.com/news.js"
var duckduckgoNewsBootstrapEndpoint = "https://duckduckgo.com/"
@@ -230,7 +233,17 @@ func (d *DuckDuckGoTool) InvokableRun(ctx context.Context, argsJSON string, _ ..
fmt.Errorf("duckduckgo: upstream returned %d", resp.StatusCode)
}
- results, err := parseDuckDuckGoHTML(resp.Body, topN)
+ body, err := io.ReadAll(io.LimitReader(resp.Body, duckduckgoMaxResponseBytes+1))
+ if err != nil {
+ return duckduckgoErrJSON(fmt.Errorf("duckduckgo: read body: %w", err)),
+ fmt.Errorf("duckduckgo: read body: %w", err)
+ }
+ if len(body) > duckduckgoMaxResponseBytes {
+ return duckduckgoErrJSON(fmt.Errorf("duckduckgo: response too large")),
+ fmt.Errorf("duckduckgo: response too large")
+ }
+
+ results, err := parseDuckDuckGoHTML(bytes.NewReader(body), topN)
if err != nil {
return duckduckgoErrJSON(fmt.Errorf("duckduckgo: parse html: %w", err)),
fmt.Errorf("duckduckgo: parse html: %w", err)
@@ -354,8 +367,18 @@ func (d *DuckDuckGoTool) runNewsSearch(ctx context.Context, query string, topN i
fmt.Errorf("duckduckgo: upstream returned %d", resp.StatusCode)
}
+ body, err := io.ReadAll(io.LimitReader(resp.Body, duckduckgoMaxResponseBytes+1))
+ if err != nil {
+ return duckduckgoErrJSON(fmt.Errorf("duckduckgo: read news response: %w", err)),
+ fmt.Errorf("duckduckgo: read news response: %w", err)
+ }
+ if len(body) > duckduckgoMaxResponseBytes {
+ return duckduckgoErrJSON(fmt.Errorf("duckduckgo: news response too large")),
+ fmt.Errorf("duckduckgo: news response too large")
+ }
+
var raw duckduckgoNewsResponse
- if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil {
+ if err := json.Unmarshal(body, &raw); err != nil {
return duckduckgoErrJSON(fmt.Errorf("duckduckgo: decode news response: %w", err)),
fmt.Errorf("duckduckgo: decode news response: %w", err)
}
@@ -398,10 +421,13 @@ func (d *DuckDuckGoTool) fetchDuckDuckGoNewsVQD(ctx context.Context, query strin
return "", fmt.Errorf("duckduckgo: news bootstrap returned %d", resp.StatusCode)
}
- body, err := io.ReadAll(resp.Body)
+ body, err := io.ReadAll(io.LimitReader(resp.Body, duckduckgoMaxResponseBytes+1))
if err != nil {
return "", fmt.Errorf("duckduckgo: read news bootstrap: %w", err)
}
+ if len(body) > duckduckgoMaxResponseBytes {
+ return "", fmt.Errorf("duckduckgo: news bootstrap response too large")
+ }
matches := duckduckgoVQDPattern.FindSubmatch(body)
if len(matches) != 2 {
return "", fmt.Errorf("duckduckgo: news bootstrap missing vqd")
diff --git a/internal/agent/tool/querit.go b/internal/agent/tool/querit.go
index f520f79e8c..6898ae11f8 100644
--- a/internal/agent/tool/querit.go
+++ b/internal/agent/tool/querit.go
@@ -34,11 +34,12 @@ import (
)
const (
- queritToolName = "querit_search"
- queritToolDescription = "Search the web via the Querit API and return the complete search response."
- queritEndpoint = "https://api.querit.ai/v1/search"
- queritMaxAttempts = 3
- queritPromptMaxTokens = 200000
+ queritToolName = "querit_search"
+ queritToolDescription = "Search the web via the Querit API and return the complete search response."
+ queritEndpoint = "https://api.querit.ai/v1/search"
+ queritMaxAttempts = 3
+ queritPromptMaxTokens = 200000
+ queritMaxResponseBytes = 4 << 20
)
var (
@@ -289,11 +290,14 @@ func doQueritRequest(
return nil, fmt.Errorf("upstream returned %d", resp.StatusCode)
}
- raw, readErr := io.ReadAll(resp.Body)
+ raw, readErr := io.ReadAll(io.LimitReader(resp.Body, queritMaxResponseBytes+1))
_ = resp.Body.Close()
if readErr != nil {
return nil, fmt.Errorf("read response: %w", readErr)
}
+ if len(raw) > queritMaxResponseBytes {
+ return nil, fmt.Errorf("response too large")
+ }
return raw, nil
}
return nil, fmt.Errorf("request exhausted retries")
diff --git a/internal/agent/tool/yahoo_finance.go b/internal/agent/tool/yahoo_finance.go
index c550bfa2a0..ee2c737ea9 100644
--- a/internal/agent/tool/yahoo_finance.go
+++ b/internal/agent/tool/yahoo_finance.go
@@ -37,6 +37,8 @@ const yahooFinanceToolDescription = "The Yahoo Finance service provides access t
const yahooFinanceStockCodeDescription = "The stock code or company name."
+const yahooFinanceMaxResponseBytes = 4 << 20
+
const (
yahooFinanceInfoDescription = "Fetch stock information."
yahooFinanceHistoryDescription = "Fetch historical market data."
@@ -435,7 +437,14 @@ func (y *YahooFinanceTool) fetchYahooFinanceJSONWithHeaders(ctx context.Context,
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("yahoo_finance: upstream returned %d", resp.StatusCode)
}
- if err := json.NewDecoder(resp.Body).Decode(target); err != nil {
+ body, err := io.ReadAll(io.LimitReader(resp.Body, yahooFinanceMaxResponseBytes+1))
+ if err != nil {
+ return fmt.Errorf("yahoo_finance: read response: %w", err)
+ }
+ if len(body) > yahooFinanceMaxResponseBytes {
+ return fmt.Errorf("yahoo_finance: response too large")
+ }
+ if err := json.Unmarshal(body, target); err != nil {
return fmt.Errorf("yahoo_finance: decode response: %w", err)
}
return nil
@@ -466,10 +475,13 @@ func (y *YahooFinanceTool) fetchYahooFinanceCrumb(ctx context.Context) (string,
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return "", "", fmt.Errorf("yahoo_finance: crumb endpoint returned %d", resp.StatusCode)
}
- body, err := io.ReadAll(resp.Body)
+ body, err := io.ReadAll(io.LimitReader(resp.Body, yahooFinanceMaxResponseBytes+1))
if err != nil {
return "", "", fmt.Errorf("yahoo_finance: read crumb: %w", err)
}
+ if len(body) > yahooFinanceMaxResponseBytes {
+ return "", "", fmt.Errorf("yahoo_finance: crumb response too large")
+ }
crumb := strings.TrimSpace(string(body))
if crumb == "" {
return "", "", fmt.Errorf("yahoo_finance: empty crumb")