RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agent): bound crawler response bodies to 4 MiB (#18677)
Lem0nTea2002bb488db9d474
diff --git a/internal/agent/tool/crawler.go b/internal/agent/tool/crawler.goindex 21db1c6709..3682647e4c 100644--- a/internal/agent/tool/crawler.go+++ b/internal/agent/tool/crawler.go@@ -35,6 +35,8 @@ const crawlerToolName = "web_crawler"const crawlerToolDescription = "Crawls a web page and returns its extracted text content and links."+const maxCrawlerResponseBytes = 4 << 20+// crawlerArgs is the JSON shape the model sends in. query is the// Python-compatible argument name; url is accepted for older Go// callers. max_depth and max_pages are accepted for API symmetry with@@ -205,11 +207,15 @@ func (c *CrawlerTool) InvokableRun(ctx context.Context, argumentsInJSON string,}defer resp.Body.Close()- body, err := io.ReadAll(resp.Body)+ body, err := io.ReadAll(io.LimitReader(resp.Body, maxCrawlerResponseBytes+1))if err != nil {return crawlerStubResult(crawlerResult{URL: targetURL, Status: resp.StatusCode, Error: "read body: " + err.Error()}),fmt.Errorf("crawler: read body: %w", err)}+ if len(body) > maxCrawlerResponseBytes {+ err = errors.New("crawler: response too large")+ return crawlerStubResult(crawlerResult{URL: targetURL, Status: resp.StatusCode, Error: err.Error()}), err+ }page, err := extractPage(body)if err != nil {diff --git a/internal/agent/tool/crawler_test.go b/internal/agent/tool/crawler_test.goindex f263ecace8..5875432deb 100644--- a/internal/agent/tool/crawler_test.go+++ b/internal/agent/tool/crawler_test.go@@ -103,6 +103,38 @@ func TestCrawler_FetchesAndExtractsText(t *testing.T) {}}+func TestCrawler_RejectsOversizedResponse(t *testing.T) {+ t.Parallel()+ ctx := t.Context()++ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = w.Write([]byte(strings.Repeat("x", maxCrawlerResponseBytes+1)))+ }))+ defer srv.Close()++ loopbackResolver := func(rawURL string) (string, net.IP, error) {+ u, err := url.Parse(rawURL)+ if err != nil {+ return "", nil, err+ }+ host := u.Hostname()+ return host, net.ParseIP(host), nil+ }+ c := NewCrawlerTool().WithResolver(loopbackResolver)+ out, err := c.InvokableRun(ctx, `{"query":`+jsonString(srv.URL)+`}`)+ if err == nil || !strings.Contains(err.Error(), "response too large") {+ t.Fatalf("err = %v, want response too large", err)+ }++ var got crawlerResult+ if jerr := json.Unmarshal([]byte(out), &got); jerr != nil {+ t.Fatalf("output is not valid JSON: %v (raw=%s)", jerr, out)+ }+ if !strings.Contains(got.Error, "response too large") {+ t.Fatalf("_ERROR = %q, want response too large", got.Error)+ }+}+func TestCrawler_RejectsMaxDepthGreaterThanZero(t *testing.T) {t.Parallel()ctx := t.Context()