Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(web-search): bound provider response bodies instead of buffering them whole (#4370)
ordinary beingc44f939211bb
diff --git a/internal/infrastructure/web_search/bing.go b/internal/infrastructure/web_search/bing.goindex b5ba35b48..fecc4bcf9 100644--- a/internal/infrastructure/web_search/bing.go+++ b/internal/infrastructure/web_search/bing.go@@ -4,7 +4,6 @@ import ("context""encoding/json""fmt"- "io""net/http""net/url""strconv"@@ -101,14 +100,15 @@ func (p *BingProvider) doSearch(ctx context.Context, req *http.Request) ([]*type}defer resp.Body.Close()- body, err := io.ReadAll(resp.Body)- if err != nil {- return nil, err+ if resp.StatusCode != http.StatusOK {+ preview := readErrorPreview(resp.Body)+ logger.Warnf(ctx, "[WebSearch][Bing] API returned status %d: %s", resp.StatusCode, preview)+ return nil, fmt.Errorf("bing API returned status %d: %s", resp.StatusCode, preview)}- if resp.StatusCode != http.StatusOK {- logger.Warnf(ctx, "[WebSearch][Bing] API returned status %d: %s", resp.StatusCode, string(body))- return nil, fmt.Errorf("bing API returned status %d: %s", resp.StatusCode, string(body))+ body, err := readCapped(resp.Body, 0)+ if err != nil {+ return nil, err}var respData bingSearchResponsediff --git a/internal/infrastructure/web_search/duckduckgo.go b/internal/infrastructure/web_search/duckduckgo.goindex 0eaa8db28..62f7e9010 100644--- a/internal/infrastructure/web_search/duckduckgo.go+++ b/internal/infrastructure/web_search/duckduckgo.go@@ -4,7 +4,6 @@ import ("context""encoding/json""fmt"- "io""net/http""net/url""strings"@@ -157,8 +156,13 @@ func (p *DuckDuckGoProvider) searchAPI(}defer resp.Body.Close()if resp.StatusCode != http.StatusOK {- body, _ := io.ReadAll(resp.Body)- return nil, fmt.Errorf("duckduckgo API returned status %d: %s", resp.StatusCode, string(body))+ preview := readErrorPreview(resp.Body)+ return nil, fmt.Errorf("duckduckgo API returned status %d: %s", resp.StatusCode, preview)+ }++ bodyBytes, err := readCapped(resp.Body, 0)+ if err != nil {+ return nil, fmt.Errorf("failed to read response: %w", err)}var apiResponse struct {@@ -174,7 +178,7 @@ func (p *DuckDuckGoProvider) searchAPI(Text string `json:"Text"`} `json:"Results"`}- if err := json.NewDecoder(resp.Body).Decode(&apiResponse); err != nil {+ if err := json.Unmarshal(bodyBytes, &apiResponse); err != nil {return nil, fmt.Errorf("failed to decode API response: %w", err)}diff --git a/internal/infrastructure/web_search/keenable.go b/internal/infrastructure/web_search/keenable.goindex fa01eb209..d4762cbc5 100644--- a/internal/infrastructure/web_search/keenable.go+++ b/internal/infrastructure/web_search/keenable.go@@ -5,7 +5,6 @@ import ("context""encoding/json""fmt"- "io""net/http""time"@@ -102,12 +101,12 @@ func (p *KeenableProvider) Search(defer resp.Body.Close()if resp.StatusCode != http.StatusOK {- respBody, _ := io.ReadAll(resp.Body)- logger.Warnf(ctx, "[WebSearch][Keenable] API returned status %d: %s", resp.StatusCode, string(respBody))- return nil, fmt.Errorf("keenable API returned status %d: %s", resp.StatusCode, string(respBody))+ preview := readErrorPreview(resp.Body)+ logger.Warnf(ctx, "[WebSearch][Keenable] API returned status %d: %s", resp.StatusCode, preview)+ return nil, fmt.Errorf("keenable API returned status %d: %s", resp.StatusCode, preview)}- respBody, err := io.ReadAll(resp.Body)+ respBody, err := readCapped(resp.Body, 0)if err != nil {return nil, fmt.Errorf("failed to read response: %w", err)}diff --git a/internal/infrastructure/web_search/ollama.go b/internal/infrastructure/web_search/ollama.goindex b4e3fa8fe..551204156 100644--- a/internal/infrastructure/web_search/ollama.go+++ b/internal/infrastructure/web_search/ollama.go@@ -5,7 +5,6 @@ import ("context""encoding/json""fmt"- "io""net/http""time"@@ -115,14 +114,15 @@ func (p *OllamaProvider) doSearch(ctx context.Context, req *http.Request) ([]*ty}defer resp.Body.Close()- body, err := io.ReadAll(resp.Body)- if err != nil {- return nil, err+ if resp.StatusCode != http.StatusOK {+ preview := readErrorPreview(resp.Body)+ logger.Warnf(ctx, "[WebSearch][Ollama] API returned status %d: %s", resp.StatusCode, preview)+ return nil, fmt.Errorf("ollama API returned status %d: %s", resp.StatusCode, preview)}- if resp.StatusCode != http.StatusOK {- logger.Warnf(ctx, "[WebSearch][Ollama] API returned status %d: %s", resp.StatusCode, string(body))- return nil, fmt.Errorf("ollama API returned status %d: %s", resp.StatusCode, string(body))+ body, err := readCapped(resp.Body, 0)+ if err != nil {+ return nil, err}var respData ollamaSearchResponsediff --git a/internal/infrastructure/web_search/response_limit.go b/internal/infrastructure/web_search/response_limit.gonew file mode 100644index 000000000..1833dfe18--- /dev/null+++ b/internal/infrastructure/web_search/response_limit.go@@ -0,0 +1,51 @@+package web_search++import (+ "errors"+ "fmt"+ "io"+)++// maxJSONResponseBytes bounds every provider response body. A search result set+// is JSON metadata over a page of hits; a larger body means a broken or+// misconfigured endpoint, not more results.+const maxJSONResponseBytes = 16 << 20++// maxErrorPreviewBytes bounds how much of a failed response is echoed into a log+// line or an error string. Without it a failing endpoint can inject an unbounded+// blob into both.+const maxErrorPreviewBytes = 4 << 10++// errResponseTooLarge marks a body over the cap. It is deterministic: the same+// request returns the same oversized body, so callers must not retry it.+var errResponseTooLarge = errors.New("response exceeds maximum size")++// readCapped reads a response body, refusing anything larger than limit instead+// of buffering it. Oversized payloads are reported as an error: a truncated body+// would otherwise be parsed as if it were the whole response. A non-positive+// limit (e.g. a provider built without one) falls back to maxJSONResponseBytes.+//+// Mirrors the cap the datasource connectors already apply to their API bodies.+func readCapped(body io.Reader, limit int64) ([]byte, error) {+ if limit <= 0 {+ limit = maxJSONResponseBytes+ }+ data, err := io.ReadAll(io.LimitReader(body, limit+1))+ if err != nil {+ return nil, err+ }+ if int64(len(data)) > limit {+ return nil, fmt.Errorf("%w (%d bytes)", errResponseTooLarge, limit)+ }+ return data, nil+}++// readErrorPreview reads just enough of a failed body to put in a log line or an+// error message. It truncates rather than refusing: a preview is diagnostic, and+// the interesting part of a large error body is its start. Read errors are+// dropped because the caller is already on the error path and is about to report+// the status code, which is the useful part.+func readErrorPreview(body io.Reader) string {+ data, _ := io.ReadAll(io.LimitReader(body, maxErrorPreviewBytes))+ return string(data)+}diff --git a/internal/infrastructure/web_search/response_limit_test.go b/internal/infrastructure/web_search/response_limit_test.gonew file mode 100644index 000000000..22c626477--- /dev/null+++ b/internal/infrastructure/web_search/response_limit_test.go@@ -0,0 +1,143 @@+package web_search++import (+ "context"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+)++// fillerReader streams size bytes without allocating them, so a test can hand an+// endpoint a body far larger than any cap without spending the memory itself.+type fillerReader struct {+ remaining int64+}++func (f *fillerReader) Read(p []byte) (int, error) {+ if f.remaining <= 0 {+ return 0, io.EOF+ }+ if int64(len(p)) > f.remaining {+ p = p[:f.remaining]+ }+ for i := range p {+ p[i] = 'a'+ }+ f.remaining -= int64(len(p))+ return len(p), nil+}++// readCapped must refuse an oversized body rather than buffering it, and must+// still pass a body at exactly the cap through untouched.+func TestReadCapped(t *testing.T) {+ t.Run("rejects over the cap", func(t *testing.T) {+ const limit = 1 << 10+ _, err := readCapped(&fillerReader{remaining: limit + 1}, limit)+ if err == nil {+ t.Fatal("expected an over-cap body to be refused")+ }+ if !errors.Is(err, errResponseTooLarge) {+ t.Errorf("err = %v, want errResponseTooLarge", err)+ }+ if !strings.Contains(err.Error(), "exceeds maximum size") {+ t.Errorf("error text = %q, want the canonical over-size message", err)+ }+ })++ t.Run("accepts exactly the cap", func(t *testing.T) {+ const limit = 1 << 10+ got, err := readCapped(strings.NewReader(strings.Repeat("a", limit)), limit)+ if err != nil {+ t.Fatalf("readCapped() = %v, want nil at exactly the cap", err)+ }+ if len(got) != limit {+ t.Errorf("len = %d, want %d", len(got), limit)+ }+ })++ t.Run("non-positive limit falls back to the production cap", func(t *testing.T) {+ got, err := readCapped(strings.NewReader("{}"), 0)+ if err != nil {+ t.Fatalf("readCapped(_, 0) = %v, want nil", err)+ }+ if string(got) != "{}" {+ t.Errorf("body = %q, want %q", got, "{}")+ }+ })+}++// An error preview ends up in a log line and in the error the caller sees, so it+// has to be bounded too. searxng already capped its preview; its siblings did not.+func TestReadErrorPreviewIsBounded(t *testing.T) {+ preview := readErrorPreview(&fillerReader{remaining: 1 << 20})+ if len(preview) != maxErrorPreviewBytes {+ t.Errorf("preview is %d bytes, want %d", len(preview), maxErrorPreviewBytes)+ }+ if preview == "" {+ t.Error("preview is empty; a truncated body is more useful than none")+ }+}++// doSearch reads the response itself, so these two drive the real call sites.+// Bing is the one that used to read the whole body before it even looked at the+// status code.+func TestBingProviderRejectsOversizedSuccessBody(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusOK)+ // Valid JSON prefix, then streamed past the cap without the test ever+ // allocating it. Only the size guard can reject this body: a parse error+ // instead would mean the body was capped but then still parsed.+ _, _ = w.Write([]byte(`{"webPages":{"value":[`))+ _, _ = io.Copy(w, &fillerReader{remaining: maxJSONResponseBytes + (1 << 20)})+ }))+ defer srv.Close()++ p := &BingProvider{client: srv.Client()}+ req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, srv.URL, nil)+ if err != nil {+ t.Fatal(err)+ }++ results, err := p.doSearch(context.Background(), req)+ if err == nil {+ t.Fatalf("expected an oversized body to be refused, got %d results", len(results))+ }+ if !errors.Is(err, errResponseTooLarge) {+ t.Errorf("err = %v, want errResponseTooLarge", err)+ }+ if results != nil {+ t.Errorf("results = %v, want nil on a refused body", results)+ }+}++// Before the fix this error carried the entire failing body, so a broken or+// hostile endpoint could push an unbounded blob into the caller's error string+// and into the log line alongside it.+func TestBingProviderBoundsFailedResponseInError(t *testing.T) {+ const bodySize = 1 << 20+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusBadGateway)+ _, _ = io.Copy(w, &fillerReader{remaining: bodySize})+ }))+ defer srv.Close()++ p := &BingProvider{client: srv.Client()}+ req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, srv.URL, nil)+ if err != nil {+ t.Fatal(err)+ }++ _, err = p.doSearch(context.Background(), req)+ if err == nil {+ t.Fatal("expected an error for a failing status")+ }+ // Room for the fixed prefix ("bing API returned status 502: ") on top of the cap.+ if limit := len("bing API returned status 502: ") + maxErrorPreviewBytes; len(err.Error()) > limit {+ t.Errorf("error is %d bytes for a %d byte body, want at most %d: the failed "++ "response is being echoed unbounded", len(err.Error()), bodySize, limit)+ }+}diff --git a/internal/infrastructure/web_search/searxng.go b/internal/infrastructure/web_search/searxng.goindex 7e57c7383..d406c0cfe 100644--- a/internal/infrastructure/web_search/searxng.go+++ b/internal/infrastructure/web_search/searxng.go@@ -4,7 +4,6 @@ import ("context""encoding/json""fmt"- "io""net/http""net/url""strings"@@ -128,12 +127,16 @@ func (p *SearxngProvider) Search(defer resp.Body.Close()if resp.StatusCode != http.StatusOK {- body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))- return nil, fmt.Errorf("searxng returned status %d: %s", resp.StatusCode, string(body))+ return nil, fmt.Errorf("searxng returned status %d: %s", resp.StatusCode, readErrorPreview(resp.Body))+ }++ bodyBytes, err := readCapped(resp.Body, 0)+ if err != nil {+ return nil, fmt.Errorf("failed to read response: %w", err)}var data searxngResponse- if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {+ if err := json.Unmarshal(bodyBytes, &data); err != nil {p.lastUnresponsive = nilreturn nil, fmt.Errorf("failed to decode SearXNG response (ensure JSON format is enabled in settings.yml): %w", err)}diff --git a/internal/infrastructure/web_search/tavily.go b/internal/infrastructure/web_search/tavily.goindex 4a1e35264..eb6e1efd4 100644--- a/internal/infrastructure/web_search/tavily.go+++ b/internal/infrastructure/web_search/tavily.go@@ -5,7 +5,6 @@ import ("context""encoding/json""fmt"- "io""net/http""time"@@ -88,12 +87,12 @@ func (p *TavilyProvider) Search(defer resp.Body.Close()if resp.StatusCode != http.StatusOK {- respBody, _ := io.ReadAll(resp.Body)- logger.Warnf(ctx, "[WebSearch][Tavily] API returned status %d: %s", resp.StatusCode, string(respBody))- return nil, fmt.Errorf("tavily API returned status %d: %s", resp.StatusCode, string(respBody))+ preview := readErrorPreview(resp.Body)+ logger.Warnf(ctx, "[WebSearch][Tavily] API returned status %d: %s", resp.StatusCode, preview)+ return nil, fmt.Errorf("tavily API returned status %d: %s", resp.StatusCode, preview)}- respBody, err := io.ReadAll(resp.Body)+ respBody, err := readCapped(resp.Body, 0)if err != nil {return nil, fmt.Errorf("failed to read response: %w", err)}