Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(web-search): bound provider response bodies instead of buffering them whole (#4370)

ordinary beingc44f939211bb
diff --git a/internal/infrastructure/web_search/bing.go b/internal/infrastructure/web_search/bing.go
index b5ba35b48..fecc4bcf9 100644
--- a/internal/infrastructure/web_search/bing.go
+++ b/internal/infrastructure/web_search/bing.go
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
- "io"
"net/http"
"net/url"
"strconv"
@@ -101,14 +100,15 @@ func (p *BingProvider) doSearch(ctx context.Context, req *http.Request) ([]*type
}
defer resp.Body.Close()
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, err
+ if resp.StatusCode != http.StatusOK {
+ preview := readErrorPreview(resp.Body)
+ logger.Warnf(ctx, "[WebSearch][Bing] API returned status %d: %s", resp.StatusCode, preview)
+ return nil, fmt.Errorf("bing API returned status %d: %s", resp.StatusCode, preview)
}
- if resp.StatusCode != http.StatusOK {
- logger.Warnf(ctx, "[WebSearch][Bing] API returned status %d: %s", resp.StatusCode, string(body))
- return nil, fmt.Errorf("bing API returned status %d: %s", resp.StatusCode, string(body))
+ body, err := readCapped(resp.Body, 0)
+ if err != nil {
+ return nil, err
}
var respData bingSearchResponse
diff --git a/internal/infrastructure/web_search/duckduckgo.go b/internal/infrastructure/web_search/duckduckgo.go
index 0eaa8db28..62f7e9010 100644
--- a/internal/infrastructure/web_search/duckduckgo.go
+++ b/internal/infrastructure/web_search/duckduckgo.go
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
- "io"
"net/http"
"net/url"
"strings"
@@ -157,8 +156,13 @@ func (p *DuckDuckGoProvider) searchAPI(
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(resp.Body)
- return nil, fmt.Errorf("duckduckgo API returned status %d: %s", resp.StatusCode, string(body))
+ preview := readErrorPreview(resp.Body)
+ return nil, fmt.Errorf("duckduckgo API returned status %d: %s", resp.StatusCode, preview)
+ }
+
+ bodyBytes, err := readCapped(resp.Body, 0)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
}
var apiResponse struct {
@@ -174,7 +178,7 @@ func (p *DuckDuckGoProvider) searchAPI(
Text string `json:"Text"`
} `json:"Results"`
}
- if err := json.NewDecoder(resp.Body).Decode(&apiResponse); err != nil {
+ if err := json.Unmarshal(bodyBytes, &apiResponse); err != nil {
return nil, fmt.Errorf("failed to decode API response: %w", err)
}
diff --git a/internal/infrastructure/web_search/keenable.go b/internal/infrastructure/web_search/keenable.go
index fa01eb209..d4762cbc5 100644
--- a/internal/infrastructure/web_search/keenable.go
+++ b/internal/infrastructure/web_search/keenable.go
@@ -5,7 +5,6 @@ import (
"context"
"encoding/json"
"fmt"
- "io"
"net/http"
"time"
@@ -102,12 +101,12 @@ func (p *KeenableProvider) Search(
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
- respBody, _ := io.ReadAll(resp.Body)
- logger.Warnf(ctx, "[WebSearch][Keenable] API returned status %d: %s", resp.StatusCode, string(respBody))
- return nil, fmt.Errorf("keenable API returned status %d: %s", resp.StatusCode, string(respBody))
+ preview := readErrorPreview(resp.Body)
+ logger.Warnf(ctx, "[WebSearch][Keenable] API returned status %d: %s", resp.StatusCode, preview)
+ return nil, fmt.Errorf("keenable API returned status %d: %s", resp.StatusCode, preview)
}
- respBody, err := io.ReadAll(resp.Body)
+ respBody, err := readCapped(resp.Body, 0)
if err != nil {
return nil, fmt.Errorf("failed to read response: %w", err)
}
diff --git a/internal/infrastructure/web_search/ollama.go b/internal/infrastructure/web_search/ollama.go
index b4e3fa8fe..551204156 100644
--- a/internal/infrastructure/web_search/ollama.go
+++ b/internal/infrastructure/web_search/ollama.go
@@ -5,7 +5,6 @@ import (
"context"
"encoding/json"
"fmt"
- "io"
"net/http"
"time"
@@ -115,14 +114,15 @@ func (p *OllamaProvider) doSearch(ctx context.Context, req *http.Request) ([]*ty
}
defer resp.Body.Close()
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, err
+ if resp.StatusCode != http.StatusOK {
+ preview := readErrorPreview(resp.Body)
+ logger.Warnf(ctx, "[WebSearch][Ollama] API returned status %d: %s", resp.StatusCode, preview)
+ return nil, fmt.Errorf("ollama API returned status %d: %s", resp.StatusCode, preview)
}
- if resp.StatusCode != http.StatusOK {
- logger.Warnf(ctx, "[WebSearch][Ollama] API returned status %d: %s", resp.StatusCode, string(body))
- return nil, fmt.Errorf("ollama API returned status %d: %s", resp.StatusCode, string(body))
+ body, err := readCapped(resp.Body, 0)
+ if err != nil {
+ return nil, err
}
var respData ollamaSearchResponse
diff --git a/internal/infrastructure/web_search/response_limit.go b/internal/infrastructure/web_search/response_limit.go
new file mode 100644
index 000000000..1833dfe18
--- /dev/null
+++ b/internal/infrastructure/web_search/response_limit.go
@@ -0,0 +1,51 @@
+package web_search
+
+import (
+ "errors"
+ "fmt"
+ "io"
+)
+
+// maxJSONResponseBytes bounds every provider response body. A search result set
+// is JSON metadata over a page of hits; a larger body means a broken or
+// misconfigured endpoint, not more results.
+const maxJSONResponseBytes = 16 << 20
+
+// maxErrorPreviewBytes bounds how much of a failed response is echoed into a log
+// line or an error string. Without it a failing endpoint can inject an unbounded
+// blob into both.
+const maxErrorPreviewBytes = 4 << 10
+
+// errResponseTooLarge marks a body over the cap. It is deterministic: the same
+// request returns the same oversized body, so callers must not retry it.
+var errResponseTooLarge = errors.New("response exceeds maximum size")
+
+// readCapped reads a response body, refusing anything larger than limit instead
+// of buffering it. Oversized payloads are reported as an error: a truncated body
+// would otherwise be parsed as if it were the whole response. A non-positive
+// limit (e.g. a provider built without one) falls back to maxJSONResponseBytes.
+//
+// Mirrors the cap the datasource connectors already apply to their API bodies.
+func readCapped(body io.Reader, limit int64) ([]byte, error) {
+ if limit <= 0 {
+ limit = maxJSONResponseBytes
+ }
+ data, err := io.ReadAll(io.LimitReader(body, limit+1))
+ if err != nil {
+ return nil, err
+ }
+ if int64(len(data)) > limit {
+ return nil, fmt.Errorf("%w (%d bytes)", errResponseTooLarge, limit)
+ }
+ return data, nil
+}
+
+// readErrorPreview reads just enough of a failed body to put in a log line or an
+// error message. It truncates rather than refusing: a preview is diagnostic, and
+// the interesting part of a large error body is its start. Read errors are
+// dropped because the caller is already on the error path and is about to report
+// the status code, which is the useful part.
+func readErrorPreview(body io.Reader) string {
+ data, _ := io.ReadAll(io.LimitReader(body, maxErrorPreviewBytes))
+ return string(data)
+}
diff --git a/internal/infrastructure/web_search/response_limit_test.go b/internal/infrastructure/web_search/response_limit_test.go
new file mode 100644
index 000000000..22c626477
--- /dev/null
+++ b/internal/infrastructure/web_search/response_limit_test.go
@@ -0,0 +1,143 @@
+package web_search
+
+import (
+ "context"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+// fillerReader streams size bytes without allocating them, so a test can hand an
+// endpoint a body far larger than any cap without spending the memory itself.
+type fillerReader struct {
+ remaining int64
+}
+
+func (f *fillerReader) Read(p []byte) (int, error) {
+ if f.remaining <= 0 {
+ return 0, io.EOF
+ }
+ if int64(len(p)) > f.remaining {
+ p = p[:f.remaining]
+ }
+ for i := range p {
+ p[i] = 'a'
+ }
+ f.remaining -= int64(len(p))
+ return len(p), nil
+}
+
+// readCapped must refuse an oversized body rather than buffering it, and must
+// still pass a body at exactly the cap through untouched.
+func TestReadCapped(t *testing.T) {
+ t.Run("rejects over the cap", func(t *testing.T) {
+ const limit = 1 << 10
+ _, err := readCapped(&fillerReader{remaining: limit + 1}, limit)
+ if err == nil {
+ t.Fatal("expected an over-cap body to be refused")
+ }
+ if !errors.Is(err, errResponseTooLarge) {
+ t.Errorf("err = %v, want errResponseTooLarge", err)
+ }
+ if !strings.Contains(err.Error(), "exceeds maximum size") {
+ t.Errorf("error text = %q, want the canonical over-size message", err)
+ }
+ })
+
+ t.Run("accepts exactly the cap", func(t *testing.T) {
+ const limit = 1 << 10
+ got, err := readCapped(strings.NewReader(strings.Repeat("a", limit)), limit)
+ if err != nil {
+ t.Fatalf("readCapped() = %v, want nil at exactly the cap", err)
+ }
+ if len(got) != limit {
+ t.Errorf("len = %d, want %d", len(got), limit)
+ }
+ })
+
+ t.Run("non-positive limit falls back to the production cap", func(t *testing.T) {
+ got, err := readCapped(strings.NewReader("{}"), 0)
+ if err != nil {
+ t.Fatalf("readCapped(_, 0) = %v, want nil", err)
+ }
+ if string(got) != "{}" {
+ t.Errorf("body = %q, want %q", got, "{}")
+ }
+ })
+}
+
+// An error preview ends up in a log line and in the error the caller sees, so it
+// has to be bounded too. searxng already capped its preview; its siblings did not.
+func TestReadErrorPreviewIsBounded(t *testing.T) {
+ preview := readErrorPreview(&fillerReader{remaining: 1 << 20})
+ if len(preview) != maxErrorPreviewBytes {
+ t.Errorf("preview is %d bytes, want %d", len(preview), maxErrorPreviewBytes)
+ }
+ if preview == "" {
+ t.Error("preview is empty; a truncated body is more useful than none")
+ }
+}
+
+// doSearch reads the response itself, so these two drive the real call sites.
+// Bing is the one that used to read the whole body before it even looked at the
+// status code.
+func TestBingProviderRejectsOversizedSuccessBody(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ // Valid JSON prefix, then streamed past the cap without the test ever
+ // allocating it. Only the size guard can reject this body: a parse error
+ // instead would mean the body was capped but then still parsed.
+ _, _ = w.Write([]byte(`{"webPages":{"value":[`))
+ _, _ = io.Copy(w, &fillerReader{remaining: maxJSONResponseBytes + (1 << 20)})
+ }))
+ defer srv.Close()
+
+ p := &BingProvider{client: srv.Client()}
+ req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, srv.URL, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ results, err := p.doSearch(context.Background(), req)
+ if err == nil {
+ t.Fatalf("expected an oversized body to be refused, got %d results", len(results))
+ }
+ if !errors.Is(err, errResponseTooLarge) {
+ t.Errorf("err = %v, want errResponseTooLarge", err)
+ }
+ if results != nil {
+ t.Errorf("results = %v, want nil on a refused body", results)
+ }
+}
+
+// Before the fix this error carried the entire failing body, so a broken or
+// hostile endpoint could push an unbounded blob into the caller's error string
+// and into the log line alongside it.
+func TestBingProviderBoundsFailedResponseInError(t *testing.T) {
+ const bodySize = 1 << 20
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusBadGateway)
+ _, _ = io.Copy(w, &fillerReader{remaining: bodySize})
+ }))
+ defer srv.Close()
+
+ p := &BingProvider{client: srv.Client()}
+ req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, srv.URL, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _, err = p.doSearch(context.Background(), req)
+ if err == nil {
+ t.Fatal("expected an error for a failing status")
+ }
+ // Room for the fixed prefix ("bing API returned status 502: ") on top of the cap.
+ if limit := len("bing API returned status 502: ") + maxErrorPreviewBytes; len(err.Error()) > limit {
+ t.Errorf("error is %d bytes for a %d byte body, want at most %d: the failed "+
+ "response is being echoed unbounded", len(err.Error()), bodySize, limit)
+ }
+}
diff --git a/internal/infrastructure/web_search/searxng.go b/internal/infrastructure/web_search/searxng.go
index 7e57c7383..d406c0cfe 100644
--- a/internal/infrastructure/web_search/searxng.go
+++ b/internal/infrastructure/web_search/searxng.go
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
- "io"
"net/http"
"net/url"
"strings"
@@ -128,12 +127,16 @@ func (p *SearxngProvider) Search(
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
- return nil, fmt.Errorf("searxng returned status %d: %s", resp.StatusCode, string(body))
+ return nil, fmt.Errorf("searxng returned status %d: %s", resp.StatusCode, readErrorPreview(resp.Body))
+ }
+
+ bodyBytes, err := readCapped(resp.Body, 0)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
}
var data searxngResponse
- if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
+ if err := json.Unmarshal(bodyBytes, &data); err != nil {
p.lastUnresponsive = nil
return nil, fmt.Errorf("failed to decode SearXNG response (ensure JSON format is enabled in settings.yml): %w", err)
}
diff --git a/internal/infrastructure/web_search/tavily.go b/internal/infrastructure/web_search/tavily.go
index 4a1e35264..eb6e1efd4 100644
--- a/internal/infrastructure/web_search/tavily.go
+++ b/internal/infrastructure/web_search/tavily.go
@@ -5,7 +5,6 @@ import (
"context"
"encoding/json"
"fmt"
- "io"
"net/http"
"time"
@@ -88,12 +87,12 @@ func (p *TavilyProvider) Search(
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
- respBody, _ := io.ReadAll(resp.Body)
- logger.Warnf(ctx, "[WebSearch][Tavily] API returned status %d: %s", resp.StatusCode, string(respBody))
- return nil, fmt.Errorf("tavily API returned status %d: %s", resp.StatusCode, string(respBody))
+ preview := readErrorPreview(resp.Body)
+ logger.Warnf(ctx, "[WebSearch][Tavily] API returned status %d: %s", resp.StatusCode, preview)
+ return nil, fmt.Errorf("tavily API returned status %d: %s", resp.StatusCode, preview)
}
- respBody, err := io.ReadAll(resp.Body)
+ respBody, err := readCapped(resp.Body, 0)
if err != nil {
return nil, fmt.Errorf("failed to read response: %w", err)
}