RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(agent): avoid unsafe HTTP retries (#20456)
Sbaaoui Idrissb754c76b16b2
diff --git a/internal/agent/tool/http_helper.go b/internal/agent/tool/http_helper.goindex c81e8f187b..5a9386ca33 100644--- a/internal/agent/tool/http_helper.go+++ b/internal/agent/tool/http_helper.go@@ -147,10 +147,10 @@ func (h *HTTPHelper) WithClient(c *http.Client) *HTTPHelper {// that sniff the body still behave sensibly.//// Retry policy:-// - 5xx responses: retried-// - network errors (connection refused, DNS, etc.): retried+// - idempotent methods: 5xx responses and network errors are retried+// - non-idempotent methods: never retried (the server may have acted already)// - 4xx responses: NOT retried (caller error, won't help to retry)-// - 2xx / 3xx: returned as-is+// - other responses: returned as-is//// The context is honored on every attempt; cancellation aborts the loop.func (h *HTTPHelper) Do(@@ -216,8 +216,8 @@ func (h *HTTPHelper) DoPinned(// pinnedClient builds a one-shot *http.Client whose transport dials// pinnedIP:port instead of originalHost:port. The transport is cloned-// from baseTransport so the rest of the connection behaviour (TLS-// config, idle pool) is identical to a non-pinned call.+// from baseTransport so the TLS config stays identical to a non-pinned+// call; keep-alives are disabled because the per-call pool cannot be reused.//// The proxy setting is explicitly disabled. Two reasons://@@ -248,6 +248,7 @@ func (h *HTTPHelper) DoPinned(func (h *HTTPHelper) pinnedClient(pinnedIP net.IP) *http.Client {base := h.baseTransport.Clone()base.Proxy = nil+ base.DisableKeepAlives = truebase.DialContext = (&pinnedDialer{pinnedIP: pinnedIP,base: &net.Dialer{@@ -295,6 +296,12 @@ func (h *HTTPHelper) doRawWithClient(if body != "" && contentType == "" {contentType = "application/octet-stream"}+ retryableMethod := false+ switch method {+ case http.MethodGet, http.MethodHead, http.MethodOptions, http.MethodTrace,+ http.MethodPut, http.MethodDelete:+ retryableMethod = true+ }var lastErr errorfor attempt := 1; attempt <= h.retry.MaxAttempts; attempt++ {@@ -316,7 +323,7 @@ func (h *HTTPHelper) doRawWithClient(resp, err := client.Do(req)if err != nil {lastErr = err- if !isRetryableNetError(err) {+ if !retryableMethod || !isRetryableNetError(err) {return nil, err}if attempt == h.retry.MaxAttempts {@@ -326,8 +333,8 @@ func (h *HTTPHelper) doRawWithClient(continue}- // 5xx is retryable, 4xx is not.- if resp.StatusCode >= 500 {+ // Retry 5xx only when repeating the request cannot duplicate an action.+ if resp.StatusCode >= 500 && retryableMethod {lastErr = fmt.Errorf("http_helper: %s %s returned %d", method, SanitizeURL(url), resp.StatusCode)// drain body so the connection can be reused_, _ = io.Copy(io.Discard, resp.Body)diff --git a/internal/agent/tool/http_helper_test.go b/internal/agent/tool/http_helper_test.goindex 369289ab7b..990db3810c 100644--- a/internal/agent/tool/http_helper_test.go+++ b/internal/agent/tool/http_helper_test.go@@ -114,6 +114,92 @@ func TestHTTPHelper_RetriesOn5xx(t *testing.T) {}}+func TestHTTPHelper_PostDoesNotRetry5xx(t *testing.T) {+ t.Parallel()+ var hits atomic.Int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ hits.Add(1)+ w.WriteHeader(http.StatusServiceUnavailable)+ }))+ defer srv.Close()++ resp, err := newTestHelper(3, time.Millisecond, 5*time.Millisecond).+ Do(t.Context(), http.MethodPost, srv.URL, "payload", "text/plain", nil)+ if err != nil {+ t.Fatalf("Do: %v", err)+ }+ defer resp.Body.Close()+ if resp.StatusCode != http.StatusServiceUnavailable || hits.Load() != 1 {+ t.Fatalf("status = %d, server hits = %d; want 503 and 1 hit", resp.StatusCode, hits.Load())+ }+}++func TestHTTPHelper_PostDoesNotRetryAfterTransportError(t *testing.T) {+ t.Parallel()+ var hits atomic.Int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = io.Copy(io.Discard, r.Body)+ hits.Add(1)+ conn, _, err := w.(http.Hijacker).Hijack()+ if err != nil {+ t.Errorf("hijack: %v", err)+ return+ }+ _ = conn.Close()+ }))+ defer srv.Close()++ _, err := newTestHelper(3, time.Millisecond, 5*time.Millisecond).+ Do(t.Context(), http.MethodPost, srv.URL, "payload", "text/plain", nil)+ if err == nil {+ t.Fatal("Do unexpectedly succeeded after server closed the connection")+ }+ if got := hits.Load(); got != 1 {+ t.Fatalf("server hits = %d, want 1 (request body was already delivered)", got)+ }+}++func TestHTTPHelper_DoPinnedClosesConnectionAfterResponse(t *testing.T) {+ t.Parallel()+ closed := make(chan struct{}, 1)+ srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = io.WriteString(w, "ok")+ }))+ srv.Config.ConnState = func(_ net.Conn, state http.ConnState) {+ if state == http.StateClosed {+ closed <- struct{}{}+ }+ }+ srv.Start()+ defer srv.Close()++ u, err := neturl.Parse(srv.URL)+ if err != nil {+ t.Fatal(err)+ }+ _, port, err := net.SplitHostPort(u.Host)+ if err != nil {+ t.Fatal(err)+ }+ h := newTestHelper(3, time.Millisecond, 5*time.Millisecond)+ resp, err := h.DoPinned(t.Context(), http.MethodGet, "http://example.test:"+port+"/",+ "", "", nil, "example.test", net.ParseIP("127.0.0.1"))+ if err != nil {+ t.Fatalf("DoPinned: %v", err)+ }+ if _, err := io.Copy(io.Discard, resp.Body); err != nil {+ t.Fatal(err)+ }+ if err := resp.Body.Close(); err != nil {+ t.Fatal(err)+ }+ select {+ case <-closed:+ case <-time.After(time.Second):+ t.Fatal("pinned connection remained idle after response body was closed")+ }+}+// TestHTTPHelper_NoRetryOn4xx verifies that 4xx is returned immediately with// no retry — the caller is responsible for fixing 4xx, retrying won't help.func TestHTTPHelper_NoRetryOn4xx(t *testing.T) {diff --git a/internal/agent/tool/querit_test.go b/internal/agent/tool/querit_test.goindex 3c9d91d4a1..1e8ba34811 100644--- a/internal/agent/tool/querit_test.go+++ b/internal/agent/tool/querit_test.go@@ -317,28 +317,6 @@ func TestQueritHTTPFailuresAreSoftErrors(t *testing.T) {}})- t.Run("HTTP helper retries temporary server errors", func(t *testing.T) {- var calls atomic.Int32- server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {- attempt := calls.Add(1)- if attempt < 3 {- writer.WriteHeader(http.StatusServiceUnavailable)- return- }- _, _ = writer.Write([]byte(`{"results":{"result":[]}}`))- }))- defer server.Close()- helper := NewHTTPHelperWithRetry(RetryConfig{- MaxAttempts: 3,- BaseBackoff: time.Nanosecond,- MaxBackoff: time.Nanosecond,- }).WithClient(&http.Client{Transport: rewriteQueritHostTransport(server.URL)})- out, err := NewQueritToolWithEnvKey(helper, func() string { return "k" }).InvokableRun(t.Context(), `{"query":"x"}`)- if err != nil || calls.Load() != 3 || strings.Contains(out, "_ERROR") {- t.Fatalf("result = %s, err = %v, calls = %d", out, err, calls.Load())- }- })-t.Run("persistent server errors are soft", func(t *testing.T) {var calls atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {@@ -353,7 +331,7 @@ func TestQueritHTTPFailuresAreSoftErrors(t *testing.T) {}).WithClient(&http.Client{Transport: rewriteQueritHostTransport(server.URL)})querit := NewQueritToolWithEnvKey(helper, func() string { return "environment-secret" })out, err := querit.InvokableRun(t.Context(), `{"query":"x"}`)- if err != nil || calls.Load() != 3 || !strings.Contains(out, "_ERROR") || !strings.Contains(out, "500") {+ if err != nil || calls.Load() != 1 || !strings.Contains(out, "_ERROR") || !strings.Contains(out, "500") {t.Fatalf("result = %s, err = %v, calls = %d", out, err, calls.Load())}})@@ -399,7 +377,7 @@ func TestQueritHTTPFailuresAreSoftErrors(t *testing.T) {return errors.New("offline")})})out, err := NewQueritToolWithEnvKey(helper, func() string { return "k" }).InvokableRun(t.Context(), `{"query":"x"}`)- if err != nil || calls.Load() != 3 || !strings.Contains(out, "_ERROR") {+ if err != nil || calls.Load() != 1 || !strings.Contains(out, "_ERROR") {t.Fatalf("result = %s, err = %v, calls = %d", out, err, calls.Load())}})