Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(review): retry a transient provider failure inside the AI review (#21590)
h3n4l6024cf918e4c
diff --git a/backend/plugin/ai/claude.go b/backend/plugin/ai/claude.goindex e6afa242b9..acf8c25045 100644--- a/backend/plugin/ai/claude.go+++ b/backend/plugin/ai/claude.go@@ -65,6 +65,25 @@ func claudeContentFromToolCalls(toolCalls []*v1pb.AIChatToolCall) []json.RawMessreturn nil}+type claudeErrorReply struct {+ Error struct {+ Details struct {+ ErrorCode string `json:"error_code"`+ } `json:"details"`+ } `json:"error"`+}++// readClaudeFailure marks the reply for a reached monthly spend cap. Anthropic+// answers it with 429 and the type rate_limit_error, like a rate limit, and+// tells the two apart by details.error_code.+func readClaudeFailure(httpErr *HTTPError) {+ var reply claudeErrorReply+ if err := json.Unmarshal([]byte(httpErr.Body), &reply); err != nil {+ return+ }+ httpErr.QuotaExhausted = reply.Error.Details.ErrorCode == "enforced_spend_limit_reached"+}+func chatClaude(ctx context.Context, aiSetting *storepb.AISetting, request *v1pb.AIChatRequest) (*v1pb.AIChatResponse, error) {payload := chatClaudeRequest{Model: aiSetting.Model,@@ -158,7 +177,7 @@ func chatClaude(ctx context.Context, aiSetting *storepb.AISetting, request *v1pbhttpReq.Header.Set("anthropic-version", "2023-06-01")}- body, err := doHTTPRequest(ctx, httpReq, "Claude")+ body, err := doHTTPRequest(ctx, httpReq, "Claude", readClaudeFailure)if err != nil {return nil, err}diff --git a/backend/plugin/ai/gemini.go b/backend/plugin/ai/gemini.goindex 35126b7160..2a1098401e 100644--- a/backend/plugin/ai/gemini.go+++ b/backend/plugin/ai/gemini.go@@ -7,6 +7,8 @@ import ("fmt""net/http""net/url"+ "strings"+ "time""github.com/google/uuid""github.com/pkg/errors"@@ -95,6 +97,37 @@ func geminiFunctionResponse(content string) map[string]any {return map[string]any{"result": parsed}}+type geminiErrorReply struct {+ Error struct {+ Details []struct {+ Type string `json:"@type"`+ RetryDelay string `json:"retryDelay"`+ } `json:"details"`+ } `json:"error"`+}++// readGeminiFailure reads the wait Gemini asks for, which it can put in a+// google.rpc.RetryInfo detail of the error body. The delay is a protobuf+// Duration in JSON: decimal seconds with an "s" suffix, such as "34.074824224s".+func readGeminiFailure(httpErr *HTTPError) {+ var reply geminiErrorReply+ if err := json.Unmarshal([]byte(httpErr.Body), &reply); err != nil {+ return+ }+ for _, detail := range reply.Error.Details {+ if detail.Type != "type.googleapis.com/google.rpc.RetryInfo" {+ continue+ }+ seconds, ok := strings.CutSuffix(detail.RetryDelay, "s")+ if !ok {+ continue+ }+ if delay, ok := parseDecimal(seconds, time.Second); ok {+ httpErr.RetryAfter = max(httpErr.RetryAfter, delay)+ }+ }+}+func chatGemini(ctx context.Context, aiSetting *storepb.AISetting, request *v1pb.AIChatRequest) (*v1pb.AIChatResponse, error) {payload := chatGeminiRequest{}@@ -195,7 +228,7 @@ func chatGemini(ctx context.Context, aiSetting *storepb.AISetting, request *v1pbhttpReq.Header.Set("Content-Type", "application/json")httpReq.Header.Set("x-goog-api-key", aiSetting.ApiKey)- body, err := doHTTPRequest(ctx, httpReq, "Gemini")+ body, err := doHTTPRequest(ctx, httpReq, "Gemini", readGeminiFailure)if err != nil {return nil, err}diff --git a/backend/plugin/ai/http_error.go b/backend/plugin/ai/http_error.gonew file mode 100644index 0000000000..90609bd8ef--- /dev/null+++ b/backend/plugin/ai/http_error.go@@ -0,0 +1,110 @@+package ai++import (+ "fmt"+ "math"+ "net/http"+ "regexp"+ "strconv"+ "strings"+ "time"+)++// HTTPError is a provider reply with a status other than 200.+type HTTPError struct {+ // Provider is the adapter's name for the vendor: "OpenAI", "Claude", or "Gemini".+ Provider string+ StatusCode int+ // RetryAfter is how long the provider asked the caller to wait before the+ // next call. It is zero when the provider named no wait, or a wait of zero.+ RetryAfter time.Duration+ // QuotaExhausted means the provider refused the call for a quota that a+ // wait does not restore, such as a spend limit. The same call keeps failing.+ QuotaExhausted bool+ // ShouldRetry is the provider's x-should-retry header: whether the provider+ // says the same call can succeed later. It is nil when the provider sent+ // none, or a value other than "true" or "false".+ ShouldRetry *bool+ Body string+}++// Error keeps a text that callers read: the frontend agent loop looks for the+// status code in it to decide whether to call again.+func (e *HTTPError) Error() string {+ return fmt.Sprintf("%s API returned status %d: %s", e.Provider, e.StatusCode, e.Body)+}++// TransportError is a provider call that got no complete reply: the request+// was not sent, or the reply could not be read. It does not unwrap to its+// cause: connect codes an uncoded error by a context error in its chain, and a+// dial timeout to the provider would then read as the caller's own deadline.+type TransportError struct {+ message string+}++func (e *TransportError) Error() string {+ return e.message+}++// readShouldRetry reads the x-should-retry header the way the official OpenAI+// and Anthropic Go SDKs do: only "true" and "false" count.+func readShouldRetry(header http.Header) *bool {+ switch header.Get("X-Should-Retry") {+ case "true":+ return new(true)+ case "false":+ return new(false)+ default:+ return nil+ }+}++// providerWait reads the wait a provider asked for in its reply headers.+// Retry-After-Ms, in milliseconds, wins over Retry-After, as it does in the+// official OpenAI and Anthropic Go SDKs.+func providerWait(header http.Header, now time.Time) time.Duration {+ if wait, ok := parseDecimal(strings.TrimSpace(header.Get("Retry-After-Ms")), time.Millisecond); ok {+ return wait+ }+ return parseRetryAfter(header.Get("Retry-After"), now)+}++// parseRetryAfter reads a Retry-After header value, which is a number of+// seconds or an HTTP date. It returns zero for a value that is absent,+// malformed, or not in the future.+func parseRetryAfter(value string, now time.Time) time.Duration {+ value = strings.TrimSpace(value)+ if wait, ok := parseDecimal(value, time.Second); ok {+ return wait+ }+ if date, err := http.ParseTime(value); err == nil {+ return max(date.Sub(now), 0)+ }+ return 0+}++var decimalNumber = regexp.MustCompile(`^[0-9]+(\.[0-9]+)?$`)++// parseDecimal reads a count of units written as digits with an optional+// fraction, to the nanosecond. The unit is a power of ten nanoseconds, such as+// time.Second or time.Millisecond. It saturates instead of overflowing: a wait+// too long for a Duration must not come out short or negative.+func parseDecimal(value string, unit time.Duration) (time.Duration, bool) {+ if !decimalNumber.MatchString(value) {+ return 0, false+ }+ whole, fraction, _ := strings.Cut(value, ".")+ count, err := strconv.ParseInt(whole, 10, 64)+ if err != nil || count > math.MaxInt64/int64(unit) {+ return math.MaxInt64, true+ }+ // Padded or cut to as many digits as the unit has zeros, the fraction+ // counts nanoseconds.+ digits := len(strconv.FormatInt(int64(unit), 10)) - 1+ nanoseconds, _ := strconv.ParseInt((fraction + strings.Repeat("0", digits))[:digits], 10, 64)+ wait := time.Duration(count) * unit+ if wait > math.MaxInt64-time.Duration(nanoseconds) {+ return math.MaxInt64, true+ }+ return wait + time.Duration(nanoseconds), true+}diff --git a/backend/plugin/ai/http_error_test.go b/backend/plugin/ai/http_error_test.gonew file mode 100644index 0000000000..ffc3f144c4--- /dev/null+++ b/backend/plugin/ai/http_error_test.go@@ -0,0 +1,410 @@+package ai++import (+ "context"+ "fmt"+ "math"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++func TestParseRetryAfter(t *testing.T) {+ t.Parallel()++ now := time.Date(2026, 10, 14, 8, 0, 0, 0, time.UTC)+ tests := []struct {+ name string+ value string+ want time.Duration+ }{+ {name: "absent", value: "", want: 0},+ {name: "seconds", value: "7", want: 7 * time.Second},+ {name: "seconds with spaces", value: " 120 ", want: 2 * time.Minute},+ {name: "fraction of a second", value: "0.5", want: 500 * time.Millisecond},+ {name: "every digit down to the nanosecond", value: "123456789.123456789", want: 123456789123456789},+ {name: "digits past the nanosecond", value: "1.0000000019", want: time.Second + time.Nanosecond},+ {name: "longest count of whole seconds", value: "9223372036", want: 9223372036 * time.Second},+ {name: "longest duration", value: "9223372036.854775807", want: math.MaxInt64},+ {name: "zero", value: "0", want: 0},+ {name: "negative", value: "-3", want: 0},+ {name: "signed", value: "+3", want: 0},+ {name: "not a number", value: "NaN", want: 0},+ {name: "infinite", value: "Inf", want: 0},+ {name: "exponent", value: "1e3", want: 0},+ {name: "hexadecimal", value: "0x10", want: 0},+ {name: "one nanosecond too long for a duration", value: "9223372036.854775808", want: math.MaxInt64},+ {name: "one second too long for a duration", value: "9223372037", want: math.MaxInt64},+ {name: "too long for an integer", value: strings.Repeat("9", 400), want: math.MaxInt64},+ {name: "HTTP date in the future", value: "Wed, 14 Oct 2026 08:00:45 GMT", want: 45 * time.Second},+ {name: "HTTP date in the past", value: "Wed, 14 Oct 2026 07:59:00 GMT", want: 0},+ {name: "obsolete HTTP date format", value: "Wednesday, 14-Oct-26 08:02:00 GMT", want: 2 * time.Minute},+ {name: "malformed", value: "soon", want: 0},+ }+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ t.Parallel()+ require.Equal(t, test.want, parseRetryAfter(test.value, now))+ })+ }+}++func TestProviderWait(t *testing.T) {+ t.Parallel()++ now := time.Date(2026, 10, 14, 8, 0, 0, 0, time.UTC)+ tests := []struct {+ name string+ header map[string]string+ want time.Duration+ }{+ {name: "no header", want: 0},+ {name: "milliseconds", header: map[string]string{"Retry-After-Ms": "1500"}, want: 1500 * time.Millisecond},+ {name: "fraction of a millisecond", header: map[string]string{"Retry-After-Ms": "250.5"}, want: 250*time.Millisecond + 500*time.Microsecond},+ {name: "digits past the nanosecond", header: map[string]string{"Retry-After-Ms": "1.0000019"}, want: time.Millisecond + time.Nanosecond},+ {name: "milliseconds win over seconds", header: map[string]string{"Retry-After-Ms": "200", "Retry-After": "20"}, want: 200 * time.Millisecond},+ {name: "zero milliseconds win over seconds", header: map[string]string{"Retry-After-Ms": "0", "Retry-After": "20"}, want: 0},+ {name: "seconds when the milliseconds do not parse", header: map[string]string{"Retry-After-Ms": "soon", "Retry-After": "3"}, want: 3 * time.Second},+ {name: "seconds when the milliseconds are negative", header: map[string]string{"Retry-After-Ms": "-200", "Retry-After": "3"}, want: 3 * time.Second},+ {name: "seconds when the milliseconds end in a point", header: map[string]string{"Retry-After-Ms": "200.", "Retry-After": "3"}, want: 3 * time.Second},+ {name: "seconds alone", header: map[string]string{"Retry-After": "3"}, want: 3 * time.Second},+ {name: "milliseconds too long for a duration", header: map[string]string{"Retry-After-Ms": strings.Repeat("9", 40)}, want: math.MaxInt64},+ {name: "longest count of whole milliseconds", header: map[string]string{"Retry-After-Ms": "9223372036854"}, want: 9223372036854 * time.Millisecond},+ {name: "one millisecond too long for a duration", header: map[string]string{"Retry-After-Ms": "9223372036855"}, want: math.MaxInt64},+ }+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ t.Parallel()+ header := http.Header{}+ for name, value := range test.header {+ header.Set(name, value)+ }+ require.Equal(t, test.want, providerWait(header, now))+ })+ }+}++// failingProvider serves one reply to every request.+func failingProvider(t *testing.T, status int, header map[string]string, body string) *httptest.Server {+ t.Helper()+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ for name, value := range header {+ w.Header().Set(name, value)+ }+ w.WriteHeader(status)+ _, _ = w.Write([]byte(body))+ }))+ t.Cleanup(server.Close)+ return server+}++func chatOnce(ctx context.Context, provider storepb.AISetting_Provider, endpoint string) error {+ content := "hello"+ _, err := Chat(ctx, &storepb.AISetting{Provider: provider, Endpoint: endpoint, Model: "test-model", ApiKey: "test-key"}, &v1pb.AIChatRequest{+ Messages: []*v1pb.AIChatMessage{{Role: v1pb.AIChatMessageRole_AI_CHAT_MESSAGE_ROLE_USER, Content: &content}},+ })+ return err+}++func TestChatReturnsAnHTTPErrorForAFailedReply(t *testing.T) {+ t.Parallel()++ const (+ openAIRateLimit = `{"error": {"message": "Rate limit reached for requests", "type": "requests", "param": null, "code": "rate_limit_exceeded"}}`+ openAIQuota = `{"error": {"message": "You exceeded your current quota, please check your plan and billing details.", "type": "insufficient_quota", "param": null, "code": "insufficient_quota"}}`+ azureRateLimit = `{"error": {"code": "429", "message": "Requests have exceeded the token rate limit of your current pricing tier."}}`+ numericCode = `{"error": {"message": "insufficient quota", "type": "insufficient_quota", "code": 429}}`+ claudeRateLimit = `{"type": "error", "error": {"type": "rate_limit_error", "message": "This request would exceed your organization's rate limit of 1,000 requests per minute."}, "request_id": "req_1"}`+ claudeSpendCap = `{"type": "error", "error": {"type": "rate_limit_error", "message": "You have reached your API usage limits.", "details": {"error_code": "enforced_spend_limit_reached"}}, "request_id": "req_2"}`+ claudeOverload = `{"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}, "request_id": "req_3"}`+ geminiRateLimit = `{"error": {"code": 429, "message": "Resource has been exhausted (e.g. check quota).", "status": "RESOURCE_EXHAUSTED", "details": [{"@type": "type.googleapis.com/google.rpc.QuotaFailure", "violations": [{"quotaId": "GenerateRequestsPerMinutePerProjectPerModel"}]}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "34.5s"}]}}`+ geminiNoDetails = `{"error": {"code": 503, "message": "The model is overloaded. Please try again later.", "status": "UNAVAILABLE"}}`+ geminiLongDelay = `{"error": {"code": 429, "status": "RESOURCE_EXHAUSTED", "details": [{"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "99999999999999999999s"}]}}`+ geminiBadDelay = `{"error": {"code": 429, "status": "RESOURCE_EXHAUSTED", "details": [{"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "-5s"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "Infs"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "7"}, {"@type": "type.googleapis.com/google.rpc.Help", "retryDelay": "9s"}]}}`+ notJSON = `upstream connect error or disconnect/reset before headers`+ )+ tests := []struct {+ name string+ provider storepb.AISetting_Provider+ path string+ status int+ header map[string]string+ body string+ want HTTPError+ }{+ {+ name: "OpenAI rate limit with the wait in the header",+ provider: storepb.AISetting_OPEN_AI,+ path: "/v1/chat/completions",+ status: http.StatusTooManyRequests,+ header: map[string]string{"Retry-After": "20"},+ body: openAIRateLimit,+ want: HTTPError{Provider: "OpenAI", RetryAfter: 20 * time.Second},+ },+ {+ name: "OpenAI rate limit with the wait in milliseconds",+ provider: storepb.AISetting_OPEN_AI,+ path: "/v1/chat/completions",+ status: http.StatusTooManyRequests,+ header: map[string]string{"Retry-After-Ms": "1500", "Retry-After": "2"},+ body: openAIRateLimit,+ want: HTTPError{Provider: "OpenAI", RetryAfter: 1500 * time.Millisecond},+ },+ {+ name: "OpenAI used-up quota",+ provider: storepb.AISetting_OPEN_AI,+ path: "/v1/chat/completions",+ status: http.StatusTooManyRequests,+ body: openAIQuota,+ want: HTTPError{Provider: "OpenAI", QuotaExhausted: true},+ },+ {+ name: "OpenAI used-up quota on the responses endpoint",+ provider: storepb.AISetting_OPEN_AI,+ path: "/v1/responses",+ status: http.StatusTooManyRequests,+ body: openAIQuota,+ want: HTTPError{Provider: "OpenAI", QuotaExhausted: true},+ },+ {+ name: "Azure OpenAI rate limit",+ provider: storepb.AISetting_AZURE_OPENAI,+ path: "/openai/deployments/test/chat/completions",+ status: http.StatusTooManyRequests,+ header: map[string]string{"Retry-After": "3"},+ body: azureRateLimit,+ want: HTTPError{Provider: "OpenAI", RetryAfter: 3 * time.Second},+ },+ {+ name: "used-up quota from a service that sends a numeric code",+ provider: storepb.AISetting_OPEN_AI,+ path: "/v1/chat/completions",+ status: http.StatusTooManyRequests,+ body: numericCode,+ want: HTTPError{Provider: "OpenAI", QuotaExhausted: true},+ },+ {+ name: "OpenAI bad key",+ provider: storepb.AISetting_OPEN_AI,+ path: "/v1/chat/completions",+ status: http.StatusUnauthorized,+ body: `{"error": {"message": "Incorrect API key provided", "type": "invalid_request_error", "code": "invalid_api_key"}}`,+ want: HTTPError{Provider: "OpenAI"},+ },+ {+ name: "Claude rate limit with the wait in the header",+ provider: storepb.AISetting_CLAUDE,+ status: http.StatusTooManyRequests,+ header: map[string]string{"Retry-After": "12"},+ body: claudeRateLimit,+ want: HTTPError{Provider: "Claude", RetryAfter: 12 * time.Second},+ },+ {+ name: "Claude spend cap",+ provider: storepb.AISetting_CLAUDE,+ status: http.StatusTooManyRequests,+ body: claudeSpendCap,+ want: HTTPError{Provider: "Claude", QuotaExhausted: true},+ },+ {+ name: "Claude rate limit without a wait",+ provider: storepb.AISetting_CLAUDE,+ status: http.StatusTooManyRequests,+ body: claudeRateLimit,+ want: HTTPError{Provider: "Claude"},+ },+ {+ name: "Claude overloaded",+ provider: storepb.AISetting_CLAUDE,+ status: 529,+ body: claudeOverload,+ want: HTTPError{Provider: "Claude"},+ },+ {+ name: "Claude says to retry",+ provider: storepb.AISetting_CLAUDE,+ status: 529,+ header: map[string]string{"X-Should-Retry": "true"},+ body: claudeOverload,+ want: HTTPError{Provider: "Claude", ShouldRetry: new(true)},+ },+ {+ name: "OpenAI says not to retry",+ provider: storepb.AISetting_OPEN_AI,+ path: "/v1/chat/completions",+ status: http.StatusServiceUnavailable,+ header: map[string]string{"X-Should-Retry": "false"},+ body: openAIRateLimit,+ want: HTTPError{Provider: "OpenAI", ShouldRetry: new(false)},+ },+ {+ name: "x-should-retry that is neither true nor false",+ provider: storepb.AISetting_OPEN_AI,+ path: "/v1/chat/completions",+ status: http.StatusServiceUnavailable,+ header: map[string]string{"X-Should-Retry": "True"},+ body: openAIRateLimit,+ want: HTTPError{Provider: "OpenAI"},+ },+ {+ name: "Gemini rate limit with the wait in the body",+ provider: storepb.AISetting_GEMINI,+ status: http.StatusTooManyRequests,+ body: geminiRateLimit,+ want: HTTPError{Provider: "Gemini", RetryAfter: 34500 * time.Millisecond},+ },+ {+ name: "Gemini takes the longer of the header and the body",+ provider: storepb.AISetting_GEMINI,+ status: http.StatusTooManyRequests,+ header: map[string]string{"Retry-After": "60"},+ body: geminiRateLimit,+ want: HTTPError{Provider: "Gemini", RetryAfter: time.Minute},+ },+ {+ name: "Gemini delay too long for a duration",+ provider: storepb.AISetting_GEMINI,+ status: http.StatusTooManyRequests,+ body: geminiLongDelay,+ want: HTTPError{Provider: "Gemini", RetryAfter: math.MaxInt64},+ },+ {+ name: "Gemini delays that are negative, malformed, or of another detail type",+ provider: storepb.AISetting_GEMINI,+ status: http.StatusTooManyRequests,+ body: geminiBadDelay,+ want: HTTPError{Provider: "Gemini"},+ },+ {+ name: "Gemini unavailable without details",+ provider: storepb.AISetting_GEMINI,+ status: http.StatusServiceUnavailable,+ body: geminiNoDetails,+ want: HTTPError{Provider: "Gemini"},+ },+ {+ name: "OpenAI reply that is not JSON",+ provider: storepb.AISetting_OPEN_AI,+ path: "/v1/chat/completions",+ status: http.StatusBadGateway,+ body: notJSON,+ want: HTTPError{Provider: "OpenAI"},+ },+ {+ name: "Claude reply that is not JSON",+ provider: storepb.AISetting_CLAUDE,+ status: http.StatusBadGateway,+ body: notJSON,+ want: HTTPError{Provider: "Claude"},+ },+ {+ name: "Gemini reply that is not JSON",+ provider: storepb.AISetting_GEMINI,+ status: http.StatusBadGateway,+ body: notJSON,+ want: HTTPError{Provider: "Gemini"},+ },+ }+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ t.Parallel()++ server := failingProvider(t, test.status, test.header, test.body)+ err := chatOnce(context.Background(), test.provider, server.URL+test.path)++ var httpErr *HTTPError+ require.ErrorAs(t, err, &httpErr)+ want := test.want+ want.StatusCode = test.status+ want.Body = test.body+ require.Equal(t, want, *httpErr)+ require.EqualError(t, err, fmt.Sprintf("%s API returned status %d: %s", want.Provider, test.status, test.body))+ })+ }+}++func TestChatReturnsATransportErrorForACallWithNoReply(t *testing.T) {+ t.Parallel()++ t.Run("canceled context", func(t *testing.T) {+ t.Parallel()++ server := failingProvider(t, http.StatusOK, nil, `{}`)+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ err := chatOnce(ctx, storepb.AISetting_GEMINI, server.URL)++ var transportErr *TransportError+ require.ErrorAs(t, err, &transportErr)+ require.NotErrorIs(t, err, context.Canceled, "the caller asks its context, the error does not unwrap")+ require.ErrorContains(t, err, "failed to send HTTP request to Gemini: Post ")+ require.ErrorContains(t, err, ": context canceled")+ })++ t.Run("expired context", func(t *testing.T) {+ t.Parallel()++ server := failingProvider(t, http.StatusOK, nil, `{}`)+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))+ defer cancel()+ err := chatOnce(ctx, storepb.AISetting_CLAUDE, server.URL)++ var transportErr *TransportError+ require.ErrorAs(t, err, &transportErr)+ require.NotErrorIs(t, err, context.DeadlineExceeded)+ require.ErrorContains(t, err, "failed to send HTTP request to Claude: Post ")+ })++ t.Run("connection closed before the reply", func(t *testing.T) {+ t.Parallel()++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ closeConnection(t, w, "")+ }))+ defer server.Close()+ err := chatOnce(context.Background(), storepb.AISetting_OPEN_AI, server.URL+"/v1/chat/completions")++ var transportErr *TransportError+ require.ErrorAs(t, err, &transportErr)+ require.NotErrorIs(t, err, context.Canceled)+ require.NotErrorIs(t, err, context.DeadlineExceeded)+ require.ErrorContains(t, err, "failed to send HTTP request to OpenAI: Post ")+ })++ t.Run("connection closed in the middle of the reply", func(t *testing.T) {+ t.Parallel()++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ closeConnection(t, w, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 100\r\n\r\n{\"candidates\": [")+ }))+ defer server.Close()+ err := chatOnce(context.Background(), storepb.AISetting_GEMINI, server.URL)++ var transportErr *TransportError+ require.ErrorAs(t, err, &transportErr)+ require.EqualError(t, err, "failed to read Gemini response body: unexpected EOF")+ })+}++// closeConnection writes the raw bytes, if any, and drops the connection.+func closeConnection(t *testing.T, w http.ResponseWriter, raw string) {+ hijacker, ok := w.(http.Hijacker)+ if !ok {+ t.Error("the response writer cannot be hijacked")+ return+ }+ conn, _, err := hijacker.Hijack()+ if err != nil {+ t.Errorf("failed to hijack the connection: %v", err)+ return+ }+ _, _ = conn.Write([]byte(raw))+ _ = conn.Close()+}diff --git a/backend/plugin/ai/openai.go b/backend/plugin/ai/openai.goindex b70bfa5301..ce9e1bd8bb 100644--- a/backend/plugin/ai/openai.go+++ b/backend/plugin/ai/openai.go@@ -193,6 +193,25 @@ func extractGeminiThoughtSignatureFromOpenAIToolCall(raw json.RawMessage) stringreturn providerFields.ExtraContent.Google.ThoughtSignature}+type openAIErrorReply struct {+ Error struct {+ Type string `json:"type"`+ // Code is any so that a numeric code from a compatible service still decodes.+ Code any `json:"code"`+ } `json:"error"`+}++// readOpenAIFailure marks the reply for a used-up quota. OpenAI answers it+// with 429, like a rate limit, and tells the two apart by the error code.+func readOpenAIFailure(httpErr *HTTPError) {+ var reply openAIErrorReply+ if err := json.Unmarshal([]byte(httpErr.Body), &reply); err != nil {+ return+ }+ const insufficientQuota = "insufficient_quota"+ httpErr.QuotaExhausted = reply.Error.Code == insufficientQuota || reply.Error.Type == insufficientQuota+}+func chatOpenAI(ctx context.Context, aiSetting *storepb.AISetting, request *v1pb.AIChatRequest) (*v1pb.AIChatResponse, error) {if isOpenAIResponsesEndpoint(aiSetting.Endpoint) {return chatOpenAIResponses(ctx, aiSetting, request)@@ -274,7 +293,7 @@ func chatOpenAIChatCompletions(ctx context.Context, aiSetting *storepb.AISettinghttpReq.Header.Set("Authorization", fmt.Sprintf("Bearer %s", aiSetting.ApiKey))}- body, err := doHTTPRequest(ctx, httpReq, "OpenAI")+ body, err := doHTTPRequest(ctx, httpReq, "OpenAI", readOpenAIFailure)if err != nil {return nil, err}@@ -395,7 +414,7 @@ func chatOpenAIResponses(ctx context.Context, aiSetting *storepb.AISetting, requhttpReq.Header.Set("Authorization", fmt.Sprintf("Bearer %s", aiSetting.ApiKey))}- body, err := doHTTPRequest(ctx, httpReq, "OpenAI")+ body, err := doHTTPRequest(ctx, httpReq, "OpenAI", readOpenAIFailure)if err != nil {return nil, err}@@ -429,9 +448,9 @@ func chatOpenAIResponses(ctx context.Context, aiSetting *storepb.AISetting, requfor _, part := range output.Content {switch part.Type {case "output_text":- content.WriteString(part.Text)+ _, _ = content.WriteString(part.Text)case "refusal":- content.WriteString(part.Refusal)+ _, _ = content.WriteString(part.Refusal)default:continue}diff --git a/backend/plugin/ai/shared.go b/backend/plugin/ai/shared.goindex b9f5b37d6f..0ab2f0f5b0 100644--- a/backend/plugin/ai/shared.go+++ b/backend/plugin/ai/shared.go@@ -2,10 +2,10 @@ package aiimport ("context"+ "fmt""io""net/http"-- "github.com/pkg/errors"+ "time"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1")@@ -19,22 +19,32 @@ func newAIChatUsage(totalTokens int32) *v1pb.AIChatUsage {}}-// doHTTPRequest executes an HTTP request and returns the response body.-func doHTTPRequest(_ context.Context, httpReq *http.Request, provider string) ([]byte, error) {+// doHTTPRequest executes an HTTP request and returns the response body. A+// status other than 200 is an *HTTPError, to which readFailure adds what the+// vendor's body format says. A call with no complete reply is a *TransportError.+func doHTTPRequest(_ context.Context, httpReq *http.Request, provider string, readFailure func(*HTTPError)) ([]byte, error) {client := &http.Client{}httpResp, err := client.Do(httpReq)if err != nil {- return nil, errors.Errorf("failed to send HTTP request to %s: %s", provider, err)+ return nil, &TransportError{message: fmt.Sprintf("failed to send HTTP request to %s: %s", provider, err)}}defer httpResp.Body.Close()body, err := io.ReadAll(httpResp.Body)if err != nil {- return nil, errors.Errorf("failed to read %s response body: %s", provider, err)+ return nil, &TransportError{message: fmt.Sprintf("failed to read %s response body: %s", provider, err)}}if httpResp.StatusCode != http.StatusOK {- return nil, errors.Errorf("%s API returned status %d: %s", provider, httpResp.StatusCode, string(body))+ httpErr := &HTTPError{+ Provider: provider,+ StatusCode: httpResp.StatusCode,+ RetryAfter: providerWait(httpResp.Header, time.Now()),+ ShouldRetry: readShouldRetry(httpResp.Header),+ Body: string(body),+ }+ readFailure(httpErr)+ return nil, httpErr}return body, nildiff --git a/backend/plugin/aireview/model_ai.go b/backend/plugin/aireview/model_ai.goindex 1675628eab..3cf01bfb8c 100644--- a/backend/plugin/aireview/model_ai.go+++ b/backend/plugin/aireview/model_ai.go@@ -2,21 +2,121 @@ package aireviewimport ("context"+ "log/slog"+ "math/rand/v2"+ "net/http"+ "time"++ "github.com/pkg/errors"storepb "github.com/bytebase/bytebase/backend/generated-go/store"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/plugin/ai")+const (+ // maxChatRetries is how many times one model call is sent again.+ maxChatRetries = 3+ // firstRetryBackoff is the middle of the first backoff. Each retry doubles it.+ firstRetryBackoff = time.Second+)+type settingModel struct {setting *storepb.AISetting+ // sleep waits for the duration, or returns the context's error as soon as+ // the context ends. Tests replace it so that a retry takes no time.+ sleep func(ctx context.Context, wait time.Duration) error}// NewModel returns the Model behind the workspace AI setting.func NewModel(setting *storepb.AISetting) Model {- return &settingModel{setting: setting}+ return &settingModel{setting: setting, sleep: sleepContext}}+// Chat sends the request again after a transient provider failure, so the+// review loop sees one model call however many attempts it takes.func (m *settingModel) Chat(ctx context.Context, request *v1pb.AIChatRequest) (*v1pb.AIChatResponse, error) {- return ai.Chat(ctx, m.setting, request)+ for retry := 0; ; retry++ {+ response, err := ai.Chat(ctx, m.setting, request)+ if err == nil {+ return response, nil+ }+ // A context that ended is reported as such, whatever the provider+ // replied. The context is asked, not the error: a dial timeout also+ // matches context.DeadlineExceeded while the context is alive.+ if ctxErr := ctx.Err(); ctxErr != nil {+ return nil, ctxErr+ }+ if retry == maxChatRetries || !isTransientFailure(err) {+ return nil, err+ }+ wait := retryWait(err, retry)+ // A sleep that ends after the deadline would trade the provider's+ // error for a timeout.+ if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) <= wait {+ return nil, err+ }+ slog.Warn("AI review model call failed, retrying",+ slog.String("provider", m.setting.GetProvider().String()),+ slog.Int("status", httpStatus(err)),+ slog.Int("attempt", retry+1),+ slog.Duration("wait", wait))+ if err := m.sleep(ctx, wait); err != nil {+ return nil, err+ }+ }+}++// isTransientFailure reports whether the failed call should be sent again.+func isTransientFailure(err error) bool {+ var httpErr *ai.HTTPError+ if errors.As(err, &httpErr) {+ if httpErr.StatusCode == http.StatusTooManyRequests && httpErr.QuotaExhausted {+ return false+ }+ // The rest follows the official OpenAI and Anthropic Go SDKs.+ if httpErr.ShouldRetry != nil {+ return *httpErr.ShouldRetry+ }+ switch httpErr.StatusCode {+ case http.StatusRequestTimeout, http.StatusConflict, http.StatusTooManyRequests:+ return true+ default:+ return httpErr.StatusCode >= http.StatusInternalServerError+ }+ }+ var transportErr *ai.TransportError+ return errors.As(err, &transportErr)+}++// retryWait is the wait the provider asked for, or else a backoff. A wait of+// zero gets the backoff too, so a failed call is never repeated at once. The+// jitter keeps the concurrent reviews of one run from retrying at the same moment.+func retryWait(err error, retry int) time.Duration {+ var httpErr *ai.HTTPError+ if errors.As(err, &httpErr) && httpErr.RetryAfter > 0 {+ return httpErr.RetryAfter+ }+ backoff := firstRetryBackoff << retry+ return backoff/2 + rand.N(backoff)+}++// httpStatus is zero for a call that got no reply.+func httpStatus(err error) int {+ var httpErr *ai.HTTPError+ if errors.As(err, &httpErr) {+ return httpErr.StatusCode+ }+ return 0+}++func sleepContext(ctx context.Context, wait time.Duration) error {+ timer := time.NewTimer(wait)+ defer timer.Stop()+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-timer.C:+ return nil+ }}diff --git a/backend/plugin/aireview/model_ai_retry_test.go b/backend/plugin/aireview/model_ai_retry_test.gonew file mode 100644index 0000000000..b6c201d604--- /dev/null+++ b/backend/plugin/aireview/model_ai_retry_test.go@@ -0,0 +1,664 @@+package aireview++import (+ "bytes"+ "context"+ "fmt"+ "io"+ "log/slog"+ "net"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "testing/synctest"+ "time"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+ "github.com/bytebase/bytebase/backend/plugin/ai"+)++const (+ testAPIKey = "test-key-never-logged"++ geminiOK = `{"candidates": [{"content": {"parts": [{"text": "ok"}]}}], "usageMetadata": {"totalTokenCount": 7}}`+ claudeOK = `{"content": [{"type": "text", "text": "ok"}], "usage": {"input_tokens": 3, "output_tokens": 4}}`+ openAIOK = `{"choices": [{"message": {"role": "assistant", "content": "ok"}}], "usage": {"total_tokens": 7}}`++ geminiUnavailable = `{"error": {"code": 503, "message": "The model is overloaded. Please try again later.", "status": "UNAVAILABLE"}}`+ geminiRateLimit = `{"error": {"code": 429, "message": "Resource has been exhausted (e.g. check quota).", "status": "RESOURCE_EXHAUSTED", "details": [{"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "13s"}]}}`+ claudeRateLimit = `{"type": "error", "error": {"type": "rate_limit_error", "message": "This request would exceed your organization's rate limit."}}`+ claudeSpendCap = `{"type": "error", "error": {"type": "rate_limit_error", "message": "You have reached your API usage limits.", "details": {"error_code": "enforced_spend_limit_reached"}}}`+ openAIRateLimit = `{"error": {"message": "Rate limit reached for requests", "type": "requests", "code": "rate_limit_exceeded"}}`+ openAIQuota = `{"error": {"message": "You exceeded your current quota, please check your plan and billing details.", "type": "insufficient_quota", "code": "insufficient_quota"}}`+)++// providerReply is one scripted HTTP reply.+type providerReply struct {+ status int+ retryAfter string+ retryAfterMs string+ shouldRetry string+ body string+ // hangUp drops the connection without a reply.+ hangUp bool+ // handle replaces the reply. It runs on the server's goroutine.+ handle func(r *http.Request)+}++// scriptedProvider serves the replies in order and repeats the last one.+type scriptedProvider struct {+ url string+ requests atomic.Int32+}++func newScriptedProvider(t *testing.T, replies ...providerReply) *scriptedProvider {+ t.Helper()+ provider := &scriptedProvider{}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ reply := replies[min(int(provider.requests.Add(1)), len(replies))-1]+ switch {+ case reply.handle != nil:+ reply.handle(r)+ case reply.hangUp:+ hijacker, ok := w.(http.Hijacker)+ if !ok {+ t.Error("the response writer cannot be hijacked")+ return+ }+ conn, _, err := hijacker.Hijack()+ if err != nil {+ t.Errorf("failed to hijack the connection: %v", err)+ return+ }+ _ = conn.Close()+ default:+ if reply.retryAfterMs != "" {+ w.Header().Set("Retry-After-Ms", reply.retryAfterMs)+ }+ if reply.shouldRetry != "" {+ w.Header().Set("X-Should-Retry", reply.shouldRetry)+ }+ if reply.retryAfter != "" {+ w.Header().Set("Retry-After", reply.retryAfter)+ }+ w.WriteHeader(reply.status)+ _, _ = w.Write([]byte(reply.body))+ }+ }))+ t.Cleanup(server.Close)+ provider.url = server.URL+ return provider+}++// sleepRecorder stands in for settingModel.sleep. It records each wait and+// returns at once, with the context's error when the context has ended.+type sleepRecorder struct {+ waits []time.Duration+}++func (s *sleepRecorder) sleep(ctx context.Context, wait time.Duration) error {+ s.waits = append(s.waits, wait)+ return ctx.Err()+}++func newRetryModel(provider storepb.AISetting_Provider, endpoint string) (*settingModel, *sleepRecorder) {+ if provider == storepb.AISetting_OPEN_AI {+ endpoint += "/v1/chat/completions"+ }+ recorder := &sleepRecorder{}+ return &settingModel{+ setting: &storepb.AISetting{Provider: provider, Endpoint: endpoint, Model: "test-model", ApiKey: testAPIKey},+ sleep: recorder.sleep,+ }, recorder+}++func chatRequest() *v1pb.AIChatRequest {+ return &v1pb.AIChatRequest{Messages: []*v1pb.AIChatMessage{textMessage(roleUser, "hello")}}+}++// requireBackoff asserts the jittered waits of the retries, which double from+// a first wait around one second.+func requireBackoff(t *testing.T, waits []time.Duration, retries int) {+ t.Helper()+ require.Len(t, waits, retries)+ for retry, wait := range waits {+ backoff := time.Second << retry+ require.GreaterOrEqual(t, wait, backoff/2, "retry %d", retry)+ require.Less(t, wait, backoff*3/2, "retry %d", retry)+ }+}++func TestSettingModelWaitsAsLongAsTheProviderAsks(t *testing.T) {+ t.Parallel()++ tests := []struct {+ name string+ provider storepb.AISetting_Provider+ failure providerReply+ success string+ want time.Duration+ }{+ {+ name: "Claude rate limit with Retry-After",+ provider: storepb.AISetting_CLAUDE,+ failure: providerReply{status: http.StatusTooManyRequests, retryAfter: "7", body: claudeRateLimit},+ success: claudeOK,+ want: 7 * time.Second,+ },+ {+ name: "Claude rate limit with Retry-After-Ms, which wins over Retry-After",+ provider: storepb.AISetting_CLAUDE,+ failure: providerReply{status: http.StatusTooManyRequests, retryAfterMs: "4500", retryAfter: "5", body: claudeRateLimit},+ success: claudeOK,+ want: 4500 * time.Millisecond,+ },+ {+ name: "OpenAI rate limit with Retry-After",+ provider: storepb.AISetting_OPEN_AI,+ failure: providerReply{status: http.StatusTooManyRequests, retryAfter: "20", body: openAIRateLimit},+ success: openAIOK,+ want: 20 * time.Second,+ },+ {+ name: "Gemini rate limit with the delay in the body",+ provider: storepb.AISetting_GEMINI,+ failure: providerReply{status: http.StatusTooManyRequests, body: geminiRateLimit},+ success: geminiOK,+ want: 13 * time.Second,+ },+ {+ name: "Gemini unavailable with Retry-After",+ provider: storepb.AISetting_GEMINI,+ failure: providerReply{status: http.StatusServiceUnavailable, retryAfter: "2", body: geminiUnavailable},+ success: geminiOK,+ want: 2 * time.Second,+ },+ }+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t, test.failure, providerReply{status: http.StatusOK, body: test.success})+ model, recorder := newRetryModel(test.provider, provider.url)+ response, err := model.Chat(context.Background(), chatRequest())+ require.NoError(t, err)+ require.Equal(t, "ok", response.GetContent())+ require.Equal(t, int32(2), provider.requests.Load())+ require.Equal(t, []time.Duration{test.want}, recorder.waits)+ })+ }+}++func TestSettingModelBacksOffWhenTheProviderNamesNoWait(t *testing.T) {+ t.Parallel()++ // The 429 here has neither a Retry-After nor the spend cap code, so nothing+ // says it will repeat and it is retried like the other statuses.+ for _, status := range []int{+ http.StatusRequestTimeout,+ http.StatusConflict,+ http.StatusTooManyRequests,+ http.StatusInternalServerError,+ http.StatusNotImplemented,+ http.StatusBadGateway,+ http.StatusServiceUnavailable,+ http.StatusGatewayTimeout,+ 529,+ } {+ t.Run(fmt.Sprintf("%d then a reply", status), func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t,+ providerReply{status: status, body: claudeRateLimit},+ providerReply{status: http.StatusOK, body: claudeOK})+ model, recorder := newRetryModel(storepb.AISetting_CLAUDE, provider.url)+ response, err := model.Chat(context.Background(), chatRequest())+ require.NoError(t, err)+ require.Equal(t, "ok", response.GetContent())+ require.Equal(t, int32(2), provider.requests.Load())+ requireBackoff(t, recorder.waits, 1)+ })+ }++ t.Run("x-should-retry true on a 400", func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t,+ providerReply{status: http.StatusBadRequest, shouldRetry: "true", body: `{"error": {"message": "try again", "type": "invalid_request_error"}}`},+ providerReply{status: http.StatusOK, body: openAIOK})+ model, recorder := newRetryModel(storepb.AISetting_OPEN_AI, provider.url)+ _, err := model.Chat(context.Background(), chatRequest())+ require.NoError(t, err)+ require.Equal(t, int32(2), provider.requests.Load())+ requireBackoff(t, recorder.waits, 1)+ })++ t.Run("a Retry-After of zero", func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t,+ providerReply{status: http.StatusTooManyRequests, retryAfter: "0", body: claudeRateLimit},+ providerReply{status: http.StatusOK, body: claudeOK})+ model, recorder := newRetryModel(storepb.AISetting_CLAUDE, provider.url)+ _, err := model.Chat(context.Background(), chatRequest())+ require.NoError(t, err)+ require.Equal(t, int32(2), provider.requests.Load())+ requireBackoff(t, recorder.waits, 1)+ })++ t.Run("a quota code on a status other than 429 does not stop the retry", func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t,+ providerReply{status: http.StatusInternalServerError, body: openAIQuota},+ providerReply{status: http.StatusOK, body: openAIOK})+ model, recorder := newRetryModel(storepb.AISetting_OPEN_AI, provider.url)+ _, err := model.Chat(context.Background(), chatRequest())+ require.NoError(t, err)+ require.Equal(t, int32(2), provider.requests.Load())+ requireBackoff(t, recorder.waits, 1)+ })++ t.Run("gives up after the fourth attempt", func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t, providerReply{status: http.StatusServiceUnavailable, body: geminiUnavailable})+ model, recorder := newRetryModel(storepb.AISetting_GEMINI, provider.url)+ _, err := model.Chat(context.Background(), chatRequest())++ var httpErr *ai.HTTPError+ require.ErrorAs(t, err, &httpErr)+ require.Equal(t, http.StatusServiceUnavailable, httpErr.StatusCode)+ require.EqualError(t, err, "Gemini API returned status 503: "+geminiUnavailable)+ require.Equal(t, int32(4), provider.requests.Load())+ requireBackoff(t, recorder.waits, 3)+ })+}++func TestSettingModelDoesNotRetryAFailureThatWillRepeat(t *testing.T) {+ t.Parallel()++ tests := []struct {+ name string+ provider storepb.AISetting_Provider+ reply providerReply+ wantText string+ }{+ {+ name: "bad request",+ provider: storepb.AISetting_GEMINI,+ reply: providerReply{status: http.StatusBadRequest, body: `{"error": {"code": 400, "status": "INVALID_ARGUMENT"}}`},+ wantText: "Gemini API returned status 400",+ },+ {+ name: "bad key",+ provider: storepb.AISetting_OPEN_AI,+ reply: providerReply{status: http.StatusUnauthorized, body: `{"error": {"message": "Incorrect API key provided", "code": "invalid_api_key"}}`},+ wantText: "OpenAI API returned status 401",+ },+ {+ name: "forbidden",+ provider: storepb.AISetting_CLAUDE,+ reply: providerReply{status: http.StatusForbidden, body: `{"type": "error", "error": {"type": "permission_error"}}`},+ wantText: "Claude API returned status 403",+ },+ {+ name: "not found",+ provider: storepb.AISetting_GEMINI,+ reply: providerReply{status: http.StatusNotFound, body: `{"error": {"code": 404, "status": "NOT_FOUND"}}`},+ wantText: "Gemini API returned status 404",+ },+ {+ name: "a status outside the retry list, even with Retry-After",+ provider: storepb.AISetting_CLAUDE,+ reply: providerReply{status: http.StatusUnprocessableEntity, retryAfter: "1", body: `{}`},+ wantText: "Claude API returned status 422",+ },+ {+ name: "x-should-retry false on a 503",+ provider: storepb.AISetting_CLAUDE,+ reply: providerReply{status: http.StatusServiceUnavailable, shouldRetry: "false", body: `{"type": "error", "error": {"type": "api_error"}}`},+ wantText: "Claude API returned status 503",+ },+ {+ name: "OpenAI used-up quota",+ provider: storepb.AISetting_OPEN_AI,+ reply: providerReply{status: http.StatusTooManyRequests, body: openAIQuota},+ wantText: "OpenAI API returned status 429",+ },+ {+ name: "OpenAI used-up quota, even with x-should-retry true",+ provider: storepb.AISetting_OPEN_AI,+ reply: providerReply{status: http.StatusTooManyRequests, shouldRetry: "true", body: openAIQuota},+ wantText: "OpenAI API returned status 429",+ },+ {+ name: "Claude spend cap",+ provider: storepb.AISetting_CLAUDE,+ reply: providerReply{status: http.StatusTooManyRequests, body: claudeSpendCap},+ wantText: "Claude API returned status 429",+ },+ {+ name: "a reply that cannot be decoded",+ provider: storepb.AISetting_GEMINI,+ reply: providerReply{status: http.StatusOK, body: `not JSON`},+ wantText: "failed to unmarshal Gemini response",+ },+ }+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t, test.reply)+ model, recorder := newRetryModel(test.provider, provider.url)+ _, err := model.Chat(context.Background(), chatRequest())+ require.ErrorContains(t, err, test.wantText)+ require.Equal(t, int32(1), provider.requests.Load())+ require.Empty(t, recorder.waits)+ })+ }+}++func TestSettingModelRetriesATransportFailure(t *testing.T) {+ t.Parallel()++ for _, test := range []struct {+ name string+ provider storepb.AISetting_Provider+ success string+ }{+ {name: "OpenAI", provider: storepb.AISetting_OPEN_AI, success: openAIOK},+ {name: "Gemini", provider: storepb.AISetting_GEMINI, success: geminiOK},+ } {+ t.Run(test.name, func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t, providerReply{hangUp: true}, providerReply{status: http.StatusOK, body: test.success})+ model, recorder := newRetryModel(test.provider, provider.url)+ response, err := model.Chat(context.Background(), chatRequest())+ require.NoError(t, err)+ require.Equal(t, "ok", response.GetContent())+ require.Equal(t, int32(2), provider.requests.Load())+ requireBackoff(t, recorder.waits, 1)+ })+ }+}++type roundTripperFunc func(*http.Request) (*http.Response, error)++func (f roundTripperFunc) RoundTrip(request *http.Request) (*http.Response, error) {+ return f(request)+}++// replaceDefaultTransport routes ai.Chat, which sends through+// http.DefaultTransport, to roundTrip. A test that calls it must not be+// parallel. It returns the transport it replaced.+func replaceDefaultTransport(t *testing.T, roundTrip roundTripperFunc) http.RoundTripper {+ t.Helper()+ defaultTransport := http.DefaultTransport+ t.Cleanup(func() { http.DefaultTransport = defaultTransport })+ http.DefaultTransport = roundTrip+ return defaultTransport+}++func TestSettingModelRetriesADialTimeout(t *testing.T) {+ provider := newScriptedProvider(t, providerReply{status: http.StatusOK, body: geminiOK})+ model, recorder := newRetryModel(storepb.AISetting_GEMINI, provider.url)++ // A dial that is given no time fails at once with the error of a real dial timeout.+ dialTimeout := &http.Transport{DialContext: (&net.Dialer{Timeout: time.Nanosecond}).DialContext}+ roundTrips := 0+ var defaultTransport http.RoundTripper+ defaultTransport = replaceDefaultTransport(t, func(request *http.Request) (*http.Response, error) {+ roundTrips+++ if roundTrips > 1 {+ return defaultTransport.RoundTrip(request)+ }+ response, err := dialTimeout.RoundTrip(request)+ if err == nil {+ _ = response.Body.Close()+ }+ // The error matches the context's sentinel while the context is alive.+ require.ErrorIs(t, err, context.DeadlineExceeded)+ require.NoError(t, request.Context().Err())+ return nil, err+ })++ response, err := model.Chat(context.Background(), chatRequest())+ require.NoError(t, err)+ require.Equal(t, "ok", response.GetContent())+ require.Equal(t, 2, roundTrips)+ require.Equal(t, int32(1), provider.requests.Load())+ requireBackoff(t, recorder.waits, 1)+}++func TestSettingModelReportsACanceledContextOverTheReply(t *testing.T) {+ for _, status := range []int{http.StatusServiceUnavailable, http.StatusUnauthorized} {+ t.Run(fmt.Sprintf("%d", status), func(t *testing.T) {+ model, recorder := newRetryModel(storepb.AISetting_GEMINI, "http://provider.test")+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ roundTrips := 0+ replaceDefaultTransport(t, func(*http.Request) (*http.Response, error) {+ roundTrips+++ cancel()+ return &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader(geminiUnavailable))}, nil+ })++ _, err := model.Chat(ctx, chatRequest())+ require.ErrorIs(t, err, context.Canceled)+ require.Equal(t, 1, roundTrips)+ require.Empty(t, recorder.waits)+ })+ }+}++func TestSettingModelStopsWhenTheContextIsCanceled(t *testing.T) {+ t.Parallel()++ t.Run("during the wait", func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t, providerReply{status: http.StatusTooManyRequests, retryAfter: "3600", body: claudeRateLimit})+ model, _ := newRetryModel(storepb.AISetting_CLAUDE, provider.url)+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ var waits []time.Duration+ model.sleep = func(waitCtx context.Context, wait time.Duration) error {+ waits = append(waits, wait)+ cancel()+ return sleepContext(waitCtx, wait)+ }++ _, err := model.Chat(ctx, chatRequest())+ require.ErrorIs(t, err, context.Canceled)+ require.Equal(t, []time.Duration{time.Hour}, waits)+ require.Equal(t, int32(1), provider.requests.Load())+ })++ t.Run("during the call", func(t *testing.T) {+ t.Parallel()++ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ provider := newScriptedProvider(t, providerReply{handle: func(r *http.Request) {+ // The server notices that the client left only after the body is read.+ _, _ = io.Copy(io.Discard, r.Body)+ cancel()+ // Holding the reply until the client is gone keeps the test from+ // racing the cancellation against a reply.+ <-r.Context().Done()+ }})+ model, recorder := newRetryModel(storepb.AISetting_GEMINI, provider.url)++ _, err := model.Chat(ctx, chatRequest())+ require.ErrorIs(t, err, context.Canceled)+ require.Equal(t, int32(1), provider.requests.Load())+ require.Empty(t, recorder.waits)+ })++ t.Run("before the call", func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t, providerReply{status: http.StatusOK, body: geminiOK})+ model, recorder := newRetryModel(storepb.AISetting_GEMINI, provider.url)+ ctx, cancel := context.WithCancel(context.Background())+ cancel()++ _, err := model.Chat(ctx, chatRequest())+ require.ErrorIs(t, err, context.Canceled)+ require.Zero(t, provider.requests.Load())+ require.Empty(t, recorder.waits)+ })+}++func TestSettingModelDoesNotWaitPastTheDeadline(t *testing.T) {+ t.Parallel()++ t.Run("a wait that ends after the deadline returns the provider's error at once", func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t, providerReply{status: http.StatusTooManyRequests, retryAfter: "3600", body: claudeRateLimit})+ model, recorder := newRetryModel(storepb.AISetting_CLAUDE, provider.url)+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)+ defer cancel()++ _, err := model.Chat(ctx, chatRequest())+ var httpErr *ai.HTTPError+ require.ErrorAs(t, err, &httpErr)+ require.Equal(t, http.StatusTooManyRequests, httpErr.StatusCode)+ require.NotErrorIs(t, err, context.DeadlineExceeded)+ require.NoError(t, ctx.Err())+ require.Equal(t, int32(1), provider.requests.Load())+ require.Empty(t, recorder.waits)+ })++ t.Run("a wait that ends before the deadline is taken", func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t,+ providerReply{status: http.StatusTooManyRequests, retryAfter: "60", body: claudeRateLimit},+ providerReply{status: http.StatusOK, body: claudeOK})+ model, recorder := newRetryModel(storepb.AISetting_CLAUDE, provider.url)+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)+ defer cancel()++ _, err := model.Chat(ctx, chatRequest())+ require.NoError(t, err)+ require.Equal(t, int32(2), provider.requests.Load())+ require.Equal(t, []time.Duration{time.Minute}, recorder.waits)+ })+}++func TestReviewCountsARetriedModelCallOnce(t *testing.T) {+ t.Parallel()++ emptyFindings := `{"candidates": [{"content": {"parts": [{"text": "{\"findings\": []}"}]}}], "usageMetadata": {"totalTokenCount": 60}}`++ t.Run("a retry that succeeds", func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t,+ providerReply{status: http.StatusServiceUnavailable, body: geminiUnavailable},+ providerReply{status: http.StatusOK, body: emptyFindings})+ model, recorder := newRetryModel(storepb.AISetting_GEMINI, provider.url)+ result, err := NewReviewer(model).Review(context.Background(), &Request{Statement: threeLineStatement}, NoTools{})+ require.NoError(t, err)+ require.Equal(t, 1, result.Calls)+ require.Equal(t, 60, result.TotalTokens)+ require.Equal(t, int32(2), provider.requests.Load())+ requireBackoff(t, recorder.waits, 1)+ })++ t.Run("retries that all fail", func(t *testing.T) {+ t.Parallel()++ provider := newScriptedProvider(t, providerReply{status: http.StatusServiceUnavailable, body: geminiUnavailable})+ model, recorder := newRetryModel(storepb.AISetting_GEMINI, provider.url)+ _, err := NewReviewer(model).Review(context.Background(), &Request{Statement: threeLineStatement}, NoTools{})+ require.ErrorContains(t, err, "model call 1 failed: Gemini API returned status 503")+ require.Equal(t, int32(4), provider.requests.Load())+ requireBackoff(t, recorder.waits, 3)+ })+}++// lockedBuffer lets the test read the log while the handler may still write.+type lockedBuffer struct {+ mu sync.Mutex+ buffer bytes.Buffer+}++func (b *lockedBuffer) Write(p []byte) (int, error) {+ b.mu.Lock()+ defer b.mu.Unlock()+ return b.buffer.Write(p)+}++func (b *lockedBuffer) String() string {+ b.mu.Lock()+ defer b.mu.Unlock()+ return b.buffer.String()+}++// Not parallel: it replaces the default logger.+func TestSettingModelLogsEachRetryWithoutTheBodyOrTheKey(t *testing.T) {+ defaultLogger := slog.Default()+ t.Cleanup(func() { slog.SetDefault(defaultLogger) })+ var log lockedBuffer+ slog.SetDefault(slog.New(slog.NewTextHandler(&log, &slog.HandlerOptions{+ ReplaceAttr: func(_ []string, attr slog.Attr) slog.Attr {+ if attr.Key == slog.TimeKey {+ return slog.Attr{}+ }+ return attr+ },+ })))++ provider := newScriptedProvider(t,+ providerReply{status: http.StatusTooManyRequests, retryAfter: "7", body: claudeRateLimit},+ providerReply{hangUp: true},+ providerReply{status: http.StatusOK, body: claudeOK})+ model, _ := newRetryModel(storepb.AISetting_CLAUDE, provider.url)+ _, err := model.Chat(context.Background(), chatRequest())+ require.NoError(t, err)++ lines := bytes.Split(bytes.TrimSpace([]byte(log.String())), []byte("\n"))+ require.Len(t, lines, 2)+ require.Equal(t, `level=WARN msg="AI review model call failed, retrying" provider=CLAUDE status=429 attempt=1 wait=7s`, string(lines[0]))+ require.Regexp(t, `^level=WARN msg="AI review model call failed, retrying" provider=CLAUDE status=0 attempt=2 wait=[0-9.]+m?s$`, string(lines[1]))+ require.NotContains(t, log.String(), testAPIKey)+ require.NotContains(t, log.String(), "rate_limit_error")+}++func TestSleepContext(t *testing.T) {+ t.Parallel()++ t.Run("waits for the whole duration", func(t *testing.T) {+ t.Parallel()++ synctest.Test(t, func(t *testing.T) {+ start := time.Now()+ require.NoError(t, sleepContext(context.Background(), time.Hour))+ require.Equal(t, time.Hour, time.Since(start))+ })+ })++ t.Run("stops as soon as the context ends", func(t *testing.T) {+ t.Parallel()++ synctest.Test(t, func(t *testing.T) {+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ time.AfterFunc(time.Minute, cancel)+ start := time.Now()+ require.ErrorIs(t, sleepContext(ctx, time.Hour), context.Canceled)+ require.Equal(t, time.Minute, time.Since(start))+ })+ })+}