Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
Implement improved model fallbacks and error handling. Fixes #232
Dane Schneider6dd3003397fc
diff --git a/app/server/model/client_stream.go b/app/server/model/client_stream.goindex 082d8a2a..503f6f78 100644--- a/app/server/model/client_stream.go+++ b/app/server/model/client_stream.go@@ -5,6 +5,7 @@ import ("fmt""io""log"+ "math/rand""plandex-server/types"shared "plandex-shared""time"@@ -20,8 +21,13 @@ func CreateChatCompletionWithInternalStream(onStream OnStreamFn,reqStarted time.Time,) (*types.ModelResponse, error) {- client, ok := clients[modelConfig.BaseModelConfig.ApiKeyEnvVar]+ _, ok := clients[modelConfig.BaseModelConfig.ApiKeyEnvVar]if !ok {+ fmt.Printf("client not found for api key env var: %s", modelConfig.BaseModelConfig.ApiKeyEnvVar)+ if modelConfig.MissingKeyFallback != nil {+ fmt.Println("using missing key fallback")+ return CreateChatCompletionWithInternalStream(clients, modelConfig.MissingKeyFallback, ctx, req, onStream, reqStarted)+ }return nil, fmt.Errorf("client not found for api key env var: %s", modelConfig.BaseModelConfig.ApiKeyEnvVar)}@@ -35,8 +41,40 @@ func CreateChatCompletionWithInternalStream(// Force streaming mode since we're using the streaming APIreq.Stream = true- return withStreamingRetries[types.ModelResponse](ctx, func() (*types.ModelResponse, error) {- return processChatCompletionStream(modelConfig, client, modelConfig.BaseModelConfig.BaseUrl, ctx, req, onStream, reqStarted)+ return withStreamingRetries(ctx, func(numTotalRetry int, modelErr *shared.ModelError) (resp *types.ModelResponse, fallbackRes shared.FallbackResult, err error) {+ fallbackRes = modelConfig.GetFallbackForModelError(numTotalRetry, modelErr)+ resolvedModelConfig := fallbackRes.ModelRoleConfig++ if resolvedModelConfig == nil {+ return nil, fallbackRes, fmt.Errorf("model config is nil")+ }++ opClient, ok := clients[resolvedModelConfig.BaseModelConfig.ApiKeyEnvVar]++ if !ok {+ if resolvedModelConfig.MissingKeyFallback != nil {+ fmt.Println("using missing key fallback")+ resolvedModelConfig = resolvedModelConfig.MissingKeyFallback+ opClient, ok = clients[resolvedModelConfig.BaseModelConfig.ApiKeyEnvVar]+ if !ok {+ return nil, fallbackRes, fmt.Errorf("client not found for api key env var: %s", resolvedModelConfig.BaseModelConfig.ApiKeyEnvVar)+ }+ } else {+ return nil, fallbackRes, fmt.Errorf("client not found for api key env var: %s", resolvedModelConfig.BaseModelConfig.ApiKeyEnvVar)+ }+ }++ modelConfig = resolvedModelConfig+ resp, err = processChatCompletionStream(resolvedModelConfig, opClient, resolvedModelConfig.BaseModelConfig.BaseUrl, ctx, req, onStream, reqStarted)+ if err != nil {+ return nil, fallbackRes, err+ }+ return resp, fallbackRes, nil+ }, func(resp *types.ModelResponse, err error) {+ if resp != nil {+ resp.Stopped = true+ resp.Error = err.Error()+ }})}@@ -50,17 +88,19 @@ func processChatCompletionStream(reqStarted time.Time,) (*types.ModelResponse, error) {streamCtx, cancel := context.WithCancel(ctx)- defer cancel()stream, err := createChatCompletionStreamExtended(modelConfig, client, baseUrl, streamCtx, req)if err != nil {+ cancel()return nil, fmt.Errorf("error creating chat completion stream: %w", err)}+defer stream.Close()+ defer cancel()accumulator := types.NewStreamCompletionAccumulator()// Create a timer that will trigger if no chunk is received within the specified duration- timer := time.NewTimer(OPENAI_STREAM_CHUNK_TIMEOUT)+ timer := time.NewTimer(ACTIVE_STREAM_CHUNK_TIMEOUT)defer timer.Stop()streamFinished := false@@ -108,7 +148,7 @@ func processChatCompletionStream(if !timer.Stop() {<-timer.C}- timer.Reset(OPENAI_STREAM_CHUNK_TIMEOUT)+ timer.Reset(ACTIVE_STREAM_CHUNK_TIMEOUT)// Process the responseif response.Usage != nil {@@ -145,7 +185,7 @@ func processChatCompletionStream(if !timer.Stop() {<-timer.C}- timer.Reset(OPENAI_USAGE_CHUNK_TIMEOUT)+ timer.Reset(USAGE_CHUNK_TIMEOUT)streamFinished = truecontinue}@@ -177,57 +217,97 @@ func processChatCompletionStream(func withStreamingRetries[T any](ctx context.Context,- operation func() (*types.ModelResponse, error),-) (*types.ModelResponse, error) {- var result *types.ModelResponse- var numRetry int+ operation func(numRetry int, modelErr *shared.ModelError) (resp *T, fallbackRes shared.FallbackResult, err error),+ onContextDone func(resp *T, err error),+) (*T, error) {+ var resp *T+ var numTotalRetry int+ var numFallbackRetry int+ var fallbackRes shared.FallbackResult+ var modelErr *shared.ModelErrorfor {if ctx.Err() != nil {- if result != nil {+ if resp != nil {// Return partial result with context error- result.Stopped = true- result.Error = ctx.Err().Error()- return result, ctx.Err()+ onContextDone(resp, ctx.Err())+ return resp, ctx.Err()}return nil, ctx.Err()}- resp, err := operation()+ var err error++ var numRetry int+ if numFallbackRetry > 0 {+ numRetry = numFallbackRetry+ } else {+ numRetry = numTotalRetry+ }++ log.Printf("withStreamingRetries - will run operation")++ resp, fallbackRes, err = operation(numTotalRetry, modelErr)if err == nil {return resp, nil}- // Store the partial result for potential return- result = resp+ log.Printf("withStreamingRetries - operation returned error: %v", err)- log.Printf("Error in streaming operation: %v, retry: %d\n", err, numRetry)-- if isNonRetriableErr(err) {- return result, err+ isFallback := fallbackRes.IsFallback+ maxRetries := MAX_RETRIES_WITHOUT_FALLBACK+ if isFallback {+ maxRetries = MAX_ADDITIONAL_RETRIES_WITH_FALLBACK}- if numRetry >= OPENAI_MAX_RETRIES {- log.Println("Max retries reached - no retry")- return result, err+ compareRetries := numTotalRetry+ if isFallback {+ compareRetries = numFallbackRetry}- // Handle retry timing- if duration := parseRetryAfter(err.Error()); duration != nil {- log.Printf("Retry duration found: %v\n", *duration)- waitDuration := time.Duration(float64(*duration) * OPENAI_BACKOFF_MULTIPLIER)-- if waitDuration > OPENAI_ABORT_WAIT_DURATION {- return result, err- } else if waitDuration > OPENAI_MAX_WAIT_DURATION {- waitDuration = OPENAI_MAX_WAIT_DURATION+ log.Printf("Error in streaming operation: %v, isFallback: %t, numTotalRetry: %d, numFallbackRetry: %d, numRetry: %d, compareRetries: %d, maxRetries: %d\n", err, isFallback, numTotalRetry, numFallbackRetry, numRetry, compareRetries, maxRetries)++ classifyRes := classifyBasicError(err)+ modelErr = &classifyRes++ newFallback := false+ if !modelErr.Retriable {+ log.Printf("withStreamingRetries - operation returned non-retriable error: %v", err)+ if modelErr.Kind == shared.ErrContextTooLong && fallbackRes.ModelRoleConfig.LargeContextFallback == nil {+ log.Printf("withStreamingRetries - non-retriable context too long error and no large context fallback is defined, returning error")+ // if it's a context too long error and no large context fallback is defined, return the error+ return resp, err+ } else if modelErr.Kind != shared.ErrContextTooLong && fallbackRes.ModelRoleConfig.ErrorFallback == nil {+ log.Printf("withStreamingRetries - non-retriable error and no error fallback is defined, returning error")+ // if it's any other error and no error fallback is defined, return the error+ return resp, err}+ log.Printf("withStreamingRetries - operation returned non-retriable error, but has fallback - resetting numFallbackRetry to 0 and continuing to retry")+ numFallbackRetry = 0+ newFallback = true+ // otherwise, continue to retry logic+ }- time.Sleep(waitDuration)+ if compareRetries >= maxRetries {+ log.Printf("withStreamingRetries - compareRetries >= maxRetries - returning error")+ return resp, err+ }++ var retryDelay time.Duration+ if modelErr != nil && modelErr.RetryAfterSeconds > 0 {+ // if the model err has a retry after, then use that with a bit of padding+ retryDelay = time.Duration(int(float64(modelErr.RetryAfterSeconds)*1.1)) * time.Second} else {- waitBackoff(numRetry)+ // otherwise, use some jitter+ retryDelay = time.Duration(1000+rand.Intn(200)) * time.Millisecond}- numRetry+++ log.Printf("withStreamingRetries - retrying stream in %v seconds", retryDelay)+ time.Sleep(retryDelay)++ numTotalRetry+++ if isFallback && !newFallback {+ numFallbackRetry+++ }}}diff --git a/app/server/model/model_error.go b/app/server/model/model_error.gonew file mode 100644index 00000000..7be992c3--- /dev/null+++ b/app/server/model/model_error.go@@ -0,0 +1,227 @@+package model++import (+ "fmt"+ "log"+ "net/http"+ shared "plandex-shared"+ "regexp"+ "strconv"+ "strings"+ "time"+)++type HTTPError struct {+ StatusCode int+ Body string+ Header http.Header+}++func (e *HTTPError) Error() string {+ return fmt.Sprintf("status code: %d, body: %s", e.StatusCode, e.Body)+}++// JSON-style `"retry_after_ms":1234`+var reJSON = regexp.MustCompile(`"retry_after_ms"\s*:\s*(\d+)`)++// Header- or text-style "Retry-After: 12" / "retry_after: 12s"+var reRetryAfter = regexp.MustCompile(+ `retry[_\-\s]?after[_\-\s]?(?:[:\s]+)?(\d+)(ms|seconds?|secs?|s)?`,+)++// Free-form Azure style "Try again in 59 seconds."+// Also matches "Retry in 10 seconds."+var reTryAgain = regexp.MustCompile(+ `(?:re)?try[_\-\s]+(?:again[_\-\s]+)?in[_\-\s]+(\d+)(ms|seconds?|secs?|s)?`,+)++func ClassifyModelError(code int, message string, headers http.Header) shared.ModelError {+ msg := strings.ToLower(message)++ var res shared.ModelError++ switch code {+ case 429, 529:++ res = shared.ModelError{+ Kind: shared.ErrRateLimited,+ Retriable: true,+ RetryAfterSeconds: 0,+ }+ case 413:+ res = shared.ModelError{+ Kind: shared.ErrContextTooLong,+ Retriable: false,+ RetryAfterSeconds: 0,+ }+ case 400:+ switch {+ case strings.Contains(msg, "model_overloaded") ||+ strings.Contains(msg, "server is overloaded") ||+ strings.Contains(msg, "model is currently overloaded") ||+ strings.Contains(msg, "overloaded_error") ||+ strings.Contains(msg, "resource has been exhausted"):+ res = shared.ModelError{+ Kind: shared.ErrOverloaded,+ Retriable: true,+ RetryAfterSeconds: 0,+ }+ case strings.Contains(msg, "maximum context length") ||+ strings.Contains(msg, "context length exceeded") ||+ strings.Contains(msg, "too many tokens") ||+ strings.Contains(msg, "payload too large"):+ res = shared.ModelError{+ Kind: shared.ErrContextTooLong,+ Retriable: false,+ RetryAfterSeconds: 0,+ }+ default:+ res = shared.ModelError{+ Kind: shared.ErrOther,+ Retriable: false,+ RetryAfterSeconds: 0,+ }+ }++ // rare codes but they never succeed on retry if they do show up+ case 501, 505:+ res = shared.ModelError{+ Kind: shared.ErrOther,+ Retriable: false,+ RetryAfterSeconds: 0,+ }+ default:++ res = shared.ModelError{+ Kind: shared.ErrOther,+ Retriable: code >= 500,+ RetryAfterSeconds: 0,+ }+ }++ log.Printf("Model error: %+v", res)++ // best‑effort parse of "Retry‑After" style hints in the message+ if res.Retriable {+ retryAfter := extractRetryAfter(headers, msg)++ // if the retry after is greater than the max delay, then the error is not retriable+ if retryAfter > MAX_RETRY_DELAY_SECONDS {+ log.Printf("Retry after %d seconds is greater than the max delay of %d seconds - not retriable", retryAfter, MAX_RETRY_DELAY_SECONDS)+ res.Retriable = false+ } else {+ res.RetryAfterSeconds = retryAfter+ }++ }++ return res+}++func extractRetryAfter(h http.Header, body string) (sec int) {+ now := time.Now()++ // Retry-After header: seconds or HTTP-date+ if v := h.Get("Retry-After"); v != "" {+ if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {+ return n+ }+ if t, err := time.Parse(http.TimeFormat, v); err == nil {+ d := int(t.Sub(now).Seconds())+ if d > 0 {+ return d+ }+ }+ }++ // X-RateLimit-Reset epoch+ if v := h.Get("X-RateLimit-Reset"); v != "" {+ if reset, _ := strconv.ParseInt(v, 10, 64); reset > now.Unix() {+ return int(reset - now.Unix())+ }+ }++ lower := strings.ToLower(strings.TrimSpace(body))++ // "retry_after_ms": 1234+ if m := reJSON.FindStringSubmatch(lower); len(m) == 2 {+ n, _ := strconv.Atoi(m[1])+ return n / 1000+ }+ // "retry after 12"+ if m := reRetryAfter.FindStringSubmatch(lower); len(m) >= 2 {+ unit := ""+ if len(m) == 3 {+ unit = m[2]+ }+ return normalizeUnit(m[1], unit)+ }++ // "try again in 8"+ if m := reTryAgain.FindStringSubmatch(lower); len(m) >= 2 {+ unit := ""+ if len(m) == 3 {+ unit = m[2]+ }+ return normalizeUnit(m[1], unit)+ }+ return 0+}++func normalizeUnit(numStr, unit string) int {+ n, _ := strconv.Atoi(numStr) // safe because the regex matched \d+++ switch unit {+ case "ms": // milliseconds+ return n / 1000+ case "sec", "secs", "second", "seconds", "s":+ return n // already in seconds+ default: // unit omitted ⇒ assume seconds+ return n+ }+}++func classifyBasicError(err error) shared.ModelError {+ if httpErr, ok := err.(*HTTPError); ok {+ me := ClassifyModelError(+ httpErr.StatusCode,+ httpErr.Body,+ httpErr.Header,+ )+ return me+ }++ // Fall back to your old heuristic – still keeps the signature identical+ if isNonRetriableBasicErr(err) {+ return shared.ModelError{Kind: shared.ErrOther, Retriable: false}+ }+ return shared.ModelError{Kind: shared.ErrOther, Retriable: true}+}++func isNonRetriableBasicErr(err error) bool {+ errStr := err.Error()++ // we don't want to retry on the errors below+ if strings.Contains(errStr, "context deadline exceeded") || strings.Contains(errStr, "context canceled") {+ log.Println("Context deadline exceeded or canceled - no retry")+ return true+ }++ if strings.Contains(errStr, "status code: 400") &&+ strings.Contains(errStr, "reduce the length of the messages") {+ log.Println("Token limit exceeded - no retry")+ return true+ }++ if strings.Contains(errStr, "status code: 401") {+ log.Println("Invalid auth or api key - no retry")+ return true+ }++ if strings.Contains(errStr, "status code: 429") && strings.Contains(errStr, "exceeded your current quota") {+ log.Println("Current quota exceeded - no retry")+ return true+ }++ return false+}diff --git a/app/shared/ai_models_errors.go b/app/shared/ai_models_errors.gonew file mode 100644index 00000000..42fbd482--- /dev/null+++ b/app/shared/ai_models_errors.go@@ -0,0 +1,64 @@+package shared++type ModelErrKind string++const (+ ErrOverloaded ModelErrKind = "ErrOverloaded"+ ErrContextTooLong ModelErrKind = "ErrContextTooLong"+ ErrRateLimited ModelErrKind = "ErrRateLimited"+ ErrOther ModelErrKind = "ErrOther"+)++type ModelError struct {+ Kind ModelErrKind+ Retriable bool+ RetryAfterSeconds int+}++// if fallback is defined, retry with main model, then remaining tries use error fallback+type FallbackType string++const (+ FallbackTypeError FallbackType = "error"+ FallbackTypeContext FallbackType = "context"+)++type FallbackResult struct {+ ModelRoleConfig *ModelRoleConfig+ HasErrorFallback bool+ IsFallback bool+ FallbackType FallbackType+}++const MAX_RETRIES_BEFORE_FALLBACK = 1++func (m *ModelRoleConfig) GetFallbackForModelError(numTotalRetry int, modelErr *ModelError) FallbackResult {+ if m == nil || modelErr == nil {+ return FallbackResult{+ ModelRoleConfig: m,+ IsFallback: false,+ }+ }+ if modelErr.Kind == ErrContextTooLong {+ if m.LargeContextFallback != nil {+ return FallbackResult{+ ModelRoleConfig: m.LargeContextFallback,+ FallbackType: FallbackTypeContext,+ IsFallback: true,+ }+ }+ } else if !modelErr.Retriable || numTotalRetry > MAX_RETRIES_BEFORE_FALLBACK {+ if m.ErrorFallback != nil {+ return FallbackResult{+ ModelRoleConfig: m.ErrorFallback,+ FallbackType: FallbackTypeError,+ IsFallback: true,+ }+ }+ }++ return FallbackResult{+ ModelRoleConfig: m,+ IsFallback: false,+ }+}diff --git a/app/shared/ai_models_large_context.go b/app/shared/ai_models_large_context.gonew file mode 100644index 00000000..aac4bba7--- /dev/null+++ b/app/shared/ai_models_large_context.go@@ -0,0 +1,95 @@+package shared++const maxFallbackDepth = 10 // max fallback depth for large context fallback - should never be reached in real scenarios, but protects against infinite loops in case of circular references etc.++func (m ModelRoleConfig) GetFinalLargeContextFallback() ModelRoleConfig {+ var currentConfig ModelRoleConfig = m+ var n int = 0++ for {+ if currentConfig.LargeContextFallback == nil {+ return currentConfig+ } else {+ currentConfig = *currentConfig.LargeContextFallback+ }+ n+++ if n > maxFallbackDepth {+ break+ }+ }++ return currentConfig+}++func (m ModelRoleConfig) GetFinalLargeOutputFallback() ModelRoleConfig {+ var currentConfig ModelRoleConfig = m+ var n int = 0++ if currentConfig.LargeOutputFallback == nil {+ return currentConfig.GetFinalLargeContextFallback()+ }++ for {+ if currentConfig.LargeOutputFallback == nil {+ return currentConfig+ } else {+ currentConfig = *currentConfig.LargeOutputFallback+ }+ n+++ if n > maxFallbackDepth {+ break+ }+ }++ return currentConfig+}++// note that if the token number exeeds all the fallback models, it will return the last fallback model++func (m ModelRoleConfig) GetRoleForInputTokens(inputTokens int) ModelRoleConfig {+ inputTokens = int(float64(inputTokens) * (1 + m.BaseModelConfig.TokenEstimatePaddingPct))+ var currentConfig ModelRoleConfig = m+ var n int = 0+ for {+ if currentConfig.BaseModelConfig.MaxTokens >= inputTokens {+ return currentConfig+ }++ if currentConfig.LargeContextFallback == nil {+ return currentConfig+ } else {+ currentConfig = *currentConfig.LargeContextFallback+ }+ n+++ if n > maxFallbackDepth {+ break+ }+ }+ return currentConfig+}++func (m ModelRoleConfig) GetRoleForOutputTokens(outputTokens int) ModelRoleConfig {+ outputTokens = int(float64(outputTokens) * (1 + m.BaseModelConfig.TokenEstimatePaddingPct))+ var currentConfig ModelRoleConfig = m+ var n int = 0+ for {+ if currentConfig.GetReservedOutputTokens() >= outputTokens {+ return currentConfig+ }++ if currentConfig.LargeOutputFallback == nil {+ if currentConfig.LargeContextFallback == nil {+ return currentConfig+ } else {+ currentConfig = *currentConfig.LargeContextFallback+ }+ } else {+ currentConfig = *currentConfig.LargeOutputFallback+ }+ n+++ if n > maxFallbackDepth {+ break+ }+ }+ return currentConfig+}