Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

Implement improved model fallbacks and error handling. Fixes #232

Dane Schneider6dd3003397fc
diff --git a/app/server/model/client_stream.go b/app/server/model/client_stream.go
index 082d8a2a..503f6f78 100644
--- a/app/server/model/client_stream.go
+++ b/app/server/model/client_stream.go
@@ -5,6 +5,7 @@ import (
"fmt"
"io"
"log"
+ "math/rand"
"plandex-server/types"
shared "plandex-shared"
"time"
@@ -20,8 +21,13 @@ func CreateChatCompletionWithInternalStream(
onStream OnStreamFn,
reqStarted time.Time,
) (*types.ModelResponse, error) {
- client, ok := clients[modelConfig.BaseModelConfig.ApiKeyEnvVar]
+ _, ok := clients[modelConfig.BaseModelConfig.ApiKeyEnvVar]
if !ok {
+ fmt.Printf("client not found for api key env var: %s", modelConfig.BaseModelConfig.ApiKeyEnvVar)
+ if modelConfig.MissingKeyFallback != nil {
+ fmt.Println("using missing key fallback")
+ return CreateChatCompletionWithInternalStream(clients, modelConfig.MissingKeyFallback, ctx, req, onStream, reqStarted)
+ }
return nil, fmt.Errorf("client not found for api key env var: %s", modelConfig.BaseModelConfig.ApiKeyEnvVar)
}
@@ -35,8 +41,40 @@ func CreateChatCompletionWithInternalStream(
// Force streaming mode since we're using the streaming API
req.Stream = true
- return withStreamingRetries[types.ModelResponse](ctx, func() (*types.ModelResponse, error) {
- return processChatCompletionStream(modelConfig, client, modelConfig.BaseModelConfig.BaseUrl, ctx, req, onStream, reqStarted)
+ return withStreamingRetries(ctx, func(numTotalRetry int, modelErr *shared.ModelError) (resp *types.ModelResponse, fallbackRes shared.FallbackResult, err error) {
+ fallbackRes = modelConfig.GetFallbackForModelError(numTotalRetry, modelErr)
+ resolvedModelConfig := fallbackRes.ModelRoleConfig
+
+ if resolvedModelConfig == nil {
+ return nil, fallbackRes, fmt.Errorf("model config is nil")
+ }
+
+ opClient, ok := clients[resolvedModelConfig.BaseModelConfig.ApiKeyEnvVar]
+
+ if !ok {
+ if resolvedModelConfig.MissingKeyFallback != nil {
+ fmt.Println("using missing key fallback")
+ resolvedModelConfig = resolvedModelConfig.MissingKeyFallback
+ opClient, ok = clients[resolvedModelConfig.BaseModelConfig.ApiKeyEnvVar]
+ if !ok {
+ return nil, fallbackRes, fmt.Errorf("client not found for api key env var: %s", resolvedModelConfig.BaseModelConfig.ApiKeyEnvVar)
+ }
+ } else {
+ return nil, fallbackRes, fmt.Errorf("client not found for api key env var: %s", resolvedModelConfig.BaseModelConfig.ApiKeyEnvVar)
+ }
+ }
+
+ modelConfig = resolvedModelConfig
+ resp, err = processChatCompletionStream(resolvedModelConfig, opClient, resolvedModelConfig.BaseModelConfig.BaseUrl, ctx, req, onStream, reqStarted)
+ if err != nil {
+ return nil, fallbackRes, err
+ }
+ return resp, fallbackRes, nil
+ }, func(resp *types.ModelResponse, err error) {
+ if resp != nil {
+ resp.Stopped = true
+ resp.Error = err.Error()
+ }
})
}
@@ -50,17 +88,19 @@ func processChatCompletionStream(
reqStarted time.Time,
) (*types.ModelResponse, error) {
streamCtx, cancel := context.WithCancel(ctx)
- defer cancel()
stream, err := createChatCompletionStreamExtended(modelConfig, client, baseUrl, streamCtx, req)
if err != nil {
+ cancel()
return nil, fmt.Errorf("error creating chat completion stream: %w", err)
}
+
defer stream.Close()
+ defer cancel()
accumulator := types.NewStreamCompletionAccumulator()
// Create a timer that will trigger if no chunk is received within the specified duration
- timer := time.NewTimer(OPENAI_STREAM_CHUNK_TIMEOUT)
+ timer := time.NewTimer(ACTIVE_STREAM_CHUNK_TIMEOUT)
defer timer.Stop()
streamFinished := false
@@ -108,7 +148,7 @@ func processChatCompletionStream(
if !timer.Stop() {
<-timer.C
}
- timer.Reset(OPENAI_STREAM_CHUNK_TIMEOUT)
+ timer.Reset(ACTIVE_STREAM_CHUNK_TIMEOUT)
// Process the response
if response.Usage != nil {
@@ -145,7 +185,7 @@ func processChatCompletionStream(
if !timer.Stop() {
<-timer.C
}
- timer.Reset(OPENAI_USAGE_CHUNK_TIMEOUT)
+ timer.Reset(USAGE_CHUNK_TIMEOUT)
streamFinished = true
continue
}
@@ -177,57 +217,97 @@ func processChatCompletionStream(
func withStreamingRetries[T any](
ctx context.Context,
- operation func() (*types.ModelResponse, error),
-) (*types.ModelResponse, error) {
- var result *types.ModelResponse
- var numRetry int
+ operation func(numRetry int, modelErr *shared.ModelError) (resp *T, fallbackRes shared.FallbackResult, err error),
+ onContextDone func(resp *T, err error),
+) (*T, error) {
+ var resp *T
+ var numTotalRetry int
+ var numFallbackRetry int
+ var fallbackRes shared.FallbackResult
+ var modelErr *shared.ModelError
for {
if ctx.Err() != nil {
- if result != nil {
+ if resp != nil {
// Return partial result with context error
- result.Stopped = true
- result.Error = ctx.Err().Error()
- return result, ctx.Err()
+ onContextDone(resp, ctx.Err())
+ return resp, ctx.Err()
}
return nil, ctx.Err()
}
- resp, err := operation()
+ var err error
+
+ var numRetry int
+ if numFallbackRetry > 0 {
+ numRetry = numFallbackRetry
+ } else {
+ numRetry = numTotalRetry
+ }
+
+ log.Printf("withStreamingRetries - will run operation")
+
+ resp, fallbackRes, err = operation(numTotalRetry, modelErr)
if err == nil {
return resp, nil
}
- // Store the partial result for potential return
- result = resp
+ log.Printf("withStreamingRetries - operation returned error: %v", err)
- log.Printf("Error in streaming operation: %v, retry: %d\n", err, numRetry)
-
- if isNonRetriableErr(err) {
- return result, err
+ isFallback := fallbackRes.IsFallback
+ maxRetries := MAX_RETRIES_WITHOUT_FALLBACK
+ if isFallback {
+ maxRetries = MAX_ADDITIONAL_RETRIES_WITH_FALLBACK
}
- if numRetry >= OPENAI_MAX_RETRIES {
- log.Println("Max retries reached - no retry")
- return result, err
+ compareRetries := numTotalRetry
+ if isFallback {
+ compareRetries = numFallbackRetry
}
- // Handle retry timing
- if duration := parseRetryAfter(err.Error()); duration != nil {
- log.Printf("Retry duration found: %v\n", *duration)
- waitDuration := time.Duration(float64(*duration) * OPENAI_BACKOFF_MULTIPLIER)
-
- if waitDuration > OPENAI_ABORT_WAIT_DURATION {
- return result, err
- } else if waitDuration > OPENAI_MAX_WAIT_DURATION {
- waitDuration = OPENAI_MAX_WAIT_DURATION
+ log.Printf("Error in streaming operation: %v, isFallback: %t, numTotalRetry: %d, numFallbackRetry: %d, numRetry: %d, compareRetries: %d, maxRetries: %d\n", err, isFallback, numTotalRetry, numFallbackRetry, numRetry, compareRetries, maxRetries)
+
+ classifyRes := classifyBasicError(err)
+ modelErr = &classifyRes
+
+ newFallback := false
+ if !modelErr.Retriable {
+ log.Printf("withStreamingRetries - operation returned non-retriable error: %v", err)
+ if modelErr.Kind == shared.ErrContextTooLong && fallbackRes.ModelRoleConfig.LargeContextFallback == nil {
+ log.Printf("withStreamingRetries - non-retriable context too long error and no large context fallback is defined, returning error")
+ // if it's a context too long error and no large context fallback is defined, return the error
+ return resp, err
+ } else if modelErr.Kind != shared.ErrContextTooLong && fallbackRes.ModelRoleConfig.ErrorFallback == nil {
+ log.Printf("withStreamingRetries - non-retriable error and no error fallback is defined, returning error")
+ // if it's any other error and no error fallback is defined, return the error
+ return resp, err
}
+ log.Printf("withStreamingRetries - operation returned non-retriable error, but has fallback - resetting numFallbackRetry to 0 and continuing to retry")
+ numFallbackRetry = 0
+ newFallback = true
+ // otherwise, continue to retry logic
+ }
- time.Sleep(waitDuration)
+ if compareRetries >= maxRetries {
+ log.Printf("withStreamingRetries - compareRetries >= maxRetries - returning error")
+ return resp, err
+ }
+
+ var retryDelay time.Duration
+ if modelErr != nil && modelErr.RetryAfterSeconds > 0 {
+ // if the model err has a retry after, then use that with a bit of padding
+ retryDelay = time.Duration(int(float64(modelErr.RetryAfterSeconds)*1.1)) * time.Second
} else {
- waitBackoff(numRetry)
+ // otherwise, use some jitter
+ retryDelay = time.Duration(1000+rand.Intn(200)) * time.Millisecond
}
- numRetry++
+ log.Printf("withStreamingRetries - retrying stream in %v seconds", retryDelay)
+ time.Sleep(retryDelay)
+
+ numTotalRetry++
+ if isFallback && !newFallback {
+ numFallbackRetry++
+ }
}
}
diff --git a/app/server/model/model_error.go b/app/server/model/model_error.go
new file mode 100644
index 00000000..7be992c3
--- /dev/null
+++ b/app/server/model/model_error.go
@@ -0,0 +1,227 @@
+package model
+
+import (
+ "fmt"
+ "log"
+ "net/http"
+ shared "plandex-shared"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+)
+
+type HTTPError struct {
+ StatusCode int
+ Body string
+ Header http.Header
+}
+
+func (e *HTTPError) Error() string {
+ return fmt.Sprintf("status code: %d, body: %s", e.StatusCode, e.Body)
+}
+
+// JSON-style `"retry_after_ms":1234`
+var reJSON = regexp.MustCompile(`"retry_after_ms"\s*:\s*(\d+)`)
+
+// Header- or text-style "Retry-After: 12" / "retry_after: 12s"
+var reRetryAfter = regexp.MustCompile(
+ `retry[_\-\s]?after[_\-\s]?(?:[:\s]+)?(\d+)(ms|seconds?|secs?|s)?`,
+)
+
+// Free-form Azure style "Try again in 59 seconds."
+// Also matches "Retry in 10 seconds."
+var reTryAgain = regexp.MustCompile(
+ `(?:re)?try[_\-\s]+(?:again[_\-\s]+)?in[_\-\s]+(\d+)(ms|seconds?|secs?|s)?`,
+)
+
+func ClassifyModelError(code int, message string, headers http.Header) shared.ModelError {
+ msg := strings.ToLower(message)
+
+ var res shared.ModelError
+
+ switch code {
+ case 429, 529:
+
+ res = shared.ModelError{
+ Kind: shared.ErrRateLimited,
+ Retriable: true,
+ RetryAfterSeconds: 0,
+ }
+ case 413:
+ res = shared.ModelError{
+ Kind: shared.ErrContextTooLong,
+ Retriable: false,
+ RetryAfterSeconds: 0,
+ }
+ case 400:
+ switch {
+ case strings.Contains(msg, "model_overloaded") ||
+ strings.Contains(msg, "server is overloaded") ||
+ strings.Contains(msg, "model is currently overloaded") ||
+ strings.Contains(msg, "overloaded_error") ||
+ strings.Contains(msg, "resource has been exhausted"):
+ res = shared.ModelError{
+ Kind: shared.ErrOverloaded,
+ Retriable: true,
+ RetryAfterSeconds: 0,
+ }
+ case strings.Contains(msg, "maximum context length") ||
+ strings.Contains(msg, "context length exceeded") ||
+ strings.Contains(msg, "too many tokens") ||
+ strings.Contains(msg, "payload too large"):
+ res = shared.ModelError{
+ Kind: shared.ErrContextTooLong,
+ Retriable: false,
+ RetryAfterSeconds: 0,
+ }
+ default:
+ res = shared.ModelError{
+ Kind: shared.ErrOther,
+ Retriable: false,
+ RetryAfterSeconds: 0,
+ }
+ }
+
+ // rare codes but they never succeed on retry if they do show up
+ case 501, 505:
+ res = shared.ModelError{
+ Kind: shared.ErrOther,
+ Retriable: false,
+ RetryAfterSeconds: 0,
+ }
+ default:
+
+ res = shared.ModelError{
+ Kind: shared.ErrOther,
+ Retriable: code >= 500,
+ RetryAfterSeconds: 0,
+ }
+ }
+
+ log.Printf("Model error: %+v", res)
+
+ // best‑effort parse of "Retry‑After" style hints in the message
+ if res.Retriable {
+ retryAfter := extractRetryAfter(headers, msg)
+
+ // if the retry after is greater than the max delay, then the error is not retriable
+ if retryAfter > MAX_RETRY_DELAY_SECONDS {
+ log.Printf("Retry after %d seconds is greater than the max delay of %d seconds - not retriable", retryAfter, MAX_RETRY_DELAY_SECONDS)
+ res.Retriable = false
+ } else {
+ res.RetryAfterSeconds = retryAfter
+ }
+
+ }
+
+ return res
+}
+
+func extractRetryAfter(h http.Header, body string) (sec int) {
+ now := time.Now()
+
+ // Retry-After header: seconds or HTTP-date
+ if v := h.Get("Retry-After"); v != "" {
+ if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {
+ return n
+ }
+ if t, err := time.Parse(http.TimeFormat, v); err == nil {
+ d := int(t.Sub(now).Seconds())
+ if d > 0 {
+ return d
+ }
+ }
+ }
+
+ // X-RateLimit-Reset epoch
+ if v := h.Get("X-RateLimit-Reset"); v != "" {
+ if reset, _ := strconv.ParseInt(v, 10, 64); reset > now.Unix() {
+ return int(reset - now.Unix())
+ }
+ }
+
+ lower := strings.ToLower(strings.TrimSpace(body))
+
+ // "retry_after_ms": 1234
+ if m := reJSON.FindStringSubmatch(lower); len(m) == 2 {
+ n, _ := strconv.Atoi(m[1])
+ return n / 1000
+ }
+ // "retry after 12"
+ if m := reRetryAfter.FindStringSubmatch(lower); len(m) >= 2 {
+ unit := ""
+ if len(m) == 3 {
+ unit = m[2]
+ }
+ return normalizeUnit(m[1], unit)
+ }
+
+ // "try again in 8"
+ if m := reTryAgain.FindStringSubmatch(lower); len(m) >= 2 {
+ unit := ""
+ if len(m) == 3 {
+ unit = m[2]
+ }
+ return normalizeUnit(m[1], unit)
+ }
+ return 0
+}
+
+func normalizeUnit(numStr, unit string) int {
+ n, _ := strconv.Atoi(numStr) // safe because the regex matched \d+
+
+ switch unit {
+ case "ms": // milliseconds
+ return n / 1000
+ case "sec", "secs", "second", "seconds", "s":
+ return n // already in seconds
+ default: // unit omitted ⇒ assume seconds
+ return n
+ }
+}
+
+func classifyBasicError(err error) shared.ModelError {
+ if httpErr, ok := err.(*HTTPError); ok {
+ me := ClassifyModelError(
+ httpErr.StatusCode,
+ httpErr.Body,
+ httpErr.Header,
+ )
+ return me
+ }
+
+ // Fall back to your old heuristic – still keeps the signature identical
+ if isNonRetriableBasicErr(err) {
+ return shared.ModelError{Kind: shared.ErrOther, Retriable: false}
+ }
+ return shared.ModelError{Kind: shared.ErrOther, Retriable: true}
+}
+
+func isNonRetriableBasicErr(err error) bool {
+ errStr := err.Error()
+
+ // we don't want to retry on the errors below
+ if strings.Contains(errStr, "context deadline exceeded") || strings.Contains(errStr, "context canceled") {
+ log.Println("Context deadline exceeded or canceled - no retry")
+ return true
+ }
+
+ if strings.Contains(errStr, "status code: 400") &&
+ strings.Contains(errStr, "reduce the length of the messages") {
+ log.Println("Token limit exceeded - no retry")
+ return true
+ }
+
+ if strings.Contains(errStr, "status code: 401") {
+ log.Println("Invalid auth or api key - no retry")
+ return true
+ }
+
+ if strings.Contains(errStr, "status code: 429") && strings.Contains(errStr, "exceeded your current quota") {
+ log.Println("Current quota exceeded - no retry")
+ return true
+ }
+
+ return false
+}
diff --git a/app/shared/ai_models_errors.go b/app/shared/ai_models_errors.go
new file mode 100644
index 00000000..42fbd482
--- /dev/null
+++ b/app/shared/ai_models_errors.go
@@ -0,0 +1,64 @@
+package shared
+
+type ModelErrKind string
+
+const (
+ ErrOverloaded ModelErrKind = "ErrOverloaded"
+ ErrContextTooLong ModelErrKind = "ErrContextTooLong"
+ ErrRateLimited ModelErrKind = "ErrRateLimited"
+ ErrOther ModelErrKind = "ErrOther"
+)
+
+type ModelError struct {
+ Kind ModelErrKind
+ Retriable bool
+ RetryAfterSeconds int
+}
+
+// if fallback is defined, retry with main model, then remaining tries use error fallback
+type FallbackType string
+
+const (
+ FallbackTypeError FallbackType = "error"
+ FallbackTypeContext FallbackType = "context"
+)
+
+type FallbackResult struct {
+ ModelRoleConfig *ModelRoleConfig
+ HasErrorFallback bool
+ IsFallback bool
+ FallbackType FallbackType
+}
+
+const MAX_RETRIES_BEFORE_FALLBACK = 1
+
+func (m *ModelRoleConfig) GetFallbackForModelError(numTotalRetry int, modelErr *ModelError) FallbackResult {
+ if m == nil || modelErr == nil {
+ return FallbackResult{
+ ModelRoleConfig: m,
+ IsFallback: false,
+ }
+ }
+ if modelErr.Kind == ErrContextTooLong {
+ if m.LargeContextFallback != nil {
+ return FallbackResult{
+ ModelRoleConfig: m.LargeContextFallback,
+ FallbackType: FallbackTypeContext,
+ IsFallback: true,
+ }
+ }
+ } else if !modelErr.Retriable || numTotalRetry > MAX_RETRIES_BEFORE_FALLBACK {
+ if m.ErrorFallback != nil {
+ return FallbackResult{
+ ModelRoleConfig: m.ErrorFallback,
+ FallbackType: FallbackTypeError,
+ IsFallback: true,
+ }
+ }
+ }
+
+ return FallbackResult{
+ ModelRoleConfig: m,
+ IsFallback: false,
+ }
+}
diff --git a/app/shared/ai_models_large_context.go b/app/shared/ai_models_large_context.go
new file mode 100644
index 00000000..aac4bba7
--- /dev/null
+++ b/app/shared/ai_models_large_context.go
@@ -0,0 +1,95 @@
+package shared
+
+const maxFallbackDepth = 10 // max fallback depth for large context fallback - should never be reached in real scenarios, but protects against infinite loops in case of circular references etc.
+
+func (m ModelRoleConfig) GetFinalLargeContextFallback() ModelRoleConfig {
+ var currentConfig ModelRoleConfig = m
+ var n int = 0
+
+ for {
+ if currentConfig.LargeContextFallback == nil {
+ return currentConfig
+ } else {
+ currentConfig = *currentConfig.LargeContextFallback
+ }
+ n++
+ if n > maxFallbackDepth {
+ break
+ }
+ }
+
+ return currentConfig
+}
+
+func (m ModelRoleConfig) GetFinalLargeOutputFallback() ModelRoleConfig {
+ var currentConfig ModelRoleConfig = m
+ var n int = 0
+
+ if currentConfig.LargeOutputFallback == nil {
+ return currentConfig.GetFinalLargeContextFallback()
+ }
+
+ for {
+ if currentConfig.LargeOutputFallback == nil {
+ return currentConfig
+ } else {
+ currentConfig = *currentConfig.LargeOutputFallback
+ }
+ n++
+ if n > maxFallbackDepth {
+ break
+ }
+ }
+
+ return currentConfig
+}
+
+// note that if the token number exeeds all the fallback models, it will return the last fallback model
+
+func (m ModelRoleConfig) GetRoleForInputTokens(inputTokens int) ModelRoleConfig {
+ inputTokens = int(float64(inputTokens) * (1 + m.BaseModelConfig.TokenEstimatePaddingPct))
+ var currentConfig ModelRoleConfig = m
+ var n int = 0
+ for {
+ if currentConfig.BaseModelConfig.MaxTokens >= inputTokens {
+ return currentConfig
+ }
+
+ if currentConfig.LargeContextFallback == nil {
+ return currentConfig
+ } else {
+ currentConfig = *currentConfig.LargeContextFallback
+ }
+ n++
+ if n > maxFallbackDepth {
+ break
+ }
+ }
+ return currentConfig
+}
+
+func (m ModelRoleConfig) GetRoleForOutputTokens(outputTokens int) ModelRoleConfig {
+ outputTokens = int(float64(outputTokens) * (1 + m.BaseModelConfig.TokenEstimatePaddingPct))
+ var currentConfig ModelRoleConfig = m
+ var n int = 0
+ for {
+ if currentConfig.GetReservedOutputTokens() >= outputTokens {
+ return currentConfig
+ }
+
+ if currentConfig.LargeOutputFallback == nil {
+ if currentConfig.LargeContextFallback == nil {
+ return currentConfig
+ } else {
+ currentConfig = *currentConfig.LargeContextFallback
+ }
+ } else {
+ currentConfig = *currentConfig.LargeOutputFallback
+ }
+ n++
+ if n > maxFallbackDepth {
+ break
+ }
+ }
+ return currentConfig
+}