Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

add final provider-based fallback when there are credentials for multiple providers for a role

Dane Schneiderbccfd262e6c8
diff --git a/app/server/model/client.go b/app/server/model/client.go
index 6ada301f..6c492335 100644
--- a/app/server/model/client.go
+++ b/app/server/model/client.go
@@ -130,8 +130,8 @@ func CreateChatCompletionStream(
req.IncludeReasoning = true
}
- return withStreamingRetries(ctx, func(numTotalRetry int, modelErr *shared.ModelError) (*ExtendedChatCompletionStream, shared.FallbackResult, error) {
- fallbackRes := modelConfig.GetFallbackForModelError(numTotalRetry, modelErr)
+ return withStreamingRetries(ctx, func(numTotalRetry int, didProviderFallback bool, modelErr *shared.ModelError) (*ExtendedChatCompletionStream, shared.FallbackResult, error) {
+ fallbackRes := modelConfig.GetFallbackForModelError(numTotalRetry, didProviderFallback, modelErr, authVars)
resolvedModelConfig := fallbackRes.ModelRoleConfig
if resolvedModelConfig == nil {
diff --git a/app/server/model/client_stream.go b/app/server/model/client_stream.go
index 24a64d37..dc06df22 100644
--- a/app/server/model/client_stream.go
+++ b/app/server/model/client_stream.go
@@ -40,8 +40,8 @@ func CreateChatCompletionWithInternalStream(
// Force streaming mode since we're using the streaming API
req.Stream = true
- return withStreamingRetries(ctx, func(numTotalRetry int, modelErr *shared.ModelError) (resp *types.ModelResponse, fallbackRes shared.FallbackResult, err error) {
- fallbackRes = modelConfig.GetFallbackForModelError(numTotalRetry, modelErr)
+ return withStreamingRetries(ctx, func(numTotalRetry int, didProviderFallback bool, modelErr *shared.ModelError) (resp *types.ModelResponse, fallbackRes shared.FallbackResult, err error) {
+ fallbackRes = modelConfig.GetFallbackForModelError(numTotalRetry, didProviderFallback, modelErr, authVars)
resolvedModelConfig := fallbackRes.ModelRoleConfig
if resolvedModelConfig == nil {
@@ -208,7 +208,7 @@ func processChatCompletionStream(
func withStreamingRetries[T any](
ctx context.Context,
- operation func(numRetry int, modelErr *shared.ModelError) (resp *T, fallbackRes shared.FallbackResult, err error),
+ operation func(numRetry int, didProviderFallback bool, modelErr *shared.ModelError) (resp *T, fallbackRes shared.FallbackResult, err error),
onContextDone func(resp *T, err error),
) (*T, error) {
var resp *T
@@ -216,6 +216,7 @@ func withStreamingRetries[T any](
var numFallbackRetry int
var fallbackRes shared.FallbackResult
var modelErr *shared.ModelError
+ var didProviderFallback bool
for {
if ctx.Err() != nil {
@@ -238,7 +239,7 @@ func withStreamingRetries[T any](
log.Printf("withStreamingRetries - will run operation")
- resp, fallbackRes, err = operation(numTotalRetry, modelErr)
+ resp, fallbackRes, err = operation(numTotalRetry, didProviderFallback, modelErr)
if err == nil {
return resp, nil
}
@@ -251,6 +252,10 @@ func withStreamingRetries[T any](
maxRetries = MAX_ADDITIONAL_RETRIES_WITH_FALLBACK
}
+ if fallbackRes.FallbackType == shared.FallbackTypeProvider {
+ didProviderFallback = true
+ }
+
compareRetries := numTotalRetry
if isFallback {
compareRetries = numFallbackRetry
diff --git a/app/server/model/plan/tell_exec.go b/app/server/model/plan/tell_exec.go
index 606b241f..e78261be 100644
--- a/app/server/model/plan/tell_exec.go
+++ b/app/server/model/plan/tell_exec.go
@@ -466,12 +466,16 @@ func (state *activeTellStreamState) doTellRequest() {
modelConfig := state.modelConfig
active := state.activePlan
- fallbackRes := modelConfig.GetFallbackForModelError(state.numErrorRetry, state.modelErr)
+ fallbackRes := modelConfig.GetFallbackForModelError(state.numErrorRetry, state.didProviderFallback, state.modelErr, authVars)
modelConfig = fallbackRes.ModelRoleConfig
stop := []string{"<PlandexFinish/>"}
baseModelConfig := modelConfig.GetBaseModelConfig(state.authVars)
+ if fallbackRes.FallbackType == shared.FallbackTypeProvider {
+ state.didProviderFallback = true
+ }
+
// log.Println("Stop:", stop)
// spew.Dump(state.messages)
diff --git a/app/server/model/plan/tell_state.go b/app/server/model/plan/tell_state.go
index df7527cb..f56f8d0c 100644
--- a/app/server/model/plan/tell_state.go
+++ b/app/server/model/plan/tell_state.go
@@ -59,10 +59,11 @@ type activeTellStreamState struct {
manualStop []string
- numErrorRetry int
- numFallbackRetry int
- modelErr *shared.ModelError
- noCacheSupportErr bool
+ numErrorRetry int
+ numFallbackRetry int
+ modelErr *shared.ModelError
+ noCacheSupportErr bool
+ didProviderFallback bool
}
type chunkProcessor struct {
diff --git a/app/shared/ai_models_errors.go b/app/shared/ai_models_errors.go
index 2fef5860..0e039887 100644
--- a/app/shared/ai_models_errors.go
+++ b/app/shared/ai_models_errors.go
@@ -1,5 +1,11 @@
package shared
+import (
+ "log"
+
+ "github.com/jinzhu/copier"
+)
+
type ModelErrKind string
const (
@@ -20,8 +26,9 @@ type ModelError struct {
type FallbackType string
const (
- FallbackTypeError FallbackType = "error"
- FallbackTypeContext FallbackType = "context"
+ FallbackTypeError FallbackType = "error"
+ FallbackTypeContext FallbackType = "context"
+ FallbackTypeProvider FallbackType = "provider"
)
type FallbackResult struct {
@@ -33,7 +40,12 @@ type FallbackResult struct {
const MAX_RETRIES_BEFORE_FALLBACK = 1
-func (m *ModelRoleConfig) GetFallbackForModelError(numTotalRetry int, modelErr *ModelError) FallbackResult {
+func (m *ModelRoleConfig) GetFallbackForModelError(
+ numTotalRetry int,
+ didProviderFallback bool,
+ modelErr *ModelError,
+ authVars map[string]string,
+) FallbackResult {
if m == nil || modelErr == nil {
return FallbackResult{
ModelRoleConfig: m,
@@ -55,6 +67,15 @@ func (m *ModelRoleConfig) GetFallbackForModelError(numTotalRetry int, modelErr *
FallbackType: FallbackTypeError,
IsFallback: true,
}
+ } else if !didProviderFallback {
+ providerFallback := m.GetProviderFallback(authVars)
+ if providerFallback != nil {
+ return FallbackResult{
+ ModelRoleConfig: providerFallback,
+ FallbackType: FallbackTypeProvider,
+ IsFallback: true,
+ }
+ }
}
}
@@ -63,3 +84,41 @@ func (m *ModelRoleConfig) GetFallbackForModelError(numTotalRetry int, modelErr *
IsFallback: false,
}
}
+
+// we just try a single provider fallback if all defined fallbacks are exhausted
+// if we've got openrouter credentials in the stack, we always use OpenRouter as the fallback since it has its own routing/fallback routing to maximize resilience
+// otherwise we just use the second provider in the stack
+func (m ModelRoleConfig) GetProviderFallback(authVars map[string]string) *ModelRoleConfig {
+ providers := m.GetProvidersForAuthVars(authVars)
+
+ if len(providers) < 2 {
+ return nil
+ }
+
+ res := ModelRoleConfig{}
+ copier.Copy(&res, m)
+
+ var provider ModelProvider
+ for _, p := range providers {
+ if p.Provider == ModelProviderOpenRouter {
+ provider = p.Provider
+ break
+ }
+ }
+
+ if provider == "" {
+ provider = providers[1].Provider
+ }
+
+ availableModel := GetAvailableModel(provider, m.ModelId)
+
+ if availableModel == nil {
+ log.Printf("no available model found for provider %s and model id %s", provider, m.ModelId)
+ return nil
+ }
+
+ c := availableModel.BaseModelConfig
+ res.BaseModelConfig = &c
+
+ return &res
+}
diff --git a/app/shared/go.mod b/app/shared/go.mod
index 705b4747..c76577cb 100644
--- a/app/shared/go.mod
+++ b/app/shared/go.mod
@@ -4,6 +4,7 @@ go 1.23.3
require (
github.com/davecgh/go-spew v1.1.1
+ github.com/jinzhu/copier v0.4.0
github.com/pkoukk/tiktoken-go v0.1.7
)
diff --git a/app/shared/go.sum b/app/shared/go.sum
index a9f74421..a5df4991 100644
--- a/app/shared/go.sum
+++ b/app/shared/go.sum
@@ -4,6 +4,8 @@ github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZ
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
+github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=