Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
add final provider-based fallback when there are credentials for multiple providers for a role
Dane Schneiderbccfd262e6c8
diff --git a/app/server/model/client.go b/app/server/model/client.goindex 6ada301f..6c492335 100644--- a/app/server/model/client.go+++ b/app/server/model/client.go@@ -130,8 +130,8 @@ func CreateChatCompletionStream(req.IncludeReasoning = true}- return withStreamingRetries(ctx, func(numTotalRetry int, modelErr *shared.ModelError) (*ExtendedChatCompletionStream, shared.FallbackResult, error) {- fallbackRes := modelConfig.GetFallbackForModelError(numTotalRetry, modelErr)+ return withStreamingRetries(ctx, func(numTotalRetry int, didProviderFallback bool, modelErr *shared.ModelError) (*ExtendedChatCompletionStream, shared.FallbackResult, error) {+ fallbackRes := modelConfig.GetFallbackForModelError(numTotalRetry, didProviderFallback, modelErr, authVars)resolvedModelConfig := fallbackRes.ModelRoleConfigif resolvedModelConfig == nil {diff --git a/app/server/model/client_stream.go b/app/server/model/client_stream.goindex 24a64d37..dc06df22 100644--- a/app/server/model/client_stream.go+++ b/app/server/model/client_stream.go@@ -40,8 +40,8 @@ func CreateChatCompletionWithInternalStream(// Force streaming mode since we're using the streaming APIreq.Stream = true- return withStreamingRetries(ctx, func(numTotalRetry int, modelErr *shared.ModelError) (resp *types.ModelResponse, fallbackRes shared.FallbackResult, err error) {- fallbackRes = modelConfig.GetFallbackForModelError(numTotalRetry, modelErr)+ return withStreamingRetries(ctx, func(numTotalRetry int, didProviderFallback bool, modelErr *shared.ModelError) (resp *types.ModelResponse, fallbackRes shared.FallbackResult, err error) {+ fallbackRes = modelConfig.GetFallbackForModelError(numTotalRetry, didProviderFallback, modelErr, authVars)resolvedModelConfig := fallbackRes.ModelRoleConfigif resolvedModelConfig == nil {@@ -208,7 +208,7 @@ func processChatCompletionStream(func withStreamingRetries[T any](ctx context.Context,- operation func(numRetry int, modelErr *shared.ModelError) (resp *T, fallbackRes shared.FallbackResult, err error),+ operation func(numRetry int, didProviderFallback bool, modelErr *shared.ModelError) (resp *T, fallbackRes shared.FallbackResult, err error),onContextDone func(resp *T, err error),) (*T, error) {var resp *T@@ -216,6 +216,7 @@ func withStreamingRetries[T any](var numFallbackRetry intvar fallbackRes shared.FallbackResultvar modelErr *shared.ModelError+ var didProviderFallback boolfor {if ctx.Err() != nil {@@ -238,7 +239,7 @@ func withStreamingRetries[T any](log.Printf("withStreamingRetries - will run operation")- resp, fallbackRes, err = operation(numTotalRetry, modelErr)+ resp, fallbackRes, err = operation(numTotalRetry, didProviderFallback, modelErr)if err == nil {return resp, nil}@@ -251,6 +252,10 @@ func withStreamingRetries[T any](maxRetries = MAX_ADDITIONAL_RETRIES_WITH_FALLBACK}+ if fallbackRes.FallbackType == shared.FallbackTypeProvider {+ didProviderFallback = true+ }+compareRetries := numTotalRetryif isFallback {compareRetries = numFallbackRetrydiff --git a/app/server/model/plan/tell_exec.go b/app/server/model/plan/tell_exec.goindex 606b241f..e78261be 100644--- a/app/server/model/plan/tell_exec.go+++ b/app/server/model/plan/tell_exec.go@@ -466,12 +466,16 @@ func (state *activeTellStreamState) doTellRequest() {modelConfig := state.modelConfigactive := state.activePlan- fallbackRes := modelConfig.GetFallbackForModelError(state.numErrorRetry, state.modelErr)+ fallbackRes := modelConfig.GetFallbackForModelError(state.numErrorRetry, state.didProviderFallback, state.modelErr, authVars)modelConfig = fallbackRes.ModelRoleConfigstop := []string{"<PlandexFinish/>"}baseModelConfig := modelConfig.GetBaseModelConfig(state.authVars)+ if fallbackRes.FallbackType == shared.FallbackTypeProvider {+ state.didProviderFallback = true+ }+// log.Println("Stop:", stop)// spew.Dump(state.messages)diff --git a/app/server/model/plan/tell_state.go b/app/server/model/plan/tell_state.goindex df7527cb..f56f8d0c 100644--- a/app/server/model/plan/tell_state.go+++ b/app/server/model/plan/tell_state.go@@ -59,10 +59,11 @@ type activeTellStreamState struct {manualStop []string- numErrorRetry int- numFallbackRetry int- modelErr *shared.ModelError- noCacheSupportErr bool+ numErrorRetry int+ numFallbackRetry int+ modelErr *shared.ModelError+ noCacheSupportErr bool+ didProviderFallback bool}type chunkProcessor struct {diff --git a/app/shared/ai_models_errors.go b/app/shared/ai_models_errors.goindex 2fef5860..0e039887 100644--- a/app/shared/ai_models_errors.go+++ b/app/shared/ai_models_errors.go@@ -1,5 +1,11 @@package shared+import (+ "log"++ "github.com/jinzhu/copier"+)+type ModelErrKind stringconst (@@ -20,8 +26,9 @@ type ModelError struct {type FallbackType stringconst (- FallbackTypeError FallbackType = "error"- FallbackTypeContext FallbackType = "context"+ FallbackTypeError FallbackType = "error"+ FallbackTypeContext FallbackType = "context"+ FallbackTypeProvider FallbackType = "provider")type FallbackResult struct {@@ -33,7 +40,12 @@ type FallbackResult struct {const MAX_RETRIES_BEFORE_FALLBACK = 1-func (m *ModelRoleConfig) GetFallbackForModelError(numTotalRetry int, modelErr *ModelError) FallbackResult {+func (m *ModelRoleConfig) GetFallbackForModelError(+ numTotalRetry int,+ didProviderFallback bool,+ modelErr *ModelError,+ authVars map[string]string,+) FallbackResult {if m == nil || modelErr == nil {return FallbackResult{ModelRoleConfig: m,@@ -55,6 +67,15 @@ func (m *ModelRoleConfig) GetFallbackForModelError(numTotalRetry int, modelErr *FallbackType: FallbackTypeError,IsFallback: true,}+ } else if !didProviderFallback {+ providerFallback := m.GetProviderFallback(authVars)+ if providerFallback != nil {+ return FallbackResult{+ ModelRoleConfig: providerFallback,+ FallbackType: FallbackTypeProvider,+ IsFallback: true,+ }+ }}}@@ -63,3 +84,41 @@ func (m *ModelRoleConfig) GetFallbackForModelError(numTotalRetry int, modelErr *IsFallback: false,}}++// we just try a single provider fallback if all defined fallbacks are exhausted+// if we've got openrouter credentials in the stack, we always use OpenRouter as the fallback since it has its own routing/fallback routing to maximize resilience+// otherwise we just use the second provider in the stack+func (m ModelRoleConfig) GetProviderFallback(authVars map[string]string) *ModelRoleConfig {+ providers := m.GetProvidersForAuthVars(authVars)++ if len(providers) < 2 {+ return nil+ }++ res := ModelRoleConfig{}+ copier.Copy(&res, m)++ var provider ModelProvider+ for _, p := range providers {+ if p.Provider == ModelProviderOpenRouter {+ provider = p.Provider+ break+ }+ }++ if provider == "" {+ provider = providers[1].Provider+ }++ availableModel := GetAvailableModel(provider, m.ModelId)++ if availableModel == nil {+ log.Printf("no available model found for provider %s and model id %s", provider, m.ModelId)+ return nil+ }++ c := availableModel.BaseModelConfig+ res.BaseModelConfig = &c++ return &res+}diff --git a/app/shared/go.mod b/app/shared/go.modindex 705b4747..c76577cb 100644--- a/app/shared/go.mod+++ b/app/shared/go.mod@@ -4,6 +4,7 @@ go 1.23.3require (github.com/davecgh/go-spew v1.1.1+ github.com/jinzhu/copier v0.4.0github.com/pkoukk/tiktoken-go v0.1.7)diff --git a/app/shared/go.sum b/app/shared/go.sumindex a9f74421..a5df4991 100644--- a/app/shared/go.sum+++ b/app/shared/go.sum@@ -4,6 +4,8 @@ github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZgithub.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=+github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=+github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=