Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
Update AI model configuration and streamline provider handling with improved token management
Dane Schneiderd062c0143fc6
diff --git a/app/server/model/client.go b/app/server/model/client.goindex cd5d2c32..f7458148 100644--- a/app/server/model/client.go+++ b/app/server/model/client.go@@ -10,7 +10,7 @@ import ("log""net/http""os"- "reflect"+ "plandex-server/types""regexp""strings""sync"@@ -72,23 +72,6 @@ func newClient(apiKey, endpoint, orgId string) ClientInfo {}}-type OpenAIPrediction struct {- Type string `json:"type"`- Content string `json:"content"`-}--type OpenRouterProviderConfig struct {- Order []string `json:"order"`- AllowFallbacks bool `json:"allow_fallbacks"`-}--type ExtendedChatCompletionRequest struct {- *openai.ChatCompletionRequest- Prediction *OpenAIPrediction `json:"prediction,omitempty"`- Provider *OpenRouterProviderConfig `json:"provider,omitempty"`- ReasoningEffort *shared.ReasoningEffort `json:"reasoning_effort,omitempty"`-}-// ExtendedChatCompletionStream can wrap either a native OpenAI stream or our custom implementationtype ExtendedChatCompletionStream struct {openaiStream *openai.ChatCompletionStream@@ -114,91 +97,69 @@ type ErrorAccumulator struct {// JSONUnmarshaler handles JSON unmarshaling for stream responsestype JSONUnmarshaler struct{}-func CreateChatCompletionStreamWithRetries[T openai.ChatCompletionRequest | ExtendedChatCompletionRequest](+func CreateChatCompletionStreamWithRetries(clients map[string]ClientInfo,modelConfig *shared.ModelRoleConfig,ctx context.Context,- req T,+ req types.ExtendedChatCompletionRequest,) (*ExtendedChatCompletionStream, error) {client, ok := clients[modelConfig.BaseModelConfig.ApiKeyEnvVar]if !ok {return nil, fmt.Errorf("client not found for api key env var: %s", modelConfig.BaseModelConfig.ApiKeyEnvVar)}- finalReq, err := getFinalReq(req, modelConfig)- if err != nil {- return nil, fmt.Errorf("error setting up request: %w", err)+ resolveReq(&req, modelConfig)++ // choose the fastest provider by latency/throughput on openrouter+ if modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenRouter {+ req.Model += ":nitro"}return withRetries(ctx, func() (*ExtendedChatCompletionStream, error) {- return createChatCompletionStreamExtended(client, modelConfig.BaseModelConfig.BaseUrl, ctx, *finalReq)+ return createChatCompletionStreamExtended(modelConfig, client, modelConfig.BaseModelConfig.BaseUrl, ctx, req)})}-func CreateChatCompletionWithRetries[T openai.ChatCompletionRequest | ExtendedChatCompletionRequest](+func CreateChatCompletionWithRetries(clients map[string]ClientInfo,modelConfig *shared.ModelRoleConfig,ctx context.Context,- req T,+ req types.ExtendedChatCompletionRequest,) (openai.ChatCompletionResponse, error) {client, ok := clients[modelConfig.BaseModelConfig.ApiKeyEnvVar]if !ok {return openai.ChatCompletionResponse{}, fmt.Errorf("client not found for api key env var: %s", modelConfig.BaseModelConfig.ApiKeyEnvVar)}- finalReq, err := getFinalReq(req, modelConfig)- if err != nil {- return openai.ChatCompletionResponse{}, fmt.Errorf("error setting up request: %w", err)+ resolveReq(&req, modelConfig)++ // choose the fastest provider by latency/throughput on openrouter+ if modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenRouter {+ req.Model += ":nitro"}return withRetries(ctx, func() (openai.ChatCompletionResponse, error) {- return createChatCompletionExtended(client, modelConfig.BaseModelConfig.BaseUrl, ctx, *finalReq)+ return createChatCompletionExtended(modelConfig, client, modelConfig.BaseModelConfig.BaseUrl, ctx, req)})}-func getFinalReq[T openai.ChatCompletionRequest | ExtendedChatCompletionRequest](req T, modelConfig *shared.ModelRoleConfig) (*ExtendedChatCompletionRequest, error) {- var baseReq *openai.ChatCompletionRequest- var finalReq *ExtendedChatCompletionRequest-- if normalReq, ok := any(req).(openai.ChatCompletionRequest); ok {- baseReq = &normalReq-- finalReq = &ExtendedChatCompletionRequest{- ChatCompletionRequest: baseReq,- }-- providerOrder := getOpenRouterProviderOrder(modelConfig)- if len(providerOrder) > 0 {- finalReq.Provider = &OpenRouterProviderConfig{- Order: providerOrder,- AllowFallbacks: modelConfig.BaseModelConfig.OpenRouterAllowFallbacks,- }- }-- if modelConfig.BaseModelConfig.OpenRouterSelfModerated {- finalReq.Model = finalReq.Model + ":beta"- }-- } else if extendedReq, ok := any(req).(ExtendedChatCompletionRequest); ok {- baseReq = extendedReq.ChatCompletionRequest- finalReq = &extendedReq- } else {- log.Println("Invalid request type")- log.Println("Request type:", reflect.TypeOf(req))- return nil, fmt.Errorf("invalid request type")- }-- resolveReq(baseReq, modelConfig)-- return finalReq, nil-}-func createChatCompletionExtended(+ modelConfig *shared.ModelRoleConfig,client ClientInfo,baseUrl string,ctx context.Context,- extendedReq ExtendedChatCompletionRequest,+ extendedReq types.ExtendedChatCompletionRequest,) (openai.ChatCompletionResponse, error) {+ if modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenAI {+ log.Println("Creating chat completion with direct OpenAI provider request")+ openaiReq := extendedReq.ToOpenAI()+ resp, err := client.Client.CreateChatCompletion(ctx, *openaiReq)+ if err != nil {+ return openai.ChatCompletionResponse{}, fmt.Errorf("error creating direct OpenAI chat completion: %w", err)+ }+ return resp, nil+ }+req, err := http.NewRequestWithContext(ctx, "POST", baseUrl+"/chat/completions", nil)if err != nil {return openai.ChatCompletionResponse{}, err@@ -244,11 +205,25 @@ func createChatCompletionExtended(}func createChatCompletionStreamExtended(+ modelConfig *shared.ModelRoleConfig,client ClientInfo,baseUrl string,ctx context.Context,- extendedReq ExtendedChatCompletionRequest,+ extendedReq types.ExtendedChatCompletionRequest,) (*ExtendedChatCompletionStream, error) {+ if modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenAI {+ openaiReq := extendedReq.ToOpenAI()+ log.Println("Creating chat completion stream with direct OpenAI provider request")+ stream, err := client.Client.CreateChatCompletionStream(ctx, *openaiReq)+ if err != nil {+ return nil, fmt.Errorf("error creating direct OpenAI chat completion stream: %w", err)+ }+ return &ExtendedChatCompletionStream{+ openaiStream: stream,+ ctx: ctx,+ }, nil+ }+// Marshal the request body to JSONjsonBody, err := json.MarshalIndent(extendedReq, "", " ")if err != nil {@@ -456,7 +431,7 @@ func parseRetryAfter(errorMessage string) *time.Duration {return nil}-func resolveReq(req *openai.ChatCompletionRequest, modelConfig *shared.ModelRoleConfig) {+func resolveReq(req *types.ExtendedChatCompletionRequest, modelConfig *shared.ModelRoleConfig) {// if system prompt is disabled, change the role of the system message to userif modelConfig.BaseModelConfig.SystemPromptDisabled {log.Println("System prompt disabled - changing role of system message to user")@@ -482,17 +457,17 @@ func resolveReq(req *openai.ChatCompletionRequest, modelConfig *shared.ModelRole// route directly to first-party providers on openrouter for the main models// seems to be much faster this way currently-func getOpenRouterProviderOrder(modelConfig *shared.ModelRoleConfig) []string {- var providerOrder []string- if modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenRouter {- if len(modelConfig.BaseModelConfig.PreferredOpenRouterProviders) > 0 {- for _, provider := range modelConfig.BaseModelConfig.PreferredOpenRouterProviders {- providerOrder = append(providerOrder, string(provider))- }- }- }- return providerOrder-}+// func getOpenRouterProviderOrder(modelConfig *shared.ModelRoleConfig) []string {+// var providerOrder []string+// if modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenRouter {+// if len(modelConfig.BaseModelConfig.PreferredOpenRouterProviders) > 0 {+// for _, provider := range modelConfig.BaseModelConfig.PreferredOpenRouterProviders {+// providerOrder = append(providerOrder, string(provider))+// }+// }+// }+// return providerOrder+// }func withRetries[T any](ctx context.Context,diff --git a/app/server/model/tokens.go b/app/server/model/tokens.gonew file mode 100644index 00000000..93535398--- /dev/null+++ b/app/server/model/tokens.go@@ -0,0 +1,50 @@+package model++import (+ "plandex-server/types"+ shared "plandex-shared"++ "github.com/sashabaranov/go-openai"+)++const (+ // Per OpenAI's documentation:+ // Every message follows this format: {"role": "role_name", "content": "content"}+ // which has a 4-token overhead per message+ TokensPerMessage = 4++ // System, user, or assistant - each role name costs 1 token+ TokensPerName = 1++ // Tokens per request+ TokensPerRequest = 3++ TokensPerExtendedPart = 6+)++func GetMessagesTokenEstimate(messages ...types.ExtendedChatMessage) int {+ tokens := 0++ for _, msg := range messages {+ tokens += TokensPerMessage // Base message overhead+ tokens += TokensPerName // Role name++ if len(msg.Content) > 0 {+ // For each extended part, we need to account for the JSON structure+ // Each part follows format: {"type": "type_value", "text": "content"}+ // or {"type": "type_value", "image_url": {"url": "url_value"}}+ for _, part := range msg.Content {+ if part.Type == openai.ChatMessagePartTypeText {+ tokens += TokensPerExtendedPart // Overhead for the part object structure+ tokens += shared.GetNumTokensEstimate(part.Text)+ }++ // images are handled separately++ }+ }++ }++ return tokens+}diff --git a/app/server/types/message.go b/app/server/types/message.gonew file mode 100644index 00000000..2510a067--- /dev/null+++ b/app/server/types/message.go@@ -0,0 +1,155 @@+package types++import (+ shared "plandex-shared"++ "github.com/sashabaranov/go-openai"+)++type CacheControlType string++const (+ CacheControlTypeEphemeral CacheControlType = "ephemeral"+)++type CacheControlSpec struct {+ Type CacheControlType `json:"type"`+}++type ExtendedChatMessagePart struct {+ Type openai.ChatMessagePartType `json:"type"`+ Text string `json:"text,omitempty"`+ ImageURL *openai.ChatMessageImageURL `json:"image_url,omitempty"`+ CacheControl *CacheControlSpec `json:"cache_control,omitempty"`+}++type ExtendedChatMessage struct {+ Role string `json:"role"`+ Content []ExtendedChatMessagePart `json:"content"`+}++func (msg *ExtendedChatMessage) ToOpenAI() *openai.ChatCompletionMessage {+ // If there's only one part and it's text, use simple Content field+ if len(msg.Content) == 1 && msg.Content[0].Type == "text" {+ return &openai.ChatCompletionMessage{+ Role: msg.Role,+ Content: msg.Content[0].Text,+ }+ }++ // Otherwise, use MultiContent for multiple parts or non-text content+ parts := make([]openai.ChatMessagePart, len(msg.Content))+ for i, part := range msg.Content {+ parts[i] = openai.ChatMessagePart{+ Type: part.Type,+ Text: part.Text,+ ImageURL: part.ImageURL,+ }+ }++ return &openai.ChatCompletionMessage{+ Role: msg.Role,+ MultiContent: parts,+ }+}++type OpenAIPrediction struct {+ Type string `json:"type"`+ Content string `json:"content"`+}++type OpenRouterProviderConfig struct {+ Order []string `json:"order"`+ AllowFallbacks bool `json:"allow_fallbacks"`+}++type ExtendedChatCompletionRequest struct {+ // copied from openai.ChatCompletionRequest+ Model string `json:"model"`+ Messages []ExtendedChatMessage `json:"messages"`+ // MaxTokens The maximum number of tokens that can be generated in the chat completion.+ // This value can be used to control costs for text generated via API.+ // This value is now deprecated in favor of max_completion_tokens, and is not compatible with o1 series models.+ // refs: https://platform.openai.com/docs/api-reference/chat/create#chat-create-max_tokens+ MaxTokens int `json:"max_tokens,omitempty"`+ // MaxCompletionTokens An upper bound for the number of tokens that can be generated for a completion,+ // including visible output tokens and reasoning tokens https://platform.openai.com/docs/guides/reasoning+ MaxCompletionTokens int `json:"max_completion_tokens,omitempty"`+ Temperature float32 `json:"temperature,omitempty"`+ TopP float32 `json:"top_p,omitempty"`+ N int `json:"n,omitempty"`+ Stream bool `json:"stream,omitempty"`+ Stop []string `json:"stop,omitempty"`+ PresencePenalty float32 `json:"presence_penalty,omitempty"`+ ResponseFormat *openai.ChatCompletionResponseFormat `json:"response_format,omitempty"`+ Seed *int `json:"seed,omitempty"`+ FrequencyPenalty float32 `json:"frequency_penalty,omitempty"`+ // LogitBias is must be a token id string (specified by their token ID in the tokenizer), not a word string.+ // incorrect: `"logit_bias":{"You": 6}`, correct: `"logit_bias":{"1639": 6}`+ // refs: https://platform.openai.com/docs/api-reference/chat/create#chat/create-logit_bias+ LogitBias map[string]int `json:"logit_bias,omitempty"`+ // LogProbs indicates whether to return log probabilities of the output tokens or not.+ // If true, returns the log probabilities of each output token returned in the content of message.+ // This option is currently not available on the gpt-4-vision-preview model.+ LogProbs bool `json:"logprobs,omitempty"`+ // TopLogProbs is an integer between 0 and 5 specifying the number of most likely tokens to return at each+ // token position, each with an associated log probability.+ // logprobs must be set to true if this parameter is used.+ TopLogProbs int `json:"top_logprobs,omitempty"`+ User string `json:"user,omitempty"`+ // Deprecated: use Tools instead.+ Functions []openai.FunctionDefinition `json:"functions,omitempty"`+ // Deprecated: use ToolChoice instead.+ FunctionCall any `json:"function_call,omitempty"`+ Tools []openai.Tool `json:"tools,omitempty"`+ // This can be either a string or an ToolChoice object.+ ToolChoice any `json:"tool_choice,omitempty"`+ // Options for streaming response. Only set this when you set stream: true.+ StreamOptions *openai.StreamOptions `json:"stream_options,omitempty"`+ // Disable the default behavior of parallel tool calls by setting it: false.+ ParallelToolCalls any `json:"parallel_tool_calls,omitempty"`+ // Store can be set to true to store the output of this completion request for use in distillations and evals.+ // https://platform.openai.com/docs/api-reference/chat/create#chat-create-store+ Store bool `json:"store,omitempty"`+ // Metadata to store with the completion.+ Metadata map[string]string `json:"metadata,omitempty"`++ Prediction *OpenAIPrediction `json:"prediction,omitempty"`+ Provider *OpenRouterProviderConfig `json:"provider,omitempty"`+ ReasoningEffort *shared.ReasoningEffort `json:"reasoning_effort,omitempty"`+}++func (req *ExtendedChatCompletionRequest) ToOpenAI() *openai.ChatCompletionRequest {+ openaiMessages := make([]openai.ChatCompletionMessage, len(req.Messages))+ for i, msg := range req.Messages {+ openaiMessages[i] = *msg.ToOpenAI()+ }++ return &openai.ChatCompletionRequest{+ Model: req.Model,+ Messages: openaiMessages,+ MaxTokens: req.MaxTokens,+ MaxCompletionTokens: req.MaxCompletionTokens,+ Temperature: req.Temperature,+ TopP: req.TopP,+ N: req.N,+ Stream: req.Stream,+ Stop: req.Stop,+ PresencePenalty: req.PresencePenalty,+ ResponseFormat: req.ResponseFormat,+ Seed: req.Seed,+ FrequencyPenalty: req.FrequencyPenalty,+ LogitBias: req.LogitBias,+ LogProbs: req.LogProbs,+ TopLogProbs: req.TopLogProbs,+ User: req.User,+ Functions: req.Functions,+ FunctionCall: req.FunctionCall,+ Tools: req.Tools,+ ToolChoice: req.ToolChoice,+ StreamOptions: req.StreamOptions,+ ParallelToolCalls: req.ParallelToolCalls,+ Store: req.Store,+ Metadata: req.Metadata,+ }+}diff --git a/app/shared/ai_models_available.go b/app/shared/ai_models_available.goindex 2da7d482..4267094f 100644--- a/app/shared/ai_models_available.go+++ b/app/shared/ai_models_available.go@@ -91,16 +91,17 @@ var AvailableModels = []*AvailableModel{DefaultMaxConvoTokens: 15000,DefaultReservedOutputTokens: 8192,BaseModelConfig: BaseModelConfig{- Provider: ModelProviderOpenRouter,- ModelName: "anthropic/claude-3.5-sonnet",- MaxTokens: 200000,- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],- ModelCompatibility: fullCompatibility,- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],- PreferredModelOutputFormat: ModelOutputFormatXml,- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyAnthropic],- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyAnthropic],- OpenRouterSelfModerated: true,+ Provider: ModelProviderOpenRouter,+ ModelName: "anthropic/claude-3.5-sonnet",+ MaxTokens: 200000,+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],+ PreferredModelOutputFormat: ModelOutputFormatXml,+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyAnthropic],+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyAnthropic],+ // OpenRouterNitro: true,+ // // OpenRouterSelfModerated: true,},},{@@ -108,15 +109,16 @@ var AvailableModels = []*AvailableModel{DefaultMaxConvoTokens: 15000,DefaultReservedOutputTokens: 8192,BaseModelConfig: BaseModelConfig{- Provider: ModelProviderOpenRouter,- ModelName: "anthropic/claude-3.5-haiku",- MaxTokens: 200000,- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],- ModelCompatibility: fullCompatibility,- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],- PreferredModelOutputFormat: ModelOutputFormatXml,- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyAnthropic],- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyAnthropic],+ Provider: ModelProviderOpenRouter,+ ModelName: "anthropic/claude-3.5-haiku",+ MaxTokens: 200000,+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],+ PreferredModelOutputFormat: ModelOutputFormatXml,+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyAnthropic],+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyAnthropic],+ // OpenRouterNitro: true,},},{@@ -124,47 +126,50 @@ var AvailableModels = []*AvailableModel{DefaultMaxConvoTokens: 100000,DefaultReservedOutputTokens: 8192,BaseModelConfig: BaseModelConfig{- Provider: ModelProviderOpenRouter,- ModelName: "google/gemini-pro-1.5",- MaxTokens: 2000000,- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],- ModelCompatibility: fullCompatibility,- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],- PreferredModelOutputFormat: ModelOutputFormatXml,- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyGoogle],- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyGoogle],+ Provider: ModelProviderOpenRouter,+ ModelName: "google/gemini-pro-1.5",+ MaxTokens: 2000000,+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],+ PreferredModelOutputFormat: ModelOutputFormatXml,+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyGoogle],+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyGoogle],+ // OpenRouterNitro: true,},},{- Description: "Google Gemini Flash 1.5 via OpenRouter",- DefaultMaxConvoTokens: 75000,+ Description: "Google Gemini Pro 2.0 Experimental via OpenRouter",+ DefaultMaxConvoTokens: 100000,DefaultReservedOutputTokens: 8192,BaseModelConfig: BaseModelConfig{- Provider: ModelProviderOpenRouter,- ModelName: "google/gemini-flash-1.5",- MaxTokens: 1000000,- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],- ModelCompatibility: fullCompatibility,- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],- PreferredModelOutputFormat: ModelOutputFormatXml,- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyGoogle],- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyGoogle],+ Provider: ModelProviderOpenRouter,+ ModelName: "google/gemini-2.0-pro-exp-02-05:free",+ MaxTokens: 2000000,+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],+ PreferredModelOutputFormat: ModelOutputFormatXml,+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyGoogle],+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyGoogle],+ // OpenRouterNitro: true,},},{- Description: "Google Gemini Flash 2.0 Experimental via OpenRouter",+ Description: "Google Gemini Flash 2.0 via OpenRouter",DefaultMaxConvoTokens: 75000,DefaultReservedOutputTokens: 8192,BaseModelConfig: BaseModelConfig{- Provider: ModelProviderOpenRouter,- ModelName: "google/gemini-2.0-flash-exp:free",- MaxTokens: 1000000,- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],- ModelCompatibility: fullCompatibility,- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],- PreferredModelOutputFormat: ModelOutputFormatXml,- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyGoogle],- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyGoogle],+ Provider: ModelProviderOpenRouter,+ ModelName: "google/gemini-2.0-flash-001",+ MaxTokens: 1000000,+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],+ PreferredModelOutputFormat: ModelOutputFormatXml,+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyGoogle],+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyGoogle],+ // OpenRouterNitro: true,},},@@ -180,10 +185,10 @@ var AvailableModels = []*AvailableModel{ModelCompatibility: ModelCompatibility{HasImageSupport: false,},- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],- PreferredModelOutputFormat: ModelOutputFormatXml,- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyDeepSeek],- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyDeepSeek],+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],+ PreferredModelOutputFormat: ModelOutputFormatXml,+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyDeepSeek],+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyDeepSeek],},},@@ -199,10 +204,10 @@ var AvailableModels = []*AvailableModel{ModelCompatibility: ModelCompatibility{HasImageSupport: false,},- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],- PreferredModelOutputFormat: ModelOutputFormatXml,- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyDeepSeek],- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyDeepSeek],+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],+ PreferredModelOutputFormat: ModelOutputFormatXml,+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyDeepSeek],+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyDeepSeek],},},@@ -218,10 +223,10 @@ var AvailableModels = []*AvailableModel{ModelCompatibility: ModelCompatibility{HasImageSupport: false,},- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],- PreferredModelOutputFormat: ModelOutputFormatXml,- PreferredOpenRouterProviders: []OpenRouterProvider{OpenRouterProviderDeepInfra},- OpenRouterAllowFallbacks: false,+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],+ PreferredModelOutputFormat: ModelOutputFormatXml,+ // PreferredOpenRouterProviders: []OpenRouterProvider{OpenRouterProviderDeepInfra},+ // OpenRouterAllowFallbacks: false,},},@@ -237,10 +242,10 @@ var AvailableModels = []*AvailableModel{ModelCompatibility: ModelCompatibility{HasImageSupport: false,},- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],- PreferredModelOutputFormat: ModelOutputFormatXml,- PreferredOpenRouterProviders: []OpenRouterProvider{OpenRouterProviderFireworks},- OpenRouterAllowFallbacks: false,+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],+ PreferredModelOutputFormat: ModelOutputFormatXml,+ // PreferredOpenRouterProviders: []OpenRouterProvider{OpenRouterProviderFireworks},+ // OpenRouterAllowFallbacks: false,},},@@ -249,15 +254,15 @@ var AvailableModels = []*AvailableModel{DefaultMaxConvoTokens: 10000,DefaultReservedOutputTokens: 8192,BaseModelConfig: BaseModelConfig{- Provider: ModelProviderOpenRouter,- ModelName: "qwen/qwen-2.5-coder-32b-instruct",- MaxTokens: 128000,- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],- ModelCompatibility: fullCompatibility,- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],- PreferredModelOutputFormat: ModelOutputFormatXml,- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyQwen],- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyQwen],+ Provider: ModelProviderOpenRouter,+ ModelName: "qwen/qwen-2.5-coder-32b-instruct",+ MaxTokens: 128000,+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],+ PreferredModelOutputFormat: ModelOutputFormatXml,+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyQwen],+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyQwen],},},@@ -267,16 +272,16 @@ var AvailableModels = []*AvailableModel{DefaultMaxConvoTokens: 10000,DefaultReservedOutputTokens: 16384,BaseModelConfig: BaseModelConfig{- Provider: ModelProviderOpenRouter,- ModelName: "openai/gpt-4o",- MaxTokens: 128000,- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],- ModelCompatibility: fullCompatibility,- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],- PreferredModelOutputFormat: ModelOutputFormatToolCallJson,- PredictedOutputEnabled: true,- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyOpenAI],- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyOpenAI],+ Provider: ModelProviderOpenRouter,+ ModelName: "openai/gpt-4o",+ MaxTokens: 128000,+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],+ PreferredModelOutputFormat: ModelOutputFormatToolCallJson,+ PredictedOutputEnabled: true,+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyOpenAI],+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyOpenAI],},},{@@ -284,16 +289,16 @@ var AvailableModels = []*AvailableModel{DefaultMaxConvoTokens: 10000,DefaultReservedOutputTokens: 16384,BaseModelConfig: BaseModelConfig{- Provider: ModelProviderOpenRouter,- ModelName: "openai/gpt-4o-mini",- MaxTokens: 128000,- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],- ModelCompatibility: fullCompatibility,- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],- PreferredModelOutputFormat: ModelOutputFormatToolCallJson,- PredictedOutputEnabled: true,- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyOpenAI],- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyOpenAI],+ Provider: ModelProviderOpenRouter,+ ModelName: "openai/gpt-4o-mini",+ MaxTokens: 128000,+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],+ PreferredModelOutputFormat: ModelOutputFormatToolCallJson,+ PredictedOutputEnabled: true,+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyOpenAI],+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyOpenAI],},},diff --git a/app/shared/ai_models_data_models.go b/app/shared/ai_models_data_models.goindex 052da47e..303a0393 100644--- a/app/shared/ai_models_data_models.go+++ b/app/shared/ai_models_data_models.go@@ -19,21 +19,21 @@ const ()type BaseModelConfig struct {- Provider ModelProvider `json:"provider"`- CustomProvider *string `json:"customProvider,omitempty"`- BaseUrl string `json:"baseUrl"`- ModelName string `json:"modelName"`- MaxTokens int `json:"maxTokens"`- ApiKeyEnvVar string `json:"apiKeyEnvVar"`- PreferredModelOutputFormat ModelOutputFormat `json:"preferredModelOutputFormat"`- PreferredOpenRouterProviders []OpenRouterProvider `json:"preferredOpenRouterProviders"`- OpenRouterAllowFallbacks bool `json:"openRouterAllowFallbacks"`- SystemPromptDisabled bool `json:"systemPromptDisabled"`- RoleParamsDisabled bool `json:"roleParamsDisabled"`- PredictedOutputEnabled bool `json:"predictedOutputEnabled"`- ReasoningEffortEnabled bool `json:"reasoningEffortEnabled"`- OpenRouterSelfModerated bool `json:"openRouterSelfModerated"`-+ Provider ModelProvider `json:"provider"`+ CustomProvider *string `json:"customProvider,omitempty"`+ BaseUrl string `json:"baseUrl"`+ ModelName string `json:"modelName"`+ MaxTokens int `json:"maxTokens"`+ ApiKeyEnvVar string `json:"apiKeyEnvVar"`+ PreferredModelOutputFormat ModelOutputFormat `json:"preferredModelOutputFormat"`+ // PreferredOpenRouterProviders []OpenRouterProvider `json:"preferredOpenRouterProviders"`+ // OpenRouterAllowFallbacks bool `json:"openRouterAllowFallbacks"`+ SystemPromptDisabled bool `json:"systemPromptDisabled"`+ RoleParamsDisabled bool `json:"roleParamsDisabled"`+ PredictedOutputEnabled bool `json:"predictedOutputEnabled"`+ ReasoningEffortEnabled bool `json:"reasoningEffortEnabled"`+ // OpenRouterSelfModerated bool `json:"openRouterSelfModerated"`+ // OpenRouterNitro bool `json:"openRouterNitro"`ModelCompatibility}diff --git a/app/shared/ai_models_openrouter.go b/app/shared/ai_models_openrouter.goindex 92d52c74..b7cb035f 100644--- a/app/shared/ai_models_openrouter.go+++ b/app/shared/ai_models_openrouter.go@@ -10,29 +10,29 @@ const (OpenRouterFamilyDeepSeek OpenRouterFamily = "deepseek")-type OpenRouterProvider string+// type OpenRouterProvider string-const (- OpenRouterProviderAnthropic OpenRouterProvider = "Anthropic"- OpenRouterProviderGoogle OpenRouterProvider = "Google Vertex"- OpenRouterProviderOpenAI OpenRouterProvider = "OpenAI"- OpenRouterProviderDeepSeek OpenRouterProvider = "DeepSeek"- OpenRouterProviderQwen OpenRouterProvider = "Hyperbolic"- OpenRouterProviderDeepInfra OpenRouterProvider = "DeepInfra"- OpenRouterProviderFireworks OpenRouterProvider = "Fireworks"-)+// const (+// OpenRouterProviderAnthropic OpenRouterProvider = "Anthropic"+// OpenRouterProviderGoogle OpenRouterProvider = "Google Vertex"+// OpenRouterProviderOpenAI OpenRouterProvider = "OpenAI"+// OpenRouterProviderDeepSeek OpenRouterProvider = "DeepSeek"+// OpenRouterProviderQwen OpenRouterProvider = "Hyperbolic"+// OpenRouterProviderDeepInfra OpenRouterProvider = "DeepInfra"+// OpenRouterProviderFireworks OpenRouterProvider = "Fireworks"+// )-var DefaultOpenRouterProvidersByFamily = map[OpenRouterFamily][]OpenRouterProvider{- OpenRouterFamilyAnthropic: {OpenRouterProviderAnthropic},- OpenRouterFamilyGoogle: {OpenRouterProviderGoogle},- OpenRouterFamilyOpenAI: {OpenRouterProviderOpenAI},- OpenRouterFamilyQwen: {OpenRouterProviderQwen},- OpenRouterFamilyDeepSeek: {OpenRouterProviderDeepSeek},-}+// var DefaultOpenRouterProvidersByFamily = map[OpenRouterFamily][]OpenRouterProvider{+// OpenRouterFamilyAnthropic: {OpenRouterProviderAnthropic},+// OpenRouterFamilyGoogle: {OpenRouterProviderGoogle},+// OpenRouterFamilyOpenAI: {OpenRouterProviderOpenAI},+// OpenRouterFamilyQwen: {OpenRouterProviderQwen},+// OpenRouterFamilyDeepSeek: {OpenRouterProviderDeepSeek},+// }-// open source models don't have fallbacks enabled for now because pricing and context limits aren't predictable across providers-var DefaultOpenRouterAllowFallbacksByFamily = map[OpenRouterFamily]bool{- OpenRouterFamilyAnthropic: false,- OpenRouterFamilyGoogle: false,- OpenRouterFamilyOpenAI: false,-}+// // open source models don't have fallbacks enabled for now because pricing and context limits aren't predictable across providers+// var DefaultOpenRouterAllowFallbacksByFamily = map[OpenRouterFamily]bool{+// OpenRouterFamilyAnthropic: true,+// OpenRouterFamilyGoogle: true,+// OpenRouterFamilyOpenAI: false,+// }diff --git a/app/shared/tokens.go b/app/shared/tokens.goindex dfa4b28d..e8fdf6fc 100644--- a/app/shared/tokens.go+++ b/app/shared/tokens.go@@ -4,7 +4,6 @@ import ("fmt""github.com/pkoukk/tiktoken-go"- "github.com/sashabaranov/go-openai")var tkm *tiktoken.Tiktoken@@ -20,28 +19,3 @@ func init() {func GetNumTokensEstimate(text string) int {return len(tkm.Encode(text, nil, nil))}--const (- // Per OpenAI's documentation:- // Every message follows this format: {"role": "role_name", "content": "content"}- // which has a 4-token overhead per message- TokensPerMessage = 4-- // System, user, or assistant - each role name costs 1 token- TokensPerName = 1-- // Tokens per request- TokensPerRequest = 3-)--func GetMessagesTokenEstimate(messages ...openai.ChatCompletionMessage) int {- tokens := 0-- for _, msg := range messages {- tokens += TokensPerMessage- tokens += TokensPerName- tokens += GetNumTokensEstimate(msg.Content)- }-- return tokens-}