Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

Update AI model configuration and streamline provider handling with improved token management

Dane Schneiderd062c0143fc6
diff --git a/app/server/model/client.go b/app/server/model/client.go
index cd5d2c32..f7458148 100644
--- a/app/server/model/client.go
+++ b/app/server/model/client.go
@@ -10,7 +10,7 @@ import (
"log"
"net/http"
"os"
- "reflect"
+ "plandex-server/types"
"regexp"
"strings"
"sync"
@@ -72,23 +72,6 @@ func newClient(apiKey, endpoint, orgId string) ClientInfo {
}
}
-type OpenAIPrediction struct {
- Type string `json:"type"`
- Content string `json:"content"`
-}
-
-type OpenRouterProviderConfig struct {
- Order []string `json:"order"`
- AllowFallbacks bool `json:"allow_fallbacks"`
-}
-
-type ExtendedChatCompletionRequest struct {
- *openai.ChatCompletionRequest
- Prediction *OpenAIPrediction `json:"prediction,omitempty"`
- Provider *OpenRouterProviderConfig `json:"provider,omitempty"`
- ReasoningEffort *shared.ReasoningEffort `json:"reasoning_effort,omitempty"`
-}
-
// ExtendedChatCompletionStream can wrap either a native OpenAI stream or our custom implementation
type ExtendedChatCompletionStream struct {
openaiStream *openai.ChatCompletionStream
@@ -114,91 +97,69 @@ type ErrorAccumulator struct {
// JSONUnmarshaler handles JSON unmarshaling for stream responses
type JSONUnmarshaler struct{}
-func CreateChatCompletionStreamWithRetries[T openai.ChatCompletionRequest | ExtendedChatCompletionRequest](
+func CreateChatCompletionStreamWithRetries(
clients map[string]ClientInfo,
modelConfig *shared.ModelRoleConfig,
ctx context.Context,
- req T,
+ req types.ExtendedChatCompletionRequest,
) (*ExtendedChatCompletionStream, error) {
client, ok := clients[modelConfig.BaseModelConfig.ApiKeyEnvVar]
if !ok {
return nil, fmt.Errorf("client not found for api key env var: %s", modelConfig.BaseModelConfig.ApiKeyEnvVar)
}
- finalReq, err := getFinalReq(req, modelConfig)
- if err != nil {
- return nil, fmt.Errorf("error setting up request: %w", err)
+ resolveReq(&req, modelConfig)
+
+ // choose the fastest provider by latency/throughput on openrouter
+ if modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenRouter {
+ req.Model += ":nitro"
}
return withRetries(ctx, func() (*ExtendedChatCompletionStream, error) {
- return createChatCompletionStreamExtended(client, modelConfig.BaseModelConfig.BaseUrl, ctx, *finalReq)
+ return createChatCompletionStreamExtended(modelConfig, client, modelConfig.BaseModelConfig.BaseUrl, ctx, req)
})
}
-func CreateChatCompletionWithRetries[T openai.ChatCompletionRequest | ExtendedChatCompletionRequest](
+func CreateChatCompletionWithRetries(
clients map[string]ClientInfo,
modelConfig *shared.ModelRoleConfig,
ctx context.Context,
- req T,
+ req types.ExtendedChatCompletionRequest,
) (openai.ChatCompletionResponse, error) {
client, ok := clients[modelConfig.BaseModelConfig.ApiKeyEnvVar]
if !ok {
return openai.ChatCompletionResponse{}, fmt.Errorf("client not found for api key env var: %s", modelConfig.BaseModelConfig.ApiKeyEnvVar)
}
- finalReq, err := getFinalReq(req, modelConfig)
- if err != nil {
- return openai.ChatCompletionResponse{}, fmt.Errorf("error setting up request: %w", err)
+ resolveReq(&req, modelConfig)
+
+ // choose the fastest provider by latency/throughput on openrouter
+ if modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenRouter {
+ req.Model += ":nitro"
}
return withRetries(ctx, func() (openai.ChatCompletionResponse, error) {
- return createChatCompletionExtended(client, modelConfig.BaseModelConfig.BaseUrl, ctx, *finalReq)
+ return createChatCompletionExtended(modelConfig, client, modelConfig.BaseModelConfig.BaseUrl, ctx, req)
})
}
-func getFinalReq[T openai.ChatCompletionRequest | ExtendedChatCompletionRequest](req T, modelConfig *shared.ModelRoleConfig) (*ExtendedChatCompletionRequest, error) {
- var baseReq *openai.ChatCompletionRequest
- var finalReq *ExtendedChatCompletionRequest
-
- if normalReq, ok := any(req).(openai.ChatCompletionRequest); ok {
- baseReq = &normalReq
-
- finalReq = &ExtendedChatCompletionRequest{
- ChatCompletionRequest: baseReq,
- }
-
- providerOrder := getOpenRouterProviderOrder(modelConfig)
- if len(providerOrder) > 0 {
- finalReq.Provider = &OpenRouterProviderConfig{
- Order: providerOrder,
- AllowFallbacks: modelConfig.BaseModelConfig.OpenRouterAllowFallbacks,
- }
- }
-
- if modelConfig.BaseModelConfig.OpenRouterSelfModerated {
- finalReq.Model = finalReq.Model + ":beta"
- }
-
- } else if extendedReq, ok := any(req).(ExtendedChatCompletionRequest); ok {
- baseReq = extendedReq.ChatCompletionRequest
- finalReq = &extendedReq
- } else {
- log.Println("Invalid request type")
- log.Println("Request type:", reflect.TypeOf(req))
- return nil, fmt.Errorf("invalid request type")
- }
-
- resolveReq(baseReq, modelConfig)
-
- return finalReq, nil
-}
-
func createChatCompletionExtended(
+ modelConfig *shared.ModelRoleConfig,
client ClientInfo,
baseUrl string,
ctx context.Context,
- extendedReq ExtendedChatCompletionRequest,
+ extendedReq types.ExtendedChatCompletionRequest,
) (openai.ChatCompletionResponse, error) {
+ if modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenAI {
+ log.Println("Creating chat completion with direct OpenAI provider request")
+ openaiReq := extendedReq.ToOpenAI()
+ resp, err := client.Client.CreateChatCompletion(ctx, *openaiReq)
+ if err != nil {
+ return openai.ChatCompletionResponse{}, fmt.Errorf("error creating direct OpenAI chat completion: %w", err)
+ }
+ return resp, nil
+ }
+
req, err := http.NewRequestWithContext(ctx, "POST", baseUrl+"/chat/completions", nil)
if err != nil {
return openai.ChatCompletionResponse{}, err
@@ -244,11 +205,25 @@ func createChatCompletionExtended(
}
func createChatCompletionStreamExtended(
+ modelConfig *shared.ModelRoleConfig,
client ClientInfo,
baseUrl string,
ctx context.Context,
- extendedReq ExtendedChatCompletionRequest,
+ extendedReq types.ExtendedChatCompletionRequest,
) (*ExtendedChatCompletionStream, error) {
+ if modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenAI {
+ openaiReq := extendedReq.ToOpenAI()
+ log.Println("Creating chat completion stream with direct OpenAI provider request")
+ stream, err := client.Client.CreateChatCompletionStream(ctx, *openaiReq)
+ if err != nil {
+ return nil, fmt.Errorf("error creating direct OpenAI chat completion stream: %w", err)
+ }
+ return &ExtendedChatCompletionStream{
+ openaiStream: stream,
+ ctx: ctx,
+ }, nil
+ }
+
// Marshal the request body to JSON
jsonBody, err := json.MarshalIndent(extendedReq, "", " ")
if err != nil {
@@ -456,7 +431,7 @@ func parseRetryAfter(errorMessage string) *time.Duration {
return nil
}
-func resolveReq(req *openai.ChatCompletionRequest, modelConfig *shared.ModelRoleConfig) {
+func resolveReq(req *types.ExtendedChatCompletionRequest, modelConfig *shared.ModelRoleConfig) {
// if system prompt is disabled, change the role of the system message to user
if modelConfig.BaseModelConfig.SystemPromptDisabled {
log.Println("System prompt disabled - changing role of system message to user")
@@ -482,17 +457,17 @@ func resolveReq(req *openai.ChatCompletionRequest, modelConfig *shared.ModelRole
// route directly to first-party providers on openrouter for the main models
// seems to be much faster this way currently
-func getOpenRouterProviderOrder(modelConfig *shared.ModelRoleConfig) []string {
- var providerOrder []string
- if modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenRouter {
- if len(modelConfig.BaseModelConfig.PreferredOpenRouterProviders) > 0 {
- for _, provider := range modelConfig.BaseModelConfig.PreferredOpenRouterProviders {
- providerOrder = append(providerOrder, string(provider))
- }
- }
- }
- return providerOrder
-}
+// func getOpenRouterProviderOrder(modelConfig *shared.ModelRoleConfig) []string {
+// var providerOrder []string
+// if modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenRouter {
+// if len(modelConfig.BaseModelConfig.PreferredOpenRouterProviders) > 0 {
+// for _, provider := range modelConfig.BaseModelConfig.PreferredOpenRouterProviders {
+// providerOrder = append(providerOrder, string(provider))
+// }
+// }
+// }
+// return providerOrder
+// }
func withRetries[T any](
ctx context.Context,
diff --git a/app/server/model/tokens.go b/app/server/model/tokens.go
new file mode 100644
index 00000000..93535398
--- /dev/null
+++ b/app/server/model/tokens.go
@@ -0,0 +1,50 @@
+package model
+
+import (
+ "plandex-server/types"
+ shared "plandex-shared"
+
+ "github.com/sashabaranov/go-openai"
+)
+
+const (
+ // Per OpenAI's documentation:
+ // Every message follows this format: {"role": "role_name", "content": "content"}
+ // which has a 4-token overhead per message
+ TokensPerMessage = 4
+
+ // System, user, or assistant - each role name costs 1 token
+ TokensPerName = 1
+
+ // Tokens per request
+ TokensPerRequest = 3
+
+ TokensPerExtendedPart = 6
+)
+
+func GetMessagesTokenEstimate(messages ...types.ExtendedChatMessage) int {
+ tokens := 0
+
+ for _, msg := range messages {
+ tokens += TokensPerMessage // Base message overhead
+ tokens += TokensPerName // Role name
+
+ if len(msg.Content) > 0 {
+ // For each extended part, we need to account for the JSON structure
+ // Each part follows format: {"type": "type_value", "text": "content"}
+ // or {"type": "type_value", "image_url": {"url": "url_value"}}
+ for _, part := range msg.Content {
+ if part.Type == openai.ChatMessagePartTypeText {
+ tokens += TokensPerExtendedPart // Overhead for the part object structure
+ tokens += shared.GetNumTokensEstimate(part.Text)
+ }
+
+ // images are handled separately
+
+ }
+ }
+
+ }
+
+ return tokens
+}
diff --git a/app/server/types/message.go b/app/server/types/message.go
new file mode 100644
index 00000000..2510a067
--- /dev/null
+++ b/app/server/types/message.go
@@ -0,0 +1,155 @@
+package types
+
+import (
+ shared "plandex-shared"
+
+ "github.com/sashabaranov/go-openai"
+)
+
+type CacheControlType string
+
+const (
+ CacheControlTypeEphemeral CacheControlType = "ephemeral"
+)
+
+type CacheControlSpec struct {
+ Type CacheControlType `json:"type"`
+}
+
+type ExtendedChatMessagePart struct {
+ Type openai.ChatMessagePartType `json:"type"`
+ Text string `json:"text,omitempty"`
+ ImageURL *openai.ChatMessageImageURL `json:"image_url,omitempty"`
+ CacheControl *CacheControlSpec `json:"cache_control,omitempty"`
+}
+
+type ExtendedChatMessage struct {
+ Role string `json:"role"`
+ Content []ExtendedChatMessagePart `json:"content"`
+}
+
+func (msg *ExtendedChatMessage) ToOpenAI() *openai.ChatCompletionMessage {
+ // If there's only one part and it's text, use simple Content field
+ if len(msg.Content) == 1 && msg.Content[0].Type == "text" {
+ return &openai.ChatCompletionMessage{
+ Role: msg.Role,
+ Content: msg.Content[0].Text,
+ }
+ }
+
+ // Otherwise, use MultiContent for multiple parts or non-text content
+ parts := make([]openai.ChatMessagePart, len(msg.Content))
+ for i, part := range msg.Content {
+ parts[i] = openai.ChatMessagePart{
+ Type: part.Type,
+ Text: part.Text,
+ ImageURL: part.ImageURL,
+ }
+ }
+
+ return &openai.ChatCompletionMessage{
+ Role: msg.Role,
+ MultiContent: parts,
+ }
+}
+
+type OpenAIPrediction struct {
+ Type string `json:"type"`
+ Content string `json:"content"`
+}
+
+type OpenRouterProviderConfig struct {
+ Order []string `json:"order"`
+ AllowFallbacks bool `json:"allow_fallbacks"`
+}
+
+type ExtendedChatCompletionRequest struct {
+ // copied from openai.ChatCompletionRequest
+ Model string `json:"model"`
+ Messages []ExtendedChatMessage `json:"messages"`
+ // MaxTokens The maximum number of tokens that can be generated in the chat completion.
+ // This value can be used to control costs for text generated via API.
+ // This value is now deprecated in favor of max_completion_tokens, and is not compatible with o1 series models.
+ // refs: https://platform.openai.com/docs/api-reference/chat/create#chat-create-max_tokens
+ MaxTokens int `json:"max_tokens,omitempty"`
+ // MaxCompletionTokens An upper bound for the number of tokens that can be generated for a completion,
+ // including visible output tokens and reasoning tokens https://platform.openai.com/docs/guides/reasoning
+ MaxCompletionTokens int `json:"max_completion_tokens,omitempty"`
+ Temperature float32 `json:"temperature,omitempty"`
+ TopP float32 `json:"top_p,omitempty"`
+ N int `json:"n,omitempty"`
+ Stream bool `json:"stream,omitempty"`
+ Stop []string `json:"stop,omitempty"`
+ PresencePenalty float32 `json:"presence_penalty,omitempty"`
+ ResponseFormat *openai.ChatCompletionResponseFormat `json:"response_format,omitempty"`
+ Seed *int `json:"seed,omitempty"`
+ FrequencyPenalty float32 `json:"frequency_penalty,omitempty"`
+ // LogitBias is must be a token id string (specified by their token ID in the tokenizer), not a word string.
+ // incorrect: `"logit_bias":{"You": 6}`, correct: `"logit_bias":{"1639": 6}`
+ // refs: https://platform.openai.com/docs/api-reference/chat/create#chat/create-logit_bias
+ LogitBias map[string]int `json:"logit_bias,omitempty"`
+ // LogProbs indicates whether to return log probabilities of the output tokens or not.
+ // If true, returns the log probabilities of each output token returned in the content of message.
+ // This option is currently not available on the gpt-4-vision-preview model.
+ LogProbs bool `json:"logprobs,omitempty"`
+ // TopLogProbs is an integer between 0 and 5 specifying the number of most likely tokens to return at each
+ // token position, each with an associated log probability.
+ // logprobs must be set to true if this parameter is used.
+ TopLogProbs int `json:"top_logprobs,omitempty"`
+ User string `json:"user,omitempty"`
+ // Deprecated: use Tools instead.
+ Functions []openai.FunctionDefinition `json:"functions,omitempty"`
+ // Deprecated: use ToolChoice instead.
+ FunctionCall any `json:"function_call,omitempty"`
+ Tools []openai.Tool `json:"tools,omitempty"`
+ // This can be either a string or an ToolChoice object.
+ ToolChoice any `json:"tool_choice,omitempty"`
+ // Options for streaming response. Only set this when you set stream: true.
+ StreamOptions *openai.StreamOptions `json:"stream_options,omitempty"`
+ // Disable the default behavior of parallel tool calls by setting it: false.
+ ParallelToolCalls any `json:"parallel_tool_calls,omitempty"`
+ // Store can be set to true to store the output of this completion request for use in distillations and evals.
+ // https://platform.openai.com/docs/api-reference/chat/create#chat-create-store
+ Store bool `json:"store,omitempty"`
+ // Metadata to store with the completion.
+ Metadata map[string]string `json:"metadata,omitempty"`
+
+ Prediction *OpenAIPrediction `json:"prediction,omitempty"`
+ Provider *OpenRouterProviderConfig `json:"provider,omitempty"`
+ ReasoningEffort *shared.ReasoningEffort `json:"reasoning_effort,omitempty"`
+}
+
+func (req *ExtendedChatCompletionRequest) ToOpenAI() *openai.ChatCompletionRequest {
+ openaiMessages := make([]openai.ChatCompletionMessage, len(req.Messages))
+ for i, msg := range req.Messages {
+ openaiMessages[i] = *msg.ToOpenAI()
+ }
+
+ return &openai.ChatCompletionRequest{
+ Model: req.Model,
+ Messages: openaiMessages,
+ MaxTokens: req.MaxTokens,
+ MaxCompletionTokens: req.MaxCompletionTokens,
+ Temperature: req.Temperature,
+ TopP: req.TopP,
+ N: req.N,
+ Stream: req.Stream,
+ Stop: req.Stop,
+ PresencePenalty: req.PresencePenalty,
+ ResponseFormat: req.ResponseFormat,
+ Seed: req.Seed,
+ FrequencyPenalty: req.FrequencyPenalty,
+ LogitBias: req.LogitBias,
+ LogProbs: req.LogProbs,
+ TopLogProbs: req.TopLogProbs,
+ User: req.User,
+ Functions: req.Functions,
+ FunctionCall: req.FunctionCall,
+ Tools: req.Tools,
+ ToolChoice: req.ToolChoice,
+ StreamOptions: req.StreamOptions,
+ ParallelToolCalls: req.ParallelToolCalls,
+ Store: req.Store,
+ Metadata: req.Metadata,
+ }
+}
diff --git a/app/shared/ai_models_available.go b/app/shared/ai_models_available.go
index 2da7d482..4267094f 100644
--- a/app/shared/ai_models_available.go
+++ b/app/shared/ai_models_available.go
@@ -91,16 +91,17 @@ var AvailableModels = []*AvailableModel{
DefaultMaxConvoTokens: 15000,
DefaultReservedOutputTokens: 8192,
BaseModelConfig: BaseModelConfig{
- Provider: ModelProviderOpenRouter,
- ModelName: "anthropic/claude-3.5-sonnet",
- MaxTokens: 200000,
- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
- ModelCompatibility: fullCompatibility,
- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
- PreferredModelOutputFormat: ModelOutputFormatXml,
- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyAnthropic],
- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyAnthropic],
- OpenRouterSelfModerated: true,
+ Provider: ModelProviderOpenRouter,
+ ModelName: "anthropic/claude-3.5-sonnet",
+ MaxTokens: 200000,
+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
+ ModelCompatibility: fullCompatibility,
+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
+ PreferredModelOutputFormat: ModelOutputFormatXml,
+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyAnthropic],
+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyAnthropic],
+ // OpenRouterNitro: true,
+ // // OpenRouterSelfModerated: true,
},
},
{
@@ -108,15 +109,16 @@ var AvailableModels = []*AvailableModel{
DefaultMaxConvoTokens: 15000,
DefaultReservedOutputTokens: 8192,
BaseModelConfig: BaseModelConfig{
- Provider: ModelProviderOpenRouter,
- ModelName: "anthropic/claude-3.5-haiku",
- MaxTokens: 200000,
- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
- ModelCompatibility: fullCompatibility,
- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
- PreferredModelOutputFormat: ModelOutputFormatXml,
- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyAnthropic],
- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyAnthropic],
+ Provider: ModelProviderOpenRouter,
+ ModelName: "anthropic/claude-3.5-haiku",
+ MaxTokens: 200000,
+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
+ ModelCompatibility: fullCompatibility,
+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
+ PreferredModelOutputFormat: ModelOutputFormatXml,
+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyAnthropic],
+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyAnthropic],
+ // OpenRouterNitro: true,
},
},
{
@@ -124,47 +126,50 @@ var AvailableModels = []*AvailableModel{
DefaultMaxConvoTokens: 100000,
DefaultReservedOutputTokens: 8192,
BaseModelConfig: BaseModelConfig{
- Provider: ModelProviderOpenRouter,
- ModelName: "google/gemini-pro-1.5",
- MaxTokens: 2000000,
- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
- ModelCompatibility: fullCompatibility,
- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
- PreferredModelOutputFormat: ModelOutputFormatXml,
- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyGoogle],
- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyGoogle],
+ Provider: ModelProviderOpenRouter,
+ ModelName: "google/gemini-pro-1.5",
+ MaxTokens: 2000000,
+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
+ ModelCompatibility: fullCompatibility,
+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
+ PreferredModelOutputFormat: ModelOutputFormatXml,
+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyGoogle],
+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyGoogle],
+ // OpenRouterNitro: true,
},
},
{
- Description: "Google Gemini Flash 1.5 via OpenRouter",
- DefaultMaxConvoTokens: 75000,
+ Description: "Google Gemini Pro 2.0 Experimental via OpenRouter",
+ DefaultMaxConvoTokens: 100000,
DefaultReservedOutputTokens: 8192,
BaseModelConfig: BaseModelConfig{
- Provider: ModelProviderOpenRouter,
- ModelName: "google/gemini-flash-1.5",
- MaxTokens: 1000000,
- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
- ModelCompatibility: fullCompatibility,
- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
- PreferredModelOutputFormat: ModelOutputFormatXml,
- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyGoogle],
- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyGoogle],
+ Provider: ModelProviderOpenRouter,
+ ModelName: "google/gemini-2.0-pro-exp-02-05:free",
+ MaxTokens: 2000000,
+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
+ ModelCompatibility: fullCompatibility,
+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
+ PreferredModelOutputFormat: ModelOutputFormatXml,
+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyGoogle],
+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyGoogle],
+ // OpenRouterNitro: true,
},
},
{
- Description: "Google Gemini Flash 2.0 Experimental via OpenRouter",
+ Description: "Google Gemini Flash 2.0 via OpenRouter",
DefaultMaxConvoTokens: 75000,
DefaultReservedOutputTokens: 8192,
BaseModelConfig: BaseModelConfig{
- Provider: ModelProviderOpenRouter,
- ModelName: "google/gemini-2.0-flash-exp:free",
- MaxTokens: 1000000,
- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
- ModelCompatibility: fullCompatibility,
- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
- PreferredModelOutputFormat: ModelOutputFormatXml,
- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyGoogle],
- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyGoogle],
+ Provider: ModelProviderOpenRouter,
+ ModelName: "google/gemini-2.0-flash-001",
+ MaxTokens: 1000000,
+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
+ ModelCompatibility: fullCompatibility,
+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
+ PreferredModelOutputFormat: ModelOutputFormatXml,
+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyGoogle],
+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyGoogle],
+ // OpenRouterNitro: true,
},
},
@@ -180,10 +185,10 @@ var AvailableModels = []*AvailableModel{
ModelCompatibility: ModelCompatibility{
HasImageSupport: false,
},
- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
- PreferredModelOutputFormat: ModelOutputFormatXml,
- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyDeepSeek],
- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyDeepSeek],
+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
+ PreferredModelOutputFormat: ModelOutputFormatXml,
+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyDeepSeek],
+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyDeepSeek],
},
},
@@ -199,10 +204,10 @@ var AvailableModels = []*AvailableModel{
ModelCompatibility: ModelCompatibility{
HasImageSupport: false,
},
- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
- PreferredModelOutputFormat: ModelOutputFormatXml,
- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyDeepSeek],
- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyDeepSeek],
+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
+ PreferredModelOutputFormat: ModelOutputFormatXml,
+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyDeepSeek],
+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyDeepSeek],
},
},
@@ -218,10 +223,10 @@ var AvailableModels = []*AvailableModel{
ModelCompatibility: ModelCompatibility{
HasImageSupport: false,
},
- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
- PreferredModelOutputFormat: ModelOutputFormatXml,
- PreferredOpenRouterProviders: []OpenRouterProvider{OpenRouterProviderDeepInfra},
- OpenRouterAllowFallbacks: false,
+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
+ PreferredModelOutputFormat: ModelOutputFormatXml,
+ // PreferredOpenRouterProviders: []OpenRouterProvider{OpenRouterProviderDeepInfra},
+ // OpenRouterAllowFallbacks: false,
},
},
@@ -237,10 +242,10 @@ var AvailableModels = []*AvailableModel{
ModelCompatibility: ModelCompatibility{
HasImageSupport: false,
},
- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
- PreferredModelOutputFormat: ModelOutputFormatXml,
- PreferredOpenRouterProviders: []OpenRouterProvider{OpenRouterProviderFireworks},
- OpenRouterAllowFallbacks: false,
+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
+ PreferredModelOutputFormat: ModelOutputFormatXml,
+ // PreferredOpenRouterProviders: []OpenRouterProvider{OpenRouterProviderFireworks},
+ // OpenRouterAllowFallbacks: false,
},
},
@@ -249,15 +254,15 @@ var AvailableModels = []*AvailableModel{
DefaultMaxConvoTokens: 10000,
DefaultReservedOutputTokens: 8192,
BaseModelConfig: BaseModelConfig{
- Provider: ModelProviderOpenRouter,
- ModelName: "qwen/qwen-2.5-coder-32b-instruct",
- MaxTokens: 128000,
- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
- ModelCompatibility: fullCompatibility,
- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
- PreferredModelOutputFormat: ModelOutputFormatXml,
- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyQwen],
- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyQwen],
+ Provider: ModelProviderOpenRouter,
+ ModelName: "qwen/qwen-2.5-coder-32b-instruct",
+ MaxTokens: 128000,
+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
+ ModelCompatibility: fullCompatibility,
+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
+ PreferredModelOutputFormat: ModelOutputFormatXml,
+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyQwen],
+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyQwen],
},
},
@@ -267,16 +272,16 @@ var AvailableModels = []*AvailableModel{
DefaultMaxConvoTokens: 10000,
DefaultReservedOutputTokens: 16384,
BaseModelConfig: BaseModelConfig{
- Provider: ModelProviderOpenRouter,
- ModelName: "openai/gpt-4o",
- MaxTokens: 128000,
- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
- ModelCompatibility: fullCompatibility,
- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
- PreferredModelOutputFormat: ModelOutputFormatToolCallJson,
- PredictedOutputEnabled: true,
- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyOpenAI],
- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyOpenAI],
+ Provider: ModelProviderOpenRouter,
+ ModelName: "openai/gpt-4o",
+ MaxTokens: 128000,
+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
+ ModelCompatibility: fullCompatibility,
+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
+ PreferredModelOutputFormat: ModelOutputFormatToolCallJson,
+ PredictedOutputEnabled: true,
+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyOpenAI],
+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyOpenAI],
},
},
{
@@ -284,16 +289,16 @@ var AvailableModels = []*AvailableModel{
DefaultMaxConvoTokens: 10000,
DefaultReservedOutputTokens: 16384,
BaseModelConfig: BaseModelConfig{
- Provider: ModelProviderOpenRouter,
- ModelName: "openai/gpt-4o-mini",
- MaxTokens: 128000,
- ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
- ModelCompatibility: fullCompatibility,
- BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
- PreferredModelOutputFormat: ModelOutputFormatToolCallJson,
- PredictedOutputEnabled: true,
- PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyOpenAI],
- OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyOpenAI],
+ Provider: ModelProviderOpenRouter,
+ ModelName: "openai/gpt-4o-mini",
+ MaxTokens: 128000,
+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderOpenRouter],
+ ModelCompatibility: fullCompatibility,
+ BaseUrl: BaseUrlByProvider[ModelProviderOpenRouter],
+ PreferredModelOutputFormat: ModelOutputFormatToolCallJson,
+ PredictedOutputEnabled: true,
+ // PreferredOpenRouterProviders: DefaultOpenRouterProvidersByFamily[OpenRouterFamilyOpenAI],
+ // OpenRouterAllowFallbacks: DefaultOpenRouterAllowFallbacksByFamily[OpenRouterFamilyOpenAI],
},
},
diff --git a/app/shared/ai_models_data_models.go b/app/shared/ai_models_data_models.go
index 052da47e..303a0393 100644
--- a/app/shared/ai_models_data_models.go
+++ b/app/shared/ai_models_data_models.go
@@ -19,21 +19,21 @@ const (
)
type BaseModelConfig struct {
- Provider ModelProvider `json:"provider"`
- CustomProvider *string `json:"customProvider,omitempty"`
- BaseUrl string `json:"baseUrl"`
- ModelName string `json:"modelName"`
- MaxTokens int `json:"maxTokens"`
- ApiKeyEnvVar string `json:"apiKeyEnvVar"`
- PreferredModelOutputFormat ModelOutputFormat `json:"preferredModelOutputFormat"`
- PreferredOpenRouterProviders []OpenRouterProvider `json:"preferredOpenRouterProviders"`
- OpenRouterAllowFallbacks bool `json:"openRouterAllowFallbacks"`
- SystemPromptDisabled bool `json:"systemPromptDisabled"`
- RoleParamsDisabled bool `json:"roleParamsDisabled"`
- PredictedOutputEnabled bool `json:"predictedOutputEnabled"`
- ReasoningEffortEnabled bool `json:"reasoningEffortEnabled"`
- OpenRouterSelfModerated bool `json:"openRouterSelfModerated"`
-
+ Provider ModelProvider `json:"provider"`
+ CustomProvider *string `json:"customProvider,omitempty"`
+ BaseUrl string `json:"baseUrl"`
+ ModelName string `json:"modelName"`
+ MaxTokens int `json:"maxTokens"`
+ ApiKeyEnvVar string `json:"apiKeyEnvVar"`
+ PreferredModelOutputFormat ModelOutputFormat `json:"preferredModelOutputFormat"`
+ // PreferredOpenRouterProviders []OpenRouterProvider `json:"preferredOpenRouterProviders"`
+ // OpenRouterAllowFallbacks bool `json:"openRouterAllowFallbacks"`
+ SystemPromptDisabled bool `json:"systemPromptDisabled"`
+ RoleParamsDisabled bool `json:"roleParamsDisabled"`
+ PredictedOutputEnabled bool `json:"predictedOutputEnabled"`
+ ReasoningEffortEnabled bool `json:"reasoningEffortEnabled"`
+ // OpenRouterSelfModerated bool `json:"openRouterSelfModerated"`
+ // OpenRouterNitro bool `json:"openRouterNitro"`
ModelCompatibility
}
diff --git a/app/shared/ai_models_openrouter.go b/app/shared/ai_models_openrouter.go
index 92d52c74..b7cb035f 100644
--- a/app/shared/ai_models_openrouter.go
+++ b/app/shared/ai_models_openrouter.go
@@ -10,29 +10,29 @@ const (
OpenRouterFamilyDeepSeek OpenRouterFamily = "deepseek"
)
-type OpenRouterProvider string
+// type OpenRouterProvider string
-const (
- OpenRouterProviderAnthropic OpenRouterProvider = "Anthropic"
- OpenRouterProviderGoogle OpenRouterProvider = "Google Vertex"
- OpenRouterProviderOpenAI OpenRouterProvider = "OpenAI"
- OpenRouterProviderDeepSeek OpenRouterProvider = "DeepSeek"
- OpenRouterProviderQwen OpenRouterProvider = "Hyperbolic"
- OpenRouterProviderDeepInfra OpenRouterProvider = "DeepInfra"
- OpenRouterProviderFireworks OpenRouterProvider = "Fireworks"
-)
+// const (
+// OpenRouterProviderAnthropic OpenRouterProvider = "Anthropic"
+// OpenRouterProviderGoogle OpenRouterProvider = "Google Vertex"
+// OpenRouterProviderOpenAI OpenRouterProvider = "OpenAI"
+// OpenRouterProviderDeepSeek OpenRouterProvider = "DeepSeek"
+// OpenRouterProviderQwen OpenRouterProvider = "Hyperbolic"
+// OpenRouterProviderDeepInfra OpenRouterProvider = "DeepInfra"
+// OpenRouterProviderFireworks OpenRouterProvider = "Fireworks"
+// )
-var DefaultOpenRouterProvidersByFamily = map[OpenRouterFamily][]OpenRouterProvider{
- OpenRouterFamilyAnthropic: {OpenRouterProviderAnthropic},
- OpenRouterFamilyGoogle: {OpenRouterProviderGoogle},
- OpenRouterFamilyOpenAI: {OpenRouterProviderOpenAI},
- OpenRouterFamilyQwen: {OpenRouterProviderQwen},
- OpenRouterFamilyDeepSeek: {OpenRouterProviderDeepSeek},
-}
+// var DefaultOpenRouterProvidersByFamily = map[OpenRouterFamily][]OpenRouterProvider{
+// OpenRouterFamilyAnthropic: {OpenRouterProviderAnthropic},
+// OpenRouterFamilyGoogle: {OpenRouterProviderGoogle},
+// OpenRouterFamilyOpenAI: {OpenRouterProviderOpenAI},
+// OpenRouterFamilyQwen: {OpenRouterProviderQwen},
+// OpenRouterFamilyDeepSeek: {OpenRouterProviderDeepSeek},
+// }
-// open source models don't have fallbacks enabled for now because pricing and context limits aren't predictable across providers
-var DefaultOpenRouterAllowFallbacksByFamily = map[OpenRouterFamily]bool{
- OpenRouterFamilyAnthropic: false,
- OpenRouterFamilyGoogle: false,
- OpenRouterFamilyOpenAI: false,
-}
+// // open source models don't have fallbacks enabled for now because pricing and context limits aren't predictable across providers
+// var DefaultOpenRouterAllowFallbacksByFamily = map[OpenRouterFamily]bool{
+// OpenRouterFamilyAnthropic: true,
+// OpenRouterFamilyGoogle: true,
+// OpenRouterFamilyOpenAI: false,
+// }
diff --git a/app/shared/tokens.go b/app/shared/tokens.go
index dfa4b28d..e8fdf6fc 100644
--- a/app/shared/tokens.go
+++ b/app/shared/tokens.go
@@ -4,7 +4,6 @@ import (
"fmt"
"github.com/pkoukk/tiktoken-go"
- "github.com/sashabaranov/go-openai"
)
var tkm *tiktoken.Tiktoken
@@ -20,28 +19,3 @@ func init() {
func GetNumTokensEstimate(text string) int {
return len(tkm.Encode(text, nil, nil))
}
-
-const (
- // Per OpenAI's documentation:
- // Every message follows this format: {"role": "role_name", "content": "content"}
- // which has a 4-token overhead per message
- TokensPerMessage = 4
-
- // System, user, or assistant - each role name costs 1 token
- TokensPerName = 1
-
- // Tokens per request
- TokensPerRequest = 3
-)
-
-func GetMessagesTokenEstimate(messages ...openai.ChatCompletionMessage) int {
- tokens := 0
-
- for _, msg := range messages {
- tokens += TokensPerMessage
- tokens += TokensPerName
- tokens += GetNumTokensEstimate(msg.Content)
- }
-
- return tokens
-}