Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
wip: litellm integration for direct model provider option
Dane Schneider0397ce911bf9
diff --git a/app/docker-compose.yml b/app/docker-compose.ymlindex 2a95833b..36944797 100644--- a/app/docker-compose.yml+++ b/app/docker-compose.yml@@ -13,11 +13,15 @@ services:networks:- plandex-networkplandex-server:- image: plandexai/plandex-server:latest+ # image: plandexai/plandex-server:latest+ build:+ context: .+ dockerfile: server/Dockerfilevolumes:- plandex-files:/plandex-serverports:- "8099:8099"+ - "4000:4000"environment:DATABASE_URL: "postgres://plandex:plandex@plandex-postgres:5432/plandex?sslmode=disable"GOENV: developmentdiff --git a/app/server/Dockerfile b/app/server/Dockerfileindex 0188e14a..35d08d77 100644--- a/app/server/Dockerfile+++ b/app/server/Dockerfile@@ -4,6 +4,16 @@ FROM golang:1.23.3RUN apt-get update && \apt-get install -y git gcc g++ make+# Install Python and create a virtual environment for litellm passthrough+RUN apt-get install -y python3 python3-venv && \+ python3 -m venv /opt/venv++# Activate the virtual environment for all following RUN commands+ENV PATH="/opt/venv/bin:$PATH"++# Now install litellm/prisma in the virtual environment+RUN pip install --no-cache-dir "litellm==1.61.1" "fastapi==0.115.12" "uvicorn==0.34.1"+WORKDIR /app# Copy go.mod and go.sum for shared and server, and install dependenciesdiff --git a/app/server/litellm_proxy.py b/app/server/litellm_proxy.pynew file mode 100644index 00000000..40fe0951--- /dev/null+++ b/app/server/litellm_proxy.py@@ -0,0 +1,83 @@+from fastapi import FastAPI, Request, HTTPException+from fastapi.responses import StreamingResponse, JSONResponse, Response+from litellm import completion+import json+import re++LOGGING_ENABLED = False++print("Litellm proxy: starting proxy server on port 4000...")++app = FastAPI()+[email protected]("/health")+async def health():+ return {"status": "ok"}+[email protected]("/v1/chat/completions")+async def passthrough(request: Request):+ if LOGGING_ENABLED:+ # Log the request data for debugging+ try:+ # Get headers (excluding authorization to avoid logging credentials)+ headers = dict(request.headers)+ if "Authorization" in headers:+ headers["Authorization"] = "Bearer [REDACTED]"++ # Get request body+ body = await request.json()++ # Create a log-friendly representation+ request_data = {+ "method": request.method,+ "url": str(request.url),+ "headers": headers,+ "body": body+ }++ # Log the request data+ print("Incoming request to /v1/chat/completions:")+ print(json.dumps(request_data, indent=2))+ except Exception as e:+ print(f"Error logging request: {str(e)}")+++ payload = await request.json()++ api_key = request.headers.get("Authorization")+ if api_key and api_key.startswith("Bearer "):+ api_key = api_key.replace("Bearer ", "")+ else:+ api_key = None # optional for local/ollama models++ try:+ if payload.get("stream"):+ def stream_generator():+ try:+ response_stream = completion(api_key=api_key, **payload)+ for chunk in response_stream:+ yield f"data: {json.dumps(chunk.to_dict())}\n\n"+ yield "data: [DONE]\n\n"+ except Exception as e:+ yield f"data: {json.dumps({'error': str(e)})}\n\n"++ print(f"Litellm proxy: Initiating streaming response for model: {payload.get('model', 'unknown')}")+ return StreamingResponse(stream_generator(), media_type="text/event-stream")++ else:+ print(f"Litellm proxy: Non-streaming response requested for model: {payload.get('model', 'unknown')}")+ result = completion(api_key=api_key, **payload)+ return JSONResponse(content=result)++ except Exception as e:+ err_msg = str(e)+ print(f"Litellm proxy: Error: {err_msg}")+ status_match = re.search(r"status code: (\d+)", err_msg)+ if status_match:+ status_code = int(status_match.group(1))+ else:+ status_code = 500+ return JSONResponse(+ status_code=status_code,+ content={"error": err_msg}+ )diff --git a/app/server/main.go b/app/server/main.goindex d44cd0ef..3916f524 100644--- a/app/server/main.go+++ b/app/server/main.go@@ -3,6 +3,7 @@ package mainimport ("log""os"+ "plandex-server/model""plandex-server/routes""plandex-server/setup"@@ -17,6 +18,11 @@ func main() {return router.HandleFunc(path, handler)})+ model.EnsureLiteLLM(2)+ setup.RegisterShutdownHook(func() {+ model.ShutdownLiteLLMServer()+ })+r := mux.NewRouter()routes.AddHealthRoutes(r)routes.AddApiRoutes(r)diff --git a/app/server/model/client.go b/app/server/model/client.goindex e31df002..9a60daa2 100644--- a/app/server/model/client.go+++ b/app/server/model/client.go@@ -154,6 +154,7 @@ func createChatCompletionExtended(ctx context.Context,extendedReq types.ExtendedChatCompletionRequest,) (openai.ChatCompletionResponse, error) {+var openaiReq *types.ExtendedOpenAIChatCompletionRequestif modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenAI && !modelConfig.BaseModelConfig.UsesOpenAIResponsesAPI {log.Println("Creating chat completion with direct OpenAI provider request")@@ -179,7 +180,9 @@ func createChatCompletionExtended(req.Header.Set("OpenAI-Organization", client.OrgId)}- addOpenRouterHeaders(req)+ if modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenRouter {+ addOpenRouterHeaders(req)+ }// Add bodyvar jsonBody []byte@@ -191,6 +194,7 @@ func createChatCompletionExtended(if err != nil {return openai.ChatCompletionResponse{}, err}+req.Body = io.NopCloser(bytes.NewReader(jsonBody))// Make request@@ -222,6 +226,7 @@ func createChatCompletionStreamExtended(ctx context.Context,extendedReq types.ExtendedChatCompletionRequest,) (*ExtendedChatCompletionStream, error) {+var openaiReq *types.ExtendedOpenAIChatCompletionRequestif modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenAI && !modelConfig.BaseModelConfig.UsesOpenAIResponsesAPI {openaiReq = extendedReq.ToOpenAI()@@ -265,7 +270,9 @@ func createChatCompletionStreamExtended(req.Header.Set("OpenAI-Organization", client.OrgId)}- addOpenRouterHeaders(req)+ if modelConfig.BaseModelConfig.Provider == shared.ModelProviderOpenRouter {+ addOpenRouterHeaders(req)+ }// Send the requestresp, err := httpClient.Do(req) //nolint:bodyclose // body is closed in stream.Close()@@ -408,6 +415,8 @@ func (stream *ExtendedChatCompletionStream) Close() error {func isNonRetriableErr(err error) bool {errStr := err.Error()+ log.Println("isNonRetriableErr errStr:\n", errStr)+// we don't want to retry on the errors belowif strings.Contains(errStr, "context deadline exceeded") || strings.Contains(errStr, "context canceled") {log.Println("Context deadline exceeded or canceled - no retry")@@ -416,7 +425,7 @@ func isNonRetriableErr(err error) bool {if strings.Contains(errStr, "status code: 400") &&strings.Contains(errStr, "reduce the length of the messages") {- log.Println("Token limit exceeded - no retry")+ log.Println("Bad request - no retry")return true}@@ -430,6 +439,8 @@ func isNonRetriableErr(err error) bool {return true}+ log.Println("isNonRetriableErr false")+return false}diff --git a/app/server/model/litellm.go b/app/server/model/litellm.gonew file mode 100644index 00000000..6e553576--- /dev/null+++ b/app/server/model/litellm.go@@ -0,0 +1,131 @@+package model++import (+ "context"+ "fmt"+ "log"+ "net/http"+ "os"+ "os/exec"+ "strconv"+ "sync"+ "time"+)++var (+ liteLLMOnce sync.Once+ liteLLMCmd *exec.Cmd+)++func EnsureLiteLLM(numWorkers int) error {+ var finalErr error+ liteLLMOnce.Do(func() {+ if isLiteLLMHealthy() {+ log.Println("LiteLLM proxy is already healthy")+ return+ }++ log.Println("LiteLLM proxy is not running. Starting...")++ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()++ err := startLiteLLMServer(numWorkers)+ if err != nil {+ log.Println("LiteLLM proxy launch failed:", err)+ finalErr = fmt.Errorf("LiteLLM proxy launch failed: %w", err)+ return+ }++ ticker := time.NewTicker(500 * time.Millisecond)+ defer ticker.Stop()++ for {+ select {+ case <-ctx.Done():+ log.Println("LiteLLM proxy launch timed out")+ finalErr = fmt.Errorf("LiteLLM proxy launch timed out")+ return+ case <-ticker.C:+ if isLiteLLMHealthy() {+ log.Println("LiteLLM proxy is healthy")+ return+ } else {+ log.Println("LiteLLM proxy is not healthy yet, retrying after 500ms...")+ }+ }+ }+ })++ return finalErr+}++func ShutdownLiteLLMServer() error {+ if liteLLMCmd != nil && liteLLMCmd.Process != nil {+ log.Println("Shutting down LiteLLM proxy gracefully...")+ if err := liteLLMCmd.Process.Signal(os.Interrupt); err != nil {+ return fmt.Errorf("failed to signal LiteLLM for shutdown: %w", err)+ }++ done := make(chan error, 1)+ go func() {+ done <- liteLLMCmd.Wait()+ }()++ select {+ case <-time.After(5 * time.Second):+ log.Println("LiteLLM proxy shutdown timed out, forcing kill")+ return liteLLMCmd.Process.Kill()+ case err := <-done:+ return err+ }+ }+ return nil+}++func isLiteLLMHealthy() bool {+ ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)+ defer cancel()++ req, err := http.NewRequestWithContext(ctx, "GET", "http://localhost:4000/health", nil)+ if err != nil {+ log.Println("LiteLLM health check request failed:", err)+ return false+ }++ resp, err := http.DefaultClient.Do(req)+ if err != nil {+ log.Println("LiteLLM health check failed:", err)+ return false+ }+ defer resp.Body.Close()++ return resp.StatusCode == 200+}++func startLiteLLMServer(numWorkers int) error {+ liteLLMCmd = exec.Command("python3",+ "-m", "uvicorn",+ "litellm_proxy:app",+ "--host", "0.0.0.0",+ "--port", "4000",+ "--workers", strconv.Itoa(numWorkers),+ )++ // clean env+ liteLLMCmd.Env = []string{+ "PATH=" + os.Getenv("PATH"),+ "HOME=" + os.Getenv("HOME"),+ }++ liteLLMCmd.Stdout = os.Stdout+ liteLLMCmd.Stderr = os.Stderr++ err := liteLLMCmd.Start()+ if err != nil {+ return err+ }++ log.Println("LiteLLM proxy launched")+ return nil+}diff --git a/app/server/model/plan/tell_stream_main.go b/app/server/model/plan/tell_stream_main.goindex 10588af3..95c79157 100644--- a/app/server/model/plan/tell_stream_main.go+++ b/app/server/model/plan/tell_stream_main.go@@ -1,6 +1,7 @@package planimport (+ "errors""fmt""log""net/http"@@ -61,6 +62,9 @@ func (state *activeTellStreamState) listenStream(stream *model.ExtendedChatComplmodelProvider := state.modelConfig.BaseModelConfig.ProvidermodelName := state.modelConfig.BaseModelConfig.ModelName+ log.Println("modelProvider", modelProvider)+ log.Println("modelName", modelName)+mainLoop:for {select {@@ -117,7 +121,7 @@ mainLoop:msg = fmt.Sprintf("The AI model (%s/%s) stopped responding: %v", modelProvider, modelName, err)}state.onError(onErrorParams{- streamErr: fmt.Errorf(msg, err),+ streamErr: errors.New(msg),storeDesc: true,canRetry: active.CurrentReplyContent == "", // if there was no output yet, we can retry})@@ -140,7 +144,7 @@ mainLoop:res := state.onError(onErrorParams{streamErr: fmt.Errorf("The AI model (%s/%s) stopped streaming with error code %d: %s", modelProvider, modelName, response.Error.Code, response.Error.Message),storeDesc: true,- canRetry: active.CurrentReplyContent == "" && response.Error.Code != 429,+ canRetry: active.CurrentReplyContent == "" && !(response.Error.Code >= 400 && response.Error.Code < 500),})if res.shouldReturn {returndiff --git a/app/shared/ai_models_available.go b/app/shared/ai_models_available.goindex 8c327d08..2b4f56d5 100644--- a/app/shared/ai_models_available.go+++ b/app/shared/ai_models_available.go@@ -230,6 +230,23 @@ var AvailableModels = []*AvailableModel{IncludeReasoning: true,},},+ {+ Description: "Anthropic Claude 3.7 Sonnet",+ DefaultMaxConvoTokens: 15000,+ BaseModelConfig: BaseModelConfig{+ Provider: ModelProviderAnthropic,+ ModelName: "claude-3-7-sonnet-latest",+ ModelId: "claude-3-7-sonnet-latest",+ MaxTokens: 200000,+ MaxOutputTokens: 128000,+ ReservedOutputTokens: 20000,+ SupportsCacheControl: true,+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderAnthropic],+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderAnthropic],+ PreferredModelOutputFormat: ModelOutputFormatXml,+ },+ },{Description: "Anthropic Claude 3.5 Sonnet via OpenRouter",DefaultMaxConvoTokens: 15000,@@ -296,6 +313,24 @@ var AvailableModels = []*AvailableModel{PreferredModelOutputFormat: ModelOutputFormatXml,},},++ {+ Description: "Google Gemini Pro 1.5 Pro via AI Studio",+ DefaultMaxConvoTokens: 75000,+ BaseModelConfig: BaseModelConfig{+ Provider: ModelProviderGoogleAIStudio,+ ModelName: "models/gemini-1.5-pro",+ ModelId: "models/gemini-1.5-pro",+ MaxTokens: 2000000,+ MaxOutputTokens: 8192,+ ReservedOutputTokens: 8192,+ ApiKeyEnvVar: GoogleAIStudioApiKeyEnvVar,+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderGoogleAIStudio],+ PreferredModelOutputFormat: ModelOutputFormatXml,+ },+ },+{Description: "Google Gemini Flash 2.0 via OpenRouter",DefaultMaxConvoTokens: 75000,@@ -579,6 +614,120 @@ var AvailableModels = []*AvailableModel{PredictedOutputEnabled: true,},},++ // OpenAI models via Azure+ {+ Description: "OpenAI o3-mini-high via Azure",+ DefaultMaxConvoTokens: 10000,+ BaseModelConfig: BaseModelConfig{+ Provider: ModelProviderAzureOpenAI,+ ModelName: "openai/o3-mini",+ ModelId: "openai/o3-mini-high",+ MaxTokens: 200000,+ MaxOutputTokens: 100000,+ ReservedOutputTokens: 40000,+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderAzureOpenAI],+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderAzureOpenAI],+ PreferredModelOutputFormat: ModelOutputFormatToolCallJson,+ SystemPromptDisabled: true,+ RoleParamsDisabled: true,+ ReasoningEffortEnabled: true,+ ReasoningEffort: ReasoningEffortHigh,+ },+ },+ {+ Description: "OpenAI o3-mini-medium via Azure",+ DefaultMaxConvoTokens: 10000,+ BaseModelConfig: BaseModelConfig{+ Provider: ModelProviderAzureOpenAI,+ ModelName: "openai/o3-mini",+ ModelId: "openai/o3-mini-medium",+ MaxTokens: 200000,+ MaxOutputTokens: 100000,+ ReservedOutputTokens: 40000,+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderAzureOpenAI],+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderAzureOpenAI],+ PreferredModelOutputFormat: ModelOutputFormatToolCallJson,+ SystemPromptDisabled: true,+ RoleParamsDisabled: true,+ ReasoningEffortEnabled: true,+ ReasoningEffort: ReasoningEffortMedium,+ },+ },+ {+ Description: "OpenAI o3-mini-low via Azure",+ DefaultMaxConvoTokens: 10000,+ BaseModelConfig: BaseModelConfig{+ Provider: ModelProviderAzureOpenAI,+ ModelName: "openai/o3-mini",+ ModelId: "openai/o3-mini-low",+ MaxTokens: 200000,+ MaxOutputTokens: 100000,+ ReservedOutputTokens: 40000,+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderAzureOpenAI],+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderAzureOpenAI],+ PreferredModelOutputFormat: ModelOutputFormatToolCallJson,+ SystemPromptDisabled: true,+ RoleParamsDisabled: true,+ ReasoningEffortEnabled: true,+ ReasoningEffort: ReasoningEffortLow,+ },+ },+ {+ Description: "OpenAI o1 via Azure",+ DefaultMaxConvoTokens: 15000,+ BaseModelConfig: BaseModelConfig{+ Provider: ModelProviderAzureOpenAI,+ ModelName: "openai/o1",+ ModelId: "openai/o1",+ MaxTokens: 200000,+ MaxOutputTokens: 100000,+ ReservedOutputTokens: 40000,+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderAzureOpenAI],+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderAzureOpenAI],+ PreferredModelOutputFormat: ModelOutputFormatXml,+ SystemPromptDisabled: true,+ RoleParamsDisabled: true,+ },+ },+ {+ Description: "OpenAI gpt-4o via Azure",+ DefaultMaxConvoTokens: 10000,+ BaseModelConfig: BaseModelConfig{+ Provider: ModelProviderAzureOpenAI,+ ModelName: "openai/gpt-4o",+ ModelId: "openai/gpt-4o",+ MaxTokens: 128000,+ MaxOutputTokens: 16384,+ ReservedOutputTokens: 16384,+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderAzureOpenAI],+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderAzureOpenAI],+ PreferredModelOutputFormat: ModelOutputFormatToolCallJson,+ PredictedOutputEnabled: true,+ },+ },+ {+ Description: "OpenAI gpt-4o-mini via Azure",+ DefaultMaxConvoTokens: 10000,+ BaseModelConfig: BaseModelConfig{+ Provider: ModelProviderAzureOpenAI,+ ModelName: "openai/gpt-4o-mini",+ ModelId: "openai/gpt-4o-mini",+ MaxTokens: 128000,+ MaxOutputTokens: 16384,+ ReservedOutputTokens: 16384,+ ApiKeyEnvVar: ApiKeyByProvider[ModelProviderAzureOpenAI],+ ModelCompatibility: fullCompatibility,+ BaseUrl: BaseUrlByProvider[ModelProviderAzureOpenAI],+ PreferredModelOutputFormat: ModelOutputFormatToolCallJson,+ PredictedOutputEnabled: true,+ },+ },}var AvailableModelsByComposite = map[string]*AvailableModel{}diff --git a/app/shared/ai_models_providers.go b/app/shared/ai_models_providers.goindex 8fad6cb2..1bf49c89 100644--- a/app/shared/ai_models_providers.go+++ b/app/shared/ai_models_providers.go@@ -1,31 +1,67 @@package shared-const OpenAIEnvVar = "OPENAI_API_KEY"const OpenAIV1BaseUrl = "https://api.openai.com/v1"-const OpenRouterApiKeyEnvVar = "OPENROUTER_API_KEY"const OpenRouterBaseUrl = "https://openrouter.ai/api/v1"+const LiteLLMBaseUrl = "http://localhost:4000/v1" // runs in the same container alongside the plandex server++const OpenAIEnvVar = "OPENAI_API_KEY"+const OpenRouterApiKeyEnvVar = "OPENROUTER_API_KEY"+const AnthropicApiKeyEnvVar = "ANTHROPIC_API_KEY"+const GoogleAIStudioApiKeyEnvVar = "GEMINI_API_KEY"+const GoogleVertexApiKeyEnvVar = "VERTEX_API_KEY"+const AzureOpenAIEnvVar = "AZURE_OPENAI_API_KEY"+const DeepSeekApiKeyEnvVar = "DEEPSEEK_API_KEY"+const PerplexityApiKeyEnvVar = "PERPLEXITY_API_KEY"type ModelProvider stringconst (ModelProviderOpenRouter ModelProvider = "openrouter"ModelProviderOpenAI ModelProvider = "openai"- ModelProviderCustom ModelProvider = "custom"++ ModelProviderAnthropic ModelProvider = "anthropic"+ ModelProviderGoogleAIStudio ModelProvider = "google-ai-studio"+ ModelProviderGoogleVertex ModelProvider = "google-vertex"+ ModelProviderAzureOpenAI ModelProvider = "azure-openai"+ ModelProviderDeepSeek ModelProvider = "deepseek"+ ModelProviderPerplexity ModelProvider = "perplexity"++ ModelProviderCustom ModelProvider = "custom")var AllModelProviders = []string{string(ModelProviderOpenAI),string(ModelProviderOpenRouter),- // string(ModelProviderTogether),+ string(ModelProviderAnthropic),+ string(ModelProviderGoogleAIStudio),+ string(ModelProviderGoogleVertex),+ string(ModelProviderAzureOpenAI),+ string(ModelProviderDeepSeek),+ string(ModelProviderPerplexity),string(ModelProviderCustom),}var BaseUrlByProvider = map[ModelProvider]string{ModelProviderOpenAI: OpenAIV1BaseUrl,ModelProviderOpenRouter: OpenRouterBaseUrl,++ // apart from openai and openrouter, the rest are supported by liteLLM+ ModelProviderAnthropic: LiteLLMBaseUrl,+ ModelProviderGoogleAIStudio: LiteLLMBaseUrl,+ ModelProviderGoogleVertex: LiteLLMBaseUrl,+ ModelProviderAzureOpenAI: LiteLLMBaseUrl,+ ModelProviderDeepSeek: LiteLLMBaseUrl,+ ModelProviderPerplexity: LiteLLMBaseUrl,}var ApiKeyByProvider = map[ModelProvider]string{ModelProviderOpenAI: OpenAIEnvVar,ModelProviderOpenRouter: OpenRouterApiKeyEnvVar,++ ModelProviderAnthropic: AnthropicApiKeyEnvVar,+ ModelProviderGoogleAIStudio: GoogleAIStudioApiKeyEnvVar,+ ModelProviderGoogleVertex: GoogleVertexApiKeyEnvVar,+ ModelProviderAzureOpenAI: AzureOpenAIEnvVar,+ ModelProviderDeepSeek: DeepSeekApiKeyEnvVar,+ ModelProviderPerplexity: PerplexityApiKeyEnvVar,}