Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

improved model call error handling / retries with exponential backoff

Dane Schneider2b939aaca35c
diff --git a/app/server/model/client.go b/app/server/model/client.go
index fbc3526e..e4bc6dfa 100644
--- a/app/server/model/client.go
+++ b/app/server/model/client.go
@@ -1,6 +1,9 @@
package model
import (
+ "context"
+ "log"
+ "strings"
"time"
"github.com/sashabaranov/go-openai"
@@ -12,3 +15,118 @@ func NewClient(apiKey string) *openai.Client {
config := openai.DefaultConfig(apiKey)
return openai.NewClientWithConfig(config)
}
+
+func CreateChatCompletionStreamWithRetries(
+ client *openai.Client,
+ ctx context.Context,
+ req openai.ChatCompletionRequest,
+) (*openai.ChatCompletionStream, error) {
+ return createChatCompletionStream(client, ctx, req, 0)
+}
+
+func createChatCompletionStream(
+ client *openai.Client,
+ ctx context.Context,
+ req openai.ChatCompletionRequest,
+ numRetry int,
+) (*openai.ChatCompletionStream, error) {
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+
+ stream, err := client.CreateChatCompletionStream(ctx, req)
+
+ if err != nil {
+ log.Printf("Error creating chat completion stream: %v, retry: %d\n", err, numRetry)
+
+ if isNonRetriableErr(err) {
+ return nil, err
+ }
+
+ // for retriable errors, retry with exponential backoff
+ if numRetry < 5 {
+ waitBackoff(numRetry)
+ return createChatCompletionStream(client, ctx, req, numRetry+1)
+ }
+
+ log.Println("Max retries reached - no retry")
+ return nil, err
+ }
+
+ return stream, nil
+}
+
+func CreateChatCompletionWithRetries(
+ client *openai.Client,
+ ctx context.Context,
+ req openai.ChatCompletionRequest,
+) (openai.ChatCompletionResponse, error) {
+ return createChatCompletion(client, ctx, req, 0)
+}
+
+func createChatCompletion(
+ client *openai.Client,
+ ctx context.Context,
+ req openai.ChatCompletionRequest,
+ numRetry int,
+) (openai.ChatCompletionResponse, error) {
+
+ if ctx.Err() != nil {
+ return openai.ChatCompletionResponse{}, ctx.Err()
+ }
+
+ resp, err := client.CreateChatCompletion(ctx, req)
+
+ if err != nil {
+ log.Printf("Error creating chat completion: %v, retry: %d\n", err, numRetry)
+
+ if isNonRetriableErr(err) {
+ return openai.ChatCompletionResponse{}, err
+ }
+
+ // for retriable errors, retry with exponential backoff
+ if numRetry < 5 {
+ waitBackoff(numRetry)
+ return createChatCompletion(client, ctx, req, numRetry+1)
+ }
+
+ log.Println("Max retries reached - no retry")
+ return openai.ChatCompletionResponse{}, err
+ }
+
+ return resp, nil
+}
+
+func isNonRetriableErr(err error) bool {
+ errStr := err.Error()
+
+ // we don't want to retry on the errors below
+ if strings.Contains(errStr, "context deadline exceeded") || strings.Contains(errStr, "context canceled") {
+ log.Println("Context deadline exceeded or canceled - no retry")
+ return true
+ }
+
+ if strings.Contains(errStr, "status code: 400") &&
+ strings.Contains(errStr, "reduce the length of the messages") {
+ log.Println("Token limit exceeded - no retry")
+ return true
+ }
+
+ if strings.Contains(errStr, "status code: 401") {
+ log.Println("Invalid auth or api key - no retry")
+ return true
+ }
+
+ if strings.Contains(errStr, "status code: 429") && strings.Contains(errStr, "exceeded your current quota") {
+ log.Println("Current quota exceeded - no retry")
+ return true
+ }
+
+ return false
+}
+
+func waitBackoff(numRetry int) {
+ d := time.Duration(1<<uint(numRetry)) * time.Second
+ log.Printf("Retrying in %v\n", d)
+ time.Sleep(d)
+}
diff --git a/app/server/model/name.go b/app/server/model/name.go
index fc31830c..48185861 100644
--- a/app/server/model/name.go
+++ b/app/server/model/name.go
@@ -23,7 +23,8 @@ func GenPlanName(client *openai.Client, config shared.TaskRoleConfig, planConten
Content: prompts.GetPlanNamePrompt(planContent),
})
- resp, err := client.CreateChatCompletion(
+ resp, err := CreateChatCompletionWithRetries(
+ client,
context.Background(),
openai.ChatCompletionRequest{
Model: config.BaseModelConfig.ModelName,
diff --git a/app/server/model/plan/build_exec.go b/app/server/model/plan/build_exec.go
index 2ccf6df2..17d2e0d2 100644
--- a/app/server/model/plan/build_exec.go
+++ b/app/server/model/plan/build_exec.go
@@ -4,6 +4,7 @@ import (
"fmt"
"log"
"plandex-server/db"
+ "plandex-server/model"
"plandex-server/model/prompts"
"plandex-server/types"
@@ -244,9 +245,10 @@ func (fileState *activeBuildStreamFileState) buildFile() {
ResponseFormat: config.OpenAIResponseFormat,
}
- stream, err := client.CreateChatCompletionStream(activePlan.Ctx, modelReq)
+ stream, err := model.CreateChatCompletionStreamWithRetries(client, activePlan.Ctx, modelReq)
if err != nil {
log.Printf("Error creating plan file stream for path '%s': %v\n", filePath, err)
+ fileState.onBuildFileError(fmt.Errorf("error creating plan file stream for path '%s': %v", filePath, err))
return
}
diff --git a/app/server/model/plan/commit_msg.go b/app/server/model/plan/commit_msg.go
index 8b174eef..01f16d92 100644
--- a/app/server/model/plan/commit_msg.go
+++ b/app/server/model/plan/commit_msg.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"plandex-server/db"
+ "plandex-server/model"
"plandex-server/model/prompts"
"github.com/plandex/plandex/shared"
@@ -13,7 +14,8 @@ import (
func genPlanDescription(client *openai.Client, config shared.TaskRoleConfig, planId, branch string, ctx context.Context) (*db.ConvoMessageDescription, error) {
- descResp, err := client.CreateChatCompletion(
+ descResp, err := model.CreateChatCompletionWithRetries(
+ client,
ctx,
openai.ChatCompletionRequest{
Model: config.BaseModelConfig.ModelName,
diff --git a/app/server/model/plan/exec_status.go b/app/server/model/plan/exec_status.go
index 01ad9196..59520435 100644
--- a/app/server/model/plan/exec_status.go
+++ b/app/server/model/plan/exec_status.go
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"log"
+ "plandex-server/model"
"plandex-server/model/prompts"
"strings"
@@ -47,7 +48,8 @@ func ExecStatusShouldContinue(client *openai.Client, config shared.TaskRoleConfi
log.Println("Calling model to check if plan should continue")
- resp, err := client.CreateChatCompletion(
+ resp, err := model.CreateChatCompletionWithRetries(
+ client,
ctx,
openai.ChatCompletionRequest{
Model: config.BaseModelConfig.ModelName,
diff --git a/app/server/model/plan/tell_exec.go b/app/server/model/plan/tell_exec.go
index b5223a0a..71004292 100644
--- a/app/server/model/plan/tell_exec.go
+++ b/app/server/model/plan/tell_exec.go
@@ -5,9 +5,9 @@ import (
"log"
"net/http"
"os"
- "strings"
"plandex-server/db"
+ "plandex-server/model"
"plandex-server/model/lib"
"plandex-server/model/prompts"
"plandex-server/types"
@@ -344,21 +344,14 @@ func execTellPlan(
TopP: state.settings.ModelSet.Planner.TopP,
}
- stream, err := client.CreateChatCompletionStream(active.ModelStreamCtx, modelReq)
+ stream, err := model.CreateChatCompletionStreamWithRetries(client, active.ModelStreamCtx, modelReq)
if err != nil {
- log.Printf("Error creating proposal GPT4 stream: %v\n", err)
- log.Println(err)
-
- errStr := err.Error()
- if strings.Contains(errStr, "status code: 400") &&
- strings.Contains(errStr, "reduce the length of the messages") {
- log.Println("Token limit exceeded")
- }
+ log.Printf("Error starting reply stream: %v\n", err)
active.StreamDoneCh <- &shared.ApiError{
Type: shared.ApiErrorTypeOther,
Status: http.StatusInternalServerError,
- Msg: "Error creating proposal GPT4 stream",
+ Msg: "Error starting reply stream: " + err.Error(),
}
return
}
diff --git a/app/server/model/plan/tell_stream.go b/app/server/model/plan/tell_stream.go
index 437e589a..cbf52251 100644
--- a/app/server/model/plan/tell_stream.go
+++ b/app/server/model/plan/tell_stream.go
@@ -136,7 +136,7 @@ func (state *activeTellStreamState) listenStream(stream *openai.ChatCompletionSt
summaries: summaries,
promptMessage: promptMessage,
currentOrgId: currentOrgId,
- })
+ }, active.Ctx)
}
log.Println("Locking repo to store assistant reply and description")
diff --git a/app/server/model/plan/tell_summary.go b/app/server/model/plan/tell_summary.go
index 33a6fab3..f8e0b1ff 100644
--- a/app/server/model/plan/tell_summary.go
+++ b/app/server/model/plan/tell_summary.go
@@ -1,6 +1,7 @@
package plan
import (
+ "context"
"errors"
"fmt"
"log"
@@ -149,7 +150,7 @@ type summarizeConvoParams struct {
currentOrgId string
}
-func summarizeConvo(client *openai.Client, config shared.ModelRoleConfig, params summarizeConvoParams) error {
+func summarizeConvo(client *openai.Client, config shared.ModelRoleConfig, params summarizeConvoParams, ctx context.Context) error {
log.Printf("summarizeConvo: Called for plan ID %s on branch %s\n", params.planId, params.branch)
log.Printf("summarizeConvo: Starting summarizeConvo for planId: %s\n", params.planId)
planId := params.planId
@@ -222,7 +223,7 @@ func summarizeConvo(client *openai.Client, config shared.ModelRoleConfig, params
NumMessages: numMessagesSummarized + 1,
OrgId: currentOrgId,
PlanId: planId,
- })
+ }, ctx)
if err != nil {
log.Printf("summarizeConvo: Error generating plan summary for plan %s: %v\n", params.planId, err)
diff --git a/app/server/model/summarize.go b/app/server/model/summarize.go
index 21eecc3d..4d387d41 100644
--- a/app/server/model/summarize.go
+++ b/app/server/model/summarize.go
@@ -20,7 +20,7 @@ type PlanSummaryParams struct {
PlanId string
}
-func PlanSummary(client *openai.Client, config shared.ModelRoleConfig, params PlanSummaryParams) (*db.ConvoSummary, error) {
+func PlanSummary(client *openai.Client, config shared.ModelRoleConfig, params PlanSummaryParams, ctx context.Context) (*db.ConvoSummary, error) {
messages := []openai.ChatCompletionMessage{
{
Role: openai.ChatMessageRoleSystem,
@@ -40,8 +40,9 @@ func PlanSummary(client *openai.Client, config shared.ModelRoleConfig, params Pl
// fmt.Println("summarizing messages:")
// spew.Dump(messages)
- resp, err := client.CreateChatCompletion(
- context.Background(),
+ resp, err := CreateChatCompletionWithRetries(
+ client,
+ ctx,
openai.ChatCompletionRequest{
Model: config.BaseModelConfig.ModelName,
Messages: messages,