Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

Refactor tell_plan implementation with manual stop sequences and error recovery

Dane Schneider7f45a8f64089
diff --git a/app/server/model/plan/tell_exec.go b/app/server/model/plan/tell_exec.go
index 0138962a..cd99d593 100644
--- a/app/server/model/plan/tell_exec.go
+++ b/app/server/model/plan/tell_exec.go
@@ -62,7 +62,6 @@ type execTellPlanParams struct {
iteration int
missingFileResponse shared.RespondMissingFileChoice
shouldBuildPending bool
- numErrorRetry int
unfinishedSubtaskReasoning string
}
@@ -78,6 +77,7 @@ func execTellPlan(params execTellPlanParams) {
unfinishedSubtaskReasoning := params.unfinishedSubtaskReasoning
log.Printf("[TellExec] Starting iteration %d for plan %s on branch %s", iteration, plan.Id, branch)
+
currentUserId := auth.User.Id
currentOrgId := auth.OrgId
@@ -136,7 +136,6 @@ func execTellPlan(params execTellPlanParams) {
state := &activeTellStreamState{
modelStreamId: active.ModelStreamId,
- execTellPlanParams: params,
clients: clients,
req: req,
auth: auth,
@@ -149,7 +148,7 @@ func execTellPlan(params execTellPlanParams) {
}
log.Println("execTellPlan - Loading tell plan")
- err = state.loadTellPlan(params.numErrorRetry > 0)
+ err = state.loadTellPlan()
if err != nil {
return
}
@@ -171,7 +170,7 @@ func execTellPlan(params execTellPlanParams) {
}
} else if state.currentStage.TellStage == shared.TellStageImplementation {
tentativeModelConfig = state.settings.ModelPack.GetCoder()
- tentativeMaxTokens = tentativeModelConfig.GetFinalLargeContextFallback().BaseModelConfig.MaxTokens
+ tentativeMaxTokens = state.settings.GetCoderEffectiveMaxTokens()
} else {
log.Printf("Tell plan - execTellPlan - unknown tell stage: %s\n", state.currentStage.TellStage)
go notify.NotifyErr(notify.SeverityError, fmt.Errorf("execTellPlan: unknown tell stage: %s", state.currentStage.TellStage))
@@ -183,7 +182,6 @@ func execTellPlan(params execTellPlanParams) {
}
return
}
- state.tenativeModelConfig = &tentativeModelConfig
ok, tokensWithoutContext := state.dryRunCalculateTokensWithoutContext(tentativeMaxTokens, unfinishedSubtaskReasoning)
if !ok {
@@ -307,7 +305,18 @@ func execTellPlan(params execTellPlanParams) {
log.Printf("Latest summary tokens: %d\n", state.latestSummaryTokens)
log.Printf("Total tokens before convo: %d\n", state.tokensBeforeConvo)
- if state.tokensBeforeConvo > state.settings.GetPlannerEffectiveMaxTokens() {
+ var effectiveMaxTokens int
+ if state.currentStage.TellStage == shared.TellStagePlanning {
+ if state.currentStage.PlanningPhase == shared.PlanningPhaseContext {
+ effectiveMaxTokens = state.settings.GetArchitectEffectiveMaxTokens()
+ } else {
+ effectiveMaxTokens = state.settings.GetPlannerEffectiveMaxTokens()
+ }
+ } else if state.currentStage.TellStage == shared.TellStageImplementation {
+ effectiveMaxTokens = state.settings.GetCoderEffectiveMaxTokens()
+ }
+
+ if state.tokensBeforeConvo > effectiveMaxTokens {
// token limit already exceeded before adding conversation
err := fmt.Errorf("token limit exceeded before adding conversation")
log.Printf("Error: %v\n", err)
@@ -343,9 +352,7 @@ func execTellPlan(params execTellPlanParams) {
state.replyId = uuid.New().String()
state.replyParser = types.NewReplyParser()
- if missingFileResponse == "" {
- state.messages = append(state.messages, *promptMessage)
- } else if !state.handleMissingFileResponse(unfinishedSubtaskReasoning) {
+ if missingFileResponse != "" && !state.handleMissingFileResponse(unfinishedSubtaskReasoning) {
return
}
@@ -360,7 +367,6 @@ func execTellPlan(params execTellPlanParams) {
requestTokens := model.GetMessagesTokenEstimate(state.messages...) + model.TokensPerRequest
state.totalRequestTokens = requestTokens
- stop := []string{"<PlandexFinish/>"}
modelConfig := tentativeModelConfig
log.Println("Tell plan - setting modelConfig")
@@ -374,7 +380,7 @@ func execTellPlan(params execTellPlanParams) {
modelConfig = state.settings.ModelPack.GetArchitect().GetRoleForInputTokens(requestTokens)
log.Println("Tell plan - got modelConfig for context phase")
} else if state.currentStage.PlanningPhase == shared.PlanningPhaseTasks {
- modelConfig = state.settings.ModelPack.Planner.GetRoleForInputTokens(requestTokens).ModelRoleConfig
+ modelConfig = state.settings.ModelPack.Planner.GetRoleForInputTokens(requestTokens)
log.Println("Tell plan - got modelConfig for tasks phase")
}
} else if state.currentStage.TellStage == shared.TellStageImplementation {
@@ -383,6 +389,7 @@ func execTellPlan(params execTellPlanParams) {
}
log.Println("Tell plan - modelConfig:", spew.Sdump(modelConfig))
+ state.modelConfig = &modelConfig
// if the model doesn't support cache control, remove the cache control spec from the messages
if !modelConfig.BaseModelConfig.SupportsCacheControl {
@@ -423,6 +430,7 @@ func execTellPlan(params execTellPlanParams) {
InputTokens: requestTokens,
OutputTokens: modelConfig.BaseModelConfig.MaxOutputTokens - requestTokens,
ModelName: modelConfig.BaseModelConfig.ModelName,
+ IsUserPrompt: true,
},
})
if apiErr != nil {
@@ -430,6 +438,28 @@ func execTellPlan(params execTellPlanParams) {
return
}
+ state.doTellRequest()
+
+ if shouldBuildPending {
+ go state.queuePendingBuilds()
+ }
+
+ UpdateActivePlan(planId, branch, func(ap *types.ActivePlan) {
+ ap.CurrentStreamingReplyId = state.replyId
+ ap.CurrentReplyDoneCh = make(chan bool, 1)
+ })
+
+}
+
+func (state *activeTellStreamState) doTellRequest() {
+ clients := state.clients
+ modelConfig := state.modelConfig
+ active := state.activePlan
+
+ fallbackRes := modelConfig.GetFallbackForModelError(state.numErrorRetry, state.modelErr)
+ modelConfig = fallbackRes.ModelRoleConfig
+ stop := []string{"<PlandexFinish/>"}
+
// log.Println("Stop:", stop)
// spew.Dump(state.messages)
@@ -444,12 +474,19 @@ func execTellPlan(params execTellPlanParams) {
},
Temperature: modelConfig.Temperature,
TopP: modelConfig.TopP,
- Stop: stop,
}
+ if modelConfig.BaseModelConfig.StopDisabled {
+ state.manualStop = stop
+ } else {
+ modelReq.Stop = stop
+ }
+
+ // update state
+ state.fallbackRes = fallbackRes
state.requestStartedAt = time.Now()
state.originalReq = &modelReq
- state.modelConfig = &modelConfig
+ state.modelConfig = modelConfig
// output the modelReq to a json file
// if jsonData, err := json.MarshalIndent(modelReq, "", " "); err == nil {
@@ -462,7 +499,11 @@ func execTellPlan(params execTellPlanParams) {
// log.Printf("Error marshaling model request to JSON: %v\n", err)
// }
- stream, err := model.CreateChatCompletionStream(clients, &modelConfig, active.ModelStreamCtx, modelReq)
+ log.Printf("[Tell] doTellRequest retry=%d fallbackRetry=%d using model=%s",
+ state.numErrorRetry, state.numFallbackRetry, state.modelConfig.BaseModelConfig.ModelName)
+
+ // start the stream
+ stream, err := model.CreateChatCompletionStream(clients, modelConfig, active.ModelStreamCtx, modelReq)
if err != nil {
log.Printf("Error starting reply stream: %v\n", err)
go notify.NotifyErr(notify.SeverityError, fmt.Errorf("error starting reply stream: %v", err))
@@ -474,22 +515,13 @@ func execTellPlan(params execTellPlanParams) {
return
}
- if shouldBuildPending {
- go state.queuePendingBuilds()
- }
-
- UpdateActivePlan(planId, branch, func(ap *types.ActivePlan) {
- ap.CurrentStreamingReplyId = state.replyId
- ap.CurrentReplyDoneCh = make(chan bool, 1)
- })
-
+ // handle stream chunks
go state.listenStream(stream)
}
func (state *activeTellStreamState) dryRunCalculateTokensWithoutContext(tentativeMaxTokens int, unfinishedSubtaskReasoning string) (bool, int) {
clone := &activeTellStreamState{
modelStreamId: state.modelStreamId,
- execTellPlanParams: state.execTellPlanParams,
clients: state.clients,
req: state.req,
auth: state.auth,
@@ -513,7 +545,6 @@ func (state *activeTellStreamState) dryRunCalculateTokensWithoutContext(tentativ
hasAssistantReply: state.hasAssistantReply,
modelContext: state.modelContext,
activePlan: state.activePlan,
- tenativeModelConfig: state.tenativeModelConfig,
}
sysParts, err := clone.getTellSysPrompt(getTellSysPromptParams{
@@ -558,7 +589,18 @@ func (state *activeTellStreamState) dryRunCalculateTokensWithoutContext(tentativ
clone.latestSummaryTokens +
model.TokensPerRequest
- if clone.tokensBeforeConvo > clone.settings.GetPlannerEffectiveMaxTokens() {
+ var effectiveMaxTokens int
+ if clone.currentStage.TellStage == shared.TellStagePlanning {
+ if clone.currentStage.PlanningPhase == shared.PlanningPhaseContext {
+ effectiveMaxTokens = clone.settings.GetArchitectEffectiveMaxTokens()
+ } else {
+ effectiveMaxTokens = clone.settings.GetPlannerEffectiveMaxTokens()
+ }
+ } else if clone.currentStage.TellStage == shared.TellStageImplementation {
+ effectiveMaxTokens = clone.settings.GetCoderEffectiveMaxTokens()
+ }
+
+ if clone.tokensBeforeConvo > effectiveMaxTokens {
log.Println("tokensBeforeConvo exceeds max tokens during dry run")
go notify.NotifyErr(notify.SeverityError, fmt.Errorf("tokensBeforeConvo exceeds max tokens during dry run"))
diff --git a/app/server/model/plan/tell_load.go b/app/server/model/plan/tell_load.go
index e556a10f..28d50101 100644
--- a/app/server/model/plan/tell_load.go
+++ b/app/server/model/plan/tell_load.go
@@ -15,7 +15,7 @@ import (
"github.com/sashabaranov/go-openai"
)
-func (state *activeTellStreamState) loadTellPlan(isRetry bool) error {
+func (state *activeTellStreamState) loadTellPlan() error {
clients := state.clients
req := state.req
auth := state.auth
@@ -154,7 +154,7 @@ func (state *activeTellStreamState) loadTellPlan(isRetry bool) error {
innerErrCh := make(chan error)
go func() {
- if iteration == 0 && missingFileResponse == "" && !req.IsUserContinue && !isRetry {
+ if iteration == 0 && missingFileResponse == "" && !req.IsUserContinue {
num := len(convo) + 1
log.Printf("[TellLoad] storing user message | len(convo): %d | num: %d\n", len(convo), num)
diff --git a/app/server/model/plan/tell_state.go b/app/server/model/plan/tell_state.go
index a8a853a7..13226707 100644
--- a/app/server/model/plan/tell_state.go
+++ b/app/server/model/plan/tell_state.go
@@ -11,12 +11,9 @@ import (
"github.com/sashabaranov/go-openai"
)
-const NumTellStreamRetries = 4
-
type activeTellStreamState struct {
activePlan *types.ActivePlan
modelStreamId string
- execTellPlanParams execTellPlanParams
clients map[string]model.ClientInfo
req *shared.TellPlanRequest
auth *types.ServerAuth
@@ -51,13 +48,19 @@ type activeTellStreamState struct {
chunkProcessor *chunkProcessor
generationId string
- requestStartedAt time.Time
- firstTokenAt time.Time
- originalReq *types.ExtendedChatCompletionRequest
- tenativeModelConfig *shared.ModelRoleConfig
- modelConfig *shared.ModelRoleConfig
+ requestStartedAt time.Time
+ firstTokenAt time.Time
+ originalReq *types.ExtendedChatCompletionRequest
+ modelConfig *shared.ModelRoleConfig
+ fallbackRes shared.FallbackResult
skipConvoMessages map[string]bool
+
+ manualStop []string
+
+ numErrorRetry int
+ numFallbackRetry int
+ modelErr *shared.ModelError
}
type chunkProcessor struct {
diff --git a/app/server/model/plan/tell_stream_error.go b/app/server/model/plan/tell_stream_error.go
index 126c531b..53adfa78 100644
--- a/app/server/model/plan/tell_stream_error.go
+++ b/app/server/model/plan/tell_stream_error.go
@@ -4,8 +4,10 @@ import (
"context"
"fmt"
"log"
+ "math/rand"
"net/http"
"plandex-server/db"
+ "plandex-server/model"
"plandex-server/notify"
"plandex-server/shutdown"
"strconv"
@@ -21,6 +23,7 @@ type onErrorParams struct {
convoMessageId string
commitMsg string
canRetry bool
+ modelErr *shared.ModelError
}
type onErrorResult struct {
@@ -34,6 +37,7 @@ func (state *activeTellStreamState) onError(params onErrorParams) onErrorResult
storeDesc := params.storeDesc
convoMessageId := params.convoMessageId
commitMsg := params.commitMsg
+ modelErr := params.modelErr
planId := state.plan.Id
branch := state.branch
@@ -41,7 +45,6 @@ func (state *activeTellStreamState) onError(params onErrorParams) onErrorResult
summarizedToMessageId := state.summarizedToMessageId
active := GetActivePlan(planId, branch)
- numRetries := state.execTellPlanParams.numErrorRetry
if active == nil {
log.Printf("tellStream onError - Active plan not found for plan ID %s on branch %s\n", planId, branch)
@@ -51,10 +54,46 @@ func (state *activeTellStreamState) onError(params onErrorParams) onErrorResult
}
canRetry := params.canRetry
+ hasFallback := state.fallbackRes.HasErrorFallback
+ isFallback := state.fallbackRes.IsFallback
+
+ maxRetries := model.MAX_RETRIES_WITHOUT_FALLBACK
+ if hasFallback {
+ maxRetries = model.MAX_ADDITIONAL_RETRIES_WITH_FALLBACK
+ }
+
+ compareRetries := state.numErrorRetry
+ if isFallback {
+ compareRetries = state.numFallbackRetry
+ }
+
+ newFallback := false
+ if modelErr != nil {
+ if !modelErr.Retriable {
+ log.Printf("tellStream onError - operation returned non-retriable error: %v", modelErr)
+ if modelErr.Kind == shared.ErrContextTooLong && state.fallbackRes.ModelRoleConfig.LargeContextFallback == nil {
+ log.Printf("tellStream onError - non-retriable context too long error and no large context fallback is defined, no retry")
+ // if it's a context too long error and no large context fallback is defined, no retry
+ canRetry = false
+
+ } else if modelErr.Kind != shared.ErrContextTooLong && state.fallbackRes.ModelRoleConfig.ErrorFallback == nil {
+ log.Printf("tellStream onError - non-retriable error and no error fallback is defined, no retry")
+ // if it's any other error and no error fallback is defined, no retry
+ canRetry = false
+ } else {
+ log.Printf("tellStream onError - operation returned non-retriable error, but has fallback - resetting numFallbackRetry to 0 and continuing to retry")
+ state.numFallbackRetry = 0
+ // otherwise, continue to retry logic
+ canRetry = true
+ newFallback = true
+ }
+ }
+ }
if canRetry {
log.Println("tellStream onError - canRetry", canRetry)
- if numRetries >= NumTellStreamRetries {
+
+ if compareRetries >= maxRetries {
log.Printf("tellStream onError - Max retries reached for plan ID %s on branch %s\n", planId, branch)
canRetry = false
@@ -68,15 +107,26 @@ func (state *activeTellStreamState) onError(params onErrorParams) onErrorResult
active.ResetModelCtx()
- retryDelaySeconds := 1 * numRetries * (numRetries / 2)
+ var retryDelay time.Duration
+ if modelErr != nil && modelErr.RetryAfterSeconds > 0 {
+ // if the model err has a retry after, then use that with a bit of padding
+ retryDelay = time.Duration(int(float64(modelErr.RetryAfterSeconds)*1.1)) * time.Second
+ } else {
+ // otherwise, use some jitter
+ retryDelay = time.Duration(1000+rand.Intn(200)) * time.Millisecond
+ }
- log.Printf("tellStream onError - Retry %d/%d - Retrying stream in %d seconds", numRetries+1, NumTellStreamRetries, retryDelaySeconds)
- time.Sleep(time.Duration(retryDelaySeconds) * time.Second)
+ log.Printf("tellStream onError - Retry %d/%d - Retrying stream in %v", state.numErrorRetry+1, maxRetries, retryDelay)
+ time.Sleep(retryDelay)
- params := state.execTellPlanParams
- params.numErrorRetry = numRetries + 1
+ state.numErrorRetry = state.numErrorRetry + 1
+ if isFallback && !newFallback {
+ state.numFallbackRetry = state.numFallbackRetry + 1
+ }
+ state.modelErr = modelErr
- execTellPlan(params)
+ // retry the request
+ state.doTellRequest()
return onErrorResult{
shouldReturn: true,
}
@@ -172,11 +222,11 @@ func (state *activeTellStreamState) onError(params onErrorParams) onErrorResult
active.StreamDoneCh <- params.streamApiErr
} else {
msg := "Stream error: " + streamErr.Error()
- if params.canRetry && numRetries >= NumTellStreamRetries {
- msg += " | Failed after " + strconv.Itoa(numRetries) + " retries."
+ if params.canRetry && state.numErrorRetry >= maxRetries {
+ msg += " | Failed after " + strconv.Itoa(state.numErrorRetry) + " retries"
}
- go notify.NotifyErr(notify.SeverityInfo, fmt.Sprintf("tellStream stream error after %d retries: %v", numRetries, streamErr))
+ go notify.NotifyErr(notify.SeverityInfo, fmt.Sprintf("tellStream stream error after %d retries: %v", state.numErrorRetry, streamErr))
active.StreamDoneCh <- &shared.ApiError{
Type: shared.ApiErrorTypeOther,
diff --git a/app/server/model/plan/tell_stream_main.go b/app/server/model/plan/tell_stream_main.go
index 10588af3..9b9c344d 100644
--- a/app/server/model/plan/tell_stream_main.go
+++ b/app/server/model/plan/tell_stream_main.go
@@ -6,6 +6,7 @@ import (
"net/http"
"plandex-server/model"
"plandex-server/notify"
+ "plandex-server/types"
"runtime/debug"
"time"
@@ -54,13 +55,30 @@ func (state *activeTellStreamState) listenStream(stream *model.ExtendedChatCompl
}
// Create a timer that will trigger if no chunk is received within the specified duration
- timer := time.NewTimer(model.OPENAI_STREAM_CHUNK_TIMEOUT)
+ firstTokenTimeout := firstTokenTimeout(state.totalRequestTokens)
+ log.Printf("listenStream - firstTokenTimeout: %s\n", firstTokenTimeout)
+ timer := time.NewTimer(firstTokenTimeout)
defer timer.Stop()
streamFinished := false
modelProvider := state.modelConfig.BaseModelConfig.Provider
modelName := state.modelConfig.BaseModelConfig.ModelName
+ respCh := make(chan *types.ExtendedChatCompletionStreamResponse)
+ streamErrCh := make(chan error)
+
+ // receive chunks from the stream in a separate goroutine so that we can handle errors and timeouts — needed because stream.Recv() blocks forever
+ go func() {
+ for {
+ resp, err := stream.Recv()
+ if err != nil {
+ streamErrCh <- err
+ return
+ }
+ respCh <- resp
+ }
+ }()
+
mainLoop:
for {
select {
@@ -78,7 +96,7 @@ mainLoop:
return
} else {
res := state.onError(onErrorParams{
- streamErr: fmt.Errorf("stream timeout due to inactivity: The underlying LLM is not responding"),
+ streamErr: fmt.Errorf("stream timeout due to inactivity: The AI model (%s/%s) is not responding", modelProvider, modelName),
storeDesc: true,
canRetry: active.CurrentReplyContent == "", // if there was no output yet, we can retry
})
@@ -91,39 +109,37 @@ mainLoop:
}
}
- default:
- response, err := stream.Recv()
+ case err := <-streamErrCh:
+ log.Printf("listenStream - received from streamErrCh: %v\n", err)
- if err == nil {
- // Successfully received a chunk, reset the timer
- if !timer.Stop() {
- <-timer.C
- }
- timer.Reset(model.OPENAI_STREAM_CHUNK_TIMEOUT)
- } else {
- if err.Error() == "context canceled" {
- log.Println("Tell: stream context canceled")
- state.execHookOnStop(false)
- return
- }
+ if err.Error() == "context canceled" {
+ log.Println("Tell: stream context canceled")
+ state.execHookOnStop(false)
+ return
+ }
- log.Printf("Tell: error receiving stream chunk: %v\n", err)
- state.execHookOnStop(true)
+ log.Printf("Tell: error receiving stream chunk: %v\n", err)
+ state.execHookOnStop(true)
- var msg string
- if active.CurrentReplyContent == "" {
- msg = fmt.Sprintf("The AI model (%s/%s) didn't respond: %v", modelProvider, modelName, err)
- } else {
- msg = fmt.Sprintf("The AI model (%s/%s) stopped responding: %v", modelProvider, modelName, err)
- }
- state.onError(onErrorParams{
- streamErr: fmt.Errorf(msg, err),
- storeDesc: true,
- canRetry: active.CurrentReplyContent == "", // if there was no output yet, we can retry
- })
- // here we want to return no matter what -- state.onError will decide whether to retry or not
- return
+ var msg string
+ if active.CurrentReplyContent == "" {
+ msg = fmt.Sprintf("The AI model (%s/%s) didn't respond: %v", modelProvider, modelName, err)
+ } else {
+ msg = fmt.Sprintf("The AI model (%s/%s) stopped responding: %v", modelProvider, modelName, err)
}
+ state.onError(onErrorParams{
+ streamErr: fmt.Errorf(msg, err),
+ storeDesc: true,
+ canRetry: active.CurrentReplyContent == "", // if there was no output yet, we can retry
+ })
+ // here we want to return no matter what -- state.onError will decide whether to retry or not
+ return
+ case response := <-respCh:
+ // Successfully received a chunk, reset the timer
+ if !timer.Stop() {
+ <-timer.C
+ }
+ timer.Reset(model.ACTIVE_STREAM_CHUNK_TIMEOUT)
// log.Println("tell stream main: received stream response", spew.Sdump(response))
@@ -137,10 +153,14 @@ mainLoop:
if response.Error != nil {
log.Println("listenStream - stream finished with error", spew.Sdump(response.Error))
+
+ modelErr := model.ClassifyModelError(response.Error.Code, response.Error.Message, nil)
+
res := state.onError(onErrorParams{
streamErr: fmt.Errorf("The AI model (%s/%s) stopped streaming with error code %d: %s", modelProvider, modelName, response.Error.Code, response.Error.Message),
storeDesc: true,
- canRetry: active.CurrentReplyContent == "" && response.Error.Code != 429,
+ canRetry: active.CurrentReplyContent == "",
+ modelErr: &modelErr,
})
if res.shouldReturn {
return
@@ -175,22 +195,6 @@ mainLoop:
continue mainLoop
}
- // We'll be more accepting of multiple choices and just take the first one
- // if len(response.Choices) > 1 {
- // res := state.onError(onErrorParams{
- // streamErr: fmt.Errorf("stream finished with more than one choice | The model failed to generate a valid response."),
- // storeDesc: true,
- // canRetry: true,
- // })
- // if res.shouldReturn {
- // return
- // }
- // if res.shouldContinueMainLoop {
- // // continue instead of returning so that context cancellation is handled
- // continue mainLoop
- // }
- // }
-
choice := response.Choices[0]
processChunkRes := state.processChunk(choice)
@@ -198,6 +202,44 @@ mainLoop:
return
}
+ handleFinished := func() handleStreamFinishedResult {
+ streamFinishResult := state.handleStreamFinished()
+ if streamFinishResult.shouldReturn || streamFinishResult.shouldContinueMainLoop {
+ return streamFinishResult
+ }
+
+ // usage can either be included in the final chunk (openrouter) or in a separate chunk (openai)
+ // if the usage chunk is included, handle it and then return out of listener
+ // otherwise keep listening for the usage chunk
+ if response.Usage != nil {
+ state.handleUsageChunk(response.Usage)
+ return handleStreamFinishedResult{
+ shouldReturn: true,
+ }
+ }
+
+ // Reset the timer for the usage chunk
+ if !timer.Stop() {
+ <-timer.C
+ }
+ timer.Reset(model.USAGE_CHUNK_TIMEOUT)
+ streamFinished = true
+
+ return handleStreamFinishedResult{
+ shouldContinueMainLoop: true,
+ }
+ }
+
+ if processChunkRes.shouldStop {
+ log.Println("Model stream reached stop sequence")
+
+ res := handleFinished()
+ if res.shouldReturn {
+ return
+ }
+ continue
+ }
+
if choice.FinishReason != "" {
log.Println("Model stream finished")
log.Println("Finish reason: ", choice.FinishReason)
@@ -218,28 +260,10 @@ mainLoop:
}
}
- streamFinishResult := state.handleStreamFinished()
- if streamFinishResult.shouldContinueMainLoop {
- continue mainLoop
- }
- if streamFinishResult.shouldReturn {
- return
- }
-
- // usage can either be included in the final chunk (openrouter) or in a separate chunk (openai)
- // if the usage chunk is included, handle it and then return out of listener
- // otherwise keep listening for the usage chunk
- if response.Usage != nil {
- state.handleUsageChunk(response.Usage)
+ res := handleFinished()
+ if res.shouldReturn {
return
}
-
- // Reset the timer for the usage chunk
- if !timer.Stop() {
- <-timer.C
- }
- timer.Reset(model.OPENAI_USAGE_CHUNK_TIMEOUT)
- streamFinished = true
continue
} else if response.Usage != nil {
state.handleUsageChunk(response.Usage)
@@ -249,3 +273,20 @@ mainLoop:
}
}
}
+
+func firstTokenTimeout(tok int) time.Duration {
+ const (
+ base = 90 * time.Second
+ slope = 90 * time.Second
+ step = 150_000
+ cap = 15 * time.Minute
+ )
+ if tok <= step {
+ return base
+ }
+ extra := time.Duration((tok-step)/step) * slope
+ if extra > cap-base {
+ extra = cap - base
+ }
+ return base + extra
+}
diff --git a/app/server/model/plan/tell_stream_processor_test.go b/app/server/model/plan/tell_stream_processor_test.go
index 8e6da9e3..45f8e468 100644
--- a/app/server/model/plan/tell_stream_processor_test.go
+++ b/app/server/model/plan/tell_stream_processor_test.go
@@ -19,6 +19,7 @@ func TestBufferOrStream(t *testing.T) {
isInResetBlock bool
want bufferOrStreamResult
wantState *chunkProcessor // To verify state transitions
+ manualStop []string
}{
{
name: "streams regular content",
@@ -367,6 +368,86 @@ func TestBufferOrStream(t *testing.T) {
fileOpen: true,
},
},
+
+ {
+ name: "stop tag entirely in one chunk",
+ initialState: &chunkProcessor{}, // empty buffer
+ chunk: "hello <PlandexFinish/>bye",
+ manualStop: []string{"<PlandexFinish/>"},
+ want: bufferOrStreamResult{
+ shouldStream: true, // stream only the prefix
+ content: "hello ", // text before the tag
+ shouldStop: true, // tell caller to stop
+ },
+ wantState: &chunkProcessor{
+ contentBuffer: "", // nothing left buffered
+ },
+ },
+ {
+ name: "stop tag split across two chunks (prefix + rest)",
+ only: true, // helper if you want to run just this one
+ initialState: &chunkProcessor{
+ contentBuffer: "", // begins empty
+ },
+ // FIRST CHUNK —— just a proper prefix
+ chunk: "<PlandexFin", // no '<' in second part
+ manualStop: []string{"<PlandexFinish/>"},
+ want: bufferOrStreamResult{
+ shouldStream: false, // nothing streams yet
+ shouldStop: false, // not complete, keep going
+ },
+ wantState: &chunkProcessor{
+ contentBuffer: "<PlandexFin", // prefix is buffered
+ },
+ },
+ {
+ // SECOND CHUNK —— completes the tag; nothing should stream
+ name: "stop tag split across two chunks (completes)",
+ initialState: &chunkProcessor{
+ contentBuffer: "<PlandexFin", // leftover from previous call
+ },
+ chunk: "ish/>\nmore text", // completes tag + trailing text
+ manualStop: []string{"<PlandexFinish/>"},
+ want: bufferOrStreamResult{
+ shouldStream: false, // do NOT leak "more text"
+ shouldStop: true, // signal caller to stop
+ },
+ wantState: &chunkProcessor{
+ contentBuffer: "<PlandexFinish/>", // may keep full tag inside
+ },
+ },
+ {
+ name: "stop prefix turns out to be different tag, falls through to other parsing logic",
+ initialState: &chunkProcessor{
+ contentBuffer: "<Plandex",
+ },
+ chunk: "Blo",
+ manualStop: []string{"<PlandexFinish/>"},
+ want: bufferOrStreamResult{
+ shouldStream: false,
+ shouldStop: false,
+ },
+ wantState: &chunkProcessor{
+ contentBuffer: "<PlandexBlo",
+ awaitingBlockOpeningTag: true,
+ },
+ },
+ {
+ name: "stop prefix turns out to be different tag, falls through to other parsing logic #2",
+ initialState: &chunkProcessor{
+ contentBuffer: "something\n<Plandex",
+ },
+ chunk: "exBlock lang=\"go\" path=\"main.go\">\npackage",
+ manualStop: []string{"<PlandexFinish/>"},
+ want: bufferOrStreamResult{
+ shouldStream: true,
+ content: "something\n```go\npackage",
+ },
+ wantState: &chunkProcessor{
+ awaitingBlockOpeningTag: false,
+ fileOpen: true,
+ },
+ },
}
only := map[int]bool{}
@@ -392,7 +473,7 @@ func TestBufferOrStream(t *testing.T) {
IsInResetBlock: tt.isInResetBlock,
}, shared.CurrentStage{
TellStage: shared.TellStageImplementation,
- })
+ }, tt.manualStop)
if got.shouldStream != tt.want.shouldStream {
t.Errorf("shouldStream = %v, want %v", got.shouldStream, tt.want.shouldStream)