Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

refactored build logic, remove retry logic in prep for better replacements strategy, fix for context being newer than current file

Dane Schneider5253308f4715
diff --git a/app/server/model/plan/build.go b/app/server/model/plan/build.go
deleted file mode 100644
index 2ee17867..00000000
--- a/app/server/model/plan/build.go
+++ /dev/null
@@ -1,966 +0,0 @@
-package plan
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "log"
- "math"
- "net/http"
- "plandex-server/db"
- "plandex-server/model"
- "plandex-server/model/prompts"
- "plandex-server/types"
- "sort"
- "strings"
- "time"
-
- "github.com/davecgh/go-spew/spew"
- "github.com/google/uuid"
- "github.com/plandex/plandex/shared"
- "github.com/sashabaranov/go-openai"
-)
-
-const MaxRetries = 3
-const MaxReplacementRetries = 1
-
-func Build(client *openai.Client, plan *db.Plan, branch string, auth *types.ServerAuth) (int, error) {
- log.Printf("Build: Called with plan ID %s on branch %s\n", plan.Id, branch)
- log.Println("Build: Starting Build operation")
-
- active, err := activatePlan(client, plan, branch, auth, "", true)
-
- if err != nil {
- log.Printf("Error activating plan: %v\n", err)
- return 0, err
- }
-
- onErr := func(err error) (int, error) {
- log.Printf("Build error: %v\n", err)
- active.StreamDoneCh <- nil
- return 0, err
- }
-
- repoLockId, err := db.LockRepo(
- db.LockRepoParams{
- OrgId: auth.OrgId,
- UserId: auth.User.Id,
- PlanId: plan.Id,
- Branch: branch,
- Scope: db.LockScopeRead,
- Ctx: active.Ctx,
- CancelFn: active.CancelFn,
- },
- )
- if err != nil {
- return onErr(fmt.Errorf("error locking repo for build: %v", err))
- }
-
- var modelContext []*db.Context
- var pendingBuildsByPath map[string][]*types.ActiveBuild
- var settings *shared.PlanSettings
-
- err = func() error {
- defer func() {
- err := db.UnlockRepo(repoLockId)
- if err != nil {
- log.Printf("Error unlocking repo: %v\n", err)
- }
- }()
-
- errCh := make(chan error)
-
- go func() {
- res, err := db.GetPlanContexts(auth.OrgId, plan.Id, true)
- if err != nil {
- log.Printf("Error getting plan modelContext: %v\n", err)
- errCh <- fmt.Errorf("error getting plan modelContext: %v", err)
- return
- }
- modelContext = res
-
- errCh <- nil
- }()
-
- go func() {
- res, err := active.PendingBuildsByPath(auth.OrgId, auth.User.Id, nil)
-
- if err != nil {
- log.Printf("Error getting pending builds by path: %v\n", err)
- errCh <- fmt.Errorf("error getting pending builds by path: %v", err)
- return
- }
-
- pendingBuildsByPath = res
-
- errCh <- nil
- }()
-
- go func() {
- res, err := db.GetPlanSettings(plan, true)
- if err != nil {
- log.Printf("Error getting plan settings: %v\n", err)
- errCh <- fmt.Errorf("error getting plan settings: %v", err)
- return
- }
-
- settings = res
- errCh <- nil
- }()
-
- for i := 0; i < 3; i++ {
- err = <-errCh
- if err != nil {
- log.Printf("Error getting plan data: %v\n", err)
- return err
- }
- }
- return nil
- }()
-
- if err != nil {
- return onErr(err)
- }
-
- UpdateActivePlan(plan.Id, branch, func(ap *types.ActivePlan) {
- ap.Contexts = modelContext
- for _, context := range modelContext {
- if context.FilePath != "" {
- ap.ContextsByPath[context.FilePath] = context
- }
- }
- })
-
- if len(pendingBuildsByPath) == 0 {
- log.Println("No pending builds")
- active.StreamDoneCh <- nil
- return 0, nil
- }
-
- err = db.SetPlanStatus(plan.Id, branch, shared.PlanStatusBuilding, "")
-
- if err != nil {
- log.Printf("Error setting plan status to building: %v\n", err)
- return onErr(fmt.Errorf("error setting plan status to building: %v", err))
- }
-
- log.Printf("Starting %d builds\n", len(pendingBuildsByPath))
-
- for _, pendingBuilds := range pendingBuildsByPath {
- go execPlanBuild(client, settings.ModelSet.Builder, auth.OrgId, auth.User.Id, branch, active, pendingBuilds)
- }
-
- return len(pendingBuildsByPath), nil
-}
-
-func queueBuilds(client *openai.Client, config shared.TaskRoleConfig, currentOrgId, currentUserId, planId, branch string, activeBuilds []*types.ActiveBuild) {
- activePlan := GetActivePlan(planId, branch)
- filePath := activeBuilds[0].Path
-
- // log.Printf("Queue:")
- // spew.Dump(activePlan.BuildQueuesByPath[filePath])
-
- UpdateActivePlan(planId, branch, func(active *types.ActivePlan) {
- active.BuildQueuesByPath[filePath] = append(active.BuildQueuesByPath[filePath], activeBuilds...)
- })
- log.Printf("Queued %d build(s) for file %s\n", len(activeBuilds), filePath)
-
- if activePlan.IsBuildingByPath[filePath] {
- log.Printf("Already building file %s\n", filePath)
- return
- } else {
- log.Printf("Not building file %s, will execute now\n", filePath)
- go execPlanBuild(client, config, currentOrgId, currentUserId, branch, activePlan, activeBuilds)
- }
-}
-
-func execPlanBuild(client *openai.Client, config shared.TaskRoleConfig, currentOrgId, currentUserId, branch string, activePlan *types.ActivePlan, activeBuilds []*types.ActiveBuild) {
- log.Printf("execPlanBuild for %d active builds\n", len(activeBuilds))
-
- if len(activeBuilds) == 0 {
- log.Println("No active builds")
- return
- }
-
- // all builds should have the same path
- filePath := activeBuilds[0].Path
- planId := activePlan.Id
- var convoMessageIds []string
- added := map[string]bool{}
-
- for _, activeBuild := range activeBuilds {
- if !added[activeBuild.ReplyId] {
- convoMessageIds = append(convoMessageIds, activeBuild.ReplyId)
- added[activeBuild.ReplyId] = true
- }
- }
-
- if !activePlan.IsBuildingByPath[filePath] {
- UpdateActivePlan(activePlan.Id, activePlan.Branch, func(ap *types.ActivePlan) {
- ap.IsBuildingByPath[filePath] = true
- })
- }
-
- buildInfo := &shared.BuildInfo{
- Path: filePath,
- NumTokens: 0,
- Finished: false,
- }
- activePlan.Stream(shared.StreamMessage{
- Type: shared.StreamMessageBuildInfo,
- BuildInfo: buildInfo,
- })
-
- build := &db.PlanBuild{
- OrgId: currentOrgId,
- PlanId: planId,
- ConvoMessageIds: convoMessageIds,
- FilePath: filePath,
- }
- err := db.StorePlanBuild(build)
-
- if err != nil {
- log.Printf("Error storing plan build: %v\n", err)
- UpdateActivePlan(activePlan.Id, activePlan.Branch, func(ap *types.ActivePlan) {
- ap.IsBuildingByPath[filePath] = false
- })
- activePlan.StreamDoneCh <- &shared.ApiError{
- Type: shared.ApiErrorTypeOther,
- Status: http.StatusInternalServerError,
- Msg: "Error storing plan build: " + err.Error(),
- }
- return
- }
-
- var currentPlan *shared.CurrentPlanState
-
- repoLockId, err := db.LockRepo(
- db.LockRepoParams{
- OrgId: currentOrgId,
- UserId: currentUserId,
- PlanId: planId,
- Branch: branch,
- PlanBuildId: build.Id,
- Scope: db.LockScopeRead,
- Ctx: activePlan.Ctx,
- CancelFn: activePlan.CancelFn,
- },
- )
- if err != nil {
- log.Printf("Error locking repo for build file: %v\n", err)
- UpdateActivePlan(activePlan.Id, activePlan.Branch, func(ap *types.ActivePlan) {
- ap.IsBuildingByPath[filePath] = false
- })
- activePlan.StreamDoneCh <- &shared.ApiError{
- Type: shared.ApiErrorTypeOther,
- Status: http.StatusInternalServerError,
- Msg: "Error locking repo for build file: " + err.Error(),
- }
- return
- }
-
- func() {
- defer func() {
- err := db.UnlockRepo(repoLockId)
- if err != nil {
- log.Printf("Error unlocking repo: %v\n", err)
- }
- }()
-
- res, err := db.GetCurrentPlanState(db.CurrentPlanStateParams{
- OrgId: currentOrgId,
- PlanId: planId,
- })
- if err != nil {
- log.Printf("Error getting current plan state: %v\n", err)
- UpdateActivePlan(activePlan.Id, activePlan.Branch, func(ap *types.ActivePlan) {
- ap.IsBuildingByPath[filePath] = false
- })
- activePlan.StreamDoneCh <- &shared.ApiError{
- Type: shared.ApiErrorTypeOther,
- Status: http.StatusInternalServerError,
- Msg: "Error getting current plan state: " + err.Error(),
- }
- return
- }
- currentPlan = res
-
- log.Println("Got current plan state")
- // spew.Dump(currentPlan)
- }()
-
- errCh := make(chan error)
-
- onFinishBuild := func() {
- log.Println("Build finished")
-
- // first check if any of the messages we're building hasen't finished streaming yet
- stillStreaming := false
- var doneCh chan bool
- ap := GetActivePlan(planId, branch)
- for _, convoMessageId := range convoMessageIds {
- if ap.CurrentStreamingReplyId == convoMessageId {
- stillStreaming = true
- doneCh = ap.CurrentReplyDoneCh
- break
- }
- }
- if stillStreaming {
- log.Println("Reply is still streaming, waiting for it to finish before finishing build")
- <-doneCh
- }
-
- // Check again if build is finished
- // (more builds could have been queued while we were waiting for the reply to finish streaming)
- ap = GetActivePlan(planId, branch)
- if !ap.BuildFinished() {
- log.Println("Build not finished after waiting for reply to finish streaming")
- return
- }
-
- repoLockId, err := db.LockRepo(
- db.LockRepoParams{
- OrgId: currentOrgId,
- UserId: currentUserId,
- PlanId: planId,
- Branch: branch,
- PlanBuildId: build.Id,
- Scope: db.LockScopeWrite,
- Ctx: activePlan.Ctx,
- CancelFn: activePlan.CancelFn,
- },
- )
-
- if err != nil {
- log.Printf("Error locking repo for finished build: %v\n", err)
- activePlan.StreamDoneCh <- &shared.ApiError{
- Type: shared.ApiErrorTypeOther,
- Status: http.StatusInternalServerError,
- Msg: "Error locking repo for finished build: " + err.Error(),
- }
- return
- }
-
- log.Println("Locked repo for finished build")
-
- err = func() error {
- var err error
- defer func() {
- if err != nil {
- log.Printf("Finish build error: %v\n", err)
- err = db.GitClearUncommittedChanges(currentOrgId, planId)
- if err != nil {
- log.Printf("Error clearing uncommitted changes: %v\n", err)
- }
- log.Println("Cleared uncommitted changes")
- }
-
- err := db.UnlockRepo(repoLockId)
- if err != nil {
- log.Printf("Error unlocking repo: %v\n", err)
- }
-
- log.Println("Unlocked repo")
- }()
-
- // get plan descriptions
- planDescs, err := db.GetPendingBuildDescriptions(currentOrgId, planId)
- if err != nil {
- errCh <- fmt.Errorf("error getting pending build descriptions: %v", err)
- return err
- }
-
- // get fresh current plan state
- currentPlan, err := db.GetCurrentPlanState(db.CurrentPlanStateParams{
- OrgId: currentOrgId,
- PlanId: planId,
- PendingBuildDescriptions: planDescs,
- })
- if err != nil {
- errCh <- fmt.Errorf("error getting current plan state: %v", err)
- return err
- }
-
- descErrCh := make(chan error)
- for _, desc := range planDescs {
- if len(desc.Files) > 0 {
- desc.DidBuild = true
- }
-
- go func(desc *db.ConvoMessageDescription) {
- err := db.StoreDescription(desc)
-
- if err != nil {
- descErrCh <- fmt.Errorf("error storing description: %v", err)
- return
- }
-
- descErrCh <- nil
- }(desc)
- }
-
- for range planDescs {
- err = <-descErrCh
- if err != nil {
- errCh <- err
- return err
- }
- }
-
- err = db.GitAddAndCommit(currentOrgId, planId, branch, currentPlan.PendingChangesSummary())
-
- if err != nil {
- log.Printf("Error committing plan build: %v\n", err)
- activePlan.StreamDoneCh <- &shared.ApiError{
- Type: shared.ApiErrorTypeOther,
- Status: http.StatusInternalServerError,
- Msg: "Error committing plan build: " + err.Error(),
- }
- return err
- }
-
- log.Println("Plan build committed")
-
- return nil
-
- }()
-
- if err != nil {
- return
- }
-
- active := GetActivePlan(planId, branch)
-
- if active != nil && (active.RepliesFinished || active.BuildOnly) {
- activePlan.Stream(shared.StreamMessage{
- Type: shared.StreamMessageFinished,
- })
- }
- }
-
- onFinishBuildFile := func(planRes *db.PlanFileResult) {
- finished := false
- log.Println("onFinishBuildFile: " + filePath)
-
- repoLockId, err := db.LockRepo(
- db.LockRepoParams{
- OrgId: currentOrgId,
- UserId: currentUserId,
- PlanId: planId,
- Branch: branch,
- PlanBuildId: build.Id,
- Scope: db.LockScopeWrite,
- Ctx: activePlan.Ctx,
- CancelFn: activePlan.CancelFn,
- },
- )
- if err != nil {
- log.Printf("Error locking repo for build file: %v\n", err)
- activePlan.StreamDoneCh <- &shared.ApiError{
- Type: shared.ApiErrorTypeOther,
- Status: http.StatusInternalServerError,
- Msg: "Error locking repo for build file: " + err.Error(),
- }
- return
- }
-
- err = func() error {
- var err error
- defer func() {
- if err != nil {
- log.Printf("Error: %v\n", err)
- err = db.GitClearUncommittedChanges(currentOrgId, planId)
- if err != nil {
- log.Printf("Error clearing uncommitted changes: %v\n", err)
- }
- }
-
- err := db.UnlockRepo(repoLockId)
- if err != nil {
- log.Printf("Error unlocking repo: %v\n", err)
- }
- }()
-
- err = db.StorePlanResult(planRes)
- if err != nil {
- log.Printf("Error storing plan result: %v\n", err)
- activePlan.StreamDoneCh <- &shared.ApiError{
- Type: shared.ApiErrorTypeOther,
- Status: http.StatusInternalServerError,
- Msg: "Error storing plan result: " + err.Error(),
- }
- return err
- }
- return nil
- }()
-
- if err != nil {
- return
- }
-
- for _, build := range activeBuilds {
- build.Success = true
- }
-
- UpdateActivePlan(planId, branch, func(ap *types.ActivePlan) {
- ap.BuiltFiles[filePath] = true
- if ap.BuildFinished() {
- finished = true
- }
- })
-
- log.Printf("Finished building file %s\n", filePath)
-
- if finished {
- UpdateActivePlan(planId, branch, func(ap *types.ActivePlan) {
- ap.IsBuildingByPath[filePath] = false
- })
- log.Println("Finished building plan, calling onFinishBuild")
- onFinishBuild()
- } else {
- if activePlan.PathFinished(filePath) {
- UpdateActivePlan(planId, branch, func(ap *types.ActivePlan) {
- ap.IsBuildingByPath[filePath] = false
- })
- log.Printf("File %s finished, but not all builds finished\n", filePath)
- } else {
- log.Printf("Processing next build for file %s\n", filePath)
- var nextBuilds []*types.ActiveBuild
- for _, build := range activePlan.BuildQueuesByPath[filePath] {
- if !build.BuildFinished() && len(nextBuilds) < 5 {
- nextBuilds = append(nextBuilds, build)
- }
- }
-
- // log.Println("Next builds:")
- // spew.Dump(nextBuilds)
- log.Printf("%d builds left for file %s\n", len(nextBuilds), filePath)
-
- if len(nextBuilds) > 0 {
- log.Println("Calling execPlanBuild for next build in queue")
- go execPlanBuild(client, config, currentOrgId, currentUserId, branch, activePlan, nextBuilds)
- }
- return
- }
- }
- }
-
- onBuildFileError := func(err error) {
- log.Printf("Error for file %s: %v\n", filePath, err)
-
- for _, build := range activeBuilds {
- build.Success = false
- build.Error = err
- }
-
- activePlan.StreamDoneCh <- &shared.ApiError{
- Type: shared.ApiErrorTypeOther,
- Status: http.StatusInternalServerError,
- Msg: err.Error(),
- }
-
- if err != nil {
- log.Printf("Error storing plan error result: %v\n", err)
- }
-
- build.Error = err.Error()
-
- err = db.SetBuildError(build)
- if err != nil {
- log.Printf("Error setting build error: %v\n", err)
- }
- }
-
- var buildFile func(filePath string, numRetry int, numReplacementRetry int, res *db.PlanFileResult)
- buildFile = func(filePath string, numRetry int, numReplacementsRetry int, res *db.PlanFileResult) {
- log.Printf("Building file %s, numRetry: %d\n", filePath, numRetry)
-
- log.Println("activePlan.ContextsByPath files:")
- for k := range activePlan.ContextsByPath {
- log.Println(k)
- }
-
- // get relevant file context (if any)
- contextPart := activePlan.ContextsByPath[filePath]
-
- var currentState string
- currentPlanFile, fileInCurrentPlan := currentPlan.CurrentPlanFiles.Files[filePath]
-
- if fileInCurrentPlan {
- currentState = currentPlanFile
-
- log.Printf("File %s found in current plan. Using current state.\n", filePath)
- // log.Println("Current state:")
- // log.Println(currentState)
- } else if contextPart != nil {
- log.Printf("File %s found in model context. Using context state.\n", filePath)
-
- currentState = contextPart.Body
-
- if currentState == "" {
- log.Println("Context state is empty. That's bad.")
- }
- }
-
- if currentState == "" {
- log.Printf("File %s not found in model context or current plan. Creating new file.\n", filePath)
-
- buildInfo := &shared.BuildInfo{
- Path: filePath,
- NumTokens: 0,
- Finished: true,
- }
-
- activePlan.Stream(shared.StreamMessage{
- Type: shared.StreamMessageBuildInfo,
- BuildInfo: buildInfo,
- })
-
- // new file
- planRes := &db.PlanFileResult{
- OrgId: currentOrgId,
- PlanId: planId,
- PlanBuildId: build.Id,
- ConvoMessageIds: build.ConvoMessageIds,
- Path: filePath,
- Content: activeBuilds[0].FileContent,
- }
- onFinishBuildFile(planRes)
- return
- }
-
- log.Println("Getting file from model: " + filePath)
- // log.Println("File context:", fileContext)
-
- replacePrompt := prompts.GetReplacePrompt(filePath)
- currentStatePrompt := prompts.GetBuildCurrentStatePrompt(filePath, currentState)
- sysPrompt := prompts.GetBuildSysPrompt(filePath, currentStatePrompt)
-
- var mergedReply string
- for _, activeBuild := range activeBuilds {
- mergedReply += "\n\n" + activeBuild.ReplyContent
- }
-
- log.Println("Num active builds: " + fmt.Sprintf("%d", len(activeBuilds)))
- // log.Println("Merged reply:")
- // log.Println(mergedReply)
-
- fileMessages := []openai.ChatCompletionMessage{
- {
- Role: openai.ChatMessageRoleSystem,
- Content: sysPrompt,
- },
- {
- Role: openai.ChatMessageRoleUser,
- Content: activePlan.Prompt,
- },
- {
- Role: openai.ChatMessageRoleAssistant,
- Content: mergedReply,
- },
- {
- Role: openai.ChatMessageRoleUser,
- Content: replacePrompt,
- },
- }
-
- if numReplacementsRetry > 0 && res != nil {
- bytes, err := json.Marshal(res.Replacements)
- if err != nil {
- onBuildFileError(fmt.Errorf("error marshalling replacements: %v", err))
- return
- }
-
- correctReplacementPrompt, err := prompts.GetCorrectReplacementPrompt(res.Replacements, currentState)
- if err != nil {
- onBuildFileError(fmt.Errorf("error getting correct replacement prompt: %v", err))
- return
- }
-
- fileMessages = append(fileMessages,
- openai.ChatCompletionMessage{
- Role: openai.ChatMessageRoleAssistant,
- Content: string(bytes),
- },
-
- openai.ChatCompletionMessage{
- Role: openai.ChatMessageRoleUser,
- Content: correctReplacementPrompt,
- })
- }
-
- log.Println("Calling model for file: " + filePath)
-
- // for _, msg := range fileMessages {
- // log.Printf("%s: %s\n", msg.Role, msg.Content)
- // }
-
- modelReq := openai.ChatCompletionRequest{
- Model: config.BaseModelConfig.ModelName,
- Tools: []openai.Tool{
- {
- Type: "function",
- Function: prompts.ReplaceFn,
- },
- },
- ToolChoice: openai.ToolChoice{
- Type: "function",
- Function: openai.ToolFunction{
- Name: prompts.ReplaceFn.Name,
- },
- },
- Messages: fileMessages,
- Temperature: config.Temperature,
- TopP: config.TopP,
- ResponseFormat: config.OpenAIResponseFormat,
- }
-
- stream, err := client.CreateChatCompletionStream(activePlan.Ctx, modelReq)
- if err != nil {
- log.Printf("Error creating plan file stream for path '%s': %v\n", filePath, err)
-
- if numRetry >= MaxRetries {
- onBuildFileError(fmt.Errorf("failed to create plan file stream for path '%s' after %d retries: %v", filePath, numRetry, err))
- } else {
- log.Println("Retrying build plan for file: " + filePath)
- buildFile(filePath, numRetry+1, numReplacementsRetry, res)
- if err != nil {
- onBuildFileError(fmt.Errorf("failed to retry build plan for file '%s': %v", filePath, err))
- }
- }
- return
- }
-
- buffer := ""
-
- go func() {
- defer stream.Close()
-
- // Create a timer that will trigger if no chunk is received within the specified duration
- timer := time.NewTimer(model.OPENAI_STREAM_CHUNK_TIMEOUT)
- defer timer.Stop()
-
- handleErrorRetry := func(maxRetryErr error, shouldSleep bool, isReplacementsRetry bool, res *db.PlanFileResult) (shouldContinue bool) {
- log.Printf("Error for file %s: %v\n", filePath, maxRetryErr)
-
- if isReplacementsRetry && numReplacementsRetry >= MaxReplacementRetries {
- // in this case, we just want to ignore the error and continue
- return true
- } else if !isReplacementsRetry && numRetry >= MaxRetries {
- onBuildFileError(maxRetryErr)
- return false
- } else {
- if shouldSleep {
- time.Sleep(1 * time.Second * time.Duration(math.Pow(float64(numRetry+1), 2)))
- }
- if isReplacementsRetry {
- buildFile(filePath, numRetry+1, numReplacementsRetry+1, res)
- } else {
- buildFile(filePath, numRetry+1, numReplacementsRetry, res)
- }
- if err != nil {
- onBuildFileError(fmt.Errorf("failed to retry build plan for file '%s': %v", filePath, err))
- }
- return false
- }
-
- }
-
- for {
- select {
- case <-activePlan.Ctx.Done():
- // The main context was canceled (not the timer)
- return
- case <-timer.C:
- // Timer triggered because no new chunk was received in time
- handleErrorRetry(
- fmt.Errorf("stream timeout due to inactivity for file '%s' after %d retries", filePath, numRetry),
- true,
- false,
- res,
- )
- return
- default:
- response, err := stream.Recv()
-
- if err == nil {
- // Successfully received a chunk, reset the timer
- if !timer.Stop() {
- <-timer.C
- }
- timer.Reset(model.OPENAI_STREAM_CHUNK_TIMEOUT)
- } else {
- log.Printf("File %s: Error receiving stream chunk: %v\n", filePath, err)
-
- if err == context.Canceled {
- log.Printf("File %s: Stream canceled\n", filePath)
- return
- }
-
- handleErrorRetry(
- fmt.Errorf("stream error for file '%s' after %d retries: %v", filePath, numRetry, err),
- true,
- false,
- res,
- )
- return
- }
-
- if len(response.Choices) == 0 {
- handleErrorRetry(fmt.Errorf("stream error: no choices"), true, false, res)
- return
- }
-
- choice := response.Choices[0]
- var content string
- delta := response.Choices[0].Delta
-
- if len(delta.ToolCalls) == 0 {
- log.Println("Stream chunk missing function call. Response:")
- spew.Dump(response)
-
- log.Println("Messages:")
- spew.Dump(fileMessages)
-
- log.Println("Buffer:")
- log.Println(buffer)
-
- handleErrorRetry(
- fmt.Errorf("stream chunk missing function call. Reason: %s, File: %s", choice.FinishReason, filePath),
- false,
- false,
- res,
- )
- return
-
- }
-
- content = delta.ToolCalls[0].Function.Arguments
- buildInfo := &shared.BuildInfo{
- Path: filePath,
- NumTokens: 1,
- Finished: false,
- }
-
- // log.Printf("%s: %s", filePath, content)
- activePlan.Stream(shared.StreamMessage{
- Type: shared.StreamMessageBuildInfo,
- BuildInfo: buildInfo,
- })
-
- buffer += content
-
- var streamed types.StreamedReplacements
- err = json.Unmarshal([]byte(buffer), &streamed)
- if err == nil && len(streamed.Replacements) > 0 {
- log.Printf("File %s: Parsed replacements\n", filePath)
-
- planFileResult, allSucceeded := getPlanResult(
- planResultParams{
- orgId: currentOrgId,
- planId: planId,
- planBuildId: build.Id,
- convoMessageIds: build.ConvoMessageIds,
- filePath: filePath,
- currentState: currentState,
- context: contextPart,
- replacements: streamed.Replacements,
- },
- )
-
- if !allSucceeded {
- log.Println("Failed replacements:")
- for _, replacement := range planFileResult.Replacements {
- if replacement.Failed {
- spew.Dump(replacement)
- }
- }
-
- if numReplacementsRetry < MaxReplacementRetries {
- shouldContinue := handleErrorRetry(
- nil, // no error -- if we reach MAX_REPLACEMENT_RETRIES, we just ignore the error and continue
- false,
- true,
- planFileResult,
- )
- if !shouldContinue {
- return
- }
- }
- }
-
- buildInfo := &shared.BuildInfo{
- Path: filePath,
- NumTokens: 0,
- Finished: true,
- }
- activePlan.Stream(shared.StreamMessage{
- Type: shared.StreamMessageBuildInfo,
- BuildInfo: buildInfo,
- })
-
- onFinishBuildFile(planFileResult)
- return
- }
- }
- }
- }()
- }
-
- buildFile(filePath, 0, 0, nil)
-}
-
-type planResultParams struct {
- orgId string
- planId string
- planBuildId string
- convoMessageIds []string
- filePath string
- currentState string
- context *db.Context
- replacements []*shared.Replacement
-}
-
-func getPlanResult(params planResultParams) (*db.PlanFileResult, bool) {
- orgId := params.orgId
- planId := params.planId
- planBuildId := params.planBuildId
- filePath := params.filePath
- currentState := params.currentState
- contextPart := params.context
- replacements := params.replacements
- updated := params.currentState
-
- sort.Slice(replacements, func(i, j int) bool {
- iIdx := strings.Index(updated, replacements[i].Old)
- jIdx := strings.Index(updated, replacements[j].Old)
- return iIdx < jIdx
- })
-
- _, allSucceeded := shared.ApplyReplacements(currentState, replacements, true)
-
- var contextSha string
- var contextBody string
- if contextPart != nil {
- contextSha = contextPart.Sha
- contextBody = contextPart.Body
- }
-
- for _, replacement := range replacements {
- id := uuid.New().String()
- replacement.Id = id
- }
-
- return &db.PlanFileResult{
- OrgId: orgId,
- PlanId: planId,
- PlanBuildId: planBuildId,
- ConvoMessageIds: params.convoMessageIds,
- Content: "",
- Path: filePath,
- Replacements: replacements,
- ContextSha: contextSha,
- ContextBody: contextBody,
- AnyFailed: !allSucceeded,
- }, allSucceeded
-}
diff --git a/app/server/model/plan/build_exec.go b/app/server/model/plan/build_exec.go
new file mode 100644
index 00000000..3434e8eb
--- /dev/null
+++ b/app/server/model/plan/build_exec.go
@@ -0,0 +1,277 @@
+package plan
+
+import (
+ "fmt"
+ "log"
+ "plandex-server/db"
+ "plandex-server/model/prompts"
+ "plandex-server/types"
+
+ "github.com/plandex/plandex/shared"
+ "github.com/sashabaranov/go-openai"
+)
+
+func Build(client *openai.Client, plan *db.Plan, branch string, auth *types.ServerAuth) (int, error) {
+ log.Printf("Build: Called with plan ID %s on branch %s\n", plan.Id, branch)
+ log.Println("Build: Starting Build operation")
+
+ state := activeBuildStreamState{
+ client: client,
+ auth: auth,
+ currentOrgId: auth.OrgId,
+ currentUserId: auth.User.Id,
+ plan: plan,
+ branch: branch,
+ }
+
+ active := GetActivePlan(plan.Id, branch)
+
+ onErr := func(err error) (int, error) {
+ log.Printf("Build error: %v\n", err)
+ active.StreamDoneCh <- nil
+ return 0, err
+ }
+
+ pendingBuildsByPath, err := state.loadBuild()
+ if err != nil {
+ return onErr(err)
+ }
+
+ if len(pendingBuildsByPath) == 0 {
+ log.Println("No pending builds")
+ active.StreamDoneCh <- nil
+ return 0, nil
+ }
+
+ err = db.SetPlanStatus(plan.Id, branch, shared.PlanStatusBuilding, "")
+
+ if err != nil {
+ log.Printf("Error setting plan status to building: %v\n", err)
+ return onErr(fmt.Errorf("error setting plan status to building: %v", err))
+ }
+
+ log.Printf("Starting %d builds\n", len(pendingBuildsByPath))
+
+ for _, pendingBuilds := range pendingBuildsByPath {
+ go state.execPlanBuild(pendingBuilds)
+ }
+
+ return len(pendingBuildsByPath), nil
+}
+
+func (state *activeBuildStreamState) queueBuilds(activeBuilds []*types.ActiveBuild) {
+ planId := state.plan.Id
+ branch := state.branch
+
+ activePlan := GetActivePlan(planId, branch)
+ filePath := activeBuilds[0].Path
+
+ // log.Printf("Queue:")
+ // spew.Dump(activePlan.BuildQueuesByPath[filePath])
+
+ UpdateActivePlan(planId, branch, func(active *types.ActivePlan) {
+ active.BuildQueuesByPath[filePath] = append(active.BuildQueuesByPath[filePath], activeBuilds...)
+ })
+ log.Printf("Queued %d build(s) for file %s\n", len(activeBuilds), filePath)
+
+ if activePlan.IsBuildingByPath[filePath] {
+ log.Printf("Already building file %s\n", filePath)
+ return
+ } else {
+ log.Printf("Not building file %s, will execute now\n", filePath)
+ go state.execPlanBuild(activeBuilds)
+ }
+}
+
+func (buildState *activeBuildStreamState) execPlanBuild(activeBuilds []*types.ActiveBuild) {
+ log.Printf("execPlanBuild for %d active builds\n", len(activeBuilds))
+
+ if len(activeBuilds) == 0 {
+ log.Println("No active builds")
+ return
+ }
+
+ planId := buildState.plan.Id
+ branch := buildState.branch
+
+ activePlan := GetActivePlan(planId, branch)
+ filePath := activeBuilds[0].Path
+
+ // stream initial status to client
+ buildInfo := &shared.BuildInfo{
+ Path: filePath,
+ NumTokens: 0,
+ Finished: false,
+ }
+ activePlan.Stream(shared.StreamMessage{
+ Type: shared.StreamMessageBuildInfo,
+ BuildInfo: buildInfo,
+ })
+
+ fileState := &activeBuildStreamFileState{
+ activeBuildStreamState: buildState,
+ filePath: filePath,
+ activeBuilds: activeBuilds,
+ }
+ err := fileState.loadBuildFile(activeBuilds)
+ if err != nil {
+ log.Printf("Error loading build file: %v\n", err)
+ return
+ }
+
+ fileState.buildFile()
+}
+
+func (fileState *activeBuildStreamFileState) buildFile() {
+ filePath := fileState.filePath
+ activeBuilds := fileState.activeBuilds
+ planId := fileState.plan.Id
+ branch := fileState.branch
+ currentPlan := fileState.currentPlanState
+ currentOrgId := fileState.currentOrgId
+ client := fileState.client
+ config := fileState.settings.ModelSet.Builder
+ build := fileState.build
+
+ activePlan := GetActivePlan(planId, branch)
+
+ log.Printf("Building file %s\n", filePath)
+
+ log.Println("activePlan.ContextsByPath files:")
+ for k := range activePlan.ContextsByPath {
+ log.Println(k)
+ }
+
+ // get relevant file context (if any)
+ contextPart := activePlan.ContextsByPath[filePath]
+
+ var currentState string
+ currentPlanFile, fileInCurrentPlan := currentPlan.CurrentPlanFiles.Files[filePath]
+
+ if fileInCurrentPlan {
+ log.Printf("File %s found in current plan.\n", filePath)
+ if contextPart == nil {
+ log.Println("No context - using current plan state.")
+ currentState = currentPlanFile
+ } else {
+ currentFileUpdatedAt := currentPlan.CurrentPlanFiles.UpdatedAtByPath[filePath]
+ contextFileUpdatedAt := contextPart.UpdatedAt
+
+ if currentFileUpdatedAt.After(contextFileUpdatedAt) {
+ log.Println("Current plan file is newer than context. Using current plan state.")
+ currentState = currentPlanFile
+ } else {
+ log.Println("Context is newer than current plan file. Using context state.")
+ currentState = contextPart.Body
+ }
+ }
+ } else if contextPart != nil {
+ log.Printf("File %s found in model context. Using context state.\n", filePath)
+
+ currentState = contextPart.Body
+
+ if currentState == "" {
+ log.Println("Context state is empty. That's bad.")
+ }
+ }
+
+ if currentState == "" {
+ log.Printf("File %s not found in model context or current plan. Creating new file.\n", filePath)
+
+ buildInfo := &shared.BuildInfo{
+ Path: filePath,
+ NumTokens: 0,
+ Finished: true,
+ }
+
+ activePlan.Stream(shared.StreamMessage{
+ Type: shared.StreamMessageBuildInfo,
+ BuildInfo: buildInfo,
+ })
+
+ // new file
+ planRes := &db.PlanFileResult{
+ OrgId: currentOrgId,
+ PlanId: planId,
+ PlanBuildId: build.Id,
+ ConvoMessageIds: build.ConvoMessageIds,
+ Path: filePath,
+ Content: activeBuilds[0].FileContent,
+ }
+ fileState.onFinishBuildFile(planRes)
+ return
+ }
+
+ fileState.currentState = currentState
+ fileState.contextPart = contextPart
+
+ log.Println("Getting file from model: " + filePath)
+ // log.Println("File context:", fileContext)
+
+ replacePrompt := prompts.GetReplacePrompt(filePath)
+ currentStatePrompt := prompts.GetBuildCurrentStatePrompt(filePath, currentState)
+ sysPrompt := prompts.GetBuildSysPrompt(filePath, currentStatePrompt)
+
+ var mergedReply string
+ for _, activeBuild := range activeBuilds {
+ mergedReply += "\n\n" + activeBuild.ReplyContent
+ }
+
+ log.Println("Num active builds: " + fmt.Sprintf("%d", len(activeBuilds)))
+ // log.Println("Merged reply:")
+ // log.Println(mergedReply)
+
+ fileMessages := []openai.ChatCompletionMessage{
+ {
+ Role: openai.ChatMessageRoleSystem,
+ Content: sysPrompt,
+ },
+ {
+ Role: openai.ChatMessageRoleUser,
+ Content: activePlan.Prompt,
+ },
+ {
+ Role: openai.ChatMessageRoleAssistant,
+ Content: mergedReply,
+ },
+ {
+ Role: openai.ChatMessageRoleUser,
+ Content: replacePrompt,
+ },
+ }
+
+ log.Println("Calling model for file: " + filePath)
+
+ // for _, msg := range fileMessages {
+ // log.Printf("%s: %s\n", msg.Role, msg.Content)
+ // }
+
+ modelReq := openai.ChatCompletionRequest{
+ Model: config.BaseModelConfig.ModelName,
+ Tools: []openai.Tool{
+ {
+ Type: "function",
+ Function: prompts.ReplaceFn,
+ },
+ },
+ ToolChoice: openai.ToolChoice{
+ Type: "function",
+ Function: openai.ToolFunction{
+ Name: prompts.ReplaceFn.Name,
+ },
+ },
+ Messages: fileMessages,
+ Temperature: config.Temperature,
+ TopP: config.TopP,
+ ResponseFormat: config.OpenAIResponseFormat,
+ }
+
+ stream, err := client.CreateChatCompletionStream(activePlan.Ctx, modelReq)
+ if err != nil {
+ log.Printf("Error creating plan file stream for path '%s': %v\n", filePath, err)
+ return
+ }
+
+ go fileState.listenStream(stream)
+
+}
diff --git a/app/server/model/plan/build_finish.go b/app/server/model/plan/build_finish.go
new file mode 100644
index 00000000..cf9de4e7
--- /dev/null
+++ b/app/server/model/plan/build_finish.go
@@ -0,0 +1,321 @@
+package plan
+
+import (
+ "fmt"
+ "log"
+ "net/http"
+ "plandex-server/db"
+ "plandex-server/types"
+
+ "github.com/plandex/plandex/shared"
+)
+
+func (state *activeBuildStreamFileState) onFinishBuild() {
+ log.Println("Build finished")
+
+ planId := state.plan.Id
+ branch := state.branch
+ currentOrgId := state.currentOrgId
+ currentUserId := state.currentUserId
+ convoMessageIds := state.convoMessageIds
+ build := state.build
+
+ activePlan := GetActivePlan(planId, branch)
+
+ // first check if any of the messages we're building hasen't finished streaming yet
+ stillStreaming := false
+ var doneCh chan bool
+ ap := GetActivePlan(planId, branch)
+ for _, convoMessageId := range convoMessageIds {
+ if ap.CurrentStreamingReplyId == convoMessageId {
+ stillStreaming = true
+ doneCh = ap.CurrentReplyDoneCh
+ break
+ }
+ }
+ if stillStreaming {
+ log.Println("Reply is still streaming, waiting for it to finish before finishing build")
+ <-doneCh
+ }
+
+ // Check again if build is finished
+ // (more builds could have been queued while we were waiting for the reply to finish streaming)
+ ap = GetActivePlan(planId, branch)
+ if !ap.BuildFinished() {
+ log.Println("Build not finished after waiting for reply to finish streaming")
+ return
+ }
+
+ repoLockId, err := db.LockRepo(
+ db.LockRepoParams{
+ OrgId: currentOrgId,
+ UserId: currentUserId,
+ PlanId: planId,
+ Branch: branch,
+ PlanBuildId: build.Id,
+ Scope: db.LockScopeWrite,
+ Ctx: activePlan.Ctx,
+ CancelFn: activePlan.CancelFn,
+ },
+ )
+
+ if err != nil {
+ log.Printf("Error locking repo for finished build: %v\n", err)
+ activePlan.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: "Error locking repo for finished build: " + err.Error(),
+ }
+ return
+ }
+
+ log.Println("Locked repo for finished build")
+
+ err = func() error {
+ var err error
+ defer func() {
+ if err != nil {
+ log.Printf("Finish build error: %v\n", err)
+ err = db.GitClearUncommittedChanges(currentOrgId, planId)
+ if err != nil {
+ log.Printf("Error clearing uncommitted changes: %v\n", err)
+ }
+ log.Println("Cleared uncommitted changes")
+ }
+
+ err := db.UnlockRepo(repoLockId)
+ if err != nil {
+ log.Printf("Error unlocking repo: %v\n", err)
+ }
+
+ log.Println("Unlocked repo")
+ }()
+
+ // get plan descriptions
+ var planDescs []*db.ConvoMessageDescription
+ planDescs, err = db.GetPendingBuildDescriptions(currentOrgId, planId)
+ if err != nil {
+ log.Printf("Error getting pending build descriptions: %v\n", err)
+ return fmt.Errorf("error getting pending build descriptions: %v", err)
+ }
+
+ // get fresh current plan state
+ var currentPlan *shared.CurrentPlanState
+ currentPlan, err = db.GetCurrentPlanState(db.CurrentPlanStateParams{
+ OrgId: currentOrgId,
+ PlanId: planId,
+ PendingBuildDescriptions: planDescs,
+ })
+ if err != nil {
+ log.Printf("Error getting current plan state: %v\n", err)
+ return fmt.Errorf("error getting current plan state: %v", err)
+ }
+
+ descErrCh := make(chan error)
+ for _, desc := range planDescs {
+ if len(desc.Files) > 0 {
+ desc.DidBuild = true
+ }
+
+ go func(desc *db.ConvoMessageDescription) {
+ err := db.StoreDescription(desc)
+
+ if err != nil {
+ descErrCh <- fmt.Errorf("error storing description: %v", err)
+ return
+ }
+
+ descErrCh <- nil
+ }(desc)
+ }
+
+ for range planDescs {
+ err = <-descErrCh
+ if err != nil {
+ log.Printf("Error storing description: %v\n", err)
+ return err
+ }
+ }
+
+ err = db.GitAddAndCommit(currentOrgId, planId, branch, currentPlan.PendingChangesSummary())
+
+ if err != nil {
+ log.Printf("Error committing plan build: %v\n", err)
+ activePlan.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: "Error committing plan build: " + err.Error(),
+ }
+ return err
+ }
+
+ log.Println("Plan build committed")
+
+ return nil
+
+ }()
+
+ if err != nil {
+ return
+ }
+
+ active := GetActivePlan(planId, branch)
+
+ if active != nil && (active.RepliesFinished || active.BuildOnly) {
+ activePlan.Stream(shared.StreamMessage{
+ Type: shared.StreamMessageFinished,
+ })
+ }
+}
+
+func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanFileResult) {
+ planId := fileState.plan.Id
+ branch := fileState.branch
+ currentOrgId := fileState.currentOrgId
+ currentUserId := fileState.currentUserId
+ build := fileState.build
+ activeBuilds := fileState.activeBuilds
+
+ activePlan := GetActivePlan(planId, branch)
+ filePath := fileState.filePath
+
+ finished := false
+ log.Println("onFinishBuildFile: " + filePath)
+
+ repoLockId, err := db.LockRepo(
+ db.LockRepoParams{
+ OrgId: currentOrgId,
+ UserId: currentUserId,
+ PlanId: planId,
+ Branch: branch,
+ PlanBuildId: build.Id,
+ Scope: db.LockScopeWrite,
+ Ctx: activePlan.Ctx,
+ CancelFn: activePlan.CancelFn,
+ },
+ )
+ if err != nil {
+ log.Printf("Error locking repo for build file: %v\n", err)
+ activePlan.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: "Error locking repo for build file: " + err.Error(),
+ }
+ return
+ }
+
+ err = func() error {
+ var err error
+ defer func() {
+ if err != nil {
+ log.Printf("Error: %v\n", err)
+ err = db.GitClearUncommittedChanges(currentOrgId, planId)
+ if err != nil {
+ log.Printf("Error clearing uncommitted changes: %v\n", err)
+ }
+ }
+
+ err := db.UnlockRepo(repoLockId)
+ if err != nil {
+ log.Printf("Error unlocking repo: %v\n", err)
+ }
+ }()
+
+ err = db.StorePlanResult(planRes)
+ if err != nil {
+ log.Printf("Error storing plan result: %v\n", err)
+ activePlan.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: "Error storing plan result: " + err.Error(),
+ }
+ return err
+ }
+ return nil
+ }()
+
+ if err != nil {
+ return
+ }
+
+ for _, build := range activeBuilds {
+ build.Success = true
+ }
+
+ UpdateActivePlan(planId, branch, func(ap *types.ActivePlan) {
+ ap.BuiltFiles[filePath] = true
+ if ap.BuildFinished() {
+ finished = true
+ }
+ })
+
+ log.Printf("Finished building file %s\n", filePath)
+
+ if finished {
+ UpdateActivePlan(planId, branch, func(ap *types.ActivePlan) {
+ ap.IsBuildingByPath[filePath] = false
+ })
+ log.Println("Finished building plan, calling onFinishBuild")
+ fileState.onFinishBuild()
+ } else {
+ if activePlan.PathFinished(filePath) {
+ UpdateActivePlan(planId, branch, func(ap *types.ActivePlan) {
+ ap.IsBuildingByPath[filePath] = false
+ })
+ log.Printf("File %s finished, but not all builds finished\n", filePath)
+ } else {
+ log.Printf("Processing next build for file %s\n", filePath)
+ var nextBuilds []*types.ActiveBuild
+ for _, build := range activePlan.BuildQueuesByPath[filePath] {
+ if !build.BuildFinished() && len(nextBuilds) < 5 {
+ nextBuilds = append(nextBuilds, build)
+ }
+ }
+
+ // log.Println("Next builds:")
+ // spew.Dump(nextBuilds)
+ log.Printf("%d builds left for file %s\n", len(nextBuilds), filePath)
+
+ if len(nextBuilds) > 0 {
+ log.Println("Calling execPlanBuild for next build in queue")
+ go fileState.execPlanBuild(nextBuilds)
+ }
+ return
+ }
+ }
+
+}
+
+func (fileState *activeBuildStreamFileState) onBuildFileError(err error) {
+ planId := fileState.plan.Id
+ branch := fileState.branch
+ filePath := fileState.filePath
+ build := fileState.build
+ activeBuilds := fileState.activeBuilds
+
+ activePlan := GetActivePlan(planId, branch)
+
+ log.Printf("Error for file %s: %v\n", filePath, err)
+
+ for _, build := range activeBuilds {
+ build.Success = false
+ build.Error = err
+ }
+
+ activePlan.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: err.Error(),
+ }
+
+ if err != nil {
+ log.Printf("Error storing plan error result: %v\n", err)
+ }
+
+ build.Error = err.Error()
+
+ err = db.SetBuildError(build)
+ if err != nil {
+ log.Printf("Error setting build error: %v\n", err)
+ }
+}
diff --git a/app/server/model/plan/build_load.go b/app/server/model/plan/build_load.go
new file mode 100644
index 00000000..c85c6997
--- /dev/null
+++ b/app/server/model/plan/build_load.go
@@ -0,0 +1,236 @@
+package plan
+
+import (
+ "fmt"
+ "log"
+ "net/http"
+ "plandex-server/db"
+ "plandex-server/types"
+
+ "github.com/plandex/plandex/shared"
+)
+
+func (state *activeBuildStreamState) loadBuild() (map[string][]*types.ActiveBuild, error) {
+ client := state.client
+ plan := state.plan
+ branch := state.branch
+ auth := state.auth
+
+ active, err := activatePlan(client, plan, branch, auth, "", true)
+
+ if err != nil {
+ log.Printf("Error activating plan: %v\n", err)
+ }
+
+ repoLockId, err := db.LockRepo(
+ db.LockRepoParams{
+ OrgId: auth.OrgId,
+ UserId: auth.User.Id,
+ PlanId: plan.Id,
+ Branch: branch,
+ Scope: db.LockScopeRead,
+ Ctx: active.Ctx,
+ CancelFn: active.CancelFn,
+ },
+ )
+ if err != nil {
+ return nil, fmt.Errorf("error locking repo for build: %v", err)
+ }
+
+ var modelContext []*db.Context
+ var pendingBuildsByPath map[string][]*types.ActiveBuild
+ var settings *shared.PlanSettings
+
+ err = func() error {
+ defer func() {
+ err := db.UnlockRepo(repoLockId)
+ if err != nil {
+ log.Printf("Error unlocking repo: %v\n", err)
+ }
+ }()
+
+ errCh := make(chan error)
+
+ go func() {
+ res, err := db.GetPlanContexts(auth.OrgId, plan.Id, true)
+ if err != nil {
+ log.Printf("Error getting plan modelContext: %v\n", err)
+ errCh <- fmt.Errorf("error getting plan modelContext: %v", err)
+ return
+ }
+ modelContext = res
+
+ errCh <- nil
+ }()
+
+ go func() {
+ res, err := active.PendingBuildsByPath(auth.OrgId, auth.User.Id, nil)
+
+ if err != nil {
+ log.Printf("Error getting pending builds by path: %v\n", err)
+ errCh <- fmt.Errorf("error getting pending builds by path: %v", err)
+ return
+ }
+
+ pendingBuildsByPath = res
+
+ errCh <- nil
+ }()
+
+ go func() {
+ res, err := db.GetPlanSettings(plan, true)
+ if err != nil {
+ log.Printf("Error getting plan settings: %v\n", err)
+ errCh <- fmt.Errorf("error getting plan settings: %v", err)
+ return
+ }
+
+ settings = res
+ errCh <- nil
+ }()
+
+ for i := 0; i < 3; i++ {
+ err = <-errCh
+ if err != nil {
+ log.Printf("Error getting plan data: %v\n", err)
+ return err
+ }
+ }
+ return nil
+ }()
+
+ if err != nil {
+ return nil, err
+ }
+
+ UpdateActivePlan(plan.Id, branch, func(ap *types.ActivePlan) {
+ ap.Contexts = modelContext
+ for _, context := range modelContext {
+ if context.FilePath != "" {
+ ap.ContextsByPath[context.FilePath] = context
+ }
+ }
+ })
+
+ state.modelContext = modelContext
+ state.settings = settings
+
+ return pendingBuildsByPath, nil
+}
+
+func (state *activeBuildStreamFileState) loadBuildFile(activeBuilds []*types.ActiveBuild) error {
+
+ currentOrgId := state.currentOrgId
+ currentUserId := state.currentUserId
+ planId := state.plan.Id
+ branch := state.branch
+ filePath := state.filePath
+
+ activePlan := GetActivePlan(planId, branch)
+
+ var convoMessageIds []string
+ added := map[string]bool{}
+
+ for _, activeBuild := range activeBuilds {
+ if !added[activeBuild.ReplyId] {
+ convoMessageIds = append(convoMessageIds, activeBuild.ReplyId)
+ added[activeBuild.ReplyId] = true
+ }
+ }
+
+ if !activePlan.IsBuildingByPath[filePath] {
+ UpdateActivePlan(activePlan.Id, activePlan.Branch, func(ap *types.ActivePlan) {
+ ap.IsBuildingByPath[filePath] = true
+ })
+ }
+
+ build := &db.PlanBuild{
+ OrgId: currentOrgId,
+ PlanId: planId,
+ ConvoMessageIds: convoMessageIds,
+ FilePath: filePath,
+ }
+ err := db.StorePlanBuild(build)
+
+ if err != nil {
+ log.Printf("Error storing plan build: %v\n", err)
+ UpdateActivePlan(activePlan.Id, activePlan.Branch, func(ap *types.ActivePlan) {
+ ap.IsBuildingByPath[filePath] = false
+ })
+ activePlan.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: "Error storing plan build: " + err.Error(),
+ }
+ return err
+ }
+
+ var currentPlan *shared.CurrentPlanState
+
+ repoLockId, err := db.LockRepo(
+ db.LockRepoParams{
+ OrgId: currentOrgId,
+ UserId: currentUserId,
+ PlanId: planId,
+ Branch: branch,
+ PlanBuildId: build.Id,
+ Scope: db.LockScopeRead,
+ Ctx: activePlan.Ctx,
+ CancelFn: activePlan.CancelFn,
+ },
+ )
+ if err != nil {
+ log.Printf("Error locking repo for build file: %v\n", err)
+ UpdateActivePlan(activePlan.Id, activePlan.Branch, func(ap *types.ActivePlan) {
+ ap.IsBuildingByPath[filePath] = false
+ })
+ activePlan.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: "Error locking repo for build file: " + err.Error(),
+ }
+ return err
+ }
+
+ err = func() error {
+ defer func() {
+ err := db.UnlockRepo(repoLockId)
+ if err != nil {
+ log.Printf("Error unlocking repo: %v\n", err)
+ }
+ }()
+
+ res, err := db.GetCurrentPlanState(db.CurrentPlanStateParams{
+ OrgId: currentOrgId,
+ PlanId: planId,
+ })
+ if err != nil {
+ log.Printf("Error getting current plan state: %v\n", err)
+ UpdateActivePlan(activePlan.Id, activePlan.Branch, func(ap *types.ActivePlan) {
+ ap.IsBuildingByPath[filePath] = false
+ })
+ activePlan.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: "Error getting current plan state: " + err.Error(),
+ }
+ return err
+ }
+ currentPlan = res
+
+ log.Println("Got current plan state")
+ return nil
+ }()
+
+ if err != nil {
+ return err
+ }
+
+ state.filePath = filePath
+ state.convoMessageIds = convoMessageIds
+ state.build = build
+ state.currentPlanState = currentPlan
+
+ return nil
+
+}
diff --git a/app/server/model/plan/build_result.go b/app/server/model/plan/build_result.go
new file mode 100644
index 00000000..bf10e36d
--- /dev/null
+++ b/app/server/model/plan/build_result.go
@@ -0,0 +1,65 @@
+package plan
+
+import (
+ "plandex-server/db"
+ "sort"
+ "strings"
+
+ "github.com/google/uuid"
+ "github.com/plandex/plandex/shared"
+)
+
+type planResultParams struct {
+ orgId string
+ planId string
+ planBuildId string
+ convoMessageIds []string
+ filePath string
+ currentState string
+ context *db.Context
+ replacements []*shared.Replacement
+}
+
+func getPlanResult(params planResultParams) (*db.PlanFileResult, bool) {
+ orgId := params.orgId
+ planId := params.planId
+ planBuildId := params.planBuildId
+ filePath := params.filePath
+ currentState := params.currentState
+ contextPart := params.context
+ replacements := params.replacements
+ updated := params.currentState
+
+ sort.Slice(replacements, func(i, j int) bool {
+ iIdx := strings.Index(updated, replacements[i].Old)
+ jIdx := strings.Index(updated, replacements[j].Old)
+ return iIdx < jIdx
+ })
+
+ _, allSucceeded := shared.ApplyReplacements(currentState, replacements, true)
+
+ var contextSha string
+ var contextBody string
+ if contextPart != nil {
+ contextSha = contextPart.Sha
+ contextBody = contextPart.Body
+ }
+
+ for _, replacement := range replacements {
+ id := uuid.New().String()
+ replacement.Id = id
+ }
+
+ return &db.PlanFileResult{
+ OrgId: orgId,
+ PlanId: planId,
+ PlanBuildId: planBuildId,
+ ConvoMessageIds: params.convoMessageIds,
+ Content: "",
+ Path: filePath,
+ Replacements: replacements,
+ ContextSha: contextSha,
+ ContextBody: contextBody,
+ AnyFailed: !allSucceeded,
+ }, allSucceeded
+}
diff --git a/app/server/model/plan/build_stream.go b/app/server/model/plan/build_stream.go
new file mode 100644
index 00000000..a7d2260a
--- /dev/null
+++ b/app/server/model/plan/build_stream.go
@@ -0,0 +1,167 @@
+package plan
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log"
+ "plandex-server/db"
+ "plandex-server/model"
+ "plandex-server/types"
+ "time"
+
+ "github.com/davecgh/go-spew/spew"
+ "github.com/plandex/plandex/shared"
+ "github.com/sashabaranov/go-openai"
+)
+
+type activeBuildStreamState struct {
+ client *openai.Client
+ auth *types.ServerAuth
+ currentOrgId string
+ currentUserId string
+ plan *db.Plan
+ branch string
+ settings *shared.PlanSettings
+ modelContext []*db.Context
+}
+
+type activeBuildStreamFileState struct {
+ *activeBuildStreamState
+ filePath string
+ convoMessageIds []string
+ build *db.PlanBuild
+ currentPlanState *shared.CurrentPlanState
+ activeBuilds []*types.ActiveBuild
+ buffer string
+ contextPart *db.Context
+ currentState string
+}
+
+func (fileState *activeBuildStreamFileState) listenStream(stream *openai.ChatCompletionStream) {
+ filePath := fileState.filePath
+ build := fileState.build
+ currentOrgId := fileState.currentOrgId
+ planId := fileState.plan.Id
+ branch := fileState.branch
+ currentState := fileState.currentState
+ contextPart := fileState.contextPart
+
+ activePlan := GetActivePlan(planId, branch)
+
+ defer stream.Close()
+
+ // Create a timer that will trigger if no chunk is received within the specified duration
+ timer := time.NewTimer(model.OPENAI_STREAM_CHUNK_TIMEOUT)
+ defer timer.Stop()
+
+ for {
+ select {
+ case <-activePlan.Ctx.Done():
+ // The main context was canceled (not the timer)
+ return
+ case <-timer.C:
+ // Timer triggered because no new chunk was received in time
+ fileState.onBuildFileError(fmt.Errorf("stream timeout due to inactivity for file '%s'", filePath))
+ return
+ default:
+ response, err := stream.Recv()
+
+ if err == nil {
+ // Successfully received a chunk, reset the timer
+ if !timer.Stop() {
+ <-timer.C
+ }
+ timer.Reset(model.OPENAI_STREAM_CHUNK_TIMEOUT)
+ } else {
+ log.Printf("File %s: Error receiving stream chunk: %v\n", filePath, err)
+
+ if err == context.Canceled {
+ log.Printf("File %s: Stream canceled\n", filePath)
+ return
+ }
+
+ fileState.onBuildFileError(fmt.Errorf("stream error for file '%s': %v", filePath, err))
+ return
+ }
+
+ if len(response.Choices) == 0 {
+ fileState.onBuildFileError(fmt.Errorf("stream error: no choices"))
+ return
+ }
+
+ choice := response.Choices[0]
+ var content string
+ delta := response.Choices[0].Delta
+
+ if len(delta.ToolCalls) == 0 {
+ log.Println("Stream chunk missing function call. Response:")
+ spew.Dump(response)
+
+ fileState.onBuildFileError(fmt.Errorf("stream chunk missing function call. Reason: %s, File: %s", choice.FinishReason, filePath))
+ return
+ }
+
+ content = delta.ToolCalls[0].Function.Arguments
+ buildInfo := &shared.BuildInfo{
+ Path: filePath,
+ NumTokens: 1,
+ Finished: false,
+ }
+
+ // log.Printf("%s: %s", filePath, content)
+ activePlan.Stream(shared.StreamMessage{
+ Type: shared.StreamMessageBuildInfo,
+ BuildInfo: buildInfo,
+ })
+
+ fileState.buffer += content
+
+ var streamed types.StreamedReplacements
+ err = json.Unmarshal([]byte(fileState.buffer), &streamed)
+ if err == nil && len(streamed.Replacements) > 0 {
+ log.Printf("File %s: Parsed replacements\n", filePath)
+
+ planFileResult, allSucceeded := getPlanResult(
+ planResultParams{
+ orgId: currentOrgId,
+ planId: planId,
+ planBuildId: build.Id,
+ convoMessageIds: build.ConvoMessageIds,
+ filePath: filePath,
+ currentState: currentState,
+ context: contextPart,
+ replacements: streamed.Replacements,
+ },
+ )
+
+ if !allSucceeded {
+ log.Println("Failed replacements:")
+ for _, replacement := range planFileResult.Replacements {
+ if replacement.Failed {
+ spew.Dump(replacement)
+ }
+ }
+
+ fileState.onBuildFileError(fmt.Errorf("replacements failed for file '%s'", filePath))
+ return
+
+ }
+
+ buildInfo := &shared.BuildInfo{
+ Path: filePath,
+ NumTokens: 0,
+ Finished: true,
+ }
+ activePlan.Stream(shared.StreamMessage{
+ Type: shared.StreamMessageBuildInfo,
+ BuildInfo: buildInfo,
+ })
+
+ fileState.onFinishBuildFile(planFileResult)
+ return
+ }
+ }
+ }
+
+}
diff --git a/app/server/model/plan/lorem_ipsum.go b/app/server/model/plan/lorem_ipsum.go
deleted file mode 100644
index 8bfb87c7..00000000
--- a/app/server/model/plan/lorem_ipsum.go
+++ /dev/null
@@ -1,121 +0,0 @@
-package plan
-
-// import (
-// "encoding/json"
-// "plandex-server/types"
-// "strings"
-// "sync"
-// "time"
-
-// lorem "github.com/drhodes/golorem"
-// "github.com/plandex/plandex/shared"
-// )
-
-// func applyLoremStyling(paragraphs []string) []string {
-// for p, paragraph := range paragraphs {
-// sentences := strings.Split(paragraph, ". ")
-// for _, sentence := range sentences {
-// words := strings.Split(sentence, " ")
-// for i, word := range words {
-// if (i+1)%5 == 0 { // Bold every 5th word
-// words[i] = "**" + word + "**"
-// }
-// if (i+1)%8 == 0 { // Italicize every 8th word
-// words[i] = "_" + word + "_"
-// }
-// if (i+1)%7 == 0 { // Color every 7th word
-// words[i] = `<span style="color:blue">` + word + `</span>`
-// }
-// }
-// }
-// paragraphs[p] = strings.Join(sentences, ". ")
-// }
-// return paragraphs
-// }
-
-// // Function to stream "lorem ipsum" text sentence by sentence with delay
-// func streamLoremIpsum(onStream types.OnStreamFunc) {
-// paragraphs := []string{lorem.Paragraph(1, 1), lorem.Paragraph(1, 1), lorem.Paragraph(1, 1)}
-// paragraphs = applyLoremStyling(paragraphs)
-
-// for _, paragraph := range paragraphs {
-// for _, line := range strings.Split(paragraph, "\n") {
-// for _, word := range strings.Split(line, " ") {
-// onStream(word+" ", nil)
-// time.Sleep(50 * time.Millisecond)
-// }
-// onStream("\n", nil)
-// time.Sleep(50 * time.Millisecond)
-// }
-// onStream("\n\n", nil)
-// time.Sleep(50 * time.Millisecond)
-// }
-// onStream(shared.STREAM_DESCRIPTION_PHASE, nil)
-
-// planDescription := &shared.PlanDescription{
-// MadePlan: true,
-// Files: []string{"file1.txt", "file2.txt"},
-// }
-// planDescriptionBytes, _ := json.Marshal(planDescription)
-// planDescriptionJson := string(planDescriptionBytes)
-// time.Sleep(2000 * time.Millisecond)
-
-// onStream(planDescriptionJson, nil)
-
-// onStream(shared.STREAM_BUILD_PHASE, nil)
-// buildPlan("", map[string]string{}, map[string]int{}, onStream)
-
-// }
-
-// func streamFilesLoremIpsum(onStream types.OnStreamFunc) {
-// writeNumTokens := func(filePath string, content string) {
-// numTokens := shared.GetNumTokens(content)
-// chunk := shared.BuildInfo{Path: filePath, NumTokens: numTokens}
-// chunkJson, _ := json.Marshal(chunk)
-// onStream(string(chunkJson), nil)
-// time.Sleep(50 * time.Millisecond) // Adding a small delay between files for effect
-// }
-
-// writeFile := func(filePath string, content string) {
-// file := shared.PlanFile{Path: filePath, Content: content}
-// fileJson, _ := json.Marshal(file)
-// onStream(string(fileJson), nil)
-// }
-
-// // For each file in the proposal, stream some unstyled lorem ipsum content
-// var wg sync.WaitGroup
-// files := []string{"file1.txt", "file2.txt"}
-// contentByPath := map[string]string{}
-// for _, filePath := range files {
-// wg.Add(1)
-// go func(filePath string) {
-// defer wg.Done()
-// text := strings.Join([]string{lorem.Paragraph(1, 1), lorem.Paragraph(1, 1), lorem.Paragraph(1, 1)}, "\n\n")
-// contentByPath[filePath] = text
-
-// for _, line := range strings.Split(text, "\n") {
-// for _, word := range strings.Split(line, " ") {
-// writeNumTokens(filePath, word+" ")
-// }
-// writeNumTokens(filePath, "\n")
-// }
-// }(filePath)
-// }
-
-// wg.Wait()
-
-// onStream(shared.STREAM_WRITE_PHASE, nil)
-
-// for _, filePath := range files {
-// wg.Add(1)
-// go func(filePath string) {
-// defer wg.Done()
-// text := contentByPath[filePath]
-// writeFile(filePath, text)
-// }(filePath)
-// }
-
-// wg.Wait()
-
-// onStream(shared.STREAM_FINISHED, nil)
-// }
diff --git a/app/server/model/plan/tell_exec.go b/app/server/model/plan/tell_exec.go
index eb135a1a..b5223a0a 100644
--- a/app/server/model/plan/tell_exec.go
+++ b/app/server/model/plan/tell_exec.go
@@ -382,8 +382,19 @@ func execTellPlan(
return
}
+ buildState := &activeBuildStreamState{
+ client: client,
+ auth: auth,
+ currentOrgId: currentOrgId,
+ currentUserId: currentUserId,
+ plan: plan,
+ branch: branch,
+ settings: state.settings,
+ modelContext: state.modelContext,
+ }
+
for _, pendingBuilds := range pendingBuildsByPath {
- queueBuilds(client, state.settings.ModelSet.Builder, auth.OrgId, auth.User.Id, planId, branch, pendingBuilds)
+ buildState.queueBuilds(pendingBuilds)
}
}()
}
diff --git a/app/server/model/plan/tell_stream.go b/app/server/model/plan/tell_stream.go
index 70f4783b..3515b1c2 100644
--- a/app/server/model/plan/tell_stream.go
+++ b/app/server/model/plan/tell_stream.go
@@ -433,7 +433,18 @@ func (state *activeTellStreamState) listenStream(stream *openai.ChatCompletionSt
log.Printf("New file: %s\n", file)
if req.BuildMode == shared.BuildModeAuto {
log.Printf("Queuing build for %s\n", file)
- queueBuilds(client, settings.ModelSet.Builder, currentOrgId, currentUserId, planId, branch, []*types.ActiveBuild{{
+ buildState := &activeBuildStreamState{
+ client: client,
+ auth: auth,
+ currentOrgId: currentOrgId,
+ currentUserId: currentUserId,
+ plan: plan,
+ branch: branch,
+ settings: settings,
+ modelContext: state.modelContext,
+ }
+
+ buildState.queueBuilds([]*types.ActiveBuild{{
ReplyId: replyId,
ReplyContent: active.CurrentReplyContent,
FileContent: fileContents[i],
diff --git a/app/shared/data_models.go b/app/shared/data_models.go
index 561c2823..faa2d6a4 100644
--- a/app/shared/data_models.go
+++ b/app/shared/data_models.go
@@ -173,8 +173,9 @@ type PlanFileResult struct {
}
type CurrentPlanFiles struct {
- Files map[string]string `json:"files"`
- ContextShas map[string]string `json:"contextShas"`
+ Files map[string]string `json:"files"`
+ ContextShas map[string]string `json:"contextShas"`
+ UpdatedAtByPath map[string]time.Time `json:"updatedAtByPath"`
}
type PlanFileResultsByPath map[string][]*PlanFileResult
diff --git a/app/shared/plan_result.go b/app/shared/plan_result.go
index aaeeeace..a2da0e79 100644
--- a/app/shared/plan_result.go
+++ b/app/shared/plan_result.go
@@ -134,6 +134,7 @@ func (planState *CurrentPlanState) GetFilesBeforeReplacement(
files := make(map[string]string)
shas := make(map[string]string)
+ updatedAtByPath := make(map[string]time.Time)
for path, planResults := range planRes.FileResultsByPath {
updated := files[path]
@@ -172,6 +173,8 @@ func (planState *CurrentPlanState) GetFilesBeforeReplacement(
if !allSucceeded {
return nil, fmt.Errorf("plan replacement failed: %s", path)
}
+
+ updatedAtByPath[path] = planRes.CreatedAt
}
files[path] = updated